summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2025-03-18 22:17:17 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2025-03-20 13:50:40 +0000
commitbdd2b0fee18c43597efc3493b9ac9ace9ccf7959 (patch)
tree622c16d7352f4b5b5207f12b0fc25bd4f1638c0f /bitbake/lib/bb
parent67566c7410e1a3ec7ed4686b45908f25209494ec (diff)
downloadpoky-bdd2b0fee18c43597efc3493b9ac9ace9ccf7959.tar.gz
bitbake: fetch: Drop multiple branch/revision support for single git urls
We used to use this for bare clones where a single git url could handle multiple revisions (which implied multiple branches). We don't use this any more and I doubt we'd want to go back to it. If we remove it, we can simplfy the looping in the code which seems desireable. This patch does change the warning for missing branch parameters to a error. The message has hinted about that for long enough. Some test cases are removed since they are no longer needed. (Bitbake rev: 2515fbd10824005fa7f34e87706000c079920366) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py29
-rw-r--r--bitbake/lib/bb/fetch2/git.py217
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py51
-rw-r--r--bitbake/lib/bb/fetch2/gomod.py7
-rw-r--r--bitbake/lib/bb/tests/fetch.py116
5 files changed, 139 insertions, 281 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index cfbbce5288..fed296097f 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -806,8 +806,8 @@ def _get_srcrev(d, method_name='sortable_revision'):
806 return "", revs 806 return "", revs
807 807
808 808
809 if len(scms) == 1 and len(urldata[scms[0]].names) == 1: 809 if len(scms) == 1:
810 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0]) 810 autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].name)
811 revs.append(rev) 811 revs.append(rev)
812 if len(rev) > 10: 812 if len(rev) > 10:
813 rev = rev[:10] 813 rev = rev[:10]
@@ -828,13 +828,12 @@ def _get_srcrev(d, method_name='sortable_revision'):
828 seenautoinc = False 828 seenautoinc = False
829 for scm in scms: 829 for scm in scms:
830 ud = urldata[scm] 830 ud = urldata[scm]
831 for name in ud.names: 831 autoinc, rev = getattr(ud.method, method_name)(ud, d, ud.name)
832 autoinc, rev = getattr(ud.method, method_name)(ud, d, name) 832 revs.append(rev)
833 revs.append(rev) 833 seenautoinc = seenautoinc or autoinc
834 seenautoinc = seenautoinc or autoinc 834 if len(rev) > 10:
835 if len(rev) > 10: 835 rev = rev[:10]
836 rev = rev[:10] 836 name_to_rev[ud.name] = rev
837 name_to_rev[name] = rev
838 # Replace names by revisions in the SRCREV_FORMAT string. The approach used 837 # Replace names by revisions in the SRCREV_FORMAT string. The approach used
839 # here can handle names being prefixes of other names and names appearing 838 # here can handle names being prefixes of other names and names appearing
840 # as substrings in revisions (in which case the name should not be 839 # as substrings in revisions (in which case the name should not be
@@ -1345,7 +1344,9 @@ class FetchData(object):
1345 setattr(self, "%s_name" % checksum_id, checksum_name) 1344 setattr(self, "%s_name" % checksum_id, checksum_name)
1346 setattr(self, "%s_expected" % checksum_id, checksum_expected) 1345 setattr(self, "%s_expected" % checksum_id, checksum_expected)
1347 1346
1348 self.names = self.parm.get("name",'default').split(',') 1347 self.name = self.parm.get("name",'default')
1348 if "," in self.name:
1349 raise ParameterError("The fetcher no longer supports multiple name parameters in a single url", self.url)
1349 1350
1350 self.method = None 1351 self.method = None
1351 for m in methods: 1352 for m in methods:
@@ -1397,13 +1398,7 @@ class FetchData(object):
1397 self.lockfile = basepath + '.lock' 1398 self.lockfile = basepath + '.lock'
1398 1399
1399 def setup_revisions(self, d): 1400 def setup_revisions(self, d):
1400 self.revisions = {} 1401 self.revision = srcrev_internal_helper(self, d, self.name)
1401 for name in self.names:
1402 self.revisions[name] = srcrev_internal_helper(self, d, name)
1403
1404 # add compatibility code for non name specified case
1405 if len(self.names) == 1:
1406 self.revision = self.revisions[self.names[0]]
1407 1402
1408 def setup_localpath(self, d): 1403 def setup_localpath(self, d):
1409 if not self.localpath: 1404 if not self.localpath:
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 7fda9d2753..3a4ae6df45 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -9,15 +9,6 @@ Supported SRC_URI options are:
9- branch 9- branch
10 The git branch to retrieve from. The default is "master" 10 The git branch to retrieve from. The default is "master"
11 11
12 This option also supports multiple branch fetching, with branches
13 separated by commas. In multiple branches case, the name option
14 must have the same number of names to match the branches, which is
15 used to specify the SRC_REV for the branch
16 e.g:
17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
20
21- tag 12- tag
22 The git tag to retrieve. The default is "master" 13 The git tag to retrieve. The default is "master"
23 14
@@ -192,13 +183,10 @@ class Git(FetchMethod):
192 if ud.bareclone: 183 if ud.bareclone:
193 ud.nocheckout = 1 184 ud.nocheckout = 1
194 185
195 ud.unresolvedrev = {} 186 ud.unresolvedrev = ""
196 branches = ud.parm.get("branch", "").split(',') 187 ud.branch = ud.parm.get("branch", "")
197 if branches == [""] and not ud.nobranch: 188 if not ud.branch and not ud.nobranch:
198 bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url) 189 raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url)
199 branches = ["master"]
200 if len(branches) != len(ud.names):
201 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
202 190
203 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" 191 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
204 192
@@ -227,32 +215,27 @@ class Git(FetchMethod):
227 215
228 revs_default = d.getVar("BB_GIT_SHALLOW_REVS") 216 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
229 ud.shallow_revs = [] 217 ud.shallow_revs = []
230 ud.branches = {} 218
231 for pos, name in enumerate(ud.names): 219 ud.unresolvedrev = ud.branch
232 branch = branches[pos] 220
233 ud.branches[name] = branch 221 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name)
234 ud.unresolvedrev[name] = branch 222 if shallow_depth is not None:
235 223 try:
236 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name) 224 shallow_depth = int(shallow_depth or 0)
237 if shallow_depth is not None: 225 except ValueError:
238 try: 226 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
239 shallow_depth = int(shallow_depth or 0) 227 else:
240 except ValueError: 228 if shallow_depth < 0:
241 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 229 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
242 else: 230 ud.shallow_depths[ud.name] = shallow_depth
243 if shallow_depth < 0: 231
244 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 232 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name)
245 ud.shallow_depths[name] = shallow_depth 233 if revs is not None:
246 234 ud.shallow_revs.extend(revs.split())
247 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) 235 elif revs_default is not None:
248 if revs is not None: 236 ud.shallow_revs.extend(revs_default.split())
249 ud.shallow_revs.extend(revs.split()) 237
250 elif revs_default is not None: 238 if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0:
251 ud.shallow_revs.extend(revs_default.split())
252
253 if (ud.shallow and
254 not ud.shallow_revs and
255 all(ud.shallow_depths[n] == 0 for n in ud.names)):
256 # Shallow disabled for this URL 239 # Shallow disabled for this URL
257 ud.shallow = False 240 ud.shallow = False
258 241
@@ -261,8 +244,7 @@ class Git(FetchMethod):
261 # rev of this repository. This will get resolved into a revision 244 # rev of this repository. This will get resolved into a revision
262 # later. If an actual revision happens to have also been provided 245 # later. If an actual revision happens to have also been provided
263 # then this setting will be overridden. 246 # then this setting will be overridden.
264 for name in ud.names: 247 ud.unresolvedrev = 'HEAD'
265 ud.unresolvedrev[name] = 'HEAD'
266 248
267 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin" 249 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin"
268 250
@@ -272,16 +254,11 @@ class Git(FetchMethod):
272 254
273 ud.setup_revisions(d) 255 ud.setup_revisions(d)
274 256
275 for name in ud.names: 257 # Ensure any revision that doesn't look like a SHA-1 is translated into one
276 # Ensure any revision that doesn't look like a SHA-1 is translated into one 258 if not sha1_re.match(ud.revision or ''):
277 if not sha1_re.match(ud.revisions[name] or ''): 259 if ud.revision:
278 if ud.revisions[name]: 260 ud.unresolvedrev = ud.revision
279 ud.unresolvedrev[name] = ud.revisions[name] 261 ud.revision = self.latest_revision(ud, d, ud.name)
280 ud.revisions[name] = self.latest_revision(ud, d, name)
281
282 if 'tag' in ud.parm:
283 if len(ud.revisions) != 1:
284 raise bb.fetch2.ParameterError("Git fetcher support for multiple tagged revisions not implemented", ud.url)
285 262
286 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_')) 263 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
287 if gitsrcname.startswith('.'): 264 if gitsrcname.startswith('.'):
@@ -292,8 +269,7 @@ class Git(FetchMethod):
292 # upstream repo in the future, the mirror will remain intact and still 269 # upstream repo in the future, the mirror will remain intact and still
293 # contain the revision 270 # contain the revision
294 if ud.rebaseable: 271 if ud.rebaseable:
295 for name in ud.names: 272 gitsrcname = gitsrcname + '_' + ud.revision
296 gitsrcname = gitsrcname + '_' + ud.revisions[name]
297 273
298 dl_dir = d.getVar("DL_DIR") 274 dl_dir = d.getVar("DL_DIR")
299 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") 275 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
@@ -311,15 +287,14 @@ class Git(FetchMethod):
311 if ud.shallow_revs: 287 if ud.shallow_revs:
312 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) 288 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
313 289
314 for name, revision in sorted(ud.revisions.items()): 290 tarballname = "%s_%s" % (tarballname, ud.revision[:7])
315 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) 291 depth = ud.shallow_depths[ud.name]
316 depth = ud.shallow_depths[name] 292 if depth:
317 if depth: 293 tarballname = "%s-%s" % (tarballname, depth)
318 tarballname = "%s-%s" % (tarballname, depth)
319 294
320 shallow_refs = [] 295 shallow_refs = []
321 if not ud.nobranch: 296 if not ud.nobranch:
322 shallow_refs.extend(ud.branches.values()) 297 shallow_refs.append(ud.branch)
323 if ud.shallow_extra_refs: 298 if ud.shallow_extra_refs:
324 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) 299 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
325 if shallow_refs: 300 if shallow_refs:
@@ -344,18 +319,16 @@ class Git(FetchMethod):
344 return True 319 return True
345 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): 320 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
346 return True 321 return True
347 for name in ud.names: 322 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
348 if not self._contains_ref(ud, d, name, ud.clonedir): 323 return True
349 return True
350 return False 324 return False
351 325
352 def lfs_need_update(self, ud, d): 326 def lfs_need_update(self, ud, d):
353 if self.clonedir_need_update(ud, d): 327 if self.clonedir_need_update(ud, d):
354 return True 328 return True
355 329
356 for name in ud.names: 330 if not self._lfs_objects_downloaded(ud, d, ud.name, ud.clonedir):
357 if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir): 331 return True
358 return True
359 return False 332 return False
360 333
361 def clonedir_need_shallow_revs(self, ud, d): 334 def clonedir_need_shallow_revs(self, ud, d):
@@ -497,9 +470,8 @@ class Git(FetchMethod):
497 if exc.errno != errno.ENOENT: 470 if exc.errno != errno.ENOENT:
498 raise 471 raise
499 472
500 for name in ud.names: 473 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
501 if not self._contains_ref(ud, d, name, ud.clonedir): 474 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch))
502 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
503 475
504 if ud.shallow and ud.write_shallow_tarballs: 476 if ud.shallow and ud.write_shallow_tarballs:
505 missing_rev = self.clonedir_need_shallow_revs(ud, d) 477 missing_rev = self.clonedir_need_shallow_revs(ud, d)
@@ -627,45 +599,44 @@ class Git(FetchMethod):
627 for revision in ud.shallow_revs: 599 for revision in ud.shallow_revs:
628 shallow_exclude += " --shallow-exclude=%s" % revision 600 shallow_exclude += " --shallow-exclude=%s" % revision
629 601
630 for name in ud.names: 602 revision = ud.revision
631 revision = ud.revisions[name] 603 depth = ud.shallow_depths[ud.name]
632 depth = ud.shallow_depths[name]
633 604
634 # The --depth and --shallow-exclude can't be used together 605 # The --depth and --shallow-exclude can't be used together
635 if depth and shallow_exclude: 606 if depth and shallow_exclude:
636 raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.") 607 raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
637 608
638 # For nobranch, we need a ref, otherwise the commits will be 609 # For nobranch, we need a ref, otherwise the commits will be
639 # removed, and for non-nobranch, we truncate the branch to our 610 # removed, and for non-nobranch, we truncate the branch to our
640 # srcrev, to avoid keeping unnecessary history beyond that. 611 # srcrev, to avoid keeping unnecessary history beyond that.
641 branch = ud.branches[name] 612 branch = ud.branch
642 if ud.nobranch: 613 if ud.nobranch:
643 ref = "refs/shallow/%s" % name 614 ref = "refs/shallow/%s" % ud.name
644 elif ud.bareclone: 615 elif ud.bareclone:
645 ref = "refs/heads/%s" % branch 616 ref = "refs/heads/%s" % branch
646 else: 617 else:
647 ref = "refs/remotes/origin/%s" % branch 618 ref = "refs/remotes/origin/%s" % branch
648 619
649 fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision) 620 fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
650 if depth: 621 if depth:
651 fetch_cmd += " --depth %s" % depth 622 fetch_cmd += " --depth %s" % depth
652 623
653 if shallow_exclude: 624 if shallow_exclude:
654 fetch_cmd += shallow_exclude 625 fetch_cmd += shallow_exclude
655 626
656 # Advertise the revision for lower version git such as 2.25.1: 627 # Advertise the revision for lower version git such as 2.25.1:
657 # error: Server does not allow request for unadvertised object. 628 # error: Server does not allow request for unadvertised object.
658 # The ud.clonedir is a local temporary dir, will be removed when 629 # The ud.clonedir is a local temporary dir, will be removed when
659 # fetch is done, so we can do anything on it. 630 # fetch is done, so we can do anything on it.
660 adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision) 631 adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
661 if ud.shallow_skip_fast: 632 if ud.shallow_skip_fast:
662 runfetchcmd(adv_cmd, d, workdir=ud.clonedir) 633 runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
663 634
664 runfetchcmd(fetch_cmd, d, workdir=dest) 635 runfetchcmd(fetch_cmd, d, workdir=dest)
665 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) 636 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
666 # Fetch Git LFS data for fast shallow clones 637 # Fetch Git LFS data for fast shallow clones
667 if not ud.shallow_skip_fast: 638 if not ud.shallow_skip_fast:
668 self.lfs_fetch(ud, d, dest, ud.revisions[ud.names[0]]) 639 self.lfs_fetch(ud, d, dest, ud.revision)
669 640
670 # Apply extra ref wildcards 641 # Apply extra ref wildcards
671 all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \ 642 all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
@@ -774,17 +745,17 @@ class Git(FetchMethod):
774 745
775 if not ud.nocheckout: 746 if not ud.nocheckout:
776 if subpath: 747 if subpath:
777 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, 748 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d,
778 workdir=destdir) 749 workdir=destdir)
779 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) 750 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
780 elif not ud.nobranch: 751 elif not ud.nobranch:
781 branchname = ud.branches[ud.names[0]] 752 branchname = ud.branch
782 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ 753 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
783 ud.revisions[ud.names[0]]), d, workdir=destdir) 754 ud.revision), d, workdir=destdir)
784 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ 755 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
785 branchname), d, workdir=destdir) 756 branchname), d, workdir=destdir)
786 else: 757 else:
787 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) 758 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
788 759
789 return True 760 return True
790 761
@@ -815,10 +786,10 @@ class Git(FetchMethod):
815 cmd = "" 786 cmd = ""
816 if ud.nobranch: 787 if ud.nobranch:
817 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( 788 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
818 ud.basecmd, ud.revisions[name]) 789 ud.basecmd, ud.revision)
819 else: 790 else:
820 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( 791 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
821 ud.basecmd, ud.revisions[name], ud.branches[name]) 792 ud.basecmd, ud.revision, ud.branch)
822 try: 793 try:
823 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 794 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
824 except bb.fetch2.FetchError: 795 except bb.fetch2.FetchError:
@@ -839,7 +810,7 @@ class Git(FetchMethod):
839 # existence. 810 # existence.
840 # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git 811 # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
841 cmd = "%s lfs ls-files -l %s" \ 812 cmd = "%s lfs ls-files -l %s" \
842 % (ud.basecmd, ud.revisions[name]) 813 % (ud.basecmd, ud.revision)
843 output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() 814 output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
844 # Do not do any further matching if no objects are managed by LFS 815 # Do not do any further matching if no objects are managed by LFS
845 if not output: 816 if not output:
@@ -866,12 +837,12 @@ class Git(FetchMethod):
866 837
867 if ud.nobranch: 838 if ud.nobranch:
868 # If no branch is specified, use the current git commit 839 # If no branch is specified, use the current git commit
869 refname = self._build_revision(ud, d, ud.names[0]) 840 refname = self._build_revision(ud, d, ud.name)
870 elif wd == ud.clonedir: 841 elif wd == ud.clonedir:
871 # The bare clonedir doesn't use the remote names; it has the branch immediately. 842 # The bare clonedir doesn't use the remote names; it has the branch immediately.
872 refname = ud.branches[ud.names[0]] 843 refname = ud.branch
873 else: 844 else:
874 refname = "origin/%s" % ud.branches[ud.names[0]] 845 refname = "origin/%s" % ud.branch
875 846
876 cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( 847 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
877 ud.basecmd, refname) 848 ud.basecmd, refname)
@@ -910,7 +881,7 @@ class Git(FetchMethod):
910 Return a unique key for the url 881 Return a unique key for the url
911 """ 882 """
912 # Collapse adjacent slashes 883 # Collapse adjacent slashes
913 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] 884 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev
914 885
915 def _lsremote(self, ud, d, search): 886 def _lsremote(self, ud, d, search):
916 """ 887 """
@@ -943,26 +914,26 @@ class Git(FetchMethod):
943 Compute the HEAD revision for the url 914 Compute the HEAD revision for the url
944 """ 915 """
945 if not d.getVar("__BBSRCREV_SEEN"): 916 if not d.getVar("__BBSRCREV_SEEN"):
946 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path)) 917 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path))
947 918
948 # Ensure we mark as not cached 919 # Ensure we mark as not cached
949 bb.fetch2.mark_recipe_nocache(d) 920 bb.fetch2.mark_recipe_nocache(d)
950 921
951 output = self._lsremote(ud, d, "") 922 output = self._lsremote(ud, d, "")
952 # Tags of the form ^{} may not work, need to fallback to other form 923 # Tags of the form ^{} may not work, need to fallback to other form
953 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: 924 if ud.unresolvedrev[:5] == "refs/" or ud.usehead:
954 head = ud.unresolvedrev[name] 925 head = ud.unresolvedrev
955 tag = ud.unresolvedrev[name] 926 tag = ud.unresolvedrev
956 else: 927 else:
957 head = "refs/heads/%s" % ud.unresolvedrev[name] 928 head = "refs/heads/%s" % ud.unresolvedrev
958 tag = "refs/tags/%s" % ud.unresolvedrev[name] 929 tag = "refs/tags/%s" % ud.unresolvedrev
959 for s in [head, tag + "^{}", tag]: 930 for s in [head, tag + "^{}", tag]:
960 for l in output.strip().split('\n'): 931 for l in output.strip().split('\n'):
961 sha1, ref = l.split() 932 sha1, ref = l.split()
962 if s == ref: 933 if s == ref:
963 return sha1 934 return sha1
964 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ 935 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
965 (ud.unresolvedrev[name], ud.host+ud.path)) 936 (ud.unresolvedrev, ud.host+ud.path))
966 937
967 def latest_versionstring(self, ud, d): 938 def latest_versionstring(self, ud, d):
968 """ 939 """
@@ -1013,7 +984,7 @@ class Git(FetchMethod):
1013 return pupver 984 return pupver
1014 985
1015 def _build_revision(self, ud, d, name): 986 def _build_revision(self, ud, d, name):
1016 return ud.revisions[name] 987 return ud.revision
1017 988
1018 def gitpkgv_revision(self, ud, d, name): 989 def gitpkgv_revision(self, ud, d, name):
1019 """ 990 """
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index 1c58230eac..f514aedaf1 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -62,36 +62,35 @@ class GitSM(Git):
62 return modules 62 return modules
63 63
64 # Collect the defined submodules, and their attributes 64 # Collect the defined submodules, and their attributes
65 for name in ud.names: 65 try:
66 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir)
67 except:
68 # No submodules to update
69 gitmodules = ""
70
71 for m, md in parse_gitmodules(gitmodules).items():
66 try: 72 try:
67 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 73 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir)
68 except: 74 except:
69 # No submodules to update 75 # If the command fails, we don't have a valid file to check. If it doesn't
76 # fail -- it still might be a failure, see next check...
77 module_hash = ""
78
79 if not module_hash:
80 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
70 continue 81 continue
71 82
72 for m, md in parse_gitmodules(gitmodules).items(): 83 submodules.append(m)
73 try: 84 paths[m] = md['path']
74 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 85 revision[m] = ud.revision
75 except: 86 uris[m] = md['url']
76 # If the command fails, we don't have a valid file to check. If it doesn't 87 subrevision[m] = module_hash.split()[2]
77 # fail -- it still might be a failure, see next check... 88
78 module_hash = "" 89 # Convert relative to absolute uri based on parent uri
79 90 if uris[m].startswith('..') or uris[m].startswith('./'):
80 if not module_hash: 91 newud = copy.copy(ud)
81 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) 92 newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
82 continue 93 uris[m] = Git._get_repo_url(self, newud)
83
84 submodules.append(m)
85 paths[m] = md['path']
86 revision[m] = ud.revisions[name]
87 uris[m] = md['url']
88 subrevision[m] = module_hash.split()[2]
89
90 # Convert relative to absolute uri based on parent uri
91 if uris[m].startswith('..') or uris[m].startswith('./'):
92 newud = copy.copy(ud)
93 newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
94 uris[m] = Git._get_repo_url(self, newud)
95 94
96 for module in submodules: 95 for module in submodules:
97 # Translate the module url into a SRC_URI 96 # Translate the module url into a SRC_URI
diff --git a/bitbake/lib/bb/fetch2/gomod.py b/bitbake/lib/bb/fetch2/gomod.py
index 6c999e8ba0..03f649bc2f 100644
--- a/bitbake/lib/bb/fetch2/gomod.py
+++ b/bitbake/lib/bb/fetch2/gomod.py
@@ -190,15 +190,14 @@ class GoModGit(Git):
190 ud.path = '' 190 ud.path = ''
191 if 'protocol' not in ud.parm: 191 if 'protocol' not in ud.parm:
192 ud.parm['protocol'] = 'https' 192 ud.parm['protocol'] = 'https'
193 name = f"{module}@{ud.parm['version']}" 193 ud.name = f"{module}@{ud.parm['version']}"
194 ud.names = [name] 194 srcrev = d.getVar('SRCREV_' + ud.name)
195 srcrev = d.getVar('SRCREV_' + name)
196 if srcrev: 195 if srcrev:
197 if 'srcrev' not in ud.parm: 196 if 'srcrev' not in ud.parm:
198 ud.parm['srcrev'] = srcrev 197 ud.parm['srcrev'] = srcrev
199 else: 198 else:
200 if 'srcrev' in ud.parm: 199 if 'srcrev' in ud.parm:
201 d.setVar('SRCREV_' + name, ud.parm['srcrev']) 200 d.setVar('SRCREV_' + ud.name, ud.parm['srcrev'])
202 if 'branch' not in ud.parm: 201 if 'branch' not in ud.parm:
203 ud.parm['nobranch'] = '1' 202 ud.parm['nobranch'] = '1'
204 203
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 8fa870c8a1..486c10cd08 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -2060,60 +2060,6 @@ class GitShallowTest(FetcherTest):
2060 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 2060 assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
2061 assert os.path.exists(os.path.join(self.gitdir, 'c')) 2061 assert os.path.exists(os.path.join(self.gitdir, 'c'))
2062 2062
2063 def test_shallow_multi_one_uri(self):
2064 # Create initial git repo
2065 self.add_empty_file('a')
2066 self.add_empty_file('b')
2067 self.git('checkout -b a_branch', cwd=self.srcdir)
2068 self.add_empty_file('c')
2069 self.git('tag v0.0 HEAD', cwd=self.srcdir)
2070 self.add_empty_file('d')
2071 self.git('checkout master', cwd=self.srcdir)
2072 self.add_empty_file('e')
2073 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
2074 self.add_empty_file('f')
2075 self.assertRevCount(7, cwd=self.srcdir)
2076
2077 uri = self.d.getVar('SRC_URI').split()[0]
2078 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
2079
2080 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2081 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
2082 self.d.setVar('SRCREV_master', '${AUTOREV}')
2083 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
2084
2085 self.fetch_shallow(uri)
2086
2087 self.assertRevCount(4)
2088 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
2089
2090 def test_shallow_multi_one_uri_depths(self):
2091 # Create initial git repo
2092 self.add_empty_file('a')
2093 self.add_empty_file('b')
2094 self.git('checkout -b a_branch', cwd=self.srcdir)
2095 self.add_empty_file('c')
2096 self.add_empty_file('d')
2097 self.git('checkout master', cwd=self.srcdir)
2098 self.add_empty_file('e')
2099 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
2100 self.add_empty_file('f')
2101 self.assertRevCount(7, cwd=self.srcdir)
2102
2103 uri = self.d.getVar('SRC_URI').split()[0]
2104 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
2105
2106 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
2107 self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
2108 self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
2109 self.d.setVar('SRCREV_master', '${AUTOREV}')
2110 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
2111
2112 self.fetch_shallow(uri)
2113
2114 self.assertRevCount(4, ['--all'])
2115 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
2116
2117 def test_shallow_clone_preferred_over_shallow(self): 2063 def test_shallow_clone_preferred_over_shallow(self):
2118 self.add_empty_file('a') 2064 self.add_empty_file('a')
2119 self.add_empty_file('b') 2065 self.add_empty_file('b')
@@ -3313,58 +3259,6 @@ class FetchPremirroronlyLocalTest(FetcherTest):
3313 with self.assertRaises(bb.fetch2.NetworkAccess): 3259 with self.assertRaises(bb.fetch2.NetworkAccess):
3314 fetcher.download() 3260 fetcher.download()
3315 3261
3316 def test_mirror_tarball_multiple_branches(self):
3317 """
3318 test if PREMIRRORS can handle multiple name/branches correctly
3319 both branches have required revisions
3320 """
3321 self.make_git_repo()
3322 branch1rev = self.git_new_branch("testbranch1")
3323 branch2rev = self.git_new_branch("testbranch2")
3324 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
3325 self.d.setVar("SRCREV_branch1", branch1rev)
3326 self.d.setVar("SRCREV_branch2", branch2rev)
3327 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3328 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
3329 fetcher.download()
3330 fetcher.unpack(os.path.join(self.tempdir, "unpacked"))
3331 unpacked = os.path.join(self.tempdir, "unpacked", "git", self.testfilename)
3332 self.assertTrue(os.path.exists(unpacked), "Repo has not been unpackaged properly!")
3333 with open(unpacked, 'r') as f:
3334 content = f.read()
3335 ## We expect to see testbranch1 in the file, not master, not testbranch2
3336 self.assertTrue(content.find("testbranch1") != -1, "Wrong branch has been checked out!")
3337
3338 def test_mirror_tarball_multiple_branches_nobranch(self):
3339 """
3340 test if PREMIRRORS can handle multiple name/branches correctly
3341 Unbalanced name/branches raises ParameterError
3342 """
3343 self.make_git_repo()
3344 branch1rev = self.git_new_branch("testbranch1")
3345 branch2rev = self.git_new_branch("testbranch2")
3346 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1;protocol=https;name=branch1,branch2"
3347 self.d.setVar("SRCREV_branch1", branch1rev)
3348 self.d.setVar("SRCREV_branch2", branch2rev)
3349 with self.assertRaises(bb.fetch2.ParameterError):
3350 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3351
3352 def test_mirror_tarball_multiple_branches_norev(self):
3353 """
3354 test if PREMIRRORS can handle multiple name/branches correctly
3355 one of the branches specifies non existing SRCREV
3356 """
3357 self.make_git_repo()
3358 branch1rev = self.git_new_branch("testbranch1")
3359 branch2rev = self.git_new_branch("testbranch2")
3360 self.recipe_url = "git://git.fake.repo/bitbake;branch=testbranch1,testbranch2;protocol=https;name=branch1,branch2"
3361 self.d.setVar("SRCREV_branch1", branch1rev)
3362 self.d.setVar("SRCREV_branch2", "0"*40)
3363 fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
3364 self.assertTrue(os.path.exists(self.mirrorfile), "Mirror file doesn't exist")
3365 with self.assertRaises(bb.fetch2.NetworkAccess):
3366 fetcher.download()
3367
3368 3262
3369class FetchPremirroronlyNetworkTest(FetcherTest): 3263class FetchPremirroronlyNetworkTest(FetcherTest):
3370 3264
@@ -3571,7 +3465,7 @@ class GoModGitTest(FetcherTest):
3571 ud = fetcher.ud[urls[0]] 3465 ud = fetcher.ud[urls[0]]
3572 self.assertEqual(ud.host, 'go.googlesource.com') 3466 self.assertEqual(ud.host, 'go.googlesource.com')
3573 self.assertEqual(ud.path, '/net') 3467 self.assertEqual(ud.path, '/net')
3574 self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0']) 3468 self.assertEqual(ud.name, 'golang.org/x/net@v0.9.0')
3575 self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1') 3469 self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1')
3576 3470
3577 fetcher.download() 3471 fetcher.download()
@@ -3597,7 +3491,7 @@ class GoModGitTest(FetcherTest):
3597 self.assertEqual(ud.host, 'github.com') 3491 self.assertEqual(ud.host, 'github.com')
3598 self.assertEqual(ud.path, '/Azure/azure-sdk-for-go') 3492 self.assertEqual(ud.path, '/Azure/azure-sdk-for-go')
3599 self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob') 3493 self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob')
3600 self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0']) 3494 self.assertEqual(ud.name, 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0')
3601 self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3') 3495 self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3')
3602 3496
3603 fetcher.download() 3497 fetcher.download()
@@ -3621,7 +3515,7 @@ class GoModGitTest(FetcherTest):
3621 ud = fetcher.ud[urls[0]] 3515 ud = fetcher.ud[urls[0]]
3622 self.assertEqual(ud.host, 'gopkg.in') 3516 self.assertEqual(ud.host, 'gopkg.in')
3623 self.assertEqual(ud.path, '/ini.v1') 3517 self.assertEqual(ud.path, '/ini.v1')
3624 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3518 self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0')
3625 self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3519 self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951')
3626 3520
3627 fetcher.download() 3521 fetcher.download()
@@ -3643,7 +3537,7 @@ class GoModGitTest(FetcherTest):
3643 ud = fetcher.ud[urls[0]] 3537 ud = fetcher.ud[urls[0]]
3644 self.assertEqual(ud.host, 'gopkg.in') 3538 self.assertEqual(ud.host, 'gopkg.in')
3645 self.assertEqual(ud.path, '/ini.v1') 3539 self.assertEqual(ud.path, '/ini.v1')
3646 self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) 3540 self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0')
3647 self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951') 3541 self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951')
3648 3542
3649 fetcher.download() 3543 fetcher.download()
@@ -3666,7 +3560,7 @@ class GoModGitTest(FetcherTest):
3666 ud = fetcher.ud[urls[0]] 3560 ud = fetcher.ud[urls[0]]
3667 self.assertEqual(ud.host, 'github.com') 3561 self.assertEqual(ud.host, 'github.com')
3668 self.assertEqual(ud.path, '/census-instrumentation/opencensus-go') 3562 self.assertEqual(ud.path, '/census-instrumentation/opencensus-go')
3669 self.assertEqual(ud.names, ['go.opencensus.io@v0.24.0']) 3563 self.assertEqual(ud.name, 'go.opencensus.io@v0.24.0')
3670 self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5') 3564 self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5')
3671 3565
3672 fetcher.download() 3566 fetcher.download()