summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/git.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r--bitbake/lib/bb/fetch2/git.py669
1 files changed, 446 insertions, 223 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index e3ba80a3f5..14ec45a3f6 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -9,15 +9,6 @@ Supported SRC_URI options are:
9- branch 9- branch
10 The git branch to retrieve from. The default is "master" 10 The git branch to retrieve from. The default is "master"
11 11
12 This option also supports multiple branch fetching, with branches
13 separated by commas. In multiple branches case, the name option
14 must have the same number of names to match the branches, which is
15 used to specify the SRC_REV for the branch
16 e.g:
17 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
18 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
19 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
20
21- tag 12- tag
22 The git tag to retrieve. The default is "master" 13 The git tag to retrieve. The default is "master"
23 14
@@ -44,13 +35,27 @@ Supported SRC_URI options are:
44 35
45- nobranch 36- nobranch
46 Don't check the SHA validation for branch. set this option for the recipe 37 Don't check the SHA validation for branch. set this option for the recipe
47 referring to commit which is valid in tag instead of branch. 38 referring to commit which is valid in any namespace (branch, tag, ...)
39 instead of branch.
48 The default is "0", set nobranch=1 if needed. 40 The default is "0", set nobranch=1 if needed.
49 41
42- subpath
43 Limit the checkout to a specific subpath of the tree.
44 By default, checkout the whole tree, set subpath=<path> if needed
45
46- destsuffix
47 The name of the path in which to place the checkout.
48 By default, the path is git/, set destsuffix=<suffix> if needed
49
50- usehead 50- usehead
51 For local git:// urls to use the current branch HEAD as the revision for use with 51 For local git:// urls to use the current branch HEAD as the revision for use with
52 AUTOREV. Implies nobranch. 52 AUTOREV. Implies nobranch.
53 53
54- lfs
55 Enable the checkout to use LFS for large files. This will download all LFS files
56 in the download step, as the unpack step does not have network access.
57 The default is "1", set lfs=0 to skip.
58
54""" 59"""
55 60
56# Copyright (C) 2005 Richard Purdie 61# Copyright (C) 2005 Richard Purdie
@@ -64,15 +69,22 @@ import fnmatch
64import os 69import os
65import re 70import re
66import shlex 71import shlex
72import shutil
67import subprocess 73import subprocess
68import tempfile 74import tempfile
75import urllib
69import bb 76import bb
70import bb.progress 77import bb.progress
78from contextlib import contextmanager
71from bb.fetch2 import FetchMethod 79from bb.fetch2 import FetchMethod
72from bb.fetch2 import runfetchcmd 80from bb.fetch2 import runfetchcmd
73from bb.fetch2 import logger 81from bb.fetch2 import logger
82from bb.fetch2 import trusted_network
74 83
75 84
85sha1_re = re.compile(r'^[0-9a-f]{40}$')
86slash_re = re.compile(r"/+")
87
76class GitProgressHandler(bb.progress.LineFilterProgressHandler): 88class GitProgressHandler(bb.progress.LineFilterProgressHandler):
77 """Extract progress information from git output""" 89 """Extract progress information from git output"""
78 def __init__(self, d): 90 def __init__(self, d):
@@ -130,6 +142,9 @@ class Git(FetchMethod):
130 def supports_checksum(self, urldata): 142 def supports_checksum(self, urldata):
131 return False 143 return False
132 144
145 def cleanup_upon_failure(self):
146 return False
147
133 def urldata_init(self, ud, d): 148 def urldata_init(self, ud, d):
134 """ 149 """
135 init git specific variable within url data 150 init git specific variable within url data
@@ -141,6 +156,11 @@ class Git(FetchMethod):
141 ud.proto = 'file' 156 ud.proto = 'file'
142 else: 157 else:
143 ud.proto = "git" 158 ud.proto = "git"
159 if ud.host == "github.com" and ud.proto == "git":
160 # github stopped supporting git protocol
161 # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
162 ud.proto = "https"
163 bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
144 164
145 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): 165 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
146 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) 166 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -162,18 +182,25 @@ class Git(FetchMethod):
162 ud.bareclone = ud.parm.get("bareclone","0") == "1" 182 ud.bareclone = ud.parm.get("bareclone","0") == "1"
163 if ud.bareclone: 183 if ud.bareclone:
164 ud.nocheckout = 1 184 ud.nocheckout = 1
165
166 ud.unresolvedrev = {}
167 branches = ud.parm.get("branch", "master").split(',')
168 if len(branches) != len(ud.names):
169 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
170 185
171 ud.cloneflags = "-s -n" 186 ud.unresolvedrev = ""
187 ud.branch = ud.parm.get("branch", "")
188 if not ud.branch and not ud.nobranch:
189 raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url)
190
191 ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
192
193 ud.cloneflags = "-n"
194 if not ud.noshared:
195 ud.cloneflags += " -s"
172 if ud.bareclone: 196 if ud.bareclone:
173 ud.cloneflags += " --mirror" 197 ud.cloneflags += " --mirror"
174 198
199 ud.shallow_skip_fast = False
175 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" 200 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
176 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() 201 ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
202 if 'tag' in ud.parm:
203 ud.shallow_extra_refs.append("refs/tags/" + ud.parm['tag'])
177 204
178 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") 205 depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
179 if depth_default is not None: 206 if depth_default is not None:
@@ -190,32 +217,27 @@ class Git(FetchMethod):
190 217
191 revs_default = d.getVar("BB_GIT_SHALLOW_REVS") 218 revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
192 ud.shallow_revs = [] 219 ud.shallow_revs = []
193 ud.branches = {} 220
194 for pos, name in enumerate(ud.names): 221 ud.unresolvedrev = ud.branch
195 branch = branches[pos] 222
196 ud.branches[name] = branch 223 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name)
197 ud.unresolvedrev[name] = branch 224 if shallow_depth is not None:
198 225 try:
199 shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name) 226 shallow_depth = int(shallow_depth or 0)
200 if shallow_depth is not None: 227 except ValueError:
201 try: 228 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
202 shallow_depth = int(shallow_depth or 0) 229 else:
203 except ValueError: 230 if shallow_depth < 0:
204 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 231 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
205 else: 232 ud.shallow_depths[ud.name] = shallow_depth
206 if shallow_depth < 0: 233
207 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 234 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name)
208 ud.shallow_depths[name] = shallow_depth 235 if revs is not None:
209 236 ud.shallow_revs.extend(revs.split())
210 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) 237 elif revs_default is not None:
211 if revs is not None: 238 ud.shallow_revs.extend(revs_default.split())
212 ud.shallow_revs.extend(revs.split()) 239
213 elif revs_default is not None: 240 if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0:
214 ud.shallow_revs.extend(revs_default.split())
215
216 if (ud.shallow and
217 not ud.shallow_revs and
218 all(ud.shallow_depths[n] == 0 for n in ud.names)):
219 # Shallow disabled for this URL 241 # Shallow disabled for this URL
220 ud.shallow = False 242 ud.shallow = False
221 243
@@ -224,10 +246,9 @@ class Git(FetchMethod):
224 # rev of this repository. This will get resolved into a revision 246 # rev of this repository. This will get resolved into a revision
225 # later. If an actual revision happens to have also been provided 247 # later. If an actual revision happens to have also been provided
226 # then this setting will be overridden. 248 # then this setting will be overridden.
227 for name in ud.names: 249 ud.unresolvedrev = 'HEAD'
228 ud.unresolvedrev[name] = 'HEAD'
229 250
230 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" 251 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin"
231 252
232 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" 253 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
233 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable 254 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -235,24 +256,22 @@ class Git(FetchMethod):
235 256
236 ud.setup_revisions(d) 257 ud.setup_revisions(d)
237 258
238 for name in ud.names: 259 # Ensure any revision that doesn't look like a SHA-1 is translated into one
239 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one 260 if not sha1_re.match(ud.revision or ''):
240 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): 261 if ud.revision:
241 if ud.revisions[name]: 262 ud.unresolvedrev = ud.revision
242 ud.unresolvedrev[name] = ud.revisions[name] 263 ud.revision = self.latest_revision(ud, d, ud.name)
243 ud.revisions[name] = self.latest_revision(ud, d, name)
244 264
245 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) 265 gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
246 if gitsrcname.startswith('.'): 266 if gitsrcname.startswith('.'):
247 gitsrcname = gitsrcname[1:] 267 gitsrcname = gitsrcname[1:]
248 268
249 # for rebaseable git repo, it is necessary to keep mirror tar ball 269 # For a rebaseable git repo, it is necessary to keep a mirror tar ball
250 # per revision, so that even the revision disappears from the 270 # per revision, so that even if the revision disappears from the
251 # upstream repo in the future, the mirror will remain intact and still 271 # upstream repo in the future, the mirror will remain intact and still
252 # contains the revision 272 # contain the revision
253 if ud.rebaseable: 273 if ud.rebaseable:
254 for name in ud.names: 274 gitsrcname = gitsrcname + '_' + ud.revision
255 gitsrcname = gitsrcname + '_' + ud.revisions[name]
256 275
257 dl_dir = d.getVar("DL_DIR") 276 dl_dir = d.getVar("DL_DIR")
258 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") 277 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
@@ -270,15 +289,14 @@ class Git(FetchMethod):
270 if ud.shallow_revs: 289 if ud.shallow_revs:
271 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) 290 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
272 291
273 for name, revision in sorted(ud.revisions.items()): 292 tarballname = "%s_%s" % (tarballname, ud.revision[:7])
274 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) 293 depth = ud.shallow_depths[ud.name]
275 depth = ud.shallow_depths[name] 294 if depth:
276 if depth: 295 tarballname = "%s-%s" % (tarballname, depth)
277 tarballname = "%s-%s" % (tarballname, depth)
278 296
279 shallow_refs = [] 297 shallow_refs = []
280 if not ud.nobranch: 298 if not ud.nobranch:
281 shallow_refs.extend(ud.branches.values()) 299 shallow_refs.append(ud.branch)
282 if ud.shallow_extra_refs: 300 if ud.shallow_extra_refs:
283 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) 301 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
284 if shallow_refs: 302 if shallow_refs:
@@ -293,16 +311,29 @@ class Git(FetchMethod):
293 return ud.clonedir 311 return ud.clonedir
294 312
295 def need_update(self, ud, d): 313 def need_update(self, ud, d):
296 return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) 314 return self.clonedir_need_update(ud, d) \
315 or self.shallow_tarball_need_update(ud) \
316 or self.tarball_need_update(ud) \
317 or self.lfs_need_update(ud, d)
297 318
298 def clonedir_need_update(self, ud, d): 319 def clonedir_need_update(self, ud, d):
299 if not os.path.exists(ud.clonedir): 320 if not os.path.exists(ud.clonedir):
300 return True 321 return True
301 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): 322 if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
302 return True 323 return True
303 for name in ud.names: 324 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
304 if not self._contains_ref(ud, d, name, ud.clonedir): 325 return True
305 return True 326 return False
327
328 def lfs_need_update(self, ud, d):
329 if not self._need_lfs(ud):
330 return False
331
332 if self.clonedir_need_update(ud, d):
333 return True
334
335 if not self._lfs_objects_downloaded(ud, d, ud.clonedir):
336 return True
306 return False 337 return False
307 338
308 def clonedir_need_shallow_revs(self, ud, d): 339 def clonedir_need_shallow_revs(self, ud, d):
@@ -319,11 +350,28 @@ class Git(FetchMethod):
319 def tarball_need_update(self, ud): 350 def tarball_need_update(self, ud):
320 return ud.write_tarballs and not os.path.exists(ud.fullmirror) 351 return ud.write_tarballs and not os.path.exists(ud.fullmirror)
321 352
353 def update_mirror_links(self, ud, origud):
354 super().update_mirror_links(ud, origud)
355 # When using shallow mode, add a symlink to the original fullshallow
356 # path to ensure a valid symlink even in the `PREMIRRORS` case
357 if ud.shallow and not os.path.exists(origud.fullshallow):
358 self.ensure_symlink(ud.localpath, origud.fullshallow)
359
322 def try_premirror(self, ud, d): 360 def try_premirror(self, ud, d):
323 # If we don't do this, updating an existing checkout with only premirrors 361 # If we don't do this, updating an existing checkout with only premirrors
324 # is not possible 362 # is not possible
325 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): 363 if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
326 return True 364 return True
365 # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
366 # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
367 # we need to try premirrors first as using upstream is destined to fail.
368 if not trusted_network(d, ud.url):
369 return True
370 # the following check is to ensure incremental fetch in downloads, this is
371 # because the premirror might be old and does not contain the new rev required,
372 # and this will cause a total removal and new clone. So if we can reach to
373 # network, we prefer upstream over premirror, though the premirror might contain
374 # the new rev.
327 if os.path.exists(ud.clonedir): 375 if os.path.exists(ud.clonedir):
328 return False 376 return False
329 return True 377 return True
@@ -337,21 +385,76 @@ class Git(FetchMethod):
337 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): 385 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
338 ud.localpath = ud.fullshallow 386 ud.localpath = ud.fullshallow
339 return 387 return
340 elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): 388 elif os.path.exists(ud.fullmirror) and self.need_update(ud, d):
341 bb.utils.mkdirhier(ud.clonedir) 389 if not os.path.exists(ud.clonedir):
342 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) 390 bb.utils.mkdirhier(ud.clonedir)
343 391 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
392 else:
393 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
394 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
395 output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
396 if 'mirror' in output:
397 runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
398 runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
399 fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd)
400 runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
344 repourl = self._get_repo_url(ud) 401 repourl = self._get_repo_url(ud)
345 402
403 needs_clone = False
404 if os.path.exists(ud.clonedir):
405 # The directory may exist, but not be the top level of a bare git
406 # repository in which case it needs to be deleted and re-cloned.
407 try:
408 # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
409 output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
410 toplevel = output.rstrip()
411
412 if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
413 logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
414 needs_clone = True
415 except bb.fetch2.FetchError as e:
416 logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
417 needs_clone = True
418 except FileNotFoundError as e:
419 logger.warning("%s", e)
420 needs_clone = True
421
422 if needs_clone:
423 shutil.rmtree(ud.clonedir)
424 else:
425 needs_clone = True
426
346 # If the repo still doesn't exist, fallback to cloning it 427 # If the repo still doesn't exist, fallback to cloning it
347 if not os.path.exists(ud.clonedir): 428 if needs_clone:
348 # We do this since git will use a "-l" option automatically for local urls where possible 429 # We do this since git will use a "-l" option automatically for local urls where possible,
430 # but it doesn't work when git/objects is a symlink, only works when it is a directory.
349 if repourl.startswith("file://"): 431 if repourl.startswith("file://"):
350 repourl = repourl[7:] 432 repourl_path = repourl[7:]
433 objects = os.path.join(repourl_path, 'objects')
434 if os.path.isdir(objects) and not os.path.islink(objects):
435 repourl = repourl_path
351 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) 436 clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
352 if ud.proto.lower() != 'file': 437 if ud.proto.lower() != 'file':
353 bb.fetch2.check_network_access(d, clone_cmd, ud.url) 438 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
354 progresshandler = GitProgressHandler(d) 439 progresshandler = GitProgressHandler(d)
440
441 # Try creating a fast initial shallow clone
442 # Enabling ud.shallow_skip_fast will skip this
443 # If the Git error "Server does not allow request for unadvertised object"
444 # occurs, shallow_skip_fast is enabled automatically.
445 # This may happen if the Git server does not allow the request
446 # or if the Git client has issues with this functionality.
447 if ud.shallow and not ud.shallow_skip_fast:
448 try:
449 self.clone_shallow_with_tarball(ud, d)
450 # When the shallow clone has succeeded, use the shallow tarball
451 ud.localpath = ud.fullshallow
452 return
453 except:
454 logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.")
455
456 # When skipping fast initial shallow or the fast inital shallow clone failed:
457 # Try again with an initial regular clone
355 runfetchcmd(clone_cmd, d, log=progresshandler) 458 runfetchcmd(clone_cmd, d, log=progresshandler)
356 459
357 # Update the checkout if needed 460 # Update the checkout if needed
@@ -361,7 +464,11 @@ class Git(FetchMethod):
361 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) 464 runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
362 465
363 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) 466 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
364 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) 467
468 if ud.nobranch:
469 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
470 else:
471 fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
365 if ud.proto.lower() != 'file': 472 if ud.proto.lower() != 'file':
366 bb.fetch2.check_network_access(d, fetch_cmd, ud.url) 473 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
367 progresshandler = GitProgressHandler(d) 474 progresshandler = GitProgressHandler(d)
@@ -375,138 +482,206 @@ class Git(FetchMethod):
375 if exc.errno != errno.ENOENT: 482 if exc.errno != errno.ENOENT:
376 raise 483 raise
377 484
378 for name in ud.names: 485 if not self._contains_ref(ud, d, ud.name, ud.clonedir):
379 if not self._contains_ref(ud, d, name, ud.clonedir): 486 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch))
380 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
381 487
382 if ud.shallow and ud.write_shallow_tarballs: 488 if ud.shallow and ud.write_shallow_tarballs:
383 missing_rev = self.clonedir_need_shallow_revs(ud, d) 489 missing_rev = self.clonedir_need_shallow_revs(ud, d)
384 if missing_rev: 490 if missing_rev:
385 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 491 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
386 492
387 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): 493 if self.lfs_need_update(ud, d):
388 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching 494 self.lfs_fetch(ud, d, ud.clonedir, ud.revision)
389 # of all LFS blobs needed at the the srcrev. 495
390 # 496 def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None):
391 # It would be nice to just do this inline here by running 'git-lfs fetch' 497 """Helper method for fetching Git LFS data"""
392 # on the bare clonedir, but that operation requires a working copy on some 498 try:
393 # releases of Git LFS. 499 if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision):
394 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 500 self._ensure_git_lfs(d, ud)
395 try: 501
396 # Do the checkout. This implicitly involves a Git LFS fetch. 502 # Using worktree with the revision because .lfsconfig may exists
397 self.unpack(ud, tmpdir, d) 503 worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision)
398 504 runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir)
399 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into 505 lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "")
400 # the bare clonedir. 506 runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt"))
401 # 507 worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd
402 # As this procedure is invoked repeatedly on incremental fetches as 508 runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
403 # a recipe's SRCREV is bumped throughout its lifetime, this will 509 except:
404 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs 510 logger.warning("Fetching LFS did not succeed.")
405 # corresponding to all the blobs reachable from the different revs 511
406 # fetched across time. 512 @contextmanager
407 # 513 def create_atomic(self, filename):
408 # Only do this if the unpack resulted in a .git/lfs directory being 514 """Create as a temp file and move atomically into position to avoid races"""
409 # created; this only happens if at least one blob needed to be 515 fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
410 # downloaded. 516 try:
411 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): 517 yield tfile
412 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) 518 umask = os.umask(0o666)
413 finally: 519 os.umask(umask)
414 bb.utils.remove(tmpdir, recurse=True) 520 os.chmod(tfile, (0o666 & ~umask))
521 os.rename(tfile, filename)
522 finally:
523 os.close(fd)
415 524
416 def build_mirror_data(self, ud, d): 525 def build_mirror_data(self, ud, d):
417 if ud.shallow and ud.write_shallow_tarballs: 526 if ud.shallow and ud.write_shallow_tarballs:
418 if not os.path.exists(ud.fullshallow): 527 if not os.path.exists(ud.fullshallow):
419 if os.path.islink(ud.fullshallow): 528 if os.path.islink(ud.fullshallow):
420 os.unlink(ud.fullshallow) 529 os.unlink(ud.fullshallow)
421 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) 530 self.clone_shallow_with_tarball(ud, d)
422 shallowclone = os.path.join(tempdir, 'git')
423 try:
424 self.clone_shallow_local(ud, shallowclone, d)
425
426 logger.info("Creating tarball of git repository")
427 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
428 runfetchcmd("touch %s.done" % ud.fullshallow, d)
429 finally:
430 bb.utils.remove(tempdir, recurse=True)
431 elif ud.write_tarballs and not os.path.exists(ud.fullmirror): 531 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
432 if os.path.islink(ud.fullmirror): 532 if os.path.islink(ud.fullmirror):
433 os.unlink(ud.fullmirror) 533 os.unlink(ud.fullmirror)
434 534
435 logger.info("Creating tarball of git repository") 535 logger.info("Creating tarball of git repository")
436 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) 536 with self.create_atomic(ud.fullmirror) as tfile:
537 mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
538 quiet=True, workdir=ud.clonedir)
539 runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
540 % (tfile, mtime), d, workdir=ud.clonedir)
437 runfetchcmd("touch %s.done" % ud.fullmirror, d) 541 runfetchcmd("touch %s.done" % ud.fullmirror, d)
438 542
543 def clone_shallow_with_tarball(self, ud, d):
544 ret = False
545 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
546 shallowclone = os.path.join(tempdir, 'git')
547 try:
548 try:
549 self.clone_shallow_local(ud, shallowclone, d)
550 except:
551 logger.warning("Fast shallow clone failed, try to skip fast mode now.")
552 bb.utils.remove(tempdir, recurse=True)
553 os.mkdir(tempdir)
554 ud.shallow_skip_fast = True
555 self.clone_shallow_local(ud, shallowclone, d)
556 logger.info("Creating tarball of git repository")
557 with self.create_atomic(ud.fullshallow) as tfile:
558 runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
559 runfetchcmd("touch %s.done" % ud.fullshallow, d)
560 ret = True
561 finally:
562 bb.utils.remove(tempdir, recurse=True)
563
564 return ret
565
439 def clone_shallow_local(self, ud, dest, d): 566 def clone_shallow_local(self, ud, dest, d):
440 """Clone the repo and make it shallow. 567 """
568 Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default):
569 - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev
570 - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev
571 """
441 572
442 The upstream url of the new clone isn't set at this time, as it'll be 573 progresshandler = GitProgressHandler(d)
443 set correctly when unpacked.""" 574 repourl = self._get_repo_url(ud)
444 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d) 575 bb.utils.mkdirhier(dest)
576 init_cmd = "%s init -q" % ud.basecmd
577 if ud.bareclone:
578 init_cmd += " --bare"
579 runfetchcmd(init_cmd, d, workdir=dest)
580 # Use repourl when creating a fast initial shallow clone
581 # Prefer already existing full bare clones if available
582 if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir):
583 remote = shlex.quote(repourl)
584 else:
585 remote = ud.clonedir
586 runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest)
445 587
446 to_parse, shallow_branches = [], [] 588 # Check the histories which should be excluded
447 for name in ud.names: 589 shallow_exclude = ''
448 revision = ud.revisions[name] 590 for revision in ud.shallow_revs:
449 depth = ud.shallow_depths[name] 591 shallow_exclude += " --shallow-exclude=%s" % revision
450 if depth:
451 to_parse.append('%s~%d^{}' % (revision, depth - 1))
452 592
453 # For nobranch, we need a ref, otherwise the commits will be 593 revision = ud.revision
454 # removed, and for non-nobranch, we truncate the branch to our 594 depth = ud.shallow_depths[ud.name]
455 # srcrev, to avoid keeping unnecessary history beyond that.
456 branch = ud.branches[name]
457 if ud.nobranch:
458 ref = "refs/shallow/%s" % name
459 elif ud.bareclone:
460 ref = "refs/heads/%s" % branch
461 else:
462 ref = "refs/remotes/origin/%s" % branch
463 595
464 shallow_branches.append(ref) 596 # The --depth and --shallow-exclude can't be used together
465 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) 597 if depth and shallow_exclude:
598 raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
466 599
467 # Map srcrev+depths to revisions 600 # For nobranch, we need a ref, otherwise the commits will be
468 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest) 601 # removed, and for non-nobranch, we truncate the branch to our
602 # srcrev, to avoid keeping unnecessary history beyond that.
603 branch = ud.branch
604 if ud.nobranch:
605 ref = "refs/shallow/%s" % ud.name
606 elif ud.bareclone:
607 ref = "refs/heads/%s" % branch
608 else:
609 ref = "refs/remotes/origin/%s" % branch
610
611 fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
612 if depth:
613 fetch_cmd += " --depth %s" % depth
614
615 if shallow_exclude:
616 fetch_cmd += shallow_exclude
469 617
470 # Resolve specified revisions 618 # Advertise the revision for lower version git such as 2.25.1:
471 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest) 619 # error: Server does not allow request for unadvertised object.
472 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines() 620 # The ud.clonedir is a local temporary dir, will be removed when
621 # fetch is done, so we can do anything on it.
622 adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
623 if ud.shallow_skip_fast:
624 runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
625
626 runfetchcmd(fetch_cmd, d, workdir=dest)
627 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
628 # Fetch Git LFS data
629 self.lfs_fetch(ud, d, dest, ud.revision)
473 630
474 # Apply extra ref wildcards 631 # Apply extra ref wildcards
475 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, 632 all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
476 d, workdir=dest).splitlines() 633 d, workdir=dest).splitlines()
634 all_refs = []
635 for line in all_refs_remote:
636 all_refs.append(line.split()[-1])
637 extra_refs = []
477 for r in ud.shallow_extra_refs: 638 for r in ud.shallow_extra_refs:
478 if not ud.bareclone: 639 if not ud.bareclone:
479 r = r.replace('refs/heads/', 'refs/remotes/origin/') 640 r = r.replace('refs/heads/', 'refs/remotes/origin/')
480 641
481 if '*' in r: 642 if '*' in r:
482 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) 643 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
483 shallow_branches.extend(matches) 644 extra_refs.extend(matches)
484 else: 645 else:
485 shallow_branches.append(r) 646 extra_refs.append(r)
647
648 for ref in extra_refs:
649 ref_fetch = ref.replace('refs/heads/', '').replace('refs/remotes/origin/', '').replace('refs/tags/', '')
650 runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest)
651 revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest)
652 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
486 653
487 # Make the repository shallow 654 # The url is local ud.clonedir, set it to upstream one
488 shallow_cmd = [self.make_shallow_path, '-s'] 655 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest)
489 for b in shallow_branches:
490 shallow_cmd.append('-r')
491 shallow_cmd.append(b)
492 shallow_cmd.extend(shallow_revisions)
493 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
494 656
495 def unpack(self, ud, destdir, d): 657 def unpack(self, ud, destdir, d):
496 """ unpack the downloaded src to destdir""" 658 """ unpack the downloaded src to destdir"""
497 659
498 subdir = ud.parm.get("subpath", "") 660 subdir = ud.parm.get("subdir")
499 if subdir != "": 661 subpath = ud.parm.get("subpath")
500 readpathspec = ":%s" % subdir 662 readpathspec = ""
501 def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) 663 def_destsuffix = (d.getVar("BB_GIT_DEFAULT_DESTSUFFIX") or "git") + "/"
502 else: 664
503 readpathspec = "" 665 if subpath:
504 def_destsuffix = "git/" 666 readpathspec = ":%s" % subpath
667 def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
668
669 if subdir:
670 # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
671 if os.path.isabs(subdir):
672 if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
673 raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
674 destdir = subdir
675 else:
676 destdir = os.path.join(destdir, subdir)
677 def_destsuffix = ""
505 678
506 destsuffix = ud.parm.get("destsuffix", def_destsuffix) 679 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
507 destdir = ud.destdir = os.path.join(destdir, destsuffix) 680 destdir = ud.destdir = os.path.join(destdir, destsuffix)
508 if os.path.exists(destdir): 681 if os.path.exists(destdir):
509 bb.utils.prunedir(destdir) 682 bb.utils.prunedir(destdir)
683 if not ud.bareclone:
684 ud.unpack_tracer.unpack("git", destdir)
510 685
511 need_lfs = self._need_lfs(ud) 686 need_lfs = self._need_lfs(ud)
512 687
@@ -516,13 +691,12 @@ class Git(FetchMethod):
516 source_found = False 691 source_found = False
517 source_error = [] 692 source_error = []
518 693
519 if not source_found: 694 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
520 clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) 695 if clonedir_is_up_to_date:
521 if clonedir_is_up_to_date: 696 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
522 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) 697 source_found = True
523 source_found = True 698 else:
524 else: 699 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
525 source_error.append("clone directory not available or not up to date: " + ud.clonedir)
526 700
527 if not source_found: 701 if not source_found:
528 if ud.shallow: 702 if ud.shallow:
@@ -538,28 +712,43 @@ class Git(FetchMethod):
538 if not source_found: 712 if not source_found:
539 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) 713 raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
540 714
715 # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag
716 # matches the revision
717 if 'tag' in ud.parm and sha1_re.match(ud.revision):
718 output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir)
719 output = output.strip()
720 if output != ud.revision:
721 # It is possible ud.revision is the revision on an annotated tag which won't match the output of rev-list
722 # If it resolves to the same thing there isn't a problem.
723 output2 = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
724 output2 = output2.strip()
725 if output != output2:
726 raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url)
727
541 repourl = self._get_repo_url(ud) 728 repourl = self._get_repo_url(ud)
542 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) 729 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
543 730
544 if self._contains_lfs(ud, d, destdir): 731 if self._contains_lfs(ud, d, destdir):
545 if need_lfs and not self._find_git_lfs(d): 732 if not need_lfs:
546 raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
547 elif not need_lfs:
548 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) 733 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
734 else:
735 self._ensure_git_lfs(d, ud)
736
737 runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
549 738
550 if not ud.nocheckout: 739 if not ud.nocheckout:
551 if subdir != "": 740 if subpath:
552 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, 741 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d,
553 workdir=destdir) 742 workdir=destdir)
554 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) 743 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
555 elif not ud.nobranch: 744 elif not ud.nobranch:
556 branchname = ud.branches[ud.names[0]] 745 branchname = ud.branch
557 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ 746 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
558 ud.revisions[ud.names[0]]), d, workdir=destdir) 747 ud.revision), d, workdir=destdir)
559 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ 748 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
560 branchname), d, workdir=destdir) 749 branchname), d, workdir=destdir)
561 else: 750 else:
562 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) 751 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
563 752
564 return True 753 return True
565 754
@@ -573,8 +762,13 @@ class Git(FetchMethod):
573 clonedir = os.path.realpath(ud.localpath) 762 clonedir = os.path.realpath(ud.localpath)
574 to_remove.append(clonedir) 763 to_remove.append(clonedir)
575 764
765 # Remove shallow mirror tarball
766 if ud.shallow:
767 to_remove.append(ud.fullshallow)
768 to_remove.append(ud.fullshallow + ".done")
769
576 for r in to_remove: 770 for r in to_remove:
577 if os.path.exists(r): 771 if os.path.exists(r) or os.path.islink(r):
578 bb.note('Removing %s' % r) 772 bb.note('Removing %s' % r)
579 bb.utils.remove(r, True) 773 bb.utils.remove(r, True)
580 774
@@ -585,10 +779,10 @@ class Git(FetchMethod):
585 cmd = "" 779 cmd = ""
586 if ud.nobranch: 780 if ud.nobranch:
587 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( 781 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
588 ud.basecmd, ud.revisions[name]) 782 ud.basecmd, ud.revision)
589 else: 783 else:
590 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( 784 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
591 ud.basecmd, ud.revisions[name], ud.branches[name]) 785 ud.basecmd, ud.revision, ud.branch)
592 try: 786 try:
593 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 787 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
594 except bb.fetch2.FetchError: 788 except bb.fetch2.FetchError:
@@ -597,6 +791,37 @@ class Git(FetchMethod):
597 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 791 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
598 return output.split()[0] != "0" 792 return output.split()[0] != "0"
599 793
794 def _lfs_objects_downloaded(self, ud, d, wd):
795 """
796 Verifies whether the LFS objects for requested revisions have already been downloaded
797 """
798 # Bail out early if this repository doesn't use LFS
799 if not self._contains_lfs(ud, d, wd):
800 return True
801
802 self._ensure_git_lfs(d, ud)
803
804 # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
805 # existence.
806 # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
807 cmd = "%s lfs ls-files -l %s" \
808 % (ud.basecmd, ud.revision)
809 output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
810 # Do not do any further matching if no objects are managed by LFS
811 if not output:
812 return True
813
814 # Match all lines beginning with the hexadecimal OID
815 oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
816 for line in output.split("\n"):
817 oid = re.search(oid_regex, line)
818 if not oid:
819 bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
820 if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
821 return False
822
823 return True
824
600 def _need_lfs(self, ud): 825 def _need_lfs(self, ud):
601 return ud.parm.get("lfs", "1") == "1" 826 return ud.parm.get("lfs", "1") == "1"
602 827
@@ -604,20 +829,8 @@ class Git(FetchMethod):
604 """ 829 """
605 Check if the repository has 'lfs' (large file) content 830 Check if the repository has 'lfs' (large file) content
606 """ 831 """
607
608 if not ud.nobranch:
609 branchname = ud.branches[ud.names[0]]
610 else:
611 branchname = "master"
612
613 # The bare clonedir doesn't use the remote names; it has the branch immediately.
614 if wd == ud.clonedir:
615 refname = ud.branches[ud.names[0]]
616 else:
617 refname = "origin/%s" % ud.branches[ud.names[0]]
618
619 cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( 832 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
620 ud.basecmd, refname) 833 ud.basecmd, ud.revision)
621 834
622 try: 835 try:
623 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 836 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
@@ -627,12 +840,14 @@ class Git(FetchMethod):
627 pass 840 pass
628 return False 841 return False
629 842
630 def _find_git_lfs(self, d): 843 def _ensure_git_lfs(self, d, ud):
631 """ 844 """
632 Return True if git-lfs can be found, False otherwise. 845 Ensures that git-lfs is available, raising a FetchError if it isn't.
633 """ 846 """
634 import shutil 847 if shutil.which("git-lfs", path=d.getVar('PATH')) is None:
635 return shutil.which("git-lfs", path=d.getVar('PATH')) is not None 848 raise bb.fetch2.FetchError(
849 "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 "
850 "to ignore it)" % self._get_repo_url(ud))
636 851
637 def _get_repo_url(self, ud): 852 def _get_repo_url(self, ud):
638 """ 853 """
@@ -640,22 +855,21 @@ class Git(FetchMethod):
640 """ 855 """
641 # Note that we do not support passwords directly in the git urls. There are several 856 # Note that we do not support passwords directly in the git urls. There are several
642 # reasons. SRC_URI can be written out to things like buildhistory and people don't 857 # reasons. SRC_URI can be written out to things like buildhistory and people don't
643 # want to leak passwords like that. Its also all too easy to share metadata without 858 # want to leak passwords like that. Its also all too easy to share metadata without
644 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as 859 # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
645 # alternatives so we will not take patches adding password support here. 860 # alternatives so we will not take patches adding password support here.
646 if ud.user: 861 if ud.user:
647 username = ud.user + '@' 862 username = ud.user + '@'
648 else: 863 else:
649 username = "" 864 username = ""
650 return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) 865 return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path))
651 866
652 def _revision_key(self, ud, d, name): 867 def _revision_key(self, ud, d, name):
653 """ 868 """
654 Return a unique key for the url 869 Return a unique key for the url
655 """ 870 """
656 # Collapse adjacent slashes 871 # Collapse adjacent slashes
657 slash_re = re.compile(r"/+") 872 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev
658 return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
659 873
660 def _lsremote(self, ud, d, search): 874 def _lsremote(self, ud, d, search):
661 """ 875 """
@@ -687,21 +901,27 @@ class Git(FetchMethod):
687 """ 901 """
688 Compute the HEAD revision for the url 902 Compute the HEAD revision for the url
689 """ 903 """
904 if not d.getVar("__BBSRCREV_SEEN"):
905 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path))
906
907 # Ensure we mark as not cached
908 bb.fetch2.mark_recipe_nocache(d)
909
690 output = self._lsremote(ud, d, "") 910 output = self._lsremote(ud, d, "")
691 # Tags of the form ^{} may not work, need to fallback to other form 911 # Tags of the form ^{} may not work, need to fallback to other form
692 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: 912 if ud.unresolvedrev[:5] == "refs/" or ud.usehead:
693 head = ud.unresolvedrev[name] 913 head = ud.unresolvedrev
694 tag = ud.unresolvedrev[name] 914 tag = ud.unresolvedrev
695 else: 915 else:
696 head = "refs/heads/%s" % ud.unresolvedrev[name] 916 head = "refs/heads/%s" % ud.unresolvedrev
697 tag = "refs/tags/%s" % ud.unresolvedrev[name] 917 tag = "refs/tags/%s" % ud.unresolvedrev
698 for s in [head, tag + "^{}", tag]: 918 for s in [head, tag + "^{}", tag]:
699 for l in output.strip().split('\n'): 919 for l in output.strip().split('\n'):
700 sha1, ref = l.split() 920 sha1, ref = l.split()
701 if s == ref: 921 if s == ref:
702 return sha1 922 return sha1
703 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ 923 raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
704 (ud.unresolvedrev[name], ud.host+ud.path)) 924 (ud.unresolvedrev, ud.host+ud.path))
705 925
706 def latest_versionstring(self, ud, d): 926 def latest_versionstring(self, ud, d):
707 """ 927 """
@@ -711,60 +931,63 @@ class Git(FetchMethod):
711 """ 931 """
712 pupver = ('', '') 932 pupver = ('', '')
713 933
714 tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
715 try: 934 try:
716 output = self._lsremote(ud, d, "refs/tags/*") 935 output = self._lsremote(ud, d, "refs/tags/*")
717 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: 936 except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
718 bb.note("Could not list remote: %s" % str(e)) 937 bb.note("Could not list remote: %s" % str(e))
719 return pupver 938 return pupver
720 939
940 rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
941 pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
942 nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
943
721 verstring = "" 944 verstring = ""
722 revision = ""
723 for line in output.split("\n"): 945 for line in output.split("\n"):
724 if not line: 946 if not line:
725 break 947 break
726 948
727 tag_head = line.split("/")[-1] 949 m = rev_tag_re.match(line)
950 if not m:
951 continue
952
953 (revision, tag) = m.groups()
954
728 # Ignore non-released branches 955 # Ignore non-released branches
729 m = re.search(r"(alpha|beta|rc|final)+", tag_head) 956 if nonrel_re.search(tag):
730 if m:
731 continue 957 continue
732 958
733 # search for version in the line 959 # search for version in the line
734 tag = tagregex.search(tag_head) 960 m = pver_re.search(tag)
735 if tag is None: 961 if not m:
736 continue 962 continue
737 963
738 tag = tag.group('pver') 964 pver = m.group('pver').replace("_", ".")
739 tag = tag.replace("_", ".")
740 965
741 if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: 966 if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
742 continue 967 continue
743 968
744 verstring = tag 969 verstring = pver
745 revision = line.split()[0]
746 pupver = (verstring, revision) 970 pupver = (verstring, revision)
747 971
748 return pupver 972 return pupver
749 973
750 def _build_revision(self, ud, d, name): 974 def _build_revision(self, ud, d, name):
751 return ud.revisions[name] 975 return ud.revision
752 976
753 def gitpkgv_revision(self, ud, d, name): 977 def gitpkgv_revision(self, ud, d, name):
754 """ 978 """
755 Return a sortable revision number by counting commits in the history 979 Return a sortable revision number by counting commits in the history
756 Based on gitpkgv.bblass in meta-openembedded 980 Based on gitpkgv.bblass in meta-openembedded
757 """ 981 """
758 rev = self._build_revision(ud, d, name) 982 rev = ud.revision
759 localpath = ud.localpath 983 localpath = ud.localpath
760 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) 984 rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
761 if not os.path.exists(localpath): 985 if not os.path.exists(localpath):
762 commits = None 986 commits = None
763 else: 987 else:
764 if not os.path.exists(rev_file) or not os.path.getsize(rev_file): 988 if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
765 from pipes import quote
766 commits = bb.fetch2.runfetchcmd( 989 commits = bb.fetch2.runfetchcmd(
767 "git rev-list %s -- | wc -l" % quote(rev), 990 "git rev-list %s -- | wc -l" % shlex.quote(rev),
768 d, quiet=True).strip().lstrip('0') 991 d, quiet=True).strip().lstrip('0')
769 if commits: 992 if commits:
770 open(rev_file, "w").write("%d\n" % int(commits)) 993 open(rev_file, "w").write("%d\n" % int(commits))