diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 669 |
1 files changed, 446 insertions, 223 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index e3ba80a3f5..14ec45a3f6 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -9,15 +9,6 @@ Supported SRC_URI options are: | |||
9 | - branch | 9 | - branch |
10 | The git branch to retrieve from. The default is "master" | 10 | The git branch to retrieve from. The default is "master" |
11 | 11 | ||
12 | This option also supports multiple branch fetching, with branches | ||
13 | separated by commas. In multiple branches case, the name option | ||
14 | must have the same number of names to match the branches, which is | ||
15 | used to specify the SRC_REV for the branch | ||
16 | e.g: | ||
17 | SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" | ||
18 | SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" | ||
19 | SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" | ||
20 | |||
21 | - tag | 12 | - tag |
22 | The git tag to retrieve. The default is "master" | 13 | The git tag to retrieve. The default is "master" |
23 | 14 | ||
@@ -44,13 +35,27 @@ Supported SRC_URI options are: | |||
44 | 35 | ||
45 | - nobranch | 36 | - nobranch |
46 | Don't check the SHA validation for branch. set this option for the recipe | 37 | Don't check the SHA validation for branch. set this option for the recipe |
47 | referring to commit which is valid in tag instead of branch. | 38 | referring to commit which is valid in any namespace (branch, tag, ...) |
39 | instead of branch. | ||
48 | The default is "0", set nobranch=1 if needed. | 40 | The default is "0", set nobranch=1 if needed. |
49 | 41 | ||
42 | - subpath | ||
43 | Limit the checkout to a specific subpath of the tree. | ||
44 | By default, checkout the whole tree, set subpath=<path> if needed | ||
45 | |||
46 | - destsuffix | ||
47 | The name of the path in which to place the checkout. | ||
48 | By default, the path is git/, set destsuffix=<suffix> if needed | ||
49 | |||
50 | - usehead | 50 | - usehead |
51 | For local git:// urls to use the current branch HEAD as the revision for use with | 51 | For local git:// urls to use the current branch HEAD as the revision for use with |
52 | AUTOREV. Implies nobranch. | 52 | AUTOREV. Implies nobranch. |
53 | 53 | ||
54 | - lfs | ||
55 | Enable the checkout to use LFS for large files. This will download all LFS files | ||
56 | in the download step, as the unpack step does not have network access. | ||
57 | The default is "1", set lfs=0 to skip. | ||
58 | |||
54 | """ | 59 | """ |
55 | 60 | ||
56 | # Copyright (C) 2005 Richard Purdie | 61 | # Copyright (C) 2005 Richard Purdie |
@@ -64,15 +69,22 @@ import fnmatch | |||
64 | import os | 69 | import os |
65 | import re | 70 | import re |
66 | import shlex | 71 | import shlex |
72 | import shutil | ||
67 | import subprocess | 73 | import subprocess |
68 | import tempfile | 74 | import tempfile |
75 | import urllib | ||
69 | import bb | 76 | import bb |
70 | import bb.progress | 77 | import bb.progress |
78 | from contextlib import contextmanager | ||
71 | from bb.fetch2 import FetchMethod | 79 | from bb.fetch2 import FetchMethod |
72 | from bb.fetch2 import runfetchcmd | 80 | from bb.fetch2 import runfetchcmd |
73 | from bb.fetch2 import logger | 81 | from bb.fetch2 import logger |
82 | from bb.fetch2 import trusted_network | ||
74 | 83 | ||
75 | 84 | ||
85 | sha1_re = re.compile(r'^[0-9a-f]{40}$') | ||
86 | slash_re = re.compile(r"/+") | ||
87 | |||
76 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): | 88 | class GitProgressHandler(bb.progress.LineFilterProgressHandler): |
77 | """Extract progress information from git output""" | 89 | """Extract progress information from git output""" |
78 | def __init__(self, d): | 90 | def __init__(self, d): |
@@ -130,6 +142,9 @@ class Git(FetchMethod): | |||
130 | def supports_checksum(self, urldata): | 142 | def supports_checksum(self, urldata): |
131 | return False | 143 | return False |
132 | 144 | ||
145 | def cleanup_upon_failure(self): | ||
146 | return False | ||
147 | |||
133 | def urldata_init(self, ud, d): | 148 | def urldata_init(self, ud, d): |
134 | """ | 149 | """ |
135 | init git specific variable within url data | 150 | init git specific variable within url data |
@@ -141,6 +156,11 @@ class Git(FetchMethod): | |||
141 | ud.proto = 'file' | 156 | ud.proto = 'file' |
142 | else: | 157 | else: |
143 | ud.proto = "git" | 158 | ud.proto = "git" |
159 | if ud.host == "github.com" and ud.proto == "git": | ||
160 | # github stopped supporting git protocol | ||
161 | # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git | ||
162 | ud.proto = "https" | ||
163 | bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url) | ||
144 | 164 | ||
145 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): | 165 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): |
146 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) | 166 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) |
@@ -162,18 +182,25 @@ class Git(FetchMethod): | |||
162 | ud.bareclone = ud.parm.get("bareclone","0") == "1" | 182 | ud.bareclone = ud.parm.get("bareclone","0") == "1" |
163 | if ud.bareclone: | 183 | if ud.bareclone: |
164 | ud.nocheckout = 1 | 184 | ud.nocheckout = 1 |
165 | |||
166 | ud.unresolvedrev = {} | ||
167 | branches = ud.parm.get("branch", "master").split(',') | ||
168 | if len(branches) != len(ud.names): | ||
169 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) | ||
170 | 185 | ||
171 | ud.cloneflags = "-s -n" | 186 | ud.unresolvedrev = "" |
187 | ud.branch = ud.parm.get("branch", "") | ||
188 | if not ud.branch and not ud.nobranch: | ||
189 | raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url) | ||
190 | |||
191 | ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1" | ||
192 | |||
193 | ud.cloneflags = "-n" | ||
194 | if not ud.noshared: | ||
195 | ud.cloneflags += " -s" | ||
172 | if ud.bareclone: | 196 | if ud.bareclone: |
173 | ud.cloneflags += " --mirror" | 197 | ud.cloneflags += " --mirror" |
174 | 198 | ||
199 | ud.shallow_skip_fast = False | ||
175 | ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" | 200 | ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1" |
176 | ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() | 201 | ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split() |
202 | if 'tag' in ud.parm: | ||
203 | ud.shallow_extra_refs.append("refs/tags/" + ud.parm['tag']) | ||
177 | 204 | ||
178 | depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") | 205 | depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH") |
179 | if depth_default is not None: | 206 | if depth_default is not None: |
@@ -190,32 +217,27 @@ class Git(FetchMethod): | |||
190 | 217 | ||
191 | revs_default = d.getVar("BB_GIT_SHALLOW_REVS") | 218 | revs_default = d.getVar("BB_GIT_SHALLOW_REVS") |
192 | ud.shallow_revs = [] | 219 | ud.shallow_revs = [] |
193 | ud.branches = {} | 220 | |
194 | for pos, name in enumerate(ud.names): | 221 | ud.unresolvedrev = ud.branch |
195 | branch = branches[pos] | 222 | |
196 | ud.branches[name] = branch | 223 | shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name) |
197 | ud.unresolvedrev[name] = branch | 224 | if shallow_depth is not None: |
198 | 225 | try: | |
199 | shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name) | 226 | shallow_depth = int(shallow_depth or 0) |
200 | if shallow_depth is not None: | 227 | except ValueError: |
201 | try: | 228 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth)) |
202 | shallow_depth = int(shallow_depth or 0) | 229 | else: |
203 | except ValueError: | 230 | if shallow_depth < 0: |
204 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) | 231 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth)) |
205 | else: | 232 | ud.shallow_depths[ud.name] = shallow_depth |
206 | if shallow_depth < 0: | 233 | |
207 | raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) | 234 | revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name) |
208 | ud.shallow_depths[name] = shallow_depth | 235 | if revs is not None: |
209 | 236 | ud.shallow_revs.extend(revs.split()) | |
210 | revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name) | 237 | elif revs_default is not None: |
211 | if revs is not None: | 238 | ud.shallow_revs.extend(revs_default.split()) |
212 | ud.shallow_revs.extend(revs.split()) | 239 | |
213 | elif revs_default is not None: | 240 | if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0: |
214 | ud.shallow_revs.extend(revs_default.split()) | ||
215 | |||
216 | if (ud.shallow and | ||
217 | not ud.shallow_revs and | ||
218 | all(ud.shallow_depths[n] == 0 for n in ud.names)): | ||
219 | # Shallow disabled for this URL | 241 | # Shallow disabled for this URL |
220 | ud.shallow = False | 242 | ud.shallow = False |
221 | 243 | ||
@@ -224,10 +246,9 @@ class Git(FetchMethod): | |||
224 | # rev of this repository. This will get resolved into a revision | 246 | # rev of this repository. This will get resolved into a revision |
225 | # later. If an actual revision happens to have also been provided | 247 | # later. If an actual revision happens to have also been provided |
226 | # then this setting will be overridden. | 248 | # then this setting will be overridden. |
227 | for name in ud.names: | 249 | ud.unresolvedrev = 'HEAD' |
228 | ud.unresolvedrev[name] = 'HEAD' | ||
229 | 250 | ||
230 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" | 251 | ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin" |
231 | 252 | ||
232 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" | 253 | write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0" |
233 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable | 254 | ud.write_tarballs = write_tarballs != "0" or ud.rebaseable |
@@ -235,24 +256,22 @@ class Git(FetchMethod): | |||
235 | 256 | ||
236 | ud.setup_revisions(d) | 257 | ud.setup_revisions(d) |
237 | 258 | ||
238 | for name in ud.names: | 259 | # Ensure any revision that doesn't look like a SHA-1 is translated into one |
239 | # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one | 260 | if not sha1_re.match(ud.revision or ''): |
240 | if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): | 261 | if ud.revision: |
241 | if ud.revisions[name]: | 262 | ud.unresolvedrev = ud.revision |
242 | ud.unresolvedrev[name] = ud.revisions[name] | 263 | ud.revision = self.latest_revision(ud, d, ud.name) |
243 | ud.revisions[name] = self.latest_revision(ud, d, name) | ||
244 | 264 | ||
245 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_')) | 265 | gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_')) |
246 | if gitsrcname.startswith('.'): | 266 | if gitsrcname.startswith('.'): |
247 | gitsrcname = gitsrcname[1:] | 267 | gitsrcname = gitsrcname[1:] |
248 | 268 | ||
249 | # for rebaseable git repo, it is necessary to keep mirror tar ball | 269 | # For a rebaseable git repo, it is necessary to keep a mirror tar ball |
250 | # per revision, so that even the revision disappears from the | 270 | # per revision, so that even if the revision disappears from the |
251 | # upstream repo in the future, the mirror will remain intact and still | 271 | # upstream repo in the future, the mirror will remain intact and still |
252 | # contains the revision | 272 | # contain the revision |
253 | if ud.rebaseable: | 273 | if ud.rebaseable: |
254 | for name in ud.names: | 274 | gitsrcname = gitsrcname + '_' + ud.revision |
255 | gitsrcname = gitsrcname + '_' + ud.revisions[name] | ||
256 | 275 | ||
257 | dl_dir = d.getVar("DL_DIR") | 276 | dl_dir = d.getVar("DL_DIR") |
258 | gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") | 277 | gitdir = d.getVar("GITDIR") or (dl_dir + "/git2") |
@@ -270,15 +289,14 @@ class Git(FetchMethod): | |||
270 | if ud.shallow_revs: | 289 | if ud.shallow_revs: |
271 | tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) | 290 | tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs))) |
272 | 291 | ||
273 | for name, revision in sorted(ud.revisions.items()): | 292 | tarballname = "%s_%s" % (tarballname, ud.revision[:7]) |
274 | tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) | 293 | depth = ud.shallow_depths[ud.name] |
275 | depth = ud.shallow_depths[name] | 294 | if depth: |
276 | if depth: | 295 | tarballname = "%s-%s" % (tarballname, depth) |
277 | tarballname = "%s-%s" % (tarballname, depth) | ||
278 | 296 | ||
279 | shallow_refs = [] | 297 | shallow_refs = [] |
280 | if not ud.nobranch: | 298 | if not ud.nobranch: |
281 | shallow_refs.extend(ud.branches.values()) | 299 | shallow_refs.append(ud.branch) |
282 | if ud.shallow_extra_refs: | 300 | if ud.shallow_extra_refs: |
283 | shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) | 301 | shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs) |
284 | if shallow_refs: | 302 | if shallow_refs: |
@@ -293,16 +311,29 @@ class Git(FetchMethod): | |||
293 | return ud.clonedir | 311 | return ud.clonedir |
294 | 312 | ||
295 | def need_update(self, ud, d): | 313 | def need_update(self, ud, d): |
296 | return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) | 314 | return self.clonedir_need_update(ud, d) \ |
315 | or self.shallow_tarball_need_update(ud) \ | ||
316 | or self.tarball_need_update(ud) \ | ||
317 | or self.lfs_need_update(ud, d) | ||
297 | 318 | ||
298 | def clonedir_need_update(self, ud, d): | 319 | def clonedir_need_update(self, ud, d): |
299 | if not os.path.exists(ud.clonedir): | 320 | if not os.path.exists(ud.clonedir): |
300 | return True | 321 | return True |
301 | if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): | 322 | if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): |
302 | return True | 323 | return True |
303 | for name in ud.names: | 324 | if not self._contains_ref(ud, d, ud.name, ud.clonedir): |
304 | if not self._contains_ref(ud, d, name, ud.clonedir): | 325 | return True |
305 | return True | 326 | return False |
327 | |||
328 | def lfs_need_update(self, ud, d): | ||
329 | if not self._need_lfs(ud): | ||
330 | return False | ||
331 | |||
332 | if self.clonedir_need_update(ud, d): | ||
333 | return True | ||
334 | |||
335 | if not self._lfs_objects_downloaded(ud, d, ud.clonedir): | ||
336 | return True | ||
306 | return False | 337 | return False |
307 | 338 | ||
308 | def clonedir_need_shallow_revs(self, ud, d): | 339 | def clonedir_need_shallow_revs(self, ud, d): |
@@ -319,11 +350,28 @@ class Git(FetchMethod): | |||
319 | def tarball_need_update(self, ud): | 350 | def tarball_need_update(self, ud): |
320 | return ud.write_tarballs and not os.path.exists(ud.fullmirror) | 351 | return ud.write_tarballs and not os.path.exists(ud.fullmirror) |
321 | 352 | ||
353 | def update_mirror_links(self, ud, origud): | ||
354 | super().update_mirror_links(ud, origud) | ||
355 | # When using shallow mode, add a symlink to the original fullshallow | ||
356 | # path to ensure a valid symlink even in the `PREMIRRORS` case | ||
357 | if ud.shallow and not os.path.exists(origud.fullshallow): | ||
358 | self.ensure_symlink(ud.localpath, origud.fullshallow) | ||
359 | |||
322 | def try_premirror(self, ud, d): | 360 | def try_premirror(self, ud, d): |
323 | # If we don't do this, updating an existing checkout with only premirrors | 361 | # If we don't do this, updating an existing checkout with only premirrors |
324 | # is not possible | 362 | # is not possible |
325 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): | 363 | if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): |
326 | return True | 364 | return True |
365 | # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0 | ||
366 | # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then | ||
367 | # we need to try premirrors first as using upstream is destined to fail. | ||
368 | if not trusted_network(d, ud.url): | ||
369 | return True | ||
370 | # the following check is to ensure incremental fetch in downloads, this is | ||
371 | # because the premirror might be old and does not contain the new rev required, | ||
372 | # and this will cause a total removal and new clone. So if we can reach to | ||
373 | # network, we prefer upstream over premirror, though the premirror might contain | ||
374 | # the new rev. | ||
327 | if os.path.exists(ud.clonedir): | 375 | if os.path.exists(ud.clonedir): |
328 | return False | 376 | return False |
329 | return True | 377 | return True |
@@ -337,21 +385,76 @@ class Git(FetchMethod): | |||
337 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): | 385 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): |
338 | ud.localpath = ud.fullshallow | 386 | ud.localpath = ud.fullshallow |
339 | return | 387 | return |
340 | elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir): | 388 | elif os.path.exists(ud.fullmirror) and self.need_update(ud, d): |
341 | bb.utils.mkdirhier(ud.clonedir) | 389 | if not os.path.exists(ud.clonedir): |
342 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | 390 | bb.utils.mkdirhier(ud.clonedir) |
343 | 391 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) | |
392 | else: | ||
393 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | ||
394 | runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir) | ||
395 | output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) | ||
396 | if 'mirror' in output: | ||
397 | runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir) | ||
398 | runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir) | ||
399 | fetch_cmd = "LANG=C %s fetch -f --update-head-ok --progress mirror " % (ud.basecmd) | ||
400 | runfetchcmd(fetch_cmd, d, workdir=ud.clonedir) | ||
344 | repourl = self._get_repo_url(ud) | 401 | repourl = self._get_repo_url(ud) |
345 | 402 | ||
403 | needs_clone = False | ||
404 | if os.path.exists(ud.clonedir): | ||
405 | # The directory may exist, but not be the top level of a bare git | ||
406 | # repository in which case it needs to be deleted and re-cloned. | ||
407 | try: | ||
408 | # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel | ||
409 | output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir) | ||
410 | toplevel = output.rstrip() | ||
411 | |||
412 | if not bb.utils.path_is_descendant(toplevel, ud.clonedir): | ||
413 | logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir) | ||
414 | needs_clone = True | ||
415 | except bb.fetch2.FetchError as e: | ||
416 | logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e) | ||
417 | needs_clone = True | ||
418 | except FileNotFoundError as e: | ||
419 | logger.warning("%s", e) | ||
420 | needs_clone = True | ||
421 | |||
422 | if needs_clone: | ||
423 | shutil.rmtree(ud.clonedir) | ||
424 | else: | ||
425 | needs_clone = True | ||
426 | |||
346 | # If the repo still doesn't exist, fallback to cloning it | 427 | # If the repo still doesn't exist, fallback to cloning it |
347 | if not os.path.exists(ud.clonedir): | 428 | if needs_clone: |
348 | # We do this since git will use a "-l" option automatically for local urls where possible | 429 | # We do this since git will use a "-l" option automatically for local urls where possible, |
430 | # but it doesn't work when git/objects is a symlink, only works when it is a directory. | ||
349 | if repourl.startswith("file://"): | 431 | if repourl.startswith("file://"): |
350 | repourl = repourl[7:] | 432 | repourl_path = repourl[7:] |
433 | objects = os.path.join(repourl_path, 'objects') | ||
434 | if os.path.isdir(objects) and not os.path.islink(objects): | ||
435 | repourl = repourl_path | ||
351 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) | 436 | clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir) |
352 | if ud.proto.lower() != 'file': | 437 | if ud.proto.lower() != 'file': |
353 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) | 438 | bb.fetch2.check_network_access(d, clone_cmd, ud.url) |
354 | progresshandler = GitProgressHandler(d) | 439 | progresshandler = GitProgressHandler(d) |
440 | |||
441 | # Try creating a fast initial shallow clone | ||
442 | # Enabling ud.shallow_skip_fast will skip this | ||
443 | # If the Git error "Server does not allow request for unadvertised object" | ||
444 | # occurs, shallow_skip_fast is enabled automatically. | ||
445 | # This may happen if the Git server does not allow the request | ||
446 | # or if the Git client has issues with this functionality. | ||
447 | if ud.shallow and not ud.shallow_skip_fast: | ||
448 | try: | ||
449 | self.clone_shallow_with_tarball(ud, d) | ||
450 | # When the shallow clone has succeeded, use the shallow tarball | ||
451 | ud.localpath = ud.fullshallow | ||
452 | return | ||
453 | except: | ||
454 | logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.") | ||
455 | |||
456 | # When skipping fast initial shallow or the fast inital shallow clone failed: | ||
457 | # Try again with an initial regular clone | ||
355 | runfetchcmd(clone_cmd, d, log=progresshandler) | 458 | runfetchcmd(clone_cmd, d, log=progresshandler) |
356 | 459 | ||
357 | # Update the checkout if needed | 460 | # Update the checkout if needed |
@@ -361,7 +464,11 @@ class Git(FetchMethod): | |||
361 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) | 464 | runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) |
362 | 465 | ||
363 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) | 466 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) |
364 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | 467 | |
468 | if ud.nobranch: | ||
469 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) | ||
470 | else: | ||
471 | fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) | ||
365 | if ud.proto.lower() != 'file': | 472 | if ud.proto.lower() != 'file': |
366 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) | 473 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) |
367 | progresshandler = GitProgressHandler(d) | 474 | progresshandler = GitProgressHandler(d) |
@@ -375,138 +482,206 @@ class Git(FetchMethod): | |||
375 | if exc.errno != errno.ENOENT: | 482 | if exc.errno != errno.ENOENT: |
376 | raise | 483 | raise |
377 | 484 | ||
378 | for name in ud.names: | 485 | if not self._contains_ref(ud, d, ud.name, ud.clonedir): |
379 | if not self._contains_ref(ud, d, name, ud.clonedir): | 486 | raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch)) |
380 | raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) | ||
381 | 487 | ||
382 | if ud.shallow and ud.write_shallow_tarballs: | 488 | if ud.shallow and ud.write_shallow_tarballs: |
383 | missing_rev = self.clonedir_need_shallow_revs(ud, d) | 489 | missing_rev = self.clonedir_need_shallow_revs(ud, d) |
384 | if missing_rev: | 490 | if missing_rev: |
385 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) | 491 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) |
386 | 492 | ||
387 | if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): | 493 | if self.lfs_need_update(ud, d): |
388 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching | 494 | self.lfs_fetch(ud, d, ud.clonedir, ud.revision) |
389 | # of all LFS blobs needed at the the srcrev. | 495 | |
390 | # | 496 | def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None): |
391 | # It would be nice to just do this inline here by running 'git-lfs fetch' | 497 | """Helper method for fetching Git LFS data""" |
392 | # on the bare clonedir, but that operation requires a working copy on some | 498 | try: |
393 | # releases of Git LFS. | 499 | if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision): |
394 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | 500 | self._ensure_git_lfs(d, ud) |
395 | try: | 501 | |
396 | # Do the checkout. This implicitly involves a Git LFS fetch. | 502 | # Using worktree with the revision because .lfsconfig may exists |
397 | self.unpack(ud, tmpdir, d) | 503 | worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision) |
398 | 504 | runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir) | |
399 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into | 505 | lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "") |
400 | # the bare clonedir. | 506 | runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt")) |
401 | # | 507 | worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd |
402 | # As this procedure is invoked repeatedly on incremental fetches as | 508 | runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir) |
403 | # a recipe's SRCREV is bumped throughout its lifetime, this will | 509 | except: |
404 | # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs | 510 | logger.warning("Fetching LFS did not succeed.") |
405 | # corresponding to all the blobs reachable from the different revs | 511 | |
406 | # fetched across time. | 512 | @contextmanager |
407 | # | 513 | def create_atomic(self, filename): |
408 | # Only do this if the unpack resulted in a .git/lfs directory being | 514 | """Create as a temp file and move atomically into position to avoid races""" |
409 | # created; this only happens if at least one blob needed to be | 515 | fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename)) |
410 | # downloaded. | 516 | try: |
411 | if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): | 517 | yield tfile |
412 | runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) | 518 | umask = os.umask(0o666) |
413 | finally: | 519 | os.umask(umask) |
414 | bb.utils.remove(tmpdir, recurse=True) | 520 | os.chmod(tfile, (0o666 & ~umask)) |
521 | os.rename(tfile, filename) | ||
522 | finally: | ||
523 | os.close(fd) | ||
415 | 524 | ||
416 | def build_mirror_data(self, ud, d): | 525 | def build_mirror_data(self, ud, d): |
417 | if ud.shallow and ud.write_shallow_tarballs: | 526 | if ud.shallow and ud.write_shallow_tarballs: |
418 | if not os.path.exists(ud.fullshallow): | 527 | if not os.path.exists(ud.fullshallow): |
419 | if os.path.islink(ud.fullshallow): | 528 | if os.path.islink(ud.fullshallow): |
420 | os.unlink(ud.fullshallow) | 529 | os.unlink(ud.fullshallow) |
421 | tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | 530 | self.clone_shallow_with_tarball(ud, d) |
422 | shallowclone = os.path.join(tempdir, 'git') | ||
423 | try: | ||
424 | self.clone_shallow_local(ud, shallowclone, d) | ||
425 | |||
426 | logger.info("Creating tarball of git repository") | ||
427 | runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone) | ||
428 | runfetchcmd("touch %s.done" % ud.fullshallow, d) | ||
429 | finally: | ||
430 | bb.utils.remove(tempdir, recurse=True) | ||
431 | elif ud.write_tarballs and not os.path.exists(ud.fullmirror): | 531 | elif ud.write_tarballs and not os.path.exists(ud.fullmirror): |
432 | if os.path.islink(ud.fullmirror): | 532 | if os.path.islink(ud.fullmirror): |
433 | os.unlink(ud.fullmirror) | 533 | os.unlink(ud.fullmirror) |
434 | 534 | ||
435 | logger.info("Creating tarball of git repository") | 535 | logger.info("Creating tarball of git repository") |
436 | runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) | 536 | with self.create_atomic(ud.fullmirror) as tfile: |
537 | mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d, | ||
538 | quiet=True, workdir=ud.clonedir) | ||
539 | runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ." | ||
540 | % (tfile, mtime), d, workdir=ud.clonedir) | ||
437 | runfetchcmd("touch %s.done" % ud.fullmirror, d) | 541 | runfetchcmd("touch %s.done" % ud.fullmirror, d) |
438 | 542 | ||
543 | def clone_shallow_with_tarball(self, ud, d): | ||
544 | ret = False | ||
545 | tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | ||
546 | shallowclone = os.path.join(tempdir, 'git') | ||
547 | try: | ||
548 | try: | ||
549 | self.clone_shallow_local(ud, shallowclone, d) | ||
550 | except: | ||
551 | logger.warning("Fast shallow clone failed, try to skip fast mode now.") | ||
552 | bb.utils.remove(tempdir, recurse=True) | ||
553 | os.mkdir(tempdir) | ||
554 | ud.shallow_skip_fast = True | ||
555 | self.clone_shallow_local(ud, shallowclone, d) | ||
556 | logger.info("Creating tarball of git repository") | ||
557 | with self.create_atomic(ud.fullshallow) as tfile: | ||
558 | runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone) | ||
559 | runfetchcmd("touch %s.done" % ud.fullshallow, d) | ||
560 | ret = True | ||
561 | finally: | ||
562 | bb.utils.remove(tempdir, recurse=True) | ||
563 | |||
564 | return ret | ||
565 | |||
439 | def clone_shallow_local(self, ud, dest, d): | 566 | def clone_shallow_local(self, ud, dest, d): |
440 | """Clone the repo and make it shallow. | 567 | """ |
568 | Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default): | ||
569 | - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev | ||
570 | - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev | ||
571 | """ | ||
441 | 572 | ||
442 | The upstream url of the new clone isn't set at this time, as it'll be | 573 | progresshandler = GitProgressHandler(d) |
443 | set correctly when unpacked.""" | 574 | repourl = self._get_repo_url(ud) |
444 | runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d) | 575 | bb.utils.mkdirhier(dest) |
576 | init_cmd = "%s init -q" % ud.basecmd | ||
577 | if ud.bareclone: | ||
578 | init_cmd += " --bare" | ||
579 | runfetchcmd(init_cmd, d, workdir=dest) | ||
580 | # Use repourl when creating a fast initial shallow clone | ||
581 | # Prefer already existing full bare clones if available | ||
582 | if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir): | ||
583 | remote = shlex.quote(repourl) | ||
584 | else: | ||
585 | remote = ud.clonedir | ||
586 | runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest) | ||
445 | 587 | ||
446 | to_parse, shallow_branches = [], [] | 588 | # Check the histories which should be excluded |
447 | for name in ud.names: | 589 | shallow_exclude = '' |
448 | revision = ud.revisions[name] | 590 | for revision in ud.shallow_revs: |
449 | depth = ud.shallow_depths[name] | 591 | shallow_exclude += " --shallow-exclude=%s" % revision |
450 | if depth: | ||
451 | to_parse.append('%s~%d^{}' % (revision, depth - 1)) | ||
452 | 592 | ||
453 | # For nobranch, we need a ref, otherwise the commits will be | 593 | revision = ud.revision |
454 | # removed, and for non-nobranch, we truncate the branch to our | 594 | depth = ud.shallow_depths[ud.name] |
455 | # srcrev, to avoid keeping unnecessary history beyond that. | ||
456 | branch = ud.branches[name] | ||
457 | if ud.nobranch: | ||
458 | ref = "refs/shallow/%s" % name | ||
459 | elif ud.bareclone: | ||
460 | ref = "refs/heads/%s" % branch | ||
461 | else: | ||
462 | ref = "refs/remotes/origin/%s" % branch | ||
463 | 595 | ||
464 | shallow_branches.append(ref) | 596 | # The --depth and --shallow-exclude can't be used together |
465 | runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) | 597 | if depth and shallow_exclude: |
598 | raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.") | ||
466 | 599 | ||
467 | # Map srcrev+depths to revisions | 600 | # For nobranch, we need a ref, otherwise the commits will be |
468 | parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest) | 601 | # removed, and for non-nobranch, we truncate the branch to our |
602 | # srcrev, to avoid keeping unnecessary history beyond that. | ||
603 | branch = ud.branch | ||
604 | if ud.nobranch: | ||
605 | ref = "refs/shallow/%s" % ud.name | ||
606 | elif ud.bareclone: | ||
607 | ref = "refs/heads/%s" % branch | ||
608 | else: | ||
609 | ref = "refs/remotes/origin/%s" % branch | ||
610 | |||
611 | fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision) | ||
612 | if depth: | ||
613 | fetch_cmd += " --depth %s" % depth | ||
614 | |||
615 | if shallow_exclude: | ||
616 | fetch_cmd += shallow_exclude | ||
469 | 617 | ||
470 | # Resolve specified revisions | 618 | # Advertise the revision for lower version git such as 2.25.1: |
471 | parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest) | 619 | # error: Server does not allow request for unadvertised object. |
472 | shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines() | 620 | # The ud.clonedir is a local temporary dir, will be removed when |
621 | # fetch is done, so we can do anything on it. | ||
622 | adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision) | ||
623 | if ud.shallow_skip_fast: | ||
624 | runfetchcmd(adv_cmd, d, workdir=ud.clonedir) | ||
625 | |||
626 | runfetchcmd(fetch_cmd, d, workdir=dest) | ||
627 | runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) | ||
628 | # Fetch Git LFS data | ||
629 | self.lfs_fetch(ud, d, dest, ud.revision) | ||
473 | 630 | ||
474 | # Apply extra ref wildcards | 631 | # Apply extra ref wildcards |
475 | all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, | 632 | all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \ |
476 | d, workdir=dest).splitlines() | 633 | d, workdir=dest).splitlines() |
634 | all_refs = [] | ||
635 | for line in all_refs_remote: | ||
636 | all_refs.append(line.split()[-1]) | ||
637 | extra_refs = [] | ||
477 | for r in ud.shallow_extra_refs: | 638 | for r in ud.shallow_extra_refs: |
478 | if not ud.bareclone: | 639 | if not ud.bareclone: |
479 | r = r.replace('refs/heads/', 'refs/remotes/origin/') | 640 | r = r.replace('refs/heads/', 'refs/remotes/origin/') |
480 | 641 | ||
481 | if '*' in r: | 642 | if '*' in r: |
482 | matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) | 643 | matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs) |
483 | shallow_branches.extend(matches) | 644 | extra_refs.extend(matches) |
484 | else: | 645 | else: |
485 | shallow_branches.append(r) | 646 | extra_refs.append(r) |
647 | |||
648 | for ref in extra_refs: | ||
649 | ref_fetch = ref.replace('refs/heads/', '').replace('refs/remotes/origin/', '').replace('refs/tags/', '') | ||
650 | runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest) | ||
651 | revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest) | ||
652 | runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) | ||
486 | 653 | ||
487 | # Make the repository shallow | 654 | # The url is local ud.clonedir, set it to upstream one |
488 | shallow_cmd = [self.make_shallow_path, '-s'] | 655 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest) |
489 | for b in shallow_branches: | ||
490 | shallow_cmd.append('-r') | ||
491 | shallow_cmd.append(b) | ||
492 | shallow_cmd.extend(shallow_revisions) | ||
493 | runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest) | ||
494 | 656 | ||
495 | def unpack(self, ud, destdir, d): | 657 | def unpack(self, ud, destdir, d): |
496 | """ unpack the downloaded src to destdir""" | 658 | """ unpack the downloaded src to destdir""" |
497 | 659 | ||
498 | subdir = ud.parm.get("subpath", "") | 660 | subdir = ud.parm.get("subdir") |
499 | if subdir != "": | 661 | subpath = ud.parm.get("subpath") |
500 | readpathspec = ":%s" % subdir | 662 | readpathspec = "" |
501 | def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/')) | 663 | def_destsuffix = (d.getVar("BB_GIT_DEFAULT_DESTSUFFIX") or "git") + "/" |
502 | else: | 664 | |
503 | readpathspec = "" | 665 | if subpath: |
504 | def_destsuffix = "git/" | 666 | readpathspec = ":%s" % subpath |
667 | def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/')) | ||
668 | |||
669 | if subdir: | ||
670 | # If 'subdir' param exists, create a dir and use it as destination for unpack cmd | ||
671 | if os.path.isabs(subdir): | ||
672 | if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)): | ||
673 | raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url) | ||
674 | destdir = subdir | ||
675 | else: | ||
676 | destdir = os.path.join(destdir, subdir) | ||
677 | def_destsuffix = "" | ||
505 | 678 | ||
506 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) | 679 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) |
507 | destdir = ud.destdir = os.path.join(destdir, destsuffix) | 680 | destdir = ud.destdir = os.path.join(destdir, destsuffix) |
508 | if os.path.exists(destdir): | 681 | if os.path.exists(destdir): |
509 | bb.utils.prunedir(destdir) | 682 | bb.utils.prunedir(destdir) |
683 | if not ud.bareclone: | ||
684 | ud.unpack_tracer.unpack("git", destdir) | ||
510 | 685 | ||
511 | need_lfs = self._need_lfs(ud) | 686 | need_lfs = self._need_lfs(ud) |
512 | 687 | ||
@@ -516,13 +691,12 @@ class Git(FetchMethod): | |||
516 | source_found = False | 691 | source_found = False |
517 | source_error = [] | 692 | source_error = [] |
518 | 693 | ||
519 | if not source_found: | 694 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) |
520 | clonedir_is_up_to_date = not self.clonedir_need_update(ud, d) | 695 | if clonedir_is_up_to_date: |
521 | if clonedir_is_up_to_date: | 696 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) |
522 | runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d) | 697 | source_found = True |
523 | source_found = True | 698 | else: |
524 | else: | 699 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) |
525 | source_error.append("clone directory not available or not up to date: " + ud.clonedir) | ||
526 | 700 | ||
527 | if not source_found: | 701 | if not source_found: |
528 | if ud.shallow: | 702 | if ud.shallow: |
@@ -538,28 +712,43 @@ class Git(FetchMethod): | |||
538 | if not source_found: | 712 | if not source_found: |
539 | raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) | 713 | raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url) |
540 | 714 | ||
715 | # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag | ||
716 | # matches the revision | ||
717 | if 'tag' in ud.parm and sha1_re.match(ud.revision): | ||
718 | output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir) | ||
719 | output = output.strip() | ||
720 | if output != ud.revision: | ||
721 | # It is possible ud.revision is the revision on an annotated tag which won't match the output of rev-list | ||
722 | # If it resolves to the same thing there isn't a problem. | ||
723 | output2 = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.revision), d, workdir=destdir) | ||
724 | output2 = output2.strip() | ||
725 | if output != output2: | ||
726 | raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url) | ||
727 | |||
541 | repourl = self._get_repo_url(ud) | 728 | repourl = self._get_repo_url(ud) |
542 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) | 729 | runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir) |
543 | 730 | ||
544 | if self._contains_lfs(ud, d, destdir): | 731 | if self._contains_lfs(ud, d, destdir): |
545 | if need_lfs and not self._find_git_lfs(d): | 732 | if not need_lfs: |
546 | raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) | ||
547 | elif not need_lfs: | ||
548 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) | 733 | bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) |
734 | else: | ||
735 | self._ensure_git_lfs(d, ud) | ||
736 | |||
737 | runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir) | ||
549 | 738 | ||
550 | if not ud.nocheckout: | 739 | if not ud.nocheckout: |
551 | if subdir != "": | 740 | if subpath: |
552 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, | 741 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d, |
553 | workdir=destdir) | 742 | workdir=destdir) |
554 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) | 743 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir) |
555 | elif not ud.nobranch: | 744 | elif not ud.nobranch: |
556 | branchname = ud.branches[ud.names[0]] | 745 | branchname = ud.branch |
557 | runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ | 746 | runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \ |
558 | ud.revisions[ud.names[0]]), d, workdir=destdir) | 747 | ud.revision), d, workdir=destdir) |
559 | runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ | 748 | runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \ |
560 | branchname), d, workdir=destdir) | 749 | branchname), d, workdir=destdir) |
561 | else: | 750 | else: |
562 | runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir) | 751 | runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir) |
563 | 752 | ||
564 | return True | 753 | return True |
565 | 754 | ||
@@ -573,8 +762,13 @@ class Git(FetchMethod): | |||
573 | clonedir = os.path.realpath(ud.localpath) | 762 | clonedir = os.path.realpath(ud.localpath) |
574 | to_remove.append(clonedir) | 763 | to_remove.append(clonedir) |
575 | 764 | ||
765 | # Remove shallow mirror tarball | ||
766 | if ud.shallow: | ||
767 | to_remove.append(ud.fullshallow) | ||
768 | to_remove.append(ud.fullshallow + ".done") | ||
769 | |||
576 | for r in to_remove: | 770 | for r in to_remove: |
577 | if os.path.exists(r): | 771 | if os.path.exists(r) or os.path.islink(r): |
578 | bb.note('Removing %s' % r) | 772 | bb.note('Removing %s' % r) |
579 | bb.utils.remove(r, True) | 773 | bb.utils.remove(r, True) |
580 | 774 | ||
@@ -585,10 +779,10 @@ class Git(FetchMethod): | |||
585 | cmd = "" | 779 | cmd = "" |
586 | if ud.nobranch: | 780 | if ud.nobranch: |
587 | cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( | 781 | cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( |
588 | ud.basecmd, ud.revisions[name]) | 782 | ud.basecmd, ud.revision) |
589 | else: | 783 | else: |
590 | cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( | 784 | cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( |
591 | ud.basecmd, ud.revisions[name], ud.branches[name]) | 785 | ud.basecmd, ud.revision, ud.branch) |
592 | try: | 786 | try: |
593 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) | 787 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) |
594 | except bb.fetch2.FetchError: | 788 | except bb.fetch2.FetchError: |
@@ -597,6 +791,37 @@ class Git(FetchMethod): | |||
597 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) | 791 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) |
598 | return output.split()[0] != "0" | 792 | return output.split()[0] != "0" |
599 | 793 | ||
794 | def _lfs_objects_downloaded(self, ud, d, wd): | ||
795 | """ | ||
796 | Verifies whether the LFS objects for requested revisions have already been downloaded | ||
797 | """ | ||
798 | # Bail out early if this repository doesn't use LFS | ||
799 | if not self._contains_lfs(ud, d, wd): | ||
800 | return True | ||
801 | |||
802 | self._ensure_git_lfs(d, ud) | ||
803 | |||
804 | # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file | ||
805 | # existence. | ||
806 | # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git | ||
807 | cmd = "%s lfs ls-files -l %s" \ | ||
808 | % (ud.basecmd, ud.revision) | ||
809 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() | ||
810 | # Do not do any further matching if no objects are managed by LFS | ||
811 | if not output: | ||
812 | return True | ||
813 | |||
814 | # Match all lines beginning with the hexadecimal OID | ||
815 | oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)") | ||
816 | for line in output.split("\n"): | ||
817 | oid = re.search(oid_regex, line) | ||
818 | if not oid: | ||
819 | bb.warn("git lfs ls-files output '%s' did not match expected format." % line) | ||
820 | if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))): | ||
821 | return False | ||
822 | |||
823 | return True | ||
824 | |||
600 | def _need_lfs(self, ud): | 825 | def _need_lfs(self, ud): |
601 | return ud.parm.get("lfs", "1") == "1" | 826 | return ud.parm.get("lfs", "1") == "1" |
602 | 827 | ||
@@ -604,20 +829,8 @@ class Git(FetchMethod): | |||
604 | """ | 829 | """ |
605 | Check if the repository has 'lfs' (large file) content | 830 | Check if the repository has 'lfs' (large file) content |
606 | """ | 831 | """ |
607 | |||
608 | if not ud.nobranch: | ||
609 | branchname = ud.branches[ud.names[0]] | ||
610 | else: | ||
611 | branchname = "master" | ||
612 | |||
613 | # The bare clonedir doesn't use the remote names; it has the branch immediately. | ||
614 | if wd == ud.clonedir: | ||
615 | refname = ud.branches[ud.names[0]] | ||
616 | else: | ||
617 | refname = "origin/%s" % ud.branches[ud.names[0]] | ||
618 | |||
619 | cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( | 832 | cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( |
620 | ud.basecmd, refname) | 833 | ud.basecmd, ud.revision) |
621 | 834 | ||
622 | try: | 835 | try: |
623 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) | 836 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) |
@@ -627,12 +840,14 @@ class Git(FetchMethod): | |||
627 | pass | 840 | pass |
628 | return False | 841 | return False |
629 | 842 | ||
630 | def _find_git_lfs(self, d): | 843 | def _ensure_git_lfs(self, d, ud): |
631 | """ | 844 | """ |
632 | Return True if git-lfs can be found, False otherwise. | 845 | Ensures that git-lfs is available, raising a FetchError if it isn't. |
633 | """ | 846 | """ |
634 | import shutil | 847 | if shutil.which("git-lfs", path=d.getVar('PATH')) is None: |
635 | return shutil.which("git-lfs", path=d.getVar('PATH')) is not None | 848 | raise bb.fetch2.FetchError( |
849 | "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 " | ||
850 | "to ignore it)" % self._get_repo_url(ud)) | ||
636 | 851 | ||
637 | def _get_repo_url(self, ud): | 852 | def _get_repo_url(self, ud): |
638 | """ | 853 | """ |
@@ -640,22 +855,21 @@ class Git(FetchMethod): | |||
640 | """ | 855 | """ |
641 | # Note that we do not support passwords directly in the git urls. There are several | 856 | # Note that we do not support passwords directly in the git urls. There are several |
642 | # reasons. SRC_URI can be written out to things like buildhistory and people don't | 857 | # reasons. SRC_URI can be written out to things like buildhistory and people don't |
643 | # want to leak passwords like that. Its also all too easy to share metadata without | 858 | # want to leak passwords like that. Its also all too easy to share metadata without |
644 | # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as | 859 | # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as |
645 | # alternatives so we will not take patches adding password support here. | 860 | # alternatives so we will not take patches adding password support here. |
646 | if ud.user: | 861 | if ud.user: |
647 | username = ud.user + '@' | 862 | username = ud.user + '@' |
648 | else: | 863 | else: |
649 | username = "" | 864 | username = "" |
650 | return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) | 865 | return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path)) |
651 | 866 | ||
652 | def _revision_key(self, ud, d, name): | 867 | def _revision_key(self, ud, d, name): |
653 | """ | 868 | """ |
654 | Return a unique key for the url | 869 | Return a unique key for the url |
655 | """ | 870 | """ |
656 | # Collapse adjacent slashes | 871 | # Collapse adjacent slashes |
657 | slash_re = re.compile(r"/+") | 872 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev |
658 | return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] | ||
659 | 873 | ||
660 | def _lsremote(self, ud, d, search): | 874 | def _lsremote(self, ud, d, search): |
661 | """ | 875 | """ |
@@ -687,21 +901,27 @@ class Git(FetchMethod): | |||
687 | """ | 901 | """ |
688 | Compute the HEAD revision for the url | 902 | Compute the HEAD revision for the url |
689 | """ | 903 | """ |
904 | if not d.getVar("__BBSRCREV_SEEN"): | ||
905 | raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path)) | ||
906 | |||
907 | # Ensure we mark as not cached | ||
908 | bb.fetch2.mark_recipe_nocache(d) | ||
909 | |||
690 | output = self._lsremote(ud, d, "") | 910 | output = self._lsremote(ud, d, "") |
691 | # Tags of the form ^{} may not work, need to fallback to other form | 911 | # Tags of the form ^{} may not work, need to fallback to other form |
692 | if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: | 912 | if ud.unresolvedrev[:5] == "refs/" or ud.usehead: |
693 | head = ud.unresolvedrev[name] | 913 | head = ud.unresolvedrev |
694 | tag = ud.unresolvedrev[name] | 914 | tag = ud.unresolvedrev |
695 | else: | 915 | else: |
696 | head = "refs/heads/%s" % ud.unresolvedrev[name] | 916 | head = "refs/heads/%s" % ud.unresolvedrev |
697 | tag = "refs/tags/%s" % ud.unresolvedrev[name] | 917 | tag = "refs/tags/%s" % ud.unresolvedrev |
698 | for s in [head, tag + "^{}", tag]: | 918 | for s in [head, tag + "^{}", tag]: |
699 | for l in output.strip().split('\n'): | 919 | for l in output.strip().split('\n'): |
700 | sha1, ref = l.split() | 920 | sha1, ref = l.split() |
701 | if s == ref: | 921 | if s == ref: |
702 | return sha1 | 922 | return sha1 |
703 | raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ | 923 | raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \ |
704 | (ud.unresolvedrev[name], ud.host+ud.path)) | 924 | (ud.unresolvedrev, ud.host+ud.path)) |
705 | 925 | ||
706 | def latest_versionstring(self, ud, d): | 926 | def latest_versionstring(self, ud, d): |
707 | """ | 927 | """ |
@@ -711,60 +931,63 @@ class Git(FetchMethod): | |||
711 | """ | 931 | """ |
712 | pupver = ('', '') | 932 | pupver = ('', '') |
713 | 933 | ||
714 | tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
715 | try: | 934 | try: |
716 | output = self._lsremote(ud, d, "refs/tags/*") | 935 | output = self._lsremote(ud, d, "refs/tags/*") |
717 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: | 936 | except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: |
718 | bb.note("Could not list remote: %s" % str(e)) | 937 | bb.note("Could not list remote: %s" % str(e)) |
719 | return pupver | 938 | return pupver |
720 | 939 | ||
940 | rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)") | ||
941 | pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") | ||
942 | nonrel_re = re.compile(r"(alpha|beta|rc|final)+") | ||
943 | |||
721 | verstring = "" | 944 | verstring = "" |
722 | revision = "" | ||
723 | for line in output.split("\n"): | 945 | for line in output.split("\n"): |
724 | if not line: | 946 | if not line: |
725 | break | 947 | break |
726 | 948 | ||
727 | tag_head = line.split("/")[-1] | 949 | m = rev_tag_re.match(line) |
950 | if not m: | ||
951 | continue | ||
952 | |||
953 | (revision, tag) = m.groups() | ||
954 | |||
728 | # Ignore non-released branches | 955 | # Ignore non-released branches |
729 | m = re.search(r"(alpha|beta|rc|final)+", tag_head) | 956 | if nonrel_re.search(tag): |
730 | if m: | ||
731 | continue | 957 | continue |
732 | 958 | ||
733 | # search for version in the line | 959 | # search for version in the line |
734 | tag = tagregex.search(tag_head) | 960 | m = pver_re.search(tag) |
735 | if tag is None: | 961 | if not m: |
736 | continue | 962 | continue |
737 | 963 | ||
738 | tag = tag.group('pver') | 964 | pver = m.group('pver').replace("_", ".") |
739 | tag = tag.replace("_", ".") | ||
740 | 965 | ||
741 | if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0: | 966 | if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0: |
742 | continue | 967 | continue |
743 | 968 | ||
744 | verstring = tag | 969 | verstring = pver |
745 | revision = line.split()[0] | ||
746 | pupver = (verstring, revision) | 970 | pupver = (verstring, revision) |
747 | 971 | ||
748 | return pupver | 972 | return pupver |
749 | 973 | ||
750 | def _build_revision(self, ud, d, name): | 974 | def _build_revision(self, ud, d, name): |
751 | return ud.revisions[name] | 975 | return ud.revision |
752 | 976 | ||
753 | def gitpkgv_revision(self, ud, d, name): | 977 | def gitpkgv_revision(self, ud, d, name): |
754 | """ | 978 | """ |
755 | Return a sortable revision number by counting commits in the history | 979 | Return a sortable revision number by counting commits in the history |
756 | Based on gitpkgv.bblass in meta-openembedded | 980 | Based on gitpkgv.bblass in meta-openembedded |
757 | """ | 981 | """ |
758 | rev = self._build_revision(ud, d, name) | 982 | rev = ud.revision |
759 | localpath = ud.localpath | 983 | localpath = ud.localpath |
760 | rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) | 984 | rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev) |
761 | if not os.path.exists(localpath): | 985 | if not os.path.exists(localpath): |
762 | commits = None | 986 | commits = None |
763 | else: | 987 | else: |
764 | if not os.path.exists(rev_file) or not os.path.getsize(rev_file): | 988 | if not os.path.exists(rev_file) or not os.path.getsize(rev_file): |
765 | from pipes import quote | ||
766 | commits = bb.fetch2.runfetchcmd( | 989 | commits = bb.fetch2.runfetchcmd( |
767 | "git rev-list %s -- | wc -l" % quote(rev), | 990 | "git rev-list %s -- | wc -l" % shlex.quote(rev), |
768 | d, quiet=True).strip().lstrip('0') | 991 | d, quiet=True).strip().lstrip('0') |
769 | if commits: | 992 | if commits: |
770 | open(rev_file, "w").write("%d\n" % int(commits)) | 993 | open(rev_file, "w").write("%d\n" % int(commits)) |