summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/git.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r--bitbake/lib/bb/fetch2/git.py133
1 files changed, 118 insertions, 15 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 01d4bbdc2e..0412f9ff51 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -73,8 +73,9 @@ Supported SRC_URI options are:
73import errno 73import errno
74import os 74import os
75import re 75import re
76import subprocess
77import tempfile
76import bb 78import bb
77import errno
78import bb.progress 79import bb.progress
79from bb.fetch2 import FetchMethod 80from bb.fetch2 import FetchMethod
80from bb.fetch2 import runfetchcmd 81from bb.fetch2 import runfetchcmd
@@ -172,6 +173,11 @@ class Git(FetchMethod):
172 branches = ud.parm.get("branch", "master").split(',') 173 branches = ud.parm.get("branch", "master").split(',')
173 if len(branches) != len(ud.names): 174 if len(branches) != len(ud.names):
174 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) 175 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
176
177 ud.cloneflags = "-s -n"
178 if ud.bareclone:
179 ud.cloneflags += " --mirror"
180
175 ud.branches = {} 181 ud.branches = {}
176 for pos, name in enumerate(ud.names): 182 for pos, name in enumerate(ud.names):
177 branch = branches[pos] 183 branch = branches[pos]
@@ -183,7 +189,9 @@ class Git(FetchMethod):
183 189
184 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" 190 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
185 191
186 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") or ud.rebaseable 192 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
193 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
194 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
187 195
188 ud.setup_revisions(d) 196 ud.setup_revisions(d)
189 197
@@ -205,13 +213,48 @@ class Git(FetchMethod):
205 if ud.rebaseable: 213 if ud.rebaseable:
206 for name in ud.names: 214 for name in ud.names:
207 gitsrcname = gitsrcname + '_' + ud.revisions[name] 215 gitsrcname = gitsrcname + '_' + ud.revisions[name]
216
217 dl_dir = d.getVar("DL_DIR")
218 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2/")
219 ud.clonedir = os.path.join(gitdir, gitsrcname)
220 ud.localfile = ud.clonedir
221
208 mirrortarball = 'git2_%s.tar.gz' % gitsrcname 222 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
209 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) 223 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
210 ud.mirrortarballs = [mirrortarball] 224 ud.mirrortarballs = [mirrortarball]
211 gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
212 ud.clonedir = os.path.join(gitdir, gitsrcname)
213 225
214 ud.localfile = ud.clonedir 226 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
227 if ud.shallow:
228 ud.shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH")
229 if ud.shallow_depth is not None:
230 try:
231 ud.shallow_depth = int(ud.shallow_depth or 0)
232 except ValueError:
233 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % ud.shallow_depth)
234 else:
235 if not ud.shallow_depth:
236 ud.shallow = False
237 elif ud.shallow_depth < 0:
238 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % ud.shallow_depth)
239 else:
240 ud.shallow_depth = 1
241
242 if ud.shallow:
243 tarballname = gitsrcname
244 if ud.bareclone:
245 tarballname = "%s_bare" % tarballname
246
247 for name, revision in sorted(ud.revisions.items()):
248 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
249 if not ud.nobranch:
250 tarballname = "%s-%s" % (tarballname, ud.branches[name])
251
252 tarballname = "%s-%s" % (tarballname, ud.shallow_depth)
253
254 fetcher = self.__class__.__name__.lower()
255 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
256 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
257 ud.mirrortarballs.insert(0, ud.shallowtarball)
215 258
216 def localpath(self, ud, d): 259 def localpath(self, ud, d):
217 return ud.clonedir 260 return ud.clonedir
@@ -222,6 +265,8 @@ class Git(FetchMethod):
222 for name in ud.names: 265 for name in ud.names:
223 if not self._contains_ref(ud, d, name, ud.clonedir): 266 if not self._contains_ref(ud, d, name, ud.clonedir):
224 return True 267 return True
268 if ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow):
269 return True
225 if ud.write_tarballs and not os.path.exists(ud.fullmirror): 270 if ud.write_tarballs and not os.path.exists(ud.fullmirror):
226 return True 271 return True
227 return False 272 return False
@@ -238,8 +283,16 @@ class Git(FetchMethod):
238 def download(self, ud, d): 283 def download(self, ud, d):
239 """Fetch url""" 284 """Fetch url"""
240 285
241 # If the checkout doesn't exist and the mirror tarball does, extract it 286 no_clone = not os.path.exists(ud.clonedir)
242 if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): 287 need_update = no_clone or self.need_update(ud, d)
288
289 # A current clone is preferred to either tarball, a shallow tarball is
290 # preferred to an out of date clone, and a missing clone will use
291 # either tarball.
292 if ud.shallow and os.path.exists(ud.fullshallow) and need_update:
293 ud.localpath = ud.fullshallow
294 return
295 elif os.path.exists(ud.fullmirror) and no_clone:
243 bb.utils.mkdirhier(ud.clonedir) 296 bb.utils.mkdirhier(ud.clonedir)
244 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) 297 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
245 298
@@ -285,9 +338,21 @@ class Git(FetchMethod):
285 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) 338 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
286 339
287 def build_mirror_data(self, ud, d): 340 def build_mirror_data(self, ud, d):
288 # Generate a mirror tarball if needed 341 if ud.shallow and ud.write_shallow_tarballs:
289 if ud.write_tarballs and not os.path.exists(ud.fullmirror): 342 if not os.path.exists(ud.fullshallow):
290 # it's possible that this symlink points to read-only filesystem with PREMIRROR 343 if os.path.islink(ud.fullshallow):
344 os.unlink(ud.fullshallow)
345 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
346 shallowclone = os.path.join(tempdir, 'git')
347 try:
348 self.clone_shallow_local(ud, shallowclone, d)
349
350 logger.info("Creating tarball of git repository")
351 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
352 runfetchcmd("touch %s.done" % ud.fullshallow, d)
353 finally:
354 bb.utils.remove(tempdir, recurse=True)
355 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
291 if os.path.islink(ud.fullmirror): 356 if os.path.islink(ud.fullmirror):
292 os.unlink(ud.fullmirror) 357 os.unlink(ud.fullmirror)
293 358
@@ -295,6 +360,43 @@ class Git(FetchMethod):
295 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) 360 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
296 runfetchcmd("touch %s.done" % ud.fullmirror, d) 361 runfetchcmd("touch %s.done" % ud.fullmirror, d)
297 362
363 def clone_shallow_local(self, ud, dest, d):
364 """Clone the repo and make it shallow.
365
366 The upstream url of the new clone isn't set at this time, as it'll be
367 set correctly when unpacked."""
368 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
369
370 to_parse, shallow_branches = [], []
371 for name in ud.names:
372 revision = ud.revisions[name]
373 to_parse.append('%s~%d^{}' % (revision, ud.shallow_depth - 1))
374
375 # For nobranch, we need a ref, otherwise the commits will be
376 # removed, and for non-nobranch, we truncate the branch to our
377 # srcrev, to avoid keeping unnecessary history beyond that.
378 branch = ud.branches[name]
379 if ud.nobranch:
380 ref = "refs/shallow/%s" % name
381 elif ud.bareclone:
382 ref = "refs/heads/%s" % branch
383 else:
384 ref = "refs/remotes/origin/%s" % branch
385
386 shallow_branches.append(ref)
387 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
388
389 # Map srcrev+depths to revisions
390 shallow_revisions = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest).splitlines()
391
392 # Make the repository shallow
393 shallow_cmd = ['git', 'make-shallow', '-s']
394 for b in shallow_branches:
395 shallow_cmd.append('-r')
396 shallow_cmd.append(b)
397 shallow_cmd.extend(shallow_revisions)
398 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
399
298 def unpack(self, ud, destdir, d): 400 def unpack(self, ud, destdir, d):
299 """ unpack the downloaded src to destdir""" 401 """ unpack the downloaded src to destdir"""
300 402
@@ -311,11 +413,12 @@ class Git(FetchMethod):
311 if os.path.exists(destdir): 413 if os.path.exists(destdir):
312 bb.utils.prunedir(destdir) 414 bb.utils.prunedir(destdir)
313 415
314 cloneflags = "-s -n" 416 if ud.shallow and (not os.path.exists(ud.clonedir) or self.need_update(ud, d)):
315 if ud.bareclone: 417 bb.utils.mkdirhier(destdir)
316 cloneflags += " --mirror" 418 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
419 else:
420 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
317 421
318 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d)
319 repourl = self._get_repo_url(ud) 422 repourl = self._get_repo_url(ud)
320 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir) 423 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
321 if not ud.nocheckout: 424 if not ud.nocheckout: