summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristopher Larson <kergoth@gmail.com>2017-05-13 02:46:28 +0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-06-02 13:36:57 +0100
commit27d56982c7ba05e86a100b0cca2411ee5ac7a85e (patch)
tree61d9ca5571770cfd093b327a7f9b7ca7ad3b56fe
parent2a60c406372d400437ecaa8712e6dc80b3d9fcec (diff)
downloadpoky-27d56982c7ba05e86a100b0cca2411ee5ac7a85e.tar.gz
bitbake: fetch/git: add support for shallow mirror tarballs
This adds support to the git fetcher for fetching, using, and generating mirror tarballs of shallow git repositories. The external git-make-shallow script is used for shallow mirror tarball creation. This implements support for shallow mirror tarballs, not shallow clones. Supporting shallow clones directly is not really doable for us, as we'd need to hardcode the depth between branch HEAD and the SRCREV, and that depth would change as the branch is updated. When BB_GIT_SHALLOW is enabled, we will always attempt to fetch a shallow mirror tarball. If the shallow mirror tarball cannot be fetched, it will try to fetch the full mirror tarball and use that. If a shallow tarball is to be used, it will be unpacked directly at `do_unpack` time, rather than extracting it to DL_DIR at `do_fetch` time and cloning from there, to keep things simple. There's no value in keeping a shallow repository in DL_DIR, and dealing with the state for when to convert the clonedir to/from shallow is not worthwhile. To clarify when shallow is used vs a real repository, a current clone is preferred to either tarball, a shallow tarball is preferred to an out of date clone, and a missing clone will use either tarball (attempting the shallow one first). All referenced branches are truncated to SRCREV (that is, commits *after* SRCREV but before HEAD are removed) to further shrink the repository. By default, the shallow construction process removes all unused refs (branches/tags) from the repository, other than those referenced by the URL. Example usage: BB_GIT_SHALLOW ?= "1" # Keep only the top commit BB_GIT_SHALLOW_DEPTH ?= "1" # This defaults to enabled if both BB_GIT_SHALLOW and # BB_GENERATE_MIRROR_TARBALLS are enabled BB_GENERATE_SHALLOW_TARBALLS ?= "1" (Bitbake rev: 5ed7d85fda7c671be10ec24d7981b87a7d0d3366) Signed-off-by: Christopher Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/fetch2/git.py133
-rw-r--r--bitbake/lib/bb/tests/fetch.py299
2 files changed, 417 insertions, 15 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 01d4bbdc2e..0412f9ff51 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -73,8 +73,9 @@ Supported SRC_URI options are:
73import errno 73import errno
74import os 74import os
75import re 75import re
76import subprocess
77import tempfile
76import bb 78import bb
77import errno
78import bb.progress 79import bb.progress
79from bb.fetch2 import FetchMethod 80from bb.fetch2 import FetchMethod
80from bb.fetch2 import runfetchcmd 81from bb.fetch2 import runfetchcmd
@@ -172,6 +173,11 @@ class Git(FetchMethod):
172 branches = ud.parm.get("branch", "master").split(',') 173 branches = ud.parm.get("branch", "master").split(',')
173 if len(branches) != len(ud.names): 174 if len(branches) != len(ud.names):
174 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) 175 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
176
177 ud.cloneflags = "-s -n"
178 if ud.bareclone:
179 ud.cloneflags += " --mirror"
180
175 ud.branches = {} 181 ud.branches = {}
176 for pos, name in enumerate(ud.names): 182 for pos, name in enumerate(ud.names):
177 branch = branches[pos] 183 branch = branches[pos]
@@ -183,7 +189,9 @@ class Git(FetchMethod):
183 189
184 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0" 190 ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
185 191
186 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") or ud.rebaseable 192 write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
193 ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
194 ud.write_shallow_tarballs = (d.getVar("BB_GENERATE_SHALLOW_TARBALLS") or write_tarballs) != "0"
187 195
188 ud.setup_revisions(d) 196 ud.setup_revisions(d)
189 197
@@ -205,13 +213,48 @@ class Git(FetchMethod):
205 if ud.rebaseable: 213 if ud.rebaseable:
206 for name in ud.names: 214 for name in ud.names:
207 gitsrcname = gitsrcname + '_' + ud.revisions[name] 215 gitsrcname = gitsrcname + '_' + ud.revisions[name]
216
217 dl_dir = d.getVar("DL_DIR")
218 gitdir = d.getVar("GITDIR") or (dl_dir + "/git2/")
219 ud.clonedir = os.path.join(gitdir, gitsrcname)
220 ud.localfile = ud.clonedir
221
208 mirrortarball = 'git2_%s.tar.gz' % gitsrcname 222 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
209 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) 223 ud.fullmirror = os.path.join(dl_dir, mirrortarball)
210 ud.mirrortarballs = [mirrortarball] 224 ud.mirrortarballs = [mirrortarball]
211 gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
212 ud.clonedir = os.path.join(gitdir, gitsrcname)
213 225
214 ud.localfile = ud.clonedir 226 ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
227 if ud.shallow:
228 ud.shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH")
229 if ud.shallow_depth is not None:
230 try:
231 ud.shallow_depth = int(ud.shallow_depth or 0)
232 except ValueError:
233 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % ud.shallow_depth)
234 else:
235 if not ud.shallow_depth:
236 ud.shallow = False
237 elif ud.shallow_depth < 0:
238 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH: %s" % ud.shallow_depth)
239 else:
240 ud.shallow_depth = 1
241
242 if ud.shallow:
243 tarballname = gitsrcname
244 if ud.bareclone:
245 tarballname = "%s_bare" % tarballname
246
247 for name, revision in sorted(ud.revisions.items()):
248 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
249 if not ud.nobranch:
250 tarballname = "%s-%s" % (tarballname, ud.branches[name])
251
252 tarballname = "%s-%s" % (tarballname, ud.shallow_depth)
253
254 fetcher = self.__class__.__name__.lower()
255 ud.shallowtarball = '%sshallow_%s.tar.gz' % (fetcher, tarballname)
256 ud.fullshallow = os.path.join(dl_dir, ud.shallowtarball)
257 ud.mirrortarballs.insert(0, ud.shallowtarball)
215 258
216 def localpath(self, ud, d): 259 def localpath(self, ud, d):
217 return ud.clonedir 260 return ud.clonedir
@@ -222,6 +265,8 @@ class Git(FetchMethod):
222 for name in ud.names: 265 for name in ud.names:
223 if not self._contains_ref(ud, d, name, ud.clonedir): 266 if not self._contains_ref(ud, d, name, ud.clonedir):
224 return True 267 return True
268 if ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow):
269 return True
225 if ud.write_tarballs and not os.path.exists(ud.fullmirror): 270 if ud.write_tarballs and not os.path.exists(ud.fullmirror):
226 return True 271 return True
227 return False 272 return False
@@ -238,8 +283,16 @@ class Git(FetchMethod):
238 def download(self, ud, d): 283 def download(self, ud, d):
239 """Fetch url""" 284 """Fetch url"""
240 285
241 # If the checkout doesn't exist and the mirror tarball does, extract it 286 no_clone = not os.path.exists(ud.clonedir)
242 if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): 287 need_update = no_clone or self.need_update(ud, d)
288
289 # A current clone is preferred to either tarball, a shallow tarball is
290 # preferred to an out of date clone, and a missing clone will use
291 # either tarball.
292 if ud.shallow and os.path.exists(ud.fullshallow) and need_update:
293 ud.localpath = ud.fullshallow
294 return
295 elif os.path.exists(ud.fullmirror) and no_clone:
243 bb.utils.mkdirhier(ud.clonedir) 296 bb.utils.mkdirhier(ud.clonedir)
244 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir) 297 runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
245 298
@@ -285,9 +338,21 @@ class Git(FetchMethod):
285 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) 338 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
286 339
287 def build_mirror_data(self, ud, d): 340 def build_mirror_data(self, ud, d):
288 # Generate a mirror tarball if needed 341 if ud.shallow and ud.write_shallow_tarballs:
289 if ud.write_tarballs and not os.path.exists(ud.fullmirror): 342 if not os.path.exists(ud.fullshallow):
290 # it's possible that this symlink points to read-only filesystem with PREMIRROR 343 if os.path.islink(ud.fullshallow):
344 os.unlink(ud.fullshallow)
345 tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
346 shallowclone = os.path.join(tempdir, 'git')
347 try:
348 self.clone_shallow_local(ud, shallowclone, d)
349
350 logger.info("Creating tarball of git repository")
351 runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
352 runfetchcmd("touch %s.done" % ud.fullshallow, d)
353 finally:
354 bb.utils.remove(tempdir, recurse=True)
355 elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
291 if os.path.islink(ud.fullmirror): 356 if os.path.islink(ud.fullmirror):
292 os.unlink(ud.fullmirror) 357 os.unlink(ud.fullmirror)
293 358
@@ -295,6 +360,43 @@ class Git(FetchMethod):
295 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir) 360 runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
296 runfetchcmd("touch %s.done" % ud.fullmirror, d) 361 runfetchcmd("touch %s.done" % ud.fullmirror, d)
297 362
363 def clone_shallow_local(self, ud, dest, d):
364 """Clone the repo and make it shallow.
365
366 The upstream url of the new clone isn't set at this time, as it'll be
367 set correctly when unpacked."""
368 runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
369
370 to_parse, shallow_branches = [], []
371 for name in ud.names:
372 revision = ud.revisions[name]
373 to_parse.append('%s~%d^{}' % (revision, ud.shallow_depth - 1))
374
375 # For nobranch, we need a ref, otherwise the commits will be
376 # removed, and for non-nobranch, we truncate the branch to our
377 # srcrev, to avoid keeping unnecessary history beyond that.
378 branch = ud.branches[name]
379 if ud.nobranch:
380 ref = "refs/shallow/%s" % name
381 elif ud.bareclone:
382 ref = "refs/heads/%s" % branch
383 else:
384 ref = "refs/remotes/origin/%s" % branch
385
386 shallow_branches.append(ref)
387 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
388
389 # Map srcrev+depths to revisions
390 shallow_revisions = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest).splitlines()
391
392 # Make the repository shallow
393 shallow_cmd = ['git', 'make-shallow', '-s']
394 for b in shallow_branches:
395 shallow_cmd.append('-r')
396 shallow_cmd.append(b)
397 shallow_cmd.extend(shallow_revisions)
398 runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
399
298 def unpack(self, ud, destdir, d): 400 def unpack(self, ud, destdir, d):
299 """ unpack the downloaded src to destdir""" 401 """ unpack the downloaded src to destdir"""
300 402
@@ -311,11 +413,12 @@ class Git(FetchMethod):
311 if os.path.exists(destdir): 413 if os.path.exists(destdir):
312 bb.utils.prunedir(destdir) 414 bb.utils.prunedir(destdir)
313 415
314 cloneflags = "-s -n" 416 if ud.shallow and (not os.path.exists(ud.clonedir) or self.need_update(ud, d)):
315 if ud.bareclone: 417 bb.utils.mkdirhier(destdir)
316 cloneflags += " --mirror" 418 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=destdir)
419 else:
420 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
317 421
318 runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, cloneflags, ud.clonedir, destdir), d)
319 repourl = self._get_repo_url(ud) 422 repourl = self._get_repo_url(ud)
320 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir) 423 runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir)
321 if not ud.nocheckout: 424 if not ud.nocheckout:
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 510071d25d..019f22a11d 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -979,3 +979,302 @@ class GitMakeShallowTest(FetcherTest):
979 orig_revs = len(self.git('rev-list --all').splitlines()) 979 orig_revs = len(self.git('rev-list --all').splitlines())
980 self.make_shallow(['refs/tags/1.10.0']) 980 self.make_shallow(['refs/tags/1.10.0'])
981 self.assertRevCount(orig_revs - 1746, ['--all']) 981 self.assertRevCount(orig_revs - 1746, ['--all'])
982
983class GitShallowTest(FetcherTest):
984 def setUp(self):
985 FetcherTest.setUp(self)
986 self.gitdir = os.path.join(self.tempdir, 'git')
987 self.srcdir = os.path.join(self.tempdir, 'gitsource')
988
989 bb.utils.mkdirhier(self.srcdir)
990 self.git('init', cwd=self.srcdir)
991 self.d.setVar('WORKDIR', self.tempdir)
992 self.d.setVar('S', self.gitdir)
993 self.d.delVar('PREMIRRORS')
994 self.d.delVar('MIRRORS')
995
996 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
997 self.d.setVar('SRC_URI', uri)
998 self.d.setVar('SRCREV', '${AUTOREV}')
999 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1000
1001 self.d.setVar('BB_GIT_SHALLOW', '1')
1002 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1003 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1004
1005 def assertRefs(self, expected_refs, cwd=None):
1006 if cwd is None:
1007 cwd = self.gitdir
1008 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1009 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1010 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1011
1012 def assertRevCount(self, expected_count, args=None, cwd=None):
1013 if args is None:
1014 args = ['HEAD']
1015 if cwd is None:
1016 cwd = self.gitdir
1017 revs = self.git(['rev-list'] + args, cwd=cwd)
1018 actual_count = len(revs.splitlines())
1019 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1020
1021 def git(self, cmd, cwd=None):
1022 if isinstance(cmd, str):
1023 cmd = 'git ' + cmd
1024 else:
1025 cmd = ['git'] + cmd
1026 if cwd is None:
1027 cwd = self.gitdir
1028 return bb.process.run(cmd, cwd=cwd)[0]
1029
1030 def add_empty_file(self, path, msg=None):
1031 if msg is None:
1032 msg = path
1033 open(os.path.join(self.srcdir, path), 'w').close()
1034 self.git(['add', path], self.srcdir)
1035 self.git(['commit', '-m', msg, path], self.srcdir)
1036
1037 def fetch(self, uri=None):
1038 if uri is None:
1039 uris = self.d.getVar('SRC_URI', True).split()
1040 uri = uris[0]
1041 d = self.d
1042 else:
1043 d = self.d.createCopy()
1044 d.setVar('SRC_URI', uri)
1045 uri = d.expand(uri)
1046 uris = [uri]
1047
1048 fetcher = bb.fetch2.Fetch(uris, d)
1049 fetcher.download()
1050 ud = fetcher.ud[uri]
1051 return fetcher, ud
1052
1053 def fetch_and_unpack(self, uri=None):
1054 fetcher, ud = self.fetch(uri)
1055 fetcher.unpack(self.d.getVar('WORKDIR'))
1056 assert os.path.exists(self.d.getVar('S'))
1057 return fetcher, ud
1058
1059 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1060 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1061 fetcher, ud = self.fetch_and_unpack(uri)
1062 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1063
1064 # Confirm that the unpacked repo is unshallow
1065 if not disabled:
1066 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1067
1068 # fetch and unpack, from the shallow tarball
1069 bb.utils.remove(self.gitdir, recurse=True)
1070 bb.utils.remove(ud.clonedir, recurse=True)
1071
1072 # confirm that the unpacked repo is used when no git clone or git
1073 # mirror tarball is available
1074 fetcher, ud = self.fetch_and_unpack(uri)
1075 if not disabled:
1076 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1077 else:
1078 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1079 return fetcher, ud
1080
1081 def test_shallow_disabled(self):
1082 self.add_empty_file('a')
1083 self.add_empty_file('b')
1084 self.assertRevCount(2, cwd=self.srcdir)
1085
1086 self.d.setVar('BB_GIT_SHALLOW', '0')
1087 self.fetch_shallow(disabled=True)
1088 self.assertRevCount(2)
1089
1090 def test_shallow_nobranch(self):
1091 self.add_empty_file('a')
1092 self.add_empty_file('b')
1093 self.assertRevCount(2, cwd=self.srcdir)
1094
1095 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1096 self.d.setVar('SRCREV', srcrev)
1097 uri = self.d.getVar('SRC_URI', True).split()[0]
1098 uri = '%s;nobranch=1;bare=1' % uri
1099
1100 self.fetch_shallow(uri)
1101 self.assertRevCount(1)
1102
1103 # shallow refs are used to ensure the srcrev sticks around when we
1104 # have no other branches referencing it
1105 self.assertRefs(['refs/shallow/default'])
1106
1107 def test_shallow_default_depth_1(self):
1108 # Create initial git repo
1109 self.add_empty_file('a')
1110 self.add_empty_file('b')
1111 self.assertRevCount(2, cwd=self.srcdir)
1112
1113 self.fetch_shallow()
1114 self.assertRevCount(1)
1115
1116 def test_shallow_depth_0_disables(self):
1117 self.add_empty_file('a')
1118 self.add_empty_file('b')
1119 self.assertRevCount(2, cwd=self.srcdir)
1120
1121 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1122 self.fetch_shallow(disabled=True)
1123 self.assertRevCount(2)
1124
1125 def test_current_shallow_out_of_date_clone(self):
1126 # Create initial git repo
1127 self.add_empty_file('a')
1128 self.add_empty_file('b')
1129 self.add_empty_file('c')
1130 self.assertRevCount(3, cwd=self.srcdir)
1131
1132 # Clone and generate mirror tarball
1133 fetcher, ud = self.fetch()
1134
1135 # Ensure we have a current mirror tarball, but an out of date clone
1136 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1137 self.assertRevCount(2, cwd=ud.clonedir)
1138
1139 # Fetch and unpack, from the current tarball, not the out of date clone
1140 bb.utils.remove(self.gitdir, recurse=True)
1141 fetcher, ud = self.fetch()
1142 fetcher.unpack(self.d.getVar('WORKDIR'))
1143 self.assertRevCount(1)
1144
1145 def test_shallow_single_branch_no_merge(self):
1146 self.add_empty_file('a')
1147 self.add_empty_file('b')
1148 self.assertRevCount(2, cwd=self.srcdir)
1149
1150 self.fetch_shallow()
1151 self.assertRevCount(1)
1152 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1153 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1154
1155 def test_shallow_no_dangling(self):
1156 self.add_empty_file('a')
1157 self.add_empty_file('b')
1158 self.assertRevCount(2, cwd=self.srcdir)
1159
1160 self.fetch_shallow()
1161 self.assertRevCount(1)
1162 assert not self.git('fsck --dangling')
1163
1164 def test_shallow_srcrev_branch_truncation(self):
1165 self.add_empty_file('a')
1166 self.add_empty_file('b')
1167 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1168 self.add_empty_file('c')
1169 self.assertRevCount(3, cwd=self.srcdir)
1170
1171 self.d.setVar('SRCREV', b_commit)
1172 self.fetch_shallow()
1173
1174 # The 'c' commit was removed entirely, and 'a' was removed from history
1175 self.assertRevCount(1, ['--all'])
1176 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1177 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1178 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1179 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1180
1181 def test_shallow_ref_pruning(self):
1182 self.add_empty_file('a')
1183 self.add_empty_file('b')
1184 self.git('branch a_branch', cwd=self.srcdir)
1185 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1186 self.assertRevCount(2, cwd=self.srcdir)
1187
1188 self.fetch_shallow()
1189
1190 self.assertRefs(['master', 'origin/master'])
1191 self.assertRevCount(1)
1192
1193 def test_shallow_multi_one_uri(self):
1194 # Create initial git repo
1195 self.add_empty_file('a')
1196 self.add_empty_file('b')
1197 self.git('checkout -b a_branch', cwd=self.srcdir)
1198 self.add_empty_file('c')
1199 self.add_empty_file('d')
1200 self.git('checkout master', cwd=self.srcdir)
1201 self.add_empty_file('e')
1202 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1203 self.add_empty_file('f')
1204 self.assertRevCount(7, cwd=self.srcdir)
1205
1206 uri = self.d.getVar('SRC_URI', True).split()[0]
1207 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1208
1209 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1210 self.d.setVar('SRCREV_master', '${AUTOREV}')
1211 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1212
1213 self.fetch_shallow(uri)
1214
1215 self.assertRevCount(3, ['--all'])
1216 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1217
1218 def test_shallow_clone_preferred_over_shallow(self):
1219 self.add_empty_file('a')
1220 self.add_empty_file('b')
1221
1222 # Fetch once to generate the shallow tarball
1223 fetcher, ud = self.fetch()
1224 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1225
1226 # Fetch and unpack with both the clonedir and shallow tarball available
1227 bb.utils.remove(self.gitdir, recurse=True)
1228 fetcher, ud = self.fetch_and_unpack()
1229
1230 # The unpacked tree should *not* be shallow
1231 self.assertRevCount(2)
1232 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
1233
1234 def test_shallow_mirrors(self):
1235 self.add_empty_file('a')
1236 self.add_empty_file('b')
1237
1238 # Fetch once to generate the shallow tarball
1239 fetcher, ud = self.fetch()
1240 mirrortarball = ud.mirrortarballs[0]
1241 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
1242
1243 # Set up the mirror
1244 mirrordir = os.path.join(self.tempdir, 'mirror')
1245 bb.utils.mkdirhier(mirrordir)
1246 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
1247
1248 os.rename(os.path.join(self.dldir, mirrortarball),
1249 os.path.join(mirrordir, mirrortarball))
1250
1251 # Fetch from the mirror
1252 bb.utils.remove(self.dldir, recurse=True)
1253 bb.utils.remove(self.gitdir, recurse=True)
1254 self.fetch_and_unpack()
1255 self.assertRevCount(1)
1256
1257 def test_shallow_invalid_depth(self):
1258 self.add_empty_file('a')
1259 self.add_empty_file('b')
1260
1261 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
1262 with self.assertRaises(bb.fetch2.FetchError):
1263 self.fetch()
1264
1265 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
1266 print("Unset BB_SKIP_NETTESTS to run network tests")
1267 else:
1268 def test_bitbake(self):
1269 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
1270 self.git('config core.bare true', cwd=self.srcdir)
1271 self.git('fetch --tags', cwd=self.srcdir)
1272
1273 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '100')
1274
1275 self.fetch_shallow()
1276
1277 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
1278 revs = len(self.git('rev-list master').splitlines())
1279 self.assertNotEqual(orig_revs, revs)
1280 self.assertRefs(['master', 'origin/master'])