summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorChristopher Larson <kergoth@gmail.com>2017-05-13 02:46:33 +0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-06-02 13:36:57 +0100
commit35ecff3cf077bd22cf7d0a6145732cdcc34f15dc (patch)
tree14705c24efa0926d8ba6ccc2123c9fb2244e4345 /bitbake/lib
parent30485b2b1a3d49e0a72c9370e2ca2e763d83ace6 (diff)
downloadpoky-35ecff3cf077bd22cf7d0a6145732cdcc34f15dc.tar.gz
bitbake: fetch/git: add support for removing arbitrary revs for shallow
In certain cases, it's valuable to be able to exert more control over what history is removed, beyond srcrev+depth. As one example, you can remove most of the upstream kernel history from a kernel repository, keeping predominently the non-publically-accessible content. If the repository is private, the history in that repo couldn't be restored via `git fetch --unshallow`, but upstream history could be. Example usage: # Remove only these revs, not at a particular depth BB_GIT_SHALLOW_DEPTH_pn-linux-foo = "0" BB_GIT_SHALLOW_REVS_pn-linux-foo = "v4.1" (Bitbake rev: 97f856f0455d014ea34c28b1c25f09e13cdc851b) Signed-off-by: Christopher Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/fetch2/git.py18
-rw-r--r--bitbake/lib/bb/tests/fetch.py66
2 files changed, 82 insertions, 2 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index aa972c5cf4..534c93d3c5 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -196,6 +196,8 @@ class Git(FetchMethod):
196 depth_default = 1 196 depth_default = 1
197 ud.shallow_depths = collections.defaultdict(lambda: depth_default) 197 ud.shallow_depths = collections.defaultdict(lambda: depth_default)
198 198
199 revs_default = d.getVar("BB_GIT_SHALLOW_REVS", True)
200 ud.shallow_revs = []
199 ud.branches = {} 201 ud.branches = {}
200 for pos, name in enumerate(ud.names): 202 for pos, name in enumerate(ud.names):
201 branch = branches[pos] 203 branch = branches[pos]
@@ -213,7 +215,14 @@ class Git(FetchMethod):
213 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth)) 215 raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
214 ud.shallow_depths[name] = shallow_depth 216 ud.shallow_depths[name] = shallow_depth
215 217
218 revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
219 if revs is not None:
220 ud.shallow_revs.extend(revs.split())
221 elif revs_default is not None:
222 ud.shallow_revs.extend(revs_default.split())
223
216 if (ud.shallow and 224 if (ud.shallow and
225 not ud.shallow_revs and
217 all(ud.shallow_depths[n] == 0 for n in ud.names)): 226 all(ud.shallow_depths[n] == 0 for n in ud.names)):
218 # Shallow disabled for this URL 227 # Shallow disabled for this URL
219 ud.shallow = False 228 ud.shallow = False
@@ -261,6 +270,9 @@ class Git(FetchMethod):
261 if ud.bareclone: 270 if ud.bareclone:
262 tarballname = "%s_bare" % tarballname 271 tarballname = "%s_bare" % tarballname
263 272
273 if ud.shallow_revs:
274 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
275
264 for name, revision in sorted(ud.revisions.items()): 276 for name, revision in sorted(ud.revisions.items()):
265 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7]) 277 tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
266 depth = ud.shallow_depths[name] 278 depth = ud.shallow_depths[name]
@@ -413,7 +425,11 @@ class Git(FetchMethod):
413 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest) 425 runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
414 426
415 # Map srcrev+depths to revisions 427 # Map srcrev+depths to revisions
416 shallow_revisions = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest).splitlines() 428 parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
429
430 # Resolve specified revisions
431 parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
432 shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
417 433
418 # Apply extra ref wildcards 434 # Apply extra ref wildcards
419 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd, 435 all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 73f7b3f78e..343ae8fe5e 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -1259,6 +1259,33 @@ class GitShallowTest(FetcherTest):
1259 self.add_empty_file('c') 1259 self.add_empty_file('c')
1260 self.add_empty_file('d') 1260 self.add_empty_file('d')
1261 self.git('checkout master', cwd=self.srcdir) 1261 self.git('checkout master', cwd=self.srcdir)
1262 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1263 self.add_empty_file('e')
1264 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1265 self.add_empty_file('f')
1266 self.assertRevCount(7, cwd=self.srcdir)
1267
1268 uri = self.d.getVar('SRC_URI', True).split()[0]
1269 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1270
1271 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1272 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1273 self.d.setVar('SRCREV_master', '${AUTOREV}')
1274 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1275
1276 self.fetch_shallow(uri)
1277
1278 self.assertRevCount(5)
1279 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1280
1281 def test_shallow_multi_one_uri_depths(self):
1282 # Create initial git repo
1283 self.add_empty_file('a')
1284 self.add_empty_file('b')
1285 self.git('checkout -b a_branch', cwd=self.srcdir)
1286 self.add_empty_file('c')
1287 self.add_empty_file('d')
1288 self.git('checkout master', cwd=self.srcdir)
1262 self.add_empty_file('e') 1289 self.add_empty_file('e')
1263 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir) 1290 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1264 self.add_empty_file('f') 1291 self.add_empty_file('f')
@@ -1375,6 +1402,38 @@ class GitShallowTest(FetcherTest):
1375 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*') 1402 self.d.setVar('BB_GIT_SHALLOW_EXTRA_REFS', 'refs/tags/*')
1376 self.fetch() 1403 self.fetch()
1377 1404
1405 def test_shallow_remove_revs(self):
1406 # Create initial git repo
1407 self.add_empty_file('a')
1408 self.add_empty_file('b')
1409 self.git('checkout -b a_branch', cwd=self.srcdir)
1410 self.add_empty_file('c')
1411 self.add_empty_file('d')
1412 self.git('checkout master', cwd=self.srcdir)
1413 self.git('tag v0.0 a_branch', cwd=self.srcdir)
1414 self.add_empty_file('e')
1415 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1416 self.git('branch -d a_branch', cwd=self.srcdir)
1417 self.add_empty_file('f')
1418 self.assertRevCount(7, cwd=self.srcdir)
1419
1420 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1421 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1422
1423 self.fetch_shallow()
1424
1425 self.assertRevCount(5)
1426
1427 def test_shallow_invalid_revs(self):
1428 self.add_empty_file('a')
1429 self.add_empty_file('b')
1430
1431 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1432 self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
1433
1434 with self.assertRaises(bb.fetch2.FetchError):
1435 self.fetch()
1436
1378 if os.environ.get("BB_SKIP_NETTESTS") == "yes": 1437 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
1379 print("Unset BB_SKIP_NETTESTS to run network tests") 1438 print("Unset BB_SKIP_NETTESTS to run network tests")
1380 else: 1439 else:
@@ -1383,11 +1442,16 @@ class GitShallowTest(FetcherTest):
1383 self.git('config core.bare true', cwd=self.srcdir) 1442 self.git('config core.bare true', cwd=self.srcdir)
1384 self.git('fetch --tags', cwd=self.srcdir) 1443 self.git('fetch --tags', cwd=self.srcdir)
1385 1444
1386 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '100') 1445 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1446 # Note that the 1.10.0 tag is annotated, so this also tests
1447 # reference of an annotated vs unannotated tag
1448 self.d.setVar('BB_GIT_SHALLOW_REVS', '1.10.0')
1387 1449
1388 self.fetch_shallow() 1450 self.fetch_shallow()
1389 1451
1452 # Confirm that the history of 1.10.0 was removed
1390 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines()) 1453 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
1391 revs = len(self.git('rev-list master').splitlines()) 1454 revs = len(self.git('rev-list master').splitlines())
1392 self.assertNotEqual(orig_revs, revs) 1455 self.assertNotEqual(orig_revs, revs)
1393 self.assertRefs(['master', 'origin/master']) 1456 self.assertRefs(['master', 'origin/master'])
1457 self.assertRevCount(orig_revs - 1758)