summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/tests
diff options
context:
space:
mode:
authorChristopher Larson <kergoth@gmail.com>2017-05-13 02:46:28 +0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-06-02 13:36:57 +0100
commit27d56982c7ba05e86a100b0cca2411ee5ac7a85e (patch)
tree61d9ca5571770cfd093b327a7f9b7ca7ad3b56fe /bitbake/lib/bb/tests
parent2a60c406372d400437ecaa8712e6dc80b3d9fcec (diff)
downloadpoky-27d56982c7ba05e86a100b0cca2411ee5ac7a85e.tar.gz
bitbake: fetch/git: add support for shallow mirror tarballs
This adds support to the git fetcher for fetching, using, and generating mirror tarballs of shallow git repositories. The external git-make-shallow script is used for shallow mirror tarball creation. This implements support for shallow mirror tarballs, not shallow clones. Supporting shallow clones directly is not really doable for us, as we'd need to hardcode the depth between branch HEAD and the SRCREV, and that depth would change as the branch is updated. When BB_GIT_SHALLOW is enabled, we will always attempt to fetch a shallow mirror tarball. If the shallow mirror tarball cannot be fetched, it will try to fetch the full mirror tarball and use that. If a shallow tarball is to be used, it will be unpacked directly at `do_unpack` time, rather than extracting it to DL_DIR at `do_fetch` time and cloning from there, to keep things simple. There's no value in keeping a shallow repository in DL_DIR, and dealing with the state for when to convert the clonedir to/from shallow is not worthwhile. To clarify when shallow is used vs a real repository, a current clone is preferred to either tarball, a shallow tarball is preferred to an out of date clone, and a missing clone will use either tarball (attempting the shallow one first). All referenced branches are truncated to SRCREV (that is, commits *after* SRCREV but before HEAD are removed) to further shrink the repository. By default, the shallow construction process removes all unused refs (branches/tags) from the repository, other than those referenced by the URL. Example usage: BB_GIT_SHALLOW ?= "1" # Keep only the top commit BB_GIT_SHALLOW_DEPTH ?= "1" # This defaults to enabled if both BB_GIT_SHALLOW and # BB_GENERATE_MIRROR_TARBALLS are enabled BB_GENERATE_SHALLOW_TARBALLS ?= "1" (Bitbake rev: 5ed7d85fda7c671be10ec24d7981b87a7d0d3366) Signed-off-by: Christopher Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/tests')
-rw-r--r--bitbake/lib/bb/tests/fetch.py299
1 files changed, 299 insertions, 0 deletions
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 510071d25d..019f22a11d 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -979,3 +979,302 @@ class GitMakeShallowTest(FetcherTest):
979 orig_revs = len(self.git('rev-list --all').splitlines()) 979 orig_revs = len(self.git('rev-list --all').splitlines())
980 self.make_shallow(['refs/tags/1.10.0']) 980 self.make_shallow(['refs/tags/1.10.0'])
981 self.assertRevCount(orig_revs - 1746, ['--all']) 981 self.assertRevCount(orig_revs - 1746, ['--all'])
982
983class GitShallowTest(FetcherTest):
984 def setUp(self):
985 FetcherTest.setUp(self)
986 self.gitdir = os.path.join(self.tempdir, 'git')
987 self.srcdir = os.path.join(self.tempdir, 'gitsource')
988
989 bb.utils.mkdirhier(self.srcdir)
990 self.git('init', cwd=self.srcdir)
991 self.d.setVar('WORKDIR', self.tempdir)
992 self.d.setVar('S', self.gitdir)
993 self.d.delVar('PREMIRRORS')
994 self.d.delVar('MIRRORS')
995
996 uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
997 self.d.setVar('SRC_URI', uri)
998 self.d.setVar('SRCREV', '${AUTOREV}')
999 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
1000
1001 self.d.setVar('BB_GIT_SHALLOW', '1')
1002 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1003 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1004
1005 def assertRefs(self, expected_refs, cwd=None):
1006 if cwd is None:
1007 cwd = self.gitdir
1008 actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
1009 full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
1010 self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
1011
1012 def assertRevCount(self, expected_count, args=None, cwd=None):
1013 if args is None:
1014 args = ['HEAD']
1015 if cwd is None:
1016 cwd = self.gitdir
1017 revs = self.git(['rev-list'] + args, cwd=cwd)
1018 actual_count = len(revs.splitlines())
1019 self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
1020
1021 def git(self, cmd, cwd=None):
1022 if isinstance(cmd, str):
1023 cmd = 'git ' + cmd
1024 else:
1025 cmd = ['git'] + cmd
1026 if cwd is None:
1027 cwd = self.gitdir
1028 return bb.process.run(cmd, cwd=cwd)[0]
1029
1030 def add_empty_file(self, path, msg=None):
1031 if msg is None:
1032 msg = path
1033 open(os.path.join(self.srcdir, path), 'w').close()
1034 self.git(['add', path], self.srcdir)
1035 self.git(['commit', '-m', msg, path], self.srcdir)
1036
1037 def fetch(self, uri=None):
1038 if uri is None:
1039 uris = self.d.getVar('SRC_URI', True).split()
1040 uri = uris[0]
1041 d = self.d
1042 else:
1043 d = self.d.createCopy()
1044 d.setVar('SRC_URI', uri)
1045 uri = d.expand(uri)
1046 uris = [uri]
1047
1048 fetcher = bb.fetch2.Fetch(uris, d)
1049 fetcher.download()
1050 ud = fetcher.ud[uri]
1051 return fetcher, ud
1052
1053 def fetch_and_unpack(self, uri=None):
1054 fetcher, ud = self.fetch(uri)
1055 fetcher.unpack(self.d.getVar('WORKDIR'))
1056 assert os.path.exists(self.d.getVar('S'))
1057 return fetcher, ud
1058
1059 def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
1060 """Fetch a uri, generating a shallow tarball, then unpack using it"""
1061 fetcher, ud = self.fetch_and_unpack(uri)
1062 assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
1063
1064 # Confirm that the unpacked repo is unshallow
1065 if not disabled:
1066 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1067
1068 # fetch and unpack, from the shallow tarball
1069 bb.utils.remove(self.gitdir, recurse=True)
1070 bb.utils.remove(ud.clonedir, recurse=True)
1071
1072 # confirm that the unpacked repo is used when no git clone or git
1073 # mirror tarball is available
1074 fetcher, ud = self.fetch_and_unpack(uri)
1075 if not disabled:
1076 assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is not shallow' % self.gitdir
1077 else:
1078 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow')), 'Unpacked git repository at %s is shallow' % self.gitdir
1079 return fetcher, ud
1080
1081 def test_shallow_disabled(self):
1082 self.add_empty_file('a')
1083 self.add_empty_file('b')
1084 self.assertRevCount(2, cwd=self.srcdir)
1085
1086 self.d.setVar('BB_GIT_SHALLOW', '0')
1087 self.fetch_shallow(disabled=True)
1088 self.assertRevCount(2)
1089
1090 def test_shallow_nobranch(self):
1091 self.add_empty_file('a')
1092 self.add_empty_file('b')
1093 self.assertRevCount(2, cwd=self.srcdir)
1094
1095 srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
1096 self.d.setVar('SRCREV', srcrev)
1097 uri = self.d.getVar('SRC_URI', True).split()[0]
1098 uri = '%s;nobranch=1;bare=1' % uri
1099
1100 self.fetch_shallow(uri)
1101 self.assertRevCount(1)
1102
1103 # shallow refs are used to ensure the srcrev sticks around when we
1104 # have no other branches referencing it
1105 self.assertRefs(['refs/shallow/default'])
1106
1107 def test_shallow_default_depth_1(self):
1108 # Create initial git repo
1109 self.add_empty_file('a')
1110 self.add_empty_file('b')
1111 self.assertRevCount(2, cwd=self.srcdir)
1112
1113 self.fetch_shallow()
1114 self.assertRevCount(1)
1115
1116 def test_shallow_depth_0_disables(self):
1117 self.add_empty_file('a')
1118 self.add_empty_file('b')
1119 self.assertRevCount(2, cwd=self.srcdir)
1120
1121 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
1122 self.fetch_shallow(disabled=True)
1123 self.assertRevCount(2)
1124
1125 def test_current_shallow_out_of_date_clone(self):
1126 # Create initial git repo
1127 self.add_empty_file('a')
1128 self.add_empty_file('b')
1129 self.add_empty_file('c')
1130 self.assertRevCount(3, cwd=self.srcdir)
1131
1132 # Clone and generate mirror tarball
1133 fetcher, ud = self.fetch()
1134
1135 # Ensure we have a current mirror tarball, but an out of date clone
1136 self.git('update-ref refs/heads/master refs/heads/master~1', cwd=ud.clonedir)
1137 self.assertRevCount(2, cwd=ud.clonedir)
1138
1139 # Fetch and unpack, from the current tarball, not the out of date clone
1140 bb.utils.remove(self.gitdir, recurse=True)
1141 fetcher, ud = self.fetch()
1142 fetcher.unpack(self.d.getVar('WORKDIR'))
1143 self.assertRevCount(1)
1144
1145 def test_shallow_single_branch_no_merge(self):
1146 self.add_empty_file('a')
1147 self.add_empty_file('b')
1148 self.assertRevCount(2, cwd=self.srcdir)
1149
1150 self.fetch_shallow()
1151 self.assertRevCount(1)
1152 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1153 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1154
1155 def test_shallow_no_dangling(self):
1156 self.add_empty_file('a')
1157 self.add_empty_file('b')
1158 self.assertRevCount(2, cwd=self.srcdir)
1159
1160 self.fetch_shallow()
1161 self.assertRevCount(1)
1162 assert not self.git('fsck --dangling')
1163
1164 def test_shallow_srcrev_branch_truncation(self):
1165 self.add_empty_file('a')
1166 self.add_empty_file('b')
1167 b_commit = self.git('rev-parse HEAD', cwd=self.srcdir).rstrip()
1168 self.add_empty_file('c')
1169 self.assertRevCount(3, cwd=self.srcdir)
1170
1171 self.d.setVar('SRCREV', b_commit)
1172 self.fetch_shallow()
1173
1174 # The 'c' commit was removed entirely, and 'a' was removed from history
1175 self.assertRevCount(1, ['--all'])
1176 self.assertEqual(self.git('rev-parse HEAD').strip(), b_commit)
1177 assert os.path.exists(os.path.join(self.gitdir, 'a'))
1178 assert os.path.exists(os.path.join(self.gitdir, 'b'))
1179 assert not os.path.exists(os.path.join(self.gitdir, 'c'))
1180
1181 def test_shallow_ref_pruning(self):
1182 self.add_empty_file('a')
1183 self.add_empty_file('b')
1184 self.git('branch a_branch', cwd=self.srcdir)
1185 self.assertRefs(['master', 'a_branch'], cwd=self.srcdir)
1186 self.assertRevCount(2, cwd=self.srcdir)
1187
1188 self.fetch_shallow()
1189
1190 self.assertRefs(['master', 'origin/master'])
1191 self.assertRevCount(1)
1192
1193 def test_shallow_multi_one_uri(self):
1194 # Create initial git repo
1195 self.add_empty_file('a')
1196 self.add_empty_file('b')
1197 self.git('checkout -b a_branch', cwd=self.srcdir)
1198 self.add_empty_file('c')
1199 self.add_empty_file('d')
1200 self.git('checkout master', cwd=self.srcdir)
1201 self.add_empty_file('e')
1202 self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
1203 self.add_empty_file('f')
1204 self.assertRevCount(7, cwd=self.srcdir)
1205
1206 uri = self.d.getVar('SRC_URI', True).split()[0]
1207 uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
1208
1209 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '2')
1210 self.d.setVar('SRCREV_master', '${AUTOREV}')
1211 self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
1212
1213 self.fetch_shallow(uri)
1214
1215 self.assertRevCount(3, ['--all'])
1216 self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
1217
1218 def test_shallow_clone_preferred_over_shallow(self):
1219 self.add_empty_file('a')
1220 self.add_empty_file('b')
1221
1222 # Fetch once to generate the shallow tarball
1223 fetcher, ud = self.fetch()
1224 assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
1225
1226 # Fetch and unpack with both the clonedir and shallow tarball available
1227 bb.utils.remove(self.gitdir, recurse=True)
1228 fetcher, ud = self.fetch_and_unpack()
1229
1230 # The unpacked tree should *not* be shallow
1231 self.assertRevCount(2)
1232 assert not os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
1233
1234 def test_shallow_mirrors(self):
1235 self.add_empty_file('a')
1236 self.add_empty_file('b')
1237
1238 # Fetch once to generate the shallow tarball
1239 fetcher, ud = self.fetch()
1240 mirrortarball = ud.mirrortarballs[0]
1241 assert os.path.exists(os.path.join(self.dldir, mirrortarball))
1242
1243 # Set up the mirror
1244 mirrordir = os.path.join(self.tempdir, 'mirror')
1245 bb.utils.mkdirhier(mirrordir)
1246 self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
1247
1248 os.rename(os.path.join(self.dldir, mirrortarball),
1249 os.path.join(mirrordir, mirrortarball))
1250
1251 # Fetch from the mirror
1252 bb.utils.remove(self.dldir, recurse=True)
1253 bb.utils.remove(self.gitdir, recurse=True)
1254 self.fetch_and_unpack()
1255 self.assertRevCount(1)
1256
1257 def test_shallow_invalid_depth(self):
1258 self.add_empty_file('a')
1259 self.add_empty_file('b')
1260
1261 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '-12')
1262 with self.assertRaises(bb.fetch2.FetchError):
1263 self.fetch()
1264
1265 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
1266 print("Unset BB_SKIP_NETTESTS to run network tests")
1267 else:
1268 def test_bitbake(self):
1269 self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
1270 self.git('config core.bare true', cwd=self.srcdir)
1271 self.git('fetch --tags', cwd=self.srcdir)
1272
1273 self.d.setVar('BB_GIT_SHALLOW_DEPTH', '100')
1274
1275 self.fetch_shallow()
1276
1277 orig_revs = len(self.git('rev-list master', cwd=self.srcdir).splitlines())
1278 revs = len(self.git('rev-list master').splitlines())
1279 self.assertNotEqual(orig_revs, revs)
1280 self.assertRefs(['master', 'origin/master'])