summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2019-01-15 16:31:36 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-16 15:35:08 +0000
commita9cf611e7a689571a5b4a70c0fe76de89476cc7d (patch)
tree6d507dbe9e5ba59be9f2f0ef2878708e50e3cb73 /bitbake/lib
parentcd1430e379c9c63bacd4d0ccce0fc567d98a1e4a (diff)
downloadpoky-a9cf611e7a689571a5b4a70c0fe76de89476cc7d.tar.gz
bitbake: gitsm.py: Rework the shallow fetcher and test case
A custom shallow submodule is no longer necessary, as the regular git fetcher is used and shallow handling works with the same code. The only general difference between the regular change is simply declaring a clone as shallow, when appropriate. This also removes a potential race condition in copying repositories vs cloning them. The gitsm shallow fetcher test was revised to verify that the submodule is shallow cloned along with the primary repository. The first step of this change was to be sure to clean the gitsubmodule download directory, as was previously done with the may gitsource directory. Additional test components were added to verify commit counts, and an obsolete (and likely incorrect) test for the .git/modules directory to be empty was also removed. (Bitbake rev: f9cc4684dcf4281acc557cda8cb35602354ac3d6) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py66
-rw-r--r--bitbake/lib/bb/tests/fetch.py10
2 files changed, 13 insertions, 63 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index c172ab1660..83571f834b 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -150,68 +150,7 @@ class GitSM(Git):
150 150
151 def download(self, ud, d): 151 def download(self, ud, d):
152 Git.download(self, ud, d) 152 Git.download(self, ud, d)
153 153 self.update_submodules(ud, d)
154 if not ud.shallow or ud.localpath != ud.fullshallow:
155 self.update_submodules(ud, d)
156
157 def copy_submodules(self, submodules, ud, name, destdir, d):
158 if ud.bareclone:
159 repo_conf = destdir
160 else:
161 repo_conf = os.path.join(destdir, '.git')
162
163 if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')):
164 os.mkdir(os.path.join(repo_conf, 'modules'))
165
166 for module, md in submodules.items():
167 srcpath = os.path.join(ud.clonedir, 'modules', md['path'])
168 modpath = os.path.join(repo_conf, 'modules', md['path'])
169
170 # Check if the module is initialized
171 try:
172 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=ud.clonedir)
173 except:
174 # If the command fails, we don't have a valid file to check. If it doesn't
175 # fail -- it still might be a failure, see next check...
176 module_hash = ""
177
178 if not module_hash:
179 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module)
180 continue
181
182 if os.path.exists(srcpath):
183 if os.path.exists(os.path.join(srcpath, '.git')):
184 srcpath = os.path.join(srcpath, '.git')
185
186 target = modpath
187 if os.path.exists(modpath):
188 target = os.path.dirname(modpath)
189
190 os.makedirs(os.path.dirname(target), exist_ok=True)
191 runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d)
192 elif os.path.exists(modpath):
193 # Module already exists, likely unpacked from a shallow mirror clone
194 pass
195 else:
196 # This is fatal, as we do NOT want git-submodule to hit the network
197 raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath))
198
199 def clone_shallow_local(self, ud, dest, d):
200 super(GitSM, self).clone_shallow_local(ud, dest, d)
201
202 # Copy over the submodules' fetched histories too.
203 repo_conf = os.path.join(dest, '.git')
204
205 submodules = []
206 for name in ud.names:
207 try:
208 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest)
209 except:
210 # No submodules to update
211 continue
212
213 submodules = self.parse_gitmodules(gitmodules)
214 self.copy_submodules(submodules, ud, name, dest, d)
215 154
216 def unpack_submodules(self, repo_conf, ud, d): 155 def unpack_submodules(self, repo_conf, ud, d):
217 submodules = [] 156 submodules = []
@@ -294,6 +233,9 @@ class GitSM(Git):
294 # Correct the submodule references to the local download version... 233 # Correct the submodule references to the local download version...
295 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) 234 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
296 235
236 if ud.shallow:
237 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
238
297 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 239 # Ensure the submodule repository is NOT set to bare, since we're checking it out...
298 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module])) 240 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module]))
299 241
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 311c70137e..5fb5d04cb0 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -1316,6 +1316,7 @@ class GitShallowTest(FetcherTest):
1316 # fetch and unpack, from the shallow tarball 1316 # fetch and unpack, from the shallow tarball
1317 bb.utils.remove(self.gitdir, recurse=True) 1317 bb.utils.remove(self.gitdir, recurse=True)
1318 bb.utils.remove(ud.clonedir, recurse=True) 1318 bb.utils.remove(ud.clonedir, recurse=True)
1319 bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
1319 1320
1320 # confirm that the unpacked repo is used when no git clone or git 1321 # confirm that the unpacked repo is used when no git clone or git
1321 # mirror tarball is available 1322 # mirror tarball is available
@@ -1470,6 +1471,7 @@ class GitShallowTest(FetcherTest):
1470 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) 1471 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1471 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) 1472 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
1472 self.add_empty_file('asub', cwd=smdir) 1473 self.add_empty_file('asub', cwd=smdir)
1474 self.add_empty_file('bsub', cwd=smdir)
1473 1475
1474 self.git('submodule init', cwd=self.srcdir) 1476 self.git('submodule init', cwd=self.srcdir)
1475 self.git('submodule add file://%s' % smdir, cwd=self.srcdir) 1477 self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
@@ -1479,10 +1481,16 @@ class GitShallowTest(FetcherTest):
1479 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir 1481 uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
1480 fetcher, ud = self.fetch_shallow(uri) 1482 fetcher, ud = self.fetch_shallow(uri)
1481 1483
1484 # Verify the main repository is shallow
1482 self.assertRevCount(1) 1485 self.assertRevCount(1)
1483 assert './.git/modules/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0] 1486
1487 # Verify the gitsubmodule directory is present
1484 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) 1488 assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule'))
1485 1489
1490 # Verify the submodule is also shallow
1491 self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule'))
1492
1493
1486 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): 1494 if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')):
1487 def test_shallow_annex(self): 1495 def test_shallow_annex(self):
1488 self.add_empty_file('a') 1496 self.add_empty_file('a')