diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/gitsm.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/gitsm.py | 126 |
1 files changed, 64 insertions, 62 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py index f7f3af7212..5869e1b99b 100644 --- a/bitbake/lib/bb/fetch2/gitsm.py +++ b/bitbake/lib/bb/fetch2/gitsm.py | |||
@@ -62,36 +62,35 @@ class GitSM(Git): | |||
62 | return modules | 62 | return modules |
63 | 63 | ||
64 | # Collect the defined submodules, and their attributes | 64 | # Collect the defined submodules, and their attributes |
65 | for name in ud.names: | 65 | try: |
66 | gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir) | ||
67 | except: | ||
68 | # No submodules to update | ||
69 | gitmodules = "" | ||
70 | |||
71 | for m, md in parse_gitmodules(gitmodules).items(): | ||
66 | try: | 72 | try: |
67 | gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) | 73 | module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir) |
68 | except: | 74 | except: |
69 | # No submodules to update | 75 | # If the command fails, we don't have a valid file to check. If it doesn't |
76 | # fail -- it still might be a failure, see next check... | ||
77 | module_hash = "" | ||
78 | |||
79 | if not module_hash: | ||
80 | logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) | ||
70 | continue | 81 | continue |
71 | 82 | ||
72 | for m, md in parse_gitmodules(gitmodules).items(): | 83 | submodules.append(m) |
73 | try: | 84 | paths[m] = md['path'] |
74 | module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) | 85 | revision[m] = ud.revision |
75 | except: | 86 | uris[m] = md['url'] |
76 | # If the command fails, we don't have a valid file to check. If it doesn't | 87 | subrevision[m] = module_hash.split()[2] |
77 | # fail -- it still might be a failure, see next check... | 88 | |
78 | module_hash = "" | 89 | # Convert relative to absolute uri based on parent uri |
79 | 90 | if uris[m].startswith('..') or uris[m].startswith('./'): | |
80 | if not module_hash: | 91 | newud = copy.copy(ud) |
81 | logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) | 92 | newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) |
82 | continue | 93 | uris[m] = Git._get_repo_url(self, newud) |
83 | |||
84 | submodules.append(m) | ||
85 | paths[m] = md['path'] | ||
86 | revision[m] = ud.revisions[name] | ||
87 | uris[m] = md['url'] | ||
88 | subrevision[m] = module_hash.split()[2] | ||
89 | |||
90 | # Convert relative to absolute uri based on parent uri | ||
91 | if uris[m].startswith('..') or uris[m].startswith('./'): | ||
92 | newud = copy.copy(ud) | ||
93 | newud.path = os.path.normpath(os.path.join(newud.path, uris[m])) | ||
94 | uris[m] = Git._get_repo_url(self, newud) | ||
95 | 94 | ||
96 | for module in submodules: | 95 | for module in submodules: |
97 | # Translate the module url into a SRC_URI | 96 | # Translate the module url into a SRC_URI |
@@ -123,7 +122,7 @@ class GitSM(Git): | |||
123 | url += ";name=%s" % module | 122 | url += ";name=%s" % module |
124 | url += ";subpath=%s" % module | 123 | url += ";subpath=%s" % module |
125 | url += ";nobranch=1" | 124 | url += ";nobranch=1" |
126 | url += ";lfs=%s" % self._need_lfs(ud) | 125 | url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0") |
127 | # Note that adding "user=" here to give credentials to the | 126 | # Note that adding "user=" here to give credentials to the |
128 | # submodule is not supported. Since using SRC_URI to give git:// | 127 | # submodule is not supported. Since using SRC_URI to give git:// |
129 | # URL a password is not supported, one have to use one of the | 128 | # URL a password is not supported, one have to use one of the |
@@ -147,6 +146,22 @@ class GitSM(Git): | |||
147 | 146 | ||
148 | return submodules != [] | 147 | return submodules != [] |
149 | 148 | ||
149 | def call_process_submodules(self, ud, d, extra_check, subfunc): | ||
150 | # If we're using a shallow mirror tarball it needs to be | ||
151 | # unpacked temporarily so that we can examine the .gitmodules file | ||
152 | # Unpack even when ud.clonedir is not available, | ||
153 | # which may occur during a fast shallow clone | ||
154 | unpack = extra_check or not os.path.exists(ud.clonedir) | ||
155 | if ud.shallow and os.path.exists(ud.fullshallow) and unpack: | ||
156 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
157 | try: | ||
158 | runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) | ||
159 | self.process_submodules(ud, tmpdir, subfunc, d) | ||
160 | finally: | ||
161 | shutil.rmtree(tmpdir) | ||
162 | else: | ||
163 | self.process_submodules(ud, ud.clonedir, subfunc, d) | ||
164 | |||
150 | def need_update(self, ud, d): | 165 | def need_update(self, ud, d): |
151 | if Git.need_update(self, ud, d): | 166 | if Git.need_update(self, ud, d): |
152 | return True | 167 | return True |
@@ -164,15 +179,7 @@ class GitSM(Git): | |||
164 | logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) | 179 | logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) |
165 | need_update_result = True | 180 | need_update_result = True |
166 | 181 | ||
167 | # If we're using a shallow mirror tarball it needs to be unpacked | 182 | self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule) |
168 | # temporarily so that we can examine the .gitmodules file | ||
169 | if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir): | ||
170 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
171 | runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) | ||
172 | self.process_submodules(ud, tmpdir, need_update_submodule, d) | ||
173 | shutil.rmtree(tmpdir) | ||
174 | else: | ||
175 | self.process_submodules(ud, ud.clonedir, need_update_submodule, d) | ||
176 | 183 | ||
177 | if need_update_list: | 184 | if need_update_list: |
178 | logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) | 185 | logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) |
@@ -195,16 +202,7 @@ class GitSM(Git): | |||
195 | raise | 202 | raise |
196 | 203 | ||
197 | Git.download(self, ud, d) | 204 | Git.download(self, ud, d) |
198 | 205 | self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule) | |
199 | # If we're using a shallow mirror tarball it needs to be unpacked | ||
200 | # temporarily so that we can examine the .gitmodules file | ||
201 | if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): | ||
202 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
203 | runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) | ||
204 | self.process_submodules(ud, tmpdir, download_submodule, d) | ||
205 | shutil.rmtree(tmpdir) | ||
206 | else: | ||
207 | self.process_submodules(ud, ud.clonedir, download_submodule, d) | ||
208 | 206 | ||
209 | def unpack(self, ud, destdir, d): | 207 | def unpack(self, ud, destdir, d): |
210 | def unpack_submodules(ud, url, module, modpath, workdir, d): | 208 | def unpack_submodules(ud, url, module, modpath, workdir, d): |
@@ -247,15 +245,27 @@ class GitSM(Git): | |||
247 | ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) | 245 | ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) |
248 | 246 | ||
249 | if not ud.bareclone and ret: | 247 | if not ud.bareclone and ret: |
250 | # All submodules should already be downloaded and configured in the tree. This simply | 248 | cmdprefix = "" |
251 | # sets up the configuration and checks out the files. The main project config should | 249 | # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed. |
252 | # remain unmodified, and no download from the internet should occur. As such, lfs smudge | 250 | if not self._need_lfs(ud): |
253 | # should also be skipped as these files were already smudged in the fetch stage if lfs | 251 | cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 " |
254 | # was enabled. | 252 | runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir) |
255 | runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) | 253 | def clean(self, ud, d): |
254 | def clean_submodule(ud, url, module, modpath, workdir, d): | ||
255 | url += ";bareclone=1;nobranch=1" | ||
256 | try: | ||
257 | newfetch = Fetch([url], d, cache=False) | ||
258 | newfetch.clean() | ||
259 | except Exception as e: | ||
260 | logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e))) | ||
261 | |||
262 | self.call_process_submodules(ud, d, True, clean_submodule) | ||
263 | |||
264 | # Clean top git dir | ||
265 | Git.clean(self, ud, d) | ||
256 | 266 | ||
257 | def implicit_urldata(self, ud, d): | 267 | def implicit_urldata(self, ud, d): |
258 | import shutil, subprocess, tempfile | 268 | import subprocess |
259 | 269 | ||
260 | urldata = [] | 270 | urldata = [] |
261 | def add_submodule(ud, url, module, modpath, workdir, d): | 271 | def add_submodule(ud, url, module, modpath, workdir, d): |
@@ -263,14 +273,6 @@ class GitSM(Git): | |||
263 | newfetch = Fetch([url], d, cache=False) | 273 | newfetch = Fetch([url], d, cache=False) |
264 | urldata.extend(newfetch.expanded_urldata()) | 274 | urldata.extend(newfetch.expanded_urldata()) |
265 | 275 | ||
266 | # If we're using a shallow mirror tarball it needs to be unpacked | 276 | self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule) |
267 | # temporarily so that we can examine the .gitmodules file | ||
268 | if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): | ||
269 | tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) | ||
270 | subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) | ||
271 | self.process_submodules(ud, tmpdir, add_submodule, d) | ||
272 | shutil.rmtree(tmpdir) | ||
273 | else: | ||
274 | self.process_submodules(ud, ud.clonedir, add_submodule, d) | ||
275 | 277 | ||
276 | return urldata | 278 | return urldata |