summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/gitsm.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/fetch2/gitsm.py')
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py126
1 files changed, 64 insertions, 62 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index f7f3af7212..5869e1b99b 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -62,36 +62,35 @@ class GitSM(Git):
62 return modules 62 return modules
63 63
64 # Collect the defined submodules, and their attributes 64 # Collect the defined submodules, and their attributes
65 for name in ud.names: 65 try:
66 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir)
67 except:
68 # No submodules to update
69 gitmodules = ""
70
71 for m, md in parse_gitmodules(gitmodules).items():
66 try: 72 try:
67 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir) 73 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir)
68 except: 74 except:
69 # No submodules to update 75 # If the command fails, we don't have a valid file to check. If it doesn't
76 # fail -- it still might be a failure, see next check...
77 module_hash = ""
78
79 if not module_hash:
80 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
70 continue 81 continue
71 82
72 for m, md in parse_gitmodules(gitmodules).items(): 83 submodules.append(m)
73 try: 84 paths[m] = md['path']
74 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir) 85 revision[m] = ud.revision
75 except: 86 uris[m] = md['url']
76 # If the command fails, we don't have a valid file to check. If it doesn't 87 subrevision[m] = module_hash.split()[2]
77 # fail -- it still might be a failure, see next check... 88
78 module_hash = "" 89 # Convert relative to absolute uri based on parent uri
79 90 if uris[m].startswith('..') or uris[m].startswith('./'):
80 if not module_hash: 91 newud = copy.copy(ud)
81 logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m) 92 newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
82 continue 93 uris[m] = Git._get_repo_url(self, newud)
83
84 submodules.append(m)
85 paths[m] = md['path']
86 revision[m] = ud.revisions[name]
87 uris[m] = md['url']
88 subrevision[m] = module_hash.split()[2]
89
90 # Convert relative to absolute uri based on parent uri
91 if uris[m].startswith('..') or uris[m].startswith('./'):
92 newud = copy.copy(ud)
93 newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
94 uris[m] = Git._get_repo_url(self, newud)
95 94
96 for module in submodules: 95 for module in submodules:
97 # Translate the module url into a SRC_URI 96 # Translate the module url into a SRC_URI
@@ -123,7 +122,7 @@ class GitSM(Git):
123 url += ";name=%s" % module 122 url += ";name=%s" % module
124 url += ";subpath=%s" % module 123 url += ";subpath=%s" % module
125 url += ";nobranch=1" 124 url += ";nobranch=1"
126 url += ";lfs=%s" % self._need_lfs(ud) 125 url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0")
127 # Note that adding "user=" here to give credentials to the 126 # Note that adding "user=" here to give credentials to the
128 # submodule is not supported. Since using SRC_URI to give git:// 127 # submodule is not supported. Since using SRC_URI to give git://
129 # URL a password is not supported, one have to use one of the 128 # URL a password is not supported, one have to use one of the
@@ -147,6 +146,22 @@ class GitSM(Git):
147 146
148 return submodules != [] 147 return submodules != []
149 148
149 def call_process_submodules(self, ud, d, extra_check, subfunc):
150 # If we're using a shallow mirror tarball it needs to be
151 # unpacked temporarily so that we can examine the .gitmodules file
152 # Unpack even when ud.clonedir is not available,
153 # which may occur during a fast shallow clone
154 unpack = extra_check or not os.path.exists(ud.clonedir)
155 if ud.shallow and os.path.exists(ud.fullshallow) and unpack:
156 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
157 try:
158 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
159 self.process_submodules(ud, tmpdir, subfunc, d)
160 finally:
161 shutil.rmtree(tmpdir)
162 else:
163 self.process_submodules(ud, ud.clonedir, subfunc, d)
164
150 def need_update(self, ud, d): 165 def need_update(self, ud, d):
151 if Git.need_update(self, ud, d): 166 if Git.need_update(self, ud, d):
152 return True 167 return True
@@ -164,15 +179,7 @@ class GitSM(Git):
164 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e))) 179 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
165 need_update_result = True 180 need_update_result = True
166 181
167 # If we're using a shallow mirror tarball it needs to be unpacked 182 self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule)
168 # temporarily so that we can examine the .gitmodules file
169 if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir):
170 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
171 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
172 self.process_submodules(ud, tmpdir, need_update_submodule, d)
173 shutil.rmtree(tmpdir)
174 else:
175 self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
176 183
177 if need_update_list: 184 if need_update_list:
178 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list))) 185 logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
@@ -195,16 +202,7 @@ class GitSM(Git):
195 raise 202 raise
196 203
197 Git.download(self, ud, d) 204 Git.download(self, ud, d)
198 205 self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule)
199 # If we're using a shallow mirror tarball it needs to be unpacked
200 # temporarily so that we can examine the .gitmodules file
201 if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
202 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
203 runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
204 self.process_submodules(ud, tmpdir, download_submodule, d)
205 shutil.rmtree(tmpdir)
206 else:
207 self.process_submodules(ud, ud.clonedir, download_submodule, d)
208 206
209 def unpack(self, ud, destdir, d): 207 def unpack(self, ud, destdir, d):
210 def unpack_submodules(ud, url, module, modpath, workdir, d): 208 def unpack_submodules(ud, url, module, modpath, workdir, d):
@@ -247,15 +245,27 @@ class GitSM(Git):
247 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d) 245 ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
248 246
249 if not ud.bareclone and ret: 247 if not ud.bareclone and ret:
250 # All submodules should already be downloaded and configured in the tree. This simply 248 cmdprefix = ""
251 # sets up the configuration and checks out the files. The main project config should 249 # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed.
252 # remain unmodified, and no download from the internet should occur. As such, lfs smudge 250 if not self._need_lfs(ud):
253 # should also be skipped as these files were already smudged in the fetch stage if lfs 251 cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 "
254 # was enabled. 252 runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir)
255 runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 253 def clean(self, ud, d):
254 def clean_submodule(ud, url, module, modpath, workdir, d):
255 url += ";bareclone=1;nobranch=1"
256 try:
257 newfetch = Fetch([url], d, cache=False)
258 newfetch.clean()
259 except Exception as e:
260 logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e)))
261
262 self.call_process_submodules(ud, d, True, clean_submodule)
263
264 # Clean top git dir
265 Git.clean(self, ud, d)
256 266
257 def implicit_urldata(self, ud, d): 267 def implicit_urldata(self, ud, d):
258 import shutil, subprocess, tempfile 268 import subprocess
259 269
260 urldata = [] 270 urldata = []
261 def add_submodule(ud, url, module, modpath, workdir, d): 271 def add_submodule(ud, url, module, modpath, workdir, d):
@@ -263,14 +273,6 @@ class GitSM(Git):
263 newfetch = Fetch([url], d, cache=False) 273 newfetch = Fetch([url], d, cache=False)
264 urldata.extend(newfetch.expanded_urldata()) 274 urldata.extend(newfetch.expanded_urldata())
265 275
266 # If we're using a shallow mirror tarball it needs to be unpacked 276 self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule)
267 # temporarily so that we can examine the .gitmodules file
268 if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
269 tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
270 subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
271 self.process_submodules(ud, tmpdir, add_submodule, d)
272 shutil.rmtree(tmpdir)
273 else:
274 self.process_submodules(ud, ud.clonedir, add_submodule, d)
275 277
276 return urldata 278 return urldata