summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2018-09-25 13:15:25 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-09-26 15:14:33 +0100
commit5cd00e3e53819f3c10d1733480077ce9f3cc2ca8 (patch)
tree2b0df39dabfafe0465827ec1211c6260d4c5e5d2 /bitbake/lib
parentf61ef5b45417382ff94e9d389d8eeda490026d73 (diff)
downloadpoky-5cd00e3e53819f3c10d1733480077ce9f3cc2ca8.tar.gz
bitbake: fetch2/gitsm.py: Rework the git submodule fetcher
The prior fetcher did not know how to work with MIRRORS, and did not honor BB_NO_NETWORK and similar. The new fetcher approach recursively calls 'gitsm' download on each submodule detected. This ensures that it will go throug the standard download process. Each downloaded submodule is then 'attached' to the original download in the 'modules' directory. This mimics the behavior of: git submodule init but there is no chance it will contact the network without permission. It then corrects upstream reference URIs. The unpack steps simply copies the items from the downloads to the destdir. Once copied the submodules are connected and we then run: git submodule update According to the git documentation, git submodule init can and will modify the project configuration and may connect to the network. Doing the work manually prevents this. (This manual process is allowed based on my reading of the documentation.) See: https://git-scm.com/book/en/v2/Git-Tools-Submodules The small change to the existing test is due to this new code always assuming the code is from a remote system, and not a 'local' repository. If this assumption proves to be incorrect -- code will need to be added to deal with local repositories without an upstream URI. (Bitbake rev: 9c6b39adf9781fa6745f48913a97c859fa37eb5b) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py279
-rw-r--r--bitbake/lib/bb/tests/fetch.py3
2 files changed, 159 insertions, 123 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index 86773094dc..51f8d0e68c 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -34,6 +34,8 @@ import bb
34from bb.fetch2.git import Git 34from bb.fetch2.git import Git
35from bb.fetch2 import runfetchcmd 35from bb.fetch2 import runfetchcmd
36from bb.fetch2 import logger 36from bb.fetch2 import logger
37from bb.fetch2 import Fetch
38from bb.fetch2 import BBFetchException
37 39
38class GitSM(Git): 40class GitSM(Git):
39 def supports(self, ud, d): 41 def supports(self, ud, d):
@@ -42,96 +44,66 @@ class GitSM(Git):
42 """ 44 """
43 return ud.type in ['gitsm'] 45 return ud.type in ['gitsm']
44 46
45 def uses_submodules(self, ud, d, wd): 47 def update_submodules(self, ud, d):
48 submodules = []
49 paths = {}
50 uris = {}
51 local_paths = {}
52
46 for name in ud.names: 53 for name in ud.names:
47 try: 54 try:
48 runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd) 55 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir)
49 return True 56 except:
50 except bb.fetch.FetchError: 57 # No submodules to update
51 pass 58 continue
52 return False
53 59
54 def _set_relative_paths(self, repopath): 60 module = ""
55 """ 61 for line in gitmodules.splitlines():
56 Fix submodule paths to be relative instead of absolute,
57 so that when we move the repo it doesn't break
58 (In Git 1.7.10+ this is done automatically)
59 """
60 submodules = []
61 with open(os.path.join(repopath, '.gitmodules'), 'r') as f:
62 for line in f.readlines():
63 if line.startswith('[submodule'): 62 if line.startswith('[submodule'):
64 submodules.append(line.split('"')[1]) 63 module = line.split('"')[1]
64 submodules.append(module)
65 elif module and line.strip().startswith('path'):
66 path = line.split('=')[1].strip()
67 paths[module] = path
68 elif module and line.strip().startswith('url'):
69 url = line.split('=')[1].strip()
70 uris[module] = url
65 71
66 for module in submodules: 72 for module in submodules:
67 repo_conf = os.path.join(repopath, module, '.git') 73 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir)
68 if os.path.exists(repo_conf): 74 module_hash = module_hash.split()[2]
69 with open(repo_conf, 'r') as f: 75
70 lines = f.readlines() 76 # Build new SRC_URI
71 newpath = '' 77 proto = uris[module].split(':', 1)[0]
72 for i, line in enumerate(lines): 78 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
73 if line.startswith('gitdir:'): 79 url += ';protocol=%s' % proto
74 oldpath = line.split(': ')[-1].rstrip() 80 url += ";name=%s" % module
75 if oldpath.startswith('/'): 81 url += ";qbareclone=1;nocheckout=1"
76 newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module 82
77 lines[i] = 'gitdir: %s\n' % newpath 83 ld = d.createCopy()
78 break 84 # Not necessary to set SRC_URI, since we're passing the URI to
79 if newpath: 85 # Fetch.
80 with open(repo_conf, 'w') as f: 86 #ld.setVar('SRC_URI', url)
81 for line in lines: 87 ld.setVar('SRCREV_%s' % module, module_hash)
82 f.write(line) 88
83 89 # Workaround for issues with SRCPV/SRCREV_FORMAT errors
84 repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config') 90 # error refer to 'multiple' repositories. Only the repository
85 if os.path.exists(repo_conf2): 91 # in the original SRC_URI actually matters...
86 with open(repo_conf2, 'r') as f: 92 ld.setVar('SRCPV', d.getVar('SRCPV'))
87 lines = f.readlines() 93 ld.setVar('SRCREV_FORMAT', module)
88 newpath = '' 94
89 for i, line in enumerate(lines): 95 newfetch = Fetch([url], ld, cache=False)
90 if line.lstrip().startswith('worktree = '): 96 newfetch.download()
91 oldpath = line.split(' = ')[-1].rstrip() 97 local_paths[module] = newfetch.localpath(url)
92 if oldpath.startswith('/'): 98
93 newpath = '../' * (module.count('/') + 3) + module 99 # Correct the submodule references to the local download version...
94 lines[i] = '\tworktree = %s\n' % newpath 100 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
95 break 101 try:
96 if newpath: 102 os.mkdir(os.path.join(ud.clonedir, 'modules'))
97 with open(repo_conf2, 'w') as f: 103 except OSError:
98 for line in lines: 104 pass
99 f.write(line) 105 if not os.path.exists(os.path.join(ud.clonedir, 'modules', paths[module])):
100 106 os.symlink(local_paths[module], os.path.join(ud.clonedir, 'modules', paths[module]))
101 def update_submodules(self, ud, d, allow_network):
102 # We have to convert bare -> full repo, do the submodule bit, then convert back
103 tmpclonedir = ud.clonedir + ".tmp"
104 gitdir = tmpclonedir + os.sep + ".git"
105 bb.utils.remove(tmpclonedir, True)
106 os.mkdir(tmpclonedir)
107 os.rename(ud.clonedir, gitdir)
108 runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d)
109 runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir)
110 runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir)
111
112 try:
113 if allow_network:
114 fetch_flags = ""
115 else:
116 fetch_flags = "--no-fetch"
117
118 # The 'git submodule sync' sandwiched between two successive 'git submodule update' commands is
119 # intentional. See the notes on the similar construction in download() for an explanation.
120 runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags' : fetch_flags}, d, workdir=tmpclonedir)
121 except bb.fetch.FetchError:
122 if allow_network:
123 raise
124 else:
125 # This method was called as a probe to see whether the submodule history
126 # is complete enough to allow the current working copy to have its
127 # modules filled in. It's not, so swallow up the exception and report
128 # the negative result.
129 return False
130 finally:
131 self._set_relative_paths(tmpclonedir)
132 runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir)
133 os.rename(gitdir, ud.clonedir,)
134 bb.utils.remove(tmpclonedir, True)
135 107
136 return True 108 return True
137 109
@@ -147,56 +119,117 @@ class GitSM(Git):
147 # Now check that the submodule histories are new enough. The git-submodule command doesn't have 119 # Now check that the submodule histories are new enough. The git-submodule command doesn't have
148 # any clean interface for doing this aside from just attempting the checkout (with network 120 # any clean interface for doing this aside from just attempting the checkout (with network
149 # fetched disabled). 121 # fetched disabled).
150 return not self.update_submodules(ud, d, allow_network=False) 122 return not self.update_submodules(ud, d)
151 123
152 def download(self, ud, d): 124 def download(self, ud, d):
153 Git.download(self, ud, d) 125 Git.download(self, ud, d)
154 126
155 if not ud.shallow or ud.localpath != ud.fullshallow: 127 if not ud.shallow or ud.localpath != ud.fullshallow:
156 submodules = self.uses_submodules(ud, d, ud.clonedir) 128 self.update_submodules(ud, d)
157 if submodules: 129
158 self.update_submodules(ud, d, allow_network=True) 130 def copy_submodules(self, submodules, ud, destdir, d):
131 if ud.bareclone:
132 repo_conf = destdir
133 else:
134 repo_conf = os.path.join(destdir, '.git')
135
136 if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')):
137 os.mkdir(os.path.join(repo_conf, 'modules'))
138
139 for module in submodules:
140 srcpath = os.path.join(ud.clonedir, 'modules', module)
141 modpath = os.path.join(repo_conf, 'modules', module)
142
143 if os.path.exists(srcpath):
144 if os.path.exists(os.path.join(srcpath, '.git')):
145 srcpath = os.path.join(srcpath, '.git')
146
147 target = modpath
148 if os.path.exists(modpath):
149 target = os.path.dirname(modpath)
150
151 runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d)
152 elif os.path.exists(modpath):
153 # Module already exists, likely unpacked from a shallow mirror clone
154 pass
155 else:
156 # This is fatal, as we do NOT want git-submodule to hit the network
157 raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath))
159 158
160 def clone_shallow_local(self, ud, dest, d): 159 def clone_shallow_local(self, ud, dest, d):
161 super(GitSM, self).clone_shallow_local(ud, dest, d) 160 super(GitSM, self).clone_shallow_local(ud, dest, d)
162 161
163 runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d) 162 # Copy over the submodules' fetched histories too.
163 repo_conf = os.path.join(dest, '.git')
164
165 submodules = []
166 for name in ud.names:
167 try:
168 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest)
169 except:
170 # No submodules to update
171 continue
172
173 for line in gitmodules.splitlines():
174 if line.startswith('[submodule'):
175 module = line.split('"')[1]
176 submodules.append(module)
177
178 self.copy_submodules(submodules, ud, dest, d)
164 179
165 def unpack(self, ud, destdir, d): 180 def unpack(self, ud, destdir, d):
166 Git.unpack(self, ud, destdir, d) 181 Git.unpack(self, ud, destdir, d)
167 182
168 if self.uses_submodules(ud, d, ud.destdir): 183 # Copy over the submodules' fetched histories too.
169 runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir) 184 if ud.bareclone:
185 repo_conf = ud.destdir
186 else:
187 repo_conf = os.path.join(ud.destdir, '.git')
170 188
171 # Copy over the submodules' fetched histories too. 189 submodules = []
172 if ud.bareclone: 190 paths = {}
173 repo_conf = ud.destdir 191 uris = {}
174 else: 192 local_paths = {}
175 repo_conf = os.path.join(ud.destdir, '.git') 193 for name in ud.names:
176 194 try:
177 if os.path.exists(ud.clonedir): 195 gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
178 # This is not a copy unpacked from a shallow mirror clone. So 196 except:
179 # the manual intervention to populate the .git/modules done 197 # No submodules to update
180 # in clone_shallow_local() won't have been done yet. 198 continue
181 runfetchcmd("cp -fpPRH %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d) 199
182 fetch_flags = "--no-fetch" 200 module = ""
183 elif os.path.exists(os.path.join(repo_conf, 'modules')): 201 for line in gitmodules.splitlines():
184 # Unpacked from a shallow mirror clone. Manual population of 202 if line.startswith('[submodule'):
185 # .git/modules is already done. 203 module = line.split('"')[1]
186 fetch_flags = "--no-fetch" 204 submodules.append(module)
187 else: 205 elif module and line.strip().startswith('path'):
188 # This isn't fatal; git-submodule will just fetch it 206 path = line.split('=')[1].strip()
189 # during do_unpack(). 207 paths[module] = path
190 fetch_flags = "" 208 elif module and line.strip().startswith('url'):
191 bb.error("submodule history not retrieved during do_fetch()") 209 url = line.split('=')[1].strip()
192 210 uris[module] = url
193 # Careful not to hit the network during unpacking; all history should already 211
194 # be fetched. 212 self.copy_submodules(submodules, ud, ud.destdir, d)
195 # 213
196 # The repeated attempts to do the submodule initialization sandwiched around a sync to 214 for module in submodules:
197 # install the correct remote URLs into the submodules' .git/config metadata are deliberate. 215 srcpath = os.path.join(ud.clonedir, 'modules', module)
198 # Bad remote URLs are leftover in the modules' .git/config files from the unpack of bare 216 modpath = os.path.join(repo_conf, 'modules', module)
199 # clone tarballs and an initial 'git submodule update' is necessary to prod them back to 217
200 # enough life so that the 'git submodule sync' realizes the existing module .git/config 218 # Determine (from the submodule) the correct url to reference
201 # files exist to be updated. 219 try:
202 runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags': fetch_flags}, d, workdir=ud.destdir) 220 output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath)
221 except bb.fetch2.FetchError as e:
222 # No remote url defined in this submodule
223 continue
224
225 local_paths[module] = output
226
227 # Setup the local URL properly (like git submodule init or sync would do...)
228 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir)
229
230 # Ensure the submodule repository is NOT set to bare, since we're checking it out...
231 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath)
232
233 if submodules:
234 # Run submodule update, this sets up the directories -- without touching the config
235 runfetchcmd("%s submodule update --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index e1c856f117..9c5601aa4d 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -1344,6 +1344,9 @@ class GitShallowTest(FetcherTest):
1344 smdir = os.path.join(self.tempdir, 'gitsubmodule') 1344 smdir = os.path.join(self.tempdir, 'gitsubmodule')
1345 bb.utils.mkdirhier(smdir) 1345 bb.utils.mkdirhier(smdir)
1346 self.git('init', cwd=smdir) 1346 self.git('init', cwd=smdir)
1347 # Make this look like it was cloned from a remote...
1348 self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
1349 self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
1347 self.add_empty_file('asub', cwd=smdir) 1350 self.add_empty_file('asub', cwd=smdir)
1348 1351
1349 self.git('submodule init', cwd=self.srcdir) 1352 self.git('submodule init', cwd=self.srcdir)