summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2019-01-15 16:31:37 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-16 15:35:08 +0000
commit7d715ae071da9f1bb8e50c264dadefd0636b69d6 (patch)
treee26c17a600b8e8c3e043d43fba529c22b707fd4c /bitbake/lib
parenta9cf611e7a689571a5b4a70c0fe76de89476cc7d (diff)
downloadpoky-7d715ae071da9f1bb8e50c264dadefd0636b69d6.tar.gz
bitbake: gitsm.py: Refactor the functions and simplify the class
The update_submodules and unpack_submodules functions were nearly indentical, so we made a common function where the different behavior could be passed in by the download and unpack users. The new function is process_submodules. Moved the parse_gitmodules function under the new process_submodules, since there are no external callers. Refactor the file relative path processing to the URL translation code. We also add a warning to the translation if a relative ssh URL has been detected. Since this can cause a problem. In the case of a relative URL that does not work after being translated, it should be possible to use the MIRROR functions to manual translate the generated relative URL into one that works properly. Remove 'git config' processing on download contents. It turns out this is not necessary since all of the later components work using the git fetcher. Limit the 'git submodule update' call to only when unpacking a non-bare repository. Submodules are always loaded as bare, so this prevents intermediate unpacks from being attempted. Finally, the test cases were updated and the new commit ids in the test repository were updates as well. (Bitbake rev: 610dbee5634677f5055e2b36a3043cd197fb8c51) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py232
-rw-r--r--bitbake/lib/bb/tests/fetch.py12
2 files changed, 98 insertions, 146 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index 83571f834b..86198ee6cd 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -45,85 +45,96 @@ class GitSM(Git):
45 """ 45 """
46 return ud.type in ['gitsm'] 46 return ud.type in ['gitsm']
47 47
48 @staticmethod 48 def process_submodules(self, ud, workdir, function, d):
49 def parse_gitmodules(gitmodules): 49 """
50 modules = {} 50 Iterate over all of the submodules in this repository and execute
51 module = "" 51 the 'function' for each of them.
52 for line in gitmodules.splitlines(): 52 """
53 if line.startswith('[submodule'): 53
54 module = line.split('"')[1]
55 modules[module] = {}
56 elif module and line.strip().startswith('path'):
57 path = line.split('=')[1].strip()
58 modules[module]['path'] = path
59 elif module and line.strip().startswith('url'):
60 url = line.split('=')[1].strip()
61 modules[module]['url'] = url
62 return modules
63
64 def update_submodules(self, ud, d):
65 submodules = [] 54 submodules = []
66 paths = {} 55 paths = {}
67 revision = {} 56 revision = {}
68 uris = {} 57 uris = {}
69 local_paths = {} 58 subrevision = {}
70 59
60 def parse_gitmodules(gitmodules):
61 modules = {}
62 module = ""
63 for line in gitmodules.splitlines():
64 if line.startswith('[submodule'):
65 module = line.split('"')[1]
66 modules[module] = {}
67 elif module and line.strip().startswith('path'):
68 path = line.split('=')[1].strip()
69 modules[module]['path'] = path
70 elif module and line.strip().startswith('url'):
71 url = line.split('=')[1].strip()
72 modules[module]['url'] = url
73 return modules
74
75 # Collect the defined submodules, and their attributes
71 for name in ud.names: 76 for name in ud.names:
72 try: 77 try:
73 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir) 78 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
74 except: 79 except:
75 # No submodules to update 80 # No submodules to update
76 continue 81 continue
77 82
78 for m, md in self.parse_gitmodules(gitmodules).items(): 83 for m, md in parse_gitmodules(gitmodules).items():
84 try:
85 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
86 except:
87 # If the command fails, we don't have a valid file to check. If it doesn't
88 # fail -- it still might be a failure, see next check...
89 module_hash = ""
90
91 if not module_hash:
92 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
93 continue
94
79 submodules.append(m) 95 submodules.append(m)
80 paths[m] = md['path'] 96 paths[m] = md['path']
81 revision[m] = ud.revisions[name] 97 revision[m] = ud.revisions[name]
82 uris[m] = md['url'] 98 uris[m] = md['url']
83 if uris[m].startswith('..'): 99 subrevision[m] = module_hash.split()[2]
84 newud = copy.copy(ud)
85 newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
86 uris[m] = Git._get_repo_url(self, newud)
87 100
88 for module in submodules: 101 for module in submodules:
89 try: 102 # Translate the module url into a SRC_URI
90 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
91 except:
92 # If the command fails, we don't have a valid file to check. If it doesn't
93 # fail -- it still might be a failure, see next check...
94 module_hash = ""
95 103
96 if not module_hash: 104 if "://" in uris[module]:
97 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module) 105 # Properly formated URL already
98 continue 106 proto = uris[module].split(':', 1)[0]
99 107 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
100 module_hash = module_hash.split()[2] 108 else:
101
102 # Build new SRC_URI
103 if "://" not in uris[module]:
104 # It's ssh if the format does NOT have "://", but has a ':'
105 if ":" in uris[module]: 109 if ":" in uris[module]:
110 # Most likely an SSH style reference
106 proto = "ssh" 111 proto = "ssh"
107 if ":/" in uris[module]: 112 if ":/" in uris[module]:
113 # Absolute reference, easy to convert..
108 url = "gitsm://" + uris[module].replace(':/', '/', 1) 114 url = "gitsm://" + uris[module].replace(':/', '/', 1)
109 else: 115 else:
116 # Relative reference, no way to know if this is right!
117 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
110 url = "gitsm://" + uris[module].replace(':', '/', 1) 118 url = "gitsm://" + uris[module].replace(':', '/', 1)
111 else: # Fall back to 'file' if there is no ':' 119 else:
120 # This has to be a file reference
112 proto = "file" 121 proto = "file"
113 url = "gitsm://" + uris[module] 122 url = "gitsm://" + uris[module]
114 else: 123 if uris[module].startswith('..'):
115 proto = uris[module].split(':', 1)[0] 124 # Local on disk relative reference
116 url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 125 newud = copy.copy(ud)
126 newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
127 url = "gitsm://" + Git._get_repo_url(self, newud)
117 128
118 url += ';protocol=%s' % proto 129 url += ';protocol=%s' % proto
119 url += ";name=%s" % module 130 url += ";name=%s" % module
120 url += ";bareclone=1;nocheckout=1;nobranch=1" 131 url += ";subpath=%s" % paths[module]
121 132
122 ld = d.createCopy() 133 ld = d.createCopy()
123 # Not necessary to set SRC_URI, since we're passing the URI to 134 # Not necessary to set SRC_URI, since we're passing the URI to
124 # Fetch. 135 # Fetch.
125 #ld.setVar('SRC_URI', url) 136 #ld.setVar('SRC_URI', url)
126 ld.setVar('SRCREV_%s' % module, module_hash) 137 ld.setVar('SRCREV_%s' % module, subrevision[module])
127 138
128 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 139 # Workaround for issues with SRCPV/SRCREV_FORMAT errors
129 # error refer to 'multiple' repositories. Only the repository 140 # error refer to 'multiple' repositories. Only the repository
@@ -131,125 +142,58 @@ class GitSM(Git):
131 ld.setVar('SRCPV', d.getVar('SRCPV')) 142 ld.setVar('SRCPV', d.getVar('SRCPV'))
132 ld.setVar('SRCREV_FORMAT', module) 143 ld.setVar('SRCREV_FORMAT', module)
133 144
134 newfetch = Fetch([url], ld, cache=False) 145 function(ud, url, module, paths[module], ld)
135 newfetch.download()
136 local_paths[module] = newfetch.localpath(url)
137 146
138 # Correct the submodule references to the local download version... 147 return submodules != []
139 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
140
141 symlink_path = os.path.join(ud.clonedir, 'modules', paths[module])
142 if not os.path.exists(symlink_path):
143 try:
144 os.makedirs(os.path.dirname(symlink_path), exist_ok=True)
145 except OSError:
146 pass
147 os.symlink(local_paths[module], symlink_path)
148
149 return True
150 148
151 def download(self, ud, d): 149 def download(self, ud, d):
152 Git.download(self, ud, d) 150 def download_submodule(ud, url, module, modpath, d):
153 self.update_submodules(ud, d) 151 url += ";bareclone=1;nobranch=1"
154
155 def unpack_submodules(self, repo_conf, ud, d):
156 submodules = []
157 paths = {}
158 revision = {}
159 uris = {}
160 local_paths = {}
161
162 for name in ud.names:
163 try:
164 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.destdir)
165 except:
166 # No submodules to update
167 continue
168
169 for m, md in self.parse_gitmodules(gitmodules).items():
170 submodules.append(m)
171 paths[m] = md['path']
172 revision[m] = ud.revisions[name]
173 uris[m] = md['url']
174 if uris[m].startswith('..'):
175 newud = copy.copy(ud)
176 newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
177 uris[m] = Git._get_repo_url(self, newud)
178 152
179 modules_updated = False 153 # Is the following still needed?
154 #url += ";nocheckout=1"
180 155
181 for module in submodules:
182 try: 156 try:
183 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.destdir) 157 newfetch = Fetch([url], d, cache=False)
184 except: 158 newfetch.download()
185 # If the command fails, we don't have a valid file to check. If it doesn't 159 except Exception as e:
186 # fail -- it still might be a failure, see next check... 160 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
187 module_hash = "" 161 raise
188 162
189 if not module_hash: 163 Git.download(self, ud, d)
190 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module) 164 self.process_submodules(ud, ud.clonedir, download_submodule, d)
191 continue
192
193 modules_updated = True
194 165
195 module_hash = module_hash.split()[2] 166 def unpack(self, ud, destdir, d):
167 def unpack_submodules(ud, url, module, modpath, d):
168 url += ";bareclone=1;nobranch=1"
196 169
197 # Build new SRC_URI 170 # Figure out where we clone over the bare submodules...
198 if "://" not in uris[module]: 171 if ud.bareclone:
199 # It's ssh if the format does NOT have "://", but has a ':' 172 repo_conf = ud.destdir
200 if ":" in uris[module]:
201 proto = "ssh"
202 if ":/" in uris[module]:
203 url = "gitsm://" + uris[module].replace(':/', '/', 1)
204 else:
205 url = "gitsm://" + uris[module].replace(':', '/', 1)
206 else: # Fall back to 'file' if there is no ':'
207 proto = "file"
208 url = "gitsm://" + uris[module]
209 else: 173 else:
210 proto = uris[module].split(':', 1)[0] 174 repo_conf = os.path.join(ud.destdir, '.git')
211 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
212
213 url += ';protocol=%s' % proto
214 url += ";name=%s" % module
215 url += ";bareclone=1;nobranch=1;subpath=%s" % paths[module]
216
217 ld = d.createCopy()
218 # Not necessary to set SRC_URI, since we're passing the URI to
219 # Fetch.
220 #ld.setVar('SRC_URI', url)
221 ld.setVar('SRCREV_%s' % module, module_hash)
222 175
223 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 176 try:
224 # error refer to 'multiple' repositories. Only the repository 177 newfetch = Fetch([url], d, cache=False)
225 # in the original SRC_URI actually matters... 178 newfetch.unpack(root=os.path.join(repo_conf, 'modules'))
226 ld.setVar('SRCPV', d.getVar('SRCPV')) 179 except Exception as e:
227 ld.setVar('SRCREV_FORMAT', module) 180 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
181 raise
228 182
229 newfetch = Fetch([url], ld, cache=False) 183 newfetch = Fetch([url], d, cache=False)
230 newfetch.unpack(root=os.path.join(repo_conf, 'modules')) 184 local_path = newfetch.localpath(url)
231 local_paths[module] = newfetch.localpath(url)
232 185
233 # Correct the submodule references to the local download version... 186 # Correct the submodule references to the local download version...
234 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) 187 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
235 188
236 if ud.shallow: 189 if ud.shallow:
237 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 190 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
238 191
239 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 192 # Ensure the submodule repository is NOT set to bare, since we're checking it out...
240 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module])) 193 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
241
242 return modules_updated
243 194
244 def unpack(self, ud, destdir, d):
245 Git.unpack(self, ud, destdir, d) 195 Git.unpack(self, ud, destdir, d)
246 196
247 # Copy over the submodules' fetched histories too. 197 if not ud.bareclone and self.process_submodules(ud, ud.destdir, unpack_submodules, d):
248 if ud.bareclone:
249 repo_conf = ud.destdir
250 else:
251 repo_conf = os.path.join(ud.destdir, '.git')
252
253 if self.unpack_submodules(repo_conf, ud, d):
254 # Run submodule update, this sets up the directories -- without touching the config 198 # Run submodule update, this sets up the directories -- without touching the config
255 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 199 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 5fb5d04cb0..1497a3cff7 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -894,15 +894,23 @@ class FetcherNetworkTest(FetcherTest):
894 @skipIfNoNetwork() 894 @skipIfNoNetwork()
895 def test_git_submodule(self): 895 def test_git_submodule(self):
896 # URL with ssh submodules 896 # URL with ssh submodules
897 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=0d3ffc14bce95e8b3a21a0a67bfe4c4a96ba6350" 897 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=f53765f515e0eeca569ed385bb1c89ce008bb058"
898 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 898 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
899 url = "gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842" 899 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=132fea6e4dee56b61bcf5721c94e8b2445c6a017"
900 fetcher = bb.fetch.Fetch([url], self.d) 900 fetcher = bb.fetch.Fetch([url], self.d)
901 fetcher.download() 901 fetcher.download()
902 # Previous cwd has been deleted 902 # Previous cwd has been deleted
903 os.chdir(os.path.dirname(self.unpackdir)) 903 os.chdir(os.path.dirname(self.unpackdir))
904 fetcher.unpack(self.unpackdir) 904 fetcher.unpack(self.unpackdir)
905 905
906 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
907 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
908 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
909 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
910
911 # Only when we're running the extended test with a submodule's submodule, can we check this.
912 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
913 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
906 914
907class TrustedNetworksTest(FetcherTest): 915class TrustedNetworksTest(FetcherTest):
908 def test_trusted_network(self): 916 def test_trusted_network(self):