summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bitbake/lib/bb/fetch2/gitsm.py232
-rw-r--r--bitbake/lib/bb/tests/fetch.py12
2 files changed, 98 insertions, 146 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index 83571f834b..86198ee6cd 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -45,85 +45,96 @@ class GitSM(Git):
45 """ 45 """
46 return ud.type in ['gitsm'] 46 return ud.type in ['gitsm']
47 47
48 @staticmethod 48 def process_submodules(self, ud, workdir, function, d):
49 def parse_gitmodules(gitmodules): 49 """
50 modules = {} 50 Iterate over all of the submodules in this repository and execute
51 module = "" 51 the 'function' for each of them.
52 for line in gitmodules.splitlines(): 52 """
53 if line.startswith('[submodule'): 53
54 module = line.split('"')[1]
55 modules[module] = {}
56 elif module and line.strip().startswith('path'):
57 path = line.split('=')[1].strip()
58 modules[module]['path'] = path
59 elif module and line.strip().startswith('url'):
60 url = line.split('=')[1].strip()
61 modules[module]['url'] = url
62 return modules
63
64 def update_submodules(self, ud, d):
65 submodules = [] 54 submodules = []
66 paths = {} 55 paths = {}
67 revision = {} 56 revision = {}
68 uris = {} 57 uris = {}
69 local_paths = {} 58 subrevision = {}
70 59
60 def parse_gitmodules(gitmodules):
61 modules = {}
62 module = ""
63 for line in gitmodules.splitlines():
64 if line.startswith('[submodule'):
65 module = line.split('"')[1]
66 modules[module] = {}
67 elif module and line.strip().startswith('path'):
68 path = line.split('=')[1].strip()
69 modules[module]['path'] = path
70 elif module and line.strip().startswith('url'):
71 url = line.split('=')[1].strip()
72 modules[module]['url'] = url
73 return modules
74
75 # Collect the defined submodules, and their attributes
71 for name in ud.names: 76 for name in ud.names:
72 try: 77 try:
73 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir) 78 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
74 except: 79 except:
75 # No submodules to update 80 # No submodules to update
76 continue 81 continue
77 82
78 for m, md in self.parse_gitmodules(gitmodules).items(): 83 for m, md in parse_gitmodules(gitmodules).items():
84 try:
85 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
86 except:
87 # If the command fails, we don't have a valid file to check. If it doesn't
88 # fail -- it still might be a failure, see next check...
89 module_hash = ""
90
91 if not module_hash:
92 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", m)
93 continue
94
79 submodules.append(m) 95 submodules.append(m)
80 paths[m] = md['path'] 96 paths[m] = md['path']
81 revision[m] = ud.revisions[name] 97 revision[m] = ud.revisions[name]
82 uris[m] = md['url'] 98 uris[m] = md['url']
83 if uris[m].startswith('..'): 99 subrevision[m] = module_hash.split()[2]
84 newud = copy.copy(ud)
85 newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
86 uris[m] = Git._get_repo_url(self, newud)
87 100
88 for module in submodules: 101 for module in submodules:
89 try: 102 # Translate the module url into a SRC_URI
90 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.clonedir)
91 except:
92 # If the command fails, we don't have a valid file to check. If it doesn't
93 # fail -- it still might be a failure, see next check...
94 module_hash = ""
95 103
96 if not module_hash: 104 if "://" in uris[module]:
97 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module) 105 # Properly formated URL already
98 continue 106 proto = uris[module].split(':', 1)[0]
99 107 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
100 module_hash = module_hash.split()[2] 108 else:
101
102 # Build new SRC_URI
103 if "://" not in uris[module]:
104 # It's ssh if the format does NOT have "://", but has a ':'
105 if ":" in uris[module]: 109 if ":" in uris[module]:
110 # Most likely an SSH style reference
106 proto = "ssh" 111 proto = "ssh"
107 if ":/" in uris[module]: 112 if ":/" in uris[module]:
113 # Absolute reference, easy to convert..
108 url = "gitsm://" + uris[module].replace(':/', '/', 1) 114 url = "gitsm://" + uris[module].replace(':/', '/', 1)
109 else: 115 else:
116 # Relative reference, no way to know if this is right!
117 logger.warning("Submodule included by %s refers to relative ssh reference %s. References may fail if not absolute." % (ud.url, uris[module]))
110 url = "gitsm://" + uris[module].replace(':', '/', 1) 118 url = "gitsm://" + uris[module].replace(':', '/', 1)
111 else: # Fall back to 'file' if there is no ':' 119 else:
120 # This has to be a file reference
112 proto = "file" 121 proto = "file"
113 url = "gitsm://" + uris[module] 122 url = "gitsm://" + uris[module]
114 else: 123 if uris[module].startswith('..'):
115 proto = uris[module].split(':', 1)[0] 124 # Local on disk relative reference
116 url = uris[module].replace('%s:' % proto, 'gitsm:', 1) 125 newud = copy.copy(ud)
126 newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
127 url = "gitsm://" + Git._get_repo_url(self, newud)
117 128
118 url += ';protocol=%s' % proto 129 url += ';protocol=%s' % proto
119 url += ";name=%s" % module 130 url += ";name=%s" % module
120 url += ";bareclone=1;nocheckout=1;nobranch=1" 131 url += ";subpath=%s" % paths[module]
121 132
122 ld = d.createCopy() 133 ld = d.createCopy()
123 # Not necessary to set SRC_URI, since we're passing the URI to 134 # Not necessary to set SRC_URI, since we're passing the URI to
124 # Fetch. 135 # Fetch.
125 #ld.setVar('SRC_URI', url) 136 #ld.setVar('SRC_URI', url)
126 ld.setVar('SRCREV_%s' % module, module_hash) 137 ld.setVar('SRCREV_%s' % module, subrevision[module])
127 138
128 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 139 # Workaround for issues with SRCPV/SRCREV_FORMAT errors
129 # error refer to 'multiple' repositories. Only the repository 140 # error refer to 'multiple' repositories. Only the repository
@@ -131,125 +142,58 @@ class GitSM(Git):
131 ld.setVar('SRCPV', d.getVar('SRCPV')) 142 ld.setVar('SRCPV', d.getVar('SRCPV'))
132 ld.setVar('SRCREV_FORMAT', module) 143 ld.setVar('SRCREV_FORMAT', module)
133 144
134 newfetch = Fetch([url], ld, cache=False) 145 function(ud, url, module, paths[module], ld)
135 newfetch.download()
136 local_paths[module] = newfetch.localpath(url)
137 146
138 # Correct the submodule references to the local download version... 147 return submodules != []
139 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir)
140
141 symlink_path = os.path.join(ud.clonedir, 'modules', paths[module])
142 if not os.path.exists(symlink_path):
143 try:
144 os.makedirs(os.path.dirname(symlink_path), exist_ok=True)
145 except OSError:
146 pass
147 os.symlink(local_paths[module], symlink_path)
148
149 return True
150 148
151 def download(self, ud, d): 149 def download(self, ud, d):
152 Git.download(self, ud, d) 150 def download_submodule(ud, url, module, modpath, d):
153 self.update_submodules(ud, d) 151 url += ";bareclone=1;nobranch=1"
154
155 def unpack_submodules(self, repo_conf, ud, d):
156 submodules = []
157 paths = {}
158 revision = {}
159 uris = {}
160 local_paths = {}
161
162 for name in ud.names:
163 try:
164 gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.destdir)
165 except:
166 # No submodules to update
167 continue
168
169 for m, md in self.parse_gitmodules(gitmodules).items():
170 submodules.append(m)
171 paths[m] = md['path']
172 revision[m] = ud.revisions[name]
173 uris[m] = md['url']
174 if uris[m].startswith('..'):
175 newud = copy.copy(ud)
176 newud.path = os.path.realpath(os.path.join(newud.path, md['url']))
177 uris[m] = Git._get_repo_url(self, newud)
178 152
179 modules_updated = False 153 # Is the following still needed?
154 #url += ";nocheckout=1"
180 155
181 for module in submodules:
182 try: 156 try:
183 module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, revision[module], paths[module]), d, quiet=True, workdir=ud.destdir) 157 newfetch = Fetch([url], d, cache=False)
184 except: 158 newfetch.download()
185 # If the command fails, we don't have a valid file to check. If it doesn't 159 except Exception as e:
186 # fail -- it still might be a failure, see next check... 160 logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e)))
187 module_hash = "" 161 raise
188 162
189 if not module_hash: 163 Git.download(self, ud, d)
190 logger.debug(1, "submodule %s is defined, but is not initialized in the repository. Skipping", module) 164 self.process_submodules(ud, ud.clonedir, download_submodule, d)
191 continue
192
193 modules_updated = True
194 165
195 module_hash = module_hash.split()[2] 166 def unpack(self, ud, destdir, d):
167 def unpack_submodules(ud, url, module, modpath, d):
168 url += ";bareclone=1;nobranch=1"
196 169
197 # Build new SRC_URI 170 # Figure out where we clone over the bare submodules...
198 if "://" not in uris[module]: 171 if ud.bareclone:
199 # It's ssh if the format does NOT have "://", but has a ':' 172 repo_conf = ud.destdir
200 if ":" in uris[module]:
201 proto = "ssh"
202 if ":/" in uris[module]:
203 url = "gitsm://" + uris[module].replace(':/', '/', 1)
204 else:
205 url = "gitsm://" + uris[module].replace(':', '/', 1)
206 else: # Fall back to 'file' if there is no ':'
207 proto = "file"
208 url = "gitsm://" + uris[module]
209 else: 173 else:
210 proto = uris[module].split(':', 1)[0] 174 repo_conf = os.path.join(ud.destdir, '.git')
211 url = uris[module].replace('%s:' % proto, 'gitsm:', 1)
212
213 url += ';protocol=%s' % proto
214 url += ";name=%s" % module
215 url += ";bareclone=1;nobranch=1;subpath=%s" % paths[module]
216
217 ld = d.createCopy()
218 # Not necessary to set SRC_URI, since we're passing the URI to
219 # Fetch.
220 #ld.setVar('SRC_URI', url)
221 ld.setVar('SRCREV_%s' % module, module_hash)
222 175
223 # Workaround for issues with SRCPV/SRCREV_FORMAT errors 176 try:
224 # error refer to 'multiple' repositories. Only the repository 177 newfetch = Fetch([url], d, cache=False)
225 # in the original SRC_URI actually matters... 178 newfetch.unpack(root=os.path.join(repo_conf, 'modules'))
226 ld.setVar('SRCPV', d.getVar('SRCPV')) 179 except Exception as e:
227 ld.setVar('SRCREV_FORMAT', module) 180 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
181 raise
228 182
229 newfetch = Fetch([url], ld, cache=False) 183 newfetch = Fetch([url], d, cache=False)
230 newfetch.unpack(root=os.path.join(repo_conf, 'modules')) 184 local_path = newfetch.localpath(url)
231 local_paths[module] = newfetch.localpath(url)
232 185
233 # Correct the submodule references to the local download version... 186 # Correct the submodule references to the local download version...
234 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) 187 runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_path}, d, workdir=ud.destdir)
235 188
236 if ud.shallow: 189 if ud.shallow:
237 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir) 190 runfetchcmd("%(basecmd)s config submodule.%(module)s.shallow true" % {'basecmd': ud.basecmd, 'module': module}, d, workdir=ud.destdir)
238 191
239 # Ensure the submodule repository is NOT set to bare, since we're checking it out... 192 # Ensure the submodule repository is NOT set to bare, since we're checking it out...
240 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', paths[module])) 193 runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=os.path.join(repo_conf, 'modules', modpath))
241
242 return modules_updated
243 194
244 def unpack(self, ud, destdir, d):
245 Git.unpack(self, ud, destdir, d) 195 Git.unpack(self, ud, destdir, d)
246 196
247 # Copy over the submodules' fetched histories too. 197 if not ud.bareclone and self.process_submodules(ud, ud.destdir, unpack_submodules, d):
248 if ud.bareclone:
249 repo_conf = ud.destdir
250 else:
251 repo_conf = os.path.join(ud.destdir, '.git')
252
253 if self.unpack_submodules(repo_conf, ud, d):
254 # Run submodule update, this sets up the directories -- without touching the config 198 # Run submodule update, this sets up the directories -- without touching the config
255 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) 199 runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 5fb5d04cb0..1497a3cff7 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -894,15 +894,23 @@ class FetcherNetworkTest(FetcherTest):
894 @skipIfNoNetwork() 894 @skipIfNoNetwork()
895 def test_git_submodule(self): 895 def test_git_submodule(self):
896 # URL with ssh submodules 896 # URL with ssh submodules
897 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=0d3ffc14bce95e8b3a21a0a67bfe4c4a96ba6350" 897 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=f53765f515e0eeca569ed385bb1c89ce008bb058"
898 # Original URL (comment this if you have ssh access to git.yoctoproject.org) 898 # Original URL (comment this if you have ssh access to git.yoctoproject.org)
899 url = "gitsm://git.yoctoproject.org/git-submodule-test;rev=f12e57f2edf0aa534cf1616fa983d165a92b0842" 899 url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=132fea6e4dee56b61bcf5721c94e8b2445c6a017"
900 fetcher = bb.fetch.Fetch([url], self.d) 900 fetcher = bb.fetch.Fetch([url], self.d)
901 fetcher.download() 901 fetcher.download()
902 # Previous cwd has been deleted 902 # Previous cwd has been deleted
903 os.chdir(os.path.dirname(self.unpackdir)) 903 os.chdir(os.path.dirname(self.unpackdir))
904 fetcher.unpack(self.unpackdir) 904 fetcher.unpack(self.unpackdir)
905 905
906 repo_path = os.path.join(self.tempdir, 'unpacked', 'git')
907 self.assertTrue(os.path.exists(repo_path), msg='Unpacked repository missing')
908 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake')), msg='bitbake submodule missing')
909 self.assertFalse(os.path.exists(os.path.join(repo_path, 'na')), msg='uninitialized submodule present')
910
911 # Only when we're running the extended test with a submodule's submodule, can we check this.
912 if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')):
913 self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
906 914
907class TrustedNetworksTest(FetcherTest): 915class TrustedNetworksTest(FetcherTest):
908 def test_trusted_network(self): 916 def test_trusted_network(self):