diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/gitsm.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/gitsm.py | 279 |
1 files changed, 156 insertions, 123 deletions
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py index 86773094dc..51f8d0e68c 100644 --- a/bitbake/lib/bb/fetch2/gitsm.py +++ b/bitbake/lib/bb/fetch2/gitsm.py | |||
@@ -34,6 +34,8 @@ import bb | |||
34 | from bb.fetch2.git import Git | 34 | from bb.fetch2.git import Git |
35 | from bb.fetch2 import runfetchcmd | 35 | from bb.fetch2 import runfetchcmd |
36 | from bb.fetch2 import logger | 36 | from bb.fetch2 import logger |
37 | from bb.fetch2 import Fetch | ||
38 | from bb.fetch2 import BBFetchException | ||
37 | 39 | ||
38 | class GitSM(Git): | 40 | class GitSM(Git): |
39 | def supports(self, ud, d): | 41 | def supports(self, ud, d): |
@@ -42,96 +44,66 @@ class GitSM(Git): | |||
42 | """ | 44 | """ |
43 | return ud.type in ['gitsm'] | 45 | return ud.type in ['gitsm'] |
44 | 46 | ||
45 | def uses_submodules(self, ud, d, wd): | 47 | def update_submodules(self, ud, d): |
48 | submodules = [] | ||
49 | paths = {} | ||
50 | uris = {} | ||
51 | local_paths = {} | ||
52 | |||
46 | for name in ud.names: | 53 | for name in ud.names: |
47 | try: | 54 | try: |
48 | runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=wd) | 55 | gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=ud.clonedir) |
49 | return True | 56 | except: |
50 | except bb.fetch.FetchError: | 57 | # No submodules to update |
51 | pass | 58 | continue |
52 | return False | ||
53 | 59 | ||
54 | def _set_relative_paths(self, repopath): | 60 | module = "" |
55 | """ | 61 | for line in gitmodules.splitlines(): |
56 | Fix submodule paths to be relative instead of absolute, | ||
57 | so that when we move the repo it doesn't break | ||
58 | (In Git 1.7.10+ this is done automatically) | ||
59 | """ | ||
60 | submodules = [] | ||
61 | with open(os.path.join(repopath, '.gitmodules'), 'r') as f: | ||
62 | for line in f.readlines(): | ||
63 | if line.startswith('[submodule'): | 62 | if line.startswith('[submodule'): |
64 | submodules.append(line.split('"')[1]) | 63 | module = line.split('"')[1] |
64 | submodules.append(module) | ||
65 | elif module and line.strip().startswith('path'): | ||
66 | path = line.split('=')[1].strip() | ||
67 | paths[module] = path | ||
68 | elif module and line.strip().startswith('url'): | ||
69 | url = line.split('=')[1].strip() | ||
70 | uris[module] = url | ||
65 | 71 | ||
66 | for module in submodules: | 72 | for module in submodules: |
67 | repo_conf = os.path.join(repopath, module, '.git') | 73 | module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], paths[module]), d, quiet=True, workdir=ud.clonedir) |
68 | if os.path.exists(repo_conf): | 74 | module_hash = module_hash.split()[2] |
69 | with open(repo_conf, 'r') as f: | 75 | |
70 | lines = f.readlines() | 76 | # Build new SRC_URI |
71 | newpath = '' | 77 | proto = uris[module].split(':', 1)[0] |
72 | for i, line in enumerate(lines): | 78 | url = uris[module].replace('%s:' % proto, 'gitsm:', 1) |
73 | if line.startswith('gitdir:'): | 79 | url += ';protocol=%s' % proto |
74 | oldpath = line.split(': ')[-1].rstrip() | 80 | url += ";name=%s" % module |
75 | if oldpath.startswith('/'): | 81 | url += ";qbareclone=1;nocheckout=1" |
76 | newpath = '../' * (module.count('/') + 1) + '.git/modules/' + module | 82 | |
77 | lines[i] = 'gitdir: %s\n' % newpath | 83 | ld = d.createCopy() |
78 | break | 84 | # Not necessary to set SRC_URI, since we're passing the URI to |
79 | if newpath: | 85 | # Fetch. |
80 | with open(repo_conf, 'w') as f: | 86 | #ld.setVar('SRC_URI', url) |
81 | for line in lines: | 87 | ld.setVar('SRCREV_%s' % module, module_hash) |
82 | f.write(line) | 88 | |
83 | 89 | # Workaround for issues with SRCPV/SRCREV_FORMAT errors | |
84 | repo_conf2 = os.path.join(repopath, '.git', 'modules', module, 'config') | 90 | # error refer to 'multiple' repositories. Only the repository |
85 | if os.path.exists(repo_conf2): | 91 | # in the original SRC_URI actually matters... |
86 | with open(repo_conf2, 'r') as f: | 92 | ld.setVar('SRCPV', d.getVar('SRCPV')) |
87 | lines = f.readlines() | 93 | ld.setVar('SRCREV_FORMAT', module) |
88 | newpath = '' | 94 | |
89 | for i, line in enumerate(lines): | 95 | newfetch = Fetch([url], ld, cache=False) |
90 | if line.lstrip().startswith('worktree = '): | 96 | newfetch.download() |
91 | oldpath = line.split(' = ')[-1].rstrip() | 97 | local_paths[module] = newfetch.localpath(url) |
92 | if oldpath.startswith('/'): | 98 | |
93 | newpath = '../' * (module.count('/') + 3) + module | 99 | # Correct the submodule references to the local download version... |
94 | lines[i] = '\tworktree = %s\n' % newpath | 100 | runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.clonedir) |
95 | break | 101 | try: |
96 | if newpath: | 102 | os.mkdir(os.path.join(ud.clonedir, 'modules')) |
97 | with open(repo_conf2, 'w') as f: | 103 | except OSError: |
98 | for line in lines: | 104 | pass |
99 | f.write(line) | 105 | if not os.path.exists(os.path.join(ud.clonedir, 'modules', paths[module])): |
100 | 106 | os.symlink(local_paths[module], os.path.join(ud.clonedir, 'modules', paths[module])) | |
101 | def update_submodules(self, ud, d, allow_network): | ||
102 | # We have to convert bare -> full repo, do the submodule bit, then convert back | ||
103 | tmpclonedir = ud.clonedir + ".tmp" | ||
104 | gitdir = tmpclonedir + os.sep + ".git" | ||
105 | bb.utils.remove(tmpclonedir, True) | ||
106 | os.mkdir(tmpclonedir) | ||
107 | os.rename(ud.clonedir, gitdir) | ||
108 | runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*true/bare = false/'", d) | ||
109 | runfetchcmd(ud.basecmd + " reset --hard", d, workdir=tmpclonedir) | ||
110 | runfetchcmd(ud.basecmd + " checkout -f " + ud.revisions[ud.names[0]], d, workdir=tmpclonedir) | ||
111 | |||
112 | try: | ||
113 | if allow_network: | ||
114 | fetch_flags = "" | ||
115 | else: | ||
116 | fetch_flags = "--no-fetch" | ||
117 | |||
118 | # The 'git submodule sync' sandwiched between two successive 'git submodule update' commands is | ||
119 | # intentional. See the notes on the similar construction in download() for an explanation. | ||
120 | runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags' : fetch_flags}, d, workdir=tmpclonedir) | ||
121 | except bb.fetch.FetchError: | ||
122 | if allow_network: | ||
123 | raise | ||
124 | else: | ||
125 | # This method was called as a probe to see whether the submodule history | ||
126 | # is complete enough to allow the current working copy to have its | ||
127 | # modules filled in. It's not, so swallow up the exception and report | ||
128 | # the negative result. | ||
129 | return False | ||
130 | finally: | ||
131 | self._set_relative_paths(tmpclonedir) | ||
132 | runfetchcmd("sed " + gitdir + "/config -i -e 's/bare.*=.*false/bare = true/'", d, workdir=tmpclonedir) | ||
133 | os.rename(gitdir, ud.clonedir,) | ||
134 | bb.utils.remove(tmpclonedir, True) | ||
135 | 107 | ||
136 | return True | 108 | return True |
137 | 109 | ||
@@ -147,56 +119,117 @@ class GitSM(Git): | |||
147 | # Now check that the submodule histories are new enough. The git-submodule command doesn't have | 119 | # Now check that the submodule histories are new enough. The git-submodule command doesn't have |
148 | # any clean interface for doing this aside from just attempting the checkout (with network | 120 | # any clean interface for doing this aside from just attempting the checkout (with network |
149 | # fetched disabled). | 121 | # fetched disabled). |
150 | return not self.update_submodules(ud, d, allow_network=False) | 122 | return not self.update_submodules(ud, d) |
151 | 123 | ||
152 | def download(self, ud, d): | 124 | def download(self, ud, d): |
153 | Git.download(self, ud, d) | 125 | Git.download(self, ud, d) |
154 | 126 | ||
155 | if not ud.shallow or ud.localpath != ud.fullshallow: | 127 | if not ud.shallow or ud.localpath != ud.fullshallow: |
156 | submodules = self.uses_submodules(ud, d, ud.clonedir) | 128 | self.update_submodules(ud, d) |
157 | if submodules: | 129 | |
158 | self.update_submodules(ud, d, allow_network=True) | 130 | def copy_submodules(self, submodules, ud, destdir, d): |
131 | if ud.bareclone: | ||
132 | repo_conf = destdir | ||
133 | else: | ||
134 | repo_conf = os.path.join(destdir, '.git') | ||
135 | |||
136 | if submodules and not os.path.exists(os.path.join(repo_conf, 'modules')): | ||
137 | os.mkdir(os.path.join(repo_conf, 'modules')) | ||
138 | |||
139 | for module in submodules: | ||
140 | srcpath = os.path.join(ud.clonedir, 'modules', module) | ||
141 | modpath = os.path.join(repo_conf, 'modules', module) | ||
142 | |||
143 | if os.path.exists(srcpath): | ||
144 | if os.path.exists(os.path.join(srcpath, '.git')): | ||
145 | srcpath = os.path.join(srcpath, '.git') | ||
146 | |||
147 | target = modpath | ||
148 | if os.path.exists(modpath): | ||
149 | target = os.path.dirname(modpath) | ||
150 | |||
151 | runfetchcmd("cp -fpLR %s %s" % (srcpath, target), d) | ||
152 | elif os.path.exists(modpath): | ||
153 | # Module already exists, likely unpacked from a shallow mirror clone | ||
154 | pass | ||
155 | else: | ||
156 | # This is fatal, as we do NOT want git-submodule to hit the network | ||
157 | raise bb.fetch2.FetchError('Submodule %s does not exist in %s or %s.' % (module, srcpath, modpath)) | ||
159 | 158 | ||
160 | def clone_shallow_local(self, ud, dest, d): | 159 | def clone_shallow_local(self, ud, dest, d): |
161 | super(GitSM, self).clone_shallow_local(ud, dest, d) | 160 | super(GitSM, self).clone_shallow_local(ud, dest, d) |
162 | 161 | ||
163 | runfetchcmd('cp -fpPRH "%s/modules" "%s/"' % (ud.clonedir, os.path.join(dest, '.git')), d) | 162 | # Copy over the submodules' fetched histories too. |
163 | repo_conf = os.path.join(dest, '.git') | ||
164 | |||
165 | submodules = [] | ||
166 | for name in ud.names: | ||
167 | try: | ||
168 | gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=dest) | ||
169 | except: | ||
170 | # No submodules to update | ||
171 | continue | ||
172 | |||
173 | for line in gitmodules.splitlines(): | ||
174 | if line.startswith('[submodule'): | ||
175 | module = line.split('"')[1] | ||
176 | submodules.append(module) | ||
177 | |||
178 | self.copy_submodules(submodules, ud, dest, d) | ||
164 | 179 | ||
165 | def unpack(self, ud, destdir, d): | 180 | def unpack(self, ud, destdir, d): |
166 | Git.unpack(self, ud, destdir, d) | 181 | Git.unpack(self, ud, destdir, d) |
167 | 182 | ||
168 | if self.uses_submodules(ud, d, ud.destdir): | 183 | # Copy over the submodules' fetched histories too. |
169 | runfetchcmd(ud.basecmd + " checkout " + ud.revisions[ud.names[0]], d, workdir=ud.destdir) | 184 | if ud.bareclone: |
185 | repo_conf = ud.destdir | ||
186 | else: | ||
187 | repo_conf = os.path.join(ud.destdir, '.git') | ||
170 | 188 | ||
171 | # Copy over the submodules' fetched histories too. | 189 | submodules = [] |
172 | if ud.bareclone: | 190 | paths = {} |
173 | repo_conf = ud.destdir | 191 | uris = {} |
174 | else: | 192 | local_paths = {} |
175 | repo_conf = os.path.join(ud.destdir, '.git') | 193 | for name in ud.names: |
176 | 194 | try: | |
177 | if os.path.exists(ud.clonedir): | 195 | gitmodules = runfetchcmd("%s show HEAD:.gitmodules" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) |
178 | # This is not a copy unpacked from a shallow mirror clone. So | 196 | except: |
179 | # the manual intervention to populate the .git/modules done | 197 | # No submodules to update |
180 | # in clone_shallow_local() won't have been done yet. | 198 | continue |
181 | runfetchcmd("cp -fpPRH %s %s" % (os.path.join(ud.clonedir, 'modules'), repo_conf), d) | 199 | |
182 | fetch_flags = "--no-fetch" | 200 | module = "" |
183 | elif os.path.exists(os.path.join(repo_conf, 'modules')): | 201 | for line in gitmodules.splitlines(): |
184 | # Unpacked from a shallow mirror clone. Manual population of | 202 | if line.startswith('[submodule'): |
185 | # .git/modules is already done. | 203 | module = line.split('"')[1] |
186 | fetch_flags = "--no-fetch" | 204 | submodules.append(module) |
187 | else: | 205 | elif module and line.strip().startswith('path'): |
188 | # This isn't fatal; git-submodule will just fetch it | 206 | path = line.split('=')[1].strip() |
189 | # during do_unpack(). | 207 | paths[module] = path |
190 | fetch_flags = "" | 208 | elif module and line.strip().startswith('url'): |
191 | bb.error("submodule history not retrieved during do_fetch()") | 209 | url = line.split('=')[1].strip() |
192 | 210 | uris[module] = url | |
193 | # Careful not to hit the network during unpacking; all history should already | 211 | |
194 | # be fetched. | 212 | self.copy_submodules(submodules, ud, ud.destdir, d) |
195 | # | 213 | |
196 | # The repeated attempts to do the submodule initialization sandwiched around a sync to | 214 | for module in submodules: |
197 | # install the correct remote URLs into the submodules' .git/config metadata are deliberate. | 215 | srcpath = os.path.join(ud.clonedir, 'modules', module) |
198 | # Bad remote URLs are leftover in the modules' .git/config files from the unpack of bare | 216 | modpath = os.path.join(repo_conf, 'modules', module) |
199 | # clone tarballs and an initial 'git submodule update' is necessary to prod them back to | 217 | |
200 | # enough life so that the 'git submodule sync' realizes the existing module .git/config | 218 | # Determine (from the submodule) the correct url to reference |
201 | # files exist to be updated. | 219 | try: |
202 | runfetchcmd("%(basecmd)s submodule update --init --recursive %(fetch_flags)s || (%(basecmd)s submodule sync --recursive && %(basecmd)s submodule update --init --recursive %(fetch_flags)s)" % {'basecmd': ud.basecmd, 'fetch_flags': fetch_flags}, d, workdir=ud.destdir) | 220 | output = runfetchcmd("%(basecmd)s config remote.origin.url" % {'basecmd': ud.basecmd}, d, workdir=modpath) |
221 | except bb.fetch2.FetchError as e: | ||
222 | # No remote url defined in this submodule | ||
223 | continue | ||
224 | |||
225 | local_paths[module] = output | ||
226 | |||
227 | # Setup the local URL properly (like git submodule init or sync would do...) | ||
228 | runfetchcmd("%(basecmd)s config submodule.%(module)s.url %(url)s" % {'basecmd': ud.basecmd, 'module': module, 'url' : local_paths[module]}, d, workdir=ud.destdir) | ||
229 | |||
230 | # Ensure the submodule repository is NOT set to bare, since we're checking it out... | ||
231 | runfetchcmd("%s config core.bare false" % (ud.basecmd), d, quiet=True, workdir=modpath) | ||
232 | |||
233 | if submodules: | ||
234 | # Run submodule update, this sets up the directories -- without touching the config | ||
235 | runfetchcmd("%s submodule update --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) | ||