diff options
author | Tudor Florea <tudor.florea@enea.com> | 2014-10-16 03:05:19 +0200 |
---|---|---|
committer | Tudor Florea <tudor.florea@enea.com> | 2014-10-16 03:05:19 +0200 |
commit | c527fd1f14c27855a37f2e8ac5346ce8d940ced2 (patch) | |
tree | bb002c1fdf011c41dbd2f0927bed23ecb5f83c97 /bitbake/lib/bb/fetch2/git.py | |
download | poky-daisy-140929.tar.gz |
initial commit for Enea Linux 4.0-140929daisy-140929
Migrated from the internal git server on the daisy-enea-point-release branch
Signed-off-by: Tudor Florea <tudor.florea@enea.com>
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 355 |
1 files changed, 355 insertions, 0 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py new file mode 100644 index 0000000000..9ca24428a1 --- /dev/null +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -0,0 +1,355 @@ | |||
1 | # ex:ts=4:sw=4:sts=4:et | ||
2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
3 | """ | ||
4 | BitBake 'Fetch' git implementation | ||
5 | |||
6 | git fetcher support the SRC_URI with format of: | ||
7 | SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..." | ||
8 | |||
9 | Supported SRC_URI options are: | ||
10 | |||
11 | - branch | ||
12 | The git branch to retrieve from. The default is "master" | ||
13 | |||
14 | This option also supports multiple branch fetching, with branches | ||
15 | separated by commas. In multiple branches case, the name option | ||
16 | must have the same number of names to match the branches, which is | ||
17 | used to specify the SRC_REV for the branch | ||
18 | e.g: | ||
19 | SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" | ||
20 | SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" | ||
21 | SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" | ||
22 | |||
23 | - tag | ||
24 | The git tag to retrieve. The default is "master" | ||
25 | |||
26 | - protocol | ||
27 | The method to use to access the repository. Common options are "git", | ||
28 | "http", "https", "file", "ssh" and "rsync". The default is "git". | ||
29 | |||
30 | - rebaseable | ||
31 | rebaseable indicates that the upstream git repo may rebase in the future, | ||
32 | and current revision may disappear from upstream repo. This option will | ||
33 | remind fetcher to preserve local cache carefully for future use. | ||
34 | The default value is "0", set rebaseable=1 for rebaseable git repo. | ||
35 | |||
36 | - nocheckout | ||
37 | Don't checkout source code when unpacking. set this option for the recipe | ||
38 | who has its own routine to checkout code. | ||
39 | The default is "0", set nocheckout=1 if needed. | ||
40 | |||
41 | - bareclone | ||
42 | Create a bare clone of the source code and don't checkout the source code | ||
43 | when unpacking. Set this option for the recipe who has its own routine to | ||
44 | checkout code and tracking branch requirements. | ||
45 | The default is "0", set bareclone=1 if needed. | ||
46 | |||
47 | - nobranch | ||
48 | Don't check the SHA validation for branch. set this option for the recipe | ||
49 | referring to commit which is valid in tag instead of branch. | ||
50 | The default is "0", set nobranch=1 if needed. | ||
51 | |||
52 | """ | ||
53 | |||
54 | #Copyright (C) 2005 Richard Purdie | ||
55 | # | ||
56 | # This program is free software; you can redistribute it and/or modify | ||
57 | # it under the terms of the GNU General Public License version 2 as | ||
58 | # published by the Free Software Foundation. | ||
59 | # | ||
60 | # This program is distributed in the hope that it will be useful, | ||
61 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
62 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
63 | # GNU General Public License for more details. | ||
64 | # | ||
65 | # You should have received a copy of the GNU General Public License along | ||
66 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
67 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
68 | |||
69 | import os | ||
70 | import bb | ||
71 | from bb import data | ||
72 | from bb.fetch2 import FetchMethod | ||
73 | from bb.fetch2 import runfetchcmd | ||
74 | from bb.fetch2 import logger | ||
75 | |||
76 | class Git(FetchMethod): | ||
77 | """Class to fetch a module or modules from git repositories""" | ||
78 | def init(self, d): | ||
79 | pass | ||
80 | |||
81 | def supports(self, ud, d): | ||
82 | """ | ||
83 | Check to see if a given url can be fetched with git. | ||
84 | """ | ||
85 | return ud.type in ['git'] | ||
86 | |||
87 | def supports_checksum(self, urldata): | ||
88 | return False | ||
89 | |||
90 | def urldata_init(self, ud, d): | ||
91 | """ | ||
92 | init git specific variable within url data | ||
93 | so that the git method like latest_revision() can work | ||
94 | """ | ||
95 | if 'protocol' in ud.parm: | ||
96 | ud.proto = ud.parm['protocol'] | ||
97 | elif not ud.host: | ||
98 | ud.proto = 'file' | ||
99 | else: | ||
100 | ud.proto = "git" | ||
101 | |||
102 | if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): | ||
103 | raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) | ||
104 | |||
105 | ud.nocheckout = ud.parm.get("nocheckout","0") == "1" | ||
106 | |||
107 | ud.rebaseable = ud.parm.get("rebaseable","0") == "1" | ||
108 | |||
109 | ud.nobranch = ud.parm.get("nobranch","0") == "1" | ||
110 | |||
111 | # bareclone implies nocheckout | ||
112 | ud.bareclone = ud.parm.get("bareclone","0") == "1" | ||
113 | if ud.bareclone: | ||
114 | ud.nocheckout = 1 | ||
115 | |||
116 | ud.unresolvedrev = {} | ||
117 | branches = ud.parm.get("branch", "master").split(',') | ||
118 | if len(branches) != len(ud.names): | ||
119 | raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) | ||
120 | ud.branches = {} | ||
121 | for name in ud.names: | ||
122 | branch = branches[ud.names.index(name)] | ||
123 | ud.branches[name] = branch | ||
124 | ud.unresolvedrev[name] = branch | ||
125 | |||
126 | ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git" | ||
127 | |||
128 | ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable | ||
129 | |||
130 | ud.setup_revisons(d) | ||
131 | |||
132 | for name in ud.names: | ||
133 | # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one | ||
134 | if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): | ||
135 | if ud.revisions[name]: | ||
136 | ud.unresolvedrev[name] = ud.revisions[name] | ||
137 | ud.revisions[name] = self.latest_revision(ud, d, name) | ||
138 | |||
139 | gitsrcname = '%s%s' % (ud.host.replace(':','.'), ud.path.replace('/', '.').replace('*', '.')) | ||
140 | # for rebaseable git repo, it is necessary to keep mirror tar ball | ||
141 | # per revision, so that even the revision disappears from the | ||
142 | # upstream repo in the future, the mirror will remain intact and still | ||
143 | # contains the revision | ||
144 | if ud.rebaseable: | ||
145 | for name in ud.names: | ||
146 | gitsrcname = gitsrcname + '_' + ud.revisions[name] | ||
147 | ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) | ||
148 | ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) | ||
149 | gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") | ||
150 | ud.clonedir = os.path.join(gitdir, gitsrcname) | ||
151 | |||
152 | ud.localfile = ud.clonedir | ||
153 | |||
154 | def localpath(self, ud, d): | ||
155 | return ud.clonedir | ||
156 | |||
157 | def need_update(self, ud, d): | ||
158 | if not os.path.exists(ud.clonedir): | ||
159 | return True | ||
160 | os.chdir(ud.clonedir) | ||
161 | for name in ud.names: | ||
162 | if not self._contains_ref(ud, d, name): | ||
163 | return True | ||
164 | if ud.write_tarballs and not os.path.exists(ud.fullmirror): | ||
165 | return True | ||
166 | return False | ||
167 | |||
168 | def try_premirror(self, ud, d): | ||
169 | # If we don't do this, updating an existing checkout with only premirrors | ||
170 | # is not possible | ||
171 | if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None: | ||
172 | return True | ||
173 | if os.path.exists(ud.clonedir): | ||
174 | return False | ||
175 | return True | ||
176 | |||
177 | def download(self, ud, d): | ||
178 | """Fetch url""" | ||
179 | |||
180 | if ud.user: | ||
181 | username = ud.user + '@' | ||
182 | else: | ||
183 | username = "" | ||
184 | |||
185 | ud.repochanged = not os.path.exists(ud.fullmirror) | ||
186 | |||
187 | # If the checkout doesn't exist and the mirror tarball does, extract it | ||
188 | if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): | ||
189 | bb.utils.mkdirhier(ud.clonedir) | ||
190 | os.chdir(ud.clonedir) | ||
191 | runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) | ||
192 | |||
193 | repourl = "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) | ||
194 | |||
195 | # If the repo still doesn't exist, fallback to cloning it | ||
196 | if not os.path.exists(ud.clonedir): | ||
197 | # We do this since git will use a "-l" option automatically for local urls where possible | ||
198 | if repourl.startswith("file://"): | ||
199 | repourl = repourl[7:] | ||
200 | clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir) | ||
201 | if ud.proto.lower() != 'file': | ||
202 | bb.fetch2.check_network_access(d, clone_cmd) | ||
203 | runfetchcmd(clone_cmd, d) | ||
204 | |||
205 | os.chdir(ud.clonedir) | ||
206 | # Update the checkout if needed | ||
207 | needupdate = False | ||
208 | for name in ud.names: | ||
209 | if not self._contains_ref(ud, d, name): | ||
210 | needupdate = True | ||
211 | if needupdate: | ||
212 | try: | ||
213 | runfetchcmd("%s remote rm origin" % ud.basecmd, d) | ||
214 | except bb.fetch2.FetchError: | ||
215 | logger.debug(1, "No Origin") | ||
216 | |||
217 | runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d) | ||
218 | fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl) | ||
219 | if ud.proto.lower() != 'file': | ||
220 | bb.fetch2.check_network_access(d, fetch_cmd, ud.url) | ||
221 | runfetchcmd(fetch_cmd, d) | ||
222 | runfetchcmd("%s prune-packed" % ud.basecmd, d) | ||
223 | runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) | ||
224 | ud.repochanged = True | ||
225 | os.chdir(ud.clonedir) | ||
226 | for name in ud.names: | ||
227 | if not self._contains_ref(ud, d, name): | ||
228 | raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) | ||
229 | |||
230 | def build_mirror_data(self, ud, d): | ||
231 | # Generate a mirror tarball if needed | ||
232 | if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)): | ||
233 | # it's possible that this symlink points to read-only filesystem with PREMIRROR | ||
234 | if os.path.islink(ud.fullmirror): | ||
235 | os.unlink(ud.fullmirror) | ||
236 | |||
237 | os.chdir(ud.clonedir) | ||
238 | logger.info("Creating tarball of git repository") | ||
239 | runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d) | ||
240 | runfetchcmd("touch %s.done" % (ud.fullmirror), d) | ||
241 | |||
242 | def unpack(self, ud, destdir, d): | ||
243 | """ unpack the downloaded src to destdir""" | ||
244 | |||
245 | subdir = ud.parm.get("subpath", "") | ||
246 | if subdir != "": | ||
247 | readpathspec = ":%s" % (subdir) | ||
248 | def_destsuffix = "%s/" % os.path.basename(subdir) | ||
249 | else: | ||
250 | readpathspec = "" | ||
251 | def_destsuffix = "git/" | ||
252 | |||
253 | destsuffix = ud.parm.get("destsuffix", def_destsuffix) | ||
254 | destdir = ud.destdir = os.path.join(destdir, destsuffix) | ||
255 | if os.path.exists(destdir): | ||
256 | bb.utils.prunedir(destdir) | ||
257 | |||
258 | cloneflags = "-s -n" | ||
259 | if ud.bareclone: | ||
260 | cloneflags += " --mirror" | ||
261 | |||
262 | # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused | ||
263 | # and you end up with some horrible union of the two when you attempt to clone it | ||
264 | # The least invasive workaround seems to be a symlink to the real directory to | ||
265 | # fool git into ignoring any .git version that may also be present. | ||
266 | # | ||
267 | # The issue is fixed in more recent versions of git so we can drop this hack in future | ||
268 | # when that version becomes common enough. | ||
269 | clonedir = ud.clonedir | ||
270 | if not ud.path.endswith(".git"): | ||
271 | indirectiondir = destdir[:-1] + ".indirectionsymlink" | ||
272 | if os.path.exists(indirectiondir): | ||
273 | os.remove(indirectiondir) | ||
274 | bb.utils.mkdirhier(os.path.dirname(indirectiondir)) | ||
275 | os.symlink(ud.clonedir, indirectiondir) | ||
276 | clonedir = indirectiondir | ||
277 | |||
278 | runfetchcmd("git clone %s %s/ %s" % (cloneflags, clonedir, destdir), d) | ||
279 | if not ud.nocheckout: | ||
280 | os.chdir(destdir) | ||
281 | if subdir != "": | ||
282 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d) | ||
283 | runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d) | ||
284 | else: | ||
285 | runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d) | ||
286 | return True | ||
287 | |||
288 | def clean(self, ud, d): | ||
289 | """ clean the git directory """ | ||
290 | |||
291 | bb.utils.remove(ud.localpath, True) | ||
292 | bb.utils.remove(ud.fullmirror) | ||
293 | bb.utils.remove(ud.fullmirror + ".done") | ||
294 | |||
295 | def supports_srcrev(self): | ||
296 | return True | ||
297 | |||
298 | def _contains_ref(self, ud, d, name): | ||
299 | cmd = "" | ||
300 | if ud.nobranch: | ||
301 | cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( | ||
302 | ud.basecmd, ud.revisions[name]) | ||
303 | else: | ||
304 | cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( | ||
305 | ud.basecmd, ud.revisions[name], ud.branches[name]) | ||
306 | try: | ||
307 | output = runfetchcmd(cmd, d, quiet=True) | ||
308 | except bb.fetch2.FetchError: | ||
309 | return False | ||
310 | if len(output.split()) > 1: | ||
311 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) | ||
312 | return output.split()[0] != "0" | ||
313 | |||
314 | def _revision_key(self, ud, d, name): | ||
315 | """ | ||
316 | Return a unique key for the url | ||
317 | """ | ||
318 | return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name] | ||
319 | |||
320 | def _lsremote(self, ud, d, search): | ||
321 | """ | ||
322 | Run git ls-remote with the specified search string | ||
323 | """ | ||
324 | if ud.user: | ||
325 | username = ud.user + '@' | ||
326 | else: | ||
327 | username = "" | ||
328 | |||
329 | cmd = "%s ls-remote %s://%s%s%s %s" % \ | ||
330 | (ud.basecmd, ud.proto, username, ud.host, ud.path, search) | ||
331 | if ud.proto.lower() != 'file': | ||
332 | bb.fetch2.check_network_access(d, cmd) | ||
333 | output = runfetchcmd(cmd, d, True) | ||
334 | if not output: | ||
335 | raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url) | ||
336 | return output | ||
337 | |||
338 | def _latest_revision(self, ud, d, name): | ||
339 | """ | ||
340 | Compute the HEAD revision for the url | ||
341 | """ | ||
342 | search = "refs/heads/%s refs/tags/%s^{}" % (ud.unresolvedrev[name], ud.unresolvedrev[name]) | ||
343 | output = self._lsremote(ud, d, search) | ||
344 | return output.split()[0] | ||
345 | |||
346 | def _build_revision(self, ud, d, name): | ||
347 | return ud.revisions[name] | ||
348 | |||
349 | def checkstatus(self, ud, d): | ||
350 | fetchcmd = "%s ls-remote %s" % (ud.basecmd, ud.url) | ||
351 | try: | ||
352 | runfetchcmd(fetchcmd, d, quiet=True) | ||
353 | return True | ||
354 | except FetchError: | ||
355 | return False | ||