diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2012-03-03 11:25:19 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2012-03-05 10:38:01 -0800 |
commit | 5419dc1bb787d338f9aec238b627ae9eea2f948d (patch) | |
tree | f4d104e4edfdff41bef735b578f260ee2c9124fd /bitbake/lib/bb/fetch/git.py | |
parent | ff73b02a7203623e5edfaaa10df17eaa63cf4ed3 (diff) | |
download | poky-5419dc1bb787d338f9aec238b627ae9eea2f948d.tar.gz |
bitbake: Drop fetch v1, v2 provides a much better codebase to build from
(Bitbake rev: 292e3430e5140b602cad86f55b5453e8cebb28a1)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch/git.py')
-rw-r--r-- | bitbake/lib/bb/fetch/git.py | 339 |
1 files changed, 0 insertions, 339 deletions
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py deleted file mode 100644 index 7160919d5a..0000000000 --- a/bitbake/lib/bb/fetch/git.py +++ /dev/null | |||
@@ -1,339 +0,0 @@ | |||
1 | # ex:ts=4:sw=4:sts=4:et | ||
2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
3 | """ | ||
4 | BitBake 'Fetch' git implementation | ||
5 | |||
6 | """ | ||
7 | |||
8 | #Copyright (C) 2005 Richard Purdie | ||
9 | # | ||
10 | # This program is free software; you can redistribute it and/or modify | ||
11 | # it under the terms of the GNU General Public License version 2 as | ||
12 | # published by the Free Software Foundation. | ||
13 | # | ||
14 | # This program is distributed in the hope that it will be useful, | ||
15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
17 | # GNU General Public License for more details. | ||
18 | # | ||
19 | # You should have received a copy of the GNU General Public License along | ||
20 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
21 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
22 | |||
23 | import os | ||
24 | import bb | ||
25 | import bb.persist_data | ||
26 | from bb import data | ||
27 | from bb.fetch import Fetch | ||
28 | from bb.fetch import runfetchcmd | ||
29 | from bb.fetch import logger | ||
30 | |||
31 | class Git(Fetch): | ||
32 | """Class to fetch a module or modules from git repositories""" | ||
33 | def init(self, d): | ||
34 | # | ||
35 | # Only enable _sortable revision if the key is set | ||
36 | # | ||
37 | if d.getVar("BB_GIT_CLONE_FOR_SRCREV", True): | ||
38 | self._sortable_buildindex = self._sortable_buildindex_disabled | ||
39 | def supports(self, url, ud, d): | ||
40 | """ | ||
41 | Check to see if a given url can be fetched with git. | ||
42 | """ | ||
43 | return ud.type in ['git'] | ||
44 | |||
45 | def localpath(self, url, ud, d): | ||
46 | |||
47 | if 'protocol' in ud.parm: | ||
48 | ud.proto = ud.parm['protocol'] | ||
49 | elif not ud.host: | ||
50 | ud.proto = 'file' | ||
51 | else: | ||
52 | ud.proto = "rsync" | ||
53 | |||
54 | ud.branch = ud.parm.get("branch", "master") | ||
55 | |||
56 | gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) | ||
57 | ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname) | ||
58 | ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) | ||
59 | |||
60 | tag = Fetch.srcrev_internal_helper(ud, d) | ||
61 | if tag is True: | ||
62 | ud.tag = self.latest_revision(url, ud, d) | ||
63 | elif tag: | ||
64 | ud.tag = tag | ||
65 | |||
66 | if not ud.tag or ud.tag == "master": | ||
67 | ud.tag = self.latest_revision(url, ud, d) | ||
68 | |||
69 | subdir = ud.parm.get("subpath", "") | ||
70 | if subdir != "": | ||
71 | if subdir.endswith("/"): | ||
72 | subdir = subdir[:-1] | ||
73 | subdirpath = os.path.join(ud.path, subdir); | ||
74 | else: | ||
75 | subdirpath = ud.path; | ||
76 | |||
77 | if 'fullclone' in ud.parm: | ||
78 | ud.localfile = ud.mirrortarball | ||
79 | else: | ||
80 | ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d) | ||
81 | |||
82 | ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git" | ||
83 | |||
84 | if 'noclone' in ud.parm: | ||
85 | ud.localfile = None | ||
86 | return None | ||
87 | |||
88 | return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile) | ||
89 | |||
90 | def forcefetch(self, url, ud, d): | ||
91 | if 'fullclone' in ud.parm: | ||
92 | return True | ||
93 | if 'noclone' in ud.parm: | ||
94 | return False | ||
95 | if os.path.exists(ud.localpath): | ||
96 | return False | ||
97 | if not self._contains_ref(ud.tag, d): | ||
98 | return True | ||
99 | return False | ||
100 | |||
101 | def try_premirror(self, u, ud, d): | ||
102 | if 'noclone' in ud.parm: | ||
103 | return False | ||
104 | if os.path.exists(ud.clonedir): | ||
105 | return False | ||
106 | if os.path.exists(ud.localpath): | ||
107 | return False | ||
108 | |||
109 | return True | ||
110 | |||
111 | def go(self, loc, ud, d): | ||
112 | """Fetch url""" | ||
113 | |||
114 | if ud.user: | ||
115 | username = ud.user + '@' | ||
116 | else: | ||
117 | username = "" | ||
118 | |||
119 | repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball) | ||
120 | |||
121 | |||
122 | coname = '%s' % (ud.tag) | ||
123 | codir = os.path.join(ud.clonedir, coname) | ||
124 | |||
125 | # If we have no existing clone and no mirror tarball, try and obtain one | ||
126 | if not os.path.exists(ud.clonedir) and not os.path.exists(repofile): | ||
127 | try: | ||
128 | Fetch.try_mirrors(ud.mirrortarball) | ||
129 | except: | ||
130 | pass | ||
131 | |||
132 | # If the checkout doesn't exist and the mirror tarball does, extract it | ||
133 | if not os.path.exists(ud.clonedir) and os.path.exists(repofile): | ||
134 | bb.utils.mkdirhier(ud.clonedir) | ||
135 | os.chdir(ud.clonedir) | ||
136 | runfetchcmd("tar -xzf %s" % (repofile), d) | ||
137 | |||
138 | # If the repo still doesn't exist, fallback to cloning it | ||
139 | if not os.path.exists(ud.clonedir): | ||
140 | runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d) | ||
141 | |||
142 | os.chdir(ud.clonedir) | ||
143 | # Update the checkout if needed | ||
144 | if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm: | ||
145 | # Remove all but the .git directory | ||
146 | runfetchcmd("rm * -Rf", d) | ||
147 | if 'fullclone' in ud.parm: | ||
148 | runfetchcmd("%s fetch --all" % (ud.basecmd), d) | ||
149 | else: | ||
150 | runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d) | ||
151 | runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) | ||
152 | runfetchcmd("%s prune-packed" % ud.basecmd, d) | ||
153 | runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) | ||
154 | |||
155 | # Generate a mirror tarball if needed | ||
156 | os.chdir(ud.clonedir) | ||
157 | mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) | ||
158 | if mirror_tarballs != "0" or 'fullclone' in ud.parm: | ||
159 | logger.info("Creating tarball of git repository") | ||
160 | runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) | ||
161 | |||
162 | if 'fullclone' in ud.parm: | ||
163 | return | ||
164 | |||
165 | if os.path.exists(codir): | ||
166 | bb.utils.prunedir(codir) | ||
167 | |||
168 | subdir = ud.parm.get("subpath", "") | ||
169 | if subdir != "": | ||
170 | if subdir.endswith("/"): | ||
171 | subdirbase = os.path.basename(subdir[:-1]) | ||
172 | else: | ||
173 | subdirbase = os.path.basename(subdir) | ||
174 | else: | ||
175 | subdirbase = "" | ||
176 | |||
177 | if subdir != "": | ||
178 | readpathspec = ":%s" % (subdir) | ||
179 | codir = os.path.join(codir, "git") | ||
180 | coprefix = os.path.join(codir, subdirbase, "") | ||
181 | else: | ||
182 | readpathspec = "" | ||
183 | coprefix = os.path.join(codir, "git", "") | ||
184 | |||
185 | scmdata = ud.parm.get("scmdata", "") | ||
186 | if scmdata == "keep": | ||
187 | runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d) | ||
188 | os.chdir(coprefix) | ||
189 | runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d) | ||
190 | else: | ||
191 | bb.utils.mkdirhier(codir) | ||
192 | os.chdir(ud.clonedir) | ||
193 | runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d) | ||
194 | runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d) | ||
195 | |||
196 | os.chdir(codir) | ||
197 | logger.info("Creating tarball of git checkout") | ||
198 | runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) | ||
199 | |||
200 | os.chdir(ud.clonedir) | ||
201 | bb.utils.prunedir(codir) | ||
202 | |||
203 | def supports_srcrev(self): | ||
204 | return True | ||
205 | |||
206 | def _contains_ref(self, tag, d): | ||
207 | basecmd = data.getVar("FETCHCMD_git", d, True) or "git" | ||
208 | output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True) | ||
209 | return output.split()[0] != "0" | ||
210 | |||
211 | def _revision_key(self, url, ud, d, branch=False): | ||
212 | """ | ||
213 | Return a unique key for the url | ||
214 | """ | ||
215 | key = 'git:' + ud.host + ud.path.replace('/', '.') | ||
216 | if branch: | ||
217 | return key + ud.branch | ||
218 | else: | ||
219 | return key | ||
220 | |||
221 | def generate_revision_key(self, url, ud, d, branch=False): | ||
222 | key = self._revision_key(url, ud, d, branch) | ||
223 | return "%s-%s" % (key, d.getVar("PN", True) or "") | ||
224 | |||
225 | def _latest_revision(self, url, ud, d): | ||
226 | """ | ||
227 | Compute the HEAD revision for the url | ||
228 | """ | ||
229 | if ud.user: | ||
230 | username = ud.user + '@' | ||
231 | else: | ||
232 | username = "" | ||
233 | |||
234 | basecmd = data.getVar("FETCHCMD_git", d, True) or "git" | ||
235 | cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch) | ||
236 | output = runfetchcmd(cmd, d, True) | ||
237 | if not output: | ||
238 | raise bb.fetch.FetchError("Fetch command %s gave empty output\n" % (cmd)) | ||
239 | return output.split()[0] | ||
240 | |||
241 | def latest_revision(self, url, ud, d): | ||
242 | """ | ||
243 | Look in the cache for the latest revision, if not present ask the SCM. | ||
244 | """ | ||
245 | revs = bb.persist_data.persist('BB_URI_HEADREVS', d) | ||
246 | |||
247 | key = self.generate_revision_key(url, ud, d, branch=True) | ||
248 | |||
249 | try: | ||
250 | return revs[key] | ||
251 | except KeyError: | ||
252 | # Compatibility with old key format, no branch included | ||
253 | oldkey = self.generate_revision_key(url, ud, d, branch=False) | ||
254 | try: | ||
255 | rev = revs[oldkey] | ||
256 | except KeyError: | ||
257 | rev = self._latest_revision(url, ud, d) | ||
258 | else: | ||
259 | del revs[oldkey] | ||
260 | revs[key] = rev | ||
261 | return rev | ||
262 | |||
263 | def sortable_revision(self, url, ud, d): | ||
264 | """ | ||
265 | |||
266 | """ | ||
267 | localcounts = bb.persist_data.persist('BB_URI_LOCALCOUNT', d) | ||
268 | key = self.generate_revision_key(url, ud, d, branch=True) | ||
269 | oldkey = self.generate_revision_key(url, ud, d, branch=False) | ||
270 | |||
271 | latest_rev = self._build_revision(url, ud, d) | ||
272 | last_rev = localcounts.get(key + '_rev') | ||
273 | if last_rev is None: | ||
274 | last_rev = localcounts.get(oldkey + '_rev') | ||
275 | if last_rev is not None: | ||
276 | del localcounts[oldkey + '_rev'] | ||
277 | localcounts[key + '_rev'] = last_rev | ||
278 | |||
279 | uselocalcount = d.getVar("BB_LOCALCOUNT_OVERRIDE", True) or False | ||
280 | count = None | ||
281 | if uselocalcount: | ||
282 | count = Fetch.localcount_internal_helper(ud, d) | ||
283 | if count is None: | ||
284 | count = localcounts.get(key + '_count') | ||
285 | if count is None: | ||
286 | count = localcounts.get(oldkey + '_count') | ||
287 | if count is not None: | ||
288 | del localcounts[oldkey + '_count'] | ||
289 | localcounts[key + '_count'] = count | ||
290 | |||
291 | if last_rev == latest_rev: | ||
292 | return str(count + "+" + latest_rev) | ||
293 | |||
294 | buildindex_provided = hasattr(self, "_sortable_buildindex") | ||
295 | if buildindex_provided: | ||
296 | count = self._sortable_buildindex(url, ud, d, latest_rev) | ||
297 | if count is None: | ||
298 | count = "0" | ||
299 | elif uselocalcount or buildindex_provided: | ||
300 | count = str(count) | ||
301 | else: | ||
302 | count = str(int(count) + 1) | ||
303 | |||
304 | localcounts[key + '_rev'] = latest_rev | ||
305 | localcounts[key + '_count'] = count | ||
306 | |||
307 | return str(count + "+" + latest_rev) | ||
308 | |||
309 | def _build_revision(self, url, ud, d): | ||
310 | return ud.tag | ||
311 | |||
312 | def _sortable_buildindex_disabled(self, url, ud, d, rev): | ||
313 | """ | ||
314 | Return a suitable buildindex for the revision specified. This is done by counting revisions | ||
315 | using "git rev-list" which may or may not work in different circumstances. | ||
316 | """ | ||
317 | |||
318 | cwd = os.getcwd() | ||
319 | |||
320 | # Check if we have the rev already | ||
321 | |||
322 | if not os.path.exists(ud.clonedir): | ||
323 | print("no repo") | ||
324 | self.go(None, ud, d) | ||
325 | if not os.path.exists(ud.clonedir): | ||
326 | logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir) | ||
327 | return None | ||
328 | |||
329 | |||
330 | os.chdir(ud.clonedir) | ||
331 | if not self._contains_ref(rev, d): | ||
332 | self.go(None, ud, d) | ||
333 | |||
334 | output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True) | ||
335 | os.chdir(cwd) | ||
336 | |||
337 | buildindex = "%s" % output.split()[0] | ||
338 | logger.debug(1, "GIT repository for %s in %s is returning %s revisions in rev-list before %s", url, ud.clonedir, buildindex, rev) | ||
339 | return buildindex | ||