summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch/git.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2012-03-03 11:25:19 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-03-05 10:38:01 -0800
commit5419dc1bb787d338f9aec238b627ae9eea2f948d (patch)
treef4d104e4edfdff41bef735b578f260ee2c9124fd /bitbake/lib/bb/fetch/git.py
parentff73b02a7203623e5edfaaa10df17eaa63cf4ed3 (diff)
downloadpoky-5419dc1bb787d338f9aec238b627ae9eea2f948d.tar.gz
bitbake: Drop fetch v1, v2 provides a much better codebase to build from
(Bitbake rev: 292e3430e5140b602cad86f55b5453e8cebb28a1) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch/git.py')
-rw-r--r--bitbake/lib/bb/fetch/git.py339
1 files changed, 0 insertions, 339 deletions
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
deleted file mode 100644
index 7160919d5a..0000000000
--- a/bitbake/lib/bb/fetch/git.py
+++ /dev/null
@@ -1,339 +0,0 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' git implementation
5
6"""
7
8#Copyright (C) 2005 Richard Purdie
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License version 2 as
12# published by the Free Software Foundation.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License along
20# with this program; if not, write to the Free Software Foundation, Inc.,
21# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22
23import os
24import bb
25import bb.persist_data
26from bb import data
27from bb.fetch import Fetch
28from bb.fetch import runfetchcmd
29from bb.fetch import logger
30
31class Git(Fetch):
32 """Class to fetch a module or modules from git repositories"""
33 def init(self, d):
34 #
35 # Only enable _sortable revision if the key is set
36 #
37 if d.getVar("BB_GIT_CLONE_FOR_SRCREV", True):
38 self._sortable_buildindex = self._sortable_buildindex_disabled
39 def supports(self, url, ud, d):
40 """
41 Check to see if a given url can be fetched with git.
42 """
43 return ud.type in ['git']
44
45 def localpath(self, url, ud, d):
46
47 if 'protocol' in ud.parm:
48 ud.proto = ud.parm['protocol']
49 elif not ud.host:
50 ud.proto = 'file'
51 else:
52 ud.proto = "rsync"
53
54 ud.branch = ud.parm.get("branch", "master")
55
56 gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
57 ud.mirrortarball = 'git_%s.tar.gz' % (gitsrcname)
58 ud.clonedir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
59
60 tag = Fetch.srcrev_internal_helper(ud, d)
61 if tag is True:
62 ud.tag = self.latest_revision(url, ud, d)
63 elif tag:
64 ud.tag = tag
65
66 if not ud.tag or ud.tag == "master":
67 ud.tag = self.latest_revision(url, ud, d)
68
69 subdir = ud.parm.get("subpath", "")
70 if subdir != "":
71 if subdir.endswith("/"):
72 subdir = subdir[:-1]
73 subdirpath = os.path.join(ud.path, subdir);
74 else:
75 subdirpath = ud.path;
76
77 if 'fullclone' in ud.parm:
78 ud.localfile = ud.mirrortarball
79 else:
80 ud.localfile = data.expand('git_%s%s_%s.tar.gz' % (ud.host, subdirpath.replace('/', '.'), ud.tag), d)
81
82 ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
83
84 if 'noclone' in ud.parm:
85 ud.localfile = None
86 return None
87
88 return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
89
90 def forcefetch(self, url, ud, d):
91 if 'fullclone' in ud.parm:
92 return True
93 if 'noclone' in ud.parm:
94 return False
95 if os.path.exists(ud.localpath):
96 return False
97 if not self._contains_ref(ud.tag, d):
98 return True
99 return False
100
101 def try_premirror(self, u, ud, d):
102 if 'noclone' in ud.parm:
103 return False
104 if os.path.exists(ud.clonedir):
105 return False
106 if os.path.exists(ud.localpath):
107 return False
108
109 return True
110
111 def go(self, loc, ud, d):
112 """Fetch url"""
113
114 if ud.user:
115 username = ud.user + '@'
116 else:
117 username = ""
118
119 repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
120
121
122 coname = '%s' % (ud.tag)
123 codir = os.path.join(ud.clonedir, coname)
124
125 # If we have no existing clone and no mirror tarball, try and obtain one
126 if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
127 try:
128 Fetch.try_mirrors(ud.mirrortarball)
129 except:
130 pass
131
132 # If the checkout doesn't exist and the mirror tarball does, extract it
133 if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
134 bb.utils.mkdirhier(ud.clonedir)
135 os.chdir(ud.clonedir)
136 runfetchcmd("tar -xzf %s" % (repofile), d)
137
138 # If the repo still doesn't exist, fallback to cloning it
139 if not os.path.exists(ud.clonedir):
140 runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d)
141
142 os.chdir(ud.clonedir)
143 # Update the checkout if needed
144 if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm:
145 # Remove all but the .git directory
146 runfetchcmd("rm * -Rf", d)
147 if 'fullclone' in ud.parm:
148 runfetchcmd("%s fetch --all" % (ud.basecmd), d)
149 else:
150 runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d)
151 runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
152 runfetchcmd("%s prune-packed" % ud.basecmd, d)
153 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
154
155 # Generate a mirror tarball if needed
156 os.chdir(ud.clonedir)
157 mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
158 if mirror_tarballs != "0" or 'fullclone' in ud.parm:
159 logger.info("Creating tarball of git repository")
160 runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
161
162 if 'fullclone' in ud.parm:
163 return
164
165 if os.path.exists(codir):
166 bb.utils.prunedir(codir)
167
168 subdir = ud.parm.get("subpath", "")
169 if subdir != "":
170 if subdir.endswith("/"):
171 subdirbase = os.path.basename(subdir[:-1])
172 else:
173 subdirbase = os.path.basename(subdir)
174 else:
175 subdirbase = ""
176
177 if subdir != "":
178 readpathspec = ":%s" % (subdir)
179 codir = os.path.join(codir, "git")
180 coprefix = os.path.join(codir, subdirbase, "")
181 else:
182 readpathspec = ""
183 coprefix = os.path.join(codir, "git", "")
184
185 scmdata = ud.parm.get("scmdata", "")
186 if scmdata == "keep":
187 runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
188 os.chdir(coprefix)
189 runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
190 else:
191 bb.utils.mkdirhier(codir)
192 os.chdir(ud.clonedir)
193 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
194 runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
195
196 os.chdir(codir)
197 logger.info("Creating tarball of git checkout")
198 runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
199
200 os.chdir(ud.clonedir)
201 bb.utils.prunedir(codir)
202
203 def supports_srcrev(self):
204 return True
205
206 def _contains_ref(self, tag, d):
207 basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
208 output = runfetchcmd("%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (basecmd, tag), d, quiet=True)
209 return output.split()[0] != "0"
210
211 def _revision_key(self, url, ud, d, branch=False):
212 """
213 Return a unique key for the url
214 """
215 key = 'git:' + ud.host + ud.path.replace('/', '.')
216 if branch:
217 return key + ud.branch
218 else:
219 return key
220
221 def generate_revision_key(self, url, ud, d, branch=False):
222 key = self._revision_key(url, ud, d, branch)
223 return "%s-%s" % (key, d.getVar("PN", True) or "")
224
225 def _latest_revision(self, url, ud, d):
226 """
227 Compute the HEAD revision for the url
228 """
229 if ud.user:
230 username = ud.user + '@'
231 else:
232 username = ""
233
234 basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
235 cmd = "%s ls-remote %s://%s%s%s %s" % (basecmd, ud.proto, username, ud.host, ud.path, ud.branch)
236 output = runfetchcmd(cmd, d, True)
237 if not output:
238 raise bb.fetch.FetchError("Fetch command %s gave empty output\n" % (cmd))
239 return output.split()[0]
240
241 def latest_revision(self, url, ud, d):
242 """
243 Look in the cache for the latest revision, if not present ask the SCM.
244 """
245 revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
246
247 key = self.generate_revision_key(url, ud, d, branch=True)
248
249 try:
250 return revs[key]
251 except KeyError:
252 # Compatibility with old key format, no branch included
253 oldkey = self.generate_revision_key(url, ud, d, branch=False)
254 try:
255 rev = revs[oldkey]
256 except KeyError:
257 rev = self._latest_revision(url, ud, d)
258 else:
259 del revs[oldkey]
260 revs[key] = rev
261 return rev
262
263 def sortable_revision(self, url, ud, d):
264 """
265
266 """
267 localcounts = bb.persist_data.persist('BB_URI_LOCALCOUNT', d)
268 key = self.generate_revision_key(url, ud, d, branch=True)
269 oldkey = self.generate_revision_key(url, ud, d, branch=False)
270
271 latest_rev = self._build_revision(url, ud, d)
272 last_rev = localcounts.get(key + '_rev')
273 if last_rev is None:
274 last_rev = localcounts.get(oldkey + '_rev')
275 if last_rev is not None:
276 del localcounts[oldkey + '_rev']
277 localcounts[key + '_rev'] = last_rev
278
279 uselocalcount = d.getVar("BB_LOCALCOUNT_OVERRIDE", True) or False
280 count = None
281 if uselocalcount:
282 count = Fetch.localcount_internal_helper(ud, d)
283 if count is None:
284 count = localcounts.get(key + '_count')
285 if count is None:
286 count = localcounts.get(oldkey + '_count')
287 if count is not None:
288 del localcounts[oldkey + '_count']
289 localcounts[key + '_count'] = count
290
291 if last_rev == latest_rev:
292 return str(count + "+" + latest_rev)
293
294 buildindex_provided = hasattr(self, "_sortable_buildindex")
295 if buildindex_provided:
296 count = self._sortable_buildindex(url, ud, d, latest_rev)
297 if count is None:
298 count = "0"
299 elif uselocalcount or buildindex_provided:
300 count = str(count)
301 else:
302 count = str(int(count) + 1)
303
304 localcounts[key + '_rev'] = latest_rev
305 localcounts[key + '_count'] = count
306
307 return str(count + "+" + latest_rev)
308
309 def _build_revision(self, url, ud, d):
310 return ud.tag
311
312 def _sortable_buildindex_disabled(self, url, ud, d, rev):
313 """
314 Return a suitable buildindex for the revision specified. This is done by counting revisions
315 using "git rev-list" which may or may not work in different circumstances.
316 """
317
318 cwd = os.getcwd()
319
320 # Check if we have the rev already
321
322 if not os.path.exists(ud.clonedir):
323 print("no repo")
324 self.go(None, ud, d)
325 if not os.path.exists(ud.clonedir):
326 logger.error("GIT repository for %s doesn't exist in %s, cannot get sortable buildnumber, using old value", url, ud.clonedir)
327 return None
328
329
330 os.chdir(ud.clonedir)
331 if not self._contains_ref(rev, d):
332 self.go(None, ud, d)
333
334 output = runfetchcmd("%s rev-list %s -- 2> /dev/null | wc -l" % (ud.basecmd, rev), d, quiet=True)
335 os.chdir(cwd)
336
337 buildindex = "%s" % output.split()[0]
338 logger.debug(1, "GIT repository for %s in %s is returning %s revisions in rev-list before %s", url, ud.clonedir, buildindex, rev)
339 return buildindex