summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/git.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r--bitbake/lib/bb/fetch2/git.py355
1 files changed, 355 insertions, 0 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
new file mode 100644
index 0000000..9ca2442
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -0,0 +1,355 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Fetch' git implementation
5
6git fetcher support the SRC_URI with format of:
7SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..."
8
9Supported SRC_URI options are:
10
11- branch
12 The git branch to retrieve from. The default is "master"
13
14 This option also supports multiple branch fetching, with branches
15 separated by commas. In multiple branches case, the name option
16 must have the same number of names to match the branches, which is
17 used to specify the SRC_REV for the branch
18 e.g:
19 SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
20 SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
21 SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
22
23- tag
24 The git tag to retrieve. The default is "master"
25
26- protocol
27 The method to use to access the repository. Common options are "git",
28 "http", "https", "file", "ssh" and "rsync". The default is "git".
29
30- rebaseable
31 rebaseable indicates that the upstream git repo may rebase in the future,
32 and current revision may disappear from upstream repo. This option will
33 remind fetcher to preserve local cache carefully for future use.
34 The default value is "0", set rebaseable=1 for rebaseable git repo.
35
36- nocheckout
37 Don't checkout source code when unpacking. set this option for the recipe
38 who has its own routine to checkout code.
39 The default is "0", set nocheckout=1 if needed.
40
41- bareclone
42 Create a bare clone of the source code and don't checkout the source code
43 when unpacking. Set this option for the recipe who has its own routine to
44 checkout code and tracking branch requirements.
45 The default is "0", set bareclone=1 if needed.
46
47- nobranch
48 Don't check the SHA validation for branch. set this option for the recipe
49 referring to commit which is valid in tag instead of branch.
50 The default is "0", set nobranch=1 if needed.
51
52"""
53
54#Copyright (C) 2005 Richard Purdie
55#
56# This program is free software; you can redistribute it and/or modify
57# it under the terms of the GNU General Public License version 2 as
58# published by the Free Software Foundation.
59#
60# This program is distributed in the hope that it will be useful,
61# but WITHOUT ANY WARRANTY; without even the implied warranty of
62# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
63# GNU General Public License for more details.
64#
65# You should have received a copy of the GNU General Public License along
66# with this program; if not, write to the Free Software Foundation, Inc.,
67# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
68
69import os
70import bb
71from bb import data
72from bb.fetch2 import FetchMethod
73from bb.fetch2 import runfetchcmd
74from bb.fetch2 import logger
75
76class Git(FetchMethod):
77 """Class to fetch a module or modules from git repositories"""
78 def init(self, d):
79 pass
80
81 def supports(self, ud, d):
82 """
83 Check to see if a given url can be fetched with git.
84 """
85 return ud.type in ['git']
86
87 def supports_checksum(self, urldata):
88 return False
89
90 def urldata_init(self, ud, d):
91 """
92 init git specific variable within url data
93 so that the git method like latest_revision() can work
94 """
95 if 'protocol' in ud.parm:
96 ud.proto = ud.parm['protocol']
97 elif not ud.host:
98 ud.proto = 'file'
99 else:
100 ud.proto = "git"
101
102 if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
103 raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
104
105 ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
106
107 ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
108
109 ud.nobranch = ud.parm.get("nobranch","0") == "1"
110
111 # bareclone implies nocheckout
112 ud.bareclone = ud.parm.get("bareclone","0") == "1"
113 if ud.bareclone:
114 ud.nocheckout = 1
115
116 ud.unresolvedrev = {}
117 branches = ud.parm.get("branch", "master").split(',')
118 if len(branches) != len(ud.names):
119 raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
120 ud.branches = {}
121 for name in ud.names:
122 branch = branches[ud.names.index(name)]
123 ud.branches[name] = branch
124 ud.unresolvedrev[name] = branch
125
126 ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
127
128 ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
129
130 ud.setup_revisons(d)
131
132 for name in ud.names:
133 # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
134 if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
135 if ud.revisions[name]:
136 ud.unresolvedrev[name] = ud.revisions[name]
137 ud.revisions[name] = self.latest_revision(ud, d, name)
138
139 gitsrcname = '%s%s' % (ud.host.replace(':','.'), ud.path.replace('/', '.').replace('*', '.'))
140 # for rebaseable git repo, it is necessary to keep mirror tar ball
141 # per revision, so that even the revision disappears from the
142 # upstream repo in the future, the mirror will remain intact and still
143 # contains the revision
144 if ud.rebaseable:
145 for name in ud.names:
146 gitsrcname = gitsrcname + '_' + ud.revisions[name]
147 ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
148 ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
149 gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
150 ud.clonedir = os.path.join(gitdir, gitsrcname)
151
152 ud.localfile = ud.clonedir
153
154 def localpath(self, ud, d):
155 return ud.clonedir
156
157 def need_update(self, ud, d):
158 if not os.path.exists(ud.clonedir):
159 return True
160 os.chdir(ud.clonedir)
161 for name in ud.names:
162 if not self._contains_ref(ud, d, name):
163 return True
164 if ud.write_tarballs and not os.path.exists(ud.fullmirror):
165 return True
166 return False
167
168 def try_premirror(self, ud, d):
169 # If we don't do this, updating an existing checkout with only premirrors
170 # is not possible
171 if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None:
172 return True
173 if os.path.exists(ud.clonedir):
174 return False
175 return True
176
177 def download(self, ud, d):
178 """Fetch url"""
179
180 if ud.user:
181 username = ud.user + '@'
182 else:
183 username = ""
184
185 ud.repochanged = not os.path.exists(ud.fullmirror)
186
187 # If the checkout doesn't exist and the mirror tarball does, extract it
188 if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
189 bb.utils.mkdirhier(ud.clonedir)
190 os.chdir(ud.clonedir)
191 runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
192
193 repourl = "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
194
195 # If the repo still doesn't exist, fallback to cloning it
196 if not os.path.exists(ud.clonedir):
197 # We do this since git will use a "-l" option automatically for local urls where possible
198 if repourl.startswith("file://"):
199 repourl = repourl[7:]
200 clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir)
201 if ud.proto.lower() != 'file':
202 bb.fetch2.check_network_access(d, clone_cmd)
203 runfetchcmd(clone_cmd, d)
204
205 os.chdir(ud.clonedir)
206 # Update the checkout if needed
207 needupdate = False
208 for name in ud.names:
209 if not self._contains_ref(ud, d, name):
210 needupdate = True
211 if needupdate:
212 try:
213 runfetchcmd("%s remote rm origin" % ud.basecmd, d)
214 except bb.fetch2.FetchError:
215 logger.debug(1, "No Origin")
216
217 runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d)
218 fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl)
219 if ud.proto.lower() != 'file':
220 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
221 runfetchcmd(fetch_cmd, d)
222 runfetchcmd("%s prune-packed" % ud.basecmd, d)
223 runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
224 ud.repochanged = True
225 os.chdir(ud.clonedir)
226 for name in ud.names:
227 if not self._contains_ref(ud, d, name):
228 raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
229
230 def build_mirror_data(self, ud, d):
231 # Generate a mirror tarball if needed
232 if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)):
233 # it's possible that this symlink points to read-only filesystem with PREMIRROR
234 if os.path.islink(ud.fullmirror):
235 os.unlink(ud.fullmirror)
236
237 os.chdir(ud.clonedir)
238 logger.info("Creating tarball of git repository")
239 runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d)
240 runfetchcmd("touch %s.done" % (ud.fullmirror), d)
241
242 def unpack(self, ud, destdir, d):
243 """ unpack the downloaded src to destdir"""
244
245 subdir = ud.parm.get("subpath", "")
246 if subdir != "":
247 readpathspec = ":%s" % (subdir)
248 def_destsuffix = "%s/" % os.path.basename(subdir)
249 else:
250 readpathspec = ""
251 def_destsuffix = "git/"
252
253 destsuffix = ud.parm.get("destsuffix", def_destsuffix)
254 destdir = ud.destdir = os.path.join(destdir, destsuffix)
255 if os.path.exists(destdir):
256 bb.utils.prunedir(destdir)
257
258 cloneflags = "-s -n"
259 if ud.bareclone:
260 cloneflags += " --mirror"
261
262 # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused
263 # and you end up with some horrible union of the two when you attempt to clone it
264 # The least invasive workaround seems to be a symlink to the real directory to
265 # fool git into ignoring any .git version that may also be present.
266 #
267 # The issue is fixed in more recent versions of git so we can drop this hack in future
268 # when that version becomes common enough.
269 clonedir = ud.clonedir
270 if not ud.path.endswith(".git"):
271 indirectiondir = destdir[:-1] + ".indirectionsymlink"
272 if os.path.exists(indirectiondir):
273 os.remove(indirectiondir)
274 bb.utils.mkdirhier(os.path.dirname(indirectiondir))
275 os.symlink(ud.clonedir, indirectiondir)
276 clonedir = indirectiondir
277
278 runfetchcmd("git clone %s %s/ %s" % (cloneflags, clonedir, destdir), d)
279 if not ud.nocheckout:
280 os.chdir(destdir)
281 if subdir != "":
282 runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d)
283 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d)
284 else:
285 runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d)
286 return True
287
288 def clean(self, ud, d):
289 """ clean the git directory """
290
291 bb.utils.remove(ud.localpath, True)
292 bb.utils.remove(ud.fullmirror)
293 bb.utils.remove(ud.fullmirror + ".done")
294
295 def supports_srcrev(self):
296 return True
297
298 def _contains_ref(self, ud, d, name):
299 cmd = ""
300 if ud.nobranch:
301 cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
302 ud.basecmd, ud.revisions[name])
303 else:
304 cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
305 ud.basecmd, ud.revisions[name], ud.branches[name])
306 try:
307 output = runfetchcmd(cmd, d, quiet=True)
308 except bb.fetch2.FetchError:
309 return False
310 if len(output.split()) > 1:
311 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
312 return output.split()[0] != "0"
313
314 def _revision_key(self, ud, d, name):
315 """
316 Return a unique key for the url
317 """
318 return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name]
319
320 def _lsremote(self, ud, d, search):
321 """
322 Run git ls-remote with the specified search string
323 """
324 if ud.user:
325 username = ud.user + '@'
326 else:
327 username = ""
328
329 cmd = "%s ls-remote %s://%s%s%s %s" % \
330 (ud.basecmd, ud.proto, username, ud.host, ud.path, search)
331 if ud.proto.lower() != 'file':
332 bb.fetch2.check_network_access(d, cmd)
333 output = runfetchcmd(cmd, d, True)
334 if not output:
335 raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url)
336 return output
337
338 def _latest_revision(self, ud, d, name):
339 """
340 Compute the HEAD revision for the url
341 """
342 search = "refs/heads/%s refs/tags/%s^{}" % (ud.unresolvedrev[name], ud.unresolvedrev[name])
343 output = self._lsremote(ud, d, search)
344 return output.split()[0]
345
346 def _build_revision(self, ud, d, name):
347 return ud.revisions[name]
348
349 def checkstatus(self, ud, d):
350 fetchcmd = "%s ls-remote %s" % (ud.basecmd, ud.url)
351 try:
352 runfetchcmd(fetchcmd, d, quiet=True)
353 return True
354 except FetchError:
355 return False