From c527fd1f14c27855a37f2e8ac5346ce8d940ced2 Mon Sep 17 00:00:00 2001 From: Tudor Florea Date: Thu, 16 Oct 2014 03:05:19 +0200 Subject: initial commit for Enea Linux 4.0-140929 Migrated from the internal git server on the daisy-enea-point-release branch Signed-off-by: Tudor Florea --- bitbake/lib/bb/fetch2/git.py | 355 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 bitbake/lib/bb/fetch2/git.py (limited to 'bitbake/lib/bb/fetch2/git.py') diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py new file mode 100644 index 0000000000..9ca24428a1 --- /dev/null +++ b/bitbake/lib/bb/fetch2/git.py @@ -0,0 +1,355 @@ +# ex:ts=4:sw=4:sts=4:et +# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +""" +BitBake 'Fetch' git implementation + +git fetcher support the SRC_URI with format of: +SRC_URI = "git://some.host/somepath;OptionA=xxx;OptionB=xxx;..." + +Supported SRC_URI options are: + +- branch + The git branch to retrieve from. The default is "master" + + This option also supports multiple branch fetching, with branches + separated by commas. In multiple branches case, the name option + must have the same number of names to match the branches, which is + used to specify the SRC_REV for the branch + e.g: + SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY" + SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx" + SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY" + +- tag + The git tag to retrieve. The default is "master" + +- protocol + The method to use to access the repository. Common options are "git", + "http", "https", "file", "ssh" and "rsync". The default is "git". + +- rebaseable + rebaseable indicates that the upstream git repo may rebase in the future, + and current revision may disappear from upstream repo. This option will + remind fetcher to preserve local cache carefully for future use. + The default value is "0", set rebaseable=1 for rebaseable git repo. + +- nocheckout + Don't checkout source code when unpacking. set this option for the recipe + who has its own routine to checkout code. + The default is "0", set nocheckout=1 if needed. + +- bareclone + Create a bare clone of the source code and don't checkout the source code + when unpacking. Set this option for the recipe who has its own routine to + checkout code and tracking branch requirements. + The default is "0", set bareclone=1 if needed. + +- nobranch + Don't check the SHA validation for branch. set this option for the recipe + referring to commit which is valid in tag instead of branch. + The default is "0", set nobranch=1 if needed. + +""" + +#Copyright (C) 2005 Richard Purdie +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import os +import bb +from bb import data +from bb.fetch2 import FetchMethod +from bb.fetch2 import runfetchcmd +from bb.fetch2 import logger + +class Git(FetchMethod): + """Class to fetch a module or modules from git repositories""" + def init(self, d): + pass + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched with git. + """ + return ud.type in ['git'] + + def supports_checksum(self, urldata): + return False + + def urldata_init(self, ud, d): + """ + init git specific variable within url data + so that the git method like latest_revision() can work + """ + if 'protocol' in ud.parm: + ud.proto = ud.parm['protocol'] + elif not ud.host: + ud.proto = 'file' + else: + ud.proto = "git" + + if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'): + raise bb.fetch2.ParameterError("Invalid protocol type", ud.url) + + ud.nocheckout = ud.parm.get("nocheckout","0") == "1" + + ud.rebaseable = ud.parm.get("rebaseable","0") == "1" + + ud.nobranch = ud.parm.get("nobranch","0") == "1" + + # bareclone implies nocheckout + ud.bareclone = ud.parm.get("bareclone","0") == "1" + if ud.bareclone: + ud.nocheckout = 1 + + ud.unresolvedrev = {} + branches = ud.parm.get("branch", "master").split(',') + if len(branches) != len(ud.names): + raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url) + ud.branches = {} + for name in ud.names: + branch = branches[ud.names.index(name)] + ud.branches[name] = branch + ud.unresolvedrev[name] = branch + + ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git" + + ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable + + ud.setup_revisons(d) + + for name in ud.names: + # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one + if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]): + if ud.revisions[name]: + ud.unresolvedrev[name] = ud.revisions[name] + ud.revisions[name] = self.latest_revision(ud, d, name) + + gitsrcname = '%s%s' % (ud.host.replace(':','.'), ud.path.replace('/', '.').replace('*', '.')) + # for rebaseable git repo, it is necessary to keep mirror tar ball + # per revision, so that even the revision disappears from the + # upstream repo in the future, the mirror will remain intact and still + # contains the revision + if ud.rebaseable: + for name in ud.names: + gitsrcname = gitsrcname + '_' + ud.revisions[name] + ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) + ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") + ud.clonedir = os.path.join(gitdir, gitsrcname) + + ud.localfile = ud.clonedir + + def localpath(self, ud, d): + return ud.clonedir + + def need_update(self, ud, d): + if not os.path.exists(ud.clonedir): + return True + os.chdir(ud.clonedir) + for name in ud.names: + if not self._contains_ref(ud, d, name): + return True + if ud.write_tarballs and not os.path.exists(ud.fullmirror): + return True + return False + + def try_premirror(self, ud, d): + # If we don't do this, updating an existing checkout with only premirrors + # is not possible + if d.getVar("BB_FETCH_PREMIRRORONLY", True) is not None: + return True + if os.path.exists(ud.clonedir): + return False + return True + + def download(self, ud, d): + """Fetch url""" + + if ud.user: + username = ud.user + '@' + else: + username = "" + + ud.repochanged = not os.path.exists(ud.fullmirror) + + # If the checkout doesn't exist and the mirror tarball does, extract it + if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): + bb.utils.mkdirhier(ud.clonedir) + os.chdir(ud.clonedir) + runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) + + repourl = "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path) + + # If the repo still doesn't exist, fallback to cloning it + if not os.path.exists(ud.clonedir): + # We do this since git will use a "-l" option automatically for local urls where possible + if repourl.startswith("file://"): + repourl = repourl[7:] + clone_cmd = "%s clone --bare --mirror %s %s" % (ud.basecmd, repourl, ud.clonedir) + if ud.proto.lower() != 'file': + bb.fetch2.check_network_access(d, clone_cmd) + runfetchcmd(clone_cmd, d) + + os.chdir(ud.clonedir) + # Update the checkout if needed + needupdate = False + for name in ud.names: + if not self._contains_ref(ud, d, name): + needupdate = True + if needupdate: + try: + runfetchcmd("%s remote rm origin" % ud.basecmd, d) + except bb.fetch2.FetchError: + logger.debug(1, "No Origin") + + runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d) + fetch_cmd = "%s fetch -f --prune %s refs/*:refs/*" % (ud.basecmd, repourl) + if ud.proto.lower() != 'file': + bb.fetch2.check_network_access(d, fetch_cmd, ud.url) + runfetchcmd(fetch_cmd, d) + runfetchcmd("%s prune-packed" % ud.basecmd, d) + runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) + ud.repochanged = True + os.chdir(ud.clonedir) + for name in ud.names: + if not self._contains_ref(ud, d, name): + raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) + + def build_mirror_data(self, ud, d): + # Generate a mirror tarball if needed + if ud.write_tarballs and (ud.repochanged or not os.path.exists(ud.fullmirror)): + # it's possible that this symlink points to read-only filesystem with PREMIRROR + if os.path.islink(ud.fullmirror): + os.unlink(ud.fullmirror) + + os.chdir(ud.clonedir) + logger.info("Creating tarball of git repository") + runfetchcmd("tar -czf %s %s" % (ud.fullmirror, os.path.join(".") ), d) + runfetchcmd("touch %s.done" % (ud.fullmirror), d) + + def unpack(self, ud, destdir, d): + """ unpack the downloaded src to destdir""" + + subdir = ud.parm.get("subpath", "") + if subdir != "": + readpathspec = ":%s" % (subdir) + def_destsuffix = "%s/" % os.path.basename(subdir) + else: + readpathspec = "" + def_destsuffix = "git/" + + destsuffix = ud.parm.get("destsuffix", def_destsuffix) + destdir = ud.destdir = os.path.join(destdir, destsuffix) + if os.path.exists(destdir): + bb.utils.prunedir(destdir) + + cloneflags = "-s -n" + if ud.bareclone: + cloneflags += " --mirror" + + # Versions of git prior to 1.7.9.2 have issues where foo.git and foo get confused + # and you end up with some horrible union of the two when you attempt to clone it + # The least invasive workaround seems to be a symlink to the real directory to + # fool git into ignoring any .git version that may also be present. + # + # The issue is fixed in more recent versions of git so we can drop this hack in future + # when that version becomes common enough. + clonedir = ud.clonedir + if not ud.path.endswith(".git"): + indirectiondir = destdir[:-1] + ".indirectionsymlink" + if os.path.exists(indirectiondir): + os.remove(indirectiondir) + bb.utils.mkdirhier(os.path.dirname(indirectiondir)) + os.symlink(ud.clonedir, indirectiondir) + clonedir = indirectiondir + + runfetchcmd("git clone %s %s/ %s" % (cloneflags, clonedir, destdir), d) + if not ud.nocheckout: + os.chdir(destdir) + if subdir != "": + runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d) + runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d) + else: + runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d) + return True + + def clean(self, ud, d): + """ clean the git directory """ + + bb.utils.remove(ud.localpath, True) + bb.utils.remove(ud.fullmirror) + bb.utils.remove(ud.fullmirror + ".done") + + def supports_srcrev(self): + return True + + def _contains_ref(self, ud, d, name): + cmd = "" + if ud.nobranch: + cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % ( + ud.basecmd, ud.revisions[name]) + else: + cmd = "%s branch --contains %s --list %s 2> /dev/null | wc -l" % ( + ud.basecmd, ud.revisions[name], ud.branches[name]) + try: + output = runfetchcmd(cmd, d, quiet=True) + except bb.fetch2.FetchError: + return False + if len(output.split()) > 1: + raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) + return output.split()[0] != "0" + + def _revision_key(self, ud, d, name): + """ + Return a unique key for the url + """ + return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name] + + def _lsremote(self, ud, d, search): + """ + Run git ls-remote with the specified search string + """ + if ud.user: + username = ud.user + '@' + else: + username = "" + + cmd = "%s ls-remote %s://%s%s%s %s" % \ + (ud.basecmd, ud.proto, username, ud.host, ud.path, search) + if ud.proto.lower() != 'file': + bb.fetch2.check_network_access(d, cmd) + output = runfetchcmd(cmd, d, True) + if not output: + raise bb.fetch2.FetchError("The command %s gave empty output unexpectedly" % cmd, ud.url) + return output + + def _latest_revision(self, ud, d, name): + """ + Compute the HEAD revision for the url + """ + search = "refs/heads/%s refs/tags/%s^{}" % (ud.unresolvedrev[name], ud.unresolvedrev[name]) + output = self._lsremote(ud, d, search) + return output.split()[0] + + def _build_revision(self, ud, d, name): + return ud.revisions[name] + + def checkstatus(self, ud, d): + fetchcmd = "%s ls-remote %s" % (ud.basecmd, ud.url) + try: + runfetchcmd(fetchcmd, d, quiet=True) + return True + except FetchError: + return False -- cgit v1.2.3-54-g00ecf