Update to latest bitbake

git-svn-id: https://svn.o-hand.com/repos/poky/trunk@309 311d38ba-8fff-0310-9ca6-ca027cbcb966

5 files changed, 83 insertions, 134 deletions

diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py
index da5b10c4b6..0515f2a5e9 100644
--- a/bitbake/lib/bb/fetch/__init__.py
+++ b/bitbake/lib/bb/fetch/__init__.py
@@ -158,18 +158,47 @@ class Fetch(object):
         return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1 )
     getSRCDate = staticmethod(getSRCDate)
 
-#if __name__ == "__main__":
+    def try_mirror(d, tarfn):
+        """
+        Try to use a mirrored version of the sources. We do this
+        to avoid massive loads on foreign cvs and svn servers.
+        This method will be used by the different fetcher
+        implementations.
+
+        d Is a bb.data instance
+        tarfn is the name of the tarball
+        """
+        tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
+        if os.access(tarpath, os.R_OK):
+            return True
+
+        pn = data.getVar('PN', d, True)
+        src_tarball_stash = None
+        if pn:
+            src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
+
+        for stash in src_tarball_stash:
+            fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
+            uri = stash + tarfn
+            bb.note("fetch " + uri)
+            fetchcmd = fetchcmd.replace("${URI}", uri)
+            ret = os.system(fetchcmd)
+            if ret == 0:
+                bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
+                return True
+        return False
+    try_mirror = staticmethod(try_mirror)
 
-import bk
 import cvs
 import git
 import local
 import svn
 import wget
+import svk
 
-methods.append(bk.Bk())
 methods.append(cvs.Cvs())
 methods.append(git.Git())
 methods.append(local.Local())
 methods.append(svn.Svn())
 methods.append(wget.Wget())
+methods.append(svk.Svk())
diff --git a/bitbake/lib/bb/fetch/bk.py b/bitbake/lib/bb/fetch/bk.py
deleted file mode 100644
index 6bd6c018fb..0000000000
--- a/bitbake/lib/bb/fetch/bk.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-# ex:ts=4:sw=4:sts=4:et
-# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
-"""
-BitBake 'Fetch' implementations
-
-Classes for obtaining upstream sources for the
-BitBake build tools.
-
-Copyright (C) 2003, 2004  Chris Larson
-
-This program is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free Software
-Foundation; either version 2 of the License, or (at your option) any later
-version.
-
-This program is distributed in the hope that it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc., 59 Temple
-Place, Suite 330, Boston, MA 02111-1307 USA. 
-
-Based on functions from the base bb module, Copyright 2003 Holger Schurig
-"""
-
-import os, re
-import bb
-from   bb import data
-from   bb.fetch import Fetch
-
-class Bk(Fetch):
-    def supports(url, d):
-        """Check to see if a given url can be fetched via bitkeeper.
-           Expects supplied url in list form, as outputted by bb.decodeurl().
-        """
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-        return type in ['bk']
-    supports = staticmethod(supports)
diff --git a/bitbake/lib/bb/fetch/cvs.py b/bitbake/lib/bb/fetch/cvs.py
index 461a2f50f9..10ec700dc9 100644
--- a/bitbake/lib/bb/fetch/cvs.py
+++ b/bitbake/lib/bb/fetch/cvs.py
@@ -133,21 +133,9 @@ class Cvs(Fetch):
                 bb.debug(1, "%s already exists, skipping cvs checkout." % tarfn)
                 continue
 
-            pn = data.getVar('PN', d, 1)
-            cvs_tarball_stash = None
-            if pn:
-                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
-            if cvs_tarball_stash == None:
-                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-            if cvs_tarball_stash:
-                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-                uri = cvs_tarball_stash + tarfn
-                bb.note("fetch " + uri)
-                fetchcmd = fetchcmd.replace("${URI}", uri)
-                ret = os.system(fetchcmd)
-                if ret == 0:
-                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-                    continue
+            # try to use the tarball stash
+            if Fetch.try_mirror(d, tarfn):
+                continue
 
             if date:
                 options.append("-D %s" % date)
@@ -194,7 +182,7 @@ class Cvs(Fetch):
                 bb.debug(1, "Running %s" % cvscmd)
                 myret = os.system(cvscmd)
 
-            if myret != 0:
+            if myret != 0 or not os.access(moddir, os.R_OK):
                 try:
                     os.rmdir(moddir)
                 except OSError:
diff --git a/bitbake/lib/bb/fetch/git.py b/bitbake/lib/bb/fetch/git.py
index 296b926392..439d522188 100644
--- a/bitbake/lib/bb/fetch/git.py
+++ b/bitbake/lib/bb/fetch/git.py
@@ -58,6 +58,28 @@ def gettag(parm):
 
     return tag
 
+def getprotocol(parm):
+    if 'protocol' in parm:
+        proto = parm['protocol']
+    else:
+        proto = ""
+    if not proto:
+        proto = "rsync"
+
+    return proto
+
+def localfile(url, d):
+    """Return the filename to cache the checkout in"""
+    (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
+
+    #if user sets localpath for file, use it instead.
+    if "localpath" in parm:
+        return parm["localpath"]
+
+    tag = gettag(parm)
+
+    return data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d)
+
 class Git(Fetch):
     """Class to fetch a module or modules from git repositories"""
     def supports(url, d):
@@ -69,17 +91,8 @@ class Git(Fetch):
     supports = staticmethod(supports)
 
     def localpath(url, d):
-        (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(url, d))
-
-        #if user sets localpath for file, use it instead.
-        if "localpath" in parm:
-            return parm["localpath"]
 
-        tag = gettag(parm)
-
-        localname = data.expand('git_%s%s_%s.tar.gz' % (host, path.replace('/', '.'), tag), d)
-
-        return os.path.join(data.getVar("DL_DIR", d, 1),data.expand('%s' % (localname), d))
+        return os.path.join(data.getVar("DL_DIR", d, 1), localfile(url, d))
 
     localpath = staticmethod(localpath)
 
@@ -92,10 +105,12 @@ class Git(Fetch):
             (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d))
 
             tag = gettag(parm)
+            proto = getprotocol(parm)
 
             gitsrcname = '%s%s' % (host, path.replace('/', '.'))
 
-            repofile = os.path.join(data.getVar("DL_DIR", d, 1), 'git_%s.tar.gz' % (gitsrcname))
+            repofilename = 'git_%s.tar.gz' % (gitsrcname)
+            repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
             repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
 
             coname = '%s' % (tag)
@@ -103,63 +118,37 @@ class Git(Fetch):
 
             cofile = self.localpath(loc, d)
 
-            # Always update to current if tag=="master"
-            #if os.access(cofile, os.R_OK) and (tag != "master"):
-            if os.access(cofile, os.R_OK):
-                bb.debug(1, "%s already exists, skipping git checkout." % cofile)
+            # tag=="master" must always update
+            if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)):
+                bb.debug(1, "%s already exists (or was stashed). Skipping git checkout." % cofile)
                 continue
 
-# Still Need to add GIT_TARBALL_STASH Support...
-#            pn = data.getVar('PN', d, 1)
-#            cvs_tarball_stash = None
-#            if pn:
-#                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH_%s' % pn, d, 1)
-#            if cvs_tarball_stash == None:
-#                cvs_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-#            if cvs_tarball_stash:
-#                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-#                uri = cvs_tarball_stash + tarfn
-#                bb.note("fetch " + uri)
-#                fetchcmd = fetchcmd.replace("${URI}", uri)
-#                ret = os.system(fetchcmd)
-#                if ret == 0:
-#                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-#                    continue
-         
-            #if os.path.exists(repodir):
-                #prunedir(repodir)
-
-            bb.mkdirhier(repodir)
-            os.chdir(repodir)
-
-            #print("Changing to %s" % repodir)
+            if not os.path.exists(repodir):
+                if Fetch.try_mirror(d, repofilename):    
+                    bb.mkdirhier(repodir)
+                    os.chdir(repodir)
+                    rungitcmd("tar -xzf %s" % (repofile),d)
+                else:
+                    rungitcmd("git clone %s://%s%s %s" % (proto, host, path, repodir),d)
 
-            if os.access(repofile, os.R_OK):
-                rungitcmd("tar -xzf %s" % (repofile),d)
-            else:
-                rungitcmd("git clone rsync://%s%s %s" % (host, path, repodir),d)
-
-            rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d)
-
-            #print("Changing to %s" % repodir)
             os.chdir(repodir)
-            rungitcmd("git pull rsync://%s%s" % (host, path),d)
+            rungitcmd("git pull %s://%s%s" % (proto, host, path),d)
+            rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d)
+            # old method of downloading tags
+            #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d)
 
-            #print("Changing to %s" % repodir)
             os.chdir(repodir)
+            bb.note("Creating tarball of git repository")
             rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d)
 
             if os.path.exists(codir):
                 prunedir(codir)
 
-            #print("Changing to %s" % repodir)
             bb.mkdirhier(codir)
             os.chdir(repodir)
             rungitcmd("git read-tree %s" % (tag),d)
-
             rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d)
 
-            #print("Changing to %s" % codir)
             os.chdir(codir)
+            bb.note("Creating tarball of git checkout")
             rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d)
-
diff --git a/bitbake/lib/bb/fetch/svn.py b/bitbake/lib/bb/fetch/svn.py
index ac5eebf5c0..6e3a9277ab 100644
--- a/bitbake/lib/bb/fetch/svn.py
+++ b/bitbake/lib/bb/fetch/svn.py
@@ -98,20 +98,14 @@ class Svn(Fetch):
 
             date = Fetch.getSRCDate(d)
 
-            if "method" in parm:
-                method = parm["method"]
-            else:
-                method = "pserver"
-
             if "proto" in parm:
                 proto = parm["proto"]
             else:
                 proto = "svn"
 
             svn_rsh = None
-            if method == "ext":
-                if "rsh" in parm:
-                    svn_rsh = parm["rsh"]
+            if proto == "svn+ssh" and "rsh" in parm:
+                svn_rsh = parm["rsh"]
 
             tarfn = data.expand('%s_%s_%s_%s_%s.tar.gz' % (module.replace('/', '.'), host, path.replace('/', '.'), revision, date), localdata)
             data.setVar('TARFILES', dlfile, localdata)
@@ -122,24 +116,13 @@ class Svn(Fetch):
                 bb.debug(1, "%s already exists, skipping svn checkout." % tarfn)
                 continue
 
-            svn_tarball_stash = data.getVar('CVS_TARBALL_STASH', d, 1)
-            if svn_tarball_stash:
-                fetchcmd = data.getVar("FETCHCOMMAND_wget", d, 1)
-                uri = svn_tarball_stash + tarfn
-                bb.note("fetch " + uri)
-                fetchcmd = fetchcmd.replace("${URI}", uri)
-                ret = os.system(fetchcmd)
-                if ret == 0:
-                    bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
-                    continue
+            # try to use the tarball stash
+            if Fetch.try_mirror(d, tarfn):
+                continue
 
             olddir = os.path.abspath(os.getcwd())
             os.chdir(data.expand(dldir, localdata))
 
-#           setup svnroot
-#            svnroot = ":" + method + ":" + user
-#            if pswd:
-#                svnroot += ":" + pswd
             svnroot = host + path
 
             data.setVar('SVNROOT', svnroot, localdata)