bitbake/fetch2: Define a new interface to the fetcher code though the 'Fetch' class

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 169 insertions, 144 deletions

diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index f1b87f9ee2..43ee1562cd 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -60,6 +60,13 @@ class FetchError(BBFetchException):
          self.url = url
          Exception.__init__(self, self.msg)
 
+class UnpackError(BBFetchException):
+    """General fetcher exception when something happens incorrectly when unpacking"""
+    def __init__(self, message, url):
+         self.msg = "Unpack failure for URL: '%s'. %s" % (url, message)
+         self.url = url
+         Exception.__init__(self, self.msg)
+
 class NoMethodError(BBFetchException):
     """Exception raised when there is no method to obtain a supplied url or set of urls"""
     def __init__(self, url):
@@ -231,26 +238,6 @@ def fetcher_compare_revisions(d):
             logger.debug(2, "%s did not change", key)
     return False
 
-# Function call order is usually:
-#   1. init
-#   2. go
-#   3. localpaths
-# localpath can be called at any time
-
-def init(urls, d):
-    urldata = {}
-
-    fn = bb.data.getVar('FILE', d, 1)
-    if fn in urldata_cache:
-        urldata = urldata_cache[fn]
-
-    for url in urls:
-        if url not in urldata:
-            urldata[url] = FetchData(url, d)
-
-    urldata_cache[fn] = urldata
-    return urldata
-
 def mirror_from_string(data):
     return [ i.split() for i in (data or "").replace('\\n','\n').split('\n') if i ]
 
@@ -304,118 +291,6 @@ def download_update(result, target):
         os.symlink(result, target)
     return
 
-def download(d, urls = None):
-    """
-    Fetch all urls
-    init must have previously been called
-    """
-    if not urls:
-        urls = d.getVar("SRC_URI", 1).split()
-    urldata = init(urls, d)
-
-    for u in urls:
-        urldata[u].setup_localpath(d)
-
-    for u in urls:
-        ud = urldata[u]
-        m = ud.method
-        localpath = ""
-
-        if not ud.localfile:
-            continue
-
-        lf = bb.utils.lockfile(ud.lockfile)
-
-        if m.try_premirror(u, ud, d):
-            # First try fetching uri, u, from PREMIRRORS
-            mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', d, True))
-            localpath = try_mirrors(d, u, mirrors, False, m.forcefetch(u, ud, d))
-        elif os.path.exists(ud.localfile):
-            localpath = ud.localfile
-
-        download_update(localpath, ud.localpath)
-
-        # Need to re-test forcefetch() which will return true if our copy is too old
-        if m.forcefetch(u, ud, d) or not localpath:
-            # Next try fetching from the original uri, u
-            try:
-                m.download(u, ud, d)
-                if hasattr(m, "build_mirror_data"):
-                    m.build_mirror_data(u, ud, d)
-                localpath = ud.localpath
-                download_update(localpath, ud.localpath)
-
-            except FetchError:
-                # Remove any incomplete file
-                bb.utils.remove(ud.localpath)
-                # Finally, try fetching uri, u, from MIRRORS
-                mirrors = mirror_from_string(bb.data.getVar('MIRRORS', d, True))
-                localpath = try_mirrors (d, u, mirrors)
-
-        if not localpath or not os.path.exists(localpath):
-            raise FetchError("Unable to fetch URL %s from any source." % u, u)
-
-        download_update(localpath, ud.localpath)
-
-        if os.path.exists(ud.donestamp):
-            # Touch the done stamp file to show active use of the download
-            try:
-                os.utime(ud.donestamp, None)
-            except:
-                # Errors aren't fatal here
-                pass
-        else:
-            # Only check the checksums if we've not seen this item before, then create the stamp
-            verify_checksum(u, ud, d)
-            open(ud.donestamp, 'w').close()
-
-
-        bb.utils.unlockfile(lf)
-
-def checkstatus(d, urls = None):
-    """
-    Check all urls exist upstream
-    init must have previously been called
-    """
-    urldata = init([], d)
-
-    if not urls:
-        urls = urldata
-
-    for u in urls:
-        ud = urldata[u]
-        ud.setup_localpath(d)
-        m = ud.method
-        logger.debug(1, "Testing URL %s", u)
-        # First try checking uri, u, from PREMIRRORS
-        mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', d, True))
-        ret = try_mirrors(d, u, mirrors, True)
-        if not ret:
-            # Next try checking from the original uri, u
-            try:
-                ret = m.checkstatus(u, ud, d)
-            except:
-                # Finally, try checking uri, u, from MIRRORS
-                mirrors = mirror_from_string(bb.data.getVar('MIRRORS', d, True))
-                ret = try_mirrors (d, u, mirrors, True)
-
-        if not ret:
-            raise FetchError("URL %s doesn't work" % u, u)
-
-def localpaths(d):
-    """
-    Return a list of the local filenames, assuming successful fetch
-    """
-    local = []
-    urldata = init([], d)
-
-    for u in urldata:
-        ud = urldata[u]
-        ud.setup_localpath(d)
-        local.append(ud.localpath)
-
-    return local
-
 def get_autorev(d):
     #  only not cache src rev in autorev case
     if bb.data.getVar('BB_SRCREV_POLICY', d, True) != "cache":
@@ -432,7 +307,8 @@ def get_srcrev(d):
     """
 
     scms = []
-    urldata = init(bb.data.getVar('SRC_URI', d, 1).split(), d)
+    fetcher = Fetch(bb.data.getVar('SRC_URI', d, True).split(), d)
+    urldata = fetcher.ud
     for u in urldata:
         if urldata[u].method.supports_srcrev():
             scms.append(u)
@@ -459,14 +335,8 @@ def get_srcrev(d):
     return format
 
 def localpath(url, d):
-    """
-    Called from public code, e.g. classes in OE e.g. patch.bbclass
-    """
-    ud = init([url], d)
-    if ud[url].method:
-        ud[url].setup_localpath(d)
-        return ud[url].localpath
-    return url
+    fetcher = bb.fetch2.Fetch([url], d)
+        return fetcher.localpath(url)
 
 def runfetchcmd(cmd, d, quiet = False, cleanup = []):
     """
@@ -796,12 +666,12 @@ class FetchMethod(object):
                 cmd = 'cp %s %s/%s/' % (file, rootdir, destdir)
 
         if not cmd:
-            return True
+            return
 
         dest = os.path.join(rootdir, os.path.basename(file))
         if os.path.exists(dest):
             if os.path.samefile(file, dest):
-                return True
+                return
 
         # Change to subdir before executing command
         save_cwd = os.getcwd();
@@ -817,7 +687,10 @@ class FetchMethod(object):
 
         os.chdir(save_cwd)
 
-        return ret == 0
+        if ret != 0:
+            raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), urldata.url)
+
+        return
 
     def try_premirror(self, url, urldata, d):
         """
@@ -915,6 +788,158 @@ class FetchMethod(object):
         key = self._revision_key(url, ud, d, name)
         return "%s-%s" % (key, bb.data.getVar("PN", d, True) or "")
 
+class Fetch(object):
+    def __init__(self, urls, d):
+        if len(urls) == 0:
+            urls = d.getVar("SRC_URI", 1).split()
+        self.urls = urls
+        self.d = d
+        self.ud = {}
+
+        fn = bb.data.getVar('FILE', d, 1)
+        if fn in urldata_cache:
+            self.ud = urldata_cache[fn]
+
+        for url in urls:
+            if url not in self.ud:
+                self.ud[url] = FetchData(url, d)
+
+        urldata_cache[fn] = self.ud
+
+    def localpath(self, url):
+        if url not in self.urls:
+            self.ud[url] = FetchData(url, self.d)
+
+        self.ud[url].setup_localpath(self.d)
+        return bb.data.expand(self.ud[url].localpath, self.d)
+
+    def localpaths(self):
+        """
+        Return a list of the local filenames, assuming successful fetch
+        """
+        local = []
+
+        for u in self.urls:
+            ud = self.ud[u]
+            ud.setup_localpath(self.d)
+            local.append(ud.localpath)
+
+        return local
+
+    def download(self, urls = []):
+        """
+        Fetch all urls
+        """
+        if len(urls) == 0:
+            urls = self.urls
+
+        for u in urls:
+            ud = self.ud[u]
+            ud.setup_localpath(self.d)
+            m = ud.method
+            localpath = ""
+
+            if not ud.localfile:
+                continue
+
+            lf = bb.utils.lockfile(ud.lockfile)
+
+            if m.try_premirror(u, ud, self.d):
+                # First try fetching uri, u, from PREMIRRORS
+                mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', self.d, True))
+                localpath = try_mirrors(self.d, u, mirrors, False, m.forcefetch(u, ud, self.d))
+            elif os.path.exists(ud.localfile):
+                localpath = ud.localfile
+
+            download_update(localpath, ud.localpath)
+
+            # Need to re-test forcefetch() which will return true if our copy is too old
+            if m.forcefetch(u, ud, self.d) or not localpath:
+                # Next try fetching from the original uri, u
+                try:
+                    m.download(u, ud, self.d)
+                    if hasattr(m, "build_mirror_data"):
+                        m.build_mirror_data(u, ud, self.d)
+                    localpath = ud.localpath
+                    download_update(localpath, ud.localpath)
+
+                except FetchError:
+                    # Remove any incomplete file
+                    bb.utils.remove(ud.localpath)
+                    # Finally, try fetching uri, u, from MIRRORS
+                    mirrors = mirror_from_string(bb.data.getVar('MIRRORS', self.d, True))
+                    localpath = try_mirrors (self.d, u, mirrors)
+
+            if not localpath or not os.path.exists(localpath):
+                raise FetchError("Unable to fetch URL %s from any source." % u, u)
+
+            download_update(localpath, ud.localpath)
+
+            if os.path.exists(ud.donestamp):
+                # Touch the done stamp file to show active use of the download
+                try:
+                    os.utime(ud.donestamp, None)
+                except:
+                    # Errors aren't fatal here
+                    pass
+            else:
+                # Only check the checksums if we've not seen this item before, then create the stamp
+                verify_checksum(u, ud, self.d)
+                open(ud.donestamp, 'w').close()
+
+            bb.utils.unlockfile(lf)
+
+    def checkstatus(self, urls = []):
+        """
+        Check all urls exist upstream
+        """
+
+        if len(urls) == 0:
+            urls = self.urls
+
+        for u in urls:
+            ud = self.ud[u]
+            ud.setup_localpath(self.d)
+            m = ud.method
+            logger.debug(1, "Testing URL %s", u)
+            # First try checking uri, u, from PREMIRRORS
+            mirrors = mirror_from_string(bb.data.getVar('PREMIRRORS', self.d, True))
+            ret = try_mirrors(self.d, u, mirrors, True)
+            if not ret:
+                # Next try checking from the original uri, u
+                try:
+                    ret = m.checkstatus(u, ud, self.d)
+                except:
+                    # Finally, try checking uri, u, from MIRRORS
+                    mirrors = mirror_from_string(bb.data.getVar('MIRRORS', self.d, True))
+                    ret = try_mirrors (self.d, u, mirrors, True)
+
+            if not ret:
+                raise FetchError("URL %s doesn't work" % u, u)
+
+    def unpack(self, root, urls = []):
+        """
+        Check all urls exist upstream
+        """
+
+        if len(urls) == 0:
+            urls = self.urls
+
+        for u in urls:
+            ud = self.ud[u]
+            ud.setup_localpath(self.d)
+
+            if bb.data.expand(self.localpath, self.d) is None:
+                continue
+
+            if ud.lockfile:
+                lf = bb.utils.lockfile(ud.lockfile)
+
+            ud.method.unpack(ud, root, self.d)
+
+            if ud.lockfile:
+                bb.utils.unlockfile(lf)
+
 from . import cvs
 from . import git
 from . import local