meta/classes: Update classes to use new fetcher API

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

5 files changed, 63 insertions, 189 deletions

diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index d8efcc0f8c..edb65eb96b 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -116,163 +116,38 @@ addtask setscene before do_fetch
 addtask fetch
 do_fetch[dirs] = "${DL_DIR}"
 python base_do_fetch() {
-        import sys
+
+        src_uri = (bb.data.getVar('SRC_URI', d, True) or "").split()
+        if len(src_uri) == 0:
+                return
 
         localdata = bb.data.createCopy(d)
         bb.data.update_data(localdata)
 
-        src_uri = bb.data.getVar('SRC_URI', localdata, 1)
-        if not src_uri:
-                return 1
-
-        try:
-                bb.fetch.init(src_uri.split(),d)
-        except bb.fetch.NoMethodError:
-                (type, value, traceback) = sys.exc_info()
-                raise bb.build.FuncFailed("No method: %s" % value)
-        except bb.MalformedUrl:
-                (type, value, traceback) = sys.exc_info()
-                raise bb.build.FuncFailed("Malformed URL: %s" % value)
-
-        try:
-                if bb.fetch.__version__ == "1":
-                        bb.fetch.go(localdata)
-                else:
-                        bb.fetch.download(localdata)
-        except bb.fetch.MissingParameterError:
-                (type, value, traceback) = sys.exc_info()
-                raise bb.build.FuncFailed("Missing parameters: %s" % value)
-        except bb.fetch.FetchError:
-                (type, value, traceback) = sys.exc_info()
-                raise bb.build.FuncFailed("Fetch failed: %s" % value)
-        except bb.fetch.MD5SumError:
-                (type, value, traceback) = sys.exc_info()
-                raise bb.build.FuncFailed("MD5  failed: %s" % value)
-        except:
-                (type, value, traceback) = sys.exc_info()
-                raise bb.build.FuncFailed("Unknown fetch Error: %s" % value)
+        try:
+            fetcher = bb.fetch2.Fetch(src_uri, localdata)
+            fetcher.download()
+        except bb.fetch2.BBFetchException, e:
+            raise bb.build.FuncFailed(e)
 }
 
-def subprocess_setup():
-        import signal
-        # Python installs a SIGPIPE handler by default. This is usually not what
-        # non-Python subprocesses expect.
-        # SIGPIPE errors are known issues with gzip/bash
-        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
-
-def oe_unpack_file(file, data, url = None):
-        import subprocess
-        if not url:
-                url = "file://%s" % file
-        dots = file.split(".")
-        if dots[-1] in ['gz', 'bz2', 'Z']:
-                efile = os.path.join(bb.data.getVar('WORKDIR', data, 1),os.path.basename('.'.join(dots[0:-1])))
-        else:
-                efile = file
-        cmd = None
-        if file.endswith('.tar'):
-                cmd = 'tar x --no-same-owner -f %s' % file
-        elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-                cmd = 'tar xz --no-same-owner -f %s' % file
-        elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
-                cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
-        elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
-                cmd = 'gzip -dc %s > %s' % (file, efile)
-        elif file.endswith('.bz2'):
-                cmd = 'bzip2 -dc %s > %s' % (file, efile)
-        elif file.endswith('.tar.xz'):
-                cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
-        elif file.endswith('.xz'):
-                cmd = 'xz -dc %s > %s' % (file, efile)
-        elif file.endswith('.zip') or file.endswith('.jar'):
-                cmd = 'unzip -q -o'
-                (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-                if 'dos' in parm:
-                        cmd = '%s -a' % cmd
-                cmd = "%s '%s'" % (cmd, file)
-        elif os.path.isdir(file):
-                filesdir = os.path.realpath(bb.data.getVar("FILESDIR", data, 1))
-                destdir = "."
-                if file[0:len(filesdir)] == filesdir:
-                        destdir = file[len(filesdir):file.rfind('/')]
-                        destdir = destdir.strip('/')
-                        if len(destdir) < 1:
-                                destdir = "."
-                        elif not os.access("%s/%s" % (os.getcwd(), destdir), os.F_OK):
-                                os.makedirs("%s/%s" % (os.getcwd(), destdir))
-                cmd = 'cp -pPR %s %s/%s/' % (file, os.getcwd(), destdir)
-        else:
-                (type, host, path, user, pswd, parm) = bb.decodeurl(url)
-                if not 'patch' in parm:
-                        # The "destdir" handling was specifically done for FILESPATH
-                        # items.  So, only do so for file:// entries.
-                        if type == "file" and path.find("/") != -1:
-                                destdir = path.rsplit("/", 1)[0]
-                        else:
-                                destdir = "."
-                        bb.mkdirhier("%s/%s" % (os.getcwd(), destdir))
-                        cmd = 'cp %s %s/%s/' % (file, os.getcwd(), destdir)
-
-        if not cmd:
-                return True
-
-        dest = os.path.join(os.getcwd(), os.path.basename(file))
-        if os.path.exists(dest):
-                if os.path.samefile(file, dest):
-                        return True
-
-        # Change to subdir before executing command
-        save_cwd = os.getcwd();
-        parm = bb.decodeurl(url)[5]
-        if 'subdir' in parm:
-                newdir = ("%s/%s" % (os.getcwd(), parm['subdir']))
-                bb.mkdirhier(newdir)
-                os.chdir(newdir)
-
-        cmd = "PATH=\"%s\" %s" % (bb.data.getVar('PATH', data, 1), cmd)
-        bb.note("Unpacking %s to %s/" % (file, os.getcwd()))
-        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
-
-        os.chdir(save_cwd)
-
-        return ret == 0
-
 addtask unpack after do_fetch
 do_unpack[dirs] = "${WORKDIR}"
 python base_do_unpack() {
-        import re
+        src_uri = (bb.data.getVar('SRC_URI', d, True) or "").split()
+        if len(src_uri) == 0:
+                return
 
         localdata = bb.data.createCopy(d)
         bb.data.update_data(localdata)
 
-        urldata = bb.fetch.init([], localdata)
+        rootdir = bb.data.getVar('WORKDIR', localdata, True)
 
-        src_uri = bb.data.getVar('SRC_URI', localdata, True)
-        if not src_uri:
-                return
-        for url in src_uri.split():
-                try:
-                        local = bb.data.expand(bb.fetch.localpath(url, localdata), localdata)
-                except bb.MalformedUrl, e:
-                        raise FuncFailed('Unable to generate local path for malformed uri: %s' % e)
-                if local is None:
-                        continue
-                local = os.path.realpath(local)
-                lockfile = urldata[url].lockfile
-                if lockfile:
-                        lf = bb.utils.lockfile(urldata[url].lockfile)
-
-                if bb.fetch.__version__ == "1":
-                        ret = oe_unpack_file(local, localdata, url)
-                else:
-                        # use bb.fetch2 unpack API
-                        ud = urldata[url]
-                        rootdir = bb.data.getVar('WORKDIR', localdata, True)
-                        ret = ud.method.unpack(ud, rootdir, localdata)
-                if lockfile:
-                        bb.utils.unlockfile(lf)
-                if not ret:
-                        raise bb.build.FuncFailed("oe_unpack_file failed with return value %s" % ret)
+        try:
+            fetcher = bb.fetch2.Fetch(src_uri, localdata)
+            fetcher.unpack(rootdir)
+        except bb.fetch2.BBFetchException, e:
+            raise bb.build.FuncFailed(e)
 }
 
 GIT_CONFIG = "${STAGING_DIR_NATIVE}/usr/etc/gitconfig"
@@ -550,7 +425,8 @@ python () {
             for s in srcuri.split():
                 if not s.startswith("file://"):
                     continue
-                local = bb.data.expand(bb.fetch.localpath(s, d), d)
+                fetcher = bb.fetch2.Fetch([s], d)
+                local = fetcher.localpath(s)
                 for mp in paths:
                     if local.startswith(mp):
                         #bb.note("overriding PACKAGE_ARCH from %s to %s" % (pkg_arch, mach_arch))
@@ -594,14 +470,12 @@ python do_cleanall() {
         dl_dir = bb.data.getVar('DL_DIR', localdata, True)
         dl_dir = os.path.realpath(dl_dir)
 
-        src_uri = bb.data.getVar('SRC_URI', localdata, True)
-        if not src_uri:
+        src_uri = (bb.data.getVar('SRC_URI', localdata, True) or "").split()
+        if len(src_uri) == 0:
                 return
-        for url in src_uri.split():
-                try:
-                        local = bb.data.expand(bb.fetch.localpath(url, localdata), localdata)
-                except bb.MalformedUrl, e:
-                        raise FuncFailed('Unable to generate local path for malformed uri: %s' % e)
+        fetcher = bb.fetch2.Fetch(src_uri, localdata)
+        for url in src_uri:
+                local = fetcher.localpath(url)
                 if local is None:
                         continue
                 local = os.path.realpath(local)
diff --git a/meta/classes/patch.bbclass b/meta/classes/patch.bbclass
index ee8a2026fc..80fd45f0e3 100644
--- a/meta/classes/patch.bbclass
+++ b/meta/classes/patch.bbclass
@@ -58,9 +58,8 @@ python patch_do_patch() {
                         continue
 
                 if not local:
-                        bb.fetch.init([url],d)
                         url = bb.encodeurl((type, host, path, user, pswd, []))
-                        local = os.path.join('/', bb.fetch.localpath(url, d))
+                        local = os.path.join('/', bb.fetch2.localpath(url, d))
                 local = bb.data.expand(local, d)
 
                 if "striplevel" in parm:
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index a754821c08..e4564e4b07 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -326,40 +326,40 @@ def sstate_package(ss, d):
     return
 
 def pstaging_fetch(sstatepkg, d):
-    import bb.fetch
-
-    # only try and fetch if the user has configured a mirror
 
+    # Only try and fetch if the user has configured a mirror
     mirrors = bb.data.getVar('SSTATE_MIRRORS', d, True)
-    if mirrors:
-        # Copy the data object and override DL_DIR and SRC_URI
-        localdata = bb.data.createCopy(d)
-        bb.data.update_data(localdata)
+    if not mirrors:
+        return
 
-        dldir = bb.data.expand("${SSTATE_DIR}", localdata)
-        srcuri = "file://" + os.path.basename(sstatepkg)
+    import bb.fetch2
+    # Copy the data object and override DL_DIR and SRC_URI
+    localdata = bb.data.createCopy(d)
+    bb.data.update_data(localdata)
 
-        bb.mkdirhier(dldir)
+    dldir = bb.data.expand("${SSTATE_DIR}", localdata)
+    srcuri = "file://" + os.path.basename(sstatepkg)
 
-        bb.data.setVar('DL_DIR', dldir, localdata)
-        bb.data.setVar('PREMIRRORS', mirrors, localdata)
-        bb.data.setVar('SRC_URI', srcuri, localdata)
+    bb.mkdirhier(dldir)
 
-        # Try a fetch from the sstate mirror, if it fails just return and
-        # we will build the package
-        try:
-            bb.fetch.init([srcuri], localdata)
-            if bb.fetch.__version__ == "1":
-                bb.fetch.go(localdata, [srcuri])
-            else:
-                bb.fetch.download(localdata, [srcuri])
-            # Need to optimise this, if using file:// urls, the fetcher just changes the local path
-            # For now work around by symlinking
-            localpath = bb.data.expand(bb.fetch.localpath(srcuri, localdata), localdata)
-            if localpath != sstatepkg and os.path.exists(localpath):
-                os.symlink(localpath, sstatepkg)
-        except:
-            pass
+    bb.data.setVar('DL_DIR', dldir, localdata)
+    bb.data.setVar('PREMIRRORS', mirrors, localdata)
+    bb.data.setVar('SRC_URI', srcuri, localdata)
+
+    # Try a fetch from the sstate mirror, if it fails just return and
+    # we will build the package
+    try:
+        fetcher = bb.fetch2.Fetch([srcuri], localdata)
+        fetcher.download()        
+
+        # Need to optimise this, if using file:// urls, the fetcher just changes the local path
+        # For now work around by symlinking
+        localpath = bb.data.expand(fetcher.localpath(srcuri), localdata)
+        if localpath != sstatepkg and os.path.exists(localpath) and not os.path.exists(sstatepkg):
+            os.symlink(localpath, sstatepkg)
+
+    except bb.fetch2.BBFetchException:
+        pass
 
 def sstate_setscene(d):
     shared_state = sstate_state_fromvars(d)
diff --git a/meta/classes/utils.bbclass b/meta/classes/utils.bbclass
index 4d4b9953e2..455b49d54a 100644
--- a/meta/classes/utils.bbclass
+++ b/meta/classes/utils.bbclass
@@ -51,11 +51,12 @@ def machine_paths(d):
 def is_machine_specific(d):
     """Determine whether the current recipe is machine specific"""
     machinepaths = set(machine_paths(d))
-    urldatadict = bb.fetch.init(d.getVar("SRC_URI", True).split(), d, True)
-    for urldata in (urldata for urldata in urldatadict.itervalues()
-                    if urldata.type == "file"):
-        if any(urldata.localpath.startswith(mp + "/") for mp in machinepaths):
-            return True
+    srcuri = d.getVar("SRC_URI", True).split()
+    for url in srcuri:
+        fetcher = bb.fetch2.Fetch([srcuri], d)
+        if url.startswith("file://"):
+            if any(fetcher.localpath(url).startswith(mp + "/") for mp in machinepaths):
+                return True
 
 def oe_popen_env(d):
     env = d.getVar("__oe_popen_env", False)
diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py
index f203d683da..c8eeb8bc4c 100644
--- a/meta/lib/oe/patch.py
+++ b/meta/lib/oe/patch.py
@@ -72,14 +72,14 @@ class PatchSet(object):
             if not patch.get("remote"):
                 raise PatchError("Patch file must be specified in patch import.")
             else:
-                patch["file"] = bb.fetch.localpath(patch["remote"], self.d)
+                patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
 
         for param in PatchSet.defaults:
             if not patch.get(param):
                 patch[param] = PatchSet.defaults[param]
 
         if patch.get("remote"):
-            patch["file"] = bb.data.expand(bb.fetch.localpath(patch["remote"], self.d), self.d)
+            patch["file"] = bb.data.expand(bb.fetch2.localpath(patch["remote"], self.d), self.d)
 
         patch["filemd5"] = bb.utils.md5_file(patch["file"])
 
@@ -293,7 +293,7 @@ class QuiltTree(PatchSet):
             if type == "file":
                 import shutil
                 if not patch.get("file") and patch.get("remote"):
-                    patch["file"] = bb.fetch.localpath(patch["remote"], self.d)
+                    patch["file"] = bb.fetch2.localpath(patch["remote"], self.d)
 
                 shutil.copyfile(patch["quiltfile"], patch["file"])
             else: