recipetool: create: reimplement fetching with normal fetch/unpack tasks

Now that we have the ability to run the tasks in a more standard context
through tinfoil, change recipetool's fetching code to use that to fetch
files using it. This has the major advantage that any dependencies of
do_fetch and do_unpack (e.g. for subversion or npm) will be handled
automatically. This also has the beneficial side-effect of fixing a
recent regression that prevented this fetch operation from working with
memory resident bitbake.

Also fix devtool's usage of fetch_uri() at the same time so that we can
completely replace it.

Fixes [YOCTO #11710].

(From OE-Core rev: 9a47a6690052ef943c0d4760630ee630fb012153)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 120 insertions, 38 deletions

diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index 92b601c7e8..1005dd495a 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -23,6 +23,8 @@ import argparse
 import subprocess
 import tempfile
 import shutil
+import random
+import string
 
 def logger_create(name, stream=None):
     logger = logging.getLogger(name)
@@ -78,50 +80,130 @@ def git_convert_standalone_clone(repodir):
             bb.process.run('git repack -a', cwd=repodir)
             os.remove(alternatesfile)
 
-def fetch_uri(d, uri, destdir, srcrev=None):
-    """Fetch a URI to a local directory"""
+def _get_temp_recipe_dir(d):
+    # This is a little bit hacky but we need to find a place where we can put
+    # the recipe so that bitbake can find it. We're going to delete it at the
+    # end so it doesn't really matter where we put it.
+    bbfiles = d.getVar('BBFILES').split()
+    fetchrecipedir = None
+    for pth in bbfiles:
+        if pth.endswith('.bb'):
+            pthdir = os.path.dirname(pth)
+            if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK):
+                fetchrecipedir = pthdir.replace('*', 'recipetool')
+                if pthdir.endswith('workspace/recipes/*'):
+                    # Prefer the workspace
+                    break
+    return fetchrecipedir
+
+class FetchUrlFailure(Exception):
+    def __init__(self, url):
+        self.url = url
+    def __str__(self):
+        return "Failed to fetch URL %s" % self.url
+
+def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False):
+    """
+    Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e.
+    any dependencies that need to be satisfied in order to support the fetch
+    operation will be taken care of
+    """
+
     import bb
-    tmpparent = d.getVar('BASE_WORKDIR')
+
+    checksums = {}
+    fetchrecipepn = None
+
+    # We need to put our temp directory under ${BASE_WORKDIR} otherwise
+    # we may have problems with the recipe-specific sysroot population
+    tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
     bb.utils.mkdirhier(tmpparent)
-    tmpworkdir = tempfile.mkdtemp(dir=tmpparent)
+    tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
     try:
-        bb.utils.mkdirhier(destdir)
-        localdata = bb.data.createCopy(d)
-
-        # Set some values to allow extend_recipe_sysroot to work here we're we are not running from a task
-        localdata.setVar('WORKDIR', tmpworkdir)
-        localdata.setVar('BB_RUNTASK', 'do_fetch')
-        localdata.setVar('PN', 'dummy')
-        localdata.setVar('BB_LIMITEDDEPS', '1')
-        bb.build.exec_func("extend_recipe_sysroot", localdata)
-
-        # Set some values for the benefit of the fetcher code
-        localdata.setVar('BB_STRICT_CHECKSUM', '')
-        localdata.setVar('SRCREV', srcrev)
-        ret = (None, None)
-        olddir = os.getcwd()
+        tmpworkdir = os.path.join(tmpdir, 'work')
+        logger.debug('fetch_url: temp dir is %s' % tmpdir)
+
+        fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data)
+        if not fetchrecipedir:
+            logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe')
+            sys.exit(1)
+        fetchrecipe = None
+        bb.utils.mkdirhier(fetchrecipedir)
         try:
-            fetcher = bb.fetch2.Fetch([uri], localdata)
-            for u in fetcher.ud:
-                ud = fetcher.ud[u]
-                ud.ignore_checksums = True
-            fetcher.download()
-            for u in fetcher.ud:
-                ud = fetcher.ud[u]
-                if ud.localpath.rstrip(os.sep) == localdata.getVar('DL_DIR').rstrip(os.sep):
-                    raise Exception('Local path is download directory - please check that the URI "%s" is correct' % uri)
-            fetcher.unpack(destdir)
-            for u in fetcher.ud:
-                ud = fetcher.ud[u]
-                if ud.method.recommends_checksum(ud):
-                    md5value = bb.utils.md5_file(ud.localpath)
-                    sha256value = bb.utils.sha256_file(ud.localpath)
-                    ret = (md5value, sha256value)
+            # Generate a dummy recipe so we can follow more or less normal paths
+            # for do_fetch and do_unpack
+            # I'd use tempfile functions here but underscores can be produced by that and those
+            # aren't allowed in recipe file names except to separate the version
+            rndstring = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8))
+            fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring)
+            fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0]
+            logger.debug('Generating initial recipe %s for fetching' % fetchrecipe)
+            with open(fetchrecipe, 'w') as f:
+                # We don't want to have to specify LIC_FILES_CHKSUM
+                f.write('LICENSE = "CLOSED"\n')
+                # We don't need the cross-compiler
+                f.write('INHIBIT_DEFAULT_DEPS = "1"\n')
+                # We don't have the checksums yet so we can't require them
+                f.write('BB_STRICT_CHECKSUM = "ignore"\n')
+                f.write('SRC_URI = "%s"\n' % srcuri)
+                f.write('SRCREV = "%s"\n' % srcrev)
+                f.write('WORKDIR = "%s"\n' % tmpworkdir)
+                # Set S out of the way so it doesn't get created under the workdir
+                f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
+
+            logger.info('Fetching %s...' % srcuri)
+
+            # FIXME this is too noisy at the moment
+
+            # Parse recipes so our new recipe gets picked up
+            tinfoil.parse_recipes()
+
+            def eventhandler(event):
+                if isinstance(event, bb.fetch2.MissingChecksumEvent):
+                    checksums.update(event.checksums)
+                    return True
+                return False
+
+            # Run the fetch + unpack tasks
+            res = tinfoil.build_targets(fetchrecipepn,
+                                        'do_unpack',
+                                        handle_events=True,
+                                        extra_events=['bb.fetch2.MissingChecksumEvent'],
+                                        event_callback=eventhandler)
+            if not res:
+                raise FetchUrlFailure(srcuri)
+
+            # Remove unneeded directories
+            rd = tinfoil.parse_recipe(fetchrecipepn)
+            if rd:
+                pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
+                for pathvar in pathvars:
+                    path = rd.getVar(pathvar)
+                    shutil.rmtree(path)
         finally:
-            os.chdir(olddir)
+            if fetchrecipe:
+                try:
+                    os.remove(fetchrecipe)
+                except FileNotFoundError:
+                    pass
+            try:
+                os.rmdir(fetchrecipedir)
+            except OSError as e:
+                import errno
+                if e.errno != errno.ENOTEMPTY:
+                    raise
+
+        bb.utils.mkdirhier(destdir)
+        for fn in os.listdir(tmpworkdir):
+            shutil.move(os.path.join(tmpworkdir, fn), destdir)
+
     finally:
-        shutil.rmtree(tmpworkdir)
-    return ret
+        if not preserve_tmp:
+            shutil.rmtree(tmpdir)
+            tmpdir = None
+
+    return checksums, tmpdir
+
 
 def run_editor(fn):
     if isinstance(fn, str):