From e4346e8be5f451d5cfe0b68a272abd15b0951831 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Thu, 20 Jul 2017 16:48:11 +0200 Subject: recipetool: create: reimplement fetching with normal fetch/unpack tasks Now that we have the ability to run the tasks in a more standard context through tinfoil, change recipetool's fetching code to use that to fetch files using it. This has the major advantage that any dependencies of do_fetch and do_unpack (e.g. for subversion or npm) will be handled automatically. This also has the beneficial side-effect of fixing a recent regression that prevented this fetch operation from working with memory resident bitbake. Also fix devtool's usage of fetch_uri() at the same time so that we can completely replace it. Fixes [YOCTO #11710]. (From OE-Core rev: 9a47a6690052ef943c0d4760630ee630fb012153) Signed-off-by: Paul Eggleton Signed-off-by: Richard Purdie --- scripts/lib/recipetool/create.py | 40 ++++++++++++++++++------------------ scripts/lib/recipetool/create_npm.py | 10 ++++----- 2 files changed, 25 insertions(+), 25 deletions(-) (limited to 'scripts/lib/recipetool') diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 2b7cc76ccb..2a6a28ba91 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py @@ -417,7 +417,7 @@ def create_recipe(args): pkgarch = "${MACHINE_ARCH}" extravalues = {} - checksums = (None, None) + checksums = {} tempsrc = '' source = args.source srcsubdir = '' @@ -439,22 +439,25 @@ def create_recipe(args): if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) - tempsrc = tempfile.mkdtemp(prefix='recipetool-') - srctree = tempsrc - d = bb.data.createCopy(tinfoil.config_data) - if fetchuri.startswith('npm://'): - # Check if npm is available - npm_bindir = check_npm(tinfoil, args.devtool) - d.prependVar('PATH', '%s:' % npm_bindir) - logger.info('Fetching %s...' % srcuri) + + tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') + bb.utils.mkdirhier(tmpparent) + tempsrc = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent) + srctree = os.path.join(tempsrc, 'source') + try: - checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev) - except bb.fetch2.BBFetchException as e: - logger.error(str(e).rstrip()) + checksums, ftmpdir = scriptutils.fetch_url(tinfoil, srcuri, srcrev, srctree, logger, preserve_tmp=args.keep_temp) + except scriptutils.FetchUrlFailure as e: + logger.error(str(e)) sys.exit(1) + + if ftmpdir and args.keep_temp: + logger.info('Fetch temp directory is %s' % ftmpdir) + dirlist = os.listdir(srctree) - if 'git.indirectionsymlink' in dirlist: - dirlist.remove('git.indirectionsymlink') + filterout = ['git.indirectionsymlink'] + dirlist = [x for x in dirlist if x not in filterout] + logger.debug('Directory listing (excluding filtered out):\n %s' % '\n '.join(dirlist)) if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): @@ -465,7 +468,7 @@ def create_recipe(args): check_single_file(dirlist[0], fetchuri) elif len(dirlist) == 0: if '/' in fetchuri: - fn = os.path.join(d.getVar('DL_DIR'), fetchuri.split('/')[-1]) + fn = os.path.join(tinfoil.config_data.getVar('DL_DIR'), fetchuri.split('/')[-1]) if os.path.isfile(fn): check_single_file(fn, fetchuri) # If we've got to here then there's no source so we might as well give up @@ -593,11 +596,8 @@ def create_recipe(args): if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) - (md5value, sha256value) = checksums - if md5value: - lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) - if sha256value: - lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) + for key, value in sorted(checksums.items()): + lines_before.append('SRC_URI[%s] = "%s"' % (key, value)) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py index cb8f338b8b..ba7e39a406 100644 --- a/scripts/lib/recipetool/create_npm.py +++ b/scripts/lib/recipetool/create_npm.py @@ -109,7 +109,6 @@ class NpmRecipeHandler(RecipeHandler): if varname == 'SRC_URI': if not origvalue.startswith('npm://'): src_uri = origvalue.split() - changed = False deplist = {} for dep, depver in optdeps.items(): depdata = self.get_npm_data(dep, depver, d) @@ -123,14 +122,15 @@ class NpmRecipeHandler(RecipeHandler): depdata = self.get_npm_data(dep, depver, d) deplist[dep] = depdata + extra_urls = [] for dep, depdata in deplist.items(): version = depdata.get('version', None) if version: url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) - scriptutils.fetch_uri(d, url, srctree) - src_uri.append(url) - changed = True - if changed: + extra_urls.append(url) + if extra_urls: + scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger) + src_uri.extend(extra_urls) return src_uri, None, -1, True return origvalue, None, 0, True updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) -- cgit v1.2.3-54-g00ecf