summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2017-04-12 22:41:27 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-04-13 10:54:10 +0100
commitbb8f141d0c2c668c4c98afe7760a3084756c92b9 (patch)
treeb75f31b6be5a0b4596deec63fadb75637f6a077c
parent0440cbccda83f34232bff9b450a2a45eda7ff596 (diff)
downloadpoky-bb8f141d0c2c668c4c98afe7760a3084756c92b9.tar.gz
devtool: add: fix node.js/npm handling with recipe specific sysroots
The change over to recipe specific sysroots means that we can no longer get a known location simply from configuration for the npm binary - we need to get the recipe sysroot for nodejs-native, look there for npm if we need to check it's present, and add that to PATH when calling out to npm. Unfortunately this means anywhere we need to get that path we have to have parsed all recipes, otherwise we have no reliable way of resolving nodejs-native. Thus we have to change recipetool create to always parse all recipes (the structure of the code does not allow us to do this conditionally). In the worst case, if npm hasn't already been added to its own sysroot and we are fetching from a source repository rather than an npm registry, this gets a bit ugly because we end up parsing recipes three times: 1) recipetool startup, which then fetches the code and determines it's a node.js module, finds that npm isn't available and then exits with a specific error to tell devtool it needs to build npm 2) when we invoke bitbake -c addto_recipe_sysroot nodejs-native 3) when we re-invoke recipetool This code is badly in need of refactoring, but now is unfortunately not the time to do that, so we're going to have to live with this ugliness for now. Fixes [YOCTO #10992]. (From OE-Core rev: acfdbd796c99882b8586023c8c6b848716105c8d) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--scripts/lib/devtool/__init__.py25
-rw-r--r--scripts/lib/devtool/standard.py2
-rw-r--r--scripts/lib/recipetool/create.py28
-rw-r--r--scripts/lib/recipetool/create_npm.py20
4 files changed, 48 insertions, 27 deletions
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 8c385beb89..d646b0cf63 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -261,23 +261,28 @@ def get_bbclassextend_targets(recipefile, pn):
261 targets.append('%s-%s' % (pn, variant)) 261 targets.append('%s-%s' % (pn, variant))
262 return targets 262 return targets
263 263
264def ensure_npm(config, basepath, fixed_setup=False): 264def ensure_npm(config, basepath, fixed_setup=False, check_exists=True):
265 """ 265 """
266 Ensure that npm is available and either build it or show a 266 Ensure that npm is available and either build it or show a
267 reasonable error message 267 reasonable error message
268 """ 268 """
269 tinfoil = setup_tinfoil(config_only=True, basepath=basepath) 269 if check_exists:
270 try: 270 tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
271 nativepath = tinfoil.config_data.getVar('STAGING_BINDIR_NATIVE') 271 try:
272 finally: 272 rd = tinfoil.parse_recipe('nodejs-native')
273 tinfoil.shutdown() 273 nativepath = rd.getVar('STAGING_BINDIR_NATIVE')
274 finally:
275 tinfoil.shutdown()
276 npmpath = os.path.join(nativepath, 'npm')
277 build_npm = not os.path.exists(npmpath)
278 else:
279 build_npm = True
274 280
275 npmpath = os.path.join(nativepath, 'npm') 281 if build_npm:
276 if not os.path.exists(npmpath):
277 logger.info('Building nodejs-native') 282 logger.info('Building nodejs-native')
278 try: 283 try:
279 exec_build_env_command(config.init_path, basepath, 284 exec_build_env_command(config.init_path, basepath,
280 'bitbake -q nodejs-native', watch=True) 285 'bitbake -q nodejs-native -c addto_recipe_sysroot', watch=True)
281 except bb.process.ExecutionError as e: 286 except bb.process.ExecutionError as e:
282 if "Nothing PROVIDES 'nodejs-native'" in e.stdout: 287 if "Nothing PROVIDES 'nodejs-native'" in e.stdout:
283 if fixed_setup: 288 if fixed_setup:
@@ -287,5 +292,3 @@ def ensure_npm(config, basepath, fixed_setup=False):
287 raise DevtoolError(msg) 292 raise DevtoolError(msg)
288 else: 293 else:
289 raise 294 raise
290 if not os.path.exists(npmpath):
291 raise DevtoolError('Built nodejs-native but npm binary still could not be found at %s' % npmpath)
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index ffca2c9ec4..73e629ca6f 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -165,7 +165,7 @@ def add(args, config, basepath, workspace):
165 # inside recipetool since recipetool keeps tinfoil active 165 # inside recipetool since recipetool keeps tinfoil active
166 # with references to it throughout the code, so we have 166 # with references to it throughout the code, so we have
167 # to exit out and come back here to do it. 167 # to exit out and come back here to do it.
168 ensure_npm(config, basepath, args.fixed_setup) 168 ensure_npm(config, basepath, args.fixed_setup, check_exists=False)
169 logger.info('Re-running recipe creation process after building nodejs') 169 logger.info('Re-running recipe creation process after building nodejs')
170 continue 170 continue
171 elif e.exitcode == 15: 171 elif e.exitcode == 15:
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 648f2d66fc..439dca0fcc 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -416,12 +416,14 @@ def create_recipe(args):
416 srcuri = rev_re.sub('', srcuri) 416 srcuri = rev_re.sub('', srcuri)
417 tempsrc = tempfile.mkdtemp(prefix='recipetool-') 417 tempsrc = tempfile.mkdtemp(prefix='recipetool-')
418 srctree = tempsrc 418 srctree = tempsrc
419 d = bb.data.createCopy(tinfoil.config_data)
419 if fetchuri.startswith('npm://'): 420 if fetchuri.startswith('npm://'):
420 # Check if npm is available 421 # Check if npm is available
421 check_npm(tinfoil.config_data, args.devtool) 422 npm_bindir = check_npm(tinfoil, args.devtool)
423 d.prependVar('PATH', '%s:' % npm_bindir)
422 logger.info('Fetching %s...' % srcuri) 424 logger.info('Fetching %s...' % srcuri)
423 try: 425 try:
424 checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) 426 checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev)
425 except bb.fetch2.BBFetchException as e: 427 except bb.fetch2.BBFetchException as e:
426 logger.error(str(e).rstrip()) 428 logger.error(str(e).rstrip())
427 sys.exit(1) 429 sys.exit(1)
@@ -1119,10 +1121,21 @@ def convert_rpm_xml(xmlfile):
1119 return values 1121 return values
1120 1122
1121 1123
1122def check_npm(d, debugonly=False): 1124def check_npm(tinfoil, debugonly=False):
1123 if not os.path.exists(os.path.join(d.getVar('STAGING_BINDIR_NATIVE'), 'npm')): 1125 try:
1124 log_error_cond('npm required to process specified source, but npm is not available - you need to build nodejs-native first', debugonly) 1126 rd = tinfoil.parse_recipe('nodejs-native')
1127 except bb.providers.NoProvider:
1128 # We still conditionally show the message and exit with the special
1129 # return code, otherwise we can't show the proper message for eSDK
1130 # users
1131 log_error_cond('nodejs-native is required for npm but is not available - you will likely need to add a layer that provides nodejs', debugonly)
1132 sys.exit(14)
1133 bindir = rd.getVar('STAGING_BINDIR_NATIVE')
1134 npmpath = os.path.join(bindir, 'npm')
1135 if not os.path.exists(npmpath):
1136 log_error_cond('npm required to process specified source, but npm is not available - you need to run bitbake -c addto_recipe_sysroot nodejs-native first', debugonly)
1125 sys.exit(14) 1137 sys.exit(14)
1138 return bindir
1126 1139
1127def register_commands(subparsers): 1140def register_commands(subparsers):
1128 parser_create = subparsers.add_parser('create', 1141 parser_create = subparsers.add_parser('create',
@@ -1141,5 +1154,8 @@ def register_commands(subparsers):
1141 parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 1154 parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
1142 parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies') 1155 parser_create.add_argument('--fetch-dev', action="store_true", help='For npm, also fetch devDependencies')
1143 parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS) 1156 parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
1144 parser_create.set_defaults(func=create_recipe) 1157 # FIXME I really hate having to set parserecipes for this, but given we may need
1158 # to call into npm (and we don't know in advance if we will or not) and in order
1159 # to do so we need to know npm's recipe sysroot path, there's not much alternative
1160 parser_create.set_defaults(func=create_recipe, parserecipes=True)
1145 1161
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index eb19555a8e..a79a9afbb1 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -65,9 +65,9 @@ class NpmRecipeHandler(RecipeHandler):
65 'SEE-LICENSE-IN-EULA') 65 'SEE-LICENSE-IN-EULA')
66 return license 66 return license
67 67
68 def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before): 68 def _shrinkwrap(self, srctree, localfilesdir, extravalues, lines_before, d):
69 try: 69 try:
70 runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH')) 70 runenv = dict(os.environ, PATH=d.getVar('PATH'))
71 bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True) 71 bb.process.run('npm shrinkwrap', cwd=srctree, stderr=subprocess.STDOUT, env=runenv, shell=True)
72 except bb.process.ExecutionError as e: 72 except bb.process.ExecutionError as e:
73 logger.warn('npm shrinkwrap failed:\n%s' % e.stdout) 73 logger.warn('npm shrinkwrap failed:\n%s' % e.stdout)
@@ -79,8 +79,8 @@ class NpmRecipeHandler(RecipeHandler):
79 extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile 79 extravalues['extrafiles']['npm-shrinkwrap.json'] = tmpfile
80 lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"') 80 lines_before.append('NPM_SHRINKWRAP := "${THISDIR}/${PN}/npm-shrinkwrap.json"')
81 81
82 def _lockdown(self, srctree, localfilesdir, extravalues, lines_before): 82 def _lockdown(self, srctree, localfilesdir, extravalues, lines_before, d):
83 runenv = dict(os.environ, PATH=tinfoil.config_data.getVar('PATH')) 83 runenv = dict(os.environ, PATH=d.getVar('PATH'))
84 if not NpmRecipeHandler.lockdownpath: 84 if not NpmRecipeHandler.lockdownpath:
85 NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown') 85 NpmRecipeHandler.lockdownpath = tempfile.mkdtemp('recipetool-npm-lockdown')
86 bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath, 86 bb.process.run('npm install lockdown --prefix %s' % NpmRecipeHandler.lockdownpath,
@@ -188,7 +188,9 @@ class NpmRecipeHandler(RecipeHandler):
188 188
189 files = RecipeHandler.checkfiles(srctree, ['package.json']) 189 files = RecipeHandler.checkfiles(srctree, ['package.json'])
190 if files: 190 if files:
191 check_npm(tinfoil.config_data) 191 d = bb.data.createCopy(tinfoil.config_data)
192 npm_bindir = check_npm(tinfoil)
193 d.prependVar('PATH', '%s:' % npm_bindir)
192 194
193 data = read_package_json(files[0]) 195 data = read_package_json(files[0])
194 if 'name' in data and 'version' in data: 196 if 'name' in data and 'version' in data:
@@ -203,17 +205,17 @@ class NpmRecipeHandler(RecipeHandler):
203 205
204 fetchdev = extravalues['fetchdev'] or None 206 fetchdev = extravalues['fetchdev'] or None
205 deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev) 207 deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
206 updated = self._handle_dependencies(tinfoil.config_data, deps, optdeps, devdeps, lines_before, srctree) 208 updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
207 if updated: 209 if updated:
208 # We need to redo the license stuff 210 # We need to redo the license stuff
209 self._replace_license_vars(srctree, lines_before, handled, extravalues, tinfoil.config_data) 211 self._replace_license_vars(srctree, lines_before, handled, extravalues, d)
210 212
211 # Shrinkwrap 213 # Shrinkwrap
212 localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') 214 localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
213 self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before) 215 self._shrinkwrap(srctree, localfilesdir, extravalues, lines_before, d)
214 216
215 # Lockdown 217 # Lockdown
216 self._lockdown(srctree, localfilesdir, extravalues, lines_before) 218 self._lockdown(srctree, localfilesdir, extravalues, lines_before, d)
217 219
218 # Split each npm module out to is own package 220 # Split each npm module out to is own package
219 npmpackages = oe.package.npm_split_package_dirs(srctree) 221 npmpackages = oe.package.npm_split_package_dirs(srctree)