summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/devtool/upgrade.py10
-rw-r--r--scripts/lib/recipetool/create.py40
-rw-r--r--scripts/lib/recipetool/create_npm.py10
-rw-r--r--scripts/lib/scriptutils.py158
4 files changed, 153 insertions, 65 deletions
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 05fb9e5ed0..f077f37726 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -207,10 +207,16 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tin
207 207
208 tmpdir = tempfile.mkdtemp(prefix='devtool') 208 tmpdir = tempfile.mkdtemp(prefix='devtool')
209 try: 209 try:
210 md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) 210 checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp)
211 except bb.fetch2.FetchError as e: 211 except scriptutils.FetchUrlFailure as e:
212 raise DevtoolError(e) 212 raise DevtoolError(e)
213 213
214 if ftmpdir and keep_temp:
215 logger.info('Fetch temp directory is %s' % ftmpdir)
216
217 md5 = checksums['md5sum']
218 sha256 = checksums['sha256sum']
219
214 tmpsrctree = _get_srctree(tmpdir) 220 tmpsrctree = _get_srctree(tmpdir)
215 srctree = os.path.abspath(srctree) 221 srctree = os.path.abspath(srctree)
216 222
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 2b7cc76ccb..2a6a28ba91 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -417,7 +417,7 @@ def create_recipe(args):
417 pkgarch = "${MACHINE_ARCH}" 417 pkgarch = "${MACHINE_ARCH}"
418 418
419 extravalues = {} 419 extravalues = {}
420 checksums = (None, None) 420 checksums = {}
421 tempsrc = '' 421 tempsrc = ''
422 source = args.source 422 source = args.source
423 srcsubdir = '' 423 srcsubdir = ''
@@ -439,22 +439,25 @@ def create_recipe(args):
439 if res: 439 if res:
440 srcrev = res.group(1) 440 srcrev = res.group(1)
441 srcuri = rev_re.sub('', srcuri) 441 srcuri = rev_re.sub('', srcuri)
442 tempsrc = tempfile.mkdtemp(prefix='recipetool-') 442
443 srctree = tempsrc 443 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
444 d = bb.data.createCopy(tinfoil.config_data) 444 bb.utils.mkdirhier(tmpparent)
445 if fetchuri.startswith('npm://'): 445 tempsrc = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
446 # Check if npm is available 446 srctree = os.path.join(tempsrc, 'source')
447 npm_bindir = check_npm(tinfoil, args.devtool) 447
448 d.prependVar('PATH', '%s:' % npm_bindir)
449 logger.info('Fetching %s...' % srcuri)
450 try: 448 try:
451 checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev) 449 checksums, ftmpdir = scriptutils.fetch_url(tinfoil, srcuri, srcrev, srctree, logger, preserve_tmp=args.keep_temp)
452 except bb.fetch2.BBFetchException as e: 450 except scriptutils.FetchUrlFailure as e:
453 logger.error(str(e).rstrip()) 451 logger.error(str(e))
454 sys.exit(1) 452 sys.exit(1)
453
454 if ftmpdir and args.keep_temp:
455 logger.info('Fetch temp directory is %s' % ftmpdir)
456
455 dirlist = os.listdir(srctree) 457 dirlist = os.listdir(srctree)
456 if 'git.indirectionsymlink' in dirlist: 458 filterout = ['git.indirectionsymlink']
457 dirlist.remove('git.indirectionsymlink') 459 dirlist = [x for x in dirlist if x not in filterout]
460 logger.debug('Directory listing (excluding filtered out):\n %s' % '\n '.join(dirlist))
458 if len(dirlist) == 1: 461 if len(dirlist) == 1:
459 singleitem = os.path.join(srctree, dirlist[0]) 462 singleitem = os.path.join(srctree, dirlist[0])
460 if os.path.isdir(singleitem): 463 if os.path.isdir(singleitem):
@@ -465,7 +468,7 @@ def create_recipe(args):
465 check_single_file(dirlist[0], fetchuri) 468 check_single_file(dirlist[0], fetchuri)
466 elif len(dirlist) == 0: 469 elif len(dirlist) == 0:
467 if '/' in fetchuri: 470 if '/' in fetchuri:
468 fn = os.path.join(d.getVar('DL_DIR'), fetchuri.split('/')[-1]) 471 fn = os.path.join(tinfoil.config_data.getVar('DL_DIR'), fetchuri.split('/')[-1])
469 if os.path.isfile(fn): 472 if os.path.isfile(fn):
470 check_single_file(fn, fetchuri) 473 check_single_file(fn, fetchuri)
471 # If we've got to here then there's no source so we might as well give up 474 # If we've got to here then there's no source so we might as well give up
@@ -593,11 +596,8 @@ def create_recipe(args):
593 if not srcuri: 596 if not srcuri:
594 lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') 597 lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
595 lines_before.append('SRC_URI = "%s"' % srcuri) 598 lines_before.append('SRC_URI = "%s"' % srcuri)
596 (md5value, sha256value) = checksums 599 for key, value in sorted(checksums.items()):
597 if md5value: 600 lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
598 lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
599 if sha256value:
600 lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
601 if srcuri and supports_srcrev(srcuri): 601 if srcuri and supports_srcrev(srcuri):
602 lines_before.append('') 602 lines_before.append('')
603 lines_before.append('# Modify these as desired') 603 lines_before.append('# Modify these as desired')
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index cb8f338b8b..ba7e39a406 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -109,7 +109,6 @@ class NpmRecipeHandler(RecipeHandler):
109 if varname == 'SRC_URI': 109 if varname == 'SRC_URI':
110 if not origvalue.startswith('npm://'): 110 if not origvalue.startswith('npm://'):
111 src_uri = origvalue.split() 111 src_uri = origvalue.split()
112 changed = False
113 deplist = {} 112 deplist = {}
114 for dep, depver in optdeps.items(): 113 for dep, depver in optdeps.items():
115 depdata = self.get_npm_data(dep, depver, d) 114 depdata = self.get_npm_data(dep, depver, d)
@@ -123,14 +122,15 @@ class NpmRecipeHandler(RecipeHandler):
123 depdata = self.get_npm_data(dep, depver, d) 122 depdata = self.get_npm_data(dep, depver, d)
124 deplist[dep] = depdata 123 deplist[dep] = depdata
125 124
125 extra_urls = []
126 for dep, depdata in deplist.items(): 126 for dep, depdata in deplist.items():
127 version = depdata.get('version', None) 127 version = depdata.get('version', None)
128 if version: 128 if version:
129 url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) 129 url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep)
130 scriptutils.fetch_uri(d, url, srctree) 130 extra_urls.append(url)
131 src_uri.append(url) 131 if extra_urls:
132 changed = True 132 scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger)
133 if changed: 133 src_uri.extend(extra_urls)
134 return src_uri, None, -1, True 134 return src_uri, None, -1, True
135 return origvalue, None, 0, True 135 return origvalue, None, 0, True
136 updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc) 136 updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index 92b601c7e8..1005dd495a 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -23,6 +23,8 @@ import argparse
23import subprocess 23import subprocess
24import tempfile 24import tempfile
25import shutil 25import shutil
26import random
27import string
26 28
27def logger_create(name, stream=None): 29def logger_create(name, stream=None):
28 logger = logging.getLogger(name) 30 logger = logging.getLogger(name)
@@ -78,50 +80,130 @@ def git_convert_standalone_clone(repodir):
78 bb.process.run('git repack -a', cwd=repodir) 80 bb.process.run('git repack -a', cwd=repodir)
79 os.remove(alternatesfile) 81 os.remove(alternatesfile)
80 82
81def fetch_uri(d, uri, destdir, srcrev=None): 83def _get_temp_recipe_dir(d):
82 """Fetch a URI to a local directory""" 84 # This is a little bit hacky but we need to find a place where we can put
85 # the recipe so that bitbake can find it. We're going to delete it at the
86 # end so it doesn't really matter where we put it.
87 bbfiles = d.getVar('BBFILES').split()
88 fetchrecipedir = None
89 for pth in bbfiles:
90 if pth.endswith('.bb'):
91 pthdir = os.path.dirname(pth)
92 if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK):
93 fetchrecipedir = pthdir.replace('*', 'recipetool')
94 if pthdir.endswith('workspace/recipes/*'):
95 # Prefer the workspace
96 break
97 return fetchrecipedir
98
99class FetchUrlFailure(Exception):
100 def __init__(self, url):
101 self.url = url
102 def __str__(self):
103 return "Failed to fetch URL %s" % self.url
104
105def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False):
106 """
107 Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e.
108 any dependencies that need to be satisfied in order to support the fetch
109 operation will be taken care of
110 """
111
83 import bb 112 import bb
84 tmpparent = d.getVar('BASE_WORKDIR') 113
114 checksums = {}
115 fetchrecipepn = None
116
117 # We need to put our temp directory under ${BASE_WORKDIR} otherwise
118 # we may have problems with the recipe-specific sysroot population
119 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
85 bb.utils.mkdirhier(tmpparent) 120 bb.utils.mkdirhier(tmpparent)
86 tmpworkdir = tempfile.mkdtemp(dir=tmpparent) 121 tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
87 try: 122 try:
88 bb.utils.mkdirhier(destdir) 123 tmpworkdir = os.path.join(tmpdir, 'work')
89 localdata = bb.data.createCopy(d) 124 logger.debug('fetch_url: temp dir is %s' % tmpdir)
90 125
91 # Set some values to allow extend_recipe_sysroot to work here we're we are not running from a task 126 fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data)
92 localdata.setVar('WORKDIR', tmpworkdir) 127 if not fetchrecipedir:
93 localdata.setVar('BB_RUNTASK', 'do_fetch') 128 logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe')
94 localdata.setVar('PN', 'dummy') 129 sys.exit(1)
95 localdata.setVar('BB_LIMITEDDEPS', '1') 130 fetchrecipe = None
96 bb.build.exec_func("extend_recipe_sysroot", localdata) 131 bb.utils.mkdirhier(fetchrecipedir)
97
98 # Set some values for the benefit of the fetcher code
99 localdata.setVar('BB_STRICT_CHECKSUM', '')
100 localdata.setVar('SRCREV', srcrev)
101 ret = (None, None)
102 olddir = os.getcwd()
103 try: 132 try:
104 fetcher = bb.fetch2.Fetch([uri], localdata) 133 # Generate a dummy recipe so we can follow more or less normal paths
105 for u in fetcher.ud: 134 # for do_fetch and do_unpack
106 ud = fetcher.ud[u] 135 # I'd use tempfile functions here but underscores can be produced by that and those
107 ud.ignore_checksums = True 136 # aren't allowed in recipe file names except to separate the version
108 fetcher.download() 137 rndstring = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8))
109 for u in fetcher.ud: 138 fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring)
110 ud = fetcher.ud[u] 139 fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0]
111 if ud.localpath.rstrip(os.sep) == localdata.getVar('DL_DIR').rstrip(os.sep): 140 logger.debug('Generating initial recipe %s for fetching' % fetchrecipe)
112 raise Exception('Local path is download directory - please check that the URI "%s" is correct' % uri) 141 with open(fetchrecipe, 'w') as f:
113 fetcher.unpack(destdir) 142 # We don't want to have to specify LIC_FILES_CHKSUM
114 for u in fetcher.ud: 143 f.write('LICENSE = "CLOSED"\n')
115 ud = fetcher.ud[u] 144 # We don't need the cross-compiler
116 if ud.method.recommends_checksum(ud): 145 f.write('INHIBIT_DEFAULT_DEPS = "1"\n')
117 md5value = bb.utils.md5_file(ud.localpath) 146 # We don't have the checksums yet so we can't require them
118 sha256value = bb.utils.sha256_file(ud.localpath) 147 f.write('BB_STRICT_CHECKSUM = "ignore"\n')
119 ret = (md5value, sha256value) 148 f.write('SRC_URI = "%s"\n' % srcuri)
149 f.write('SRCREV = "%s"\n' % srcrev)
150 f.write('WORKDIR = "%s"\n' % tmpworkdir)
151 # Set S out of the way so it doesn't get created under the workdir
152 f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
153
154 logger.info('Fetching %s...' % srcuri)
155
156 # FIXME this is too noisy at the moment
157
158 # Parse recipes so our new recipe gets picked up
159 tinfoil.parse_recipes()
160
161 def eventhandler(event):
162 if isinstance(event, bb.fetch2.MissingChecksumEvent):
163 checksums.update(event.checksums)
164 return True
165 return False
166
167 # Run the fetch + unpack tasks
168 res = tinfoil.build_targets(fetchrecipepn,
169 'do_unpack',
170 handle_events=True,
171 extra_events=['bb.fetch2.MissingChecksumEvent'],
172 event_callback=eventhandler)
173 if not res:
174 raise FetchUrlFailure(srcuri)
175
176 # Remove unneeded directories
177 rd = tinfoil.parse_recipe(fetchrecipepn)
178 if rd:
179 pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
180 for pathvar in pathvars:
181 path = rd.getVar(pathvar)
182 shutil.rmtree(path)
120 finally: 183 finally:
121 os.chdir(olddir) 184 if fetchrecipe:
185 try:
186 os.remove(fetchrecipe)
187 except FileNotFoundError:
188 pass
189 try:
190 os.rmdir(fetchrecipedir)
191 except OSError as e:
192 import errno
193 if e.errno != errno.ENOTEMPTY:
194 raise
195
196 bb.utils.mkdirhier(destdir)
197 for fn in os.listdir(tmpworkdir):
198 shutil.move(os.path.join(tmpworkdir, fn), destdir)
199
122 finally: 200 finally:
123 shutil.rmtree(tmpworkdir) 201 if not preserve_tmp:
124 return ret 202 shutil.rmtree(tmpdir)
203 tmpdir = None
204
205 return checksums, tmpdir
206
125 207
126def run_editor(fn): 208def run_editor(fn):
127 if isinstance(fn, str): 209 if isinstance(fn, str):