summaryrefslogtreecommitdiffstats
path: root/scripts/lib/scriptutils.py
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2017-07-20 16:48:11 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-07-21 08:44:25 +0100
commite4346e8be5f451d5cfe0b68a272abd15b0951831 (patch)
treefee60446d4319e5e771c2e809fc8a21278d00983 /scripts/lib/scriptutils.py
parent7d474e27bc851b22b7c00ace66ac40ea45588054 (diff)
downloadpoky-e4346e8be5f451d5cfe0b68a272abd15b0951831.tar.gz
recipetool: create: reimplement fetching with normal fetch/unpack tasks
Now that we have the ability to run the tasks in a more standard context through tinfoil, change recipetool's fetching code to use that to fetch files using it. This has the major advantage that any dependencies of do_fetch and do_unpack (e.g. for subversion or npm) will be handled automatically. This also has the beneficial side-effect of fixing a recent regression that prevented this fetch operation from working with memory resident bitbake. Also fix devtool's usage of fetch_uri() at the same time so that we can completely replace it. Fixes [YOCTO #11710]. (From OE-Core rev: 9a47a6690052ef943c0d4760630ee630fb012153) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/scriptutils.py')
-rw-r--r--scripts/lib/scriptutils.py158
1 files changed, 120 insertions, 38 deletions
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index 92b601c7e8..1005dd495a 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -23,6 +23,8 @@ import argparse
23import subprocess 23import subprocess
24import tempfile 24import tempfile
25import shutil 25import shutil
26import random
27import string
26 28
27def logger_create(name, stream=None): 29def logger_create(name, stream=None):
28 logger = logging.getLogger(name) 30 logger = logging.getLogger(name)
@@ -78,50 +80,130 @@ def git_convert_standalone_clone(repodir):
78 bb.process.run('git repack -a', cwd=repodir) 80 bb.process.run('git repack -a', cwd=repodir)
79 os.remove(alternatesfile) 81 os.remove(alternatesfile)
80 82
81def fetch_uri(d, uri, destdir, srcrev=None): 83def _get_temp_recipe_dir(d):
82 """Fetch a URI to a local directory""" 84 # This is a little bit hacky but we need to find a place where we can put
85 # the recipe so that bitbake can find it. We're going to delete it at the
86 # end so it doesn't really matter where we put it.
87 bbfiles = d.getVar('BBFILES').split()
88 fetchrecipedir = None
89 for pth in bbfiles:
90 if pth.endswith('.bb'):
91 pthdir = os.path.dirname(pth)
92 if os.access(os.path.dirname(os.path.dirname(pthdir)), os.W_OK):
93 fetchrecipedir = pthdir.replace('*', 'recipetool')
94 if pthdir.endswith('workspace/recipes/*'):
95 # Prefer the workspace
96 break
97 return fetchrecipedir
98
99class FetchUrlFailure(Exception):
100 def __init__(self, url):
101 self.url = url
102 def __str__(self):
103 return "Failed to fetch URL %s" % self.url
104
105def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False):
106 """
107 Fetch the specified URL using normal do_fetch and do_unpack tasks, i.e.
108 any dependencies that need to be satisfied in order to support the fetch
109 operation will be taken care of
110 """
111
83 import bb 112 import bb
84 tmpparent = d.getVar('BASE_WORKDIR') 113
114 checksums = {}
115 fetchrecipepn = None
116
117 # We need to put our temp directory under ${BASE_WORKDIR} otherwise
118 # we may have problems with the recipe-specific sysroot population
119 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
85 bb.utils.mkdirhier(tmpparent) 120 bb.utils.mkdirhier(tmpparent)
86 tmpworkdir = tempfile.mkdtemp(dir=tmpparent) 121 tmpdir = tempfile.mkdtemp(prefix='recipetool-', dir=tmpparent)
87 try: 122 try:
88 bb.utils.mkdirhier(destdir) 123 tmpworkdir = os.path.join(tmpdir, 'work')
89 localdata = bb.data.createCopy(d) 124 logger.debug('fetch_url: temp dir is %s' % tmpdir)
90 125
91 # Set some values to allow extend_recipe_sysroot to work here we're we are not running from a task 126 fetchrecipedir = _get_temp_recipe_dir(tinfoil.config_data)
92 localdata.setVar('WORKDIR', tmpworkdir) 127 if not fetchrecipedir:
93 localdata.setVar('BB_RUNTASK', 'do_fetch') 128 logger.error('Searched BBFILES but unable to find a writeable place to put temporary recipe')
94 localdata.setVar('PN', 'dummy') 129 sys.exit(1)
95 localdata.setVar('BB_LIMITEDDEPS', '1') 130 fetchrecipe = None
96 bb.build.exec_func("extend_recipe_sysroot", localdata) 131 bb.utils.mkdirhier(fetchrecipedir)
97
98 # Set some values for the benefit of the fetcher code
99 localdata.setVar('BB_STRICT_CHECKSUM', '')
100 localdata.setVar('SRCREV', srcrev)
101 ret = (None, None)
102 olddir = os.getcwd()
103 try: 132 try:
104 fetcher = bb.fetch2.Fetch([uri], localdata) 133 # Generate a dummy recipe so we can follow more or less normal paths
105 for u in fetcher.ud: 134 # for do_fetch and do_unpack
106 ud = fetcher.ud[u] 135 # I'd use tempfile functions here but underscores can be produced by that and those
107 ud.ignore_checksums = True 136 # aren't allowed in recipe file names except to separate the version
108 fetcher.download() 137 rndstring = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8))
109 for u in fetcher.ud: 138 fetchrecipe = os.path.join(fetchrecipedir, 'tmp-recipetool-%s.bb' % rndstring)
110 ud = fetcher.ud[u] 139 fetchrecipepn = os.path.splitext(os.path.basename(fetchrecipe))[0]
111 if ud.localpath.rstrip(os.sep) == localdata.getVar('DL_DIR').rstrip(os.sep): 140 logger.debug('Generating initial recipe %s for fetching' % fetchrecipe)
112 raise Exception('Local path is download directory - please check that the URI "%s" is correct' % uri) 141 with open(fetchrecipe, 'w') as f:
113 fetcher.unpack(destdir) 142 # We don't want to have to specify LIC_FILES_CHKSUM
114 for u in fetcher.ud: 143 f.write('LICENSE = "CLOSED"\n')
115 ud = fetcher.ud[u] 144 # We don't need the cross-compiler
116 if ud.method.recommends_checksum(ud): 145 f.write('INHIBIT_DEFAULT_DEPS = "1"\n')
117 md5value = bb.utils.md5_file(ud.localpath) 146 # We don't have the checksums yet so we can't require them
118 sha256value = bb.utils.sha256_file(ud.localpath) 147 f.write('BB_STRICT_CHECKSUM = "ignore"\n')
119 ret = (md5value, sha256value) 148 f.write('SRC_URI = "%s"\n' % srcuri)
149 f.write('SRCREV = "%s"\n' % srcrev)
150 f.write('WORKDIR = "%s"\n' % tmpworkdir)
151 # Set S out of the way so it doesn't get created under the workdir
152 f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
153
154 logger.info('Fetching %s...' % srcuri)
155
156 # FIXME this is too noisy at the moment
157
158 # Parse recipes so our new recipe gets picked up
159 tinfoil.parse_recipes()
160
161 def eventhandler(event):
162 if isinstance(event, bb.fetch2.MissingChecksumEvent):
163 checksums.update(event.checksums)
164 return True
165 return False
166
167 # Run the fetch + unpack tasks
168 res = tinfoil.build_targets(fetchrecipepn,
169 'do_unpack',
170 handle_events=True,
171 extra_events=['bb.fetch2.MissingChecksumEvent'],
172 event_callback=eventhandler)
173 if not res:
174 raise FetchUrlFailure(srcuri)
175
176 # Remove unneeded directories
177 rd = tinfoil.parse_recipe(fetchrecipepn)
178 if rd:
179 pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
180 for pathvar in pathvars:
181 path = rd.getVar(pathvar)
182 shutil.rmtree(path)
120 finally: 183 finally:
121 os.chdir(olddir) 184 if fetchrecipe:
185 try:
186 os.remove(fetchrecipe)
187 except FileNotFoundError:
188 pass
189 try:
190 os.rmdir(fetchrecipedir)
191 except OSError as e:
192 import errno
193 if e.errno != errno.ENOTEMPTY:
194 raise
195
196 bb.utils.mkdirhier(destdir)
197 for fn in os.listdir(tmpworkdir):
198 shutil.move(os.path.join(tmpworkdir, fn), destdir)
199
122 finally: 200 finally:
123 shutil.rmtree(tmpworkdir) 201 if not preserve_tmp:
124 return ret 202 shutil.rmtree(tmpdir)
203 tmpdir = None
204
205 return checksums, tmpdir
206
125 207
126def run_editor(fn): 208def run_editor(fn):
127 if isinstance(fn, str): 209 if isinstance(fn, str):