summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-07-25 20:41:39 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-26 08:56:31 +0100
commit0b2ca6687404999a3f663b116259c041a39d389d (patch)
treed7a58b786b23e1547d5395527d239cdb6e6a8156 /meta
parent2ac1fdf53741f1943623df47c7a2bfa2a91544b9 (diff)
downloadpoky-0b2ca6687404999a3f663b116259c041a39d389d.tar.gz
classes/populate_sdk_ext: filter sstate within the extensible SDK
Use the new oe-check-sstate to filter the sstate artifacts shipped with the extensible SDK by effectively running bitbake within the produced eSDK and and getting it to tell us which tasks it will restore from sstate. This has several benefits: 1) We drop the *-initial artifacts from the minimal + toolchain eSDK. This still leaves us with a reasonably large SDK for this configuration, however it does pave the way for future reductions since we are actually filtering by what will be expected to be there on install rather than hoping that whatever cuts we make will match. 2) We verify bitbake's basic operation within the eSDK, i.e. that we haven't messed up the configuration 3) We verify that the sstate artifacts we expect to be present are present (at least in the sstate cache for the build producing the eSDK). Outside deletion of sstate artifacts has been a problem up to now, and this should at least catch that earlier i.e. during the build rather than when someone tries to install the eSDK. This does add a couple of minutes to the do_populate_sdk_ext time, but it seems like the most appropriate way to handle this. Should mostly address [YOCTO #9083] and [YOCTO #9626]. (From OE-Core rev: 4b7b48fcb9b39fccf8222650c2608325df2a4507) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/populate_sdk_ext.bbclass78
-rw-r--r--meta/lib/oe/copy_buildsystem.py22
2 files changed, 92 insertions, 8 deletions
diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 211a02286e..ce8c8ff3c4 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -85,6 +85,60 @@ SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
85 85
86SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME', True) or d.getVar('DISTRO', True)} Extensible SDK" 86SDK_TITLE_task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME', True) or d.getVar('DISTRO', True)} Extensible SDK"
87 87
88def clean_esdk_builddir(sdkbasepath):
89 """Clean up traces of the fake build for create_filtered_tasklist()"""
90 import shutil
91 cleanpaths = 'tmp cache conf/sanity_info conf/templateconf.cfg downloads'.split()
92 for pth in cleanpaths:
93 fullpth = os.path.join(sdkbasepath, pth)
94 if os.path.isdir(fullpth):
95 shutil.rmtree(fullpth)
96 elif os.path.isfile(fullpth):
97 os.remove(fullpth)
98
99def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
100 """
101 Create a filtered list of tasks. Also double-checks that the build system
102 within the SDK basically works and required sstate artifacts are available.
103 """
104 import tempfile
105 import shutil
106 import oe.copy_buildsystem
107
108 # Create a temporary build directory that we can pass to the env setup script
109 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak')
110 try:
111 with open(sdkbasepath + '/conf/local.conf', 'a') as f:
112 f.write('\nSSTATE_DIR_forcevariable = "%s"\n' % d.getVar('SSTATE_DIR', True))
113 f.write('SSTATE_MIRRORS_forcevariable = ""\n')
114
115 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake
116 # will not allow in its COREBASE path, so we need to rename the directory temporarily
117 temp_sdkbasepath = d.getVar('SDK_OUTPUT', True) + '/tmp-renamed-sdk'
118 # Delete any existing temp dir
119 try:
120 shutil.rmtree(temp_sdkbasepath)
121 except FileNotFoundError:
122 pass
123 os.rename(sdkbasepath, temp_sdkbasepath)
124 try:
125 cmdprefix = '. %s .; ' % conf_initpath
126 logfile = d.getVar('WORKDIR', True) + '/tasklist_bb_log.txt'
127 try:
128 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile)
129 except bb.process.ExecutionError as e:
130 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip()
131 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout:
132 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n'
133 bb.fatal(msg)
134 finally:
135 os.rename(temp_sdkbasepath, sdkbasepath)
136 # Clean out residue of running bitbake, which check_sstate_task_list()
137 # will effectively do
138 clean_esdk_builddir(sdkbasepath)
139 finally:
140 os.replace(sdkbasepath + '/conf/local.conf.bak', sdkbasepath + '/conf/local.conf')
141
88python copy_buildsystem () { 142python copy_buildsystem () {
89 import re 143 import re
90 import shutil 144 import shutil
@@ -301,6 +355,15 @@ python copy_buildsystem () {
301 # uninative.bbclass sets NATIVELSBSTRING to 'universal' 355 # uninative.bbclass sets NATIVELSBSTRING to 'universal'
302 fixedlsbstring = 'universal' 356 fixedlsbstring = 'universal'
303 357
358 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN', True) == '1')
359 sdk_ext_type = d.getVar('SDK_EXT_TYPE', True)
360 if sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative:
361 # Create the filtered task list used to generate the sstate cache shipped with the SDK
362 tasklistfn = d.getVar('WORKDIR', True) + '/tasklist.txt'
363 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath)
364 else:
365 tasklistfn = None
366
304 # Add packagedata if enabled 367 # Add packagedata if enabled
305 if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1': 368 if d.getVar('SDK_INCLUDE_PKGDATA', True) == '1':
306 lockedsigs_base = d.getVar('WORKDIR', True) + '/locked-sigs-base.inc' 369 lockedsigs_base = d.getVar('WORKDIR', True) + '/locked-sigs-base.inc'
@@ -312,20 +375,21 @@ python copy_buildsystem () {
312 lockedsigs_pruned, 375 lockedsigs_pruned,
313 lockedsigs_copy) 376 lockedsigs_copy)
314 377
315 if d.getVar('SDK_INCLUDE_TOOLCHAIN', True) == '1': 378 if sdk_include_toolchain:
316 lockedsigs_base = d.getVar('WORKDIR', True) + '/locked-sigs-base2.inc' 379 lockedsigs_base = d.getVar('WORKDIR', True) + '/locked-sigs-base2.inc'
317 lockedsigs_toolchain = d.getVar('STAGING_DIR_HOST', True) + '/locked-sigs/locked-sigs-extsdk-toolchain.inc' 380 lockedsigs_toolchain = d.getVar('STAGING_DIR_HOST', True) + '/locked-sigs/locked-sigs-extsdk-toolchain.inc'
318 shutil.move(lockedsigs_pruned, lockedsigs_base) 381 shutil.move(lockedsigs_pruned, lockedsigs_base)
319 oe.copy_buildsystem.merge_lockedsigs(['do_populate_sysroot'], 382 oe.copy_buildsystem.merge_lockedsigs([],
320 lockedsigs_base, 383 lockedsigs_base,
321 lockedsigs_toolchain, 384 lockedsigs_toolchain,
322 lockedsigs_pruned) 385 lockedsigs_pruned)
323 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain, 386 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain,
324 d.getVar('SSTATE_DIR', True), 387 d.getVar('SSTATE_DIR', True),
325 sstate_out, d, 388 sstate_out, d,
326 fixedlsbstring) 389 fixedlsbstring,
390 filterfile=tasklistfn)
327 391
328 if d.getVar('SDK_EXT_TYPE', True) == 'minimal': 392 if sdk_ext_type == 'minimal':
329 if derivative: 393 if derivative:
330 # Assume the user is not going to set up an additional sstate 394 # Assume the user is not going to set up an additional sstate
331 # mirror, thus we need to copy the additional artifacts (from 395 # mirror, thus we need to copy the additional artifacts (from
@@ -341,12 +405,14 @@ python copy_buildsystem () {
341 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra, 405 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
342 d.getVar('SSTATE_DIR', True), 406 d.getVar('SSTATE_DIR', True),
343 sstate_out, d, 407 sstate_out, d,
344 fixedlsbstring) 408 fixedlsbstring,
409 filterfile=tasklistfn)
345 else: 410 else:
346 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, 411 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
347 d.getVar('SSTATE_DIR', True), 412 d.getVar('SSTATE_DIR', True),
348 sstate_out, d, 413 sstate_out, d,
349 fixedlsbstring) 414 fixedlsbstring,
415 filterfile=tasklistfn)
350 416
351 # We don't need sstate do_package files 417 # We don't need sstate do_package files
352 for root, dirs, files in os.walk(sstate_out): 418 for root, dirs, files in os.walk(sstate_out):
diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py
index b5f546f99f..4d3faf6681 100644
--- a/meta/lib/oe/copy_buildsystem.py
+++ b/meta/lib/oe/copy_buildsystem.py
@@ -200,11 +200,11 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu
200 if merged_output: 200 if merged_output:
201 write_sigs_file(merged_output, arch_order, merged) 201 write_sigs_file(merged_output, arch_order, merged)
202 202
203def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""): 203def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring="", filterfile=None):
204 bb.note('Generating sstate-cache...') 204 bb.note('Generating sstate-cache...')
205 205
206 nativelsbstring = d.getVar('NATIVELSBSTRING', True) 206 nativelsbstring = d.getVar('NATIVELSBSTRING', True)
207 bb.process.run("gen-lockedsig-cache %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring)) 207 bb.process.run("gen-lockedsig-cache %s %s %s %s %s" % (lockedsigs, input_sstate_cache, output_sstate_cache, nativelsbstring, filterfile or ''))
208 if fixedlsbstring: 208 if fixedlsbstring:
209 nativedir = output_sstate_cache + '/' + nativelsbstring 209 nativedir = output_sstate_cache + '/' + nativelsbstring
210 if os.path.isdir(nativedir): 210 if os.path.isdir(nativedir):
@@ -216,3 +216,21 @@ def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cac
216 src = os.path.join(nativedir, i) 216 src = os.path.join(nativedir, i)
217 dest = os.path.join(destdir, i) 217 dest = os.path.join(destdir, i)
218 os.rename(src, dest) 218 os.rename(src, dest)
219
220def check_sstate_task_list(d, targets, filteroutfile, cmdprefix='', cwd=None, logfile=None):
221 import subprocess
222
223 bb.note('Generating sstate task list...')
224
225 if not cwd:
226 cwd = os.getcwd()
227 if logfile:
228 logparam = '-l %s' % logfile
229 else:
230 logparam = ''
231 cmd = "%sBB_SETSCENE_ENFORCE=1 PSEUDO_DISABLED=1 oe-check-sstate %s -s -o %s %s" % (cmdprefix, targets, filteroutfile, logparam)
232 env = dict(d.getVar('BB_ORIGENV', False))
233 env.pop('BUILDDIR', '')
234 pathitems = env['PATH'].split(':')
235 env['PATH'] = ':'.join([item for item in pathitems if not item.endswith('/bitbake/bin')])
236 bb.process.run(cmd, stderr=subprocess.STDOUT, env=env, cwd=cwd, executable='/bin/bash')