summaryrefslogtreecommitdiffstats
path: root/meta/classes-recipe/populate_sdk_ext.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes-recipe/populate_sdk_ext.bbclass')
-rw-r--r--meta/classes-recipe/populate_sdk_ext.bbclass854
1 files changed, 0 insertions, 854 deletions
diff --git a/meta/classes-recipe/populate_sdk_ext.bbclass b/meta/classes-recipe/populate_sdk_ext.bbclass
deleted file mode 100644
index 2859320ddf..0000000000
--- a/meta/classes-recipe/populate_sdk_ext.bbclass
+++ /dev/null
@@ -1,854 +0,0 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: MIT
5#
6
7# Extensible SDK
8
9inherit populate_sdk_base
10
11# Used to override TOOLCHAIN_HOST_TASK in the eSDK case
12TOOLCHAIN_HOST_TASK_ESDK = " \
13 meta-environment-extsdk-${MACHINE} \
14 "
15
16SDK_RELOCATE_AFTER_INSTALL:task-populate-sdk-ext = "0"
17
18SDK_EXT = ""
19SDK_EXT:task-populate-sdk-ext = "-ext"
20
21# Options are full or minimal
22SDK_EXT_TYPE ?= "full"
23SDK_INCLUDE_PKGDATA ?= "0"
24SDK_INCLUDE_TOOLCHAIN ?= "${@'1' if d.getVar('SDK_EXT_TYPE') == 'full' else '0'}"
25SDK_INCLUDE_NATIVESDK ?= "0"
26SDK_INCLUDE_BUILDTOOLS ?= '1'
27
28SDK_RECRDEP_TASKS ?= ""
29SDK_CUSTOM_TEMPLATECONF ?= "0"
30
31ESDK_LOCALCONF_ALLOW ?= ""
32ESDK_LOCALCONF_REMOVE ?= "CONF_VERSION \
33 BB_NUMBER_THREADS \
34 BB_NUMBER_PARSE_THREADS \
35 PARALLEL_MAKE \
36 PRSERV_HOST \
37 SSTATE_MIRRORS \
38 DL_DIR \
39 SSTATE_DIR \
40 TMPDIR \
41 BB_SERVER_TIMEOUT \
42 "
43ESDK_CLASS_INHERIT_DISABLE ?= "buildhistory"
44SDK_UPDATE_URL ?= ""
45
46SDK_TARGETS ?= "${PN}"
47
48def get_sdk_install_targets(d, images_only=False):
49 sdk_install_targets = ''
50 if images_only or d.getVar('SDK_EXT_TYPE') != 'minimal':
51 sdk_install_targets = d.getVar('SDK_TARGETS')
52
53 depd = d.getVar('BB_TASKDEPDATA', False)
54 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d)
55 tasklist.remove('do_build')
56 for v in depd.values():
57 if v[1] in tasklist:
58 if v[0] not in sdk_install_targets:
59 sdk_install_targets += ' {}'.format(v[0])
60
61 if not images_only:
62 if d.getVar('SDK_INCLUDE_PKGDATA') == '1':
63 sdk_install_targets += ' meta-world-pkgdata:do_allpackagedata'
64 if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1':
65 sdk_install_targets += ' meta-extsdk-toolchain:do_populate_sysroot'
66
67 return sdk_install_targets
68
69get_sdk_install_targets[vardepsexclude] = "BB_TASKDEPDATA"
70
71OE_INIT_ENV_SCRIPT ?= "oe-init-build-env"
72
73# The files from COREBASE that you want preserved in the COREBASE copied
74# into the sdk. This allows someone to have their own setup scripts in
75# COREBASE be preserved as well as untracked files.
76COREBASE_FILES ?= " \
77 oe-init-build-env \
78 scripts \
79 LICENSE \
80 .templateconf \
81"
82
83SDK_DIR:task-populate-sdk-ext = "${WORKDIR}/sdk-ext"
84B:task-populate-sdk-ext = "${SDK_DIR}"
85TOOLCHAINEXT_OUTPUTNAME ?= "${SDK_NAME}-toolchain-ext-${SDK_VERSION}"
86TOOLCHAIN_OUTPUTNAME:task-populate-sdk-ext = "${TOOLCHAINEXT_OUTPUTNAME}"
87
88SDK_EXT_TARGET_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"
89SDK_EXT_HOST_MANIFEST = "${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"
90
91python write_target_sdk_ext_manifest () {
92 from oe.sdk import get_extra_sdkinfo
93 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
94 extra_info = get_extra_sdkinfo(sstate_dir)
95
96 target = d.getVar('TARGET_SYS')
97 target_multimach = d.getVar('MULTIMACH_TARGET_SYS')
98 real_target_multimach = d.getVar('REAL_MULTIMACH_TARGET_SYS')
99
100 pkgs = {}
101 os.makedirs(os.path.dirname(d.getVar('SDK_EXT_TARGET_MANIFEST')), exist_ok=True)
102 with open(d.getVar('SDK_EXT_TARGET_MANIFEST'), 'w') as f:
103 for fn in extra_info['filesizes']:
104 info = fn.split(':')
105 if info[2] in (target, target_multimach, real_target_multimach) \
106 or info[5] == 'allarch':
107 if not info[1] in pkgs:
108 f.write("%s %s %s\n" % (info[1], info[2], info[3]))
109 pkgs[info[1]] = {}
110}
111python write_host_sdk_ext_manifest () {
112 from oe.sdk import get_extra_sdkinfo
113 sstate_dir = d.expand('${SDK_OUTPUT}/${SDKPATH}/sstate-cache')
114 extra_info = get_extra_sdkinfo(sstate_dir)
115 host = d.getVar('BUILD_SYS')
116 with open(d.getVar('SDK_EXT_HOST_MANIFEST'), 'w') as f:
117 for fn in extra_info['filesizes']:
118 info = fn.split(':')
119 if info[2] == host:
120 f.write("%s %s %s\n" % (info[1], info[2], info[3]))
121}
122
123SDK_POSTPROCESS_COMMAND:append:task-populate-sdk-ext = " write_target_sdk_ext_manifest write_host_sdk_ext_manifest"
124
125SDK_TITLE:task-populate-sdk-ext = "${@d.getVar('DISTRO_NAME') or d.getVar('DISTRO')} Extensible SDK"
126
127def clean_esdk_builddir(d, sdkbasepath):
128 """Clean up traces of the fake build for create_filtered_tasklist()"""
129 import shutil
130 cleanpaths = ['cache', 'tmp']
131 for pth in cleanpaths:
132 fullpth = os.path.join(sdkbasepath, pth)
133 if os.path.isdir(fullpth):
134 shutil.rmtree(fullpth)
135 elif os.path.isfile(fullpth):
136 os.remove(fullpth)
137
138def create_filtered_tasklist(d, sdkbasepath, tasklistfile, conf_initpath):
139 """
140 Create a filtered list of tasks. Also double-checks that the build system
141 within the SDK basically works and required sstate artifacts are available.
142 """
143 import tempfile
144 import shutil
145 import oe.copy_buildsystem
146
147 # Create a temporary build directory that we can pass to the env setup script
148 shutil.copyfile(sdkbasepath + '/conf/local.conf', sdkbasepath + '/conf/local.conf.bak')
149 try:
150 with open(sdkbasepath + '/conf/local.conf', 'a') as f:
151 # Force the use of sstate from the build system
152 f.write('\nSSTATE_DIR:forcevariable = "%s"\n' % d.getVar('SSTATE_DIR'))
153 # Ensure TMPDIR is the default so that clean_esdk_builddir() can delete it
154 f.write('TMPDIR:forcevariable = "${TOPDIR}/tmp"\n')
155 # Drop uninative if the build isn't using it (or else NATIVELSBSTRING will
156 # be different and we won't be able to find our native sstate)
157 if not bb.data.inherits_class('uninative', d):
158 f.write('INHERIT:remove = "uninative"\n')
159
160 # Unfortunately the default SDKPATH (or even a custom value) may contain characters that bitbake
161 # will not allow in its COREBASE path, so we need to rename the directory temporarily
162 temp_sdkbasepath = d.getVar('SDK_OUTPUT') + '/tmp-renamed-sdk'
163 # Delete any existing temp dir
164 try:
165 shutil.rmtree(temp_sdkbasepath)
166 except FileNotFoundError:
167 pass
168 bb.utils.rename(sdkbasepath, temp_sdkbasepath)
169 cmdprefix = '. %s .; ' % conf_initpath
170 logfile = d.getVar('WORKDIR') + '/tasklist_bb_log.txt'
171 try:
172 oe.copy_buildsystem.check_sstate_task_list(d, get_sdk_install_targets(d), tasklistfile, cmdprefix=cmdprefix, cwd=temp_sdkbasepath, logfile=logfile)
173 except bb.process.ExecutionError as e:
174 msg = 'Failed to generate filtered task list for extensible SDK:\n%s' % e.stdout.rstrip()
175 if 'attempted to execute unexpectedly and should have been setscened' in e.stdout:
176 msg += '\n----------\n\nNOTE: "attempted to execute unexpectedly and should have been setscened" errors indicate this may be caused by missing sstate artifacts that were likely produced in earlier builds, but have been subsequently deleted for some reason.\n'
177 bb.fatal(msg)
178 bb.utils.rename(temp_sdkbasepath, sdkbasepath)
179 # Clean out residue of running bitbake, which check_sstate_task_list()
180 # will effectively do
181 clean_esdk_builddir(d, sdkbasepath)
182 finally:
183 localconf = sdkbasepath + '/conf/local.conf'
184 if os.path.exists(localconf + '.bak'):
185 os.replace(localconf + '.bak', localconf)
186
187def copy_bitbake_and_layers(d, baseoutpath, derivative):
188 oe_init_env_script = d.getVar('OE_INIT_ENV_SCRIPT')
189
190 conf_bbpath = ''
191 conf_initpath = ''
192 core_meta_subdir = ''
193
194 # Copy in all metadata layers + bitbake (as repositories)
195 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
196
197 if derivative:
198 workspace_name = 'orig-workspace'
199 else:
200 workspace_name = None
201
202 corebase, sdkbblayers = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name)
203 conf_bbpath = os.path.join('layers', corebase, 'bitbake')
204
205 for path in os.listdir(baseoutpath + '/layers'):
206 relpath = os.path.join('layers', path, oe_init_env_script)
207 if os.path.exists(os.path.join(baseoutpath, relpath)):
208 conf_initpath = relpath
209
210 relpath = os.path.join('layers', path, 'scripts', 'esdk-tools', 'devtool')
211 if os.path.exists(os.path.join(baseoutpath, relpath)):
212 esdk_tools_path = os.path.dirname(relpath)
213
214 relpath = os.path.join('layers', path, 'meta')
215 if os.path.exists(os.path.join(baseoutpath, relpath, 'lib', 'oe')):
216 core_meta_subdir = relpath
217
218 d.setVar('oe_init_build_env_path', conf_initpath)
219 d.setVar('esdk_tools_path', esdk_tools_path)
220
221 return (conf_initpath, conf_bbpath, core_meta_subdir, sdkbblayers)
222
223def write_devtool_config(d, baseoutpath, conf_bbpath, conf_initpath, core_meta_subdir):
224 # Write out config file for devtool
225 import configparser
226 config = configparser.ConfigParser()
227 config.add_section('General')
228 config.set('General', 'bitbake_subdir', conf_bbpath)
229 config.set('General', 'init_path', conf_initpath)
230 config.set('General', 'core_meta_subdir', core_meta_subdir)
231 config.add_section('SDK')
232 config.set('SDK', 'sdk_targets', d.getVar('SDK_TARGETS'))
233 updateurl = d.getVar('SDK_UPDATE_URL')
234 if updateurl:
235 config.set('SDK', 'updateserver', updateurl)
236 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf'))
237 with open(os.path.join(baseoutpath, 'conf', 'devtool.conf'), 'w') as f:
238 config.write(f)
239
240def write_unlocked_sigs(d, baseoutpath):
241 unlockedsigs = os.path.join(baseoutpath, 'conf', 'unlocked-sigs.inc')
242 with open(unlockedsigs, 'w') as f:
243 pass
244
245def write_bblayers_conf(d, baseoutpath, sdkbblayers):
246 # Create a layer for new recipes / appends
247 bbpath = d.getVar('BBPATH')
248 env = os.environ.copy()
249 env['PYTHONDONTWRITEBYTECODE'] = '1'
250 bb.process.run(['devtool', '--bbpath', bbpath, '--basepath', baseoutpath, 'create-workspace', '--layerseries', d.getVar("LAYERSERIES_CORENAMES"), '--create-only', os.path.join(baseoutpath, 'workspace')], env=env)
251
252 # Create bblayers.conf
253 bb.utils.mkdirhier(baseoutpath + '/conf')
254 with open(baseoutpath + '/conf/bblayers.conf', 'w') as f:
255 f.write('# WARNING: this configuration has been automatically generated and in\n')
256 f.write('# most cases should not be edited. If you need more flexibility than\n')
257 f.write('# this configuration provides, it is strongly suggested that you set\n')
258 f.write('# up a proper instance of the full build system and use that instead.\n\n')
259
260 # LCONF_VERSION may not be set, for example when using meta-poky
261 # so don't error if it isn't found
262 lconf_version = d.getVar('LCONF_VERSION', False)
263 if lconf_version is not None:
264 f.write('LCONF_VERSION = "%s"\n\n' % lconf_version)
265
266 f.write('BBPATH = "$' + '{TOPDIR}"\n')
267 f.write('SDKBASEMETAPATH = "$' + '{TOPDIR}"\n')
268 f.write('BBLAYERS := " \\\n')
269 for layerrelpath in sdkbblayers:
270 f.write(' $' + '{SDKBASEMETAPATH}/layers/%s \\\n' % layerrelpath)
271 f.write(' $' + '{SDKBASEMETAPATH}/workspace \\\n')
272 f.write(' "\n')
273
274def copy_uninative(d, baseoutpath):
275 import shutil
276
277 uninative_checksum = None
278
279 # Copy uninative tarball
280 # For now this is where uninative.bbclass expects the tarball
281 if bb.data.inherits_class('uninative', d):
282 uninative_file = d.expand('${UNINATIVE_DLDIR}/' + d.getVarFlag("UNINATIVE_CHECKSUM", d.getVar("BUILD_ARCH")) + '/${UNINATIVE_TARBALL}')
283 uninative_checksum = bb.utils.sha256_file(uninative_file)
284 uninative_outdir = '%s/downloads/uninative/%s' % (baseoutpath, uninative_checksum)
285 bb.utils.mkdirhier(uninative_outdir)
286 shutil.copy(uninative_file, uninative_outdir)
287
288 return uninative_checksum
289
290def write_local_conf(d, baseoutpath, derivative, core_meta_subdir, uninative_checksum):
291 import shutil
292
293 #check if custome templateconf path is set
294 use_custom_templateconf = d.getVar('SDK_CUSTOM_TEMPLATECONF')
295
296 env_passthrough = (d.getVar('BB_ENV_PASSTHROUGH_ADDITIONS') or '').split()
297 env_passthrough_values = {}
298
299 # Create local.conf
300 builddir = d.getVar('TOPDIR')
301 if derivative and os.path.exists(builddir + '/conf/site.conf'):
302 shutil.copyfile(builddir + '/conf/site.conf', baseoutpath + '/conf/site.conf')
303 if derivative and os.path.exists(builddir + '/conf/auto.conf'):
304 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf')
305 if derivative:
306 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf')
307 else:
308 local_conf_allowed = (d.getVar('ESDK_LOCALCONF_ALLOW') or '').split()
309 local_conf_remove = (d.getVar('ESDK_LOCALCONF_REMOVE') or '').split()
310 def handle_var(varname, origvalue, op, newlines):
311 if varname in local_conf_remove or (origvalue.strip().startswith('/') and not varname in local_conf_allowed):
312 newlines.append('# Removed original setting of %s\n' % varname)
313 return None, op, 0, True
314 else:
315 if varname in env_passthrough:
316 env_passthrough_values[varname] = origvalue
317 return origvalue, op, 0, True
318 varlist = ['[^#=+ ]*']
319 oldlines = []
320 for conffile in ['site.conf', 'auto.conf', 'toolcfg.conf', 'local.conf']:
321 if os.path.exists(builddir + '/conf/' + conffile):
322 with open(builddir + '/conf/' + conffile, 'r') as f:
323 oldlines += f.readlines()
324 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
325
326 with open(baseoutpath + '/conf/local.conf', 'w') as f:
327 f.write('# WARNING: this configuration has been automatically generated and in\n')
328 f.write('# most cases should not be edited. If you need more flexibility than\n')
329 f.write('# this configuration provides, it is strongly suggested that you set\n')
330 f.write('# up a proper instance of the full build system and use that instead.\n\n')
331 for line in newlines:
332 if line.strip() and not line.startswith('#'):
333 f.write(line)
334 # Write a newline just in case there's none at the end of the original
335 f.write('\n')
336
337 f.write('TMPDIR = "${TOPDIR}/tmp"\n')
338 f.write('DL_DIR = "${TOPDIR}/downloads"\n')
339
340 if bb.data.inherits_class('uninative', d):
341 f.write('INHERIT += "%s"\n' % 'uninative')
342 f.write('UNINATIVE_CHECKSUM[%s] = "%s"\n\n' % (d.getVar('BUILD_ARCH'), uninative_checksum))
343
344 # CONF_VERSION may not be set, for example when using an empty local.conf
345 # generated with bitbake-setup, and it is not otherwise required to exist.
346 # Write it out only if it's defined.
347 conf_version = d.getVar('CONF_VERSION', False)
348 if conf_version is not None:
349 f.write('CONF_VERSION = "%s"\n\n' % conf_version)
350
351 # Some classes are not suitable for SDK, remove them from INHERIT
352 f.write('INHERIT:remove = "%s"\n' % d.getVar('ESDK_CLASS_INHERIT_DISABLE', False))
353
354 # Bypass the default connectivity check if any
355 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')
356
357 # This warning will come out if reverse dependencies for a task
358 # don't have sstate as well as the task itself. We already know
359 # this will be the case for the extensible sdk, so turn off the
360 # warning.
361 f.write('SIGGEN_LOCKEDSIGS_SSTATE_EXISTS_CHECK = "none"\n\n')
362
363 # Warn if the sigs in the locked-signature file don't match
364 # the sig computed from the metadata.
365 f.write('SIGGEN_LOCKEDSIGS_TASKSIG_CHECK = "warn"\n\n')
366
367 # We want to be able to set this without a full reparse
368 f.write('BB_HASHCONFIG_IGNORE_VARS:append = " SIGGEN_UNLOCKED_RECIPES"\n\n')
369
370 # Set up which tasks are ignored for run on install
371 f.write('BB_SETSCENE_ENFORCE_IGNORE_TASKS = "%:* *:do_shared_workdir *:do_rm_work wic-tools:* *:do_addto_recipe_sysroot"\n\n')
372
373 # Hide the config information from bitbake output (since it's fixed within the SDK)
374 f.write('BUILDCFG_HEADER = ""\n\n')
375
376 # Write METADATA_REVISION
377 # Needs distro override so it can override the value set in the bbclass code (later than local.conf)
378 f.write('METADATA_REVISION:%s = "%s"\n\n' % (d.getVar('DISTRO'), d.getVar('METADATA_REVISION')))
379
380 f.write('# Provide a flag to indicate we are in the EXT_SDK Context\n')
381 f.write('WITHIN_EXT_SDK = "1"\n\n')
382
383 if d.getVar("PRSERV_HOST"):
384 # Override this, we now include PR data, so it should only point ot the local database
385 f.write('PRSERV_HOST = "localhost:0"\n\n')
386
387 # Allow additional config through sdk-extra.conf
388 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
389 if fn:
390 with open(fn, 'r') as xf:
391 for line in xf:
392 f.write(line)
393
394 # If you define a sdk_extraconf() function then it can contain additional config
395 # (Though this is awkward; sdk-extra.conf should probably be used instead)
396 extraconf = (d.getVar('sdk_extraconf') or '').strip()
397 if extraconf:
398 # Strip off any leading / trailing spaces
399 for line in extraconf.splitlines():
400 f.write(line.strip() + '\n')
401
402 f.write('require conf/locked-sigs.inc\n')
403 f.write('require conf/unlocked-sigs.inc\n')
404
405 # Copy multiple configurations if they exist in the users config directory
406 if d.getVar('BBMULTICONFIG') is not None:
407 bb.utils.mkdirhier(os.path.join(baseoutpath, 'conf', 'multiconfig'))
408 for mc in d.getVar('BBMULTICONFIG').split():
409 dest_stub = "/conf/multiconfig/%s.conf" % (mc,)
410 if os.path.exists(builddir + dest_stub):
411 shutil.copyfile(builddir + dest_stub, baseoutpath + dest_stub)
412
413 # If PR Service is in use, we need to export this as well
414 bb.note('Do we have a pr database?')
415 if d.getVar("PRSERV_HOST"):
416 bb.note('Writing PR database...')
417 # Based on the code in classes/prexport.bbclass
418 import oe.prservice
419 #dump meta info of tables
420 localdata = d.createCopy()
421 localdata.setVar('PRSERV_DUMPOPT_COL', "1")
422 localdata.setVar('PRSERV_DUMPDIR', os.path.join(baseoutpath, 'conf'))
423 localdata.setVar('PRSERV_DUMPFILE', '${PRSERV_DUMPDIR}/prserv.inc')
424
425 bb.note('PR Database write to %s' % (localdata.getVar('PRSERV_DUMPFILE')))
426
427 retval = oe.prservice.prserv_dump_db(localdata)
428 if not retval:
429 bb.error("prexport_handler: export failed!")
430 return
431 (metainfo, datainfo) = retval
432 oe.prservice.prserv_export_tofile(localdata, metainfo, datainfo, True)
433
434 # Use templateconf.cfg file from builddir if exists
435 if os.path.exists(builddir + '/conf/templateconf.cfg') and use_custom_templateconf == '1':
436 shutil.copyfile(builddir + '/conf/templateconf.cfg', baseoutpath + '/conf/templateconf.cfg')
437 else:
438 # Write a templateconf.cfg
439 with open(baseoutpath + '/conf/templateconf.cfg', 'w') as f:
440 f.write('meta/conf/templates/default\n')
441 os.makedirs(os.path.join(baseoutpath, core_meta_subdir, 'conf/templates/default'), exist_ok=True)
442
443 # Ensure any variables set from the external environment (by way of
444 # BB_ENV_PASSTHROUGH_ADDITIONS) are set in the SDK's configuration
445 extralines = []
446 for name, value in env_passthrough_values.items():
447 actualvalue = d.getVar(name) or ''
448 if value != actualvalue:
449 extralines.append('%s = "%s"\n' % (name, actualvalue))
450 if extralines:
451 with open(baseoutpath + '/conf/local.conf', 'a') as f:
452 f.write('\n')
453 f.write('# Extra settings from environment:\n')
454 for line in extralines:
455 f.write(line)
456 f.write('\n')
457
458def prepare_locked_cache(d, baseoutpath, derivative, conf_initpath):
459 import shutil
460
461 # Filter the locked signatures file to just the sstate tasks we are interested in
462 excluded_targets = get_sdk_install_targets(d, images_only=True)
463 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc'
464 lockedsigs_pruned = baseoutpath + '/conf/locked-sigs.inc'
465 #nativesdk-only sigfile to merge into locked-sigs.inc
466 sdk_include_nativesdk = (d.getVar("SDK_INCLUDE_NATIVESDK") == '1')
467 nativesigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc'
468 nativesigfile_pruned = d.getVar('WORKDIR') + '/locked-sigs_nativesdk_pruned.inc'
469
470 if sdk_include_nativesdk:
471 oe.copy_buildsystem.prune_lockedsigs([],
472 excluded_targets.split(),
473 nativesigfile,
474 True,
475 nativesigfile_pruned)
476
477 oe.copy_buildsystem.merge_lockedsigs([],
478 sigfile,
479 nativesigfile_pruned,
480 sigfile)
481
482 oe.copy_buildsystem.prune_lockedsigs([],
483 excluded_targets.split(),
484 sigfile,
485 False,
486 lockedsigs_pruned)
487
488 sstate_out = baseoutpath + '/sstate-cache'
489 bb.utils.remove(sstate_out, True)
490
491 # uninative.bbclass sets NATIVELSBSTRING to 'universal'
492 fixedlsbstring = "universal" if bb.data.inherits_class('uninative', d) else ""
493
494 sdk_include_toolchain = (d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1')
495 sdk_ext_type = d.getVar('SDK_EXT_TYPE')
496 if (sdk_ext_type != 'minimal' or sdk_include_toolchain or derivative) and not sdk_include_nativesdk:
497 # Create the filtered task list used to generate the sstate cache shipped with the SDK
498 tasklistfn = d.getVar('WORKDIR') + '/tasklist.txt'
499 create_filtered_tasklist(d, baseoutpath, tasklistfn, conf_initpath)
500 else:
501 tasklistfn = None
502
503 # Add packagedata if enabled
504 if d.getVar('SDK_INCLUDE_PKGDATA') == '1':
505 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base.inc'
506 lockedsigs_copy = d.getVar('WORKDIR') + '/locked-sigs-copy.inc'
507 shutil.move(lockedsigs_pruned, lockedsigs_base)
508 oe.copy_buildsystem.merge_lockedsigs(['do_packagedata'],
509 lockedsigs_base,
510 d.getVar('STAGING_DIR_HOST') + '/world-pkgdata/locked-sigs-pkgdata.inc',
511 lockedsigs_pruned,
512 lockedsigs_copy)
513
514 if sdk_include_toolchain:
515 lockedsigs_base = d.getVar('WORKDIR') + '/locked-sigs-base2.inc'
516 lockedsigs_toolchain = d.expand("${STAGING_DIR}/${TUNE_PKGARCH}/meta-extsdk-toolchain/locked-sigs/locked-sigs-extsdk-toolchain.inc")
517 shutil.move(lockedsigs_pruned, lockedsigs_base)
518 oe.copy_buildsystem.merge_lockedsigs([],
519 lockedsigs_base,
520 lockedsigs_toolchain,
521 lockedsigs_pruned)
522 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_toolchain,
523 d.getVar('SSTATE_DIR'),
524 sstate_out, d,
525 fixedlsbstring,
526 filterfile=tasklistfn)
527
528 if sdk_ext_type == 'minimal':
529 if derivative:
530 # Assume the user is not going to set up an additional sstate
531 # mirror, thus we need to copy the additional artifacts (from
532 # workspace recipes) into the derivative SDK
533 lockedsigs_orig = d.getVar('TOPDIR') + '/conf/locked-sigs.inc'
534 if os.path.exists(lockedsigs_orig):
535 lockedsigs_extra = d.getVar('WORKDIR') + '/locked-sigs-extra.inc'
536 oe.copy_buildsystem.merge_lockedsigs(None,
537 lockedsigs_orig,
538 lockedsigs_pruned,
539 None,
540 lockedsigs_extra)
541 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
542 d.getVar('SSTATE_DIR'),
543 sstate_out, d,
544 fixedlsbstring,
545 filterfile=tasklistfn)
546 else:
547 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
548 d.getVar('SSTATE_DIR'),
549 sstate_out, d,
550 fixedlsbstring,
551 filterfile=tasklistfn)
552
553 # We don't need sstate do_package files
554 for root, dirs, files in os.walk(sstate_out):
555 for name in files:
556 if name.endswith("_package.tar.zst"):
557 f = os.path.join(root, name)
558 os.remove(f)
559
560def write_manifest(d, baseoutpath):
561 import glob
562
563 # Write manifest file
564 # Note: at the moment we cannot include the env setup script here to keep
565 # it updated, since it gets modified during SDK installation (see
566 # sdk_ext_postinst() below) thus the checksum we take here would always
567 # be different.
568 manifest_file_list = ['conf/*']
569 if d.getVar('BBMULTICONFIG') is not None:
570 manifest_file_list.append('conf/multiconfig/*')
571
572 esdk_manifest_excludes = (d.getVar('ESDK_MANIFEST_EXCLUDES') or '').split()
573 esdk_manifest_excludes_list = []
574 for exclude_item in esdk_manifest_excludes:
575 esdk_manifest_excludes_list += glob.glob(os.path.join(baseoutpath, exclude_item))
576 manifest_file = os.path.join(baseoutpath, 'conf', 'sdk-conf-manifest')
577 with open(manifest_file, 'w') as f:
578 for item in manifest_file_list:
579 for fn in glob.glob(os.path.join(baseoutpath, item)):
580 if fn == manifest_file or os.path.isdir(fn):
581 continue
582 if fn in esdk_manifest_excludes_list:
583 continue
584 chksum = bb.utils.sha256_file(fn)
585 f.write('%s\t%s\n' % (chksum, os.path.relpath(fn, baseoutpath)))
586
587
588python copy_buildsystem () {
589 import oe.copy_buildsystem
590
591 baseoutpath = d.getVar('SDK_OUTPUT') + '/' + d.getVar('SDKPATH')
592
593 # Determine if we're building a derivative extensible SDK (from devtool build-sdk)
594 derivative = (d.getVar('SDK_DERIVATIVE') or '') == '1'
595
596 conf_initpath, conf_bbpath, core_meta_subdir, sdkbblayers = copy_bitbake_and_layers(d, baseoutpath, derivative)
597
598 write_devtool_config(d, baseoutpath, conf_bbpath, conf_initpath, core_meta_subdir)
599
600 write_unlocked_sigs(d, baseoutpath)
601
602 write_bblayers_conf(d, baseoutpath, sdkbblayers)
603
604 uninative_checksum = copy_uninative(d, baseoutpath)
605
606 write_local_conf(d, baseoutpath, derivative, core_meta_subdir, uninative_checksum)
607
608 prepare_locked_cache(d, baseoutpath, derivative, conf_initpath)
609
610 write_manifest(d, baseoutpath)
611
612}
613
614def get_current_buildtools(d):
615 """Get the file name of the current buildtools installer"""
616 import glob
617 btfiles = glob.glob(os.path.join(d.getVar('SDK_DEPLOY'), '*-buildtools-nativesdk-standalone-*.sh'))
618 btfiles.sort(key=os.path.getctime)
619 return os.path.basename(btfiles[-1])
620
621def get_sdk_required_utilities(buildtools_fn, d):
622 """Find required utilities that aren't provided by the buildtools"""
623 sanity_required_utilities = (d.getVar('SANITY_REQUIRED_UTILITIES') or '').split()
624 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}gcc'))
625 sanity_required_utilities.append(d.expand('${BUILD_PREFIX}g++'))
626 if buildtools_fn:
627 buildtools_installer = os.path.join(d.getVar('SDK_DEPLOY'), buildtools_fn)
628 filelist, _ = bb.process.run('%s -l' % buildtools_installer)
629 else:
630 buildtools_installer = None
631 filelist = ""
632 localdata = bb.data.createCopy(d)
633 localdata.setVar('SDKPATH', '.')
634 sdkpathnative = localdata.getVar('SDKPATHNATIVE')
635 sdkbindirs = [localdata.getVar('bindir_nativesdk'),
636 localdata.getVar('sbindir_nativesdk'),
637 localdata.getVar('base_bindir_nativesdk'),
638 localdata.getVar('base_sbindir_nativesdk')]
639 for line in filelist.splitlines():
640 splitline = line.split()
641 if len(splitline) > 5:
642 fn = splitline[5]
643 if not fn.startswith('./'):
644 fn = './%s' % fn
645 if fn.startswith(sdkpathnative):
646 relpth = '/' + os.path.relpath(fn, sdkpathnative)
647 for bindir in sdkbindirs:
648 if relpth.startswith(bindir):
649 relpth = os.path.relpath(relpth, bindir)
650 if relpth in sanity_required_utilities:
651 sanity_required_utilities.remove(relpth)
652 break
653 return ' '.join(sanity_required_utilities)
654
655install_tools() {
656 touch ${SDK_OUTPUT}/${SDKPATH}/.devtoolbase
657
658 # find latest buildtools-tarball and install it
659 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
660 install ${SDK_DEPLOY}/${SDK_BUILDTOOLS_INSTALLER} ${SDK_OUTPUT}/${SDKPATH}
661 fi
662
663 install -m 0644 ${COREBASE}/meta/files/ext-sdk-prepare.py ${SDK_OUTPUT}/${SDKPATH}
664}
665do_populate_sdk_ext[file-checksums] += "${COREBASE}/meta/files/ext-sdk-prepare.py:True"
666
667sdk_ext_preinst() {
668 # Since bitbake won't run as root it doesn't make sense to try and install
669 # the extensible sdk as root.
670 if [ "`id -u`" = "0" ]; then
671 echo "ERROR: The extensible sdk cannot be installed as root."
672 exit 1
673 fi
674 if ! command -v locale > /dev/null; then
675 echo "ERROR: The installer requires the locale command, please install it first"
676 exit 1
677 fi
678 # Check setting of LC_ALL set above
679 canonicalised_locale=`echo $LC_ALL | sed 's/UTF-8/utf8/'`
680 if ! locale -a | grep -q $canonicalised_locale ; then
681 echo "ERROR: the installer requires the $LC_ALL locale to be installed (but not selected), please install it first"
682 exit 1
683 fi
684 # The relocation script used by buildtools installer requires python
685 if ! command -v python3 > /dev/null; then
686 echo "ERROR: The installer requires python3, please install it first"
687 exit 1
688 fi
689 missing_utils=""
690 for util in ${SDK_REQUIRED_UTILITIES}; do
691 if ! command -v $util > /dev/null; then
692 missing_utils="$missing_utils $util"
693 fi
694 done
695 if [ -n "$missing_utils" ] ; then
696 echo "ERROR: the SDK requires the following missing utilities, please install them: $missing_utils"
697 exit 1
698 fi
699 SDK_EXTENSIBLE="1"
700 if [ "$publish" = "1" ] && [ "${SDK_EXT_TYPE}" = "minimal" ] ; then
701 EXTRA_TAR_OPTIONS="$EXTRA_TAR_OPTIONS --exclude=sstate-cache"
702 fi
703}
704SDK_PRE_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_preinst}"
705
706# FIXME this preparation should be done as part of the SDK construction
707sdk_ext_postinst() {
708 printf "\nExtracting buildtools...\n"
709 cd $target_sdk_dir
710 env_setup_script="$target_sdk_dir/environment-setup-${REAL_MULTIMACH_TARGET_SYS}"
711 if [ -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
712 printf "buildtools\ny" | ./${SDK_BUILDTOOLS_INSTALLER} > buildtools.log || { printf 'ERROR: buildtools installation failed:\n' ; cat buildtools.log ; echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
713
714 # Delete the buildtools tar file since it won't be used again
715 rm -f ./${SDK_BUILDTOOLS_INSTALLER}
716 # We don't need the log either since it succeeded
717 rm -f buildtools.log
718
719 # Make sure when the user sets up the environment, they also get
720 # the buildtools-tarball tools in their path.
721 echo "# Save and reset OECORE_NATIVE_SYSROOT as buildtools may change it" >> $env_setup_script
722 echo "SAVED=\"\$OECORE_NATIVE_SYSROOT\"" >> $env_setup_script
723 echo ". $target_sdk_dir/buildtools/environment-setup*" >> $env_setup_script
724 echo "export OECORE_NATIVE_SYSROOT=\"\$SAVED\"" >> $env_setup_script
725 fi
726
727 # Allow bitbake environment setup to be ran as part of this sdk.
728 echo "export OE_SKIP_SDK_CHECK=1" >> $env_setup_script
729 # Work around runqemu not knowing how to get this information within the eSDK
730 echo "export DEPLOY_DIR_IMAGE=$target_sdk_dir/tmp/${@os.path.relpath(d.getVar('DEPLOY_DIR_IMAGE'), d.getVar('TMPDIR'))}" >> $env_setup_script
731
732 # A bit of another hack, but we need this in the path only for devtool
733 # so put it at the end of $PATH.
734 echo "export PATH=\"$target_sdk_dir/${esdk_tools_path}:\$PATH\"" >> $env_setup_script
735
736 echo "printf 'SDK environment now set up; additionally you may now run devtool to perform development tasks.\nRun devtool --help for further details.\n'" >> $env_setup_script
737
738 # Warn if trying to use external bitbake and the ext SDK together
739 echo "(which bitbake > /dev/null 2>&1 && echo 'WARNING: attempting to use the extensible SDK in an environment set up to run bitbake - this may lead to unexpected results. Please source this script in a new shell session instead.') || true" >> $env_setup_script
740
741 if [ "$prepare_buildsystem" != "no" -a -n "${SDK_BUILDTOOLS_INSTALLER}" ]; then
742 printf "Preparing build system...\n"
743 # dash which is /bin/sh on Ubuntu will not preserve the
744 # current working directory when first ran, nor will it set $1 when
745 # sourcing a script. That is why this has to look so ugly.
746 LOGFILE="$target_sdk_dir/preparing_build_system.log"
747 sh -c ". buildtools/environment-setup* > $LOGFILE 2>&1 && cd $target_sdk_dir/`dirname ${oe_init_build_env_path}` && set $target_sdk_dir && . $target_sdk_dir/${oe_init_build_env_path} $target_sdk_dir >> $LOGFILE 2>&1 && python3 $target_sdk_dir/ext-sdk-prepare.py $LOGFILE '${SDK_INSTALL_TARGETS}'" || { echo "printf 'ERROR: this SDK was not fully installed and needs reinstalling\n'" >> $env_setup_script ; exit 1 ; }
748 fi
749 if [ -e $target_sdk_dir/ext-sdk-prepare.py ]; then
750 rm $target_sdk_dir/ext-sdk-prepare.py
751 fi
752 echo done
753}
754
755SDK_POST_INSTALL_COMMAND:task-populate-sdk-ext = "${sdk_ext_postinst}"
756
757SDK_POSTPROCESS_COMMAND:prepend:task-populate-sdk-ext = "copy_buildsystem install_tools "
758
759SDK_INSTALL_TARGETS = ""
760fakeroot python do_populate_sdk_ext() {
761 # FIXME hopefully we can remove this restriction at some point, but uninative
762 # currently forces this upon us
763 if d.getVar('SDK_ARCH') != d.getVar('BUILD_ARCH'):
764 bb.fatal('The extensible SDK can currently only be built for the same architecture as the machine being built on - SDK_ARCH is set to %s (likely via setting SDKMACHINE) which is different from the architecture of the build machine (%s). Unable to continue.' % (d.getVar('SDK_ARCH'), d.getVar('BUILD_ARCH')))
765
766 # FIXME hopefully we can remove this restriction at some point, but the eSDK
767 # can only be built for the primary (default) multiconfig
768 if d.getVar('BB_CURRENT_MC') != '':
769 bb.fatal('The extensible SDK can currently only be built for the default multiconfig. Currently trying to build for %s.' % d.getVar('BB_CURRENT_MC'))
770
771 # eSDK dependencies don't use the traditional variables and things don't work properly if they are set
772 d.setVar("TOOLCHAIN_HOST_TASK", "${TOOLCHAIN_HOST_TASK_ESDK}")
773 d.setVar("TOOLCHAIN_TARGET_TASK", "")
774
775 d.setVar('SDK_INSTALL_TARGETS', get_sdk_install_targets(d))
776 if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1':
777 buildtools_fn = get_current_buildtools(d)
778 else:
779 buildtools_fn = None
780 d.setVar('SDK_REQUIRED_UTILITIES', get_sdk_required_utilities(buildtools_fn, d))
781 d.setVar('SDK_BUILDTOOLS_INSTALLER', buildtools_fn)
782 d.setVar('SDKDEPLOYDIR', '${SDKEXTDEPLOYDIR}')
783 # ESDKs have a libc from the buildtools so ensure we don't ship linguas twice
784 d.delVar('SDKIMAGE_LINGUAS')
785 if d.getVar("SDK_INCLUDE_NATIVESDK") == '1':
786 generate_nativesdk_lockedsigs(d)
787 populate_sdk_common(d)
788}
789
790def generate_nativesdk_lockedsigs(d):
791 import oe.copy_buildsystem
792 sigfile = d.getVar('WORKDIR') + '/locked-sigs_nativesdk.inc'
793 oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
794
795def get_ext_sdk_depends(d):
796 # Note: the deps varflag is a list not a string, so we need to specify expand=False
797 deps = d.getVarFlag('do_image_complete', 'deps', False)
798 pn = d.getVar('PN')
799 deplist = ['%s:%s' % (pn, dep) for dep in deps]
800 tasklist = bb.build.tasksbetween('do_image_complete', 'do_build', d)
801 tasklist.append('do_rootfs')
802 for task in tasklist:
803 deplist.extend((d.getVarFlag(task, 'depends') or '').split())
804 return ' '.join(deplist)
805
806python do_sdk_depends() {
807 # We have to do this separately in its own task so we avoid recursing into
808 # dependencies we don't need to (e.g. buildtools-tarball) and bringing those
809 # into the SDK's sstate-cache
810 import oe.copy_buildsystem
811 sigfile = d.getVar('WORKDIR') + '/locked-sigs.inc'
812 oe.copy_buildsystem.generate_locked_sigs(sigfile, d)
813}
814addtask sdk_depends
815
816do_sdk_depends[dirs] = "${WORKDIR}"
817do_sdk_depends[depends] = "${@get_ext_sdk_depends(d)} meta-extsdk-toolchain:do_populate_sysroot"
818do_sdk_depends[recrdeptask] = "${@d.getVarFlag('do_populate_sdk', 'recrdeptask', False)}"
819do_sdk_depends[recrdeptask] += "do_populate_lic do_package_qa do_populate_sysroot do_deploy ${SDK_RECRDEP_TASKS}"
820do_sdk_depends[rdepends] = "${@' '.join([x + ':do_package_write_${IMAGE_PKGTYPE} ' + x + ':do_packagedata' for x in d.getVar('TOOLCHAIN_HOST_TASK_ESDK').split()])}"
821
822do_populate_sdk_ext[dirs] = "${@d.getVarFlag('do_populate_sdk', 'dirs', False)}"
823
824do_populate_sdk_ext[depends] = "${@d.getVarFlag('do_populate_sdk', 'depends', False)} \
825 ${@'buildtools-tarball:do_populate_sdk' if d.getVar('SDK_INCLUDE_BUILDTOOLS') == '1' else ''} \
826 ${@'meta-world-pkgdata:do_collect_packagedata' if d.getVar('SDK_INCLUDE_PKGDATA') == '1' else ''} \
827 ${@'meta-extsdk-toolchain:do_locked_sigs' if d.getVar('SDK_INCLUDE_TOOLCHAIN') == '1' else ''}"
828
829# We must avoid depending on do_build here if rm_work.bbclass is active,
830# because otherwise do_rm_work may run before do_populate_sdk_ext itself.
831# We can't mark do_populate_sdk_ext and do_sdk_depends as having to
832# run before do_rm_work, because then they would also run as part
833# of normal builds.
834do_populate_sdk_ext[rdepends] += "${@' '.join([x + ':' + (d.getVar('RM_WORK_BUILD_WITHOUT') or 'do_build') for x in d.getVar('SDK_TARGETS').split()])}"
835
836# Make sure code changes can result in rebuild
837do_populate_sdk_ext[vardeps] += "copy_buildsystem \
838 sdk_ext_postinst"
839
840# Since any change in the metadata of any layer should cause a rebuild of the
841# sdk(since the layers are put in the sdk) set the task to nostamp so it
842# always runs.
843do_populate_sdk_ext[nostamp] = "1"
844
845SDKEXTDEPLOYDIR = "${WORKDIR}/deploy-${PN}-populate-sdk-ext"
846
847SSTATETASKS += "do_populate_sdk_ext"
848SSTATE_SKIP_CREATION:task-populate-sdk-ext = '1'
849do_populate_sdk_ext[cleandirs] = "${SDKEXTDEPLOYDIR}"
850do_populate_sdk_ext[sstate-inputdirs] = "${SDKEXTDEPLOYDIR}"
851do_populate_sdk_ext[sstate-outputdirs] = "${SDK_DEPLOY}"
852do_populate_sdk_ext[stamp-extra-info] = "${MACHINE_ARCH}"
853
854addtask populate_sdk_ext after do_sdk_depends