From 378f67bd825b4926113fb60588c81b4ffb05b704 Mon Sep 17 00:00:00 2001 From: Ross Burton Date: Tue, 18 Oct 2022 19:21:19 +0100 Subject: externalsrc: move back to classes The externalsrc class was moved to classes-recipe as part of oe-core f5c1280, but it can be used in both recipe and global contexts so move it back to classes/. (From OE-Core rev: 7a2edcd4b7cb5a2d829289a11eff62663268fbf3) Signed-off-by: Ross Burton Signed-off-by: Richard Purdie --- meta/classes/externalsrc.bbclass | 272 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 meta/classes/externalsrc.bbclass (limited to 'meta/classes/externalsrc.bbclass') diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass new file mode 100644 index 0000000000..06a9548a20 --- /dev/null +++ b/meta/classes/externalsrc.bbclass @@ -0,0 +1,272 @@ +# Copyright (C) 2012 Linux Foundation +# Author: Richard Purdie +# Some code and influence taken from srctree.bbclass: +# Copyright (C) 2009 Chris Larson +# +# SPDX-License-Identifier: MIT +# +# externalsrc.bbclass enables use of an existing source tree, usually external to +# the build system to build a piece of software rather than the usual fetch/unpack/patch +# process. +# +# To use, add externalsrc to the global inherit and set EXTERNALSRC to point at the +# directory you want to use containing the sources e.g. from local.conf for a recipe +# called "myrecipe" you would do: +# +# INHERIT += "externalsrc" +# EXTERNALSRC:pn-myrecipe = "/path/to/my/source/tree" +# +# In order to make this class work for both target and native versions (or with +# multilibs/cross or other BBCLASSEXTEND variants), B is set to point to a separate +# directory under the work directory (split source and build directories). This is +# the default, but the build directory can be set to the source directory if +# circumstances dictate by setting EXTERNALSRC_BUILD to the same value, e.g.: +# +# EXTERNALSRC_BUILD:pn-myrecipe = "/path/to/my/source/tree" +# + +SRCTREECOVEREDTASKS ?= "do_patch do_unpack do_fetch" +EXTERNALSRC_SYMLINKS ?= "oe-workdir:${WORKDIR} oe-logs:${T}" + +python () { + externalsrc = d.getVar('EXTERNALSRC') + externalsrcbuild = d.getVar('EXTERNALSRC_BUILD') + + if externalsrc and not externalsrc.startswith("/"): + bb.error("EXTERNALSRC must be an absolute path") + if externalsrcbuild and not externalsrcbuild.startswith("/"): + bb.error("EXTERNALSRC_BUILD must be an absolute path") + + # If this is the base recipe and EXTERNALSRC is set for it or any of its + # derivatives, then enable BB_DONT_CACHE to force the recipe to always be + # re-parsed so that the file-checksums function for do_compile is run every + # time. + bpn = d.getVar('BPN') + classextend = (d.getVar('BBCLASSEXTEND') or '').split() + if bpn == d.getVar('PN') or not classextend: + if (externalsrc or + ('native' in classextend and + d.getVar('EXTERNALSRC:pn-%s-native' % bpn)) or + ('nativesdk' in classextend and + d.getVar('EXTERNALSRC:pn-nativesdk-%s' % bpn)) or + ('cross' in classextend and + d.getVar('EXTERNALSRC:pn-%s-cross' % bpn))): + d.setVar('BB_DONT_CACHE', '1') + + if externalsrc: + import oe.recipeutils + import oe.path + + d.setVar('S', externalsrc) + if externalsrcbuild: + d.setVar('B', externalsrcbuild) + else: + d.setVar('B', '${WORKDIR}/${BPN}-${PV}/') + + local_srcuri = [] + fetch = bb.fetch2.Fetch((d.getVar('SRC_URI') or '').split(), d) + for url in fetch.urls: + url_data = fetch.ud[url] + parm = url_data.parm + if (url_data.type == 'file' or + url_data.type == 'npmsw' or url_data.type == 'crate' or + 'type' in parm and parm['type'] == 'kmeta'): + local_srcuri.append(url) + + d.setVar('SRC_URI', ' '.join(local_srcuri)) + + # Dummy value because the default function can't be called with blank SRC_URI + d.setVar('SRCPV', '999') + + if d.getVar('CONFIGUREOPT_DEPTRACK') == '--disable-dependency-tracking': + d.setVar('CONFIGUREOPT_DEPTRACK', '') + + tasks = filter(lambda k: d.getVarFlag(k, "task"), d.keys()) + + for task in tasks: + if task.endswith("_setscene"): + # sstate is never going to work for external source trees, disable it + bb.build.deltask(task, d) + elif os.path.realpath(d.getVar('S')) == os.path.realpath(d.getVar('B')): + # Since configure will likely touch ${S}, ensure only we lock so one task has access at a time + d.appendVarFlag(task, "lockfiles", " ${S}/singletask.lock") + + for v in d.keys(): + cleandirs = d.getVarFlag(v, "cleandirs", False) + if cleandirs: + # We do not want our source to be wiped out, ever (kernel.bbclass does this for do_clean) + cleandirs = oe.recipeutils.split_var_value(cleandirs) + setvalue = False + for cleandir in cleandirs[:]: + if oe.path.is_path_parent(externalsrc, d.expand(cleandir)): + cleandirs.remove(cleandir) + setvalue = True + if setvalue: + d.setVarFlag(v, 'cleandirs', ' '.join(cleandirs)) + + fetch_tasks = ['do_fetch', 'do_unpack'] + # If we deltask do_patch, there's no dependency to ensure do_unpack gets run, so add one + # Note that we cannot use d.appendVarFlag() here because deps is expected to be a list object, not a string + d.setVarFlag('do_configure', 'deps', (d.getVarFlag('do_configure', 'deps', False) or []) + ['do_unpack']) + + for task in d.getVar("SRCTREECOVEREDTASKS").split(): + if local_srcuri and task in fetch_tasks: + continue + bb.build.deltask(task, d) + if task == 'do_unpack': + # The reproducible build create_source_date_epoch_stamp function must + # be run after the source is available and before the + # do_deploy_source_date_epoch task. In the normal case, it's attached + # to do_unpack as a postfuncs, but since we removed do_unpack (above) + # we need to move the function elsewhere. The easiest thing to do is + # move it into the prefuncs of the do_deploy_source_date_epoch task. + # This is safe, as externalsrc runs with the source already unpacked. + d.prependVarFlag('do_deploy_source_date_epoch', 'prefuncs', 'create_source_date_epoch_stamp ') + + d.prependVarFlag('do_compile', 'prefuncs', "externalsrc_compile_prefunc ") + d.prependVarFlag('do_configure', 'prefuncs', "externalsrc_configure_prefunc ") + + d.setVarFlag('do_compile', 'file-checksums', '${@srctree_hash_files(d)}') + d.setVarFlag('do_configure', 'file-checksums', '${@srctree_configure_hash_files(d)}') + + # We don't want the workdir to go away + d.appendVar('RM_WORK_EXCLUDE', ' ' + d.getVar('PN')) + + bb.build.addtask('do_buildclean', + 'do_clean' if d.getVar('S') == d.getVar('B') else None, + None, d) + + # If B=S the same builddir is used even for different architectures. + # Thus, use a shared CONFIGURESTAMPFILE and STAMP directory so that + # change of do_configure task hash is correctly detected and stamps are + # invalidated if e.g. MACHINE changes. + if d.getVar('S') == d.getVar('B'): + configstamp = '${TMPDIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}/configure.sstate' + d.setVar('CONFIGURESTAMPFILE', configstamp) + d.setVar('STAMP', '${STAMPS_DIR}/work-shared/${PN}/${EXTENDPE}${PV}-${PR}') + d.setVar('STAMPCLEAN', '${STAMPS_DIR}/work-shared/${PN}/*-*') +} + +python externalsrc_configure_prefunc() { + s_dir = d.getVar('S') + # Create desired symlinks + symlinks = (d.getVar('EXTERNALSRC_SYMLINKS') or '').split() + newlinks = [] + for symlink in symlinks: + symsplit = symlink.split(':', 1) + lnkfile = os.path.join(s_dir, symsplit[0]) + target = d.expand(symsplit[1]) + if len(symsplit) > 1: + if os.path.islink(lnkfile): + # Link already exists, leave it if it points to the right location already + if os.readlink(lnkfile) == target: + continue + os.unlink(lnkfile) + elif os.path.exists(lnkfile): + # File/dir exists with same name as link, just leave it alone + continue + os.symlink(target, lnkfile) + newlinks.append(symsplit[0]) + # Hide the symlinks from git + try: + git_exclude_file = os.path.join(s_dir, '.git/info/exclude') + if os.path.exists(git_exclude_file): + with open(git_exclude_file, 'r+') as efile: + elines = efile.readlines() + for link in newlinks: + if link in elines or '/'+link in elines: + continue + efile.write('/' + link + '\n') + except IOError as ioe: + bb.note('Failed to hide EXTERNALSRC_SYMLINKS from git') +} + +python externalsrc_compile_prefunc() { + # Make it obvious that this is happening, since forgetting about it could lead to much confusion + bb.plain('NOTE: %s: compiling from external source tree %s' % (d.getVar('PN'), d.getVar('EXTERNALSRC'))) +} + +do_buildclean[dirs] = "${S} ${B}" +do_buildclean[nostamp] = "1" +do_buildclean[doc] = "Call 'make clean' or equivalent in ${B}" +externalsrc_do_buildclean() { + if [ -e Makefile -o -e makefile -o -e GNUmakefile ]; then + rm -f ${@' '.join([x.split(':')[0] for x in (d.getVar('EXTERNALSRC_SYMLINKS') or '').split()])} + if [ "${CLEANBROKEN}" != "1" ]; then + oe_runmake clean || die "make failed" + fi + else + bbnote "nothing to do - no makefile found" + fi +} + +def srctree_hash_files(d, srcdir=None): + import shutil + import subprocess + import tempfile + import hashlib + + s_dir = srcdir or d.getVar('EXTERNALSRC') + git_dir = None + + try: + git_dir = os.path.join(s_dir, + subprocess.check_output(['git', '-C', s_dir, 'rev-parse', '--git-dir'], stderr=subprocess.DEVNULL).decode("utf-8").rstrip()) + top_git_dir = os.path.join(s_dir, subprocess.check_output(['git', '-C', d.getVar("TOPDIR"), 'rev-parse', '--git-dir'], + stderr=subprocess.DEVNULL).decode("utf-8").rstrip()) + if git_dir == top_git_dir: + git_dir = None + except subprocess.CalledProcessError: + pass + + ret = " " + if git_dir is not None: + oe_hash_file = os.path.join(git_dir, 'oe-devtool-tree-sha1-%s' % d.getVar('PN')) + with tempfile.NamedTemporaryFile(prefix='oe-devtool-index') as tmp_index: + # Clone index + shutil.copyfile(os.path.join(git_dir, 'index'), tmp_index.name) + # Update our custom index + env = os.environ.copy() + env['GIT_INDEX_FILE'] = tmp_index.name + subprocess.check_output(['git', 'add', '-A', '.'], cwd=s_dir, env=env) + git_sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8") + if os.path.exists(".gitmodules"): + submodule_helper = subprocess.check_output(["git", "config", "--file", ".gitmodules", "--get-regexp", "path"], cwd=s_dir, env=env).decode("utf-8") + for line in submodule_helper.splitlines(): + module_dir = os.path.join(s_dir, line.rsplit(maxsplit=1)[1]) + if os.path.isdir(module_dir): + proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + proc.communicate() + proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + stdout, _ = proc.communicate() + git_sha1 += stdout.decode("utf-8") + sha1 = hashlib.sha1(git_sha1.encode("utf-8")).hexdigest() + with open(oe_hash_file, 'w') as fobj: + fobj.write(sha1) + ret = oe_hash_file + ':True' + else: + ret = s_dir + '/*:True' + return ret + +def srctree_configure_hash_files(d): + """ + Get the list of files that should trigger do_configure to re-execute, + based on the value of CONFIGURE_FILES + """ + in_files = (d.getVar('CONFIGURE_FILES') or '').split() + out_items = [] + search_files = [] + for entry in in_files: + if entry.startswith('/'): + out_items.append('%s:%s' % (entry, os.path.exists(entry))) + else: + search_files.append(entry) + if search_files: + s_dir = d.getVar('EXTERNALSRC') + for root, _, files in os.walk(s_dir): + for f in files: + if f in search_files: + out_items.append('%s:True' % os.path.join(root, f)) + return ' '.join(out_items) + +EXPORT_FUNCTIONS do_buildclean -- cgit v1.2.3-54-g00ecf