base/bitbake.conf: Filter contents of PATH to only allow whitelisted tools

We currently have a determinism problem in that the host tools present
in PATH can influence the build. In particular, the presence of pkg-config
on the build host can mask missing pkgconfig class dependencies.

This adds in a new HOSTTOOLS variable and then uses it to set up a directory
of symlinks to the whitelisted host tools. This directory is placed as PATH
instead of the usual /usr/bin:/bin and so on.

This should improve determinism of builds and avoid the issues which have
been particularly obvious since the introduction of recipe specific sysroots.

If users find there is a tool missing, they can extend HOSTTOOLS from a global
class or global conf file.

Right now the settings should be enough to build everything in OE-Core.

(From OE-Core rev: fa764a403da34bb0ca9fa3767a9e9dba8d685965)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

3 files changed, 42 insertions, 0 deletions

diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 14293f83c4..fec351a890 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -119,6 +119,25 @@ def get_lic_checksum_file_list(d):
             bb.fatal(d.getVar('PN') + ": LIC_FILES_CHKSUM contains an invalid URL: " + url)
     return " ".join(filelist)
 
+def setup_hosttools_dir(dest, toolsvar, d, fatal=True):
+    tools = d.getVar(toolsvar).split()
+    origbbenv = d.getVar("BB_ORIGENV", False)
+    path = origbbenv.getVar("PATH")
+    bb.utils.mkdirhier(dest)
+    notfound = []
+    for tool in tools:
+        desttool = os.path.join(dest, tool)
+        if not os.path.exists(desttool):
+            srctool = bb.utils.which(path, tool)
+            if "ccache" in srctool:
+                srctool = bb.utils.which(path, tool, direction=1)
+            if srctool:
+                os.symlink(srctool, desttool)
+            else:
+                notfound.append(tool)
+    if notfound and fatal:
+        bb.fatal("These tools appear to be unavailable in PATH, please install them in order to proceed:\n%s" % " ".join(notfound))
+
 addtask fetch
 do_fetch[dirs] = "${DL_DIR}"
 do_fetch[file-checksums] = "${@bb.fetch.get_checksum_file_list(d)}"
@@ -219,6 +238,9 @@ python base_eventhandler() {
         pkgarch_mapping(e.data)
         oe.utils.features_backfill("DISTRO_FEATURES", e.data)
         oe.utils.features_backfill("MACHINE_FEATURES", e.data)
+        # Works with the line in layer.conf which changes PATH to point here
+        setup_hosttools_dir(d.expand('${TMPDIR}/hosttools'), 'HOSTTOOLS', d)
+        setup_hosttools_dir(d.expand('${TMPDIR}/hosttools'), 'HOSTTOOLS_NONFATAL', d, fatal=False)
 
     if isinstance(e, bb.event.BuildStarted):
         localdata = bb.data.createCopy(e.data)
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index f9df7cacd1..bc115117c9 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -453,6 +453,24 @@ export PATH
 # Build utility info.
 ##################################################################
 
+# Tools needed to run builds with OE-Core
+HOSTTOOLS += " \
+    bash sh cut sed gcc ld git rm install which find xargs cat true mktemp \
+    grep tar gzip touch cp mv basename dirname tr getopt sort awk head tail \
+    mkdir patch uniq perl python chmod python3 ar strip expr ls make as \
+    ranlib egrep echo chown cpio tee wc wget bzip2 stat date rmdir od diff \
+    md5sum dd chrpath file pod2man gunzip python2.7 ln g++ [ false true \
+    uname test hostname nm objdump objcopy cmp printf env readlink gawk fgrep \
+    expand pwd sleep diffstat chgrp flock ldd strings rpcgen du makeinfo \
+    getconf mknod cpp readelf split \
+"
+
+# Tools needed to run testimage runtime image testing
+HOSTTOOLS += "ps stty ip ssh scp ping vi"
+
+# Link to these if present
+HOSTTOOLS_NONFATAL += "ccache pip3 ld.bfd ld.gold gcc-ar gpg sftp"
+
 CCACHE ??= ""
 # Disable ccache explicitly if CCACHE is null since gcc may be a symlink
 # of ccache some distributions (e.g., Fedora 17).
diff --git a/meta/conf/layer.conf b/meta/conf/layer.conf
index 87c235fe12..739d82ea56 100644
--- a/meta/conf/layer.conf
+++ b/meta/conf/layer.conf
@@ -59,3 +59,5 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS += " \
   oprofile->virtual/kernel \
 "
 
+# We need to keep bitbake tools in PATH
+PATH := "${@os.path.dirname(bb.utils.which(d.getVar('PATH'),'bitbake'))}:${TMPDIR}/hosttools"