summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2020-09-24 22:44:43 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-02-03 14:13:53 +0000
commit4e7c21160b74d0e63d90cae535ff73579cc4fa8b (patch)
tree865dc279ca0a00d6b3caee044d0b6f64aea80f91 /meta
parent74e91462754d459c23ad40b4867a82e2b6529de1 (diff)
downloadpoky-4e7c21160b74d0e63d90cae535ff73579cc4fa8b.tar.gz
base/bitbake.conf: Enable pseudo path filtering
This is a pretty big change to the way pseudo operates when used in OpenEmbedded. Normally, pseudo monitors and logs (adds to its database) any file created or modified whilst in a fakeroot environment. There are large numbers of files we simply don't care about the permissions of whilst in that fakeroot context, for example ${S}, ${B}, ${T}, ${SSTATE_DIR}, the central sstate control directories, This change uses new functionality in pseudo to ignore these directory trees, resulting in a cleaner database with less chance of "stray" mismatches if files are modified outside pseudo context. It also should reduce some overhead from pseudo as the interprocess round trip to the server is avoided. There is a possible complication where some existing recipe may break, for example, we found a recipe which was writing to "${B}/install" for "make install" in do_install and since we listed ${B} as not to be tracked, there were errors trying to chown root for files in this location. This patch fixes a few corner cases in OE-Core when used with this new ignore list: * The archiver directory matched a "${WORKDIR}/deploy*" pattern so was renamed to something else since that directory does need its root permissions * The ${S} and ${B} ignoring is conditional on them being different to ${WORKDIR} * package_write_* task output (the debs/rpms/ipks) are now owned by the build user so we don't want the file ownership information in the hashequiv outhash calculation even if they are built under pseudo. * The fontcache postinstall intercept is run under qemu outside of pseudo context so delete files it may delete up front where pseudo can see this. * SSTATE_DIR is in PSEUDO_PATHS_IGNORE, which is in FAKEROOTENV which is cached by bitbake. We therefore need to trigger reparsing if this changes, which means SSTATE_DIR can be in BB_HASHBASE_WHITELIST but not BB_HASHCONFIG_WHITELIST. Rework the variables to handle this. This otherwise breaks some of our sstate tests in oe-selftest. * Ignore the temp directory wic uses for rebuilding rootfs. (From OE-Core rev: 61f8cf0940f8b05e9399a062eddb8055ea69bc5e) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> (cherry picked from commit ad8f5532ffaead9a5ad13e1034fe9e5e1b7979f4) Signed-off-by: Steve Sakoman <steve@sakoman.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/archiver.bbclass2
-rw-r--r--meta/classes/base.bbclass5
-rw-r--r--meta/classes/image_types_wic.bbclass2
-rw-r--r--meta/classes/populate_sdk_base.bbclass2
-rw-r--r--meta/conf/bitbake.conf13
-rw-r--r--meta/lib/oe/sstatesig.py2
6 files changed, 20 insertions, 6 deletions
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 990f1d4674..7ca35a573b 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -53,7 +53,7 @@ ARCHIVER_MODE[recipe] ?= "0"
53ARCHIVER_MODE[mirror] ?= "split" 53ARCHIVER_MODE[mirror] ?= "split"
54 54
55DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources" 55DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources"
56ARCHIVER_TOPDIR ?= "${WORKDIR}/deploy-sources" 56ARCHIVER_TOPDIR ?= "${WORKDIR}/archiver-sources"
57ARCHIVER_OUTDIR = "${ARCHIVER_TOPDIR}/${TARGET_SYS}/${PF}/" 57ARCHIVER_OUTDIR = "${ARCHIVER_TOPDIR}/${TARGET_SYS}/${PF}/"
58ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm" 58ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm"
59ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${TARGET_SYS}/${PF}/" 59ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${TARGET_SYS}/${PF}/"
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 7aa2e144eb..a710abdbd7 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -388,6 +388,11 @@ python () {
388 oe.utils.features_backfill("DISTRO_FEATURES", d) 388 oe.utils.features_backfill("DISTRO_FEATURES", d)
389 oe.utils.features_backfill("MACHINE_FEATURES", d) 389 oe.utils.features_backfill("MACHINE_FEATURES", d)
390 390
391 if d.getVar("WORKDIR") != d.getVar("S"):
392 d.appendVar("PSEUDO_IGNORE_PATHS", ",${S}")
393 if d.getVar("WORKDIR") != d.getVar("B"):
394 d.appendVar("PSEUDO_IGNORE_PATHS", ",${B}")
395
391 # Handle PACKAGECONFIG 396 # Handle PACKAGECONFIG
392 # 397 #
393 # These take the form: 398 # These take the form:
diff --git a/meta/classes/image_types_wic.bbclass b/meta/classes/image_types_wic.bbclass
index 196c86814e..70ea46ef5b 100644
--- a/meta/classes/image_types_wic.bbclass
+++ b/meta/classes/image_types_wic.bbclass
@@ -40,6 +40,8 @@ IMAGE_CMD_wic () {
40IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR" 40IMAGE_CMD_wic[vardepsexclude] = "WKS_FULL_PATH WKS_FILES TOPDIR"
41do_image_wic[cleandirs] = "${WORKDIR}/build-wic" 41do_image_wic[cleandirs] = "${WORKDIR}/build-wic"
42 42
43PSEUDO_IGNORE_PATHS .= ",${WORKDIR}/build-wic"
44
43# Rebuild when the wks file or vars in WICVARS change 45# Rebuild when the wks file or vars in WICVARS change
44USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${CONVERSIONTYPES}'.split()), '1', '', d)}" 46USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${CONVERSIONTYPES}'.split()), '1', '', d)}"
45WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}" 47WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}"
diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
index 39a7cadaf8..dea272c441 100644
--- a/meta/classes/populate_sdk_base.bbclass
+++ b/meta/classes/populate_sdk_base.bbclass
@@ -178,6 +178,8 @@ do_populate_sdk[sstate-inputdirs] = "${SDKDEPLOYDIR}"
178do_populate_sdk[sstate-outputdirs] = "${SDK_DEPLOY}" 178do_populate_sdk[sstate-outputdirs] = "${SDK_DEPLOY}"
179do_populate_sdk[stamp-extra-info] = "${MACHINE_ARCH}${SDKMACHINE}" 179do_populate_sdk[stamp-extra-info] = "${MACHINE_ARCH}${SDKMACHINE}"
180 180
181PSEUDO_IGNORE_PATHS .= ",${SDKDEPLOYDIR}"
182
181fakeroot create_sdk_files() { 183fakeroot create_sdk_files() {
182 cp ${COREBASE}/scripts/relocate_sdk.py ${SDK_OUTPUT}/${SDKPATH}/ 184 cp ${COREBASE}/scripts/relocate_sdk.py ${SDK_OUTPUT}/${SDKPATH}/
183 185
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index bdade79abe..9cdadb1269 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -686,13 +686,15 @@ SRC_URI = ""
686PSEUDO_LOCALSTATEDIR ?= "${WORKDIR}/pseudo/" 686PSEUDO_LOCALSTATEDIR ?= "${WORKDIR}/pseudo/"
687PSEUDO_PASSWD ?= "${STAGING_DIR_TARGET}:${PSEUDO_SYSROOT}" 687PSEUDO_PASSWD ?= "${STAGING_DIR_TARGET}:${PSEUDO_SYSROOT}"
688PSEUDO_SYSROOT = "${COMPONENTS_DIR}/${BUILD_ARCH}/pseudo-native" 688PSEUDO_SYSROOT = "${COMPONENTS_DIR}/${BUILD_ARCH}/pseudo-native"
689PSEUDO_IGNORE_PATHS = "/usr/,/etc/,/lib,/dev/,${T},${WORKDIR}/recipe-sysroot,${SSTATE_DIR},${STAMPS_DIR},${WORKDIR}/pkgdata-sysroot,${TMPDIR}/sstate-control,${DEPLOY_DIR},${WORKDIR}/deploy-,${TMPDIR}/buildstats,${WORKDIR}/sstate-build-package_,${WORKDIR}/sstate-install-package_,${WORKDIR}/sstate-build-image_complete,${TMPDIR}/sysroots-components,${BUILDHISTORY_DIR},${TMPDIR}/pkgdata,${TOPDIR}/cache,${COREBASE}/scripts"
690
689export PSEUDO_DISABLED = "1" 691export PSEUDO_DISABLED = "1"
690#export PSEUDO_PREFIX = "${STAGING_DIR_NATIVE}${prefix_native}" 692#export PSEUDO_PREFIX = "${STAGING_DIR_NATIVE}${prefix_native}"
691#export PSEUDO_BINDIR = "${STAGING_DIR_NATIVE}${bindir_native}" 693#export PSEUDO_BINDIR = "${STAGING_DIR_NATIVE}${bindir_native}"
692#export PSEUDO_LIBDIR = "${STAGING_DIR_NATIVE}$PSEUDOBINDIR/../lib/pseudo/lib 694#export PSEUDO_LIBDIR = "${STAGING_DIR_NATIVE}$PSEUDOBINDIR/../lib/pseudo/lib
693FAKEROOTBASEENV = "PSEUDO_BINDIR=${PSEUDO_SYSROOT}${bindir_native} PSEUDO_LIBDIR=${PSEUDO_SYSROOT}${prefix_native}/lib/pseudo/lib PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_DISABLED=1" 695FAKEROOTBASEENV = "PSEUDO_BINDIR=${PSEUDO_SYSROOT}${bindir_native} PSEUDO_LIBDIR=${PSEUDO_SYSROOT}${prefix_native}/lib/pseudo/lib PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_IGNORE_PATHS=${PSEUDO_IGNORE_PATHS} PSEUDO_DISABLED=1"
694FAKEROOTCMD = "${PSEUDO_SYSROOT}${bindir_native}/pseudo" 696FAKEROOTCMD = "${PSEUDO_SYSROOT}${bindir_native}/pseudo"
695FAKEROOTENV = "PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_LOCALSTATEDIR=${PSEUDO_LOCALSTATEDIR} PSEUDO_PASSWD=${PSEUDO_PASSWD} PSEUDO_NOSYMLINKEXP=1 PSEUDO_DISABLED=0" 697FAKEROOTENV = "PSEUDO_PREFIX=${PSEUDO_SYSROOT}${prefix_native} PSEUDO_LOCALSTATEDIR=${PSEUDO_LOCALSTATEDIR} PSEUDO_PASSWD=${PSEUDO_PASSWD} PSEUDO_NOSYMLINKEXP=1 PSEUDO_IGNORE_PATHS=${PSEUDO_IGNORE_PATHS} PSEUDO_DISABLED=0"
696FAKEROOTNOENV = "PSEUDO_UNLOAD=1" 698FAKEROOTNOENV = "PSEUDO_UNLOAD=1"
697FAKEROOTDIRS = "${PSEUDO_LOCALSTATEDIR}" 699FAKEROOTDIRS = "${PSEUDO_LOCALSTATEDIR}"
698PREFERRED_PROVIDER_virtual/fakeroot-native ?= "pseudo-native" 700PREFERRED_PROVIDER_virtual/fakeroot-native ?= "pseudo-native"
@@ -874,8 +876,8 @@ BB_CONSOLELOG ?= "${LOG_DIR}/cooker/${MACHINE}/${DATETIME}.log"
874 876
875# Setup our default hash policy 877# Setup our default hash policy
876BB_SIGNATURE_HANDLER ?= "OEBasicHash" 878BB_SIGNATURE_HANDLER ?= "OEBasicHash"
877BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \ 879BB_HASHEXCLUDE_COMMON ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DIR \
878 SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \ 880 THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL TERM \
879 USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \ 881 USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \
880 STAMPS_DIR PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \ 882 STAMPS_DIR PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \
881 CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_NOHASHDIR LICENSE_PATH SDKPKGSUFFIX \ 883 CCACHE_DIR EXTERNAL_TOOLCHAIN CCACHE CCACHE_NOHASHDIR LICENSE_PATH SDKPKGSUFFIX \
@@ -883,7 +885,8 @@ BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH BBSERVER DL_DI
883 BB_WORKERCONTEXT BB_LIMITEDDEPS BB_UNIHASH extend_recipe_sysroot DEPLOY_DIR \ 885 BB_WORKERCONTEXT BB_LIMITEDDEPS BB_UNIHASH extend_recipe_sysroot DEPLOY_DIR \
884 SSTATE_HASHEQUIV_METHOD SSTATE_HASHEQUIV_REPORT_TASKDATA \ 886 SSTATE_HASHEQUIV_METHOD SSTATE_HASHEQUIV_REPORT_TASKDATA \
885 SSTATE_HASHEQUIV_OWNER CCACHE_TOP_DIR BB_HASHSERVE GIT_CEILING_DIRECTORIES" 887 SSTATE_HASHEQUIV_OWNER CCACHE_TOP_DIR BB_HASHSERVE GIT_CEILING_DIRECTORIES"
886BB_HASHCONFIG_WHITELIST ?= "${BB_HASHBASE_WHITELIST} DATE TIME SSH_AGENT_PID \ 888BB_HASHBASE_WHITELIST ?= "${BB_HASHEXCLUDE_COMMON} PSEUDO_IGNORE_PATHS BUILDHISTORY_DIR SSTATE_DIR "
889BB_HASHCONFIG_WHITELIST ?= "${BB_HASHEXCLUDE_COMMON} DATE TIME SSH_AGENT_PID \
887 SSH_AUTH_SOCK PSEUDO_BUILD BB_ENV_EXTRAWHITE DISABLE_SANITY_CHECKS \ 890 SSH_AUTH_SOCK PSEUDO_BUILD BB_ENV_EXTRAWHITE DISABLE_SANITY_CHECKS \
888 PARALLEL_MAKE BB_NUMBER_THREADS BB_ORIGENV BB_INVALIDCONF BBINCLUDED \ 891 PARALLEL_MAKE BB_NUMBER_THREADS BB_ORIGENV BB_INVALIDCONF BBINCLUDED \
889 GIT_PROXY_COMMAND ALL_PROXY all_proxy NO_PROXY no_proxy FTP_PROXY ftp_proxy \ 892 GIT_PROXY_COMMAND ALL_PROXY all_proxy NO_PROXY no_proxy FTP_PROXY ftp_proxy \
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index d5a6200562..beaccdc909 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -477,6 +477,8 @@ def OEOuthashBasic(path, sigfile, task, d):
477 h = hashlib.sha256() 477 h = hashlib.sha256()
478 prev_dir = os.getcwd() 478 prev_dir = os.getcwd()
479 include_owners = os.environ.get('PSEUDO_DISABLED') == '0' 479 include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
480 if "package_write_" in task or task == "package_qa":
481 include_owners = False
480 include_timestamps = False 482 include_timestamps = False
481 if task == "package": 483 if task == "package":
482 include_timestamps = d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1' 484 include_timestamps = d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1'