summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorKhem Raj <raj.khem@gmail.com>2019-02-11 21:47:54 -0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-14 11:20:55 +0000
commitc6a243dbb2708ecc0cca722c273d56cbbb24d918 (patch)
treec0cf3dad355d0b97789d9c27c70d41041b4dc451 /meta
parent715c9e3ad6ab4200cff4895ea3fa09b3044db2a3 (diff)
downloadpoky-c6a243dbb2708ecc0cca722c273d56cbbb24d918.tar.gz
image_types.bbclass: Set memory usage limit and CPU threads for xz
when building with opkg backend and huge packages e.g. chromium/llvm all going in parallel, memory pressure causes xz to catapult with do_package_write_ipk: Failed to create package, opkg-build failed with: xz: (stdin): Cannot allocate memory since there are many tasks going on in parallel, xz adds to memory pressure and it wants it all, put an upper limit for memory xz can use We add a variable XZ_MAXRAM with 30% of RAM limit and can be customized if builders have more memory one can set it like XZ_DEFAULTS = "-M 0 -T 0" (From OE-Core rev: 28b277a93a34bba033d9d0d9f3227c9453efd384) Signed-off-by: Khem Raj <raj.khem@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/image_types.bbclass3
-rw-r--r--meta/classes/package_ipk.bbclass2
-rw-r--r--meta/classes/populate_sdk_base.bbclass2
-rw-r--r--meta/conf/bitbake.conf3
4 files changed, 6 insertions, 4 deletions
diff --git a/meta/classes/image_types.bbclass b/meta/classes/image_types.bbclass
index ddca5b624e..6b9407e78b 100644
--- a/meta/classes/image_types.bbclass
+++ b/meta/classes/image_types.bbclass
@@ -56,7 +56,6 @@ def imagetypes_getdepends(d):
56 56
57XZ_COMPRESSION_LEVEL ?= "-3" 57XZ_COMPRESSION_LEVEL ?= "-3"
58XZ_INTEGRITY_CHECK ?= "crc32" 58XZ_INTEGRITY_CHECK ?= "crc32"
59XZ_THREADS ?= "-T 0"
60 59
61ZIP_COMPRESSION_LEVEL ?= "-9" 60ZIP_COMPRESSION_LEVEL ?= "-9"
62 61
@@ -284,7 +283,7 @@ CONVERSIONTYPES = "gz bz2 lzma xz lz4 lzo zip sum md5sum sha1sum sha224sum sha25
284CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" 283CONVERSION_CMD_lzma = "lzma -k -f -7 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
285CONVERSION_CMD_gz = "pigz -f -9 -n -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz" 284CONVERSION_CMD_gz = "pigz -f -9 -n -c ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.gz"
286CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" 285CONVERSION_CMD_bz2 = "pbzip2 -f -k ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
287CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_THREADS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz" 286CONVERSION_CMD_xz = "xz -f -k -c ${XZ_COMPRESSION_LEVEL} ${XZ_DEFAULTS} --check=${XZ_INTEGRITY_CHECK} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} > ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.xz"
288CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4" 287CONVERSION_CMD_lz4 = "lz4 -9 -z -l ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.lz4"
289CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" 288CONVERSION_CMD_lzo = "lzop -9 ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
290CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" 289CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index 508b7dcaff..d1b317b42b 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -8,7 +8,7 @@ IPKGCONF_SDK = "${WORKDIR}/opkg-sdk.conf"
8PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks" 8PKGWRITEDIRIPK = "${WORKDIR}/deploy-ipks"
9 9
10# Program to be used to build opkg packages 10# Program to be used to build opkg packages
11OPKGBUILDCMD ??= 'opkg-build -Z xz -a "--threads 0"' 11OPKGBUILDCMD ??= 'opkg-build -Z xz -a "${XZ_DEFAULTS}"'
12 12
13OPKG_ARGS += "--force_postinstall --prefer-arch-to-version" 13OPKG_ARGS += "--force_postinstall --prefer-arch-to-version"
14OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}" 14OPKG_ARGS += "${@['', '--no-install-recommends'][d.getVar("NO_RECOMMENDATIONS") == "1"]}"
diff --git a/meta/classes/populate_sdk_base.bbclass b/meta/classes/populate_sdk_base.bbclass
index 80fa443e4c..ebc30d39b3 100644
--- a/meta/classes/populate_sdk_base.bbclass
+++ b/meta/classes/populate_sdk_base.bbclass
@@ -226,7 +226,7 @@ fakeroot tar_sdk() {
226 # Package it up 226 # Package it up
227 mkdir -p ${SDKDEPLOYDIR} 227 mkdir -p ${SDKDEPLOYDIR}
228 cd ${SDK_OUTPUT}/${SDKPATH} 228 cd ${SDK_OUTPUT}/${SDKPATH}
229 tar ${SDKTAROPTS} -cf - . | xz -T 0 > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz 229 tar ${SDKTAROPTS} -cf - . | xz ${XZ_DEFAULTS} > ${SDKDEPLOYDIR}/${TOOLCHAIN_OUTPUTNAME}.tar.xz
230} 230}
231 231
232TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh" 232TOOLCHAIN_SHAR_EXT_TMPL ?= "${COREBASE}/meta/files/toolchain-shar-extract.sh"
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 21fd93e58d..6e063dc14d 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -791,6 +791,9 @@ BB_NUMBER_THREADS ?= "${@oe.utils.cpu_count()}"
791# Default to setting automatically based on cpu count 791# Default to setting automatically based on cpu count
792PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}" 792PARALLEL_MAKE ?= "-j ${@oe.utils.cpu_count()}"
793 793
794# Default parallelism and resource usage for xz
795XZ_DEFAULTS ?= "--memlimit=50% --threads=${@oe.utils.cpu_count()}"
796
794################################################################## 797##################################################################
795# Magic Cookie for SANITY CHECK 798# Magic Cookie for SANITY CHECK
796################################################################## 799##################################################################