summaryrefslogtreecommitdiffstats
path: root/meta/classes-global/sstate.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes-global/sstate.bbclass')
-rw-r--r--meta/classes-global/sstate.bbclass176
1 files changed, 80 insertions, 96 deletions
diff --git a/meta/classes-global/sstate.bbclass b/meta/classes-global/sstate.bbclass
index 04539bbb99..53bc2e3940 100644
--- a/meta/classes-global/sstate.bbclass
+++ b/meta/classes-global/sstate.bbclass
@@ -4,7 +4,7 @@
4# SPDX-License-Identifier: MIT 4# SPDX-License-Identifier: MIT
5# 5#
6 6
7SSTATE_VERSION = "12" 7SSTATE_VERSION = "14"
8 8
9SSTATE_ZSTD_CLEVEL ??= "8" 9SSTATE_ZSTD_CLEVEL ??= "8"
10 10
@@ -76,7 +76,7 @@ SSTATE_SCAN_CMD_NATIVE ??= 'grep -Irl -e ${RECIPE_SYSROOT} -e ${RECIPE_SYSROOT_N
76SSTATE_HASHEQUIV_FILEMAP ?= " \ 76SSTATE_HASHEQUIV_FILEMAP ?= " \
77 populate_sysroot:*/postinst-useradd-*:${TMPDIR} \ 77 populate_sysroot:*/postinst-useradd-*:${TMPDIR} \
78 populate_sysroot:*/postinst-useradd-*:${COREBASE} \ 78 populate_sysroot:*/postinst-useradd-*:${COREBASE} \
79 populate_sysroot:*/postinst-useradd-*:regex-\s(PATH|PSEUDO_IGNORE_PATHS|HOME|LOGNAME|OMP_NUM_THREADS|USER)=.*\s \ 79 populate_sysroot:*/postinst-useradd-*:regex-\s(PATH|PSEUDO_INCLUDE_PATHS|HOME|LOGNAME|OMP_NUM_THREADS|USER)=.*\s \
80 populate_sysroot:*/crossscripts/*:${TMPDIR} \ 80 populate_sysroot:*/crossscripts/*:${TMPDIR} \
81 populate_sysroot:*/crossscripts/*:${COREBASE} \ 81 populate_sysroot:*/crossscripts/*:${COREBASE} \
82 " 82 "
@@ -103,7 +103,6 @@ SSTATECREATEFUNCS[vardeps] = "SSTATE_SCAN_FILES"
103SSTATEPOSTCREATEFUNCS = "" 103SSTATEPOSTCREATEFUNCS = ""
104SSTATEPREINSTFUNCS = "" 104SSTATEPREINSTFUNCS = ""
105SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack" 105SSTATEPOSTUNPACKFUNCS = "sstate_hardcode_path_unpack"
106SSTATEPOSTINSTFUNCS = ""
107EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR" 106EXTRA_STAGING_FIXMES ?= "HOSTTOOLS_DIR"
108 107
109# Check whether sstate exists for tasks that support sstate and are in the 108# Check whether sstate exists for tasks that support sstate and are in the
@@ -161,7 +160,10 @@ python () {
161 d.setVar('SSTATETASKS', " ".join(unique_tasks)) 160 d.setVar('SSTATETASKS', " ".join(unique_tasks))
162 for task in unique_tasks: 161 for task in unique_tasks:
163 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ") 162 d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ")
164 d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc") 163 # Generally sstate should be last, execpt for buildhistory functions
164 postfuncs = (d.getVarFlag(task, 'postfuncs') or "").split()
165 newpostfuncs = [p for p in postfuncs if "buildhistory" not in p] + ["sstate_task_postfunc"] + [p for p in postfuncs if "buildhistory" in p]
166 d.setVarFlag(task, 'postfuncs', " ".join(newpostfuncs))
165 d.setVarFlag(task, 'network', '1') 167 d.setVarFlag(task, 'network', '1')
166 d.setVarFlag(task + "_setscene", 'network', '1') 168 d.setVarFlag(task + "_setscene", 'network', '1')
167} 169}
@@ -189,7 +191,6 @@ def sstate_state_fromvars(d, task = None):
189 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split() 191 plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs') or "").split()
190 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split() 192 lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile') or "").split()
191 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split() 193 lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared') or "").split()
192 interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs') or "").split()
193 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or "" 194 fixmedir = d.getVarFlag("do_" + task, 'sstate-fixmedir') or ""
194 if not task or len(inputs) != len(outputs): 195 if not task or len(inputs) != len(outputs):
195 bb.fatal("sstate variables not setup correctly?!") 196 bb.fatal("sstate variables not setup correctly?!")
@@ -205,7 +206,6 @@ def sstate_state_fromvars(d, task = None):
205 ss['lockfiles'] = lockfiles 206 ss['lockfiles'] = lockfiles
206 ss['lockfiles-shared'] = lockfilesshared 207 ss['lockfiles-shared'] = lockfilesshared
207 ss['plaindirs'] = plaindirs 208 ss['plaindirs'] = plaindirs
208 ss['interceptfuncs'] = interceptfuncs
209 ss['fixmedir'] = fixmedir 209 ss['fixmedir'] = fixmedir
210 return ss 210 return ss
211 211
@@ -225,12 +225,23 @@ def sstate_install(ss, d):
225 import oe.sstatesig 225 import oe.sstatesig
226 import subprocess 226 import subprocess
227 227
228 def prepdir(dir):
229 # remove dir if it exists, ensure any parent directories do exist
230 if os.path.exists(dir):
231 oe.path.remove(dir)
232 bb.utils.mkdirhier(dir)
233 oe.path.remove(dir)
234
235 sstateinst = d.getVar("SSTATE_INSTDIR")
236
237 for state in ss['dirs']:
238 prepdir(state[1])
239 bb.utils.rename(sstateinst + state[0], state[1])
240
228 sharedfiles = [] 241 sharedfiles = []
229 shareddirs = [] 242 shareddirs = []
230 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) 243 bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}"))
231 244
232 sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task'])
233
234 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d) 245 manifest, d2 = oe.sstatesig.sstate_get_manifest_filename(ss['task'], d)
235 246
236 if os.access(manifest, os.R_OK): 247 if os.access(manifest, os.R_OK):
@@ -329,15 +340,21 @@ def sstate_install(ss, d):
329 if os.path.exists(state[1]): 340 if os.path.exists(state[1]):
330 oe.path.copyhardlinktree(state[1], state[2]) 341 oe.path.copyhardlinktree(state[1], state[2])
331 342
332 for postinst in (d.getVar('SSTATEPOSTINSTFUNCS') or '').split(): 343 for plain in ss['plaindirs']:
333 # All hooks should run in the SSTATE_INSTDIR 344 workdir = d.getVar('WORKDIR')
334 bb.build.exec_func(postinst, d, (sstateinst,)) 345 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
346 src = sstateinst + "/" + plain.replace(workdir, '')
347 if sharedworkdir in plain:
348 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
349 dest = plain
350 bb.utils.mkdirhier(src)
351 prepdir(dest)
352 bb.utils.rename(src, dest)
335 353
336 for lock in locks: 354 for lock in locks:
337 bb.utils.unlockfile(lock) 355 bb.utils.unlockfile(lock)
338 356
339sstate_install[vardepsexclude] += "SSTATE_ALLOW_OVERLAP_FILES SSTATE_MANMACH SSTATE_MANFILEPREFIX" 357sstate_install[vardepsexclude] += "SSTATE_ALLOW_OVERLAP_FILES SSTATE_MANMACH SSTATE_MANFILEPREFIX STAMP"
340sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}"
341 358
342def sstate_installpkg(ss, d): 359def sstate_installpkg(ss, d):
343 from oe.gpg_sign import get_signer 360 from oe.gpg_sign import get_signer
@@ -393,29 +410,8 @@ def sstate_installpkgdir(ss, d):
393 # All hooks should run in the SSTATE_INSTDIR 410 # All hooks should run in the SSTATE_INSTDIR
394 bb.build.exec_func(f, d, (sstateinst,)) 411 bb.build.exec_func(f, d, (sstateinst,))
395 412
396 def prepdir(dir):
397 # remove dir if it exists, ensure any parent directories do exist
398 if os.path.exists(dir):
399 oe.path.remove(dir)
400 bb.utils.mkdirhier(dir)
401 oe.path.remove(dir)
402
403 for state in ss['dirs']:
404 prepdir(state[1])
405 bb.utils.rename(sstateinst + state[0], state[1])
406 sstate_install(ss, d) 413 sstate_install(ss, d)
407 414
408 for plain in ss['plaindirs']:
409 workdir = d.getVar('WORKDIR')
410 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
411 src = sstateinst + "/" + plain.replace(workdir, '')
412 if sharedworkdir in plain:
413 src = sstateinst + "/" + plain.replace(sharedworkdir, '')
414 dest = plain
415 bb.utils.mkdirhier(src)
416 prepdir(dest)
417 bb.utils.rename(src, dest)
418
419 return True 415 return True
420 416
421python sstate_hardcode_path_unpack () { 417python sstate_hardcode_path_unpack () {
@@ -645,20 +641,12 @@ def sstate_package(ss, d):
645 641
646 tmpdir = d.getVar('TMPDIR') 642 tmpdir = d.getVar('TMPDIR')
647 643
648 fixtime = False
649 if ss['task'] == "package":
650 fixtime = True
651
652 def fixtimestamp(root, path):
653 f = os.path.join(root, path)
654 if os.lstat(f).st_mtime > sde:
655 os.utime(f, (sde, sde), follow_symlinks=False)
656
657 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task']) 644 sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task'])
658 sde = int(d.getVar("SOURCE_DATE_EPOCH") or time.time()) 645 sde = int(d.getVar("SOURCE_DATE_EPOCH") or time.time())
659 d.setVar("SSTATE_CURRTASK", ss['task']) 646 d.setVar("SSTATE_CURRTASK", ss['task'])
660 bb.utils.remove(sstatebuild, recurse=True) 647 bb.utils.remove(sstatebuild, recurse=True)
661 bb.utils.mkdirhier(sstatebuild) 648 bb.utils.mkdirhier(sstatebuild)
649 exit = False
662 for state in ss['dirs']: 650 for state in ss['dirs']:
663 if not os.path.exists(state[1]): 651 if not os.path.exists(state[1]):
664 continue 652 continue
@@ -668,8 +656,6 @@ def sstate_package(ss, d):
668 # to sstate tasks but there aren't many of these so better just avoid them entirely. 656 # to sstate tasks but there aren't many of these so better just avoid them entirely.
669 for walkroot, dirs, files in os.walk(state[1]): 657 for walkroot, dirs, files in os.walk(state[1]):
670 for file in files + dirs: 658 for file in files + dirs:
671 if fixtime:
672 fixtimestamp(walkroot, file)
673 srcpath = os.path.join(walkroot, file) 659 srcpath = os.path.join(walkroot, file)
674 if not os.path.islink(srcpath): 660 if not os.path.islink(srcpath):
675 continue 661 continue
@@ -679,8 +665,11 @@ def sstate_package(ss, d):
679 if not link.startswith(tmpdir): 665 if not link.startswith(tmpdir):
680 continue 666 continue
681 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link)) 667 bb.error("sstate found an absolute path symlink %s pointing at %s. Please replace this with a relative link." % (srcpath, link))
668 exit = True
682 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0])) 669 bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0]))
683 bb.utils.rename(state[1], sstatebuild + state[0]) 670 bb.utils.rename(state[1], sstatebuild + state[0])
671 if exit:
672 bb.fatal("Failing task due to absolute path symlinks")
684 673
685 workdir = d.getVar('WORKDIR') 674 workdir = d.getVar('WORKDIR')
686 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared") 675 sharedworkdir = os.path.join(d.getVar('TMPDIR'), "work-shared")
@@ -691,11 +680,6 @@ def sstate_package(ss, d):
691 bb.utils.mkdirhier(plain) 680 bb.utils.mkdirhier(plain)
692 bb.utils.mkdirhier(pdir) 681 bb.utils.mkdirhier(pdir)
693 bb.utils.rename(plain, pdir) 682 bb.utils.rename(plain, pdir)
694 if fixtime:
695 fixtimestamp(pdir, "")
696 for walkroot, dirs, files in os.walk(pdir):
697 for file in files + dirs:
698 fixtimestamp(walkroot, file)
699 683
700 d.setVar('SSTATE_BUILDDIR', sstatebuild) 684 d.setVar('SSTATE_BUILDDIR', sstatebuild)
701 d.setVar('SSTATE_INSTDIR', sstatebuild) 685 d.setVar('SSTATE_INSTDIR', sstatebuild)
@@ -728,7 +712,7 @@ def sstate_package(ss, d):
728 712
729 return 713 return
730 714
731sstate_package[vardepsexclude] += "SSTATE_SIG_KEY" 715sstate_package[vardepsexclude] += "SSTATE_SIG_KEY SSTATE_PKG"
732 716
733def pstaging_fetch(sstatefetch, d): 717def pstaging_fetch(sstatefetch, d):
734 import bb.fetch2 718 import bb.fetch2
@@ -742,7 +726,6 @@ def pstaging_fetch(sstatefetch, d):
742 localdata = bb.data.createCopy(d) 726 localdata = bb.data.createCopy(d)
743 727
744 dldir = localdata.expand("${SSTATE_DIR}") 728 dldir = localdata.expand("${SSTATE_DIR}")
745 bb.utils.mkdirhier(dldir)
746 729
747 localdata.delVar('MIRRORS') 730 localdata.delVar('MIRRORS')
748 localdata.setVar('FILESPATH', dldir) 731 localdata.setVar('FILESPATH', dldir)
@@ -762,16 +745,19 @@ def pstaging_fetch(sstatefetch, d):
762 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False): 745 if bb.utils.to_boolean(d.getVar("SSTATE_VERIFY_SIG"), False):
763 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)] 746 uris += ['file://{0}.sig;downloadfilename={0}.sig'.format(sstatefetch)]
764 747
765 for srcuri in uris: 748 with bb.utils.umask(bb.utils.to_filemode(d.getVar("OE_SHARED_UMASK"))):
766 localdata.delVar('SRC_URI') 749 bb.utils.mkdirhier(dldir)
767 localdata.setVar('SRC_URI', srcuri)
768 try:
769 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
770 fetcher.checkstatus()
771 fetcher.download()
772 750
773 except bb.fetch2.BBFetchException: 751 for srcuri in uris:
774 pass 752 localdata.delVar('SRC_URI')
753 localdata.setVar('SRC_URI', srcuri)
754 try:
755 fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False)
756 fetcher.checkstatus()
757 fetcher.download()
758
759 except bb.fetch2.BBFetchException:
760 pass
775 761
776def sstate_setscene(d): 762def sstate_setscene(d):
777 shared_state = sstate_state_fromvars(d) 763 shared_state = sstate_state_fromvars(d)
@@ -790,12 +776,10 @@ sstate_task_prefunc[dirs] = "${WORKDIR}"
790python sstate_task_postfunc () { 776python sstate_task_postfunc () {
791 shared_state = sstate_state_fromvars(d) 777 shared_state = sstate_state_fromvars(d)
792 778
793 for intercept in shared_state['interceptfuncs']: 779 shared_umask = bb.utils.to_filemode(d.getVar("OE_SHARED_UMASK"))
794 bb.build.exec_func(intercept, d, (d.getVar("WORKDIR"),)) 780 omask = os.umask(shared_umask)
795 781 if omask != shared_umask:
796 omask = os.umask(0o002) 782 bb.note("Using umask %0o (not %0o) for sstate packaging" % (shared_umask, omask))
797 if omask != 0o002:
798 bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
799 sstate_package(shared_state, d) 783 sstate_package(shared_state, d)
800 os.umask(omask) 784 os.umask(omask)
801 785
@@ -860,15 +844,15 @@ python sstate_create_and_sign_package () {
860 844
861 # Create the required sstate directory if it is not present. 845 # Create the required sstate directory if it is not present.
862 if not sstate_pkg.parent.is_dir(): 846 if not sstate_pkg.parent.is_dir():
863 with bb.utils.umask(0o002): 847 shared_umask = bb.utils.to_filemode(d.getVar("OE_SHARED_UMASK"))
848 with bb.utils.umask(shared_umask):
864 bb.utils.mkdirhier(str(sstate_pkg.parent)) 849 bb.utils.mkdirhier(str(sstate_pkg.parent))
865 850
866 if sign_pkg: 851 if sign_pkg:
867 from tempfile import TemporaryDirectory 852 from tempfile import TemporaryDirectory
868 with TemporaryDirectory(dir=sstate_pkg.parent) as tmp_dir: 853 with TemporaryDirectory(dir=sstate_pkg.parent) as tmp_dir:
869 tmp_pkg = Path(tmp_dir) / sstate_pkg.name 854 tmp_pkg = Path(tmp_dir) / sstate_pkg.name
870 d.setVar("TMP_SSTATE_PKG", str(tmp_pkg)) 855 sstate_archive_package(tmp_pkg, d)
871 bb.build.exec_func('sstate_archive_package', d)
872 856
873 from oe.gpg_sign import get_signer 857 from oe.gpg_sign import get_signer
874 signer = get_signer(d, 'local') 858 signer = get_signer(d, 'local')
@@ -888,8 +872,7 @@ python sstate_create_and_sign_package () {
888 from tempfile import NamedTemporaryFile 872 from tempfile import NamedTemporaryFile
889 with NamedTemporaryFile(prefix=sstate_pkg.name, dir=sstate_pkg.parent) as tmp_pkg_fd: 873 with NamedTemporaryFile(prefix=sstate_pkg.name, dir=sstate_pkg.parent) as tmp_pkg_fd:
890 tmp_pkg = tmp_pkg_fd.name 874 tmp_pkg = tmp_pkg_fd.name
891 d.setVar("TMP_SSTATE_PKG", str(tmp_pkg)) 875 sstate_archive_package(tmp_pkg, d)
892 bb.build.exec_func('sstate_archive_package',d)
893 update_file(tmp_pkg, sstate_pkg) 876 update_file(tmp_pkg, sstate_pkg)
894 # update_file() may have renamed tmp_pkg, which must exist when the 877 # update_file() may have renamed tmp_pkg, which must exist when the
895 # NamedTemporaryFile() context handler ends. 878 # NamedTemporaryFile() context handler ends.
@@ -897,32 +880,33 @@ python sstate_create_and_sign_package () {
897 880
898} 881}
899 882
900# Shell function to generate a sstate package from a directory 883# Function to generate a sstate package from the current directory.
901# set as SSTATE_BUILDDIR. Will be run from within SSTATE_BUILDDIR.
902# The calling function handles moving the sstate package into the final 884# The calling function handles moving the sstate package into the final
903# destination. 885# destination.
904sstate_archive_package () { 886def sstate_archive_package(sstate_pkg, d):
905 OPT="-cS" 887 import subprocess
906 ZSTD="zstd -${SSTATE_ZSTD_CLEVEL} -T${ZSTD_THREADS}"
907 # Use pzstd if available
908 if [ -x "$(command -v pzstd)" ]; then
909 ZSTD="pzstd -${SSTATE_ZSTD_CLEVEL} -p ${ZSTD_THREADS}"
910 fi
911 888
912 # Need to handle empty directories 889 cmd = [
913 if [ "$(ls -A)" ]; then 890 "tar",
914 set +e 891 "-I", d.expand("pzstd -${SSTATE_ZSTD_CLEVEL} -p${ZSTD_THREADS}"),
915 tar -I "$ZSTD" $OPT -f ${TMP_SSTATE_PKG} * 892 "-cS",
916 ret=$? 893 "-f", sstate_pkg,
917 if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then 894 ]
918 exit 1 895
919 fi 896 # tar refuses to create an empty archive unless told explicitly
920 set -e 897 files = sorted(os.listdir("."))
921 else 898 if not files:
922 tar -I "$ZSTD" $OPT --file=${TMP_SSTATE_PKG} --files-from=/dev/null 899 files = ["--files-from=/dev/null"]
923 fi 900
924 chmod 0664 ${TMP_SSTATE_PKG} 901 try:
925} 902 subprocess.run(cmd + files, check=True)
903 except subprocess.CalledProcessError as e:
904 # Ignore error 1 as this is caused by files changing
905 # (link count increasing from hardlinks being created).
906 if e.returncode != 1:
907 raise
908
909 os.chmod(sstate_pkg, 0o664)
926 910
927 911
928python sstate_report_unihash() { 912python sstate_report_unihash() {
@@ -1115,7 +1099,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True,
1115 bb.parse.siggen.checkhashes(sq_data, missed, found, d) 1099 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
1116 1100
1117 return found 1101 return found
1118setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT" 1102setscene_depvalid[vardepsexclude] = "SSTATE_EXCLUDEDEPS_SYSROOT _SSTATE_EXCLUDEDEPS_SYSROOT"
1119 1103
1120BB_SETSCENE_DEPVALID = "setscene_depvalid" 1104BB_SETSCENE_DEPVALID = "setscene_depvalid"
1121 1105