summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2021-10-14 12:34:17 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-10-16 17:41:59 +0100
commit4b424bdfdbbe16b6e41f6cd1de75a51e3799cf94 (patch)
treef59682b14ffa5de9fef7dd62b88dddd43e3de400
parentf47a396964c4de76838f8c59b8226625817aa369 (diff)
downloadpoky-4b424bdfdbbe16b6e41f6cd1de75a51e3799cf94.tar.gz
reproducible: Merge code into base.bbclass
Reproducibility is here to stay and needs to be part of our default workflow. Move the remaining code to base.bbclass so it is always a first class citizen and it is clear people need to be mindful of it. (From OE-Core rev: abb0671d2cebfd7e8df94796404bbe9c7f961058) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/base.bbclass38
-rw-r--r--meta/classes/externalsrc.bbclass4
-rw-r--r--meta/classes/image-artifact-names.bbclass2
-rw-r--r--meta/classes/package_deb.bbclass3
-rw-r--r--meta/classes/package_ipk.bbclass3
-rw-r--r--meta/classes/package_rpm.bbclass3
-rw-r--r--meta/classes/reproducible_build.bbclass96
-rw-r--r--meta/lib/oe/reproducible.py51
-rw-r--r--meta/lib/oeqa/selftest/cases/reproducible.py1
9 files changed, 95 insertions, 106 deletions
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index bca3944ae7..a65fcc6c1d 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -180,6 +180,44 @@ python base_do_unpack() {
180 bb.fatal("Bitbake Fetcher Error: " + repr(e)) 180 bb.fatal("Bitbake Fetcher Error: " + repr(e))
181} 181}
182 182
183SSTATETASKS += "do_deploy_source_date_epoch"
184
185do_deploy_source_date_epoch () {
186 mkdir -p ${SDE_DEPLOYDIR}
187 if [ -e ${SDE_FILE} ]; then
188 echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}."
189 cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt
190 else
191 echo "${SDE_FILE} not found!"
192 fi
193}
194
195python do_deploy_source_date_epoch_setscene () {
196 sstate_setscene(d)
197 bb.utils.mkdirhier(d.getVar('SDE_DIR'))
198 sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt')
199 if os.path.exists(sde_file):
200 target = d.getVar('SDE_FILE')
201 bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target))
202 bb.utils.rename(sde_file, target)
203 else:
204 bb.debug(1, "%s not found!" % sde_file)
205}
206
207do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}"
208do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}"
209addtask do_deploy_source_date_epoch_setscene
210addtask do_deploy_source_date_epoch before do_configure after do_patch
211
212python create_source_date_epoch_stamp() {
213 source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S'))
214 oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d)
215}
216do_unpack[postfuncs] += "create_source_date_epoch_stamp"
217
218def get_source_date_epoch_value(d):
219 return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d)
220
183def get_layers_branch_rev(d): 221def get_layers_branch_rev(d):
184 layers = (d.getVar("BBLAYERS") or "").split() 222 layers = (d.getVar("BBLAYERS") or "").split()
185 layers_branch_rev = ["%-20s = \"%s:%s\"" % (os.path.basename(i), \ 223 layers_branch_rev = ["%-20s = \"%s:%s\"" % (os.path.basename(i), \
diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass
index ad93b2d2ab..abfe24bace 100644
--- a/meta/classes/externalsrc.bbclass
+++ b/meta/classes/externalsrc.bbclass
@@ -109,8 +109,8 @@ python () {
109 if local_srcuri and task in fetch_tasks: 109 if local_srcuri and task in fetch_tasks:
110 continue 110 continue
111 bb.build.deltask(task, d) 111 bb.build.deltask(task, d)
112 if bb.data.inherits_class('reproducible_build', d) and task == 'do_unpack': 112 if task == 'do_unpack':
113 # The reproducible_build's create_source_date_epoch_stamp function must 113 # The reproducible build create_source_date_epoch_stamp function must
114 # be run after the source is available and before the 114 # be run after the source is available and before the
115 # do_deploy_source_date_epoch task. In the normal case, it's attached 115 # do_deploy_source_date_epoch task. In the normal case, it's attached
116 # to do_unpack as a postfuncs, but since we removed do_unpack (above) 116 # to do_unpack as a postfuncs, but since we removed do_unpack (above)
diff --git a/meta/classes/image-artifact-names.bbclass b/meta/classes/image-artifact-names.bbclass
index 67f04e8165..f5769e520f 100644
--- a/meta/classes/image-artifact-names.bbclass
+++ b/meta/classes/image-artifact-names.bbclass
@@ -15,7 +15,7 @@ IMAGE_LINK_NAME ?= "${IMAGE_BASENAME}-${MACHINE}"
15IMAGE_NAME_SUFFIX ??= ".rootfs" 15IMAGE_NAME_SUFFIX ??= ".rootfs"
16 16
17python () { 17python () {
18 if bb.data.inherits_class('reproducible_build', d) and bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}": 18 if bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}":
19 import datetime 19 import datetime
20 d.setVar("IMAGE_VERSION_SUFFIX", "-" + datetime.datetime.fromtimestamp(int(d.getVar("SOURCE_DATE_EPOCH")), datetime.timezone.utc).strftime('%Y%m%d%H%M%S')) 20 d.setVar("IMAGE_VERSION_SUFFIX", "-" + datetime.datetime.fromtimestamp(int(d.getVar("SOURCE_DATE_EPOCH")), datetime.timezone.utc).strftime('%Y%m%d%H%M%S'))
21 d.setVarFlag("IMAGE_VERSION_SUFFIX", "vardepvalue", "") 21 d.setVarFlag("IMAGE_VERSION_SUFFIX", "vardepvalue", "")
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index 65dbe6c392..1ae6393d37 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -315,8 +315,7 @@ python do_package_write_deb () {
315do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}" 315do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}"
316do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}" 316do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}"
317do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" 317do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
318EPOCHTASK ??= "" 318addtask package_write_deb after do_packagedata do_package do_deploy_source_date_epoch before do_build
319addtask package_write_deb after do_packagedata do_package ${EPOCHTASK} before do_build
320 319
321PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot" 320PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot"
322PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot" 321PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot"
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index 776fe8ed23..902b7f94c8 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -274,8 +274,7 @@ python do_package_write_ipk () {
274do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}" 274do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}"
275do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" 275do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}"
276do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" 276do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
277EPOCHTASK ??= "" 277addtask package_write_ipk after do_packagedata do_package do_deploy_source_date_epoch before do_build
278addtask package_write_ipk after do_packagedata do_package ${EPOCHTASK} before do_build
279 278
280PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" 279PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot"
281PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot" 280PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot"
diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index e738806fd7..b0754421a3 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -748,8 +748,7 @@ python do_package_write_rpm () {
748do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}" 748do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}"
749do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" 749do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}"
750do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" 750do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}"
751EPOCHTASK ??= "" 751addtask package_write_rpm after do_packagedata do_package do_deploy_source_date_epoch before do_build
752addtask package_write_rpm after do_packagedata do_package ${EPOCHTASK} before do_build
753 752
754PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" 753PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot"
755PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot" 754PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot"
diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass
deleted file mode 100644
index f38be1a765..0000000000
--- a/meta/classes/reproducible_build.bbclass
+++ /dev/null
@@ -1,96 +0,0 @@
1# reproducible_build.bbclass
2#
3# Sets the default SOURCE_DATE_EPOCH in each component's build environment.
4# The format is number of seconds since the system epoch.
5#
6# Upstream components (generally) respect this environment variable,
7# using it in place of the "current" date and time.
8# See https://reproducible-builds.org/specs/source-date-epoch/
9#
10# The default value of SOURCE_DATE_EPOCH comes from the function
11# get_source_date_epoch_value which reads from the SDE_FILE, or if the file
12# is not available (or set to 0) will use the fallback of
13# SOURCE_DATE_EPOCH_FALLBACK.
14#
15# The SDE_FILE is normally constructed from the function
16# create_source_date_epoch_stamp which is typically added as a postfuncs to
17# the do_unpack task. If a recipe does NOT have do_unpack, it should be added
18# to a task that runs after the source is available and before the
19# do_deploy_source_date_epoch task is executed.
20#
21# If a recipe wishes to override the default behavior it should set it's own
22# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task
23# with recipe-specific functionality to write the appropriate
24# SOURCE_DATE_EPOCH into the SDE_FILE.
25#
26# SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should
27# be reproducible for anyone who builds the same revision from the same
28# sources.
29#
30# There are 4 ways the create_source_date_epoch_stamp function determines what
31# becomes SOURCE_DATE_EPOCH:
32#
33# 1. Use the value from __source_date_epoch.txt file if this file exists.
34# This file was most likely created in the previous build by one of the
35# following methods 2,3,4.
36# Alternatively, it can be provided by a recipe via SRC_URI.
37#
38# If the file does not exist:
39#
40# 2. If there is a git checkout, use the last git commit timestamp.
41# Git does not preserve file timestamps on checkout.
42#
43# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
44# This works for well-kept repositories distributed via tarball.
45#
46# 4. Use the modification time of the youngest file in the source tree, if
47# there is one.
48# This will be the newest file from the distribution tarball, if any.
49#
50# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK).
51#
52# Once the value is determined, it is stored in the recipe's SDE_FILE.
53
54
55SSTATETASKS += "do_deploy_source_date_epoch"
56
57do_deploy_source_date_epoch () {
58 mkdir -p ${SDE_DEPLOYDIR}
59 if [ -e ${SDE_FILE} ]; then
60 echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}."
61 cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt
62 else
63 echo "${SDE_FILE} not found!"
64 fi
65}
66
67python do_deploy_source_date_epoch_setscene () {
68 sstate_setscene(d)
69 bb.utils.mkdirhier(d.getVar('SDE_DIR'))
70 sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt')
71 if os.path.exists(sde_file):
72 target = d.getVar('SDE_FILE')
73 bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target))
74 bb.utils.rename(sde_file, target)
75 else:
76 bb.debug(1, "%s not found!" % sde_file)
77}
78
79do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}"
80do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}"
81addtask do_deploy_source_date_epoch_setscene
82addtask do_deploy_source_date_epoch before do_configure after do_patch
83
84python create_source_date_epoch_stamp() {
85 source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S'))
86 oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d)
87}
88
89EPOCHTASK = "do_deploy_source_date_epoch"
90
91# Generate the stamp after do_unpack runs
92do_unpack[postfuncs] += "create_source_date_epoch_stamp"
93
94def get_source_date_epoch_value(d):
95 return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d)
96
diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py
index a5000574cf..4fb99d963c 100644
--- a/meta/lib/oe/reproducible.py
+++ b/meta/lib/oe/reproducible.py
@@ -5,6 +5,57 @@ import os
5import subprocess 5import subprocess
6import bb 6import bb
7 7
8# For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each
9# component's build environment. The format is number of seconds since the
10# system epoch.
11#
12# Upstream components (generally) respect this environment variable,
13# using it in place of the "current" date and time.
14# See https://reproducible-builds.org/specs/source-date-epoch/
15#
16# The default value of SOURCE_DATE_EPOCH comes from the function
17# get_source_date_epoch_value which reads from the SDE_FILE, or if the file
18# is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK.
19#
20# The SDE_FILE is normally constructed from the function
21# create_source_date_epoch_stamp which is typically added as a postfuncs to
22# the do_unpack task. If a recipe does NOT have do_unpack, it should be added
23# to a task that runs after the source is available and before the
24# do_deploy_source_date_epoch task is executed.
25#
26# If a recipe wishes to override the default behavior it should set it's own
27# SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task
28# with recipe-specific functionality to write the appropriate
29# SOURCE_DATE_EPOCH into the SDE_FILE.
30#
31# SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should
32# be reproducible for anyone who builds the same revision from the same
33# sources.
34#
35# There are 4 ways the create_source_date_epoch_stamp function determines what
36# becomes SOURCE_DATE_EPOCH:
37#
38# 1. Use the value from __source_date_epoch.txt file if this file exists.
39# This file was most likely created in the previous build by one of the
40# following methods 2,3,4.
41# Alternatively, it can be provided by a recipe via SRC_URI.
42#
43# If the file does not exist:
44#
45# 2. If there is a git checkout, use the last git commit timestamp.
46# Git does not preserve file timestamps on checkout.
47#
48# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
49# This works for well-kept repositories distributed via tarball.
50#
51# 4. Use the modification time of the youngest file in the source tree, if
52# there is one.
53# This will be the newest file from the distribution tarball, if any.
54#
55# 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK).
56#
57# Once the value is determined, it is stored in the recipe's SDE_FILE.
58
8def get_source_date_epoch_from_known_files(d, sourcedir): 59def get_source_date_epoch_from_known_files(d, sourcedir):
9 source_date_epoch = None 60 source_date_epoch = None
10 newest_file = None 61 newest_file = None
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
index e4582cb82a..2e983d2f17 100644
--- a/meta/lib/oeqa/selftest/cases/reproducible.py
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -219,7 +219,6 @@ class ReproducibleTests(OESelftestTestCase):
219 bb.utils.remove(tmpdir, recurse=True) 219 bb.utils.remove(tmpdir, recurse=True)
220 220
221 config = textwrap.dedent('''\ 221 config = textwrap.dedent('''\
222 INHERIT += "reproducible_build"
223 PACKAGE_CLASSES = "{package_classes}" 222 PACKAGE_CLASSES = "{package_classes}"
224 INHIBIT_PACKAGE_STRIP = "1" 223 INHIBIT_PACKAGE_STRIP = "1"
225 TMPDIR = "{tmpdir}" 224 TMPDIR = "{tmpdir}"