diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-10-14 12:34:17 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-10-16 17:41:59 +0100 |
commit | 4b424bdfdbbe16b6e41f6cd1de75a51e3799cf94 (patch) | |
tree | f59682b14ffa5de9fef7dd62b88dddd43e3de400 | |
parent | f47a396964c4de76838f8c59b8226625817aa369 (diff) | |
download | poky-4b424bdfdbbe16b6e41f6cd1de75a51e3799cf94.tar.gz |
reproducible: Merge code into base.bbclass
Reproducibility is here to stay and needs to be part of our default workflow.
Move the remaining code to base.bbclass so it is always a first class citizen
and it is clear people need to be mindful of it.
(From OE-Core rev: abb0671d2cebfd7e8df94796404bbe9c7f961058)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | meta/classes/base.bbclass | 38 | ||||
-rw-r--r-- | meta/classes/externalsrc.bbclass | 4 | ||||
-rw-r--r-- | meta/classes/image-artifact-names.bbclass | 2 | ||||
-rw-r--r-- | meta/classes/package_deb.bbclass | 3 | ||||
-rw-r--r-- | meta/classes/package_ipk.bbclass | 3 | ||||
-rw-r--r-- | meta/classes/package_rpm.bbclass | 3 | ||||
-rw-r--r-- | meta/classes/reproducible_build.bbclass | 96 | ||||
-rw-r--r-- | meta/lib/oe/reproducible.py | 51 | ||||
-rw-r--r-- | meta/lib/oeqa/selftest/cases/reproducible.py | 1 |
9 files changed, 95 insertions, 106 deletions
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass index bca3944ae7..a65fcc6c1d 100644 --- a/meta/classes/base.bbclass +++ b/meta/classes/base.bbclass | |||
@@ -180,6 +180,44 @@ python base_do_unpack() { | |||
180 | bb.fatal("Bitbake Fetcher Error: " + repr(e)) | 180 | bb.fatal("Bitbake Fetcher Error: " + repr(e)) |
181 | } | 181 | } |
182 | 182 | ||
183 | SSTATETASKS += "do_deploy_source_date_epoch" | ||
184 | |||
185 | do_deploy_source_date_epoch () { | ||
186 | mkdir -p ${SDE_DEPLOYDIR} | ||
187 | if [ -e ${SDE_FILE} ]; then | ||
188 | echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}." | ||
189 | cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt | ||
190 | else | ||
191 | echo "${SDE_FILE} not found!" | ||
192 | fi | ||
193 | } | ||
194 | |||
195 | python do_deploy_source_date_epoch_setscene () { | ||
196 | sstate_setscene(d) | ||
197 | bb.utils.mkdirhier(d.getVar('SDE_DIR')) | ||
198 | sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt') | ||
199 | if os.path.exists(sde_file): | ||
200 | target = d.getVar('SDE_FILE') | ||
201 | bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target)) | ||
202 | bb.utils.rename(sde_file, target) | ||
203 | else: | ||
204 | bb.debug(1, "%s not found!" % sde_file) | ||
205 | } | ||
206 | |||
207 | do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}" | ||
208 | do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}" | ||
209 | addtask do_deploy_source_date_epoch_setscene | ||
210 | addtask do_deploy_source_date_epoch before do_configure after do_patch | ||
211 | |||
212 | python create_source_date_epoch_stamp() { | ||
213 | source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S')) | ||
214 | oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d) | ||
215 | } | ||
216 | do_unpack[postfuncs] += "create_source_date_epoch_stamp" | ||
217 | |||
218 | def get_source_date_epoch_value(d): | ||
219 | return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d) | ||
220 | |||
183 | def get_layers_branch_rev(d): | 221 | def get_layers_branch_rev(d): |
184 | layers = (d.getVar("BBLAYERS") or "").split() | 222 | layers = (d.getVar("BBLAYERS") or "").split() |
185 | layers_branch_rev = ["%-20s = \"%s:%s\"" % (os.path.basename(i), \ | 223 | layers_branch_rev = ["%-20s = \"%s:%s\"" % (os.path.basename(i), \ |
diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass index ad93b2d2ab..abfe24bace 100644 --- a/meta/classes/externalsrc.bbclass +++ b/meta/classes/externalsrc.bbclass | |||
@@ -109,8 +109,8 @@ python () { | |||
109 | if local_srcuri and task in fetch_tasks: | 109 | if local_srcuri and task in fetch_tasks: |
110 | continue | 110 | continue |
111 | bb.build.deltask(task, d) | 111 | bb.build.deltask(task, d) |
112 | if bb.data.inherits_class('reproducible_build', d) and task == 'do_unpack': | 112 | if task == 'do_unpack': |
113 | # The reproducible_build's create_source_date_epoch_stamp function must | 113 | # The reproducible build create_source_date_epoch_stamp function must |
114 | # be run after the source is available and before the | 114 | # be run after the source is available and before the |
115 | # do_deploy_source_date_epoch task. In the normal case, it's attached | 115 | # do_deploy_source_date_epoch task. In the normal case, it's attached |
116 | # to do_unpack as a postfuncs, but since we removed do_unpack (above) | 116 | # to do_unpack as a postfuncs, but since we removed do_unpack (above) |
diff --git a/meta/classes/image-artifact-names.bbclass b/meta/classes/image-artifact-names.bbclass index 67f04e8165..f5769e520f 100644 --- a/meta/classes/image-artifact-names.bbclass +++ b/meta/classes/image-artifact-names.bbclass | |||
@@ -15,7 +15,7 @@ IMAGE_LINK_NAME ?= "${IMAGE_BASENAME}-${MACHINE}" | |||
15 | IMAGE_NAME_SUFFIX ??= ".rootfs" | 15 | IMAGE_NAME_SUFFIX ??= ".rootfs" |
16 | 16 | ||
17 | python () { | 17 | python () { |
18 | if bb.data.inherits_class('reproducible_build', d) and bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}": | 18 | if bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}": |
19 | import datetime | 19 | import datetime |
20 | d.setVar("IMAGE_VERSION_SUFFIX", "-" + datetime.datetime.fromtimestamp(int(d.getVar("SOURCE_DATE_EPOCH")), datetime.timezone.utc).strftime('%Y%m%d%H%M%S')) | 20 | d.setVar("IMAGE_VERSION_SUFFIX", "-" + datetime.datetime.fromtimestamp(int(d.getVar("SOURCE_DATE_EPOCH")), datetime.timezone.utc).strftime('%Y%m%d%H%M%S')) |
21 | d.setVarFlag("IMAGE_VERSION_SUFFIX", "vardepvalue", "") | 21 | d.setVarFlag("IMAGE_VERSION_SUFFIX", "vardepvalue", "") |
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass index 65dbe6c392..1ae6393d37 100644 --- a/meta/classes/package_deb.bbclass +++ b/meta/classes/package_deb.bbclass | |||
@@ -315,8 +315,7 @@ python do_package_write_deb () { | |||
315 | do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}" | 315 | do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}" |
316 | do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}" | 316 | do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}" |
317 | do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" | 317 | do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" |
318 | EPOCHTASK ??= "" | 318 | addtask package_write_deb after do_packagedata do_package do_deploy_source_date_epoch before do_build |
319 | addtask package_write_deb after do_packagedata do_package ${EPOCHTASK} before do_build | ||
320 | 319 | ||
321 | PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot" | 320 | PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot" |
322 | PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot" | 321 | PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot" |
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass index 776fe8ed23..902b7f94c8 100644 --- a/meta/classes/package_ipk.bbclass +++ b/meta/classes/package_ipk.bbclass | |||
@@ -274,8 +274,7 @@ python do_package_write_ipk () { | |||
274 | do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}" | 274 | do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}" |
275 | do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" | 275 | do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" |
276 | do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" | 276 | do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" |
277 | EPOCHTASK ??= "" | 277 | addtask package_write_ipk after do_packagedata do_package do_deploy_source_date_epoch before do_build |
278 | addtask package_write_ipk after do_packagedata do_package ${EPOCHTASK} before do_build | ||
279 | 278 | ||
280 | PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" | 279 | PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" |
281 | PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot" | 280 | PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot" |
diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass index e738806fd7..b0754421a3 100644 --- a/meta/classes/package_rpm.bbclass +++ b/meta/classes/package_rpm.bbclass | |||
@@ -748,8 +748,7 @@ python do_package_write_rpm () { | |||
748 | do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}" | 748 | do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}" |
749 | do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" | 749 | do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" |
750 | do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" | 750 | do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" |
751 | EPOCHTASK ??= "" | 751 | addtask package_write_rpm after do_packagedata do_package do_deploy_source_date_epoch before do_build |
752 | addtask package_write_rpm after do_packagedata do_package ${EPOCHTASK} before do_build | ||
753 | 752 | ||
754 | PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" | 753 | PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" |
755 | PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot" | 754 | PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot" |
diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass deleted file mode 100644 index f38be1a765..0000000000 --- a/meta/classes/reproducible_build.bbclass +++ /dev/null | |||
@@ -1,96 +0,0 @@ | |||
1 | # reproducible_build.bbclass | ||
2 | # | ||
3 | # Sets the default SOURCE_DATE_EPOCH in each component's build environment. | ||
4 | # The format is number of seconds since the system epoch. | ||
5 | # | ||
6 | # Upstream components (generally) respect this environment variable, | ||
7 | # using it in place of the "current" date and time. | ||
8 | # See https://reproducible-builds.org/specs/source-date-epoch/ | ||
9 | # | ||
10 | # The default value of SOURCE_DATE_EPOCH comes from the function | ||
11 | # get_source_date_epoch_value which reads from the SDE_FILE, or if the file | ||
12 | # is not available (or set to 0) will use the fallback of | ||
13 | # SOURCE_DATE_EPOCH_FALLBACK. | ||
14 | # | ||
15 | # The SDE_FILE is normally constructed from the function | ||
16 | # create_source_date_epoch_stamp which is typically added as a postfuncs to | ||
17 | # the do_unpack task. If a recipe does NOT have do_unpack, it should be added | ||
18 | # to a task that runs after the source is available and before the | ||
19 | # do_deploy_source_date_epoch task is executed. | ||
20 | # | ||
21 | # If a recipe wishes to override the default behavior it should set it's own | ||
22 | # SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task | ||
23 | # with recipe-specific functionality to write the appropriate | ||
24 | # SOURCE_DATE_EPOCH into the SDE_FILE. | ||
25 | # | ||
26 | # SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should | ||
27 | # be reproducible for anyone who builds the same revision from the same | ||
28 | # sources. | ||
29 | # | ||
30 | # There are 4 ways the create_source_date_epoch_stamp function determines what | ||
31 | # becomes SOURCE_DATE_EPOCH: | ||
32 | # | ||
33 | # 1. Use the value from __source_date_epoch.txt file if this file exists. | ||
34 | # This file was most likely created in the previous build by one of the | ||
35 | # following methods 2,3,4. | ||
36 | # Alternatively, it can be provided by a recipe via SRC_URI. | ||
37 | # | ||
38 | # If the file does not exist: | ||
39 | # | ||
40 | # 2. If there is a git checkout, use the last git commit timestamp. | ||
41 | # Git does not preserve file timestamps on checkout. | ||
42 | # | ||
43 | # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... | ||
44 | # This works for well-kept repositories distributed via tarball. | ||
45 | # | ||
46 | # 4. Use the modification time of the youngest file in the source tree, if | ||
47 | # there is one. | ||
48 | # This will be the newest file from the distribution tarball, if any. | ||
49 | # | ||
50 | # 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). | ||
51 | # | ||
52 | # Once the value is determined, it is stored in the recipe's SDE_FILE. | ||
53 | |||
54 | |||
55 | SSTATETASKS += "do_deploy_source_date_epoch" | ||
56 | |||
57 | do_deploy_source_date_epoch () { | ||
58 | mkdir -p ${SDE_DEPLOYDIR} | ||
59 | if [ -e ${SDE_FILE} ]; then | ||
60 | echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}." | ||
61 | cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt | ||
62 | else | ||
63 | echo "${SDE_FILE} not found!" | ||
64 | fi | ||
65 | } | ||
66 | |||
67 | python do_deploy_source_date_epoch_setscene () { | ||
68 | sstate_setscene(d) | ||
69 | bb.utils.mkdirhier(d.getVar('SDE_DIR')) | ||
70 | sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt') | ||
71 | if os.path.exists(sde_file): | ||
72 | target = d.getVar('SDE_FILE') | ||
73 | bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target)) | ||
74 | bb.utils.rename(sde_file, target) | ||
75 | else: | ||
76 | bb.debug(1, "%s not found!" % sde_file) | ||
77 | } | ||
78 | |||
79 | do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}" | ||
80 | do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}" | ||
81 | addtask do_deploy_source_date_epoch_setscene | ||
82 | addtask do_deploy_source_date_epoch before do_configure after do_patch | ||
83 | |||
84 | python create_source_date_epoch_stamp() { | ||
85 | source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S')) | ||
86 | oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d) | ||
87 | } | ||
88 | |||
89 | EPOCHTASK = "do_deploy_source_date_epoch" | ||
90 | |||
91 | # Generate the stamp after do_unpack runs | ||
92 | do_unpack[postfuncs] += "create_source_date_epoch_stamp" | ||
93 | |||
94 | def get_source_date_epoch_value(d): | ||
95 | return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d) | ||
96 | |||
diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py index a5000574cf..4fb99d963c 100644 --- a/meta/lib/oe/reproducible.py +++ b/meta/lib/oe/reproducible.py | |||
@@ -5,6 +5,57 @@ import os | |||
5 | import subprocess | 5 | import subprocess |
6 | import bb | 6 | import bb |
7 | 7 | ||
8 | # For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each | ||
9 | # component's build environment. The format is number of seconds since the | ||
10 | # system epoch. | ||
11 | # | ||
12 | # Upstream components (generally) respect this environment variable, | ||
13 | # using it in place of the "current" date and time. | ||
14 | # See https://reproducible-builds.org/specs/source-date-epoch/ | ||
15 | # | ||
16 | # The default value of SOURCE_DATE_EPOCH comes from the function | ||
17 | # get_source_date_epoch_value which reads from the SDE_FILE, or if the file | ||
18 | # is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK. | ||
19 | # | ||
20 | # The SDE_FILE is normally constructed from the function | ||
21 | # create_source_date_epoch_stamp which is typically added as a postfuncs to | ||
22 | # the do_unpack task. If a recipe does NOT have do_unpack, it should be added | ||
23 | # to a task that runs after the source is available and before the | ||
24 | # do_deploy_source_date_epoch task is executed. | ||
25 | # | ||
26 | # If a recipe wishes to override the default behavior it should set it's own | ||
27 | # SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task | ||
28 | # with recipe-specific functionality to write the appropriate | ||
29 | # SOURCE_DATE_EPOCH into the SDE_FILE. | ||
30 | # | ||
31 | # SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should | ||
32 | # be reproducible for anyone who builds the same revision from the same | ||
33 | # sources. | ||
34 | # | ||
35 | # There are 4 ways the create_source_date_epoch_stamp function determines what | ||
36 | # becomes SOURCE_DATE_EPOCH: | ||
37 | # | ||
38 | # 1. Use the value from __source_date_epoch.txt file if this file exists. | ||
39 | # This file was most likely created in the previous build by one of the | ||
40 | # following methods 2,3,4. | ||
41 | # Alternatively, it can be provided by a recipe via SRC_URI. | ||
42 | # | ||
43 | # If the file does not exist: | ||
44 | # | ||
45 | # 2. If there is a git checkout, use the last git commit timestamp. | ||
46 | # Git does not preserve file timestamps on checkout. | ||
47 | # | ||
48 | # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... | ||
49 | # This works for well-kept repositories distributed via tarball. | ||
50 | # | ||
51 | # 4. Use the modification time of the youngest file in the source tree, if | ||
52 | # there is one. | ||
53 | # This will be the newest file from the distribution tarball, if any. | ||
54 | # | ||
55 | # 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). | ||
56 | # | ||
57 | # Once the value is determined, it is stored in the recipe's SDE_FILE. | ||
58 | |||
8 | def get_source_date_epoch_from_known_files(d, sourcedir): | 59 | def get_source_date_epoch_from_known_files(d, sourcedir): |
9 | source_date_epoch = None | 60 | source_date_epoch = None |
10 | newest_file = None | 61 | newest_file = None |
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py index e4582cb82a..2e983d2f17 100644 --- a/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/meta/lib/oeqa/selftest/cases/reproducible.py | |||
@@ -219,7 +219,6 @@ class ReproducibleTests(OESelftestTestCase): | |||
219 | bb.utils.remove(tmpdir, recurse=True) | 219 | bb.utils.remove(tmpdir, recurse=True) |
220 | 220 | ||
221 | config = textwrap.dedent('''\ | 221 | config = textwrap.dedent('''\ |
222 | INHERIT += "reproducible_build" | ||
223 | PACKAGE_CLASSES = "{package_classes}" | 222 | PACKAGE_CLASSES = "{package_classes}" |
224 | INHIBIT_PACKAGE_STRIP = "1" | 223 | INHIBIT_PACKAGE_STRIP = "1" |
225 | TMPDIR = "{tmpdir}" | 224 | TMPDIR = "{tmpdir}" |