diff options
| author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-10-14 12:34:17 +0100 |
|---|---|---|
| committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-10-16 17:41:59 +0100 |
| commit | 4b424bdfdbbe16b6e41f6cd1de75a51e3799cf94 (patch) | |
| tree | f59682b14ffa5de9fef7dd62b88dddd43e3de400 /meta | |
| parent | f47a396964c4de76838f8c59b8226625817aa369 (diff) | |
| download | poky-4b424bdfdbbe16b6e41f6cd1de75a51e3799cf94.tar.gz | |
reproducible: Merge code into base.bbclass
Reproducibility is here to stay and needs to be part of our default workflow.
Move the remaining code to base.bbclass so it is always a first class citizen
and it is clear people need to be mindful of it.
(From OE-Core rev: abb0671d2cebfd7e8df94796404bbe9c7f961058)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
| -rw-r--r-- | meta/classes/base.bbclass | 38 | ||||
| -rw-r--r-- | meta/classes/externalsrc.bbclass | 4 | ||||
| -rw-r--r-- | meta/classes/image-artifact-names.bbclass | 2 | ||||
| -rw-r--r-- | meta/classes/package_deb.bbclass | 3 | ||||
| -rw-r--r-- | meta/classes/package_ipk.bbclass | 3 | ||||
| -rw-r--r-- | meta/classes/package_rpm.bbclass | 3 | ||||
| -rw-r--r-- | meta/classes/reproducible_build.bbclass | 96 | ||||
| -rw-r--r-- | meta/lib/oe/reproducible.py | 51 | ||||
| -rw-r--r-- | meta/lib/oeqa/selftest/cases/reproducible.py | 1 |
9 files changed, 95 insertions, 106 deletions
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass index bca3944ae7..a65fcc6c1d 100644 --- a/meta/classes/base.bbclass +++ b/meta/classes/base.bbclass | |||
| @@ -180,6 +180,44 @@ python base_do_unpack() { | |||
| 180 | bb.fatal("Bitbake Fetcher Error: " + repr(e)) | 180 | bb.fatal("Bitbake Fetcher Error: " + repr(e)) |
| 181 | } | 181 | } |
| 182 | 182 | ||
| 183 | SSTATETASKS += "do_deploy_source_date_epoch" | ||
| 184 | |||
| 185 | do_deploy_source_date_epoch () { | ||
| 186 | mkdir -p ${SDE_DEPLOYDIR} | ||
| 187 | if [ -e ${SDE_FILE} ]; then | ||
| 188 | echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}." | ||
| 189 | cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt | ||
| 190 | else | ||
| 191 | echo "${SDE_FILE} not found!" | ||
| 192 | fi | ||
| 193 | } | ||
| 194 | |||
| 195 | python do_deploy_source_date_epoch_setscene () { | ||
| 196 | sstate_setscene(d) | ||
| 197 | bb.utils.mkdirhier(d.getVar('SDE_DIR')) | ||
| 198 | sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt') | ||
| 199 | if os.path.exists(sde_file): | ||
| 200 | target = d.getVar('SDE_FILE') | ||
| 201 | bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target)) | ||
| 202 | bb.utils.rename(sde_file, target) | ||
| 203 | else: | ||
| 204 | bb.debug(1, "%s not found!" % sde_file) | ||
| 205 | } | ||
| 206 | |||
| 207 | do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}" | ||
| 208 | do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}" | ||
| 209 | addtask do_deploy_source_date_epoch_setscene | ||
| 210 | addtask do_deploy_source_date_epoch before do_configure after do_patch | ||
| 211 | |||
| 212 | python create_source_date_epoch_stamp() { | ||
| 213 | source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S')) | ||
| 214 | oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d) | ||
| 215 | } | ||
| 216 | do_unpack[postfuncs] += "create_source_date_epoch_stamp" | ||
| 217 | |||
| 218 | def get_source_date_epoch_value(d): | ||
| 219 | return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d) | ||
| 220 | |||
| 183 | def get_layers_branch_rev(d): | 221 | def get_layers_branch_rev(d): |
| 184 | layers = (d.getVar("BBLAYERS") or "").split() | 222 | layers = (d.getVar("BBLAYERS") or "").split() |
| 185 | layers_branch_rev = ["%-20s = \"%s:%s\"" % (os.path.basename(i), \ | 223 | layers_branch_rev = ["%-20s = \"%s:%s\"" % (os.path.basename(i), \ |
diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass index ad93b2d2ab..abfe24bace 100644 --- a/meta/classes/externalsrc.bbclass +++ b/meta/classes/externalsrc.bbclass | |||
| @@ -109,8 +109,8 @@ python () { | |||
| 109 | if local_srcuri and task in fetch_tasks: | 109 | if local_srcuri and task in fetch_tasks: |
| 110 | continue | 110 | continue |
| 111 | bb.build.deltask(task, d) | 111 | bb.build.deltask(task, d) |
| 112 | if bb.data.inherits_class('reproducible_build', d) and task == 'do_unpack': | 112 | if task == 'do_unpack': |
| 113 | # The reproducible_build's create_source_date_epoch_stamp function must | 113 | # The reproducible build create_source_date_epoch_stamp function must |
| 114 | # be run after the source is available and before the | 114 | # be run after the source is available and before the |
| 115 | # do_deploy_source_date_epoch task. In the normal case, it's attached | 115 | # do_deploy_source_date_epoch task. In the normal case, it's attached |
| 116 | # to do_unpack as a postfuncs, but since we removed do_unpack (above) | 116 | # to do_unpack as a postfuncs, but since we removed do_unpack (above) |
diff --git a/meta/classes/image-artifact-names.bbclass b/meta/classes/image-artifact-names.bbclass index 67f04e8165..f5769e520f 100644 --- a/meta/classes/image-artifact-names.bbclass +++ b/meta/classes/image-artifact-names.bbclass | |||
| @@ -15,7 +15,7 @@ IMAGE_LINK_NAME ?= "${IMAGE_BASENAME}-${MACHINE}" | |||
| 15 | IMAGE_NAME_SUFFIX ??= ".rootfs" | 15 | IMAGE_NAME_SUFFIX ??= ".rootfs" |
| 16 | 16 | ||
| 17 | python () { | 17 | python () { |
| 18 | if bb.data.inherits_class('reproducible_build', d) and bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}": | 18 | if bb.data.inherits_class('deploy', d) and d.getVar("IMAGE_VERSION_SUFFIX") == "-${DATETIME}": |
| 19 | import datetime | 19 | import datetime |
| 20 | d.setVar("IMAGE_VERSION_SUFFIX", "-" + datetime.datetime.fromtimestamp(int(d.getVar("SOURCE_DATE_EPOCH")), datetime.timezone.utc).strftime('%Y%m%d%H%M%S')) | 20 | d.setVar("IMAGE_VERSION_SUFFIX", "-" + datetime.datetime.fromtimestamp(int(d.getVar("SOURCE_DATE_EPOCH")), datetime.timezone.utc).strftime('%Y%m%d%H%M%S')) |
| 21 | d.setVarFlag("IMAGE_VERSION_SUFFIX", "vardepvalue", "") | 21 | d.setVarFlag("IMAGE_VERSION_SUFFIX", "vardepvalue", "") |
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass index 65dbe6c392..1ae6393d37 100644 --- a/meta/classes/package_deb.bbclass +++ b/meta/classes/package_deb.bbclass | |||
| @@ -315,8 +315,7 @@ python do_package_write_deb () { | |||
| 315 | do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}" | 315 | do_package_write_deb[dirs] = "${PKGWRITEDIRDEB}" |
| 316 | do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}" | 316 | do_package_write_deb[cleandirs] = "${PKGWRITEDIRDEB}" |
| 317 | do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" | 317 | do_package_write_deb[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" |
| 318 | EPOCHTASK ??= "" | 318 | addtask package_write_deb after do_packagedata do_package do_deploy_source_date_epoch before do_build |
| 319 | addtask package_write_deb after do_packagedata do_package ${EPOCHTASK} before do_build | ||
| 320 | 319 | ||
| 321 | PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot" | 320 | PACKAGEINDEXDEPS += "dpkg-native:do_populate_sysroot" |
| 322 | PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot" | 321 | PACKAGEINDEXDEPS += "apt-native:do_populate_sysroot" |
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass index 776fe8ed23..902b7f94c8 100644 --- a/meta/classes/package_ipk.bbclass +++ b/meta/classes/package_ipk.bbclass | |||
| @@ -274,8 +274,7 @@ python do_package_write_ipk () { | |||
| 274 | do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}" | 274 | do_package_write_ipk[dirs] = "${PKGWRITEDIRIPK}" |
| 275 | do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" | 275 | do_package_write_ipk[cleandirs] = "${PKGWRITEDIRIPK}" |
| 276 | do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" | 276 | do_package_write_ipk[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" |
| 277 | EPOCHTASK ??= "" | 277 | addtask package_write_ipk after do_packagedata do_package do_deploy_source_date_epoch before do_build |
| 278 | addtask package_write_ipk after do_packagedata do_package ${EPOCHTASK} before do_build | ||
| 279 | 278 | ||
| 280 | PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" | 279 | PACKAGEINDEXDEPS += "opkg-utils-native:do_populate_sysroot" |
| 281 | PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot" | 280 | PACKAGEINDEXDEPS += "opkg-native:do_populate_sysroot" |
diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass index e738806fd7..b0754421a3 100644 --- a/meta/classes/package_rpm.bbclass +++ b/meta/classes/package_rpm.bbclass | |||
| @@ -748,8 +748,7 @@ python do_package_write_rpm () { | |||
| 748 | do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}" | 748 | do_package_write_rpm[dirs] = "${PKGWRITEDIRRPM}" |
| 749 | do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" | 749 | do_package_write_rpm[cleandirs] = "${PKGWRITEDIRRPM}" |
| 750 | do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" | 750 | do_package_write_rpm[depends] += "${@oe.utils.build_depends_string(d.getVar('PACKAGE_WRITE_DEPS'), 'do_populate_sysroot')}" |
| 751 | EPOCHTASK ??= "" | 751 | addtask package_write_rpm after do_packagedata do_package do_deploy_source_date_epoch before do_build |
| 752 | addtask package_write_rpm after do_packagedata do_package ${EPOCHTASK} before do_build | ||
| 753 | 752 | ||
| 754 | PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" | 753 | PACKAGEINDEXDEPS += "rpm-native:do_populate_sysroot" |
| 755 | PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot" | 754 | PACKAGEINDEXDEPS += "createrepo-c-native:do_populate_sysroot" |
diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass deleted file mode 100644 index f38be1a765..0000000000 --- a/meta/classes/reproducible_build.bbclass +++ /dev/null | |||
| @@ -1,96 +0,0 @@ | |||
| 1 | # reproducible_build.bbclass | ||
| 2 | # | ||
| 3 | # Sets the default SOURCE_DATE_EPOCH in each component's build environment. | ||
| 4 | # The format is number of seconds since the system epoch. | ||
| 5 | # | ||
| 6 | # Upstream components (generally) respect this environment variable, | ||
| 7 | # using it in place of the "current" date and time. | ||
| 8 | # See https://reproducible-builds.org/specs/source-date-epoch/ | ||
| 9 | # | ||
| 10 | # The default value of SOURCE_DATE_EPOCH comes from the function | ||
| 11 | # get_source_date_epoch_value which reads from the SDE_FILE, or if the file | ||
| 12 | # is not available (or set to 0) will use the fallback of | ||
| 13 | # SOURCE_DATE_EPOCH_FALLBACK. | ||
| 14 | # | ||
| 15 | # The SDE_FILE is normally constructed from the function | ||
| 16 | # create_source_date_epoch_stamp which is typically added as a postfuncs to | ||
| 17 | # the do_unpack task. If a recipe does NOT have do_unpack, it should be added | ||
| 18 | # to a task that runs after the source is available and before the | ||
| 19 | # do_deploy_source_date_epoch task is executed. | ||
| 20 | # | ||
| 21 | # If a recipe wishes to override the default behavior it should set it's own | ||
| 22 | # SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task | ||
| 23 | # with recipe-specific functionality to write the appropriate | ||
| 24 | # SOURCE_DATE_EPOCH into the SDE_FILE. | ||
| 25 | # | ||
| 26 | # SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should | ||
| 27 | # be reproducible for anyone who builds the same revision from the same | ||
| 28 | # sources. | ||
| 29 | # | ||
| 30 | # There are 4 ways the create_source_date_epoch_stamp function determines what | ||
| 31 | # becomes SOURCE_DATE_EPOCH: | ||
| 32 | # | ||
| 33 | # 1. Use the value from __source_date_epoch.txt file if this file exists. | ||
| 34 | # This file was most likely created in the previous build by one of the | ||
| 35 | # following methods 2,3,4. | ||
| 36 | # Alternatively, it can be provided by a recipe via SRC_URI. | ||
| 37 | # | ||
| 38 | # If the file does not exist: | ||
| 39 | # | ||
| 40 | # 2. If there is a git checkout, use the last git commit timestamp. | ||
| 41 | # Git does not preserve file timestamps on checkout. | ||
| 42 | # | ||
| 43 | # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... | ||
| 44 | # This works for well-kept repositories distributed via tarball. | ||
| 45 | # | ||
| 46 | # 4. Use the modification time of the youngest file in the source tree, if | ||
| 47 | # there is one. | ||
| 48 | # This will be the newest file from the distribution tarball, if any. | ||
| 49 | # | ||
| 50 | # 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). | ||
| 51 | # | ||
| 52 | # Once the value is determined, it is stored in the recipe's SDE_FILE. | ||
| 53 | |||
| 54 | |||
| 55 | SSTATETASKS += "do_deploy_source_date_epoch" | ||
| 56 | |||
| 57 | do_deploy_source_date_epoch () { | ||
| 58 | mkdir -p ${SDE_DEPLOYDIR} | ||
| 59 | if [ -e ${SDE_FILE} ]; then | ||
| 60 | echo "Deploying SDE from ${SDE_FILE} -> ${SDE_DEPLOYDIR}." | ||
| 61 | cp -p ${SDE_FILE} ${SDE_DEPLOYDIR}/__source_date_epoch.txt | ||
| 62 | else | ||
| 63 | echo "${SDE_FILE} not found!" | ||
| 64 | fi | ||
| 65 | } | ||
| 66 | |||
| 67 | python do_deploy_source_date_epoch_setscene () { | ||
| 68 | sstate_setscene(d) | ||
| 69 | bb.utils.mkdirhier(d.getVar('SDE_DIR')) | ||
| 70 | sde_file = os.path.join(d.getVar('SDE_DEPLOYDIR'), '__source_date_epoch.txt') | ||
| 71 | if os.path.exists(sde_file): | ||
| 72 | target = d.getVar('SDE_FILE') | ||
| 73 | bb.debug(1, "Moving setscene SDE file %s -> %s" % (sde_file, target)) | ||
| 74 | bb.utils.rename(sde_file, target) | ||
| 75 | else: | ||
| 76 | bb.debug(1, "%s not found!" % sde_file) | ||
| 77 | } | ||
| 78 | |||
| 79 | do_deploy_source_date_epoch[dirs] = "${SDE_DEPLOYDIR}" | ||
| 80 | do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}" | ||
| 81 | addtask do_deploy_source_date_epoch_setscene | ||
| 82 | addtask do_deploy_source_date_epoch before do_configure after do_patch | ||
| 83 | |||
| 84 | python create_source_date_epoch_stamp() { | ||
| 85 | source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S')) | ||
| 86 | oe.reproducible.epochfile_write(source_date_epoch, d.getVar('SDE_FILE'), d) | ||
| 87 | } | ||
| 88 | |||
| 89 | EPOCHTASK = "do_deploy_source_date_epoch" | ||
| 90 | |||
| 91 | # Generate the stamp after do_unpack runs | ||
| 92 | do_unpack[postfuncs] += "create_source_date_epoch_stamp" | ||
| 93 | |||
| 94 | def get_source_date_epoch_value(d): | ||
| 95 | return oe.reproducible.epochfile_read(d.getVar('SDE_FILE'), d) | ||
| 96 | |||
diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py index a5000574cf..4fb99d963c 100644 --- a/meta/lib/oe/reproducible.py +++ b/meta/lib/oe/reproducible.py | |||
| @@ -5,6 +5,57 @@ import os | |||
| 5 | import subprocess | 5 | import subprocess |
| 6 | import bb | 6 | import bb |
| 7 | 7 | ||
| 8 | # For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each | ||
| 9 | # component's build environment. The format is number of seconds since the | ||
| 10 | # system epoch. | ||
| 11 | # | ||
| 12 | # Upstream components (generally) respect this environment variable, | ||
| 13 | # using it in place of the "current" date and time. | ||
| 14 | # See https://reproducible-builds.org/specs/source-date-epoch/ | ||
| 15 | # | ||
| 16 | # The default value of SOURCE_DATE_EPOCH comes from the function | ||
| 17 | # get_source_date_epoch_value which reads from the SDE_FILE, or if the file | ||
| 18 | # is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK. | ||
| 19 | # | ||
| 20 | # The SDE_FILE is normally constructed from the function | ||
| 21 | # create_source_date_epoch_stamp which is typically added as a postfuncs to | ||
| 22 | # the do_unpack task. If a recipe does NOT have do_unpack, it should be added | ||
| 23 | # to a task that runs after the source is available and before the | ||
| 24 | # do_deploy_source_date_epoch task is executed. | ||
| 25 | # | ||
| 26 | # If a recipe wishes to override the default behavior it should set it's own | ||
| 27 | # SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task | ||
| 28 | # with recipe-specific functionality to write the appropriate | ||
| 29 | # SOURCE_DATE_EPOCH into the SDE_FILE. | ||
| 30 | # | ||
| 31 | # SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should | ||
| 32 | # be reproducible for anyone who builds the same revision from the same | ||
| 33 | # sources. | ||
| 34 | # | ||
| 35 | # There are 4 ways the create_source_date_epoch_stamp function determines what | ||
| 36 | # becomes SOURCE_DATE_EPOCH: | ||
| 37 | # | ||
| 38 | # 1. Use the value from __source_date_epoch.txt file if this file exists. | ||
| 39 | # This file was most likely created in the previous build by one of the | ||
| 40 | # following methods 2,3,4. | ||
| 41 | # Alternatively, it can be provided by a recipe via SRC_URI. | ||
| 42 | # | ||
| 43 | # If the file does not exist: | ||
| 44 | # | ||
| 45 | # 2. If there is a git checkout, use the last git commit timestamp. | ||
| 46 | # Git does not preserve file timestamps on checkout. | ||
| 47 | # | ||
| 48 | # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... | ||
| 49 | # This works for well-kept repositories distributed via tarball. | ||
| 50 | # | ||
| 51 | # 4. Use the modification time of the youngest file in the source tree, if | ||
| 52 | # there is one. | ||
| 53 | # This will be the newest file from the distribution tarball, if any. | ||
| 54 | # | ||
| 55 | # 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). | ||
| 56 | # | ||
| 57 | # Once the value is determined, it is stored in the recipe's SDE_FILE. | ||
| 58 | |||
| 8 | def get_source_date_epoch_from_known_files(d, sourcedir): | 59 | def get_source_date_epoch_from_known_files(d, sourcedir): |
| 9 | source_date_epoch = None | 60 | source_date_epoch = None |
| 10 | newest_file = None | 61 | newest_file = None |
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py index e4582cb82a..2e983d2f17 100644 --- a/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/meta/lib/oeqa/selftest/cases/reproducible.py | |||
| @@ -219,7 +219,6 @@ class ReproducibleTests(OESelftestTestCase): | |||
| 219 | bb.utils.remove(tmpdir, recurse=True) | 219 | bb.utils.remove(tmpdir, recurse=True) |
| 220 | 220 | ||
| 221 | config = textwrap.dedent('''\ | 221 | config = textwrap.dedent('''\ |
| 222 | INHERIT += "reproducible_build" | ||
| 223 | PACKAGE_CLASSES = "{package_classes}" | 222 | PACKAGE_CLASSES = "{package_classes}" |
| 224 | INHIBIT_PACKAGE_STRIP = "1" | 223 | INHIBIT_PACKAGE_STRIP = "1" |
| 225 | TMPDIR = "{tmpdir}" | 224 | TMPDIR = "{tmpdir}" |
