diff options
Diffstat (limited to 'meta/lib/oe/reproducible.py')
-rw-r--r-- | meta/lib/oe/reproducible.py | 97 |
1 files changed, 93 insertions, 4 deletions
diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py index 204b9bd734..a9f717159e 100644 --- a/meta/lib/oe/reproducible.py +++ b/meta/lib/oe/reproducible.py | |||
@@ -1,10 +1,63 @@ | |||
1 | # | 1 | # |
2 | # Copyright OpenEmbedded Contributors | ||
3 | # | ||
2 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
3 | # | 5 | # |
4 | import os | 6 | import os |
5 | import subprocess | 7 | import subprocess |
6 | import bb | 8 | import bb |
7 | 9 | ||
10 | # For reproducible builds, this code sets the default SOURCE_DATE_EPOCH in each | ||
11 | # component's build environment. The format is number of seconds since the | ||
12 | # system epoch. | ||
13 | # | ||
14 | # Upstream components (generally) respect this environment variable, | ||
15 | # using it in place of the "current" date and time. | ||
16 | # See https://reproducible-builds.org/specs/source-date-epoch/ | ||
17 | # | ||
18 | # The default value of SOURCE_DATE_EPOCH comes from the function | ||
19 | # get_source_date_epoch_value which reads from the SDE_FILE, or if the file | ||
20 | # is not available will use the fallback of SOURCE_DATE_EPOCH_FALLBACK. | ||
21 | # | ||
22 | # The SDE_FILE is normally constructed from the function | ||
23 | # create_source_date_epoch_stamp which is typically added as a postfuncs to | ||
24 | # the do_unpack task. If a recipe does NOT have do_unpack, it should be added | ||
25 | # to a task that runs after the source is available and before the | ||
26 | # do_deploy_source_date_epoch task is executed. | ||
27 | # | ||
28 | # If a recipe wishes to override the default behavior it should set it's own | ||
29 | # SOURCE_DATE_EPOCH or override the do_deploy_source_date_epoch_stamp task | ||
30 | # with recipe-specific functionality to write the appropriate | ||
31 | # SOURCE_DATE_EPOCH into the SDE_FILE. | ||
32 | # | ||
33 | # SOURCE_DATE_EPOCH is intended to be a reproducible value. This value should | ||
34 | # be reproducible for anyone who builds the same revision from the same | ||
35 | # sources. | ||
36 | # | ||
37 | # There are 4 ways the create_source_date_epoch_stamp function determines what | ||
38 | # becomes SOURCE_DATE_EPOCH: | ||
39 | # | ||
40 | # 1. Use the value from __source_date_epoch.txt file if this file exists. | ||
41 | # This file was most likely created in the previous build by one of the | ||
42 | # following methods 2,3,4. | ||
43 | # Alternatively, it can be provided by a recipe via SRC_URI. | ||
44 | # | ||
45 | # If the file does not exist: | ||
46 | # | ||
47 | # 2. If there is a git checkout, use the last git commit timestamp. | ||
48 | # Git does not preserve file timestamps on checkout. | ||
49 | # | ||
50 | # 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ... | ||
51 | # This works for well-kept repositories distributed via tarball. | ||
52 | # | ||
53 | # 4. Use the modification time of the youngest file in the source tree, if | ||
54 | # there is one. | ||
55 | # This will be the newest file from the distribution tarball, if any. | ||
56 | # | ||
57 | # 5. Fall back to a fixed timestamp (SOURCE_DATE_EPOCH_FALLBACK). | ||
58 | # | ||
59 | # Once the value is determined, it is stored in the recipe's SDE_FILE. | ||
60 | |||
8 | def get_source_date_epoch_from_known_files(d, sourcedir): | 61 | def get_source_date_epoch_from_known_files(d, sourcedir): |
9 | source_date_epoch = None | 62 | source_date_epoch = None |
10 | newest_file = None | 63 | newest_file = None |
@@ -41,7 +94,7 @@ def find_git_folder(d, sourcedir): | |||
41 | for root, dirs, files in os.walk(workdir, topdown=True): | 94 | for root, dirs, files in os.walk(workdir, topdown=True): |
42 | dirs[:] = [d for d in dirs if d not in exclude] | 95 | dirs[:] = [d for d in dirs if d not in exclude] |
43 | if '.git' in dirs: | 96 | if '.git' in dirs: |
44 | return root | 97 | return os.path.join(root, ".git") |
45 | 98 | ||
46 | bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) | 99 | bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) |
47 | return None | 100 | return None |
@@ -62,11 +115,12 @@ def get_source_date_epoch_from_git(d, sourcedir): | |||
62 | return None | 115 | return None |
63 | 116 | ||
64 | bb.debug(1, "git repository: %s" % gitpath) | 117 | bb.debug(1, "git repository: %s" % gitpath) |
65 | p = subprocess.run(['git', '--git-dir', gitpath, 'log', '-1', '--pretty=%ct'], check=True, stdout=subprocess.PIPE) | 118 | p = subprocess.run(['git', '-c', 'log.showSignature=false', '--git-dir', gitpath, 'log', '-1', '--pretty=%ct'], |
119 | check=True, stdout=subprocess.PIPE) | ||
66 | return int(p.stdout.decode('utf-8')) | 120 | return int(p.stdout.decode('utf-8')) |
67 | 121 | ||
68 | def get_source_date_epoch_from_youngest_file(d, sourcedir): | 122 | def get_source_date_epoch_from_youngest_file(d, sourcedir): |
69 | if sourcedir == d.getVar('WORKDIR'): | 123 | if sourcedir == d.getVar('UNPACKDIR'): |
70 | # These sources are almost certainly not from a tarball | 124 | # These sources are almost certainly not from a tarball |
71 | return None | 125 | return None |
72 | 126 | ||
@@ -77,6 +131,9 @@ def get_source_date_epoch_from_youngest_file(d, sourcedir): | |||
77 | files = [f for f in files if not f[0] == '.'] | 131 | files = [f for f in files if not f[0] == '.'] |
78 | 132 | ||
79 | for fname in files: | 133 | for fname in files: |
134 | if fname == "singletask.lock": | ||
135 | # Ignore externalsrc/devtool lockfile [YOCTO #14921] | ||
136 | continue | ||
80 | filename = os.path.join(root, fname) | 137 | filename = os.path.join(root, fname) |
81 | try: | 138 | try: |
82 | mtime = int(os.lstat(filename).st_mtime) | 139 | mtime = int(os.lstat(filename).st_mtime) |
@@ -101,8 +158,40 @@ def fixed_source_date_epoch(d): | |||
101 | def get_source_date_epoch(d, sourcedir): | 158 | def get_source_date_epoch(d, sourcedir): |
102 | return ( | 159 | return ( |
103 | get_source_date_epoch_from_git(d, sourcedir) or | 160 | get_source_date_epoch_from_git(d, sourcedir) or |
104 | get_source_date_epoch_from_known_files(d, sourcedir) or | ||
105 | get_source_date_epoch_from_youngest_file(d, sourcedir) or | 161 | get_source_date_epoch_from_youngest_file(d, sourcedir) or |
106 | fixed_source_date_epoch(d) # Last resort | 162 | fixed_source_date_epoch(d) # Last resort |
107 | ) | 163 | ) |
108 | 164 | ||
165 | def epochfile_read(epochfile, d): | ||
166 | cached, efile = d.getVar('__CACHED_SOURCE_DATE_EPOCH') or (None, None) | ||
167 | if cached and efile == epochfile: | ||
168 | return cached | ||
169 | |||
170 | if cached and epochfile != efile: | ||
171 | bb.debug(1, "Epoch file changed from %s to %s" % (efile, epochfile)) | ||
172 | |||
173 | source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) | ||
174 | try: | ||
175 | with open(epochfile, 'r') as f: | ||
176 | s = f.read() | ||
177 | try: | ||
178 | source_date_epoch = int(s) | ||
179 | except ValueError: | ||
180 | bb.warn("SOURCE_DATE_EPOCH value '%s' is invalid. Reverting to SOURCE_DATE_EPOCH_FALLBACK" % s) | ||
181 | source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK')) | ||
182 | bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) | ||
183 | except FileNotFoundError: | ||
184 | bb.debug(1, "Cannot find %s. SOURCE_DATE_EPOCH will default to %d" % (epochfile, source_date_epoch)) | ||
185 | |||
186 | d.setVar('__CACHED_SOURCE_DATE_EPOCH', (str(source_date_epoch), epochfile)) | ||
187 | return str(source_date_epoch) | ||
188 | |||
189 | def epochfile_write(source_date_epoch, epochfile, d): | ||
190 | |||
191 | bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) | ||
192 | bb.utils.mkdirhier(os.path.dirname(epochfile)) | ||
193 | |||
194 | tmp_file = "%s.new" % epochfile | ||
195 | with open(tmp_file, 'w') as f: | ||
196 | f.write(str(source_date_epoch)) | ||
197 | os.rename(tmp_file, epochfile) | ||