diff options
author | Joshua Watt <JPEWhacker@gmail.com> | 2020-07-20 12:56:31 -0500 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-11-22 13:27:37 +0000 |
commit | 9f46c1a1ad2efcebd8ddc34f3ef6be58706c126d (patch) | |
tree | 2cbe2b03cca172bb4ff9b55f47f29874736ab833 /meta | |
parent | 4cd047d6e33607ca32207daf61bedbfc9f0bed65 (diff) | |
download | poky-9f46c1a1ad2efcebd8ddc34f3ef6be58706c126d.tar.gz |
classes/reproducible: Move to library code
Moves most of the python code used for dealing with the source date
epoch to library code.
(From OE-Core rev: bb957547fbd3f6670220706642b49fee560c6b75)
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit a7ede90955bc0c8bec1cbb3cab498ef2583b2f4e)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r-- | meta/classes/reproducible_build.bbclass | 90 | ||||
-rw-r--r-- | meta/lib/oe/reproducible.py | 95 |
2 files changed, 98 insertions, 87 deletions
diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass index 8da40f656a..2f3bd90b07 100644 --- a/meta/classes/reproducible_build.bbclass +++ b/meta/classes/reproducible_build.bbclass | |||
@@ -70,100 +70,16 @@ do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DEPLOYDIR}" | |||
70 | addtask do_deploy_source_date_epoch_setscene | 70 | addtask do_deploy_source_date_epoch_setscene |
71 | addtask do_deploy_source_date_epoch before do_configure after do_patch | 71 | addtask do_deploy_source_date_epoch before do_configure after do_patch |
72 | 72 | ||
73 | def get_source_date_epoch_from_known_files(d, sourcedir): | ||
74 | source_date_epoch = None | ||
75 | newest_file = None | ||
76 | known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"]) | ||
77 | for file in known_files: | ||
78 | filepath = os.path.join(sourcedir, file) | ||
79 | if os.path.isfile(filepath): | ||
80 | mtime = int(os.lstat(filepath).st_mtime) | ||
81 | # There may be more than one "known_file" present, if so, use the youngest one | ||
82 | if not source_date_epoch or mtime > source_date_epoch: | ||
83 | source_date_epoch = mtime | ||
84 | newest_file = filepath | ||
85 | if newest_file: | ||
86 | bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file) | ||
87 | return source_date_epoch | ||
88 | |||
89 | def find_git_folder(d, sourcedir): | ||
90 | # First guess: WORKDIR/git | ||
91 | # This is the default git fetcher unpack path | ||
92 | workdir = d.getVar('WORKDIR') | ||
93 | gitpath = os.path.join(workdir, "git/.git") | ||
94 | if os.path.isdir(gitpath): | ||
95 | return gitpath | ||
96 | |||
97 | # Second guess: ${S} | ||
98 | gitpath = os.path.join(sourcedir, ".git") | ||
99 | if os.path.isdir(gitpath): | ||
100 | return gitpath | ||
101 | |||
102 | # Perhaps there was a subpath or destsuffix specified. | ||
103 | # Go looking in the WORKDIR | ||
104 | exclude = set(["build", "image", "license-destdir", "patches", "pseudo", | ||
105 | "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"]) | ||
106 | for root, dirs, files in os.walk(workdir, topdown=True): | ||
107 | dirs[:] = [d for d in dirs if d not in exclude] | ||
108 | if '.git' in dirs: | ||
109 | return root | ||
110 | |||
111 | bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) | ||
112 | return None | ||
113 | |||
114 | def get_source_date_epoch_from_git(d, sourcedir): | ||
115 | source_date_epoch = None | ||
116 | if "git://" in d.getVar('SRC_URI'): | ||
117 | gitpath = find_git_folder(d, sourcedir) | ||
118 | if gitpath: | ||
119 | import subprocess | ||
120 | source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath)) | ||
121 | bb.debug(1, "git repository: %s" % gitpath) | ||
122 | return source_date_epoch | ||
123 | |||
124 | def get_source_date_epoch_from_youngest_file(d, sourcedir): | ||
125 | if sourcedir == d.getVar('WORKDIR'): | ||
126 | # These sources are almost certainly not from a tarball | ||
127 | return None | ||
128 | |||
129 | # Do it the hard way: check all files and find the youngest one... | ||
130 | source_date_epoch = None | ||
131 | newest_file = None | ||
132 | for root, dirs, files in os.walk(sourcedir, topdown=True): | ||
133 | files = [f for f in files if not f[0] == '.'] | ||
134 | |||
135 | for fname in files: | ||
136 | filename = os.path.join(root, fname) | ||
137 | try: | ||
138 | mtime = int(os.lstat(filename).st_mtime) | ||
139 | except ValueError: | ||
140 | mtime = 0 | ||
141 | if not source_date_epoch or mtime > source_date_epoch: | ||
142 | source_date_epoch = mtime | ||
143 | newest_file = filename | ||
144 | |||
145 | if newest_file: | ||
146 | bb.debug(1, "Newest file found: %s" % newest_file) | ||
147 | return source_date_epoch | ||
148 | |||
149 | def fixed_source_date_epoch(): | ||
150 | bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH") | ||
151 | return 0 | ||
152 | |||
153 | python create_source_date_epoch_stamp() { | 73 | python create_source_date_epoch_stamp() { |
74 | import oe.reproducible | ||
75 | |||
154 | epochfile = d.getVar('SDE_FILE') | 76 | epochfile = d.getVar('SDE_FILE') |
155 | # If it exists we need to regenerate as the sources may have changed | 77 | # If it exists we need to regenerate as the sources may have changed |
156 | if os.path.isfile(epochfile): | 78 | if os.path.isfile(epochfile): |
157 | bb.debug(1, "Deleting existing SOURCE_DATE_EPOCH from: %s" % epochfile) | 79 | bb.debug(1, "Deleting existing SOURCE_DATE_EPOCH from: %s" % epochfile) |
158 | os.remove(epochfile) | 80 | os.remove(epochfile) |
159 | 81 | ||
160 | sourcedir = d.getVar('S') | 82 | source_date_epoch = oe.reproducible.get_source_date_epoch(d, d.getVar('S')) |
161 | source_date_epoch = ( | ||
162 | get_source_date_epoch_from_git(d, sourcedir) or | ||
163 | get_source_date_epoch_from_known_files(d, sourcedir) or | ||
164 | get_source_date_epoch_from_youngest_file(d, sourcedir) or | ||
165 | fixed_source_date_epoch() # Last resort | ||
166 | ) | ||
167 | 83 | ||
168 | bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) | 84 | bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch) |
169 | bb.utils.mkdirhier(d.getVar('SDE_DIR')) | 85 | bb.utils.mkdirhier(d.getVar('SDE_DIR')) |
diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py new file mode 100644 index 0000000000..f80a85ddef --- /dev/null +++ b/meta/lib/oe/reproducible.py | |||
@@ -0,0 +1,95 @@ | |||
1 | # | ||
2 | # SPDX-License-Identifier: GPL-2.0-only | ||
3 | # | ||
4 | import os | ||
5 | import subprocess | ||
6 | import bb | ||
7 | |||
8 | def get_source_date_epoch_from_known_files(d, sourcedir): | ||
9 | source_date_epoch = None | ||
10 | newest_file = None | ||
11 | known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"]) | ||
12 | for file in known_files: | ||
13 | filepath = os.path.join(sourcedir, file) | ||
14 | if os.path.isfile(filepath): | ||
15 | mtime = int(os.lstat(filepath).st_mtime) | ||
16 | # There may be more than one "known_file" present, if so, use the youngest one | ||
17 | if not source_date_epoch or mtime > source_date_epoch: | ||
18 | source_date_epoch = mtime | ||
19 | newest_file = filepath | ||
20 | if newest_file: | ||
21 | bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file) | ||
22 | return source_date_epoch | ||
23 | |||
24 | def find_git_folder(d, sourcedir): | ||
25 | # First guess: WORKDIR/git | ||
26 | # This is the default git fetcher unpack path | ||
27 | workdir = d.getVar('WORKDIR') | ||
28 | gitpath = os.path.join(workdir, "git/.git") | ||
29 | if os.path.isdir(gitpath): | ||
30 | return gitpath | ||
31 | |||
32 | # Second guess: ${S} | ||
33 | gitpath = os.path.join(sourcedir, ".git") | ||
34 | if os.path.isdir(gitpath): | ||
35 | return gitpath | ||
36 | |||
37 | # Perhaps there was a subpath or destsuffix specified. | ||
38 | # Go looking in the WORKDIR | ||
39 | exclude = set(["build", "image", "license-destdir", "patches", "pseudo", | ||
40 | "recipe-sysroot", "recipe-sysroot-native", "sysroot-destdir", "temp"]) | ||
41 | for root, dirs, files in os.walk(workdir, topdown=True): | ||
42 | dirs[:] = [d for d in dirs if d not in exclude] | ||
43 | if '.git' in dirs: | ||
44 | return root | ||
45 | |||
46 | bb.warn("Failed to find a git repository in WORKDIR: %s" % workdir) | ||
47 | return None | ||
48 | |||
49 | def get_source_date_epoch_from_git(d, sourcedir): | ||
50 | source_date_epoch = None | ||
51 | if "git://" in d.getVar('SRC_URI'): | ||
52 | gitpath = find_git_folder(d, sourcedir) | ||
53 | if gitpath: | ||
54 | import subprocess | ||
55 | source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=gitpath)) | ||
56 | bb.debug(1, "git repository: %s" % gitpath) | ||
57 | return source_date_epoch | ||
58 | |||
59 | def get_source_date_epoch_from_youngest_file(d, sourcedir): | ||
60 | if sourcedir == d.getVar('WORKDIR'): | ||
61 | # These sources are almost certainly not from a tarball | ||
62 | return None | ||
63 | |||
64 | # Do it the hard way: check all files and find the youngest one... | ||
65 | source_date_epoch = None | ||
66 | newest_file = None | ||
67 | for root, dirs, files in os.walk(sourcedir, topdown=True): | ||
68 | files = [f for f in files if not f[0] == '.'] | ||
69 | |||
70 | for fname in files: | ||
71 | filename = os.path.join(root, fname) | ||
72 | try: | ||
73 | mtime = int(os.lstat(filename).st_mtime) | ||
74 | except ValueError: | ||
75 | mtime = 0 | ||
76 | if not source_date_epoch or mtime > source_date_epoch: | ||
77 | source_date_epoch = mtime | ||
78 | newest_file = filename | ||
79 | |||
80 | if newest_file: | ||
81 | bb.debug(1, "Newest file found: %s" % newest_file) | ||
82 | return source_date_epoch | ||
83 | |||
84 | def fixed_source_date_epoch(): | ||
85 | bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH") | ||
86 | return 0 | ||
87 | |||
88 | def get_source_date_epoch(d, sourcedir): | ||
89 | return ( | ||
90 | get_source_date_epoch_from_git(d, sourcedir) or | ||
91 | get_source_date_epoch_from_known_files(d, sourcedir) or | ||
92 | get_source_date_epoch_from_youngest_file(d, sourcedir) or | ||
93 | fixed_source_date_epoch() # Last resort | ||
94 | ) | ||
95 | |||