summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJuro Bystricky <juro.bystricky@intel.com>2018-03-20 15:34:19 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-03-25 09:40:42 +0100
commitcf6f3c023cfa731ce061949135e255458d474964 (patch)
treedd569929bfc225460f0c845b7d887c82a030bd8a
parent7cd6442613353b93ab6a2fe8f0dd6f2c9fa18afb (diff)
downloadpoky-cf6f3c023cfa731ce061949135e255458d474964.tar.gz
reproducible_build.bbclass: support for binary reproducibility
Setup environment for builds requiring binary reproducibility. Determine and export SOURCE_DATE_EPOCH per each recipe. This is a crucial step to achieve binary reproducibility. The value for this variable (timestamp) is obtained after source code for a recipe has been unpacked, but before it is patched. If the code sources come from a GIT repo, we get the timestamp from the top commit. (GIT repo does not preserve file mktime timestamps). Otherwise, if GIT repo is not present, we try to get mtime from known files such as NEWS, ChangeLog, etc. If this also fails, we go through all files and get the timestamp from the youngest one. We create an individual timestamp for each recipe. The timestamp is stored in the file '__source_date_epoch.txt' (in the folder source-date-epoch_). Later on, each task reads this file and sets the exported value of SOURCE_DATE_EPOCH to the value found in the file. Uasge: INHERIT += "reproducible_build" [YOCTO#11178] [YOCTO#11179] (From OE-Core rev: cc438ac7711dedbe05d654e99af9316c9215b02e) Signed-off-by: Juro Bystricky <juro.bystricky@intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/reproducible_build.bbclass150
1 files changed, 150 insertions, 0 deletions
diff --git a/meta/classes/reproducible_build.bbclass b/meta/classes/reproducible_build.bbclass
new file mode 100644
index 0000000000..2df805330a
--- /dev/null
+++ b/meta/classes/reproducible_build.bbclass
@@ -0,0 +1,150 @@
1#
2# reproducible_build.bbclass
3#
4# This bbclass is mainly responsible to determine SOURCE_DATE_EPOCH on a per recipe base.
5# We need to set a recipe specific SOURCE_DATE_EPOCH in each recipe environment for various tasks.
6# One way would be to modify all recipes one-by-one to specify SOURCE_DATE_EPOCH explicitly,
7# but that is not realistic as there are hundreds (probably thousands) of recipes in various meta-layers.
8# Therefore we do it this class.
9# After sources are unpacked but before they are patched, we try to determine the value for SOURCE_DATE_EPOCH.
10#
11# There are 4 ways to determine SOURCE_DATE_EPOCH:
12#
13# 1. Use value from __source_date_epoch.txt file if this file exists.
14# This file was most likely created in the previous build by one of the following methods 2,3,4.
15# In principle, it could actually provided by a recipe via SRC_URI
16#
17# If the file does not exist, first try to determine the value for SOURCE_DATE_EPOCH:
18#
19# 2. If we detected a folder .git, use .git last commit date timestamp, as git does not allow checking out
20# files and preserving their timestamps.
21#
22# 3. Use the mtime of "known" files such as NEWS, CHANGLELOG, ...
23# This will work fine for any well kept repository distributed via tarballs.
24#
25# 4. If the above steps fail, we need to check all package source files and use the youngest file of the source tree.
26#
27# Once the value of SOURCE_DATE_EPOCH is determined, it is stored in the recipe ${WORKDIR}/source_date_epoch folder
28# in a text file "__source_date_epoch.txt'. If this file is found by other recipe task, the value is exported in
29# the SOURCE_DATE_EPOCH variable in the task environment. This is done in an anonymous python function,
30# so SOURCE_DATE_EPOCH is guaranteed to exist for all tasks the may use it (do_configure, do_compile, do_package, ...)
31
32BUILD_REPRODUCIBLE_BINARIES ??= '1'
33inherit ${@oe.utils.ifelse(d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1', 'reproducible_build_simple', '')}
34
35SDE_DIR ="${WORKDIR}/source-date-epoch"
36SDE_FILE = "${SDE_DIR}/__source_date_epoch.txt"
37
38SSTATETASKS += "do_deploy_source_date_epoch"
39
40do_deploy_source_date_epoch () {
41 echo "Deploying SDE to ${SDE_DIR}."
42}
43
44python do_deploy_source_date_epoch_setscene () {
45 sstate_setscene(d)
46}
47
48do_deploy_source_date_epoch[dirs] = "${SDE_DIR}"
49do_deploy_source_date_epoch[sstate-plaindirs] = "${SDE_DIR}"
50addtask do_deploy_source_date_epoch_setscene
51addtask do_deploy_source_date_epoch before do_configure after do_patch
52
53def get_source_date_epoch_known_files(d, path):
54 source_date_epoch = 0
55 known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"])
56 for file in known_files:
57 filepath = os.path.join(path,file)
58 if os.path.isfile(filepath):
59 mtime = int(os.path.getmtime(filepath))
60 # There may be more than one "known_file" present, if so, use the youngest one
61 if mtime > source_date_epoch:
62 source_date_epoch = mtime
63 return source_date_epoch
64
65def find_git_folder(path):
66 exclude = set(["temp", "license-destdir", "patches", "recipe-sysroot-native", "recipe-sysroot", "pseudo", "build", "image", "sysroot-destdir"])
67 for root, dirs, files in os.walk(path, topdown=True):
68 dirs[:] = [d for d in dirs if d not in exclude]
69 if '.git' in dirs:
70 #bb.warn("found root:%s" % (str(root)))
71 return root
72
73def get_source_date_epoch_git(d, path):
74 source_date_epoch = 0
75 if "git://" in d.getVar('SRC_URI'):
76 gitpath = find_git_folder(d.getVar('WORKDIR'))
77 if gitpath != None:
78 import subprocess
79 if os.path.isdir(os.path.join(gitpath,".git")):
80 try:
81 source_date_epoch = int(subprocess.check_output(['git','log','-1','--pretty=%ct'], cwd=path))
82 #bb.warn("JB *** gitpath:%s sde: %d" % (gitpath,source_date_epoch))
83 bb.debug(1, "git repo path:%s sde: %d" % (gitpath,source_date_epoch))
84 except subprocess.CalledProcessError as grepexc:
85 #bb.warn( "Expected git repository not found, (path: %s) error:%d" % (gitpath, grepexc.returncode))
86 bb.debug(1, "Expected git repository not found, (path: %s) error:%d" % (gitpath, grepexc.returncode))
87 else:
88 bb.warn("Failed to find a git repository for path:%s" % (path))
89 return source_date_epoch
90
91python do_create_source_date_epoch_stamp() {
92 path = d.getVar('S')
93 if not os.path.isdir(path):
94 bb.warn("Unable to determine source_date_epoch! path:%s" % path)
95 return
96
97 epochfile = d.getVar('SDE_FILE')
98 if os.path.isfile(epochfile):
99 bb.debug(1, " path: %s reusing __source_date_epoch.txt" % epochfile)
100 return
101
102 # Try to detect/find a git repository
103 source_date_epoch = get_source_date_epoch_git(d, path)
104 if source_date_epoch == 0:
105 source_date_epoch = get_source_date_epoch_known_files(d, path)
106 if source_date_epoch == 0:
107 # Do it the hard way: check all files and find the youngest one...
108 filename_dbg = None
109 exclude = set(["temp", "license-destdir", "patches", "recipe-sysroot-native", "recipe-sysroot", "pseudo", "build", "image", "sysroot-destdir"])
110 for root, dirs, files in os.walk(path, topdown=True):
111 files = [f for f in files if not f[0] == '.']
112 dirs[:] = [d for d in dirs if d not in exclude]
113
114 for fname in files:
115 filename = os.path.join(root, fname)
116 try:
117 mtime = int(os.path.getmtime(filename))
118 except ValueError:
119 mtime = 0
120 if mtime > source_date_epoch:
121 source_date_epoch = mtime
122 filename_dbg = filename
123
124 if filename_dbg != None:
125 bb.debug(1," SOURCE_DATE_EPOCH %d derived from: %s" % (source_date_epoch, filename_dbg))
126
127 if source_date_epoch == 0:
128 # empty folder, not a single file ...
129 # kernel source do_unpack is special cased
130 if not bb.data.inherits_class('kernel', d):
131 bb.debug(1, "Unable to determine source_date_epoch! path:%s" % path)
132
133 bb.utils.mkdirhier(d.getVar('SDE_DIR'))
134 with open(epochfile, 'w') as f:
135 f.write(str(source_date_epoch))
136}
137
138BB_HASHBASE_WHITELIST += "SOURCE_DATE_EPOCH"
139
140python () {
141 if d.getVar('BUILD_REPRODUCIBLE_BINARIES') == '1':
142 d.appendVarFlag("do_unpack", "postfuncs", " do_create_source_date_epoch_stamp")
143 epochfile = d.getVar('SDE_FILE')
144 source_date_epoch = "0"
145 if os.path.isfile(epochfile):
146 with open(epochfile, 'r') as f:
147 source_date_epoch = f.read()
148 bb.debug(1, "source_date_epoch stamp found ---> stamp %s" % source_date_epoch)
149 d.setVar('SOURCE_DATE_EPOCH', source_date_epoch)
150}