summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-07-17 20:32:47 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-26 08:56:29 +0100
commitb36753b1c9c5471ccc43735957248f2a5bb76aba (patch)
tree497da32f5395edc3f4569cabee78b6d8c0ab9e80 /meta
parentcbf7902030ba367baaf0b8551b11c882aaefae3d (diff)
downloadpoky-b36753b1c9c5471ccc43735957248f2a5bb76aba.tar.gz
packagefeed-stability: add class to help reduce package feed churn
When a dependency causes a recipe to effectively be rebuilt, its output may in fact not change; but new packages (with an increased PR value, if using the PR server) will be generated nonetheless. There's no practical way for us to predict whether or not this is going to be the case based solely on the inputs, but we can compare the package output and see if that is materially different and based upon that decide to replace the old package with the new one. This class effectively intercepts packages as they are written out by do_package_write_*, causing them to be written into a different directory where we can compare them to whatever older packages might be in the "real" package feed directory, and avoid copying the new package to the feed if it has not materially changed. We use build-compare to do the package comparison. (From OE-Core rev: cc8b1a93912f830e605e6249c446b3764e550863) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/packagefeed-stability.bbclass270
1 files changed, 270 insertions, 0 deletions
diff --git a/meta/classes/packagefeed-stability.bbclass b/meta/classes/packagefeed-stability.bbclass
new file mode 100644
index 0000000000..b5207d9f84
--- /dev/null
+++ b/meta/classes/packagefeed-stability.bbclass
@@ -0,0 +1,270 @@
1# Class to avoid copying packages into the feed if they haven't materially changed
2#
3# Copyright (C) 2015 Intel Corporation
4# Released under the MIT license (see COPYING.MIT for details)
5#
6# This class effectively intercepts packages as they are written out by
7# do_package_write_*, causing them to be written into a different
8# directory where we can compare them to whatever older packages might
9# be in the "real" package feed directory, and avoid copying the new
10# package to the feed if it has not materially changed. The idea is to
11# avoid unnecessary churn in the packages when dependencies trigger task
12# reexecution (and thus repackaging). Enabling the class is simple:
13#
14# INHERIT += "packagefeed-stability"
15#
16# Caveats:
17# 1) Latest PR values in the build system may not match those in packages
18# seen on the target (naturally)
19# 2) If you rebuild from sstate without the existing package feed present,
20# you will lose the "state" of the package feed i.e. the preserved old
21# package versions. Not the end of the world, but would negate the
22# entire purpose of this class.
23#
24# Note that running -c cleanall on a recipe will purposely delete the old
25# package files so they will definitely be copied the next time.
26
27python() {
28 # Package backend agnostic intercept
29 # This assumes that the package_write task is called package_write_<pkgtype>
30 # and that the directory in which packages should be written is
31 # pointed to by the variable DEPLOY_DIR_<PKGTYPE>
32 for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
33 if pkgclass.startswith('package_'):
34 pkgtype = pkgclass.split('_', 1)[1]
35 pkgwritefunc = 'do_package_write_%s' % pkgtype
36 sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False)
37 deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper()
38 deploydirvarref = '${' + deploydirvar + '}'
39 pkgcomparefunc = 'do_package_compare_%s' % pkgtype
40
41 if bb.data.inherits_class('image', d):
42 d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc)
43
44 if bb.data.inherits_class('populate_sdk_base', d):
45 d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc)
46
47 if bb.data.inherits_class('populate_sdk_ext', d):
48 d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc)
49
50 d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc)
51
52 if d.getVarFlag(pkgwritefunc, 'noexec', True) or (not d.getVarFlag(pkgwritefunc, 'task', True)) or pkgwritefunc in (d.getVar('__BBDELTASKS', True) or []):
53 # Packaging is disabled for this recipe, we shouldn't do anything
54 continue
55
56 if deploydirvarref in sstate_outputdirs:
57 # Set intermediate output directory
58 d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deploydirvarref + '-prediff'))
59
60 d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False))
61 d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False))
62 d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot')
63 bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d)
64}
65
66# This isn't the real task function - it's a template that we use in the
67# anonymous python code above
68fakeroot python do_package_compare () {
69 currenttask = d.getVar('BB_CURRENTTASK', True)
70 pkgtype = currenttask.rsplit('_', 1)[1]
71 package_compare_impl(pkgtype, d)
72}
73
74def package_compare_impl(pkgtype, d):
75 import errno
76 import fnmatch
77 import glob
78 import subprocess
79 import oe.sstatesig
80
81 pn = d.getVar('PN', True)
82 deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
83 prepath = deploydir + '-prediff/'
84
85 # Find out PKGR values are
86 pkgdatadir = d.getVar('PKGDATA_DIR', True)
87 packages = []
88 try:
89 with open(os.path.join(pkgdatadir, pn), 'r') as f:
90 for line in f:
91 if line.startswith('PACKAGES:'):
92 packages = line.split(':', 1)[1].split()
93 break
94 except IOError as e:
95 if e.errno == errno.ENOENT:
96 pass
97
98 if not packages:
99 bb.debug(2, '%s: no packages, nothing to do' % pn)
100 return
101
102 pkgrvalues = {}
103 rpkgnames = {}
104 rdepends = {}
105 pkgvvalues = {}
106 for pkg in packages:
107 with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f:
108 for line in f:
109 if line.startswith('PKGR:'):
110 pkgrvalues[pkg] = line.split(':', 1)[1].strip()
111 if line.startswith('PKGV:'):
112 pkgvvalues[pkg] = line.split(':', 1)[1].strip()
113 elif line.startswith('PKG_%s:' % pkg):
114 rpkgnames[pkg] = line.split(':', 1)[1].strip()
115 elif line.startswith('RDEPENDS_%s:' % pkg):
116 rdepends[pkg] = line.split(':', 1)[1].strip()
117
118 # Prepare a list of the runtime package names for packages that were
119 # actually produced
120 rpkglist = []
121 for pkg, rpkg in rpkgnames.iteritems():
122 if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')):
123 rpkglist.append((rpkg, pkg))
124 rpkglist.sort(key=lambda x: len(x[0]), reverse=True)
125
126 pvu = d.getVar('PV', False)
127 if '$' + '{SRCPV}' in pvu:
128 pvprefix = pvu.split('$' + '{SRCPV}', 1)[0]
129 else:
130 pvprefix = None
131
132 pkgwritetask = 'package_write_%s' % pkgtype
133 files = []
134 copypkgs = []
135 manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d)
136 with open(manifest, 'r') as f:
137 for line in f:
138 if line.startswith(prepath):
139 srcpath = line.rstrip()
140 if os.path.isfile(srcpath):
141 destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath))
142
143 # This is crude but should work assuming the output
144 # package file name starts with the package name
145 # and rpkglist is sorted by length (descending)
146 pkgbasename = os.path.basename(destpath)
147 pkgname = None
148 for rpkg, pkg in rpkglist:
149 if pkgbasename.startswith(rpkg):
150 pkgr = pkgrvalues[pkg]
151 destpathspec = destpath.replace(pkgr, '*')
152 if pvprefix:
153 pkgv = pkgvvalues[pkg]
154 if pkgv.startswith(pvprefix):
155 pkgvsuffix = pkgv[len(pvprefix):]
156 if '+' in pkgvsuffix:
157 newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1]
158 destpathspec = destpathspec.replace(pkgv, newpkgv)
159 pkgname = pkg
160 break
161 else:
162 bb.warn('Unable to map %s back to package' % pkgbasename)
163 destpathspec = destpath
164
165 oldfiles = glob.glob(destpathspec)
166 oldfile = None
167 docopy = True
168 if oldfiles:
169 oldfile = oldfiles[-1]
170 result = subprocess.call(['pkg-diff.sh', oldfile, srcpath])
171 if result == 0:
172 docopy = False
173
174 files.append((pkgname, pkgbasename, srcpath, oldfile, destpath))
175 bb.debug(2, '%s: package %s %s' % (pn, files[-1], docopy))
176 if docopy:
177 copypkgs.append(pkgname)
178
179 # Ensure that dependencies on specific versions (such as -dev on the
180 # main package) are copied in lock-step
181 changed = True
182 while changed:
183 rpkgdict = {x[0]: x[1] for x in rpkglist}
184 changed = False
185 for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
186 rdeps = rdepends.get(pkgname, None)
187 if not rdeps:
188 continue
189 rdepvers = bb.utils.explode_dep_versions2(rdeps)
190 for rdep, versions in rdepvers.iteritems():
191 dep = rpkgdict.get(rdep, None)
192 for version in versions:
193 if version and version.startswith('= '):
194 if dep in copypkgs and not pkgname in copypkgs:
195 bb.debug(2, '%s: copying %s because it has a fixed version dependency on %s and that package is going to be copied' % (pn, pkgname, dep))
196 changed = True
197 copypkgs.append(pkgname)
198 elif pkgname in copypkgs and not dep in copypkgs:
199 bb.debug(2, '%s: copying %s because %s has a fixed version dependency on it and that package is going to be copied' % (pn, dep, pkgname))
200 changed = True
201 copypkgs.append(dep)
202
203 # Read in old manifest so we can delete any packages we aren't going to replace or preserve
204 pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
205 try:
206 with open(pcmanifest, 'r') as f:
207 knownfiles = [x[3] for x in files if x[3]]
208 for line in f:
209 fn = line.rstrip()
210 if fn:
211 if fn in knownfiles:
212 knownfiles.remove(fn)
213 else:
214 try:
215 os.remove(fn)
216 bb.warn('Removed old package %s' % fn)
217 except OSError as e:
218 if e.errno == errno.ENOENT:
219 pass
220 except IOError as e:
221 if e.errno == errno.ENOENT:
222 pass
223
224 # Create new manifest
225 with open(pcmanifest, 'w') as f:
226 for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
227 if pkgname in copypkgs:
228 bb.warn('Copying %s' % pkgbasename)
229 destdir = os.path.dirname(destpath)
230 bb.utils.mkdirhier(destdir)
231 if oldfile:
232 try:
233 os.remove(oldfile)
234 except OSError as e:
235 if e.errno == errno.ENOENT:
236 pass
237 if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev):
238 # Use a hard link to save space
239 os.link(srcpath, destpath)
240 else:
241 shutil.copyfile(srcpath, destpath)
242 f.write('%s\n' % destpath)
243 else:
244 bb.warn('Not copying %s' % pkgbasename)
245 f.write('%s\n' % oldfile)
246
247
248do_cleanall_append() {
249 import errno
250 for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
251 if pkgclass.startswith('package_'):
252 pkgtype = pkgclass.split('_', 1)[1]
253 deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
254 prepath = deploydir + '-prediff'
255 pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
256 try:
257 with open(pcmanifest, 'r') as f:
258 for line in f:
259 fn = line.rstrip()
260 if fn:
261 try:
262 os.remove(fn)
263 except OSError as e:
264 if e.errno == errno.ENOENT:
265 pass
266 os.remove(pcmanifest)
267 except IOError as e:
268 if e.errno == errno.ENOENT:
269 pass
270}