summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/packagefeed-stability.bbclass270
1 files changed, 270 insertions, 0 deletions
diff --git a/meta/classes/packagefeed-stability.bbclass b/meta/classes/packagefeed-stability.bbclass
new file mode 100644
index 0000000000..b5207d9f84
--- /dev/null
+++ b/meta/classes/packagefeed-stability.bbclass
@@ -0,0 +1,270 @@
1# Class to avoid copying packages into the feed if they haven't materially changed
2#
3# Copyright (C) 2015 Intel Corporation
4# Released under the MIT license (see COPYING.MIT for details)
5#
6# This class effectively intercepts packages as they are written out by
7# do_package_write_*, causing them to be written into a different
8# directory where we can compare them to whatever older packages might
9# be in the "real" package feed directory, and avoid copying the new
10# package to the feed if it has not materially changed. The idea is to
11# avoid unnecessary churn in the packages when dependencies trigger task
12# reexecution (and thus repackaging). Enabling the class is simple:
13#
14# INHERIT += "packagefeed-stability"
15#
16# Caveats:
17# 1) Latest PR values in the build system may not match those in packages
18# seen on the target (naturally)
19# 2) If you rebuild from sstate without the existing package feed present,
20# you will lose the "state" of the package feed i.e. the preserved old
21# package versions. Not the end of the world, but would negate the
22# entire purpose of this class.
23#
24# Note that running -c cleanall on a recipe will purposely delete the old
25# package files so they will definitely be copied the next time.
26
27python() {
28 # Package backend agnostic intercept
29 # This assumes that the package_write task is called package_write_<pkgtype>
30 # and that the directory in which packages should be written is
31 # pointed to by the variable DEPLOY_DIR_<PKGTYPE>
32 for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
33 if pkgclass.startswith('package_'):
34 pkgtype = pkgclass.split('_', 1)[1]
35 pkgwritefunc = 'do_package_write_%s' % pkgtype
36 sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False)
37 deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper()
38 deploydirvarref = '${' + deploydirvar + '}'
39 pkgcomparefunc = 'do_package_compare_%s' % pkgtype
40
41 if bb.data.inherits_class('image', d):
42 d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc)
43
44 if bb.data.inherits_class('populate_sdk_base', d):
45 d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc)
46
47 if bb.data.inherits_class('populate_sdk_ext', d):
48 d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc)
49
50 d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc)
51
52 if d.getVarFlag(pkgwritefunc, 'noexec', True) or (not d.getVarFlag(pkgwritefunc, 'task', True)) or pkgwritefunc in (d.getVar('__BBDELTASKS', True) or []):
53 # Packaging is disabled for this recipe, we shouldn't do anything
54 continue
55
56 if deploydirvarref in sstate_outputdirs:
57 # Set intermediate output directory
58 d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deploydirvarref + '-prediff'))
59
60 d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False))
61 d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False))
62 d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot')
63 bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d)
64}
65
66# This isn't the real task function - it's a template that we use in the
67# anonymous python code above
68fakeroot python do_package_compare () {
69 currenttask = d.getVar('BB_CURRENTTASK', True)
70 pkgtype = currenttask.rsplit('_', 1)[1]
71 package_compare_impl(pkgtype, d)
72}
73
74def package_compare_impl(pkgtype, d):
75 import errno
76 import fnmatch
77 import glob
78 import subprocess
79 import oe.sstatesig
80
81 pn = d.getVar('PN', True)
82 deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
83 prepath = deploydir + '-prediff/'
84
85 # Find out PKGR values are
86 pkgdatadir = d.getVar('PKGDATA_DIR', True)
87 packages = []
88 try:
89 with open(os.path.join(pkgdatadir, pn), 'r') as f:
90 for line in f:
91 if line.startswith('PACKAGES:'):
92 packages = line.split(':', 1)[1].split()
93 break
94 except IOError as e:
95 if e.errno == errno.ENOENT:
96 pass
97
98 if not packages:
99 bb.debug(2, '%s: no packages, nothing to do' % pn)
100 return
101
102 pkgrvalues = {}
103 rpkgnames = {}
104 rdepends = {}
105 pkgvvalues = {}
106 for pkg in packages:
107 with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f:
108 for line in f:
109 if line.startswith('PKGR:'):
110 pkgrvalues[pkg] = line.split(':', 1)[1].strip()
111 if line.startswith('PKGV:'):
112 pkgvvalues[pkg] = line.split(':', 1)[1].strip()
113 elif line.startswith('PKG_%s:' % pkg):
114 rpkgnames[pkg] = line.split(':', 1)[1].strip()
115 elif line.startswith('RDEPENDS_%s:' % pkg):
116 rdepends[pkg] = line.split(':', 1)[1].strip()
117
118 # Prepare a list of the runtime package names for packages that were
119 # actually produced
120 rpkglist = []
121 for pkg, rpkg in rpkgnames.iteritems():
122 if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')):
123 rpkglist.append((rpkg, pkg))
124 rpkglist.sort(key=lambda x: len(x[0]), reverse=True)
125
126 pvu = d.getVar('PV', False)
127 if '$' + '{SRCPV}' in pvu:
128 pvprefix = pvu.split('$' + '{SRCPV}', 1)[0]
129 else:
130 pvprefix = None
131
132 pkgwritetask = 'package_write_%s' % pkgtype
133 files = []
134 copypkgs = []
135 manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d)
136 with open(manifest, 'r') as f:
137 for line in f:
138 if line.startswith(prepath):
139 srcpath = line.rstrip()
140 if os.path.isfile(srcpath):
141 destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath))
142
143 # This is crude but should work assuming the output
144 # package file name starts with the package name
145 # and rpkglist is sorted by length (descending)
146 pkgbasename = os.path.basename(destpath)
147 pkgname = None
148 for rpkg, pkg in rpkglist:
149 if pkgbasename.startswith(rpkg):
150 pkgr = pkgrvalues[pkg]
151 destpathspec = destpath.replace(pkgr, '*')
152 if pvprefix:
153 pkgv = pkgvvalues[pkg]
154 if pkgv.startswith(pvprefix):
155 pkgvsuffix = pkgv[len(pvprefix):]
156 if '+' in pkgvsuffix:
157 newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1]
158 destpathspec = destpathspec.replace(pkgv, newpkgv)
159 pkgname = pkg
160 break
161 else:
162 bb.warn('Unable to map %s back to package' % pkgbasename)
163 destpathspec = destpath
164
165 oldfiles = glob.glob(destpathspec)
166 oldfile = None
167 docopy = True
168 if oldfiles:
169 oldfile = oldfiles[-1]
170 result = subprocess.call(['pkg-diff.sh', oldfile, srcpath])
171 if result == 0:
172 docopy = False
173
174 files.append((pkgname, pkgbasename, srcpath, oldfile, destpath))
175 bb.debug(2, '%s: package %s %s' % (pn, files[-1], docopy))
176 if docopy:
177 copypkgs.append(pkgname)
178
179 # Ensure that dependencies on specific versions (such as -dev on the
180 # main package) are copied in lock-step
181 changed = True
182 while changed:
183 rpkgdict = {x[0]: x[1] for x in rpkglist}
184 changed = False
185 for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
186 rdeps = rdepends.get(pkgname, None)
187 if not rdeps:
188 continue
189 rdepvers = bb.utils.explode_dep_versions2(rdeps)
190 for rdep, versions in rdepvers.iteritems():
191 dep = rpkgdict.get(rdep, None)
192 for version in versions:
193 if version and version.startswith('= '):
194 if dep in copypkgs and not pkgname in copypkgs:
195 bb.debug(2, '%s: copying %s because it has a fixed version dependency on %s and that package is going to be copied' % (pn, pkgname, dep))
196 changed = True
197 copypkgs.append(pkgname)
198 elif pkgname in copypkgs and not dep in copypkgs:
199 bb.debug(2, '%s: copying %s because %s has a fixed version dependency on it and that package is going to be copied' % (pn, dep, pkgname))
200 changed = True
201 copypkgs.append(dep)
202
203 # Read in old manifest so we can delete any packages we aren't going to replace or preserve
204 pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
205 try:
206 with open(pcmanifest, 'r') as f:
207 knownfiles = [x[3] for x in files if x[3]]
208 for line in f:
209 fn = line.rstrip()
210 if fn:
211 if fn in knownfiles:
212 knownfiles.remove(fn)
213 else:
214 try:
215 os.remove(fn)
216 bb.warn('Removed old package %s' % fn)
217 except OSError as e:
218 if e.errno == errno.ENOENT:
219 pass
220 except IOError as e:
221 if e.errno == errno.ENOENT:
222 pass
223
224 # Create new manifest
225 with open(pcmanifest, 'w') as f:
226 for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
227 if pkgname in copypkgs:
228 bb.warn('Copying %s' % pkgbasename)
229 destdir = os.path.dirname(destpath)
230 bb.utils.mkdirhier(destdir)
231 if oldfile:
232 try:
233 os.remove(oldfile)
234 except OSError as e:
235 if e.errno == errno.ENOENT:
236 pass
237 if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev):
238 # Use a hard link to save space
239 os.link(srcpath, destpath)
240 else:
241 shutil.copyfile(srcpath, destpath)
242 f.write('%s\n' % destpath)
243 else:
244 bb.warn('Not copying %s' % pkgbasename)
245 f.write('%s\n' % oldfile)
246
247
248do_cleanall_append() {
249 import errno
250 for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
251 if pkgclass.startswith('package_'):
252 pkgtype = pkgclass.split('_', 1)[1]
253 deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
254 prepath = deploydir + '-prediff'
255 pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
256 try:
257 with open(pcmanifest, 'r') as f:
258 for line in f:
259 fn = line.rstrip()
260 if fn:
261 try:
262 os.remove(fn)
263 except OSError as e:
264 if e.errno == errno.ENOENT:
265 pass
266 os.remove(pcmanifest)
267 except IOError as e:
268 if e.errno == errno.ENOENT:
269 pass
270}