1 files changed, 270 insertions, 0 deletions

diff --git a/meta/classes/packagefeed-stability.bbclass b/meta/classes/packagefeed-stability.bbclass
new file mode 100644
index 0000000000..b5207d9f84
--- /dev/null
+++ b/meta/classes/packagefeed-stability.bbclass
@@ -0,0 +1,270 @@
+# Class to avoid copying packages into the feed if they haven't materially changed
+#
+# Copyright (C) 2015 Intel Corporation
+# Released under the MIT license (see COPYING.MIT for details)
+#
+# This class effectively intercepts packages as they are written out by
+# do_package_write_*, causing them to be written into a different
+# directory where we can compare them to whatever older packages might
+# be in the "real" package feed directory, and avoid copying the new
+# package to the feed if it has not materially changed. The idea is to
+# avoid unnecessary churn in the packages when dependencies trigger task
+# reexecution (and thus repackaging). Enabling the class is simple:
+#
+# INHERIT += "packagefeed-stability"
+#
+# Caveats:
+# 1) Latest PR values in the build system may not match those in packages
+#    seen on the target (naturally)
+# 2) If you rebuild from sstate without the existing package feed present,
+#    you will lose the "state" of the package feed i.e. the preserved old
+#    package versions. Not the end of the world, but would negate the
+#    entire purpose of this class.
+#
+# Note that running -c cleanall on a recipe will purposely delete the old
+# package files so they will definitely be copied the next time.
+
+python() {
+    # Package backend agnostic intercept
+    # This assumes that the package_write task is called package_write_<pkgtype>
+    # and that the directory in which packages should be written is
+    # pointed to by the variable DEPLOY_DIR_<PKGTYPE>
+    for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
+        if pkgclass.startswith('package_'):
+            pkgtype = pkgclass.split('_', 1)[1]
+            pkgwritefunc = 'do_package_write_%s' % pkgtype
+            sstate_outputdirs = d.getVarFlag(pkgwritefunc, 'sstate-outputdirs', False)
+            deploydirvar = 'DEPLOY_DIR_%s' % pkgtype.upper()
+            deploydirvarref = '${' + deploydirvar + '}'
+            pkgcomparefunc = 'do_package_compare_%s' % pkgtype
+
+            if bb.data.inherits_class('image', d):
+                d.appendVarFlag('do_rootfs', 'recrdeptask', ' ' + pkgcomparefunc)
+
+            if bb.data.inherits_class('populate_sdk_base', d):
+                d.appendVarFlag('do_populate_sdk', 'recrdeptask', ' ' + pkgcomparefunc)
+
+            if bb.data.inherits_class('populate_sdk_ext', d):
+                d.appendVarFlag('do_populate_sdk_ext', 'recrdeptask', ' ' + pkgcomparefunc)
+
+            d.appendVarFlag('do_build', 'recrdeptask', ' ' + pkgcomparefunc)
+
+            if d.getVarFlag(pkgwritefunc, 'noexec', True) or (not d.getVarFlag(pkgwritefunc, 'task', True)) or pkgwritefunc in (d.getVar('__BBDELTASKS', True) or []):
+                # Packaging is disabled for this recipe, we shouldn't do anything
+                continue
+
+            if deploydirvarref in sstate_outputdirs:
+                # Set intermediate output directory
+                d.setVarFlag(pkgwritefunc, 'sstate-outputdirs', sstate_outputdirs.replace(deploydirvarref, deploydirvarref + '-prediff'))
+
+            d.setVar(pkgcomparefunc, d.getVar('do_package_compare', False))
+            d.setVarFlags(pkgcomparefunc, d.getVarFlags('do_package_compare', False))
+            d.appendVarFlag(pkgcomparefunc, 'depends', ' build-compare-native:do_populate_sysroot')
+            bb.build.addtask(pkgcomparefunc, 'do_build', 'do_packagedata ' + pkgwritefunc, d)
+}
+
+# This isn't the real task function - it's a template that we use in the
+# anonymous python code above
+fakeroot python do_package_compare () {
+    currenttask = d.getVar('BB_CURRENTTASK', True)
+    pkgtype = currenttask.rsplit('_', 1)[1]
+    package_compare_impl(pkgtype, d)
+}
+
+def package_compare_impl(pkgtype, d):
+    import errno
+    import fnmatch
+    import glob
+    import subprocess
+    import oe.sstatesig
+
+    pn = d.getVar('PN', True)
+    deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
+    prepath = deploydir + '-prediff/'
+
+    # Find out PKGR values are
+    pkgdatadir = d.getVar('PKGDATA_DIR', True)
+    packages = []
+    try:
+        with open(os.path.join(pkgdatadir, pn), 'r') as f:
+            for line in f:
+                if line.startswith('PACKAGES:'):
+                    packages = line.split(':', 1)[1].split()
+                    break
+    except IOError as e:
+        if e.errno == errno.ENOENT:
+            pass
+
+    if not packages:
+        bb.debug(2, '%s: no packages, nothing to do' % pn)
+        return
+
+    pkgrvalues = {}
+    rpkgnames = {}
+    rdepends = {}
+    pkgvvalues = {}
+    for pkg in packages:
+        with open(os.path.join(pkgdatadir, 'runtime', pkg), 'r') as f:
+            for line in f:
+                if line.startswith('PKGR:'):
+                    pkgrvalues[pkg] = line.split(':', 1)[1].strip()
+                if line.startswith('PKGV:'):
+                    pkgvvalues[pkg] = line.split(':', 1)[1].strip()
+                elif line.startswith('PKG_%s:' % pkg):
+                    rpkgnames[pkg] = line.split(':', 1)[1].strip()
+                elif line.startswith('RDEPENDS_%s:' % pkg):
+                    rdepends[pkg] = line.split(':', 1)[1].strip()
+
+    # Prepare a list of the runtime package names for packages that were
+    # actually produced
+    rpkglist = []
+    for pkg, rpkg in rpkgnames.iteritems():
+        if os.path.exists(os.path.join(pkgdatadir, 'runtime', pkg + '.packaged')):
+            rpkglist.append((rpkg, pkg))
+    rpkglist.sort(key=lambda x: len(x[0]), reverse=True)
+
+    pvu = d.getVar('PV', False)
+    if '$' + '{SRCPV}' in pvu:
+        pvprefix = pvu.split('$' + '{SRCPV}', 1)[0]
+    else:
+        pvprefix = None
+
+    pkgwritetask = 'package_write_%s' % pkgtype
+    files = []
+    copypkgs = []
+    manifest, _ = oe.sstatesig.sstate_get_manifest_filename(pkgwritetask, d)
+    with open(manifest, 'r') as f:
+        for line in f:
+            if line.startswith(prepath):
+                srcpath = line.rstrip()
+                if os.path.isfile(srcpath):
+                    destpath = os.path.join(deploydir, os.path.relpath(srcpath, prepath))
+
+                    # This is crude but should work assuming the output
+                    # package file name starts with the package name
+                    # and rpkglist is sorted by length (descending)
+                    pkgbasename = os.path.basename(destpath)
+                    pkgname = None
+                    for rpkg, pkg in rpkglist:
+                        if pkgbasename.startswith(rpkg):
+                            pkgr = pkgrvalues[pkg]
+                            destpathspec = destpath.replace(pkgr, '*')
+                            if pvprefix:
+                                pkgv = pkgvvalues[pkg]
+                                if pkgv.startswith(pvprefix):
+                                    pkgvsuffix = pkgv[len(pvprefix):]
+                                    if '+' in pkgvsuffix:
+                                        newpkgv = pvprefix + '*+' + pkgvsuffix.split('+', 1)[1]
+                                        destpathspec = destpathspec.replace(pkgv, newpkgv)
+                            pkgname = pkg
+                            break
+                    else:
+                        bb.warn('Unable to map %s back to package' % pkgbasename)
+                        destpathspec = destpath
+
+                    oldfiles = glob.glob(destpathspec)
+                    oldfile = None
+                    docopy = True
+                    if oldfiles:
+                        oldfile = oldfiles[-1]
+                        result = subprocess.call(['pkg-diff.sh', oldfile, srcpath])
+                        if result == 0:
+                            docopy = False
+
+                    files.append((pkgname, pkgbasename, srcpath, oldfile, destpath))
+                    bb.debug(2, '%s: package %s %s' % (pn, files[-1], docopy))
+                    if docopy:
+                        copypkgs.append(pkgname)
+
+    # Ensure that dependencies on specific versions (such as -dev on the
+    # main package) are copied in lock-step
+    changed = True
+    while changed:
+        rpkgdict = {x[0]: x[1] for x in rpkglist}
+        changed = False
+        for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
+            rdeps = rdepends.get(pkgname, None)
+            if not rdeps:
+                continue
+            rdepvers = bb.utils.explode_dep_versions2(rdeps)
+            for rdep, versions in rdepvers.iteritems():
+                dep = rpkgdict.get(rdep, None)
+                for version in versions:
+                    if version and version.startswith('= '):
+                        if dep in copypkgs and not pkgname in copypkgs:
+                            bb.debug(2, '%s: copying %s because it has a fixed version dependency on %s and that package is going to be copied' % (pn, pkgname, dep))
+                            changed = True
+                            copypkgs.append(pkgname)
+                        elif pkgname in copypkgs and not dep in copypkgs:
+                            bb.debug(2, '%s: copying %s because %s has a fixed version dependency on it and that package is going to be copied' % (pn, dep, pkgname))
+                            changed = True
+                            copypkgs.append(dep)
+
+    # Read in old manifest so we can delete any packages we aren't going to replace or preserve
+    pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
+    try:
+        with open(pcmanifest, 'r') as f:
+            knownfiles = [x[3] for x in files if x[3]]
+            for line in f:
+                fn = line.rstrip()
+                if fn:
+                    if fn in knownfiles:
+                        knownfiles.remove(fn)
+                    else:
+                        try:
+                            os.remove(fn)
+                            bb.warn('Removed old package %s' % fn)
+                        except OSError as e:
+                            if e.errno == errno.ENOENT:
+                                pass
+    except IOError as e:
+        if e.errno == errno.ENOENT:
+            pass
+
+    # Create new manifest
+    with open(pcmanifest, 'w') as f:
+        for pkgname, pkgbasename, srcpath, oldfile, destpath in files:
+            if pkgname in copypkgs:
+                bb.warn('Copying %s' % pkgbasename)
+                destdir = os.path.dirname(destpath)
+                bb.utils.mkdirhier(destdir)
+                if oldfile:
+                    try:
+                        os.remove(oldfile)
+                    except OSError as e:
+                        if e.errno == errno.ENOENT:
+                            pass
+                if (os.stat(srcpath).st_dev == os.stat(destdir).st_dev):
+                    # Use a hard link to save space
+                    os.link(srcpath, destpath)
+                else:
+                    shutil.copyfile(srcpath, destpath)
+                f.write('%s\n' % destpath)
+            else:
+                bb.warn('Not copying %s' % pkgbasename)
+                f.write('%s\n' % oldfile)
+
+
+do_cleanall_append() {
+    import errno
+    for pkgclass in (d.getVar('PACKAGE_CLASSES', True) or '').split():
+        if pkgclass.startswith('package_'):
+            pkgtype = pkgclass.split('_', 1)[1]
+            deploydir = d.getVar('DEPLOY_DIR_%s' % pkgtype.upper(), True)
+            prepath = deploydir + '-prediff'
+            pcmanifest = os.path.join(prepath, d.expand('pkg-compare-manifest-${MULTIMACH_TARGET_SYS}-${PN}'))
+            try:
+                with open(pcmanifest, 'r') as f:
+                    for line in f:
+                        fn = line.rstrip()
+                        if fn:
+                            try:
+                                os.remove(fn)
+                            except OSError as e:
+                                if e.errno == errno.ENOENT:
+                                    pass
+                os.remove(pcmanifest)
+            except IOError as e:
+                if e.errno == errno.ENOENT:
+                    pass
+}