classes/buildhistory: implement history collection for SDKs

SDKs are constructed in a similar manner to images, and the contents can
be influenced by a number of different factors, thus tracking the
contents of produced SDKs when buildhistory is enabled can help detect
the same kinds of issues as with images.

This required adding POPULATE_SDK_POST_HOST_COMMAND and
SDK_POSTPROCESS_COMMAND variables so that data collection functions can
be injected at the appropriate points in the SDK construction process,
as well as moving the list_installed_packages and
rootfs_list_installed_depends functions from the rootfs_{rpm,ipk,deb} to
the package_{rpm,ipk,deb} classes so they can also be called during
do_populate_sdk as well as do_rootfs.

Implements [YOCTO #3964].

(From OE-Core rev: c3736064483d4840e38cb1b8c13d2dd3a26b36aa)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 90 insertions, 40 deletions

diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 07b3c1edaf..b559ebf8ac 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -3,14 +3,15 @@
 #
 # Based in part on testlab.bbclass and packagehistory.bbclass
 #
-# Copyright (C) 2011 Intel Corporation
+# Copyright (C) 2013 Intel Corporation
 # Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
 #
 
-BUILDHISTORY_FEATURES ?= "image package"
+BUILDHISTORY_FEATURES ?= "image package sdk"
 BUILDHISTORY_DIR ?= "${TMPDIR}/buildhistory"
 BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
 BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}"
+BUILDHISTORY_DIR_SDK = "${BUILDHISTORY_DIR}/sdk/${SDK_NAME}"
 BUILDHISTORY_COMMIT ?= "0"
 BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
 BUILDHISTORY_PUSH_REPO ?= ""
@@ -315,69 +316,91 @@ def write_pkghistory(pkginfo, d):
                 os.unlink(filevarpath)
 
 
-buildhistory_get_image_installed() {
-        # Anything requiring the use of the packaging system should be done in here
-        # in case the packaging files are going to be removed for this image
-
-        if [ "${@base_contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
-                return
-        fi
-
-        mkdir -p ${BUILDHISTORY_DIR_IMAGE}
+buildhistory_get_installed() {
+        mkdir -p $1
 
         # Get list of installed packages
-        pkgcache="${BUILDHISTORY_DIR_IMAGE}/installed-packages.tmp"
+        pkgcache="$1/installed-packages.tmp"
         list_installed_packages file | sort > $pkgcache
 
-        cat $pkgcache | awk '{ print $1 }' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt
-        cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
+        cat $pkgcache | awk '{ print $1 }' > $1/installed-package-names.txt
+        cat $pkgcache | awk '{ print $2 }' | xargs -n1 basename > $1/installed-packages.txt
 
         # Produce dependency graph
         # First, filter out characters that cause issues for dot
-        rootfs_list_installed_depends | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' > ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
+        rootfs_list_installed_depends | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' > $1/depends.tmp
         # Change delimiter from pipe to -> and set style for recommend lines
-        sed -i -e 's:|: -> :' -e 's:\[REC\]:[style=dotted]:' -e 's:$:;:' ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
+        sed -i -e 's:|: -> :' -e 's:\[REC\]:[style=dotted]:' -e 's:$:;:' $1/depends.tmp
         # Add header, sorted and de-duped contents and footer and then delete the temp file
-        printf "digraph depends {\n    node [shape=plaintext]\n" > ${BUILDHISTORY_DIR_IMAGE}/depends.dot
-        cat ${BUILDHISTORY_DIR_IMAGE}/depends.tmp | sort | uniq >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
-        echo "}" >>  ${BUILDHISTORY_DIR_IMAGE}/depends.dot
-        rm ${BUILDHISTORY_DIR_IMAGE}/depends.tmp
+        printf "digraph depends {\n    node [shape=plaintext]\n" > $1/depends.dot
+        cat $1/depends.tmp | sort | uniq >> $1/depends.dot
+        echo "}" >>  $1/depends.dot
+        rm $1/depends.tmp
 
         # Produce installed package sizes list
-        printf "" > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
+        printf "" > $1/installed-package-sizes.tmp
         cat $pkgcache | while read pkg pkgfile
         do
                 if [ -f $pkgfile ] ; then
                         pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'`
-                        echo $pkgsize $pkg >> ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
+                        echo $pkgsize $pkg >> $1/installed-package-sizes.tmp
                 fi
         done
-        cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.txt
-        rm ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
+        cat $1/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > $1/installed-package-sizes.txt
+        rm $1/installed-package-sizes.tmp
 
         # We're now done with the cache, delete it
         rm $pkgcache
 
-        # Produce some cut-down graphs (for readability)
-        grep -v kernel_image ${BUILDHISTORY_DIR_IMAGE}/depends.dot | grep -v kernel_2 | grep -v kernel_3 > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot
-        grep -v libc6 ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot | grep -v libgcc > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot
-        grep -v update_ ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot
-        grep -v kernel_module ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate-nomodules.dot
+        if [ "$2" != "sdk" ] ; then
+                # Produce some cut-down graphs (for readability)
+                grep -v kernel_image $1/depends.dot | grep -v kernel_2 | grep -v kernel_3 > $1/depends-nokernel.dot
+                grep -v libc6 $1/depends-nokernel.dot | grep -v libgcc > $1/depends-nokernel-nolibc.dot
+                grep -v update_ $1/depends-nokernel-nolibc.dot > $1/depends-nokernel-nolibc-noupdate.dot
+                grep -v kernel_module $1/depends-nokernel-nolibc-noupdate.dot > $1/depends-nokernel-nolibc-noupdate-nomodules.dot
+        fi
 
         # add complementary package information
         if [ -e ${WORKDIR}/complementary_pkgs.txt ]; then
-                cp ${WORKDIR}/complementary_pkgs.txt ${BUILDHISTORY_DIR_IMAGE}
+                cp ${WORKDIR}/complementary_pkgs.txt $1
         fi
 }
 
-buildhistory_get_imageinfo() {
+buildhistory_get_image_installed() {
+        # Anything requiring the use of the packaging system should be done in here
+        # in case the packaging files are going to be removed for this image
+
         if [ "${@base_contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
                 return
         fi
 
-        # List the files in the image, but exclude date/time etc.
+        buildhistory_get_installed ${BUILDHISTORY_DIR_IMAGE}
+}
+
+buildhistory_get_sdk_installed() {
+        # Anything requiring the use of the packaging system should be done in here
+        # in case the packaging files are going to be removed for this SDK
+
+        if [ "${@base_contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
+                return
+        fi
+
+        buildhistory_get_installed ${BUILDHISTORY_DIR_SDK}/$1 sdk
+}
+
+buildhistory_list_files() {
+        # List the files in the specified directory, but exclude date/time etc.
         # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo
-        ( cd ${IMAGE_ROOTFS} && find . -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' | sort -k5 > ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt )
+        ( cd $1 && find . -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' | sort -k5 > $2 )
+}
+
+
+buildhistory_get_imageinfo() {
+        if [ "${@base_contains('BUILDHISTORY_FEATURES', 'image', '1', '0', d)}" = "0" ] ; then
+                return
+        fi
+
+        buildhistory_list_files ${IMAGE_ROOTFS} ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
 
         # Record some machine-readable meta-information about the image
         printf ""  > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
@@ -395,11 +418,32 @@ ${@buildhistory_get_layers(d)}
 END
 }
 
+buildhistory_get_sdkinfo() {
+        if [ "${@base_contains('BUILDHISTORY_FEATURES', 'sdk', '1', '0', d)}" = "0" ] ; then
+                return
+        fi
+
+        buildhistory_list_files ${SDK_OUTPUT} ${BUILDHISTORY_DIR_SDK}/files-in-sdk.txt
+
+        # Record some machine-readable meta-information about the SDK
+        printf ""  > ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
+        cat >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt <<END
+${@buildhistory_get_sdkvars(d)}
+END
+        sdksize=`du -ks ${SDK_OUTPUT} | awk '{ print $1 }'`
+        echo "SDKSIZE = $sdksize" >> ${BUILDHISTORY_DIR_SDK}/sdk-info.txt
+}
+
 # By prepending we get in before the removal of packaging files
 ROOTFS_POSTPROCESS_COMMAND =+ "buildhistory_get_image_installed ; "
 
 IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; "
 
+POPULATE_SDK_POST_TARGET_COMMAND += "buildhistory_get_sdk_installed target ; "
+POPULATE_SDK_POST_HOST_COMMAND += "buildhistory_get_sdk_installed host ; "
+
+SDK_POSTPROCESS_COMMAND += "buildhistory_get_sdkinfo ; "
+
 def buildhistory_get_layers(d):
     layertext = "Configured metadata layers:\n%s\n" % '\n'.join(get_layers_branch_rev(d))
     return layertext
@@ -418,15 +462,11 @@ def squashspaces(string):
     import re
     return re.sub("\s+", " ", string).strip()
 
-
-def buildhistory_get_imagevars(d):
-    imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
-    listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS"
-
-    imagevars = imagevars.split()
+def outputvars(vars, listvars, d):
+    vars = vars.split()
     listvars = listvars.split()
     ret = ""
-    for var in imagevars:
+    for var in vars:
         value = d.getVar(var, True) or ""
         if var in listvars:
             # Squash out spaces
@@ -434,6 +474,16 @@ def buildhistory_get_imagevars(d):
         ret += "%s = %s\n" % (var, value)
     return ret.rstrip('\n')
 
+def buildhistory_get_imagevars(d):
+    imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
+    listvars = "USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS"
+    return outputvars(imagevars, listvars, d)
+
+def buildhistory_get_sdkvars(d):
+    sdkvars = "DISTRO DISTRO_VERSION SDK_NAME SDK_VERSION SDKMACHINE SDKIMAGE_FEATURES BAD_RECOMMENDATIONS"
+    listvars = "SDKIMAGE_FEATURES BAD_RECOMMENDATIONS"
+    return outputvars(sdkvars, listvars, d)
+
 
 buildhistory_commit() {
         if [ ! -d ${BUILDHISTORY_DIR} ] ; then