summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2011-11-30 16:48:47 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-12-05 16:23:53 (GMT)
commit6ba6f4a6f744895741c82f93756439292f563bf6 (patch)
tree64b35044e9730c94fb12b747cc658d1a2d23ef51
parent45c4f0a5870cf3a089b0edd02f52297865f5b5db (diff)
downloadpoky-6ba6f4a6f744895741c82f93756439292f563bf6.tar.gz
classes/buildhistory: add new output history collection class
Create a new build output history reporting class, using testlab.bbclass from meta-oe as a base. This records information from images produced by the build process in text files structured suitably for tracking within a git repository, thus enabling monitoring of changes over time. Build history collection can be enabled simply by adding the following to your local.conf: INHERIT += "buildhistory" The output after a build can then be found in BUILDHISTORY_DIR (defaults to TMPDIR/buildhistory). If you set up this directory as a git repository and set BUILDHISTORY_COMMIT to "1" in local.conf, the build history data will be committed on every build. (From OE-Core rev: 14acb530a27a3b088d0bfd56db291f4e72ace8ab) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/buildhistory.bbclass105
-rw-r--r--meta/classes/rootfs_ipk.bbclass27
-rw-r--r--meta/classes/rootfs_rpm.bbclass41
3 files changed, 167 insertions, 6 deletions
diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
new file mode 100644
index 0000000..21f8493
--- /dev/null
+++ b/meta/classes/buildhistory.bbclass
@@ -0,0 +1,105 @@
1#
2# Records history of build output in order to detect regressions
3#
4# Based in part on testlab.bbclass
5#
6# Copyright (C) 2011 Intel Corporation
7# Copyright (C) 2007-2011 Koen Kooi <koen@openembedded.org>
8#
9
10BUILDHISTORY_DIR ?= "${TMPDIR}/buildhistory"
11BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}"
12BUILDHISTORY_COMMIT ?= "0"
13BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory <buildhistory@${DISTRO}>"
14BUILDHISTORY_PUSH_REPO ?= ""
15
16buildhistory_get_image_installed() {
17 # Anything requiring the use of the packaging system should be done in here
18 # in case the packaging files are going to be removed for this image
19
20 mkdir -p ${BUILDHISTORY_DIR_IMAGE}
21
22 # Get list of installed packages
23 list_installed_packages | sort > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt
24 INSTALLED_PKGS=`cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt`
25
26 # Produce installed package file and size lists and dependency graph
27 echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
28 echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
29 echo -e "digraph depends {\n node [shape=plaintext]" > ${BUILDHISTORY_DIR_IMAGE}/depends.dot
30 for pkg in $INSTALLED_PKGS; do
31 pkgfile=`get_package_filename $pkg`
32 echo `basename $pkgfile` >> ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt
33 if [ -f $pkgfile ] ; then
34 pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'`
35 echo $pkgsize $pkg >> ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
36 fi
37
38 deps=`list_package_depends $pkg`
39 for dep in $deps ; do
40 echo "$pkg OPP $dep;" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g' >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
41 done
42
43 recs=`list_package_recommends $pkg`
44 for rec in $recs ; do
45 echo "$pkg OPP $rec [style=dotted];" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g' >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
46 done
47 done
48 echo "}" >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot
49
50 cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.txt
51 rm ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp
52
53 # Produce some cut-down graphs (for readability)
54 grep -v kernel_image ${BUILDHISTORY_DIR_IMAGE}/depends.dot | grep -v kernel_2 | grep -v kernel_3 > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot
55 grep -v libc6 ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot | grep -v libgcc > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot
56 grep -v update_ ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot
57 grep -v kernel_module ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate-nomodules.dot
58
59 # Workaround for broken shell function dependencies
60 if false ; then
61 get_package_filename
62 list_package_depends
63 list_package_recommends
64 fi
65}
66
67buildhistory_get_imageinfo() {
68 # List the files in the image, but exclude date/time etc.
69 # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo
70 find ${IMAGE_ROOTFS} -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' > ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt
71
72 # Add some configuration information
73 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id
74 echo "${@buildhistory_get_layers(d)}" >> ${BUILDHISTORY_DIR_IMAGE}/build-id
75}
76
77# By prepending we get in before the removal of packaging files
78ROOTFS_POSTPROCESS_COMMAND =+ "buildhistory_get_image_installed ; "
79
80IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; "
81
82def buildhistory_get_layers(d):
83 layertext = "Configured metadata layers:\n%s\n" % '\n'.join(get_layers_branch_rev(d))
84 return layertext
85
86
87buildhistory_commit() {
88 ( cd ${BUILDHISTORY_DIR}/
89 git add ${BUILDHISTORY_DIR}/*
90 git commit ${BUILDHISTORY_DIR}/ -m "Build ${BUILDNAME} for machine ${MACHINE} configured for ${DISTRO} ${DISTRO_VERSION}" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null
91 if [ "${BUILDHISTORY_PUSH_REPO}" != "" ] ; then
92 git push -q ${BUILDHISTORY_PUSH_REPO}
93 fi) || true
94}
95
96python buildhistory_eventhandler() {
97 import bb.build
98 import bb.event
99
100 if isinstance(e, bb.event.BuildCompleted):
101 if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1":
102 bb.build.exec_func("buildhistory_commit", e.data)
103}
104
105addhandler buildhistory_eventhandler
diff --git a/meta/classes/rootfs_ipk.bbclass b/meta/classes/rootfs_ipk.bbclass
index 4a5a2dd..b4b95c5 100644
--- a/meta/classes/rootfs_ipk.bbclass
+++ b/meta/classes/rootfs_ipk.bbclass
@@ -143,11 +143,36 @@ remove_packaging_data_files() {
143 mkdir ${IMAGE_ROOTFS}${opkglibdir} 143 mkdir ${IMAGE_ROOTFS}${opkglibdir}
144} 144}
145 145
146list_installed_packages() {
147 grep ^Package: ${IMAGE_ROOTFS}${opkglibdir}/status | sed "s/^Package: //"
148}
149
150get_package_filename() {
151 name=`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" | awk '/^Package/ {printf $2"_"}'`
152 name=$name`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" | awk -F: '/^Version/ {printf $NF"_"}' | sed 's/^\s*//g'`
153 name=$name`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" | awk '/^Archi/ {print $2".ipk"}'`
154
155 fullname=`find ${DEPLOY_DIR_IPK} -name "$name" || true`
156 if [ "$fullname" = "" ] ; then
157 echo $name
158 else
159 echo $fullname
160 fi
161}
162
163list_package_depends() {
164 opkg-cl ${IPKG_ARGS} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [0-9a-zA-Z.~\-]*)::g'
165}
166
167list_package_recommends() {
168 opkg-cl ${IPKG_ARGS} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [0-9a-zA-Z.~\-]*)::g'
169}
170
146install_all_locales() { 171install_all_locales() {
147 172
148 PACKAGES_TO_INSTALL="" 173 PACKAGES_TO_INSTALL=""
149 174
150 INSTALLED_PACKAGES=`grep ^Package: ${IMAGE_ROOTFS}${opkglibdir}/status |sed "s/^Package: //"|egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"` 175 INSTALLED_PACKAGES=`list_installed_packages | egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"`
151 176
152 for pkg in $INSTALLED_PACKAGES 177 for pkg in $INSTALLED_PACKAGES
153 do 178 do
diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass
index 6973008..5fd45d7 100644
--- a/meta/classes/rootfs_rpm.bbclass
+++ b/meta/classes/rootfs_rpm.bbclass
@@ -160,16 +160,47 @@ remove_packaging_data_files() {
160 rm -rf ${IMAGE_ROOTFS}${opkglibdir} 160 rm -rf ${IMAGE_ROOTFS}${opkglibdir}
161} 161}
162 162
163RPM_QUERY_CMD = '${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \
164 -D "__dbi_txn create nofsync private"'
165
166list_installed_packages() {
167 ${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]"
168}
169
170get_package_filename() {
171 resolve_package_rpm ${RPMCONF_TARGET_BASE}-base_archs.conf $1
172}
173
174list_package_depends() {
175 pkglist=`list_installed_packages`
176
177 for req in `${RPM_QUERY_CMD} -q --qf "[%{REQUIRES}\n]" $1`; do
178 if echo "$req" | grep -q "^rpmlib" ; then continue ; fi
179
180 realpkg=""
181 for dep in $pkglist; do
182 if [ "$dep" = "$req" ] ; then
183 realpkg="1"
184 echo $req
185 break
186 fi
187 done
188
189 if [ "$realdep" = "" ] ; then
190 ${RPM_QUERY_CMD} -q --whatprovides $req --qf "%{NAME}\n"
191 fi
192 done
193}
194
195list_package_recommends() {
196 :
197}
163 198
164install_all_locales() { 199install_all_locales() {
165 PACKAGES_TO_INSTALL="" 200 PACKAGES_TO_INSTALL=""
166 201
167 # Generate list of installed packages... 202 # Generate list of installed packages...
168 INSTALLED_PACKAGES=$( \ 203 INSTALLED_PACKAGES=`list_installed_packages | egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"`
169 ${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \
170 -D "__dbi_txn create nofsync private" \
171 -qa --qf "[%{NAME}\n]" | egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)" \
172 )
173 204
174 # This would likely be faster if we did it in one transaction 205 # This would likely be faster if we did it in one transaction
175 # but this should be good enough for the few users of this function... 206 # but this should be good enough for the few users of this function...