From b187b2775c854f6cf96e7eb4d20371340f5b152c Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 1 Dec 2011 23:00:52 +0000 Subject: Revert "classes/buildhistory: add new output history collection class" This reverts commit 508ff624fea705eb93cf2cc1e0c9c42cb817acf8. RP accidentally pulled the wrong commit in, it was supposed to be a *package*history bbclass change. Signed-off-by: Richard Purdie --- meta/classes/buildhistory.bbclass | 350 -------------------------------------- meta/classes/rootfs_ipk.bbclass | 27 +-- meta/classes/rootfs_rpm.bbclass | 41 +---- 3 files changed, 6 insertions(+), 412 deletions(-) delete mode 100644 meta/classes/buildhistory.bbclass diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass deleted file mode 100644 index 89b2cb1462..0000000000 --- a/meta/classes/buildhistory.bbclass +++ /dev/null @@ -1,350 +0,0 @@ -# -# Records history of build output in order to detect regressions -# -# Based in part on testlab.bbclass and packagehistory.bbclass -# -# Copyright (C) 2011 Intel Corporation -# Copyright (C) 2007, 2008 Koen Kooi -# - -BUILDHISTORY_DIR ?= "${TMPDIR}/buildhistory" -BUILDHISTORY_DIR_IMAGE = "${BUILDHISTORY_DIR}/images/${MACHINE_ARCH}/${TCLIBC}/${IMAGE_BASENAME}" -BUILDHISTORY_DIR_PACKAGE = "${BUILDHISTORY_DIR}/packages/${MULTIMACH_TARGET_SYS}/${PN}" -BUILDHISTORY_COMMIT ?= "0" -BUILDHISTORY_COMMIT_AUTHOR ?= "buildhistory " - -# Must inherit package first before changing PACKAGEFUNCS -inherit package -PACKAGEFUNCS += "buildhistory_emit_pkghistory" - -# -# Called during do_package to write out metadata about this package -# for comparision when writing future packages -# -python buildhistory_emit_pkghistory() { - import re - - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) - - class RecipeInfo: - def __init__(self, name): - self.name = name - self.pe = "0" - self.pv = "0" - self.pr = "r0" - self.depends = "" - self.packages = "" - - class PackageInfo: - def __init__(self, name): - self.name = name - self.pe = "0" - self.pv = "0" - self.pr = "r0" - self.size = 0 - self.depends = "" - self.rdepends = "" - self.rrecommends = "" - self.files = "" - self.filelist = "" - - # Should check PACKAGES here to see if anything removed - - def getpkgvar(pkg, var): - val = bb.data.getVar('%s_%s' % (var, pkg), d, 1) - if val: - return val - val = bb.data.getVar('%s' % (var), d, 1) - - return val - - def readRecipeInfo(pn, histfile): - rcpinfo = RecipeInfo(pn) - f = open(histfile, "r") - try: - for line in f: - lns = line.split('=') - name = lns[0].strip() - value = lns[1].strip(" \t\r\n").strip('"') - if name == "PE": - rcpinfo.pe = value - elif name == "PV": - rcpinfo.pv = value - elif name == "PR": - rcpinfo.pr = value - elif name == "DEPENDS": - rcpinfo.depends = value - elif name == "PACKAGES": - rcpinfo.packages = value - finally: - f.close() - return rcpinfo - - def readPackageInfo(pkg, histfile): - pkginfo = PackageInfo(pkg) - f = open(histfile, "r") - try: - for line in f: - lns = line.split('=') - name = lns[0].strip() - value = lns[1].strip(" \t\r\n").strip('"') - if name == "PE": - pkginfo.pe = value - elif name == "PV": - pkginfo.pv = value - elif name == "PR": - pkginfo.pr = value - elif name == "RDEPENDS": - pkginfo.rdepends = value - elif name == "RRECOMMENDS": - pkginfo.rrecommends = value - elif name == "PKGSIZE": - pkginfo.size = long(value) - elif name == "FILES": - pkginfo.files = value - elif name == "FILELIST": - pkginfo.filelist = value - finally: - f.close() - return pkginfo - - def getlastrecipeversion(pn): - try: - histfile = os.path.join(pkghistdir, "latest") - return readRecipeInfo(pn, histfile) - except EnvironmentError: - return None - - def getlastpkgversion(pkg): - try: - histfile = os.path.join(pkghistdir, pkg, "latest") - return readPackageInfo(pkg, histfile) - except EnvironmentError: - return None - - def squashspaces(string): - return re.sub("\s+", " ", string) - - pn = d.getVar('PN', True) - pe = d.getVar('PE', True) or "0" - pv = d.getVar('PV', True) - pr = d.getVar('PR', True) - packages = squashspaces(d.getVar('PACKAGES', True)) - - rcpinfo = RecipeInfo(pn) - rcpinfo.pe = pe - rcpinfo.pv = pv - rcpinfo.pr = pr - rcpinfo.depends = squashspaces(d.getVar('DEPENDS', True) or "") - rcpinfo.packages = packages - write_recipehistory(rcpinfo, d) - write_latestlink(None, pe, pv, pr, d) - - # Apparently the version can be different on a per-package basis (see Python) - pkgdest = d.getVar('PKGDEST', True) - for pkg in packages.split(): - pe = getpkgvar(pkg, 'PE') or "0" - pv = getpkgvar(pkg, 'PV') - pr = getpkgvar(pkg, 'PR') - # - # Find out what the last version was - # Make sure the version did not decrease - # - lastversion = getlastpkgversion(pkg) - if lastversion: - last_pe = lastversion.pe - last_pv = lastversion.pv - last_pr = lastversion.pr - r = bb.utils.vercmp((pe, pv, pr), (last_pe, last_pv, last_pr)) - if r < 0: - bb.fatal("Package version for package %s went backwards which would break package feeds from (%s:%s-%s to %s:%s-%s)" % (pkg, last_pe, last_pv, last_pr, pe, pv, pr)) - - pkginfo = PackageInfo(pkg) - pkginfo.pe = pe - pkginfo.pv = pv - pkginfo.pr = pr - pkginfo.rdepends = squashspaces(getpkgvar(pkg, 'RDEPENDS') or "") - pkginfo.rrecommends = squashspaces(getpkgvar(pkg, 'RRECOMMENDS') or "") - pkginfo.files = squashspaces(getpkgvar(pkg, 'FILES') or "") - - # Gather information about packaged files - pkgdestpkg = os.path.join(pkgdest, pkg) - filelist = [] - pkginfo.size = 0 - for root, dirs, files in os.walk(pkgdestpkg): - relpth = os.path.relpath(root, pkgdestpkg) - for f in files: - fstat = os.lstat(os.path.join(root, f)) - pkginfo.size += fstat.st_size - filelist.append(os.sep + os.path.join(relpth, f)) - pkginfo.filelist = " ".join(filelist) - - write_pkghistory(pkginfo, d) - - if lastversion: - check_pkghistory(pkginfo, lastversion) - - write_latestlink(pkg, pe, pv, pr, d) -} - - -def check_pkghistory(pkginfo, lastversion): - - bb.debug(2, "Checking package history") - # RDEPENDS removed? - # PKG changed? - # Each file list of each package for file removals? - - -def write_recipehistory(rcpinfo, d): - bb.debug(2, "Writing recipe history") - - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) - - if not os.path.exists(pkghistdir): - os.makedirs(pkghistdir) - - verfile = os.path.join(pkghistdir, "%s:%s-%s" % (rcpinfo.pe, rcpinfo.pv, rcpinfo.pr)) - f = open(verfile, "w") - try: - if rcpinfo.pe != "0": - f.write("PE = %s\n" % rcpinfo.pe) - f.write("PV = %s\n" % rcpinfo.pv) - f.write("PR = %s\n" % rcpinfo.pr) - f.write("DEPENDS = %s\n" % rcpinfo.depends) - f.write("PACKAGES = %s\n" % rcpinfo.packages) - finally: - f.close() - - -def write_pkghistory(pkginfo, d): - bb.debug(2, "Writing package history") - - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) - - verpath = os.path.join(pkghistdir, pkginfo.name) - if not os.path.exists(verpath): - os.makedirs(verpath) - - verfile = os.path.join(verpath, "%s:%s-%s" % (pkginfo.pe, pkginfo.pv, pkginfo.pr)) - f = open(verfile, "w") - try: - if pkginfo.pe != "0": - f.write("PE = %s\n" % pkginfo.pe) - f.write("PV = %s\n" % pkginfo.pv) - f.write("PR = %s\n" % pkginfo.pr) - f.write("RDEPENDS = %s\n" % pkginfo.rdepends) - f.write("RRECOMMENDS = %s\n" % pkginfo.rrecommends) - f.write("PKGSIZE = %d\n" % pkginfo.size) - f.write("FILES = %s\n" % pkginfo.files) - f.write("FILELIST = %s\n" % pkginfo.filelist) - finally: - f.close() - - -def write_latestlink(pkg, pe, pv, pr, d): - import shutil - - pkghistdir = d.getVar('BUILDHISTORY_DIR_PACKAGE', True) - - def rm_link(path): - try: - os.unlink(path) - except OSError: - return - - if pkg: - filedir = os.path.join(pkghistdir, pkg) - else: - filedir = pkghistdir - rm_link(os.path.join(filedir, "latest")) - shutil.copy(os.path.join(filedir, "%s:%s-%s" % (pe, pv, pr)), os.path.join(filedir, "latest")) - - -buildhistory_get_image_installed() { - # Anything requiring the use of the packaging system should be done in here - # in case the packaging files are going to be removed for this image - - mkdir -p ${BUILDHISTORY_DIR_IMAGE} - - # Get list of installed packages - list_installed_packages | sort > ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt - INSTALLED_PKGS=`cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-names.txt` - - # Produce installed package file and size lists and dependency graph - echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt - echo -n > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp - echo -e "digraph depends {\n node [shape=plaintext]" > ${BUILDHISTORY_DIR_IMAGE}/depends.dot - for pkg in $INSTALLED_PKGS; do - pkgfile=`get_package_filename $pkg` - echo `basename $pkgfile` >> ${BUILDHISTORY_DIR_IMAGE}/installed-packages.txt - if [ -f $pkgfile ] ; then - pkgsize=`du -k $pkgfile | head -n1 | awk '{ print $1 }'` - echo $pkgsize $pkg >> ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp - fi - - deps=`list_package_depends $pkg` - for dep in $deps ; do - echo "$pkg OPP $dep;" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g' >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot - done - - recs=`list_package_recommends $pkg` - for rec in $recs ; do - echo "$pkg OPP $rec [style=dotted];" | sed -e 's:-:_:g' -e 's:\.:_:g' -e 's:+::g' | sed 's:OPP:->:g' >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot - done - done - echo "}" >> ${BUILDHISTORY_DIR_IMAGE}/depends.dot - - cat ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp | sort -n -r | awk '{print $1 "\tKiB " $2}' > ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.txt - rm ${BUILDHISTORY_DIR_IMAGE}/installed-package-sizes.tmp - - # Produce some cut-down graphs (for readability) - grep -v kernel_image ${BUILDHISTORY_DIR_IMAGE}/depends.dot | grep -v kernel_2 | grep -v kernel_3 > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot - grep -v libc6 ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel.dot | grep -v libgcc > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot - grep -v update_ ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot - grep -v kernel_module ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate.dot > ${BUILDHISTORY_DIR_IMAGE}/depends-nokernel-nolibc-noupdate-nomodules.dot - - # Workaround for broken shell function dependencies - if false ; then - get_package_filename - list_package_depends - list_package_recommends - fi -} - -buildhistory_get_imageinfo() { - # List the files in the image, but exclude date/time etc. - # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo - find ${IMAGE_ROOTFS} -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' > ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt - - # Add some configuration information - echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id - echo "${@buildhistory_get_layers(d)}" >> ${BUILDHISTORY_DIR_IMAGE}/build-id -} - -# By prepending we get in before the removal of packaging files -ROOTFS_POSTPROCESS_COMMAND =+ "buildhistory_get_image_installed ; " - -IMAGE_POSTPROCESS_COMMAND += " buildhistory_get_imageinfo ; " - -def buildhistory_get_layers(d): - layertext = "Configured metadata layers:\n%s\n" % '\n'.join(get_layers_branch_rev(d)) - return layertext - - -buildhistory_commit() { - ( cd ${BUILDHISTORY_DIR}/ - git add ${BUILDHISTORY_DIR}/* - git commit ${BUILDHISTORY_DIR}/ -m "Build ${BUILDNAME} for machine ${MACHINE} configured for ${DISTRO} ${DISTRO_VERSION}" --author "${BUILDHISTORY_COMMIT_AUTHOR}" > /dev/null || true) -} - -python buildhistory_eventhandler() { - import bb.build - import bb.event - - if isinstance(e, bb.event.BuildCompleted): - if e.data.getVar("BUILDHISTORY_COMMIT", True) == "1": - bb.build.exec_func("buildhistory_commit", e.data) -} - -addhandler buildhistory_eventhandler diff --git a/meta/classes/rootfs_ipk.bbclass b/meta/classes/rootfs_ipk.bbclass index b4b95c5645..4a5a2dd3be 100644 --- a/meta/classes/rootfs_ipk.bbclass +++ b/meta/classes/rootfs_ipk.bbclass @@ -143,36 +143,11 @@ remove_packaging_data_files() { mkdir ${IMAGE_ROOTFS}${opkglibdir} } -list_installed_packages() { - grep ^Package: ${IMAGE_ROOTFS}${opkglibdir}/status | sed "s/^Package: //" -} - -get_package_filename() { - name=`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" | awk '/^Package/ {printf $2"_"}'` - name=$name`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" | awk -F: '/^Version/ {printf $NF"_"}' | sed 's/^\s*//g'` - name=$name`opkg-cl ${IPKG_ARGS} info $1 | grep -B 7 -A 7 "^Status.* \(\(installed\)\|\(unpacked\)\)" | awk '/^Archi/ {print $2".ipk"}'` - - fullname=`find ${DEPLOY_DIR_IPK} -name "$name" || true` - if [ "$fullname" = "" ] ; then - echo $name - else - echo $fullname - fi -} - -list_package_depends() { - opkg-cl ${IPKG_ARGS} info $1 | grep ^Depends | sed -e 's/^Depends: //' -e 's/,//g' -e 's:([=<>]* [0-9a-zA-Z.~\-]*)::g' -} - -list_package_recommends() { - opkg-cl ${IPKG_ARGS} info $1 | grep ^Recommends | sed -e 's/^Recommends: //' -e 's/,//g' -e 's:([=<>]* [0-9a-zA-Z.~\-]*)::g' -} - install_all_locales() { PACKAGES_TO_INSTALL="" - INSTALLED_PACKAGES=`list_installed_packages | egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"` + INSTALLED_PACKAGES=`grep ^Package: ${IMAGE_ROOTFS}${opkglibdir}/status |sed "s/^Package: //"|egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"` for pkg in $INSTALLED_PACKAGES do diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass index 5fd45d758c..6973008c59 100644 --- a/meta/classes/rootfs_rpm.bbclass +++ b/meta/classes/rootfs_rpm.bbclass @@ -160,47 +160,16 @@ remove_packaging_data_files() { rm -rf ${IMAGE_ROOTFS}${opkglibdir} } -RPM_QUERY_CMD = '${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \ - -D "__dbi_txn create nofsync private"' - -list_installed_packages() { - ${RPM_QUERY_CMD} -qa --qf "[%{NAME}\n]" -} - -get_package_filename() { - resolve_package_rpm ${RPMCONF_TARGET_BASE}-base_archs.conf $1 -} - -list_package_depends() { - pkglist=`list_installed_packages` - - for req in `${RPM_QUERY_CMD} -q --qf "[%{REQUIRES}\n]" $1`; do - if echo "$req" | grep -q "^rpmlib" ; then continue ; fi - - realpkg="" - for dep in $pkglist; do - if [ "$dep" = "$req" ] ; then - realpkg="1" - echo $req - break - fi - done - - if [ "$realdep" = "" ] ; then - ${RPM_QUERY_CMD} -q --whatprovides $req --qf "%{NAME}\n" - fi - done -} - -list_package_recommends() { - : -} install_all_locales() { PACKAGES_TO_INSTALL="" # Generate list of installed packages... - INSTALLED_PACKAGES=`list_installed_packages | egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)"` + INSTALLED_PACKAGES=$( \ + ${RPM} --root ${IMAGE_ROOTFS} -D "_dbpath ${rpmlibdir}" \ + -D "__dbi_txn create nofsync private" \ + -qa --qf "[%{NAME}\n]" | egrep -v -- "(-locale-|-dev$|-doc$|^kernel|^glibc|^ttf|^task|^perl|^python)" \ + ) # This would likely be faster if we did it in one transaction # but this should be good enough for the few users of this function... -- cgit v1.2.3-54-g00ecf