classes/rootfs_rpm: improve speed of RPM rootfs construction

Improve the performance of the RPM backend during do_rootfs by
performing most of the package name to file resolution in a separate
utility written in C, processing the entire list of packages at once
rather than running rpm on the command line which loads the RPM database
for every package.

(From OE-Core rev: 9135d351ba7cb21e50239d2b310565680bd4fdca)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

4 files changed, 341 insertions, 64 deletions

diff --git a/meta/classes/package_rpm.bbclass b/meta/classes/package_rpm.bbclass
index 983be4cfd0..29018e9cca 100644
--- a/meta/classes/package_rpm.bbclass
+++ b/meta/classes/package_rpm.bbclass
@@ -197,6 +197,45 @@ rpm_update_pkg () {
     fi
 }
 
+process_pkg_list_rpm() {
+        local insttype=$1
+        shift
+        local pkgs="$@"
+        local confbase=${INSTALL_CONFBASE_RPM}
+
+        echo -n > ${target_rootfs}/install/base_archs.pkglist
+        echo -n > ${target_rootfs}/install/ml_archs.pkglist
+
+        for pkg in $pkgs; do
+                echo "Processing $pkg..."
+
+                archvar=base_archs
+                ml_pkg=$pkg
+                for i in ${MULTILIB_PREFIX_LIST} ; do
+                                subst=${pkg#${i}-}
+                                if [ $subst != $pkg ] ; then
+                                                ml_pkg=$subst
+                                                archvar=ml_archs
+                                                break
+                                fi
+                done
+
+                echo $ml_pkg >> ${target_rootfs}/install/$archvar.pkglist
+        done
+
+        local manifestpfx="install"
+        local extraopt=""
+        if [ "$insttype" = "attemptonly" ] ; then
+                manifestpfx="install_attemptonly"
+                extraopt="-i"
+        fi
+
+        rpmresolve $extraopt ${confbase}-base_archs.conf ${target_rootfs}/install/base_archs.pkglist >> ${target_rootfs}/install/${manifestpfx}.manifest
+        if [ -s ${target_rootfs}/install/ml_archs.pkglist ] ; then
+                rpmresolve $extraopt ${confbase}-ml_archs.conf ${target_rootfs}/install/ml_archs.pkglist >> ${target_rootfs}/install/${manifestpfx}_multilib.manifest
+        fi
+}
+
 #
 # install a bunch of packages using rpm
 # the following shell variables needs to be set before calling this func:
@@ -256,55 +295,12 @@ package_install_internal_rpm () {
         # Uclibc builds don't provide this stuff...
         if [ x${TARGET_OS} = "xlinux" ] || [ x${TARGET_OS} = "xlinux-gnueabi" ] ; then
                 if [ ! -z "${package_linguas}" ]; then
-                        for pkg in ${package_linguas}; do
-                                echo "Processing $pkg..."
-
-                                archvar=base_archs
-                                manifest=install.manifest
-                                ml_prefix=`echo ${pkg} | cut -d'-' -f1`
-                                ml_pkg=$pkg
-                                for i in ${MULTILIB_PREFIX_LIST} ; do
-                                        if [ ${ml_prefix} = ${i} ]; then
-                                                ml_pkg=$(echo ${pkg} | sed "s,^${ml_prefix}-\(.*\),\1,")
-                                                archvar=ml_archs
-                                                manifest=install_multilib.manifest
-                                                break
-                                        fi
-                                done
-
-                                pkg_name=$(resolve_package_rpm ${confbase}-${archvar}.conf ${ml_pkg})
-                                if [ -z "$pkg_name" ]; then
-                                        echo "Unable to find package $pkg ($ml_pkg)!"
-                                        exit 1
-                                fi
-                                echo $pkg_name >> ${target_rootfs}/install/${manifest}
-                        done
+                        process_pkg_list_rpm linguas ${package_linguas}
                 fi
         fi
-        if [ ! -z "${package_to_install}" ]; then
-                for pkg in ${package_to_install} ; do
-                        echo "Processing $pkg..."
 
-                        archvar=base_archs
-                        manifest=install.manifest
-                        ml_prefix=`echo ${pkg} | cut -d'-' -f1`
-                        ml_pkg=$pkg
-                        for i in ${MULTILIB_PREFIX_LIST} ; do
-                                if [ ${ml_prefix} = ${i} ]; then
-                                        ml_pkg=$(echo ${pkg} | sed "s,^${ml_prefix}-\(.*\),\1,")
-                                        archvar=ml_archs
-                                        manifest=install_multilib.manifest
-                                        break
-                                fi
-                        done
-
-                        pkg_name=$(resolve_package_rpm ${confbase}-${archvar}.conf ${ml_pkg})
-                        if [ -z "$pkg_name" ]; then
-                                echo "Unable to find package $pkg ($ml_pkg)!"
-                                exit 1
-                        fi
-                        echo $pkg_name >> ${target_rootfs}/install/${manifest}
-                done
+        if [ ! -z "${package_to_install}" ]; then
+                process_pkg_list_rpm default ${package_to_install}
         fi
 
         # Normal package installation
@@ -324,24 +320,9 @@ package_install_internal_rpm () {
 
         if [ ! -z "${package_attemptonly}" ]; then
                 echo "Adding attempt only packages..."
-                for pkg in ${package_attemptonly} ; do
-                        echo "Processing $pkg..."
-                        archvar=base_archs
-                        ml_prefix=`echo ${pkg} | cut -d'-' -f1`
-                        ml_pkg=$pkg
-                        for i in ${MULTILIB_PREFIX_LIST} ; do
-                                if [ ${ml_prefix} = ${i} ]; then
-                                        ml_pkg=$(echo ${pkg} | sed "s,^${ml_prefix}-\(.*\),\1,")
-                                        archvar=ml_archs
-                                        break
-                                fi
-                        done
-
-                        pkg_name=$(resolve_package_rpm ${confbase}-${archvar}.conf ${ml_pkg})
-                        if [ -z "$pkg_name" ]; then
-                                echo "Note: Unable to find package $pkg ($ml_pkg) -- PACKAGE_INSTALL_ATTEMPTONLY"
-                                continue
-                        fi
+                process_pkg_list_rpm attemptonly ${package_attemptonly}
+                cat ${target_rootfs}/install/install_attemptonly.manifest | while read pkg_name
+                do
                         echo "Attempting $pkg_name..." >> "`dirname ${BB_LOGFILE}`/log.do_${task}_attemptonly.${PID}"
                         ${RPM} --predefine "_rpmds_sysinfo_path ${target_rootfs}/etc/rpm/sysinfo" \
                                 --predefine "_rpmrc_platform_path ${target_rootfs}/etc/rpm/platform" \
diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass
index 4551f7a608..cd9c5ab778 100644
--- a/meta/classes/rootfs_rpm.bbclass
+++ b/meta/classes/rootfs_rpm.bbclass
@@ -11,6 +11,7 @@ IMAGE_ROOTFS_EXTRA_SPACE_append = "${@base_contains("PACKAGE_INSTALL", "zypper",
 ROOTFS_PKGMANAGE_BOOTSTRAP = ""
 
 do_rootfs[depends] += "rpm-native:do_populate_sysroot"
+do_rootfs[depends] += "rpmresolve-native:do_populate_sysroot"
 
 # Needed for update-alternatives
 do_rootfs[depends] += "opkg-native:do_populate_sysroot"
diff --git a/meta/recipes-devtools/rpm/rpmresolve/rpmresolve.c b/meta/recipes-devtools/rpm/rpmresolve/rpmresolve.c
new file mode 100644
index 0000000000..9f6cdf28b8
--- /dev/null
+++ b/meta/recipes-devtools/rpm/rpmresolve/rpmresolve.c
@@ -0,0 +1,273 @@
+/* OpenEmbedded RPM resolver utility
+
+  Written by: Paul Eggleton <paul.eggleton@linux.intel.com>
+
+  Copyright 2012 Intel Corporation
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License version 2 as
+  published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+*/
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+#include <rpmdb.h>
+#include <rpmtypes.h>
+#include <rpmtag.h>
+#include <rpmts.h>
+#include <rpmmacro.h>
+#include <rpmcb.h>
+#include <rpmlog.h>
+#include <argv.h>
+#include <mire.h>
+
+int getPackageStr(rpmts ts, const char *NVRA, rpmTag tag, char **value)
+{
+    int rc = -1;
+    rpmmi mi = rpmtsInitIterator(ts, RPMTAG_NVRA, NVRA, 0);
+    Header h;
+    if ((h = rpmmiNext(mi)) != NULL) {
+        HE_t he = (HE_t) memset(alloca(sizeof(*he)), 0, sizeof(*he));
+        he->tag = tag;
+        rc = (headerGet(h, he, 0) != 1);
+        if(rc==0)
+            *value = strdup((char *)he->p.ptr);
+    }
+    (void)rpmmiFree(mi);
+    return rc;
+}
+
+int loadTs(rpmts **ts, int *tsct, const char *dblistfn)
+{
+    int count = 0;
+    int sz = 5;
+    int rc = 0;
+    int listfile = 1;
+    struct stat st_buf;
+
+    rc = stat(dblistfn, &st_buf);
+    if(rc != 0) {
+        perror("stat");
+        return 1;
+    }
+    if(S_ISDIR(st_buf.st_mode))
+        listfile = 0;
+
+    if(listfile) {
+        *ts = malloc(sz * sizeof(rpmts));
+        FILE *f = fopen(dblistfn, "r" );
+        if(f) {
+            char line[2048];
+            while(fgets(line, sizeof(line), f)) {
+                int len = strlen(line) - 1;
+                if(len > 0)
+                    // Trim trailing whitespace
+                    while(len > 0 && isspace(line[len]))
+                        line[len--] = '\0';
+
+                if(len > 0) {
+                    // Expand array if needed
+                    if(count == sz) {
+                        sz += 5;
+                        *ts = (rpmts *)realloc(*ts, sz);
+                    }
+
+                    char *dbpathm = malloc(strlen(line) + 10);
+                    sprintf(dbpathm, "_dbpath %s", line);
+                    rpmDefineMacro(NULL, dbpathm, RMIL_CMDLINE);
+                    free(dbpathm);
+
+                    rpmts tsi = rpmtsCreate();
+                    (*ts)[count] = tsi;
+                    rc = rpmtsOpenDB(tsi, O_RDONLY);
+                    if( rc ) {
+                        fprintf(stderr, "Failed to open database %s\n", line);
+                        rc = -1;
+                        break;
+                    }
+
+                    count++;
+                }
+            }
+            fclose(f);
+            *tsct = count;
+        }
+        else {
+            perror(dblistfn);
+            rc = -1;
+        }
+    }
+    else {
+        // Load from single database
+        *ts = malloc(sizeof(rpmts));
+        char *dbpathm = malloc(strlen(dblistfn) + 10);
+        sprintf(dbpathm, "_dbpath %s", dblistfn);
+        rpmDefineMacro(NULL, dbpathm, RMIL_CMDLINE);
+        free(dbpathm);
+
+        rpmts tsi = rpmtsCreate();
+        (*ts)[0] = tsi;
+        rc = rpmtsOpenDB(tsi, O_RDONLY);
+        if( rc ) {
+            fprintf(stderr, "Failed to open database %s\n", dblistfn);
+            rc = -1;
+        }
+        *tsct = 1;
+    }
+
+    return rc;
+}
+
+int processPackages(rpmts *ts, int tscount, const char *packagelistfn, int ignoremissing)
+{
+    int rc = 0;
+    int count = 0;
+    int sz = 100;
+    int i = 0;
+    int missing = 0;
+
+    FILE *f = fopen(packagelistfn, "r" );
+    if(f) {
+        char line[255];
+        while(fgets(line, sizeof(line), f)) {
+            int len = strlen(line) - 1;
+            if(len > 0)
+                // Trim trailing whitespace
+                while(len > 0 && isspace(line[len]))
+                    line[len--] = '\0';
+
+            if(len > 0) {
+                int found = 0;
+                for(i=0; i<tscount; i++) {
+                    ARGV_t keys = NULL;
+                    rpmdb db = rpmtsGetRdb(ts[i]);
+                    rc = rpmdbMireApply(db, RPMTAG_NAME,
+                                RPMMIRE_STRCMP, line, &keys);
+                    if (keys) {
+                        int nkeys = argvCount(keys);
+                        if( nkeys == 1 ) {
+                            char *value = NULL;
+                            rc = getPackageStr(ts[i], keys[0], RPMTAG_PACKAGEORIGIN, &value);
+                            if(rc == 0)
+                                printf("%s\n", value);
+                            else
+                                fprintf(stderr, "Failed to get package origin for %s\n", line);
+                            found = 1;
+                        }
+                        else if( nkeys > 1 ) {
+                            fprintf(stderr, "Multiple matches for %s!\n", line);
+                        }
+                    }
+                    if(found)
+                        break;
+                }
+
+                if( !found ) {
+                    if( ignoremissing ) {
+                        fprintf(stderr, "unable to find package %s - ignoring\n", line);
+                    }
+                    else {
+                        fprintf(stderr, "unable to find package %s\n", line);
+                        missing = 1;
+                    }
+                }
+            }
+            count++;
+        }
+        fclose(f);
+
+        if( missing ) {
+            fprintf(stderr, "ERROR: some packages were missing\n");
+            rc = 1;
+        }
+    }
+    else {
+        perror(packagelistfn);
+        rc = -1;
+    }
+
+    return rc;
+}
+
+void usage()
+{
+    fprintf(stderr, "OpenEmbedded rpm resolver utility\n");
+    fprintf(stderr, "syntax: rpmresolve [-i] <dblistfile> <packagelistfile>\n");
+}
+
+int main(int argc, char **argv)
+{
+    rpmts *ts = NULL;
+    int tscount = 0;
+    int rc = 0;
+    int i;
+    int c;
+    int ignoremissing = 0;
+
+    opterr = 0;
+    while ((c = getopt (argc, argv, "i")) != -1) {
+        switch (c) {
+            case 'i':
+                ignoremissing = 1;
+                break;
+            case '?':
+                if(isprint(optopt))
+                    fprintf(stderr, "Unknown option `-%c'.\n", optopt);
+                else
+                    fprintf(stderr, "Unknown option character `\\x%x'.\n",
+                        optopt);
+                usage();
+                return 1;
+            default:
+                abort();
+        }
+    }
+
+    if( argc - optind < 1 ) {
+        usage();
+        return 1;
+    }
+
+    const char *dblistfn = argv[optind];
+
+    //rpmSetVerbosity(RPMLOG_DEBUG);
+
+    rpmReadConfigFiles( NULL, NULL );
+    rpmDefineMacro(NULL, "__dbi_txn create nofsync", RMIL_CMDLINE);
+
+    rc = loadTs(&ts, &tscount, dblistfn);
+    if( rc )
+        return 1;
+    if( tscount == 0 ) {
+        fprintf(stderr, "Please specify database list file or database location\n");
+        return 1;
+    }
+
+    if( argc - optind < 2 ) {
+        fprintf(stderr, "Please specify package list file\n");
+        return 1;
+    }
+    const char *pkglistfn = argv[optind+1];
+    rc = processPackages(ts, tscount, pkglistfn, ignoremissing);
+
+    for(i=0; i<tscount; i++)
+        (void) rpmtsCloseDB(ts[i]);
+    free(ts);
+
+    return rc;
+}
diff --git a/meta/recipes-devtools/rpm/rpmresolve_1.0.bb b/meta/recipes-devtools/rpm/rpmresolve_1.0.bb
new file mode 100644
index 0000000000..f8750e02ba
--- /dev/null
+++ b/meta/recipes-devtools/rpm/rpmresolve_1.0.bb
@@ -0,0 +1,22 @@
+SUMMARY = "OpenEmbedded RPM resolver utility"
+DESCRIPTION = "OpenEmbedded RPM resolver - performs RPM database lookups in batches to avoid \
+ repeated invocations of rpm on the command line."
+DEPENDS = "rpm"
+LICENSE = "GPLv2"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/GPL-2.0;md5=801f80980d171dd6425610833a22dbe6"
+PR = "r0"
+
+SRC_URI = "file://rpmresolve.c"
+
+S = "${WORKDIR}"
+
+do_compile() {
+        ${CC} ${CFLAGS} -ggdb -I${STAGING_INCDIR}/rpm ${LDFLAGS} rpmresolve.c -o rpmresolve -lrpmbuild -lrpm -lrpmio -lrpmdb -lpopt
+}
+
+do_install() {
+        install -d ${D}${bindir}
+        install -m 0755 rpmresolve ${D}${bindir}
+}
+
+BBCLASSEXTEND = "native"