2 files changed, 362 insertions, 0 deletions

diff --git a/meta/recipes-devtools/build-compare/build-compare_git.bb b/meta/recipes-devtools/build-compare/build-compare_git.bb
index 88996e27f8..9ec8c2392f 100644
--- a/meta/recipes-devtools/build-compare/build-compare_git.bb
+++ b/meta/recipes-devtools/build-compare/build-compare_git.bb
@@ -10,6 +10,7 @@ SRC_URI = "git://github.com/openSUSE/build-compare.git \
            file://Ignore-DWARF-sections.patch;striplevel=1 \
            file://0001-Add-support-for-deb-and-ipk-packaging.patch \
            file://functions.sh-remove-space-at-head.patch \
+           file://functions.sh-run-rpm-once-to-make-it-faster.patch \
            "
 
 SRCREV = "c5352c054c6ef15735da31b76d6d88620f4aff0a"
diff --git a/meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch b/meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch
new file mode 100644
index 0000000000..f5b99d9447
--- /dev/null
+++ b/meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch
@@ -0,0 +1,361 @@
+From f34d632c427b1e1570ef9136454fc01d8c9f10a6 Mon Sep 17 00:00:00 2001
+From: Robert Yang <liezhi.yang@windriver.com>
+Date: Thu, 7 Jul 2016 19:34:09 -0700
+Subject: [PATCH] functions.sh: run rpm once to make it faster
+
+The rpm tool is a heavy process, it ran 16 (or 17 for kernel)
+"rpm -qp" times when the pkgs are identical, now we only run
+"rpm -qp --qf <all we need>" twice (one is for old pkg, and one is for
+new), save the results to spec_old and spec_new, then use sed command to
+get what we need later, this can make it 75% faster when the pkgs are
+identical. Here is the rough data on my host Ubuntu 14.04.4, 32 cores
+CPU and 128G mem:
+* When the pkgs are identical:
+  - Before the patch: 1s
+  - After the patch: 0.26s
+  I compare the whole spec firstly, and return 0 if they are the same,
+  or go on checking one by one if not, without this, it would be 0.46s,
+  the gain is great when there are lot of packages, usually, we have
+  more than 10,000 rpms to compare.
+
+* When the pkgs are different:
+  That depends on where is the different, if the different is at the
+  comparing rpmtags stage:
+  - Before the patch: 0.26s
+  - After the patch: 0.29s
+  Increased 0.03s, but if the different is happend later than comparing
+  rpmtags, it will save time.
+
+Upstream-Status: Submitted [https://github.com/openSUSE/build-compare/pull/9]
+
+Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
+---
+ functions.sh | 245 ++++++++++++++++++++++++++++++++---------------------------
+ 1 file changed, 132 insertions(+), 113 deletions(-)
+ mode change 100644 => 100755 functions.sh
+
+diff --git a/functions.sh b/functions.sh
+old mode 100644
+new mode 100755
+index b1069d2..aa572f9
+--- a/functions.sh
++++ b/functions.sh
+@@ -10,9 +10,63 @@
+ 
+ RPM="rpm -qp --nodigest --nosignature"
+ 
+-check_header() 
++# Name, Version, Release
++QF_NAME="%{NAME}"
++QF_VER_REL="%{VERSION}-%{RELEASE}"
++QF_NAME_VER_REL="%{NAME}-%{VERSION}-%{RELEASE}"
++
++# provides destroy this because at least the self-provide includes the
++# -buildnumber :-(
++QF_PROVIDES="[%{PROVIDENAME} %{PROVIDEFLAGS} %{PROVIDEVERSION}\\n]\\n"
++QF_PROVIDES="$QF_PROVIDES [%{REQUIRENAME} %{REQUIREFLAGS} %{REQUIREVERSION}\\n]\\n"
++QF_PROVIDES="$QF_PROVIDES [%{CONFLICTNAME} %{CONFLICTFLAGS} %{CONFLICTVERSION}\\n]\\n"
++QF_PROVIDES="$QF_PROVIDES [%{OBSOLETENAME} %{OBSOLETEFLAGS} %{OBSOLETEVERSION}\\n]\\n"
++
++# don't look at RELEASE, it contains our build number
++QF_TAGS="%{NAME} %{VERSION} %{EPOCH}\\n"
++QF_TAGS="$QF_TAGS %{SUMMARY}\\n%{DESCRIPTION}\\n"
++# the DISTURL tag can be used as checkin ID
++QF_TAGS="$QF_TAGS %{VENDOR} %{DISTRIBUTION} %{DISTURL}"
++QF_TAGS="$QF_TAGS %{LICENSE} %{LICENSE}\\n"
++QF_TAGS="$QF_TAGS %{GROUP} %{URL} %{EXCLUDEARCH} %{EXCLUDEOS} %{EXCLUSIVEARCH}\\n"
++QF_TAGS="$QF_TAGS %{EXCLUSIVEOS} %{RPMVERSION} %{PLATFORM}\\n"
++QF_TAGS="$QF_TAGS %{PAYLOADFORMAT} %{PAYLOADCOMPRESSOR} %{PAYLOADFLAGS}\\n"
++
++# XXX We also need to check the existence (but not the content (!))
++# of SIGGPG (and perhaps the other SIG*)
++# XXX We don't look at triggers
++QF_TAGS="$QF_TAGS [%{VERIFYSCRIPTPROG} %{VERIFYSCRIPT}]\\n"
++# Only the first ChangeLog entry; should be enough
++QF_TAGS="$QF_TAGS %{CHANGELOGTIME} %{CHANGELOGNAME} %{CHANGELOGTEXT}\\n"
++
++# scripts, might contain release number
++QF_SCRIPT="[%{PREINPROG} %{PREIN}\\n]\\n[%{POSTINPROG} %{POSTIN}\\n]\\n[%{PREUNPROG} %{PREUN}\\n]\\n[%{POSTUNPROG} %{POSTUN}\\n]\\n"
++
++# Now the files. We leave out mtime and size.  For normal files
++# the size will influence the MD5 anyway.  For directories the sizes can
++# differ, depending on which file system the package was built.  To not
++# have to filter out directories we simply ignore all sizes.
++# Also leave out FILEDEVICES, FILEINODES (depends on the build host),
++# FILECOLORS, FILECLASS (normally useful but file output contains mtimes),
++# FILEDEPENDSX and FILEDEPENDSN.
++# Also FILELANGS (or?)
++QF_FILELIST="[%{FILENAMES} %{FILEFLAGS} %{FILESTATES} %{FILEMODES:octal} %{FILEUSERNAME} %{FILEGROUPNAME} %{FILERDEVS} %{FILEVERIFYFLAGS} %{FILELINKTOS}\n]\\n"
++# ??? what to do with FILEPROVIDE and FILEREQUIRE?
++
++QF_CHECKSUM="[%{FILENAMES} %{FILEMD5S} %{FILEFLAGS}\n]\\n"
++
++QF_ALL="\n___QF_NAME___\n${QF_NAME}\n___QF_NAME___\n"
++QF_ALL="$QF_ALL\n___QF_TAGS___\n${QF_TAGS}\n___QF_TAGS___\n"
++QF_ALL="$QF_ALL\n___QF_VER_REL___\n${QF_VER_REL}\n___QF_VER_REL___\n"
++QF_ALL="$QF_ALL\n___QF_NAME_VER_REL___\n${QF_NAME_VER_REL}\n___QF_NAME_VER_REL___\n"
++QF_ALL="$QF_ALL\n___QF_PROVIDES___\n${QF_PROVIDES}\n___QF_PROVIDES___\n"
++QF_ALL="$QF_ALL\n___QF_SCRIPT___\n${QF_SCRIPT}\n___QF_SCRIPT___\n"
++QF_ALL="$QF_ALL\n___QF_FILELIST___\n${QF_FILELIST}\n___QF_FILELIST___\n"
++QF_ALL="$QF_ALL\n___QF_CHECKSUM___\n${QF_CHECKSUM}\n___QF_CHECKSUM___\n"
++
++check_header()
+ {
+-   $RPM --qf "$QF" "$1"
++   $RPM --qf "$1" "$2"
+ }
+ 
+ # Trim version-release string:
+@@ -47,18 +101,6 @@ function grep_release_new()
+   grep -E "(/boot|/lib/modules|/lib/firmware|/usr/src)/[^/]+(${version_release_new_regex_l}(\$|[^/]+\$)|${version_release_new_regex_s}(\$|[^/]+\$))"
+ }
+ 
+-function check_provides()
+-{
+-  local pkg=$1
+-  # provides destroy this because at least the self-provide includes the
+-  # -buildnumber :-(
+-  QF="[%{PROVIDENAME} %{PROVIDEFLAGS} %{PROVIDEVERSION}\\n]\\n"
+-  QF="$QF [%{REQUIRENAME} %{REQUIREFLAGS} %{REQUIREVERSION}\\n]\\n"
+-  QF="$QF [%{CONFLICTNAME} %{CONFLICTFLAGS} %{CONFLICTVERSION}\\n]\\n"
+-  QF="$QF [%{OBSOLETENAME} %{OBSOLETEFLAGS} %{OBSOLETEVERSION}\\n]\\n"
+-  check_header "$pkg"
+-}
+-
+ #usage unpackage <file> $dir
+ # Unpack files in directory $dir
+ # like /usr/bin/unpackage - just for one file and with no options
+@@ -98,6 +140,30 @@ function unpackage()
+     popd 1>/dev/null
+ }
+ 
++# Run diff command on the files
++# $1: printed info
++# $2: file1
++# $3: file2
++function comp_file()
++{
++    echo "comparing $1"
++    if ! diff -au $2 $3; then
++      if test -z "$check_all"; then
++        rm $2 $3 $spec_old $spec_new
++        return 1
++      fi
++    fi
++    return 0
++}
++
++# Get var's value from specfile.
++# $1: var name
++# $2: specfile
++function get_value()
++{
++    sed -n -e "/^___${1}___/,/^___${1}___/p" $2 | sed -e "/^___${1}___/d"
++}
++
+ # Compare just the rpm meta data of two rpms
+ # Returns:
+ # 0 in case of same content
+@@ -107,56 +173,29 @@ function unpackage()
+ function cmp_spec ()
+ {
+     local RES
+-    local file1 file2
++    local file_old file_new
+     local f
+     local sh=$1
+     local oldrpm=$2
+     local newrpm=$3
+ 
+-    QF="%{NAME}"
+-    
+-    # don't look at RELEASE, it contains our build number
+-    QF="$QF %{VERSION} %{EPOCH}\\n"
+-    QF="$QF %{SUMMARY}\\n%{DESCRIPTION}\\n"
+-    QF="$QF %{VENDOR} %{DISTRIBUTION} %{DISTURL}"
+-    QF="$QF %{LICENSE} %{LICENSE}\\n"
+-    QF="$QF %{GROUP} %{URL} %{EXCLUDEARCH} %{EXCLUDEOS} %{EXCLUSIVEARCH}\\n"
+-    QF="$QF %{EXCLUSIVEOS} %{RPMVERSION} %{PLATFORM}\\n"
+-    QF="$QF %{PAYLOADFORMAT} %{PAYLOADCOMPRESSOR} %{PAYLOADFLAGS}\\n"
+-    
+- 
+-    # XXX We also need to check the existence (but not the content (!))
+-    # of SIGGPG (and perhaps the other SIG*)
+-    
+-    # XXX We don't look at triggers
+-    
+-    QF="$QF [%{VERIFYSCRIPTPROG} %{VERIFYSCRIPT}]\\n"
+-    
+-    # Only the first ChangeLog entry; should be enough
+-    QF="$QF %{CHANGELOGTIME} %{CHANGELOGNAME} %{CHANGELOGTEXT}\\n"
+-    
+-    file1=`mktemp`
+-    file2=`mktemp`
+-    
+-    check_header $oldrpm > $file1
+-    check_header $newrpm > $file2
+-    
+-    # the DISTURL tag can be used as checkin ID
+-    #echo "$QF"
+-    echo "comparing rpmtags"
+-    if ! diff -au $file1 $file2; then
+-      if test -z "$check_all"; then
+-        rm $file1 $file2
+-        return 1
+-      fi
+-    fi
+-    
++    file_old=`mktemp`
++    file_new=`mktemp`
++    spec_old=`mktemp`
++    spec_new=`mktemp`
++
++    check_header "$QF_ALL" $oldrpm > $spec_old
++    check_header "$QF_ALL" $newrpm > $spec_new
++
++    name_new="$(get_value QF_NAME $spec_new)"
++    version_release_new="$(get_value QF_VER_REL $spec_new)"
++    name_ver_rel_new="$(get_value QF_NAME_VER_REL $spec_new)"
++
++    version_release_old="$(get_value QF_VER_REL $spec_old)"
++    name_ver_rel_old="$(get_value QF_NAME_VER_REL $spec_old)"
++
+     # Remember to quote the . which is in release
+-    version_release_old=$($RPM --qf "%{VERSION}-%{RELEASE}" "$oldrpm")
+-    version_release_new=$($RPM --qf "%{VERSION}-%{RELEASE}" "$newrpm")
+-    name_ver_rel_old=$($RPM --qf "%{NAME}-%{VERSION}-%{RELEASE}" "$oldrpm")
+-    name_ver_rel_new=$($RPM --qf "%{NAME}-%{VERSION}-%{RELEASE}" "$newrpm")
+-    # Short version without B_CNT
++    # Short version without B_CN
+     version_release_old_regex_s=${version_release_old%.*}
+     version_release_old_regex_s=${version_release_old_regex_s//./\\.}
+     version_release_new_regex_s=${version_release_new%.*}
+@@ -166,10 +205,27 @@ function cmp_spec ()
+     version_release_new_regex_l=${version_release_new//./\\.}
+     name_ver_rel_old_regex_l=${name_ver_rel_old//./\\.}
+     name_ver_rel_new_regex_l=${name_ver_rel_new//./\\.}
++
++    # Check the whole spec file at first, return 0 immediately if the
++    # are the same.
++    cat $spec_old | trim_release_old > $file_old
++    cat $spec_new | trim_release_new > $file_new
++    echo "comparing the whole specfile"
++    if diff -au $spec_old $spec_new; then
++      if test -z "$check_all"; then
++        rm $file_old $file_new $spec_old $spec_new
++        return 0
++      fi
++    fi
++
++    get_value QF_TAGS $spec_old > $file_old
++    get_value QF_TAGS $spec_new > $file_new
++    comp_file rpmtags $file_old $file_new || return 1
++
+     # This might happen when?!
+     echo "comparing RELEASE"
+     if [ "${version_release_old%.*}" != "${version_release_new%.*}" ] ; then
+-      case $($RPM --qf '%{NAME}' "$newrpm") in
++      case $name_new in
+         kernel-*)
+           # Make sure all kernel packages have the same %RELEASE
+           echo "release prefix mismatch"
+@@ -181,71 +237,34 @@ function cmp_spec ()
+         *) ;;
+       esac
+     fi
+-    
+-    check_provides $oldrpm | trim_release_old | sort > $file1
+-    check_provides $newrpm | trim_release_new | sort > $file2
+-    
+-    echo "comparing PROVIDES"
+-    if ! diff -au $file1 $file2; then
+-      if test -z "$check_all"; then
+-        rm $file1 $file2
+-        return 1
+-      fi
+-    fi
+ 
+-    # scripts, might contain release number
+-    QF="[%{PREINPROG} %{PREIN}\\n]\\n[%{POSTINPROG} %{POSTIN}\\n]\\n[%{PREUNPROG} %{PREUN}\\n]\\n[%{POSTUNPROG} %{POSTUN}\\n]\\n"
+-    check_header $oldrpm | trim_release_old > $file1
+-    check_header $newrpm | trim_release_new > $file2
++    get_value QF_PROVIDES $spec_old | trim_release_old | sort > $file_old
++    get_value QF_PROVIDES $spec_new | trim_release_new | sort > $file_new
++    comp_file PROVIDES $file_old $file_new || return 1
++
++    get_value QF_SCRIPT $spec_old | trim_release_old > $file_old
++    get_value QF_SCRIPT $spec_new | trim_release_new > $file_new
++    comp_file scripts $file_old $file_new || return 1
+ 
+-    echo "comparing scripts"
+-    if ! diff -au $file1 $file2; then
+-      if test -z "$check_all"; then
+-        rm $file1 $file2
+-        return 1
+-      fi
+-    fi
+-    
+     # First check the file attributes and later the md5s
+-    
+-    # Now the files.  We leave out mtime and size.  For normal files
+-    # the size will influence the MD5 anyway.  For directories the sizes can
+-    # differ, depending on which file system the package was built.  To not
+-    # have to filter out directories we simply ignore all sizes.
+-    # Also leave out FILEDEVICES, FILEINODES (depends on the build host),
+-    # FILECOLORS, FILECLASS (normally useful but file output contains mtimes), 
+-    # FILEDEPENDSX and FILEDEPENDSN. 
+-    # Also FILELANGS (or?)
+-    QF="[%{FILENAMES} %{FILEFLAGS} %{FILESTATES} %{FILEMODES:octal} %{FILEUSERNAME} %{FILEGROUPNAME} %{FILERDEVS} %{FILEVERIFYFLAGS} %{FILELINKTOS}\n]\\n"
+-    # ??? what to do with FILEPROVIDE and FILEREQUIRE?
+-
+-    check_header $oldrpm | trim_release_old > $file1
+-    check_header $newrpm | trim_release_new > $file2
+-    
+-    echo "comparing filelist"
+-    if ! diff -au $file1 $file2; then
+-      if test -z "$check_all"; then
+-        rm $file1 $file2
+-        return 1
+-      fi
+-    fi
+-    
++    get_value QF_FILELIST $spec_old | trim_release_old > $file_old
++    get_value QF_FILELIST $spec_new | trim_release_new > $file_new
++    comp_file filelist $file_old $file_new || return 1
++
+     # now the md5sums. if they are different, we check more detailed
+     # if there are different filenames, we will already have aborted before
+     # file flag 64 means "ghost", filter those out.
+-    QF="[%{FILENAMES} %{FILEMD5S} %{FILEFLAGS}\n]\\n"
+-    check_header $oldrpm |grep -v " 64$"| trim_release_old > $file1
+-    check_header $newrpm |grep -v " 64$"| trim_release_new > $file2
+-    
++    get_value QF_CHECKSUM $spec_old | grep -v " 64$" | trim_release_old > $file_old
++    get_value QF_CHECKSUM $spec_new | grep -v " 64$" | trim_release_new > $file_new
+     RES=2
+     # done if the same
+     echo "comparing file checksum"
+-    if cmp -s $file1 $file2; then
++    if cmp -s $file_old $file_new; then
+       RES=0
+     fi
+-    
++
+     # Get only files with different MD5sums
+-    files=`diff -U0 $file1 $file2 | fgrep -v +++ | grep ^+ | cut -b2- | awk '{print $1}'`
++    files=`diff -U0 $file_old $file_new | fgrep -v +++ | grep ^+ | cut -b2- | awk '{print $1}'`
+ 
+     if test -f "$sh"; then
+       echo "creating rename script"
+@@ -261,7 +280,7 @@ function cmp_spec ()
+       done >> "${sh}"
+     fi
+     #
+-    rm $file1 $file2
++    rm $file_old $file_new
+     return $RES
+ }
+ 
+-- 
+2.9.0
+