summaryrefslogtreecommitdiffstats
path: root/meta/recipes-devtools/build-compare/files
diff options
context:
space:
mode:
Diffstat (limited to 'meta/recipes-devtools/build-compare/files')
-rw-r--r--meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch361
1 files changed, 361 insertions, 0 deletions
diff --git a/meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch b/meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch
new file mode 100644
index 0000000000..f5b99d9447
--- /dev/null
+++ b/meta/recipes-devtools/build-compare/files/functions.sh-run-rpm-once-to-make-it-faster.patch
@@ -0,0 +1,361 @@
1From f34d632c427b1e1570ef9136454fc01d8c9f10a6 Mon Sep 17 00:00:00 2001
2From: Robert Yang <liezhi.yang@windriver.com>
3Date: Thu, 7 Jul 2016 19:34:09 -0700
4Subject: [PATCH] functions.sh: run rpm once to make it faster
5
6The rpm tool is a heavy process, it ran 16 (or 17 for kernel)
7"rpm -qp" times when the pkgs are identical, now we only run
8"rpm -qp --qf <all we need>" twice (one is for old pkg, and one is for
9new), save the results to spec_old and spec_new, then use sed command to
10get what we need later, this can make it 75% faster when the pkgs are
11identical. Here is the rough data on my host Ubuntu 14.04.4, 32 cores
12CPU and 128G mem:
13* When the pkgs are identical:
14 - Before the patch: 1s
15 - After the patch: 0.26s
16 I compare the whole spec firstly, and return 0 if they are the same,
17 or go on checking one by one if not, without this, it would be 0.46s,
18 the gain is great when there are lot of packages, usually, we have
19 more than 10,000 rpms to compare.
20
21* When the pkgs are different:
22 That depends on where is the different, if the different is at the
23 comparing rpmtags stage:
24 - Before the patch: 0.26s
25 - After the patch: 0.29s
26 Increased 0.03s, but if the different is happend later than comparing
27 rpmtags, it will save time.
28
29Upstream-Status: Submitted [https://github.com/openSUSE/build-compare/pull/9]
30
31Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
32---
33 functions.sh | 245 ++++++++++++++++++++++++++++++++---------------------------
34 1 file changed, 132 insertions(+), 113 deletions(-)
35 mode change 100644 => 100755 functions.sh
36
37diff --git a/functions.sh b/functions.sh
38old mode 100644
39new mode 100755
40index b1069d2..aa572f9
41--- a/functions.sh
42+++ b/functions.sh
43@@ -10,9 +10,63 @@
44
45 RPM="rpm -qp --nodigest --nosignature"
46
47-check_header()
48+# Name, Version, Release
49+QF_NAME="%{NAME}"
50+QF_VER_REL="%{VERSION}-%{RELEASE}"
51+QF_NAME_VER_REL="%{NAME}-%{VERSION}-%{RELEASE}"
52+
53+# provides destroy this because at least the self-provide includes the
54+# -buildnumber :-(
55+QF_PROVIDES="[%{PROVIDENAME} %{PROVIDEFLAGS} %{PROVIDEVERSION}\\n]\\n"
56+QF_PROVIDES="$QF_PROVIDES [%{REQUIRENAME} %{REQUIREFLAGS} %{REQUIREVERSION}\\n]\\n"
57+QF_PROVIDES="$QF_PROVIDES [%{CONFLICTNAME} %{CONFLICTFLAGS} %{CONFLICTVERSION}\\n]\\n"
58+QF_PROVIDES="$QF_PROVIDES [%{OBSOLETENAME} %{OBSOLETEFLAGS} %{OBSOLETEVERSION}\\n]\\n"
59+
60+# don't look at RELEASE, it contains our build number
61+QF_TAGS="%{NAME} %{VERSION} %{EPOCH}\\n"
62+QF_TAGS="$QF_TAGS %{SUMMARY}\\n%{DESCRIPTION}\\n"
63+# the DISTURL tag can be used as checkin ID
64+QF_TAGS="$QF_TAGS %{VENDOR} %{DISTRIBUTION} %{DISTURL}"
65+QF_TAGS="$QF_TAGS %{LICENSE} %{LICENSE}\\n"
66+QF_TAGS="$QF_TAGS %{GROUP} %{URL} %{EXCLUDEARCH} %{EXCLUDEOS} %{EXCLUSIVEARCH}\\n"
67+QF_TAGS="$QF_TAGS %{EXCLUSIVEOS} %{RPMVERSION} %{PLATFORM}\\n"
68+QF_TAGS="$QF_TAGS %{PAYLOADFORMAT} %{PAYLOADCOMPRESSOR} %{PAYLOADFLAGS}\\n"
69+
70+# XXX We also need to check the existence (but not the content (!))
71+# of SIGGPG (and perhaps the other SIG*)
72+# XXX We don't look at triggers
73+QF_TAGS="$QF_TAGS [%{VERIFYSCRIPTPROG} %{VERIFYSCRIPT}]\\n"
74+# Only the first ChangeLog entry; should be enough
75+QF_TAGS="$QF_TAGS %{CHANGELOGTIME} %{CHANGELOGNAME} %{CHANGELOGTEXT}\\n"
76+
77+# scripts, might contain release number
78+QF_SCRIPT="[%{PREINPROG} %{PREIN}\\n]\\n[%{POSTINPROG} %{POSTIN}\\n]\\n[%{PREUNPROG} %{PREUN}\\n]\\n[%{POSTUNPROG} %{POSTUN}\\n]\\n"
79+
80+# Now the files. We leave out mtime and size. For normal files
81+# the size will influence the MD5 anyway. For directories the sizes can
82+# differ, depending on which file system the package was built. To not
83+# have to filter out directories we simply ignore all sizes.
84+# Also leave out FILEDEVICES, FILEINODES (depends on the build host),
85+# FILECOLORS, FILECLASS (normally useful but file output contains mtimes),
86+# FILEDEPENDSX and FILEDEPENDSN.
87+# Also FILELANGS (or?)
88+QF_FILELIST="[%{FILENAMES} %{FILEFLAGS} %{FILESTATES} %{FILEMODES:octal} %{FILEUSERNAME} %{FILEGROUPNAME} %{FILERDEVS} %{FILEVERIFYFLAGS} %{FILELINKTOS}\n]\\n"
89+# ??? what to do with FILEPROVIDE and FILEREQUIRE?
90+
91+QF_CHECKSUM="[%{FILENAMES} %{FILEMD5S} %{FILEFLAGS}\n]\\n"
92+
93+QF_ALL="\n___QF_NAME___\n${QF_NAME}\n___QF_NAME___\n"
94+QF_ALL="$QF_ALL\n___QF_TAGS___\n${QF_TAGS}\n___QF_TAGS___\n"
95+QF_ALL="$QF_ALL\n___QF_VER_REL___\n${QF_VER_REL}\n___QF_VER_REL___\n"
96+QF_ALL="$QF_ALL\n___QF_NAME_VER_REL___\n${QF_NAME_VER_REL}\n___QF_NAME_VER_REL___\n"
97+QF_ALL="$QF_ALL\n___QF_PROVIDES___\n${QF_PROVIDES}\n___QF_PROVIDES___\n"
98+QF_ALL="$QF_ALL\n___QF_SCRIPT___\n${QF_SCRIPT}\n___QF_SCRIPT___\n"
99+QF_ALL="$QF_ALL\n___QF_FILELIST___\n${QF_FILELIST}\n___QF_FILELIST___\n"
100+QF_ALL="$QF_ALL\n___QF_CHECKSUM___\n${QF_CHECKSUM}\n___QF_CHECKSUM___\n"
101+
102+check_header()
103 {
104- $RPM --qf "$QF" "$1"
105+ $RPM --qf "$1" "$2"
106 }
107
108 # Trim version-release string:
109@@ -47,18 +101,6 @@ function grep_release_new()
110 grep -E "(/boot|/lib/modules|/lib/firmware|/usr/src)/[^/]+(${version_release_new_regex_l}(\$|[^/]+\$)|${version_release_new_regex_s}(\$|[^/]+\$))"
111 }
112
113-function check_provides()
114-{
115- local pkg=$1
116- # provides destroy this because at least the self-provide includes the
117- # -buildnumber :-(
118- QF="[%{PROVIDENAME} %{PROVIDEFLAGS} %{PROVIDEVERSION}\\n]\\n"
119- QF="$QF [%{REQUIRENAME} %{REQUIREFLAGS} %{REQUIREVERSION}\\n]\\n"
120- QF="$QF [%{CONFLICTNAME} %{CONFLICTFLAGS} %{CONFLICTVERSION}\\n]\\n"
121- QF="$QF [%{OBSOLETENAME} %{OBSOLETEFLAGS} %{OBSOLETEVERSION}\\n]\\n"
122- check_header "$pkg"
123-}
124-
125 #usage unpackage <file> $dir
126 # Unpack files in directory $dir
127 # like /usr/bin/unpackage - just for one file and with no options
128@@ -98,6 +140,30 @@ function unpackage()
129 popd 1>/dev/null
130 }
131
132+# Run diff command on the files
133+# $1: printed info
134+# $2: file1
135+# $3: file2
136+function comp_file()
137+{
138+ echo "comparing $1"
139+ if ! diff -au $2 $3; then
140+ if test -z "$check_all"; then
141+ rm $2 $3 $spec_old $spec_new
142+ return 1
143+ fi
144+ fi
145+ return 0
146+}
147+
148+# Get var's value from specfile.
149+# $1: var name
150+# $2: specfile
151+function get_value()
152+{
153+ sed -n -e "/^___${1}___/,/^___${1}___/p" $2 | sed -e "/^___${1}___/d"
154+}
155+
156 # Compare just the rpm meta data of two rpms
157 # Returns:
158 # 0 in case of same content
159@@ -107,56 +173,29 @@ function unpackage()
160 function cmp_spec ()
161 {
162 local RES
163- local file1 file2
164+ local file_old file_new
165 local f
166 local sh=$1
167 local oldrpm=$2
168 local newrpm=$3
169
170- QF="%{NAME}"
171-
172- # don't look at RELEASE, it contains our build number
173- QF="$QF %{VERSION} %{EPOCH}\\n"
174- QF="$QF %{SUMMARY}\\n%{DESCRIPTION}\\n"
175- QF="$QF %{VENDOR} %{DISTRIBUTION} %{DISTURL}"
176- QF="$QF %{LICENSE} %{LICENSE}\\n"
177- QF="$QF %{GROUP} %{URL} %{EXCLUDEARCH} %{EXCLUDEOS} %{EXCLUSIVEARCH}\\n"
178- QF="$QF %{EXCLUSIVEOS} %{RPMVERSION} %{PLATFORM}\\n"
179- QF="$QF %{PAYLOADFORMAT} %{PAYLOADCOMPRESSOR} %{PAYLOADFLAGS}\\n"
180-
181-
182- # XXX We also need to check the existence (but not the content (!))
183- # of SIGGPG (and perhaps the other SIG*)
184-
185- # XXX We don't look at triggers
186-
187- QF="$QF [%{VERIFYSCRIPTPROG} %{VERIFYSCRIPT}]\\n"
188-
189- # Only the first ChangeLog entry; should be enough
190- QF="$QF %{CHANGELOGTIME} %{CHANGELOGNAME} %{CHANGELOGTEXT}\\n"
191-
192- file1=`mktemp`
193- file2=`mktemp`
194-
195- check_header $oldrpm > $file1
196- check_header $newrpm > $file2
197-
198- # the DISTURL tag can be used as checkin ID
199- #echo "$QF"
200- echo "comparing rpmtags"
201- if ! diff -au $file1 $file2; then
202- if test -z "$check_all"; then
203- rm $file1 $file2
204- return 1
205- fi
206- fi
207-
208+ file_old=`mktemp`
209+ file_new=`mktemp`
210+ spec_old=`mktemp`
211+ spec_new=`mktemp`
212+
213+ check_header "$QF_ALL" $oldrpm > $spec_old
214+ check_header "$QF_ALL" $newrpm > $spec_new
215+
216+ name_new="$(get_value QF_NAME $spec_new)"
217+ version_release_new="$(get_value QF_VER_REL $spec_new)"
218+ name_ver_rel_new="$(get_value QF_NAME_VER_REL $spec_new)"
219+
220+ version_release_old="$(get_value QF_VER_REL $spec_old)"
221+ name_ver_rel_old="$(get_value QF_NAME_VER_REL $spec_old)"
222+
223 # Remember to quote the . which is in release
224- version_release_old=$($RPM --qf "%{VERSION}-%{RELEASE}" "$oldrpm")
225- version_release_new=$($RPM --qf "%{VERSION}-%{RELEASE}" "$newrpm")
226- name_ver_rel_old=$($RPM --qf "%{NAME}-%{VERSION}-%{RELEASE}" "$oldrpm")
227- name_ver_rel_new=$($RPM --qf "%{NAME}-%{VERSION}-%{RELEASE}" "$newrpm")
228- # Short version without B_CNT
229+ # Short version without B_CN
230 version_release_old_regex_s=${version_release_old%.*}
231 version_release_old_regex_s=${version_release_old_regex_s//./\\.}
232 version_release_new_regex_s=${version_release_new%.*}
233@@ -166,10 +205,27 @@ function cmp_spec ()
234 version_release_new_regex_l=${version_release_new//./\\.}
235 name_ver_rel_old_regex_l=${name_ver_rel_old//./\\.}
236 name_ver_rel_new_regex_l=${name_ver_rel_new//./\\.}
237+
238+ # Check the whole spec file at first, return 0 immediately if the
239+ # are the same.
240+ cat $spec_old | trim_release_old > $file_old
241+ cat $spec_new | trim_release_new > $file_new
242+ echo "comparing the whole specfile"
243+ if diff -au $spec_old $spec_new; then
244+ if test -z "$check_all"; then
245+ rm $file_old $file_new $spec_old $spec_new
246+ return 0
247+ fi
248+ fi
249+
250+ get_value QF_TAGS $spec_old > $file_old
251+ get_value QF_TAGS $spec_new > $file_new
252+ comp_file rpmtags $file_old $file_new || return 1
253+
254 # This might happen when?!
255 echo "comparing RELEASE"
256 if [ "${version_release_old%.*}" != "${version_release_new%.*}" ] ; then
257- case $($RPM --qf '%{NAME}' "$newrpm") in
258+ case $name_new in
259 kernel-*)
260 # Make sure all kernel packages have the same %RELEASE
261 echo "release prefix mismatch"
262@@ -181,71 +237,34 @@ function cmp_spec ()
263 *) ;;
264 esac
265 fi
266-
267- check_provides $oldrpm | trim_release_old | sort > $file1
268- check_provides $newrpm | trim_release_new | sort > $file2
269-
270- echo "comparing PROVIDES"
271- if ! diff -au $file1 $file2; then
272- if test -z "$check_all"; then
273- rm $file1 $file2
274- return 1
275- fi
276- fi
277
278- # scripts, might contain release number
279- QF="[%{PREINPROG} %{PREIN}\\n]\\n[%{POSTINPROG} %{POSTIN}\\n]\\n[%{PREUNPROG} %{PREUN}\\n]\\n[%{POSTUNPROG} %{POSTUN}\\n]\\n"
280- check_header $oldrpm | trim_release_old > $file1
281- check_header $newrpm | trim_release_new > $file2
282+ get_value QF_PROVIDES $spec_old | trim_release_old | sort > $file_old
283+ get_value QF_PROVIDES $spec_new | trim_release_new | sort > $file_new
284+ comp_file PROVIDES $file_old $file_new || return 1
285+
286+ get_value QF_SCRIPT $spec_old | trim_release_old > $file_old
287+ get_value QF_SCRIPT $spec_new | trim_release_new > $file_new
288+ comp_file scripts $file_old $file_new || return 1
289
290- echo "comparing scripts"
291- if ! diff -au $file1 $file2; then
292- if test -z "$check_all"; then
293- rm $file1 $file2
294- return 1
295- fi
296- fi
297-
298 # First check the file attributes and later the md5s
299-
300- # Now the files. We leave out mtime and size. For normal files
301- # the size will influence the MD5 anyway. For directories the sizes can
302- # differ, depending on which file system the package was built. To not
303- # have to filter out directories we simply ignore all sizes.
304- # Also leave out FILEDEVICES, FILEINODES (depends on the build host),
305- # FILECOLORS, FILECLASS (normally useful but file output contains mtimes),
306- # FILEDEPENDSX and FILEDEPENDSN.
307- # Also FILELANGS (or?)
308- QF="[%{FILENAMES} %{FILEFLAGS} %{FILESTATES} %{FILEMODES:octal} %{FILEUSERNAME} %{FILEGROUPNAME} %{FILERDEVS} %{FILEVERIFYFLAGS} %{FILELINKTOS}\n]\\n"
309- # ??? what to do with FILEPROVIDE and FILEREQUIRE?
310-
311- check_header $oldrpm | trim_release_old > $file1
312- check_header $newrpm | trim_release_new > $file2
313-
314- echo "comparing filelist"
315- if ! diff -au $file1 $file2; then
316- if test -z "$check_all"; then
317- rm $file1 $file2
318- return 1
319- fi
320- fi
321-
322+ get_value QF_FILELIST $spec_old | trim_release_old > $file_old
323+ get_value QF_FILELIST $spec_new | trim_release_new > $file_new
324+ comp_file filelist $file_old $file_new || return 1
325+
326 # now the md5sums. if they are different, we check more detailed
327 # if there are different filenames, we will already have aborted before
328 # file flag 64 means "ghost", filter those out.
329- QF="[%{FILENAMES} %{FILEMD5S} %{FILEFLAGS}\n]\\n"
330- check_header $oldrpm |grep -v " 64$"| trim_release_old > $file1
331- check_header $newrpm |grep -v " 64$"| trim_release_new > $file2
332-
333+ get_value QF_CHECKSUM $spec_old | grep -v " 64$" | trim_release_old > $file_old
334+ get_value QF_CHECKSUM $spec_new | grep -v " 64$" | trim_release_new > $file_new
335 RES=2
336 # done if the same
337 echo "comparing file checksum"
338- if cmp -s $file1 $file2; then
339+ if cmp -s $file_old $file_new; then
340 RES=0
341 fi
342-
343+
344 # Get only files with different MD5sums
345- files=`diff -U0 $file1 $file2 | fgrep -v +++ | grep ^+ | cut -b2- | awk '{print $1}'`
346+ files=`diff -U0 $file_old $file_new | fgrep -v +++ | grep ^+ | cut -b2- | awk '{print $1}'`
347
348 if test -f "$sh"; then
349 echo "creating rename script"
350@@ -261,7 +280,7 @@ function cmp_spec ()
351 done >> "${sh}"
352 fi
353 #
354- rm $file1 $file2
355+ rm $file_old $file_new
356 return $RES
357 }
358
359--
3602.9.0
361
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-11-12 02:02:15 +0000