summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorMartin Jansa <Martin.Jansa@gmail.com>2013-07-06 15:43:00 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-07-29 13:09:05 +0100
commit5c4513daf653b632096176f4af21706c66e716cc (patch)
tree7d9e57b5d44ad28af83344013cabaf87747ceb75 /scripts
parent462acd212a9e7364ce329a4fc70aa09ab93628b3 (diff)
downloadpoky-5c4513daf653b632096176f4af21706c66e716cc.tar.gz
test-dependencies: add simple script to detect missing or autoenabled dependencies
(From OE-Core rev: a2b3c9e01c871a395a93e162731db77a618306cb) Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com> Signed-off-by: Saul Wold <sgw@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/test-dependencies.sh256
1 files changed, 256 insertions, 0 deletions
diff --git a/scripts/test-dependencies.sh b/scripts/test-dependencies.sh
new file mode 100755
index 0000000000..e4cf4d366b
--- /dev/null
+++ b/scripts/test-dependencies.sh
@@ -0,0 +1,256 @@
1#!/bin/sh
2
3# Author: Martin Jansa <martin.jansa@gmail.com>
4#
5# Copyright (c) 2013 Martin Jansa <Martin.Jansa@gmail.com>
6
7# Used to detect missing dependencies or automagically
8# enabled dependencies which aren't explicitly enabled
9# or disabled.
10
11# It does 3 builds of <target>
12# 1st to populate sstate-cache directory and sysroot
13# 2nd to rebuild each recipe with every possible
14# dependency found in sysroot (which stays populated
15# from 1st build
16# 3rd to rebuild each recipe only with dependencies defined
17# in DEPENDS
18# 4th (optional) repeat build like 3rd to make sure that
19# minimal versions of dependencies defined in DEPENDS
20# is also enough
21
22# Global vars
23tmpdir=
24targets=
25recipes=
26buildhistory=
27buildtype=
28default_targets="world"
29default_buildhistory="buildhistory"
30default_buildtype="1 2 3 c"
31
32usage () {
33 cat << EOF
34Welcome to utility to detect missing or autoenabled dependencies.
35WARNING: this utility will completely remove your tmpdir (make sure
36 you don't have important buildhistory or persistent dir there).
37$0 <OPTION>
38
39Options:
40 -h, --help
41 Display this help and exit.
42
43 --tmpdir=<tmpdir>
44 Specify tmpdir, will use the environment variable TMPDIR if it is not specified.
45 Something like /OE/oe-core/tmp-eglibc (no / at the end).
46
47 --targets=<targets>
48 List of targets separated by space, will use the environment variable TARGETS if it is not specified.
49 It will run "bitbake <targets>" to populate sysroots.
50 Default value is "world".
51
52 --recipes=<recipes>
53 File with list of recipes we want to rebuild with minimal and maximal sysroot.
54 Will use the environment variable RECIPES if it is not specified.
55 Default value will use all packages ever recorded in buildhistory directory.
56
57 --buildhistory=<buildhistory>
58 Path to buildhistory directory, it needs to be enabled in your config,
59 because it's used to detect different dependencies and to create list
60 of recipes to rebuild when it's not specified.
61 Will use the environment variable BUILDHISTORY if it is not specified.
62 Default value is "buildhistory"
63
64 --buildtype=<buildtype>
65 There are 4 types of build:
66 1: build to populate sstate-cache directory and sysroot
67 2: build to rebuild each recipe with every possible dep
68 3: build to rebuild each recipe with minimal dependencies
69 4: build to rebuild each recipe again with minimal dependencies
70 c: compare buildhistory directories from build 2 and 3
71 Will use the environment variable BUILDTYPE if it is not specified.
72 Default value is "1 2 3 c", order is important, type 4 is optional.
73EOF
74}
75
76# Print error information and exit.
77echo_error () {
78 echo "ERROR: $1" >&2
79 exit 1
80}
81
82while [ -n "$1" ]; do
83 case $1 in
84 --tmpdir=*)
85 tmpdir=`echo $1 | sed -e 's#^--tmpdir=##' | xargs readlink -e`
86 [ -d "$tmpdir" ] || echo_error "Invalid argument to --tmpdir"
87 shift
88 ;;
89 --targets=*)
90 targets=`echo $1 | sed -e 's#^--targets="*\([^"]*\)"*#\1#'`
91 shift
92 ;;
93 --recipes=*)
94 recipes=`echo $1 | sed -e 's#^--recipes="*\([^"]*\)"*#\1#'`
95 shift
96 ;;
97 --buildhistory=*)
98 buildhistory=`echo $1 | sed -e 's#^--buildhistory="*\([^"]*\)"*#\1#'`
99 shift
100 ;;
101 --buildtype=*)
102 buildtype=`echo $1 | sed -e 's#^--buildtype="*\([^"]*\)"*#\1#'`
103 shift
104 ;;
105 --help|-h)
106 usage
107 exit 0
108 ;;
109 *)
110 echo "Invalid arguments $*"
111 echo_error "Try '$0 -h' for more information."
112 ;;
113 esac
114done
115
116# tmpdir directory, use environment variable TMPDIR
117# if it was not specified, otherwise, error.
118[ -n "$tmpdir" ] || tmpdir=$TMPDIR
119[ -n "$tmpdir" ] || echo_error "No tmpdir found!"
120[ -d "$tmpdir" ] || echo_error "Invalid tmpdir \"$tmpdir\""
121[ -n "$targets" ] || targets=$TARGETS
122[ -n "$targets" ] || targets=$default_targets
123[ -n "$recipes" ] || recipes=$RECIPES
124[ -n "$recipes" -a ! -f "$recipes" ] && echo_error "Invalid file with list of recipes to rebuild"
125[ -n "$recipes" ] || echo "All packages ever recorded in buildhistory directory will be rebuilt"
126[ -n "$buildhistory" ] || buildhistory=$BUILDHISTORY
127[ -n "$buildhistory" ] || buildhistory=$default_buildhistory
128[ -d "$buildhistory" ] || echo_error "Invalid buildhistory directory \"$buildhistory\""
129[ -n "$buildtype" ] || buildtype=$BUILDTYPE
130[ -n "$buildtype" ] || buildtype=$default_buildtype
131echo "$buildtype" | grep -v '^[1234c ]*$' && echo_error "Invalid buildtype \"$buildtype\", only some combination of 1, 2, 3, 4, c separated by space is allowed"
132
133OUTPUT_BASE=test-dependencies/`date "+%s"`
134
135build_all() {
136 echo "===== 1st build to populate sstate-cache directory and sysroot ====="
137 OUTPUT1=${OUTPUT_BASE}/${TYPE}_all
138 mkdir -p ${OUTPUT1}
139 echo "Logs will be stored in ${OUTPUT1} directory"
140 bitbake -k $targets | tee -a ${OUTPUT1}/complete.log
141}
142
143build_every_recipe() {
144 if [ "${TYPE}" = "2" ] ; then
145 echo "===== 2nd build to rebuild each recipe with every possible dep ====="
146 OUTPUT_MAX=${OUTPUT_BASE}/${TYPE}_max
147 OUTPUTB=${OUTPUT_MAX}
148 else
149 echo "===== 3rd or 4th build to rebuild each recipe with minimal dependencies ====="
150 OUTPUT_MIN=${OUTPUT_BASE}/${TYPE}_min
151 OUTPUTB=${OUTPUT_MIN}
152 fi
153
154 mkdir -p ${OUTPUTB} ${OUTPUTB}/failed ${OUTPUTB}/ok
155 echo "Logs will be stored in ${OUTPUTB} directory"
156 if [ -z "$recipes" ]; then
157 ls -d $buildhistory/packages/*/* | xargs -n 1 basename | sort -u > ${OUTPUTB}/recipe.list
158 recipes=${OUTPUTB}/recipe.list
159 fi
160 if [ "${TYPE}" != "2" ] ; then
161 echo "!!!Removing tmpdir \"$tmpdir\"!!!"
162 rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
163 fi
164 i=1
165 count=`cat $recipes | wc -l`
166 for recipe in `cat $recipes`; do
167 echo "Building recipe: ${recipe} ($i/$count)"
168 bitbake -c cleansstate ${recipe} > ${OUTPUTB}/log.${recipe} 2>&1;
169 bitbake ${recipe} >> ${OUTPUTB}/log.${recipe} 2>&1;
170 grep "ERROR: Task.*failed" ${OUTPUTB}/log.${recipe} && mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/failed/${recipe} || mv ${OUTPUTB}/log.${recipe} ${OUTPUTB}/ok/${recipe}
171 if [ "${TYPE}" != "2" ] ; then
172 rm -rf $tmpdir/deploy $tmpdir/pkgdata $tmpdir/sstate-control $tmpdir/stamps $tmpdir/sysroots $tmpdir/work $tmpdir/work-shared 2>/dev/null
173 fi
174 i=`expr $i + 1`
175 done
176 echo "Copying buildhistory/packages to ${OUTPUTB}"
177 cp -ra $buildhistory/packages ${OUTPUTB}
178 # This will be usefull to see which library is pulling new dependency
179 echo "Copying do_package logs to ${OUTPUTB}/do_package/"
180 mkdir ${OUTPUTB}/do_package
181 find $tmpdir/work/ -name log.do_package | while read f; do
182 # pn is 3 levels back, but we don't know if there is just one log per pn (only one arch and version)
183 # dest=`echo $f | sed 's#^.*/\([^/]*\)/\([^/]*\)/\([^/]*\)/log.do_package#\1#g'`
184 dest=`echo $f | sed "s#$tmpdir/work/##g; s#/#_#g"`
185 cp $f ${OUTPUTB}/do_package/$dest
186 done
187 grep "ERROR: Task.*failed" ${OUTPUTB}/failed/*
188 ls -1 ${OUTPUTB}/failed/* >> ${OUTPUT_BASE}/failed.recipes
189}
190
191compare_deps() {
192 # you can run just compare task with command like this
193 # OUTPUT_BASE=test-dependencies/1373140172 \
194 # OUTPUT_MAX=${OUTPUT_BASE}/2_max \
195 # OUTPUT_MIN=${OUTPUT_BASE}/3_min \
196 # openembedded-core/scripts/test-dependencies.sh --tmpdir=tmp-eglibc --targets=glib-2.0 --recipes=recipe_list --buildtype=c
197 echo "===== Compare dependencies recorded in \"${OUTPUT_MAX}\" and \"${OUTPUT_MIN}\" ====="
198 [ -n "${OUTPUTC}" ] || OUTPUTC=${OUTPUT_BASE}
199 mkdir -p ${OUTPUTC}
200 OUTPUT_FILE=${OUTPUTC}/dependency-changes
201 echo "Differences will be stored in ${OUTPUT_FILE}, dot is shown for every 100 of checked packages"
202 echo > ${OUTPUT_FILE}
203
204 [ -d ${OUTPUT_MAX} ] || echo_error "Directory with output from build 2 \"${OUTPUT_MAX}\" does not exist"
205 [ -d ${OUTPUT_MIN} ] || echo_error "Directory with output from build 3 \"${OUTPUT_MIN}\" does not exist"
206 [ -d ${OUTPUT_MAX}/packages/ ] || echo_error "Directory with packages from build 2 \"${OUTPUT_MAX}/packages/\" does not exist"
207 [ -d ${OUTPUT_MIN}/packages/ ] || echo_error "Directory with packages from build 3 \"${OUTPUT_MIN}/packages/\" does not exist"
208 i=0
209 find ${OUTPUT_MAX}/packages/ -name latest | sed "s#${OUTPUT_MAX}/##g" | while read pkg; do
210 max_pkg=${OUTPUT_MAX}/${pkg}
211 min_pkg=${OUTPUT_MIN}/${pkg}
212 if [ ! -f "${min_pkg}" ] ; then
213 echo "ERROR: ${min_pkg} doesn't exist" | tee -a ${OUTPUT_FILE}
214 continue
215 fi
216 # strip version information in parenthesis
217 max_deps=`grep "^RDEPENDS = " ${max_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
218 min_deps=`grep "^RDEPENDS = " ${min_pkg} | sed 's/^RDEPENDS = / /g; s/$/ /g; s/([^(]*)//g'`
219 if [ "$i" = 100 ] ; then
220 echo -n "." # cheap progressbar
221 i=0
222 fi
223 if [ "${max_deps}" = "${min_deps}" ] ; then
224 # it's annoying long, but at least it's showing some progress, warnings are grepped at the end
225 echo "NOTE: ${pkg} dependencies weren't changed" >> ${OUTPUT_FILE}
226 else
227 missing_deps=
228 for dep in ${max_deps}; do
229 echo "${min_deps}" | grep -q " ${dep} " || missing_deps="${missing_deps} ${dep}"
230 done
231 if [ -n "${missing_deps}" ] ; then
232 echo # to get rid of dots on last line
233 echo "WARN: ${pkg} lost dependency on ${missing_deps}" | tee -a ${OUTPUT_FILE}
234 fi
235 fi
236 i=`expr $i + 1`
237 done
238 echo # to get rid of dots on last line
239 echo "Found differences: "
240 grep "^WARN: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.warn
241 echo "Found errors: "
242 grep "^ERROR: " ${OUTPUT_FILE} | tee ${OUTPUT_FILE}.error
243 # useful for reexecuting this script with only small subset of recipes with known issues
244 sed 's#.*[ /]packages/\([^/]*\)/\([^/]*\)/.*#\2#g' ${OUTPUT_FILE}.warn ${OUTPUT_FILE}.error | sort -u >> ${OUTPUT_BASE}/failed.recipes
245}
246
247for TYPE in $buildtype; do
248 case ${TYPE} in
249 1) build_all;;
250 2) build_every_recipe;;
251 3) build_every_recipe;;
252 4) build_every_recipe;;
253 c) compare_deps;;
254 *) echo_error "Invalid buildtype \"$TYPE\""
255 esac
256done