summaryrefslogtreecommitdiffstats
path: root/scripts/contrib/test_build_time.sh
blob: 9e5725ae54eabc0ea456764cd762e5c359167420 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
#!/bin/bash

# Build performance regression test script
#
# Copyright 2011 Intel Corporation
# All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#
# DESCRIPTION
# This script is intended to be used in conjunction with "git bisect run"
# in order to find regressions in build time, however it can also be used
# independently. It cleans out the build output directories, runs a
# specified worker script (an example is test_build_time_worker.sh) under
# TIME(1), logs the results to TEST_LOGDIR (default /tmp) and returns a
# value telling "git bisect run" whether the build time is good (under
# the specified threshold) or bad (over it). There is also a tolerance
# option but it is not particularly useful as it only subtracts the
# tolerance from the given threshold and uses it as the actual threshold.
#
# It is also capable of taking a file listing git revision hashes to be
# test-applied to the repository in order to get past build failures that
# would otherwise cause certain revisions to have to be skipped; if a
# revision does not apply cleanly then the script assumes it does not
# need to be applied and ignores it.
#
# Please see the help output (syntax below) for some important setup
# instructions.
#
# AUTHORS
# Paul Eggleton <paul.eggleton@linux.intel.com>


syntax() {
    echo "syntax: $0 <script> <time> <tolerance> [patchrevlist]"
    echo ""
    echo "  script       - worker script file (if in current dir, prefix with ./)"
    echo "  time         - time threshold (in seconds, suffix m for minutes)"
    echo "  tolerance    - tolerance (in seconds, suffix m for minutes or % for"
    echo "                 percentage, can be 0)"
    echo "  patchrevlist - optional file listing revisions to apply as patches on top"
    echo ""
    echo "You must set TEST_BUILDDIR to point to a previously created build directory,"
    echo "however please note that this script will wipe out the TMPDIR defined in"
    echo "TEST_BUILDDIR/conf/local.conf as part of its initial setup (as well as your"
    echo "~/.ccache)"
    echo ""
    echo "To get rid of the sudo prompt, please add the following line to /etc/sudoers"
    echo "(use 'visudo' to edit this; also it is assumed that the user you are running"
    echo "as is a member of the 'wheel' group):"
    echo ""
    echo "%wheel ALL=(ALL) NOPASSWD: /sbin/sysctl -w vm.drop_caches=[1-3]"
    echo ""
    echo "Note: it is recommended that you disable crond and any other process that"
    echo "may cause significant CPU or I/O usage during build performance tests."
}

# Note - we exit with 250 here because that will tell git bisect run that
# something bad happened and stop
if [ "$1" = "" ] ; then
   syntax
   exit 250
fi

if [ "$2" = "" ] ; then
   syntax
   exit 250
fi

if [ "$3" = "" ] ; then
   syntax
   exit 250
fi

if ! [[ "$2" =~ ^[0-9][0-9m.]*$ ]] ; then
   echo "'$2' is not a valid number for threshold"
   exit 250
fi

if ! [[ "$3" =~ ^[0-9][0-9m.%]*$ ]] ; then
   echo "'$3' is not a valid number for tolerance"
   exit 250
fi

if [ "$TEST_BUILDDIR" = "" ] ; then
   echo "Please set TEST_BUILDDIR to a previously created build directory"
   exit 250
fi

if [ ! -d "$TEST_BUILDDIR" ] ; then
   echo "TEST_BUILDDIR $TEST_BUILDDIR not found"
   exit 250
fi

git diff --quiet
if [ $? != 0 ] ; then
    echo "Working tree is dirty, cannot proceed"
    exit 251
fi

if [ "$BB_ENV_EXTRAWHITE" != "" ] ; then
   echo "WARNING: you are running after sourcing the build environment script, this is not recommended"
fi

runscript=$1
timethreshold=$2
tolerance=$3

if [ "$4" != "" ] ; then
    patchrevlist=`cat $4`
else
    patchrevlist=""
fi

if [[ timethreshold == *m* ]] ; then
    timethreshold=`echo $timethreshold | sed s/m/*60/ | bc`
fi

if [[ $tolerance == *m* ]] ; then
    tolerance=`echo $tolerance | sed s/m/*60/ | bc`
elif [[ $tolerance == *%* ]] ; then
    tolerance=`echo $tolerance | sed s/%//`
    tolerance=`echo "scale = 2; (($tolerance * $timethreshold) / 100)" | bc`
fi

tmpdir=`grep "^TMPDIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/TMPDIR[ \t]*=[ \t\?]*"//' -e 's/"//'`
if [ "x$tmpdir" = "x" ]; then
    echo "Unable to determine TMPDIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
    exit 250
fi
sstatedir=`grep "^SSTATE_DIR" $TEST_BUILDDIR/conf/local.conf | sed -e 's/SSTATE_DIR[ \t\?]*=[ \t]*"//' -e 's/"//'`
if [ "x$sstatedir" = "x" ]; then
    echo "Unable to determine SSTATE_DIR from $TEST_BUILDDIR/conf/local.conf, bailing out"
    exit 250
fi

if [ `expr length $tmpdir` -lt 4 ] ; then
    echo "TMPDIR $tmpdir is less than 4 characters, bailing out"
    exit 250
fi

if [ `expr length $sstatedir` -lt 4 ] ; then
    echo "SSTATE_DIR $sstatedir is less than 4 characters, bailing out"
    exit 250
fi

echo -n "About to wipe out TMPDIR $tmpdir, press Ctrl+C to break out...  "
for i in 9 8 7 6 5 4 3 2 1
do
    echo -ne "\x08$i"
    sleep 1
done
echo

pushd . > /dev/null

rm -f pseudodone
echo "Removing TMPDIR $tmpdir..."
rm -rf $tmpdir
echo "Removing TMPDIR $tmpdir-*libc..."
rm -rf $tmpdir-*libc
echo "Removing SSTATE_DIR $sstatedir..."
rm -rf $sstatedir
echo "Removing ~/.ccache..."
rm -rf ~/.ccache

echo "Syncing..."
sync
sync
echo "Dropping VM cache..."
#echo 3 > /proc/sys/vm/drop_caches
sudo /sbin/sysctl -w vm.drop_caches=3 > /dev/null

if [ "$TEST_LOGDIR" = "" ] ; then
    logdir="/tmp"
else
    logdir="$TEST_LOGDIR"
fi
rev=`git rev-parse HEAD`
logfile="$logdir/timelog_$rev.log"
echo -n > $logfile

gitroot=`git rev-parse --show-toplevel`
cd $gitroot
for patchrev in $patchrevlist ; do
    echo "Applying $patchrev"
    patchfile=`mktemp`
    git show $patchrev > $patchfile
    git apply --check $patchfile &> /dev/null
    if [ $? != 0 ] ; then
        echo " ... patch does not apply without errors, ignoring"
    else
        echo "Applied $patchrev" >> $logfile
        git apply $patchfile &> /dev/null
    fi
    rm $patchfile
done

sync
echo "Quiescing for 5s..."
sleep 5

echo "Running $runscript at $rev..."
timeoutfile=`mktemp`
/usr/bin/time -o $timeoutfile -f "%e\nreal\t%E\nuser\t%Us\nsys\t%Ss\nmaxm\t%Mk" $runscript 2>&1 | tee -a $logfile
exitstatus=$PIPESTATUS

git reset --hard HEAD > /dev/null
popd > /dev/null

timeresult=`head -n1 $timeoutfile`
cat $timeoutfile | tee -a $logfile
rm $timeoutfile

if [ $exitstatus != 0 ] ; then
    # Build failed, exit with 125 to tell git bisect run to skip this rev
    echo "*** Build failed (exit code $exitstatus), skipping..." | tee -a $logfile
    exit 125
fi

ret=`echo "scale = 2; $timeresult > $timethreshold - $tolerance" | bc`
echo "Returning $ret" | tee -a $logfile
exit $ret