summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorRobert Yang <liezhi.yang@windriver.com>2013-01-31 16:45:33 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-03-07 11:14:37 +0000
commit1e6d0da8d5456d04c2354d37900a59d292bcd898 (patch)
treef38562f5cb8f7a482deb66abea4ebe7b30abe49b /scripts
parent4d5ef2af504635c60ffd2d971191bc111035c3a2 (diff)
downloadpoky-1e6d0da8d5456d04c2354d37900a59d292bcd898.tar.gz
bitbake-whatchanged: print what is about to happen
* Contents: - Summary - Usage - Implementation summary - Output - TODO * Summary: This is used for printing what is about to happen between the current and last builds, for example: $ bitbake core-image-sato # Edit some recipes $ bitbake-whatchanged core-image-sato The changes will be printed. * Usage: bitbake-whatchanged [[opts] recipe] * Implementation summary: - Use the "STAMPS_DIR=<path> bitbake -S recipe" to generate the new stamps, compare these stamps to the one in the old stamps dir (tmp/stamps), so we will get what are changed. - When the "-v" (verbose) is not specified: > Figure out the newly added tasks > Figure out the PV (including PE) and PR changed tasks > The left tasks are the ones that the "Dependencies" changed tasks - When "-v" is specified: > Figure out the newly added tasks > Use bb.siggen.compare_sigfiles to figure out the details * Output, for example (core-image-sato with different git tags) and with recipes upgraded): - without "-v": Figuring out the STAMPS_DIR ... Generating the new stamps ... (need several minutes) === Newly added tasks: (5 tasks) core-image-sato: do_configure do_populate_lic do_install do_rootfs do_compile # Note: This is because the "bitbake -S" always generate the sigdata for # do_compile, do_rootfs and other task, we may need fix this from "bitbake -S" === PV changed: (130 tasks) alsa-utils: 1.0.25 -> 1.0.26 cross-localedef-native: 2.16 -> 2.17 eglibc-initial: 2.16 -> 2.17 [snip] === Dependencies changed: (3593 tasks) busybox: do_package do_package_write do_build do_packagedata do_populate_sysroot do_install do_compile do_package_write_rpm do_configure do_populate_lic atk-native: do_compile do_package_write_rpm do_package do_configure do_populate_sysroot do_install do_populate_lic do_patch do_packagedata do_build do_package_write do_unpack [snip] === Summary: (3728 changed, 1134 unchanged) Newly added: 5 PV changed: 130 PR changed: 0 Dependencies changed: 3593 Removing the newly generated stamps dir ... - with "-v": === Newly added tasks: (5 tasks) core-image-sato: do_configure do_populate_lic do_install do_rootfs do_compile === The verbose changes of glib-2.0-native.do_do_install: Hash for dependent task virtual:native:glib-2.0_2.34.3.bb.do_compile changed from bab8b8dd95be1b83dcec93f755b1812b to 70f746df7809acaa52de204b0685abb4 [snip] === Summary: (3728 changed, 1134 unchanged) Newly added: 5 Dependencies changed: 3723 Removing the newly generated stamps dir ... * TODO - It seems that the "bitbake -S core-image-sato" has bugs, it would always report errors, but doesn't fatal errors - The gcc-cross' stamps are in tmp/stamps/work-shared, but the "bitbake -S" doesn't put the stamps in work-shared. - The "bitbake -S" always generates the sigdata for image recipe's do_compile, do_install and other tasks, we may need fix this from "bitbake -S". - Print the ones which can be installed from the sstate. [YOCTO #1659] (From OE-Core rev: 8783fcc23ccbd829ecb0dc59cf71ee44376094cc) Signed-off-by: Robert Yang <liezhi.yang@windriver.com> Signed-off-by: Saul Wold <sgw@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/bitbake-whatchanged339
1 files changed, 339 insertions, 0 deletions
diff --git a/scripts/bitbake-whatchanged b/scripts/bitbake-whatchanged
new file mode 100755
index 0000000000..90ad2f850c
--- /dev/null
+++ b/scripts/bitbake-whatchanged
@@ -0,0 +1,339 @@
1#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4
5# Copyright (c) 2013 Wind River Systems, Inc.
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 2 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
14# See the GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program; if not, write to the Free Software
18# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
20from __future__ import print_function
21import os
22import sys
23import getopt
24import shutil
25import re
26import warnings
27import subprocess
28from optparse import OptionParser
29
30# Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process
31p = subprocess.Popen("bash -c 'echo $(dirname $(which bitbake-diffsigs | grep -v \'^alias\'))/../lib'",
32 shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
33
34err = p.stderr.read()
35if err:
36 print("ERROR: Failed to locate bitbake-diffsigs:", file=sys.stderr)
37 print(err, file=sys.stderr)
38 sys.exit(1)
39
40sys.path.insert(0, p.stdout.read().rstrip('\n'))
41
42import bb.siggen
43import bb.process
44
45# Match the stamp's filename
46# group(1): PE_PV (may no PE)
47# group(2): PR
48# group(3): TASK
49# group(4): HASH
50stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)")
51sigdata_re = re.compile(".*\.sigdata\..*")
52
53def gen_dict(stamps):
54 """
55 Generate the dict from the stamps dir.
56 The output dict format is:
57 {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}}
58 Where:
59 fake_f: pv + task + hash
60 path: the path to the stamp file
61 """
62 # The member of the sub dict (A "path" will be appended below)
63 sub_mem = ("pv", "pr", "task")
64 d = {}
65 for dirpath, _, files in os.walk(stamps):
66 for f in files:
67 # The "bitbake -S" would generate ".sigdata", but no "_setscene".
68 fake_f = re.sub('_setscene.', '.', f)
69 fake_f = re.sub('.sigdata', '', fake_f)
70 subdict = {}
71 tmp = stamp_re.match(fake_f)
72 if tmp:
73 for i in sub_mem:
74 subdict[i] = tmp.group(i)
75 if len(subdict) != 0:
76 pn = os.path.basename(dirpath)
77 subdict['pn'] = pn
78 # The path will be used by os.stat() and bb.siggen
79 subdict['path'] = dirpath + "/" + f
80 fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash')
81 d[fake_f] = subdict
82 return d
83
84# Re-construct the dict
85def recon_dict(dict_in):
86 """
87 The output dict format is:
88 {pn_task: {pv: PV, pr: PR, path: PATH}}
89 """
90 dict_out = {}
91 for k in dict_in.keys():
92 subdict = {}
93 # The key
94 pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task'))
95 # If more than one stamps are found, use the latest one.
96 if pn_task in dict_out:
97 full_path_pre = dict_out.get(pn_task).get('path')
98 full_path_cur = dict_in.get(k).get('path')
99 if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime:
100 continue
101 subdict['pv'] = dict_in.get(k).get('pv')
102 subdict['pr'] = dict_in.get(k).get('pr')
103 subdict['path'] = dict_in.get(k).get('path')
104 dict_out[pn_task] = subdict
105
106 return dict_out
107
108def split_pntask(s):
109 """
110 Split the pn_task in to (pn, task) and return it
111 """
112 tmp = re.match("(.*)_(do_.*)", s)
113 return (tmp.group(1), tmp.group(2))
114
115
116def print_added(d_new = None, d_old = None):
117 """
118 Print the newly added tasks
119 """
120 added = {}
121 for k in d_new.keys():
122 if k not in d_old:
123 # Add the new one to added dict, and remove it from
124 # d_new, so the remaining ones are the changed ones
125 added[k] = d_new.get(k)
126 del(d_new[k])
127
128 if not added:
129 return 0
130
131 # Format the output, the dict format is:
132 # {pn: task1, task2 ...}
133 added_format = {}
134 counter = 0
135 for k in added.keys():
136 pn, task = split_pntask(k)
137 if pn in added_format:
138 # Append the value
139 added_format[pn] = "%s %s" % (added_format.get(pn), task)
140 else:
141 added_format[pn] = task
142 counter += 1
143 print("=== Newly added tasks: (%s tasks)" % counter)
144 for k in added_format.keys():
145 print(" %s: %s" % (k, added_format.get(k)))
146
147 return counter
148
149def print_vrchanged(d_new = None, d_old = None, vr = None):
150 """
151 Print the pv or pr changed tasks.
152 The arg "vr" is "pv" or "pr"
153 """
154 pvchanged = {}
155 counter = 0
156 for k in d_new.keys():
157 if d_new.get(k).get(vr) != d_old.get(k).get(vr):
158 counter += 1
159 pn, task = split_pntask(k)
160 if pn not in pvchanged:
161 # Format the output, we only print pn (no task) since
162 # all the tasks would be changed when pn or pr changed,
163 # the dict format is:
164 # {pn: pv/pr_old -> pv/pr_new}
165 pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr))
166 del(d_new[k])
167
168 if not pvchanged:
169 return 0
170
171 print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter))
172 for k in pvchanged.keys():
173 print(" %s: %s" % (k, pvchanged.get(k)))
174
175 return counter
176
177def print_depchanged(d_new = None, d_old = None, verbose = False):
178 """
179 Print the dependency changes
180 """
181 depchanged = {}
182 counter = 0
183 for k in d_new.keys():
184 counter += 1
185 pn, task = split_pntask(k)
186 if (verbose):
187 full_path_old = d_old.get(k).get("path")
188 full_path_new = d_new.get(k).get("path")
189 # No counter since it is not ready here
190 if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new):
191 output = bb.siggen.compare_sigfiles(full_path_old, full_path_new)
192 if output:
193 print("\n=== The verbose changes of %s.do_%s:" % (pn, task))
194 print('\n'.join(output))
195 else:
196 # Format the output, the format is:
197 # {pn: task1, task2, ...}
198 if pn in depchanged:
199 depchanged[pn] = "%s %s" % (depchanged.get(pn), task)
200 else:
201 depchanged[pn] = task
202
203 if len(depchanged) > 0:
204 print("\n=== Dependencies changed: (%s tasks)" % counter)
205 for k in depchanged.keys():
206 print(" %s: %s" % (k, depchanged[k]))
207
208 return counter
209
210
211def main():
212 """
213 Print what will be done between the current and last builds:
214 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps
215 2) Figure out what are newly added and changed, can't figure out
216 what are removed since we can't know the previous stamps
217 clearly, for example, if there are several builds, we can't know
218 which stamps the last build has used exactly.
219 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps
220 """
221
222 parser = OptionParser(
223 version = "1.0",
224 usage = """%prog [options] [package ...]
225print what will be done between the current and last builds, for example:
226
227 $ bitbake core-image-sato
228 # Edit the recipes
229 $ bitbake-whatchanged core-image-sato
230
231The changes will be printed"
232
233Note:
234 The amount of tasks is not accurate when the task is "do_build" since
235 it usually depends on other tasks.
236 The "nostamp" task is not included.
237"""
238)
239 parser.add_option("-v", "--verbose", help = "print the verbose changes",
240 action = "store_true", dest = "verbose")
241
242 options, args = parser.parse_args(sys.argv)
243
244 verbose = options.verbose
245
246 if len(args) != 2:
247 parser.error("Incorrect number of arguments")
248 else:
249 recipe = args[1]
250
251 # Get the STAMPS_DIR
252 print("Figuring out the STAMPS_DIR ...")
253 cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'"
254 try:
255 stampsdir, err = bb.process.run(cmdline)
256 except:
257 raise
258 if not stampsdir:
259 print("ERROR: No STAMPS_DIR found for '%s'" % recipe, file=sys.stderr)
260 return 2
261 stampsdir = stampsdir.rstrip("\n")
262 if not os.path.isdir(stampsdir):
263 print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr)
264 return 2
265
266 # The new stamps dir
267 new_stampsdir = stampsdir + ".bbs"
268 if os.path.exists(new_stampsdir):
269 print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr)
270 return 2
271
272 try:
273 # Generate the new stamps dir
274 print("Generating the new stamps ... (need several minutes)")
275 cmdline = "STAMPS_DIR=%s bitbake -S %s" % (new_stampsdir, recipe)
276 # FIXME
277 # The "bitbake -S" may fail, not fatal error, the stamps will still
278 # be generated, this might be a bug of "bitbake -S".
279 try:
280 bb.process.run(cmdline)
281 except Exception as exc:
282 print(exc)
283
284 # The dict for the new and old stamps.
285 old_dict = gen_dict(stampsdir)
286 new_dict = gen_dict(new_stampsdir)
287
288 # Remove the same one from both stamps.
289 cnt_unchanged = 0
290 for k in new_dict.keys():
291 if k in old_dict:
292 cnt_unchanged += 1
293 del(new_dict[k])
294 del(old_dict[k])
295
296 # Re-construct the dict to easily find out what is added or changed.
297 # The dict format is:
298 # {pn_task: {pv: PV, pr: PR, path: PATH}}
299 new_recon = recon_dict(new_dict)
300 old_recon = recon_dict(old_dict)
301
302 del new_dict
303 del old_dict
304
305 # Figure out what are changed, the new_recon would be changed
306 # by the print_xxx function.
307 # Newly added
308 cnt_added = print_added(new_recon, old_recon)
309
310 # PV (including PE) and PR changed
311 # Let the bb.siggen handle them if verbose
312 cnt_rv = {}
313 if not verbose:
314 for i in ('pv', 'pr'):
315 cnt_rv[i] = print_vrchanged(new_recon, old_recon, i)
316
317 # Dependencies changed (use bitbake-diffsigs)
318 cnt_dep = print_depchanged(new_recon, old_recon, verbose)
319
320 total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep
321
322 print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged))
323 if verbose:
324 print("Newly added: %s\nDependencies changed: %s\n" % \
325 (cnt_added, cnt_dep))
326 else:
327 print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \
328 (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep))
329 except:
330 print("ERROR occurred!")
331 raise
332 finally:
333 # Remove the newly generated stamps dir
334 if os.path.exists(new_stampsdir):
335 print("Removing the newly generated stamps dir ...")
336 shutil.rmtree(new_stampsdir)
337
338if __name__ == "__main__":
339 sys.exit(main())