diff options
Diffstat (limited to 'scripts/bitbake-whatchanged')
| -rwxr-xr-x | scripts/bitbake-whatchanged | 320 |
1 files changed, 0 insertions, 320 deletions
diff --git a/scripts/bitbake-whatchanged b/scripts/bitbake-whatchanged deleted file mode 100755 index 6f4b268119..0000000000 --- a/scripts/bitbake-whatchanged +++ /dev/null | |||
| @@ -1,320 +0,0 @@ | |||
| 1 | #!/usr/bin/env python3 | ||
| 2 | # ex:ts=4:sw=4:sts=4:et | ||
| 3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
| 4 | |||
| 5 | # Copyright (c) 2013 Wind River Systems, Inc. | ||
| 6 | # | ||
| 7 | # SPDX-License-Identifier: GPL-2.0-only | ||
| 8 | # | ||
| 9 | |||
| 10 | import os | ||
| 11 | import sys | ||
| 12 | import getopt | ||
| 13 | import shutil | ||
| 14 | import re | ||
| 15 | import warnings | ||
| 16 | import subprocess | ||
| 17 | import argparse | ||
| 18 | |||
| 19 | scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) | ||
| 20 | lib_path = scripts_path + '/lib' | ||
| 21 | sys.path = sys.path + [lib_path] | ||
| 22 | |||
| 23 | import scriptpath | ||
| 24 | |||
| 25 | # Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process | ||
| 26 | bitbakepath = scriptpath.add_bitbake_lib_path() | ||
| 27 | if not bitbakepath: | ||
| 28 | sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") | ||
| 29 | sys.exit(1) | ||
| 30 | scriptpath.add_oe_lib_path() | ||
| 31 | import argparse_oe | ||
| 32 | |||
| 33 | import bb.siggen | ||
| 34 | import bb.process | ||
| 35 | |||
| 36 | # Match the stamp's filename | ||
| 37 | # group(1): PE_PV (may no PE) | ||
| 38 | # group(2): PR | ||
| 39 | # group(3): TASK | ||
| 40 | # group(4): HASH | ||
| 41 | stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)") | ||
| 42 | sigdata_re = re.compile(".*\.sigdata\..*") | ||
| 43 | |||
| 44 | def gen_dict(stamps): | ||
| 45 | """ | ||
| 46 | Generate the dict from the stamps dir. | ||
| 47 | The output dict format is: | ||
| 48 | {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}} | ||
| 49 | Where: | ||
| 50 | fake_f: pv + task + hash | ||
| 51 | path: the path to the stamp file | ||
| 52 | """ | ||
| 53 | # The member of the sub dict (A "path" will be appended below) | ||
| 54 | sub_mem = ("pv", "pr", "task") | ||
| 55 | d = {} | ||
| 56 | for dirpath, _, files in os.walk(stamps): | ||
| 57 | for f in files: | ||
| 58 | # The "bitbake -S" would generate ".sigdata", but no "_setscene". | ||
| 59 | fake_f = re.sub('_setscene.', '.', f) | ||
| 60 | fake_f = re.sub('.sigdata', '', fake_f) | ||
| 61 | subdict = {} | ||
| 62 | tmp = stamp_re.match(fake_f) | ||
| 63 | if tmp: | ||
| 64 | for i in sub_mem: | ||
| 65 | subdict[i] = tmp.group(i) | ||
| 66 | if len(subdict) != 0: | ||
| 67 | pn = os.path.basename(dirpath) | ||
| 68 | subdict['pn'] = pn | ||
| 69 | # The path will be used by os.stat() and bb.siggen | ||
| 70 | subdict['path'] = dirpath + "/" + f | ||
| 71 | fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash') | ||
| 72 | d[fake_f] = subdict | ||
| 73 | return d | ||
| 74 | |||
| 75 | # Re-construct the dict | ||
| 76 | def recon_dict(dict_in): | ||
| 77 | """ | ||
| 78 | The output dict format is: | ||
| 79 | {pn_task: {pv: PV, pr: PR, path: PATH}} | ||
| 80 | """ | ||
| 81 | dict_out = {} | ||
| 82 | for k in dict_in.keys(): | ||
| 83 | subdict = {} | ||
| 84 | # The key | ||
| 85 | pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task')) | ||
| 86 | # If more than one stamps are found, use the latest one. | ||
| 87 | if pn_task in dict_out: | ||
| 88 | full_path_pre = dict_out.get(pn_task).get('path') | ||
| 89 | full_path_cur = dict_in.get(k).get('path') | ||
| 90 | if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime: | ||
| 91 | continue | ||
| 92 | subdict['pv'] = dict_in.get(k).get('pv') | ||
| 93 | subdict['pr'] = dict_in.get(k).get('pr') | ||
| 94 | subdict['path'] = dict_in.get(k).get('path') | ||
| 95 | dict_out[pn_task] = subdict | ||
| 96 | |||
| 97 | return dict_out | ||
| 98 | |||
| 99 | def split_pntask(s): | ||
| 100 | """ | ||
| 101 | Split the pn_task in to (pn, task) and return it | ||
| 102 | """ | ||
| 103 | tmp = re.match("(.*)_(do_.*)", s) | ||
| 104 | return (tmp.group(1), tmp.group(2)) | ||
| 105 | |||
| 106 | |||
| 107 | def print_added(d_new = None, d_old = None): | ||
| 108 | """ | ||
| 109 | Print the newly added tasks | ||
| 110 | """ | ||
| 111 | added = {} | ||
| 112 | for k in list(d_new.keys()): | ||
| 113 | if k not in d_old: | ||
| 114 | # Add the new one to added dict, and remove it from | ||
| 115 | # d_new, so the remaining ones are the changed ones | ||
| 116 | added[k] = d_new.get(k) | ||
| 117 | del(d_new[k]) | ||
| 118 | |||
| 119 | if not added: | ||
| 120 | return 0 | ||
| 121 | |||
| 122 | # Format the output, the dict format is: | ||
| 123 | # {pn: task1, task2 ...} | ||
| 124 | added_format = {} | ||
| 125 | counter = 0 | ||
| 126 | for k in added.keys(): | ||
| 127 | pn, task = split_pntask(k) | ||
| 128 | if pn in added_format: | ||
| 129 | # Append the value | ||
| 130 | added_format[pn] = "%s %s" % (added_format.get(pn), task) | ||
| 131 | else: | ||
| 132 | added_format[pn] = task | ||
| 133 | counter += 1 | ||
| 134 | print("=== Newly added tasks: (%s tasks)" % counter) | ||
| 135 | for k in added_format.keys(): | ||
| 136 | print(" %s: %s" % (k, added_format.get(k))) | ||
| 137 | |||
| 138 | return counter | ||
| 139 | |||
| 140 | def print_vrchanged(d_new = None, d_old = None, vr = None): | ||
| 141 | """ | ||
| 142 | Print the pv or pr changed tasks. | ||
| 143 | The arg "vr" is "pv" or "pr" | ||
| 144 | """ | ||
| 145 | pvchanged = {} | ||
| 146 | counter = 0 | ||
| 147 | for k in list(d_new.keys()): | ||
| 148 | if d_new.get(k).get(vr) != d_old.get(k).get(vr): | ||
| 149 | counter += 1 | ||
| 150 | pn, task = split_pntask(k) | ||
| 151 | if pn not in pvchanged: | ||
| 152 | # Format the output, we only print pn (no task) since | ||
| 153 | # all the tasks would be changed when pn or pr changed, | ||
| 154 | # the dict format is: | ||
| 155 | # {pn: pv/pr_old -> pv/pr_new} | ||
| 156 | pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr)) | ||
| 157 | del(d_new[k]) | ||
| 158 | |||
| 159 | if not pvchanged: | ||
| 160 | return 0 | ||
| 161 | |||
| 162 | print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter)) | ||
| 163 | for k in pvchanged.keys(): | ||
| 164 | print(" %s: %s" % (k, pvchanged.get(k))) | ||
| 165 | |||
| 166 | return counter | ||
| 167 | |||
| 168 | def print_depchanged(d_new = None, d_old = None, verbose = False): | ||
| 169 | """ | ||
| 170 | Print the dependency changes | ||
| 171 | """ | ||
| 172 | depchanged = {} | ||
| 173 | counter = 0 | ||
| 174 | for k in d_new.keys(): | ||
| 175 | counter += 1 | ||
| 176 | pn, task = split_pntask(k) | ||
| 177 | if (verbose): | ||
| 178 | full_path_old = d_old.get(k).get("path") | ||
| 179 | full_path_new = d_new.get(k).get("path") | ||
| 180 | # No counter since it is not ready here | ||
| 181 | if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new): | ||
| 182 | output = bb.siggen.compare_sigfiles(full_path_old, full_path_new) | ||
| 183 | if output: | ||
| 184 | print("\n=== The verbose changes of %s.%s:" % (pn, task)) | ||
| 185 | print('\n'.join(output)) | ||
| 186 | else: | ||
| 187 | # Format the output, the format is: | ||
| 188 | # {pn: task1, task2, ...} | ||
| 189 | if pn in depchanged: | ||
| 190 | depchanged[pn] = "%s %s" % (depchanged.get(pn), task) | ||
| 191 | else: | ||
| 192 | depchanged[pn] = task | ||
| 193 | |||
| 194 | if len(depchanged) > 0: | ||
| 195 | print("\n=== Dependencies changed: (%s tasks)" % counter) | ||
| 196 | for k in depchanged.keys(): | ||
| 197 | print(" %s: %s" % (k, depchanged[k])) | ||
| 198 | |||
| 199 | return counter | ||
| 200 | |||
| 201 | |||
| 202 | def main(): | ||
| 203 | """ | ||
| 204 | Print what will be done between the current and last builds: | ||
| 205 | 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps | ||
| 206 | 2) Figure out what are newly added and changed, can't figure out | ||
| 207 | what are removed since we can't know the previous stamps | ||
| 208 | clearly, for example, if there are several builds, we can't know | ||
| 209 | which stamps the last build has used exactly. | ||
| 210 | 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps | ||
| 211 | """ | ||
| 212 | |||
| 213 | parser = argparse_oe.ArgumentParser(usage = """%(prog)s [options] [package ...] | ||
| 214 | print what will be done between the current and last builds, for example: | ||
| 215 | |||
| 216 | $ bitbake core-image-sato | ||
| 217 | # Edit the recipes | ||
| 218 | $ bitbake-whatchanged core-image-sato | ||
| 219 | |||
| 220 | The changes will be printed. | ||
| 221 | |||
| 222 | Note: | ||
| 223 | The amount of tasks is not accurate when the task is "do_build" since | ||
| 224 | it usually depends on other tasks. | ||
| 225 | The "nostamp" task is not included. | ||
| 226 | """ | ||
| 227 | ) | ||
| 228 | parser.add_argument("recipe", help="recipe to check") | ||
| 229 | parser.add_argument("-v", "--verbose", help = "print the verbose changes", action = "store_true") | ||
| 230 | args = parser.parse_args() | ||
| 231 | |||
| 232 | # Get the STAMPS_DIR | ||
| 233 | print("Figuring out the STAMPS_DIR ...") | ||
| 234 | cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'" | ||
| 235 | try: | ||
| 236 | stampsdir, err = bb.process.run(cmdline) | ||
| 237 | except: | ||
| 238 | raise | ||
| 239 | if not stampsdir: | ||
| 240 | print("ERROR: No STAMPS_DIR found for '%s'" % args.recipe, file=sys.stderr) | ||
| 241 | return 2 | ||
| 242 | stampsdir = stampsdir.rstrip("\n") | ||
| 243 | if not os.path.isdir(stampsdir): | ||
| 244 | print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr) | ||
| 245 | return 2 | ||
| 246 | |||
| 247 | # The new stamps dir | ||
| 248 | new_stampsdir = stampsdir + ".bbs" | ||
| 249 | if os.path.exists(new_stampsdir): | ||
| 250 | print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr) | ||
| 251 | return 2 | ||
| 252 | |||
| 253 | try: | ||
| 254 | # Generate the new stamps dir | ||
| 255 | print("Generating the new stamps ... (need several minutes)") | ||
| 256 | cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, args.recipe) | ||
| 257 | # FIXME | ||
| 258 | # The "bitbake -S" may fail, not fatal error, the stamps will still | ||
| 259 | # be generated, this might be a bug of "bitbake -S". | ||
| 260 | try: | ||
| 261 | bb.process.run(cmdline) | ||
| 262 | except Exception as exc: | ||
| 263 | print(exc) | ||
| 264 | |||
| 265 | # The dict for the new and old stamps. | ||
| 266 | old_dict = gen_dict(stampsdir) | ||
| 267 | new_dict = gen_dict(new_stampsdir) | ||
| 268 | |||
| 269 | # Remove the same one from both stamps. | ||
| 270 | cnt_unchanged = 0 | ||
| 271 | for k in list(new_dict.keys()): | ||
| 272 | if k in old_dict: | ||
| 273 | cnt_unchanged += 1 | ||
| 274 | del(new_dict[k]) | ||
| 275 | del(old_dict[k]) | ||
| 276 | |||
| 277 | # Re-construct the dict to easily find out what is added or changed. | ||
| 278 | # The dict format is: | ||
| 279 | # {pn_task: {pv: PV, pr: PR, path: PATH}} | ||
| 280 | new_recon = recon_dict(new_dict) | ||
| 281 | old_recon = recon_dict(old_dict) | ||
| 282 | |||
| 283 | del new_dict | ||
| 284 | del old_dict | ||
| 285 | |||
| 286 | # Figure out what are changed, the new_recon would be changed | ||
| 287 | # by the print_xxx function. | ||
| 288 | # Newly added | ||
| 289 | cnt_added = print_added(new_recon, old_recon) | ||
| 290 | |||
| 291 | # PV (including PE) and PR changed | ||
| 292 | # Let the bb.siggen handle them if verbose | ||
| 293 | cnt_rv = {} | ||
| 294 | if not args.verbose: | ||
| 295 | for i in ('pv', 'pr'): | ||
| 296 | cnt_rv[i] = print_vrchanged(new_recon, old_recon, i) | ||
| 297 | |||
| 298 | # Dependencies changed (use bitbake-diffsigs) | ||
| 299 | cnt_dep = print_depchanged(new_recon, old_recon, args.verbose) | ||
| 300 | |||
| 301 | total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep | ||
| 302 | |||
| 303 | print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged)) | ||
| 304 | if args.verbose: | ||
| 305 | print("Newly added: %s\nDependencies changed: %s\n" % \ | ||
| 306 | (cnt_added, cnt_dep)) | ||
| 307 | else: | ||
| 308 | print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \ | ||
| 309 | (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep)) | ||
| 310 | except: | ||
| 311 | print("ERROR occurred!") | ||
| 312 | raise | ||
| 313 | finally: | ||
| 314 | # Remove the newly generated stamps dir | ||
| 315 | if os.path.exists(new_stampsdir): | ||
| 316 | print("Removing the newly generated stamps dir ...") | ||
| 317 | shutil.rmtree(new_stampsdir) | ||
| 318 | |||
| 319 | if __name__ == "__main__": | ||
| 320 | sys.exit(main()) | ||
