diff options
| -rwxr-xr-x | scripts/bitbake-whatchanged | 318 |
1 files changed, 0 insertions, 318 deletions
diff --git a/scripts/bitbake-whatchanged b/scripts/bitbake-whatchanged deleted file mode 100755 index cdb730dbdb..0000000000 --- a/scripts/bitbake-whatchanged +++ /dev/null | |||
| @@ -1,318 +0,0 @@ | |||
| 1 | #!/usr/bin/env python3 | ||
| 2 | # | ||
| 3 | # Copyright (c) 2013 Wind River Systems, Inc. | ||
| 4 | # | ||
| 5 | # SPDX-License-Identifier: GPL-2.0-only | ||
| 6 | # | ||
| 7 | |||
| 8 | import os | ||
| 9 | import sys | ||
| 10 | import getopt | ||
| 11 | import shutil | ||
| 12 | import re | ||
| 13 | import warnings | ||
| 14 | import subprocess | ||
| 15 | import argparse | ||
| 16 | |||
| 17 | scripts_path = os.path.abspath(os.path.dirname(os.path.abspath(sys.argv[0]))) | ||
| 18 | lib_path = scripts_path + '/lib' | ||
| 19 | sys.path = sys.path + [lib_path] | ||
| 20 | |||
| 21 | import scriptpath | ||
| 22 | |||
| 23 | # Figure out where is the bitbake/lib/bb since we need bb.siggen and bb.process | ||
| 24 | bitbakepath = scriptpath.add_bitbake_lib_path() | ||
| 25 | if not bitbakepath: | ||
| 26 | sys.stderr.write("Unable to find bitbake by searching parent directory of this script or PATH\n") | ||
| 27 | sys.exit(1) | ||
| 28 | scriptpath.add_oe_lib_path() | ||
| 29 | import argparse_oe | ||
| 30 | |||
| 31 | import bb.siggen | ||
| 32 | import bb.process | ||
| 33 | |||
| 34 | # Match the stamp's filename | ||
| 35 | # group(1): PE_PV (may no PE) | ||
| 36 | # group(2): PR | ||
| 37 | # group(3): TASK | ||
| 38 | # group(4): HASH | ||
| 39 | stamp_re = re.compile("(?P<pv>.*)-(?P<pr>r\d+)\.(?P<task>do_\w+)\.(?P<hash>[^\.]*)") | ||
| 40 | sigdata_re = re.compile(".*\.sigdata\..*") | ||
| 41 | |||
| 42 | def gen_dict(stamps): | ||
| 43 | """ | ||
| 44 | Generate the dict from the stamps dir. | ||
| 45 | The output dict format is: | ||
| 46 | {fake_f: {pn: PN, pv: PV, pr: PR, task: TASK, path: PATH}} | ||
| 47 | Where: | ||
| 48 | fake_f: pv + task + hash | ||
| 49 | path: the path to the stamp file | ||
| 50 | """ | ||
| 51 | # The member of the sub dict (A "path" will be appended below) | ||
| 52 | sub_mem = ("pv", "pr", "task") | ||
| 53 | d = {} | ||
| 54 | for dirpath, _, files in os.walk(stamps): | ||
| 55 | for f in files: | ||
| 56 | # The "bitbake -S" would generate ".sigdata", but no "_setscene". | ||
| 57 | fake_f = re.sub('_setscene.', '.', f) | ||
| 58 | fake_f = re.sub('.sigdata', '', fake_f) | ||
| 59 | subdict = {} | ||
| 60 | tmp = stamp_re.match(fake_f) | ||
| 61 | if tmp: | ||
| 62 | for i in sub_mem: | ||
| 63 | subdict[i] = tmp.group(i) | ||
| 64 | if len(subdict) != 0: | ||
| 65 | pn = os.path.basename(dirpath) | ||
| 66 | subdict['pn'] = pn | ||
| 67 | # The path will be used by os.stat() and bb.siggen | ||
| 68 | subdict['path'] = dirpath + "/" + f | ||
| 69 | fake_f = tmp.group('pv') + tmp.group('task') + tmp.group('hash') | ||
| 70 | d[fake_f] = subdict | ||
| 71 | return d | ||
| 72 | |||
| 73 | # Re-construct the dict | ||
| 74 | def recon_dict(dict_in): | ||
| 75 | """ | ||
| 76 | The output dict format is: | ||
| 77 | {pn_task: {pv: PV, pr: PR, path: PATH}} | ||
| 78 | """ | ||
| 79 | dict_out = {} | ||
| 80 | for k in dict_in.keys(): | ||
| 81 | subdict = {} | ||
| 82 | # The key | ||
| 83 | pn_task = "%s_%s" % (dict_in.get(k).get('pn'), dict_in.get(k).get('task')) | ||
| 84 | # If more than one stamps are found, use the latest one. | ||
| 85 | if pn_task in dict_out: | ||
| 86 | full_path_pre = dict_out.get(pn_task).get('path') | ||
| 87 | full_path_cur = dict_in.get(k).get('path') | ||
| 88 | if os.stat(full_path_pre).st_mtime > os.stat(full_path_cur).st_mtime: | ||
| 89 | continue | ||
| 90 | subdict['pv'] = dict_in.get(k).get('pv') | ||
| 91 | subdict['pr'] = dict_in.get(k).get('pr') | ||
| 92 | subdict['path'] = dict_in.get(k).get('path') | ||
| 93 | dict_out[pn_task] = subdict | ||
| 94 | |||
| 95 | return dict_out | ||
| 96 | |||
| 97 | def split_pntask(s): | ||
| 98 | """ | ||
| 99 | Split the pn_task in to (pn, task) and return it | ||
| 100 | """ | ||
| 101 | tmp = re.match("(.*)_(do_.*)", s) | ||
| 102 | return (tmp.group(1), tmp.group(2)) | ||
| 103 | |||
| 104 | |||
| 105 | def print_added(d_new = None, d_old = None): | ||
| 106 | """ | ||
| 107 | Print the newly added tasks | ||
| 108 | """ | ||
| 109 | added = {} | ||
| 110 | for k in list(d_new.keys()): | ||
| 111 | if k not in d_old: | ||
| 112 | # Add the new one to added dict, and remove it from | ||
| 113 | # d_new, so the remaining ones are the changed ones | ||
| 114 | added[k] = d_new.get(k) | ||
| 115 | del(d_new[k]) | ||
| 116 | |||
| 117 | if not added: | ||
| 118 | return 0 | ||
| 119 | |||
| 120 | # Format the output, the dict format is: | ||
| 121 | # {pn: task1, task2 ...} | ||
| 122 | added_format = {} | ||
| 123 | counter = 0 | ||
| 124 | for k in added.keys(): | ||
| 125 | pn, task = split_pntask(k) | ||
| 126 | if pn in added_format: | ||
| 127 | # Append the value | ||
| 128 | added_format[pn] = "%s %s" % (added_format.get(pn), task) | ||
| 129 | else: | ||
| 130 | added_format[pn] = task | ||
| 131 | counter += 1 | ||
| 132 | print("=== Newly added tasks: (%s tasks)" % counter) | ||
| 133 | for k in added_format.keys(): | ||
| 134 | print(" %s: %s" % (k, added_format.get(k))) | ||
| 135 | |||
| 136 | return counter | ||
| 137 | |||
| 138 | def print_vrchanged(d_new = None, d_old = None, vr = None): | ||
| 139 | """ | ||
| 140 | Print the pv or pr changed tasks. | ||
| 141 | The arg "vr" is "pv" or "pr" | ||
| 142 | """ | ||
| 143 | pvchanged = {} | ||
| 144 | counter = 0 | ||
| 145 | for k in list(d_new.keys()): | ||
| 146 | if d_new.get(k).get(vr) != d_old.get(k).get(vr): | ||
| 147 | counter += 1 | ||
| 148 | pn, task = split_pntask(k) | ||
| 149 | if pn not in pvchanged: | ||
| 150 | # Format the output, we only print pn (no task) since | ||
| 151 | # all the tasks would be changed when pn or pr changed, | ||
| 152 | # the dict format is: | ||
| 153 | # {pn: pv/pr_old -> pv/pr_new} | ||
| 154 | pvchanged[pn] = "%s -> %s" % (d_old.get(k).get(vr), d_new.get(k).get(vr)) | ||
| 155 | del(d_new[k]) | ||
| 156 | |||
| 157 | if not pvchanged: | ||
| 158 | return 0 | ||
| 159 | |||
| 160 | print("\n=== %s changed: (%s tasks)" % (vr.upper(), counter)) | ||
| 161 | for k in pvchanged.keys(): | ||
| 162 | print(" %s: %s" % (k, pvchanged.get(k))) | ||
| 163 | |||
| 164 | return counter | ||
| 165 | |||
| 166 | def print_depchanged(d_new = None, d_old = None, verbose = False): | ||
| 167 | """ | ||
| 168 | Print the dependency changes | ||
| 169 | """ | ||
| 170 | depchanged = {} | ||
| 171 | counter = 0 | ||
| 172 | for k in d_new.keys(): | ||
| 173 | counter += 1 | ||
| 174 | pn, task = split_pntask(k) | ||
| 175 | if (verbose): | ||
| 176 | full_path_old = d_old.get(k).get("path") | ||
| 177 | full_path_new = d_new.get(k).get("path") | ||
| 178 | # No counter since it is not ready here | ||
| 179 | if sigdata_re.match(full_path_old) and sigdata_re.match(full_path_new): | ||
| 180 | output = bb.siggen.compare_sigfiles(full_path_old, full_path_new) | ||
| 181 | if output: | ||
| 182 | print("\n=== The verbose changes of %s.%s:" % (pn, task)) | ||
| 183 | print('\n'.join(output)) | ||
| 184 | else: | ||
| 185 | # Format the output, the format is: | ||
| 186 | # {pn: task1, task2, ...} | ||
| 187 | if pn in depchanged: | ||
| 188 | depchanged[pn] = "%s %s" % (depchanged.get(pn), task) | ||
| 189 | else: | ||
| 190 | depchanged[pn] = task | ||
| 191 | |||
| 192 | if len(depchanged) > 0: | ||
| 193 | print("\n=== Dependencies changed: (%s tasks)" % counter) | ||
| 194 | for k in depchanged.keys(): | ||
| 195 | print(" %s: %s" % (k, depchanged[k])) | ||
| 196 | |||
| 197 | return counter | ||
| 198 | |||
| 199 | |||
| 200 | def main(): | ||
| 201 | """ | ||
| 202 | Print what will be done between the current and last builds: | ||
| 203 | 1) Run "STAMPS_DIR=<path> bitbake -S recipe" to re-generate the stamps | ||
| 204 | 2) Figure out what are newly added and changed, can't figure out | ||
| 205 | what are removed since we can't know the previous stamps | ||
| 206 | clearly, for example, if there are several builds, we can't know | ||
| 207 | which stamps the last build has used exactly. | ||
| 208 | 3) Use bb.siggen.compare_sigfiles to diff the old and new stamps | ||
| 209 | """ | ||
| 210 | |||
| 211 | parser = argparse_oe.ArgumentParser(usage = """%(prog)s [options] [package ...] | ||
| 212 | print what will be done between the current and last builds, for example: | ||
| 213 | |||
| 214 | $ bitbake core-image-sato | ||
| 215 | # Edit the recipes | ||
| 216 | $ bitbake-whatchanged core-image-sato | ||
| 217 | |||
| 218 | The changes will be printed. | ||
| 219 | |||
| 220 | Note: | ||
| 221 | The amount of tasks is not accurate when the task is "do_build" since | ||
| 222 | it usually depends on other tasks. | ||
| 223 | The "nostamp" task is not included. | ||
| 224 | """ | ||
| 225 | ) | ||
| 226 | parser.add_argument("recipe", help="recipe to check") | ||
| 227 | parser.add_argument("-v", "--verbose", help = "print the verbose changes", action = "store_true") | ||
| 228 | args = parser.parse_args() | ||
| 229 | |||
| 230 | # Get the STAMPS_DIR | ||
| 231 | print("Figuring out the STAMPS_DIR ...") | ||
| 232 | cmdline = "bitbake -e | sed -ne 's/^STAMPS_DIR=\"\(.*\)\"/\\1/p'" | ||
| 233 | try: | ||
| 234 | stampsdir, err = bb.process.run(cmdline) | ||
| 235 | except: | ||
| 236 | raise | ||
| 237 | if not stampsdir: | ||
| 238 | print("ERROR: No STAMPS_DIR found for '%s'" % args.recipe, file=sys.stderr) | ||
| 239 | return 2 | ||
| 240 | stampsdir = stampsdir.rstrip("\n") | ||
| 241 | if not os.path.isdir(stampsdir): | ||
| 242 | print("ERROR: stamps directory \"%s\" not found!" % stampsdir, file=sys.stderr) | ||
| 243 | return 2 | ||
| 244 | |||
| 245 | # The new stamps dir | ||
| 246 | new_stampsdir = stampsdir + ".bbs" | ||
| 247 | if os.path.exists(new_stampsdir): | ||
| 248 | print("ERROR: %s already exists!" % new_stampsdir, file=sys.stderr) | ||
| 249 | return 2 | ||
| 250 | |||
| 251 | try: | ||
| 252 | # Generate the new stamps dir | ||
| 253 | print("Generating the new stamps ... (need several minutes)") | ||
| 254 | cmdline = "STAMPS_DIR=%s bitbake -S none %s" % (new_stampsdir, args.recipe) | ||
| 255 | # FIXME | ||
| 256 | # The "bitbake -S" may fail, not fatal error, the stamps will still | ||
| 257 | # be generated, this might be a bug of "bitbake -S". | ||
| 258 | try: | ||
| 259 | bb.process.run(cmdline) | ||
| 260 | except Exception as exc: | ||
| 261 | print(exc) | ||
| 262 | |||
| 263 | # The dict for the new and old stamps. | ||
| 264 | old_dict = gen_dict(stampsdir) | ||
| 265 | new_dict = gen_dict(new_stampsdir) | ||
| 266 | |||
| 267 | # Remove the same one from both stamps. | ||
| 268 | cnt_unchanged = 0 | ||
| 269 | for k in list(new_dict.keys()): | ||
| 270 | if k in old_dict: | ||
| 271 | cnt_unchanged += 1 | ||
| 272 | del(new_dict[k]) | ||
| 273 | del(old_dict[k]) | ||
| 274 | |||
| 275 | # Re-construct the dict to easily find out what is added or changed. | ||
| 276 | # The dict format is: | ||
| 277 | # {pn_task: {pv: PV, pr: PR, path: PATH}} | ||
| 278 | new_recon = recon_dict(new_dict) | ||
| 279 | old_recon = recon_dict(old_dict) | ||
| 280 | |||
| 281 | del new_dict | ||
| 282 | del old_dict | ||
| 283 | |||
| 284 | # Figure out what are changed, the new_recon would be changed | ||
| 285 | # by the print_xxx function. | ||
| 286 | # Newly added | ||
| 287 | cnt_added = print_added(new_recon, old_recon) | ||
| 288 | |||
| 289 | # PV (including PE) and PR changed | ||
| 290 | # Let the bb.siggen handle them if verbose | ||
| 291 | cnt_rv = {} | ||
| 292 | if not args.verbose: | ||
| 293 | for i in ('pv', 'pr'): | ||
| 294 | cnt_rv[i] = print_vrchanged(new_recon, old_recon, i) | ||
| 295 | |||
| 296 | # Dependencies changed (use bitbake-diffsigs) | ||
| 297 | cnt_dep = print_depchanged(new_recon, old_recon, args.verbose) | ||
| 298 | |||
| 299 | total_changed = cnt_added + (cnt_rv.get('pv') or 0) + (cnt_rv.get('pr') or 0) + cnt_dep | ||
| 300 | |||
| 301 | print("\n=== Summary: (%s changed, %s unchanged)" % (total_changed, cnt_unchanged)) | ||
| 302 | if args.verbose: | ||
| 303 | print("Newly added: %s\nDependencies changed: %s\n" % \ | ||
| 304 | (cnt_added, cnt_dep)) | ||
| 305 | else: | ||
| 306 | print("Newly added: %s\nPV changed: %s\nPR changed: %s\nDependencies changed: %s\n" % \ | ||
| 307 | (cnt_added, cnt_rv.get('pv') or 0, cnt_rv.get('pr') or 0, cnt_dep)) | ||
| 308 | except: | ||
| 309 | print("ERROR occurred!") | ||
| 310 | raise | ||
| 311 | finally: | ||
| 312 | # Remove the newly generated stamps dir | ||
| 313 | if os.path.exists(new_stampsdir): | ||
| 314 | print("Removing the newly generated stamps dir ...") | ||
| 315 | shutil.rmtree(new_stampsdir) | ||
| 316 | |||
| 317 | if __name__ == "__main__": | ||
| 318 | sys.exit(main()) | ||
