diff options
Diffstat (limited to 'scripts/contrib/patchreview.py')
-rwxr-xr-x | scripts/contrib/patchreview.py | 71 |
1 files changed, 55 insertions, 16 deletions
diff --git a/scripts/contrib/patchreview.py b/scripts/contrib/patchreview.py index 62c509f51c..bceae06561 100755 --- a/scripts/contrib/patchreview.py +++ b/scripts/contrib/patchreview.py | |||
@@ -1,14 +1,25 @@ | |||
1 | #! /usr/bin/env python3 | 1 | #! /usr/bin/env python3 |
2 | # | 2 | # |
3 | # Copyright OpenEmbedded Contributors | ||
4 | # | ||
3 | # SPDX-License-Identifier: GPL-2.0-only | 5 | # SPDX-License-Identifier: GPL-2.0-only |
4 | # | 6 | # |
5 | 7 | ||
8 | import argparse | ||
9 | import collections | ||
10 | import json | ||
11 | import os | ||
12 | import os.path | ||
13 | import pathlib | ||
14 | import re | ||
15 | import subprocess | ||
16 | |||
6 | # TODO | 17 | # TODO |
7 | # - option to just list all broken files | 18 | # - option to just list all broken files |
8 | # - test suite | 19 | # - test suite |
9 | # - validate signed-off-by | 20 | # - validate signed-off-by |
10 | 21 | ||
11 | status_values = ("accepted", "pending", "inappropriate", "backport", "submitted", "denied") | 22 | status_values = ("accepted", "pending", "inappropriate", "backport", "submitted", "denied", "inactive-upstream") |
12 | 23 | ||
13 | class Result: | 24 | class Result: |
14 | # Whether the patch has an Upstream-Status or not | 25 | # Whether the patch has an Upstream-Status or not |
@@ -33,20 +44,18 @@ def blame_patch(patch): | |||
33 | From a patch filename, return a list of "commit summary (author name <author | 44 | From a patch filename, return a list of "commit summary (author name <author |
34 | email>)" strings representing the history. | 45 | email>)" strings representing the history. |
35 | """ | 46 | """ |
36 | import subprocess | ||
37 | return subprocess.check_output(("git", "log", | 47 | return subprocess.check_output(("git", "log", |
38 | "--follow", "--find-renames", "--diff-filter=A", | 48 | "--follow", "--find-renames", "--diff-filter=A", |
39 | "--format=%s (%aN <%aE>)", | 49 | "--format=%s (%aN <%aE>)", |
40 | "--", patch)).decode("utf-8").splitlines() | 50 | "--", patch)).decode("utf-8").splitlines() |
41 | 51 | ||
42 | def patchreview(path, patches): | 52 | def patchreview(patches): |
43 | import re, os.path | ||
44 | 53 | ||
45 | # General pattern: start of line, optional whitespace, tag with optional | 54 | # General pattern: start of line, optional whitespace, tag with optional |
46 | # hyphen or spaces, maybe a colon, some whitespace, then the value, all case | 55 | # hyphen or spaces, maybe a colon, some whitespace, then the value, all case |
47 | # insensitive. | 56 | # insensitive. |
48 | sob_re = re.compile(r"^[\t ]*(Signed[-_ ]off[-_ ]by:?)[\t ]*(.+)", re.IGNORECASE | re.MULTILINE) | 57 | sob_re = re.compile(r"^[\t ]*(Signed[-_ ]off[-_ ]by:?)[\t ]*(.+)", re.IGNORECASE | re.MULTILINE) |
49 | status_re = re.compile(r"^[\t ]*(Upstream[-_ ]Status:?)[\t ]*(\w*)", re.IGNORECASE | re.MULTILINE) | 58 | status_re = re.compile(r"^[\t ]*(Upstream[-_ ]Status:?)[\t ]*([\w-]*)", re.IGNORECASE | re.MULTILINE) |
50 | cve_tag_re = re.compile(r"^[\t ]*(CVE:)[\t ]*(.*)", re.IGNORECASE | re.MULTILINE) | 59 | cve_tag_re = re.compile(r"^[\t ]*(CVE:)[\t ]*(.*)", re.IGNORECASE | re.MULTILINE) |
51 | cve_re = re.compile(r"cve-[0-9]{4}-[0-9]{4,6}", re.IGNORECASE) | 60 | cve_re = re.compile(r"cve-[0-9]{4}-[0-9]{4,6}", re.IGNORECASE) |
52 | 61 | ||
@@ -54,11 +63,10 @@ def patchreview(path, patches): | |||
54 | 63 | ||
55 | for patch in patches: | 64 | for patch in patches: |
56 | 65 | ||
57 | fullpath = os.path.join(path, patch) | ||
58 | result = Result() | 66 | result = Result() |
59 | results[fullpath] = result | 67 | results[patch] = result |
60 | 68 | ||
61 | content = open(fullpath, encoding='ascii', errors='ignore').read() | 69 | content = open(patch, encoding='ascii', errors='ignore').read() |
62 | 70 | ||
63 | # Find the Signed-off-by tag | 71 | # Find the Signed-off-by tag |
64 | match = sob_re.search(content) | 72 | match = sob_re.search(content) |
@@ -191,29 +199,56 @@ Patches in Pending state: %s""" % (total_patches, | |||
191 | def histogram(results): | 199 | def histogram(results): |
192 | from toolz import recipes, dicttoolz | 200 | from toolz import recipes, dicttoolz |
193 | import math | 201 | import math |
202 | |||
194 | counts = recipes.countby(lambda r: r.upstream_status, results.values()) | 203 | counts = recipes.countby(lambda r: r.upstream_status, results.values()) |
195 | bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts) | 204 | bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts) |
196 | for k in bars: | 205 | for k in bars: |
197 | print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k])) | 206 | print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k])) |
198 | 207 | ||
208 | def find_layers(candidate): | ||
209 | # candidate can either be the path to a layer directly (eg meta-intel), or a | ||
210 | # repository that contains other layers (meta-arm). We can determine what by | ||
211 | # looking for a conf/layer.conf file. If that file exists then it's a layer, | ||
212 | # otherwise its a repository of layers and we can assume they're called | ||
213 | # meta-*. | ||
214 | |||
215 | if (candidate / "conf" / "layer.conf").exists(): | ||
216 | return [candidate.absolute()] | ||
217 | else: | ||
218 | return [d.absolute() for d in candidate.iterdir() if d.is_dir() and (d.name == "meta" or d.name.startswith("meta-"))] | ||
219 | |||
220 | # TODO these don't actually handle dynamic-layers/ | ||
221 | |||
222 | def gather_patches(layers): | ||
223 | patches = [] | ||
224 | for directory in layers: | ||
225 | filenames = subprocess.check_output(("git", "-C", directory, "ls-files", "recipes-*/**/*.patch", "recipes-*/**/*.diff"), universal_newlines=True).split() | ||
226 | patches += [os.path.join(directory, f) for f in filenames] | ||
227 | return patches | ||
228 | |||
229 | def count_recipes(layers): | ||
230 | count = 0 | ||
231 | for directory in layers: | ||
232 | output = subprocess.check_output(["git", "-C", directory, "ls-files", "recipes-*/**/*.bb"], universal_newlines=True) | ||
233 | count += len(output.splitlines()) | ||
234 | return count | ||
199 | 235 | ||
200 | if __name__ == "__main__": | 236 | if __name__ == "__main__": |
201 | import argparse, subprocess, os | ||
202 | |||
203 | args = argparse.ArgumentParser(description="Patch Review Tool") | 237 | args = argparse.ArgumentParser(description="Patch Review Tool") |
204 | args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches") | 238 | args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches") |
205 | args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results") | 239 | args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results") |
206 | args.add_argument("-g", "--histogram", action="store_true", help="show patch histogram") | 240 | args.add_argument("-g", "--histogram", action="store_true", help="show patch histogram") |
207 | args.add_argument("-j", "--json", help="update JSON") | 241 | args.add_argument("-j", "--json", help="update JSON") |
208 | args.add_argument("directory", help="directory to scan") | 242 | args.add_argument("directory", type=pathlib.Path, metavar="DIRECTORY", help="directory to scan (layer, or repository of layers)") |
209 | args = args.parse_args() | 243 | args = args.parse_args() |
210 | 244 | ||
211 | patches = subprocess.check_output(("git", "-C", args.directory, "ls-files", "recipes-*/**/*.patch", "recipes-*/**/*.diff")).decode("utf-8").split() | 245 | layers = find_layers(args.directory) |
212 | results = patchreview(args.directory, patches) | 246 | print(f"Found layers {' '.join((d.name for d in layers))}") |
247 | patches = gather_patches(layers) | ||
248 | results = patchreview(patches) | ||
213 | analyse(results, want_blame=args.blame, verbose=args.verbose) | 249 | analyse(results, want_blame=args.blame, verbose=args.verbose) |
214 | 250 | ||
215 | if args.json: | 251 | if args.json: |
216 | import json, os.path, collections | ||
217 | if os.path.isfile(args.json): | 252 | if os.path.isfile(args.json): |
218 | data = json.load(open(args.json)) | 253 | data = json.load(open(args.json)) |
219 | else: | 254 | else: |
@@ -221,7 +256,11 @@ if __name__ == "__main__": | |||
221 | 256 | ||
222 | row = collections.Counter() | 257 | row = collections.Counter() |
223 | row["total"] = len(results) | 258 | row["total"] = len(results) |
224 | row["date"] = subprocess.check_output(["git", "-C", args.directory, "show", "-s", "--pretty=format:%cd", "--date=format:%s"]).decode("utf-8").strip() | 259 | row["date"] = subprocess.check_output(["git", "-C", args.directory, "show", "-s", "--pretty=format:%cd", "--date=format:%s"], universal_newlines=True).strip() |
260 | row["commit"] = subprocess.check_output(["git", "-C", args.directory, "rev-parse", "HEAD"], universal_newlines=True).strip() | ||
261 | row['commit_count'] = subprocess.check_output(["git", "-C", args.directory, "rev-list", "--count", "HEAD"], universal_newlines=True).strip() | ||
262 | row['recipe_count'] = count_recipes(layers) | ||
263 | |||
225 | for r in results.values(): | 264 | for r in results.values(): |
226 | if r.upstream_status in status_values: | 265 | if r.upstream_status in status_values: |
227 | row[r.upstream_status] += 1 | 266 | row[r.upstream_status] += 1 |
@@ -231,7 +270,7 @@ if __name__ == "__main__": | |||
231 | row['malformed-sob'] += 1 | 270 | row['malformed-sob'] += 1 |
232 | 271 | ||
233 | data.append(row) | 272 | data.append(row) |
234 | json.dump(data, open(args.json, "w")) | 273 | json.dump(data, open(args.json, "w"), sort_keys=True, indent="\t") |
235 | 274 | ||
236 | if args.histogram: | 275 | if args.histogram: |
237 | print() | 276 | print() |