diff options
| author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2025-11-07 13:31:53 +0000 |
|---|---|---|
| committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2025-11-07 13:31:53 +0000 |
| commit | 8c22ff0d8b70d9b12f0487ef696a7e915b9e3173 (patch) | |
| tree | efdc32587159d0050a69009bdf2330a531727d95 /scripts/contrib/patchreview.py | |
| parent | d412d2747595c1cc4a5e3ca975e3adc31b2f7891 (diff) | |
| download | poky-8c22ff0d8b70d9b12f0487ef696a7e915b9e3173.tar.gz | |
The poky repository master branch is no longer being updated.
You can either:
a) switch to individual clones of bitbake, openembedded-core, meta-yocto and yocto-docs
b) use the new bitbake-setup
You can find information about either approach in our documentation:
https://docs.yoctoproject.org/
Note that "poky" the distro setting is still available in meta-yocto as
before and we continue to use and maintain that.
Long live Poky!
Some further information on the background of this change can be found
in: https://lists.openembedded.org/g/openembedded-architecture/message/2179
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/contrib/patchreview.py')
| -rwxr-xr-x | scripts/contrib/patchreview.py | 280 |
1 files changed, 0 insertions, 280 deletions
diff --git a/scripts/contrib/patchreview.py b/scripts/contrib/patchreview.py deleted file mode 100755 index d8d7b214e5..0000000000 --- a/scripts/contrib/patchreview.py +++ /dev/null | |||
| @@ -1,280 +0,0 @@ | |||
| 1 | #! /usr/bin/env python3 | ||
| 2 | # | ||
| 3 | # Copyright OpenEmbedded Contributors | ||
| 4 | # | ||
| 5 | # SPDX-License-Identifier: GPL-2.0-only | ||
| 6 | # | ||
| 7 | |||
| 8 | import argparse | ||
| 9 | import collections | ||
| 10 | import json | ||
| 11 | import os | ||
| 12 | import os.path | ||
| 13 | import pathlib | ||
| 14 | import re | ||
| 15 | import subprocess | ||
| 16 | |||
| 17 | import sys | ||
| 18 | sys.path.append(os.path.join(sys.path[0], '../../meta/lib')) | ||
| 19 | import oe.qa | ||
| 20 | |||
| 21 | # TODO | ||
| 22 | # - option to just list all broken files | ||
| 23 | # - test suite | ||
| 24 | # - validate signed-off-by | ||
| 25 | |||
| 26 | status_values = ("accepted", "pending", "inappropriate", "backport", "submitted", "denied", "inactive-upstream") | ||
| 27 | |||
| 28 | class Result: | ||
| 29 | # Whether the patch has an Upstream-Status or not | ||
| 30 | missing_upstream_status = False | ||
| 31 | # If the Upstream-Status tag is malformed in some way (string for bad bit) | ||
| 32 | malformed_upstream_status = None | ||
| 33 | # If the Upstream-Status value is unknown (boolean) | ||
| 34 | unknown_upstream_status = False | ||
| 35 | # The upstream status value (Pending, etc) | ||
| 36 | upstream_status = None | ||
| 37 | # Whether the patch has a Signed-off-by or not | ||
| 38 | missing_sob = False | ||
| 39 | # Whether the Signed-off-by tag is malformed in some way | ||
| 40 | malformed_sob = False | ||
| 41 | # The Signed-off-by tag value | ||
| 42 | sob = None | ||
| 43 | # Whether a patch looks like a CVE but doesn't have a CVE tag | ||
| 44 | missing_cve = False | ||
| 45 | |||
| 46 | def blame_patch(patch): | ||
| 47 | """ | ||
| 48 | From a patch filename, return a list of "commit summary (author name <author | ||
| 49 | email>)" strings representing the history. | ||
| 50 | """ | ||
| 51 | return subprocess.check_output(("git", "log", | ||
| 52 | "--follow", "--find-renames", "--diff-filter=A", | ||
| 53 | "--format=%s (%aN <%aE>)", | ||
| 54 | "--", patch), cwd=os.path.dirname(patch)).decode("utf-8").splitlines() | ||
| 55 | |||
| 56 | def patchreview(patches): | ||
| 57 | |||
| 58 | # General pattern: start of line, optional whitespace, tag with optional | ||
| 59 | # hyphen or spaces, maybe a colon, some whitespace, then the value, all case | ||
| 60 | # insensitive. | ||
| 61 | sob_re = re.compile(r"^[\t ]*(Signed[-_ ]off[-_ ]by:?)[\t ]*(.+)", re.IGNORECASE | re.MULTILINE) | ||
| 62 | status_re = re.compile(r"^[\t ]*(Upstream[-_ ]Status:?)[\t ]*([\w-]*)", re.IGNORECASE | re.MULTILINE) | ||
| 63 | cve_tag_re = re.compile(r"^[\t ]*(CVE:)[\t ]*(.*)", re.IGNORECASE | re.MULTILINE) | ||
| 64 | cve_re = re.compile(r"cve-[0-9]{4}-[0-9]{4,6}", re.IGNORECASE) | ||
| 65 | |||
| 66 | results = {} | ||
| 67 | |||
| 68 | for patch in patches: | ||
| 69 | |||
| 70 | result = Result() | ||
| 71 | results[patch] = result | ||
| 72 | |||
| 73 | content = open(patch, encoding='ascii', errors='ignore').read() | ||
| 74 | |||
| 75 | # Find the Signed-off-by tag | ||
| 76 | match = sob_re.search(content) | ||
| 77 | if match: | ||
| 78 | value = match.group(1) | ||
| 79 | if value != "Signed-off-by:": | ||
| 80 | result.malformed_sob = value | ||
| 81 | result.sob = match.group(2) | ||
| 82 | else: | ||
| 83 | result.missing_sob = True | ||
| 84 | |||
| 85 | # Find the Upstream-Status tag | ||
| 86 | match = status_re.search(content) | ||
| 87 | if match: | ||
| 88 | value = oe.qa.check_upstream_status(patch) | ||
| 89 | if value: | ||
| 90 | result.malformed_upstream_status = value | ||
| 91 | |||
| 92 | value = match.group(2).lower() | ||
| 93 | # TODO: check case | ||
| 94 | if value not in status_values: | ||
| 95 | result.unknown_upstream_status = True | ||
| 96 | result.upstream_status = value | ||
| 97 | else: | ||
| 98 | result.missing_upstream_status = True | ||
| 99 | |||
| 100 | # Check that patches which looks like CVEs have CVE tags | ||
| 101 | if cve_re.search(patch) or cve_re.search(content): | ||
| 102 | if not cve_tag_re.search(content): | ||
| 103 | result.missing_cve = True | ||
| 104 | # TODO: extract CVE list | ||
| 105 | |||
| 106 | return results | ||
| 107 | |||
| 108 | |||
| 109 | def analyse(results, want_blame=False, verbose=True): | ||
| 110 | """ | ||
| 111 | want_blame: display blame data for each malformed patch | ||
| 112 | verbose: display per-file results instead of just summary | ||
| 113 | """ | ||
| 114 | |||
| 115 | # want_blame requires verbose, so disable blame if we're not verbose | ||
| 116 | if want_blame and not verbose: | ||
| 117 | want_blame = False | ||
| 118 | |||
| 119 | total_patches = 0 | ||
| 120 | missing_sob = 0 | ||
| 121 | malformed_sob = 0 | ||
| 122 | missing_status = 0 | ||
| 123 | malformed_status = 0 | ||
| 124 | missing_cve = 0 | ||
| 125 | pending_patches = 0 | ||
| 126 | |||
| 127 | for patch in sorted(results): | ||
| 128 | r = results[patch] | ||
| 129 | total_patches += 1 | ||
| 130 | need_blame = False | ||
| 131 | |||
| 132 | # Build statistics | ||
| 133 | if r.missing_sob: | ||
| 134 | missing_sob += 1 | ||
| 135 | if r.malformed_sob: | ||
| 136 | malformed_sob += 1 | ||
| 137 | if r.missing_upstream_status: | ||
| 138 | missing_status += 1 | ||
| 139 | if r.malformed_upstream_status or r.unknown_upstream_status: | ||
| 140 | malformed_status += 1 | ||
| 141 | # Count patches with no status as pending | ||
| 142 | pending_patches +=1 | ||
| 143 | if r.missing_cve: | ||
| 144 | missing_cve += 1 | ||
| 145 | if r.upstream_status == "pending": | ||
| 146 | pending_patches += 1 | ||
| 147 | |||
| 148 | # Output warnings | ||
| 149 | if r.missing_sob: | ||
| 150 | need_blame = True | ||
| 151 | if verbose: | ||
| 152 | print("Missing Signed-off-by tag (%s)" % patch) | ||
| 153 | if r.malformed_sob: | ||
| 154 | need_blame = True | ||
| 155 | if verbose: | ||
| 156 | print("Malformed Signed-off-by '%s' (%s)" % (r.malformed_sob, patch)) | ||
| 157 | if r.missing_cve: | ||
| 158 | need_blame = True | ||
| 159 | if verbose: | ||
| 160 | print("Missing CVE tag (%s)" % patch) | ||
| 161 | if r.missing_upstream_status: | ||
| 162 | need_blame = True | ||
| 163 | if verbose: | ||
| 164 | print("Missing Upstream-Status tag (%s)" % patch) | ||
| 165 | if r.malformed_upstream_status: | ||
| 166 | need_blame = True | ||
| 167 | if verbose: | ||
| 168 | print("Malformed Upstream-Status '%s' (%s)" % (r.malformed_upstream_status, patch)) | ||
| 169 | if r.unknown_upstream_status: | ||
| 170 | need_blame = True | ||
| 171 | if verbose: | ||
| 172 | print("Unknown Upstream-Status value '%s' (%s)" % (r.upstream_status, patch)) | ||
| 173 | |||
| 174 | if want_blame and need_blame: | ||
| 175 | print("\n".join(blame_patch(patch)) + "\n") | ||
| 176 | |||
| 177 | def percent(num): | ||
| 178 | try: | ||
| 179 | return "%d (%d%%)" % (num, round(num * 100.0 / total_patches)) | ||
| 180 | except ZeroDivisionError: | ||
| 181 | return "N/A" | ||
| 182 | |||
| 183 | if verbose: | ||
| 184 | print() | ||
| 185 | |||
| 186 | print("""Total patches found: %d | ||
| 187 | Patches missing Signed-off-by: %s | ||
| 188 | Patches with malformed Signed-off-by: %s | ||
| 189 | Patches missing CVE: %s | ||
| 190 | Patches missing Upstream-Status: %s | ||
| 191 | Patches with malformed Upstream-Status: %s | ||
| 192 | Patches in Pending state: %s""" % (total_patches, | ||
| 193 | percent(missing_sob), | ||
| 194 | percent(malformed_sob), | ||
| 195 | percent(missing_cve), | ||
| 196 | percent(missing_status), | ||
| 197 | percent(malformed_status), | ||
| 198 | percent(pending_patches))) | ||
| 199 | |||
| 200 | |||
| 201 | |||
| 202 | def histogram(results): | ||
| 203 | from toolz import recipes, dicttoolz | ||
| 204 | import math | ||
| 205 | |||
| 206 | counts = recipes.countby(lambda r: r.upstream_status, results.values()) | ||
| 207 | bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts) | ||
| 208 | for k in bars: | ||
| 209 | print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k])) | ||
| 210 | |||
| 211 | def find_layers(candidate): | ||
| 212 | # candidate can either be the path to a layer directly (eg meta-intel), or a | ||
| 213 | # repository that contains other layers (meta-arm). We can determine what by | ||
| 214 | # looking for a conf/layer.conf file. If that file exists then it's a layer, | ||
| 215 | # otherwise its a repository of layers and we can assume they're called | ||
| 216 | # meta-*. | ||
| 217 | |||
| 218 | if (candidate / "conf" / "layer.conf").exists(): | ||
| 219 | return [candidate.absolute()] | ||
| 220 | else: | ||
| 221 | return [d.absolute() for d in candidate.iterdir() if d.is_dir() and (d.name == "meta" or d.name.startswith("meta-"))] | ||
| 222 | |||
| 223 | # TODO these don't actually handle dynamic-layers/ | ||
| 224 | |||
| 225 | def gather_patches(layers): | ||
| 226 | patches = [] | ||
| 227 | for directory in layers: | ||
| 228 | filenames = subprocess.check_output(("git", "-C", directory, "ls-files", "recipes-*/**/*.patch", "recipes-*/**/*.diff"), universal_newlines=True).split() | ||
| 229 | patches += [os.path.join(directory, f) for f in filenames] | ||
| 230 | return patches | ||
| 231 | |||
| 232 | def count_recipes(layers): | ||
| 233 | count = 0 | ||
| 234 | for directory in layers: | ||
| 235 | output = subprocess.check_output(["git", "-C", directory, "ls-files", "recipes-*/**/*.bb"], universal_newlines=True) | ||
| 236 | count += len(output.splitlines()) | ||
| 237 | return count | ||
| 238 | |||
| 239 | if __name__ == "__main__": | ||
| 240 | args = argparse.ArgumentParser(description="Patch Review Tool") | ||
| 241 | args.add_argument("-b", "--blame", action="store_true", help="show blame for malformed patches") | ||
| 242 | args.add_argument("-v", "--verbose", action="store_true", help="show per-patch results") | ||
| 243 | args.add_argument("-g", "--histogram", action="store_true", help="show patch histogram") | ||
| 244 | args.add_argument("-j", "--json", help="update JSON") | ||
| 245 | args.add_argument("directory", type=pathlib.Path, metavar="DIRECTORY", help="directory to scan (layer, or repository of layers)") | ||
| 246 | args = args.parse_args() | ||
| 247 | |||
| 248 | layers = find_layers(args.directory) | ||
| 249 | print(f"Found layers {' '.join((d.name for d in layers))}") | ||
| 250 | patches = gather_patches(layers) | ||
| 251 | results = patchreview(patches) | ||
| 252 | analyse(results, want_blame=args.blame, verbose=args.verbose) | ||
| 253 | |||
| 254 | if args.json: | ||
| 255 | if os.path.isfile(args.json): | ||
| 256 | data = json.load(open(args.json)) | ||
| 257 | else: | ||
| 258 | data = [] | ||
| 259 | |||
| 260 | row = collections.Counter() | ||
| 261 | row["total"] = len(results) | ||
| 262 | row["date"] = subprocess.check_output(["git", "-C", args.directory, "show", "-s", "--pretty=format:%cd", "--date=format:%s"], universal_newlines=True).strip() | ||
| 263 | row["commit"] = subprocess.check_output(["git", "-C", args.directory, "rev-parse", "HEAD"], universal_newlines=True).strip() | ||
| 264 | row['commit_count'] = subprocess.check_output(["git", "-C", args.directory, "rev-list", "--count", "HEAD"], universal_newlines=True).strip() | ||
| 265 | row['recipe_count'] = count_recipes(layers) | ||
| 266 | |||
| 267 | for r in results.values(): | ||
| 268 | if r.upstream_status in status_values: | ||
| 269 | row[r.upstream_status] += 1 | ||
| 270 | if r.malformed_upstream_status or r.missing_upstream_status: | ||
| 271 | row['malformed-upstream-status'] += 1 | ||
| 272 | if r.malformed_sob or r.missing_sob: | ||
| 273 | row['malformed-sob'] += 1 | ||
| 274 | |||
| 275 | data.append(row) | ||
| 276 | json.dump(data, open(args.json, "w"), sort_keys=True, indent="\t") | ||
| 277 | |||
| 278 | if args.histogram: | ||
| 279 | print() | ||
| 280 | histogram(results) | ||
