diff options
author | Marta Rybczynska <rybczynska@gmail.com> | 2022-04-22 16:17:50 +0200 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-05-03 17:50:06 +0100 |
commit | dcd40cfa375c272eda1ccc3063a48c5ec0a50ab5 (patch) | |
tree | 51411d9ff97fc5923fb1652ac0a41d4ce9906b01 | |
parent | 5b0093ecee4b249da588524ec13c5a86029fe1c1 (diff) | |
download | poky-dcd40cfa375c272eda1ccc3063a48c5ec0a50ab5.tar.gz |
cve-check: add json format
Backport to dunfell from master df567de36ae5964bee433ebb97e8bf702034994a
Add an option to output the CVE check in a JSON-based format.
This format is easier to parse in software than the original
text-based one and allows post-processing by other tools.
Output formats are now handed by CVE_CHECK_FORMAT_TEXT and
CVE_CHECK_FORMAT_JSON. The text format is enabled by default
to maintain compatibility, while the JSON format is disabled
by default.
The JSON output format gets generated in a similar way to the
text format with the exception of the manifest: appending to
JSON arrays requires parsing the file. Because of that we
first write JSON fragments and then assemble them in one pass
at the end.
(From OE-Core rev: 92b6011ab25fd36e2f8900a4db6883cdebc3cd3d)
Signed-off-by: Marta Rybczynska <marta.rybczynska@huawei.com>
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | meta/classes/cve-check.bbclass | 144 | ||||
-rw-r--r-- | meta/lib/oe/cve_check.py | 16 |
2 files changed, 159 insertions, 1 deletions
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass index 75c5b92b96..a7156cbdfb 100644 --- a/meta/classes/cve-check.bbclass +++ b/meta/classes/cve-check.bbclass | |||
@@ -34,15 +34,27 @@ CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check" | |||
34 | CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve" | 34 | CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve" |
35 | CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary" | 35 | CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary" |
36 | CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}" | 36 | CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}" |
37 | CVE_CHECK_SUMMARY_FILE_NAME_JSON = "cve-summary.json" | ||
38 | CVE_CHECK_SUMMARY_INDEX_PATH = "${CVE_CHECK_SUMMARY_DIR}/cve-summary-index.txt" | ||
39 | |||
40 | CVE_CHECK_LOG_JSON ?= "${T}/cve.json" | ||
37 | 41 | ||
38 | CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" | 42 | CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" |
39 | CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}" | 43 | CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}" |
44 | CVE_CHECK_RECIPE_FILE_JSON ?= "${CVE_CHECK_DIR}/${PN}_cve.json" | ||
40 | CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" | 45 | CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" |
46 | CVE_CHECK_MANIFEST_JSON ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.json" | ||
41 | CVE_CHECK_COPY_FILES ??= "1" | 47 | CVE_CHECK_COPY_FILES ??= "1" |
42 | CVE_CHECK_CREATE_MANIFEST ??= "1" | 48 | CVE_CHECK_CREATE_MANIFEST ??= "1" |
43 | 49 | ||
44 | CVE_CHECK_REPORT_PATCHED ??= "1" | 50 | CVE_CHECK_REPORT_PATCHED ??= "1" |
45 | 51 | ||
52 | # Provide text output | ||
53 | CVE_CHECK_FORMAT_TEXT ??= "1" | ||
54 | |||
55 | # Provide JSON output - disabled by default for backward compatibility | ||
56 | CVE_CHECK_FORMAT_JSON ??= "0" | ||
57 | |||
46 | # Whitelist for packages (PN) | 58 | # Whitelist for packages (PN) |
47 | CVE_CHECK_PN_WHITELIST ?= "" | 59 | CVE_CHECK_PN_WHITELIST ?= "" |
48 | 60 | ||
@@ -118,6 +130,7 @@ python cve_check_cleanup () { | |||
118 | Delete the file used to gather all the CVE information. | 130 | Delete the file used to gather all the CVE information. |
119 | """ | 131 | """ |
120 | bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE")) | 132 | bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE")) |
133 | bb.utils.remove(e.data.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")) | ||
121 | } | 134 | } |
122 | 135 | ||
123 | addhandler cve_check_cleanup | 136 | addhandler cve_check_cleanup |
@@ -129,11 +142,15 @@ python cve_check_write_rootfs_manifest () { | |||
129 | """ | 142 | """ |
130 | 143 | ||
131 | import shutil | 144 | import shutil |
145 | from oe.cve_check import cve_check_merge_jsons | ||
132 | 146 | ||
133 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": | 147 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": |
134 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE") | 148 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE") |
135 | if os.path.exists(deploy_file): | 149 | if os.path.exists(deploy_file): |
136 | bb.utils.remove(deploy_file) | 150 | bb.utils.remove(deploy_file) |
151 | deploy_file_json = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") | ||
152 | if os.path.exists(deploy_file_json): | ||
153 | bb.utils.remove(deploy_file_json) | ||
137 | 154 | ||
138 | if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")): | 155 | if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")): |
139 | bb.note("Writing rootfs CVE manifest") | 156 | bb.note("Writing rootfs CVE manifest") |
@@ -152,6 +169,26 @@ python cve_check_write_rootfs_manifest () { | |||
152 | os.remove(manifest_link) | 169 | os.remove(manifest_link) |
153 | os.symlink(os.path.basename(manifest_name), manifest_link) | 170 | os.symlink(os.path.basename(manifest_name), manifest_link) |
154 | bb.plain("Image CVE report stored in: %s" % manifest_name) | 171 | bb.plain("Image CVE report stored in: %s" % manifest_name) |
172 | |||
173 | if os.path.exists(d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")): | ||
174 | import json | ||
175 | bb.note("Generating JSON CVE manifest") | ||
176 | deploy_dir = d.getVar("DEPLOY_DIR_IMAGE") | ||
177 | link_name = d.getVar("IMAGE_LINK_NAME") | ||
178 | manifest_name = d.getVar("CVE_CHECK_MANIFEST_JSON") | ||
179 | index_file = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") | ||
180 | manifest = {"version":"1", "package": []} | ||
181 | with open(index_file) as f: | ||
182 | filename = f.readline() | ||
183 | while filename: | ||
184 | with open(filename.rstrip()) as j: | ||
185 | data = json.load(j) | ||
186 | cve_check_merge_jsons(manifest, data) | ||
187 | filename = f.readline() | ||
188 | |||
189 | with open(manifest_name, "w") as f: | ||
190 | json.dump(manifest, f, indent=2) | ||
191 | bb.plain("Image CVE report stored in: %s" % manifest_name) | ||
155 | } | 192 | } |
156 | 193 | ||
157 | ROOTFS_POSTPROCESS_COMMAND_prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" | 194 | ROOTFS_POSTPROCESS_COMMAND_prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" |
@@ -337,7 +374,7 @@ def get_cve_info(d, cves): | |||
337 | conn.close() | 374 | conn.close() |
338 | return cve_data | 375 | return cve_data |
339 | 376 | ||
340 | def cve_write_data(d, patched, unpatched, whitelisted, cve_data): | 377 | def cve_write_data_text(d, patched, unpatched, whitelisted, cve_data): |
341 | """ | 378 | """ |
342 | Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and | 379 | Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and |
343 | CVE manifest if enabled. | 380 | CVE manifest if enabled. |
@@ -403,3 +440,108 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data): | |||
403 | 440 | ||
404 | with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f: | 441 | with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f: |
405 | f.write("%s" % write_string) | 442 | f.write("%s" % write_string) |
443 | |||
444 | def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file): | ||
445 | """ | ||
446 | Write CVE information in the JSON format: to WORKDIR; and to | ||
447 | CVE_CHECK_DIR, if CVE manifest if enabled, write fragment | ||
448 | files that will be assembled at the end in cve_check_write_rootfs_manifest. | ||
449 | """ | ||
450 | |||
451 | import json | ||
452 | |||
453 | write_string = json.dumps(output, indent=2) | ||
454 | with open(direct_file, "w") as f: | ||
455 | bb.note("Writing file %s with CVE information" % direct_file) | ||
456 | f.write(write_string) | ||
457 | |||
458 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": | ||
459 | bb.utils.mkdirhier(os.path.dirname(deploy_file)) | ||
460 | with open(deploy_file, "w") as f: | ||
461 | f.write(write_string) | ||
462 | |||
463 | if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1": | ||
464 | cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR") | ||
465 | index_path = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") | ||
466 | bb.utils.mkdirhier(cvelogpath) | ||
467 | fragment_file = os.path.basename(deploy_file) | ||
468 | fragment_path = os.path.join(cvelogpath, fragment_file) | ||
469 | with open(fragment_path, "w") as f: | ||
470 | f.write(write_string) | ||
471 | with open(index_path, "a+") as f: | ||
472 | f.write("%s\n" % fragment_path) | ||
473 | |||
474 | def cve_write_data_json(d, patched, unpatched, ignored, cve_data): | ||
475 | """ | ||
476 | Prepare CVE data for the JSON format, then write it. | ||
477 | """ | ||
478 | |||
479 | output = {"version":"1", "package": []} | ||
480 | nvd_link = "https://nvd.nist.gov/vuln/detail/" | ||
481 | |||
482 | fdir_name = d.getVar("FILE_DIRNAME") | ||
483 | layer = fdir_name.split("/")[-3] | ||
484 | |||
485 | include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split() | ||
486 | exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split() | ||
487 | |||
488 | if exclude_layers and layer in exclude_layers: | ||
489 | return | ||
490 | |||
491 | if include_layers and layer not in include_layers: | ||
492 | return | ||
493 | |||
494 | unpatched_cves = [] | ||
495 | |||
496 | package_version = "%s%s" % (d.getVar("EXTENDPE"), d.getVar("PV")) | ||
497 | package_data = { | ||
498 | "name" : d.getVar("PN"), | ||
499 | "layer" : layer, | ||
500 | "version" : package_version | ||
501 | } | ||
502 | cve_list = [] | ||
503 | |||
504 | for cve in sorted(cve_data): | ||
505 | is_patched = cve in patched | ||
506 | status = "Unpatched" | ||
507 | if is_patched and (d.getVar("CVE_CHECK_REPORT_PATCHED") != "1"): | ||
508 | continue | ||
509 | if cve in ignored: | ||
510 | status = "Ignored" | ||
511 | elif is_patched: | ||
512 | status = "Patched" | ||
513 | else: | ||
514 | # default value of status is Unpatched | ||
515 | unpatched_cves.append(cve) | ||
516 | |||
517 | issue_link = "%s%s" % (nvd_link, cve) | ||
518 | |||
519 | cve_item = { | ||
520 | "id" : cve, | ||
521 | "summary" : cve_data[cve]["summary"], | ||
522 | "scorev2" : cve_data[cve]["scorev2"], | ||
523 | "scorev3" : cve_data[cve]["scorev3"], | ||
524 | "vector" : cve_data[cve]["vector"], | ||
525 | "status" : status, | ||
526 | "link": issue_link | ||
527 | } | ||
528 | cve_list.append(cve_item) | ||
529 | |||
530 | package_data["issue"] = cve_list | ||
531 | output["package"].append(package_data) | ||
532 | |||
533 | direct_file = d.getVar("CVE_CHECK_LOG_JSON") | ||
534 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") | ||
535 | manifest_file = d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON") | ||
536 | |||
537 | cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file) | ||
538 | |||
539 | def cve_write_data(d, patched, unpatched, ignored, cve_data): | ||
540 | """ | ||
541 | Write CVE data in each enabled format. | ||
542 | """ | ||
543 | |||
544 | if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1": | ||
545 | cve_write_data_text(d, patched, unpatched, ignored, cve_data) | ||
546 | if d.getVar("CVE_CHECK_FORMAT_JSON") == "1": | ||
547 | cve_write_data_json(d, patched, unpatched, ignored, cve_data) | ||
diff --git a/meta/lib/oe/cve_check.py b/meta/lib/oe/cve_check.py index a1d7c292af..1d3c775bbe 100644 --- a/meta/lib/oe/cve_check.py +++ b/meta/lib/oe/cve_check.py | |||
@@ -63,3 +63,19 @@ def _cmpkey(release, patch_l, pre_l, pre_v): | |||
63 | else: | 63 | else: |
64 | _pre = float(pre_v) if pre_v else float('-inf') | 64 | _pre = float(pre_v) if pre_v else float('-inf') |
65 | return _release, _patch, _pre | 65 | return _release, _patch, _pre |
66 | |||
67 | def cve_check_merge_jsons(output, data): | ||
68 | """ | ||
69 | Merge the data in the "package" property to the main data file | ||
70 | output | ||
71 | """ | ||
72 | if output["version"] != data["version"]: | ||
73 | bb.error("Version mismatch when merging JSON outputs") | ||
74 | return | ||
75 | |||
76 | for product in output["package"]: | ||
77 | if product["name"] == data["package"][0]["name"]: | ||
78 | bb.error("Error adding the same package twice") | ||
79 | return | ||
80 | |||
81 | output["package"].append(data["package"][0]) | ||