diff options
Diffstat (limited to 'meta/classes/cve-check.bbclass')
-rw-r--r-- | meta/classes/cve-check.bbclass | 466 |
1 files changed, 335 insertions, 131 deletions
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass index 8086cf05e9..5e6bae1757 100644 --- a/meta/classes/cve-check.bbclass +++ b/meta/classes/cve-check.bbclass | |||
@@ -20,13 +20,13 @@ | |||
20 | # the only method to check against CVEs. Running this tool | 20 | # the only method to check against CVEs. Running this tool |
21 | # doesn't guarantee your packages are free of CVEs. | 21 | # doesn't guarantee your packages are free of CVEs. |
22 | 22 | ||
23 | # The product name that the CVE database uses. Defaults to BPN, but may need to | 23 | # The product name that the CVE database uses defaults to BPN, but may need to |
24 | # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). | 24 | # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). |
25 | CVE_PRODUCT ??= "${BPN}" | 25 | CVE_PRODUCT ??= "${BPN}" |
26 | CVE_VERSION ??= "${PV}" | 26 | CVE_VERSION ??= "${PV}" |
27 | 27 | ||
28 | CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" | 28 | CVE_CHECK_DB_DIR ?= "${DL_DIR}/CVE_CHECK" |
29 | CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_1.1.db" | 29 | CVE_CHECK_DB_FILE ?= "${CVE_CHECK_DB_DIR}/nvdcve_2.db" |
30 | CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" | 30 | CVE_CHECK_DB_FILE_LOCK ?= "${CVE_CHECK_DB_FILE}.lock" |
31 | 31 | ||
32 | CVE_CHECK_LOG ?= "${T}/cve.log" | 32 | CVE_CHECK_LOG ?= "${T}/cve.log" |
@@ -34,15 +34,33 @@ CVE_CHECK_TMP_FILE ?= "${TMPDIR}/cve_check" | |||
34 | CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve" | 34 | CVE_CHECK_SUMMARY_DIR ?= "${LOG_DIR}/cve" |
35 | CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary" | 35 | CVE_CHECK_SUMMARY_FILE_NAME ?= "cve-summary" |
36 | CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}" | 36 | CVE_CHECK_SUMMARY_FILE ?= "${CVE_CHECK_SUMMARY_DIR}/${CVE_CHECK_SUMMARY_FILE_NAME}" |
37 | CVE_CHECK_SUMMARY_FILE_NAME_JSON = "cve-summary.json" | ||
38 | CVE_CHECK_SUMMARY_INDEX_PATH = "${CVE_CHECK_SUMMARY_DIR}/cve-summary-index.txt" | ||
39 | |||
40 | CVE_CHECK_LOG_JSON ?= "${T}/cve.json" | ||
37 | 41 | ||
38 | CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" | 42 | CVE_CHECK_DIR ??= "${DEPLOY_DIR}/cve" |
39 | CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}" | 43 | CVE_CHECK_RECIPE_FILE ?= "${CVE_CHECK_DIR}/${PN}" |
40 | CVE_CHECK_MANIFEST ?= "${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" | 44 | CVE_CHECK_RECIPE_FILE_JSON ?= "${CVE_CHECK_DIR}/${PN}_cve.json" |
45 | CVE_CHECK_MANIFEST ?= "${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.cve" | ||
46 | CVE_CHECK_MANIFEST_JSON ?= "${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.json" | ||
41 | CVE_CHECK_COPY_FILES ??= "1" | 47 | CVE_CHECK_COPY_FILES ??= "1" |
42 | CVE_CHECK_CREATE_MANIFEST ??= "1" | 48 | CVE_CHECK_CREATE_MANIFEST ??= "1" |
43 | 49 | ||
50 | # Report Patched or Ignored/Whitelisted CVEs | ||
44 | CVE_CHECK_REPORT_PATCHED ??= "1" | 51 | CVE_CHECK_REPORT_PATCHED ??= "1" |
45 | 52 | ||
53 | CVE_CHECK_SHOW_WARNINGS ??= "1" | ||
54 | |||
55 | # Provide text output | ||
56 | CVE_CHECK_FORMAT_TEXT ??= "1" | ||
57 | |||
58 | # Provide JSON output - disabled by default for backward compatibility | ||
59 | CVE_CHECK_FORMAT_JSON ??= "0" | ||
60 | |||
61 | # Check for packages without CVEs (no issues or missing product name) | ||
62 | CVE_CHECK_COVERAGE ??= "1" | ||
63 | |||
46 | # Whitelist for packages (PN) | 64 | # Whitelist for packages (PN) |
47 | CVE_CHECK_PN_WHITELIST ?= "" | 65 | CVE_CHECK_PN_WHITELIST ?= "" |
48 | 66 | ||
@@ -53,12 +71,43 @@ CVE_CHECK_PN_WHITELIST ?= "" | |||
53 | # | 71 | # |
54 | CVE_CHECK_WHITELIST ?= "" | 72 | CVE_CHECK_WHITELIST ?= "" |
55 | 73 | ||
56 | # set to "alphabetical" for version using single alphabetical character as increament release | 74 | # Layers to be excluded |
75 | CVE_CHECK_LAYER_EXCLUDELIST ??= "" | ||
76 | |||
77 | # Layers to be included | ||
78 | CVE_CHECK_LAYER_INCLUDELIST ??= "" | ||
79 | |||
80 | |||
81 | # set to "alphabetical" for version using single alphabetical character as increment release | ||
57 | CVE_VERSION_SUFFIX ??= "" | 82 | CVE_VERSION_SUFFIX ??= "" |
58 | 83 | ||
84 | def generate_json_report(d, out_path, link_path): | ||
85 | if os.path.exists(d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")): | ||
86 | import json | ||
87 | from oe.cve_check import cve_check_merge_jsons, update_symlinks | ||
88 | |||
89 | bb.note("Generating JSON CVE summary") | ||
90 | index_file = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") | ||
91 | summary = {"version":"1", "package": []} | ||
92 | with open(index_file) as f: | ||
93 | filename = f.readline() | ||
94 | while filename: | ||
95 | with open(filename.rstrip()) as j: | ||
96 | data = json.load(j) | ||
97 | cve_check_merge_jsons(summary, data) | ||
98 | filename = f.readline() | ||
99 | |||
100 | summary["package"].sort(key=lambda d: d['name']) | ||
101 | |||
102 | with open(out_path, "w") as f: | ||
103 | json.dump(summary, f, indent=2) | ||
104 | |||
105 | update_symlinks(out_path, link_path) | ||
106 | |||
59 | python cve_save_summary_handler () { | 107 | python cve_save_summary_handler () { |
60 | import shutil | 108 | import shutil |
61 | import datetime | 109 | import datetime |
110 | from oe.cve_check import update_symlinks | ||
62 | 111 | ||
63 | cve_tmp_file = d.getVar("CVE_CHECK_TMP_FILE") | 112 | cve_tmp_file = d.getVar("CVE_CHECK_TMP_FILE") |
64 | 113 | ||
@@ -71,13 +120,15 @@ python cve_save_summary_handler () { | |||
71 | 120 | ||
72 | if os.path.exists(cve_tmp_file): | 121 | if os.path.exists(cve_tmp_file): |
73 | shutil.copyfile(cve_tmp_file, cve_summary_file) | 122 | shutil.copyfile(cve_tmp_file, cve_summary_file) |
74 | 123 | cvefile_link = os.path.join(cvelogpath, cve_summary_name) | |
75 | if cve_summary_file and os.path.exists(cve_summary_file): | 124 | update_symlinks(cve_summary_file, cvefile_link) |
76 | cvefile_link = os.path.join(cvelogpath, cve_summary_name) | 125 | bb.plain("Complete CVE report summary created at: %s" % cvefile_link) |
77 | 126 | ||
78 | if os.path.exists(os.path.realpath(cvefile_link)): | 127 | if d.getVar("CVE_CHECK_FORMAT_JSON") == "1": |
79 | os.remove(cvefile_link) | 128 | json_summary_link_name = os.path.join(cvelogpath, d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON")) |
80 | os.symlink(os.path.basename(cve_summary_file), cvefile_link) | 129 | json_summary_name = os.path.join(cvelogpath, "%s-%s.json" % (cve_summary_name, timestamp)) |
130 | generate_json_report(d, json_summary_name, json_summary_link_name) | ||
131 | bb.plain("Complete CVE JSON report summary created at: %s" % json_summary_link_name) | ||
81 | } | 132 | } |
82 | 133 | ||
83 | addhandler cve_save_summary_handler | 134 | addhandler cve_save_summary_handler |
@@ -87,23 +138,25 @@ python do_cve_check () { | |||
87 | """ | 138 | """ |
88 | Check recipe for patched and unpatched CVEs | 139 | Check recipe for patched and unpatched CVEs |
89 | """ | 140 | """ |
141 | from oe.cve_check import get_patched_cves | ||
90 | 142 | ||
91 | if os.path.exists(d.getVar("CVE_CHECK_DB_FILE")): | 143 | with bb.utils.fileslocked([d.getVar("CVE_CHECK_DB_FILE_LOCK")], shared=True): |
92 | try: | 144 | if os.path.exists(d.getVar("CVE_CHECK_DB_FILE")): |
93 | patched_cves = get_patches_cves(d) | 145 | try: |
94 | except FileNotFoundError: | 146 | patched_cves = get_patched_cves(d) |
95 | bb.fatal("Failure in searching patches") | 147 | except FileNotFoundError: |
96 | whitelisted, patched, unpatched = check_cves(d, patched_cves) | 148 | bb.fatal("Failure in searching patches") |
97 | if patched or unpatched: | 149 | ignored, patched, unpatched, status = check_cves(d, patched_cves) |
98 | cve_data = get_cve_info(d, patched + unpatched) | 150 | if patched or unpatched or (d.getVar("CVE_CHECK_COVERAGE") == "1" and status): |
99 | cve_write_data(d, patched, unpatched, whitelisted, cve_data) | 151 | cve_data = get_cve_info(d, patched + unpatched + ignored) |
100 | else: | 152 | cve_write_data(d, patched, unpatched, ignored, cve_data, status) |
101 | bb.note("No CVE database found, skipping CVE check") | 153 | else: |
154 | bb.note("No CVE database found, skipping CVE check") | ||
102 | 155 | ||
103 | } | 156 | } |
104 | 157 | ||
105 | addtask cve_check before do_build after do_fetch | 158 | addtask cve_check before do_build |
106 | do_cve_check[depends] = "cve-update-db-native:do_populate_cve_db" | 159 | do_cve_check[depends] = "cve-update-nvd2-native:do_fetch" |
107 | do_cve_check[nostamp] = "1" | 160 | do_cve_check[nostamp] = "1" |
108 | 161 | ||
109 | python cve_check_cleanup () { | 162 | python cve_check_cleanup () { |
@@ -111,10 +164,11 @@ python cve_check_cleanup () { | |||
111 | Delete the file used to gather all the CVE information. | 164 | Delete the file used to gather all the CVE information. |
112 | """ | 165 | """ |
113 | bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE")) | 166 | bb.utils.remove(e.data.getVar("CVE_CHECK_TMP_FILE")) |
167 | bb.utils.remove(e.data.getVar("CVE_CHECK_SUMMARY_INDEX_PATH")) | ||
114 | } | 168 | } |
115 | 169 | ||
116 | addhandler cve_check_cleanup | 170 | addhandler cve_check_cleanup |
117 | cve_check_cleanup[eventmask] = "bb.cooker.CookerExit" | 171 | cve_check_cleanup[eventmask] = "bb.event.BuildCompleted" |
118 | 172 | ||
119 | python cve_check_write_rootfs_manifest () { | 173 | python cve_check_write_rootfs_manifest () { |
120 | """ | 174 | """ |
@@ -122,115 +176,107 @@ python cve_check_write_rootfs_manifest () { | |||
122 | """ | 176 | """ |
123 | 177 | ||
124 | import shutil | 178 | import shutil |
179 | import json | ||
180 | from oe.rootfs import image_list_installed_packages | ||
181 | from oe.cve_check import cve_check_merge_jsons, update_symlinks | ||
125 | 182 | ||
126 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": | 183 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": |
127 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE") | 184 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE") |
128 | if os.path.exists(deploy_file): | 185 | if os.path.exists(deploy_file): |
129 | bb.utils.remove(deploy_file) | 186 | bb.utils.remove(deploy_file) |
130 | 187 | deploy_file_json = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") | |
131 | if os.path.exists(d.getVar("CVE_CHECK_TMP_FILE")): | 188 | if os.path.exists(deploy_file_json): |
132 | bb.note("Writing rootfs CVE manifest") | 189 | bb.utils.remove(deploy_file_json) |
133 | deploy_dir = d.getVar("DEPLOY_DIR_IMAGE") | 190 | |
134 | link_name = d.getVar("IMAGE_LINK_NAME") | 191 | # Create a list of relevant recipies |
192 | recipies = set() | ||
193 | for pkg in list(image_list_installed_packages(d)): | ||
194 | pkg_info = os.path.join(d.getVar('PKGDATA_DIR'), | ||
195 | 'runtime-reverse', pkg) | ||
196 | pkg_data = oe.packagedata.read_pkgdatafile(pkg_info) | ||
197 | recipies.add(pkg_data["PN"]) | ||
198 | |||
199 | bb.note("Writing rootfs CVE manifest") | ||
200 | deploy_dir = d.getVar("IMGDEPLOYDIR") | ||
201 | link_name = d.getVar("IMAGE_LINK_NAME") | ||
202 | |||
203 | json_data = {"version":"1", "package": []} | ||
204 | text_data = "" | ||
205 | enable_json = d.getVar("CVE_CHECK_FORMAT_JSON") == "1" | ||
206 | enable_text = d.getVar("CVE_CHECK_FORMAT_TEXT") == "1" | ||
207 | |||
208 | save_pn = d.getVar("PN") | ||
209 | |||
210 | for pkg in recipies: | ||
211 | # To be able to use the CVE_CHECK_RECIPE_FILE variable we have to evaluate | ||
212 | # it with the different PN names set each time. | ||
213 | d.setVar("PN", pkg) | ||
214 | if enable_text: | ||
215 | pkgfilepath = d.getVar("CVE_CHECK_RECIPE_FILE") | ||
216 | if os.path.exists(pkgfilepath): | ||
217 | with open(pkgfilepath) as pfile: | ||
218 | text_data += pfile.read() | ||
219 | |||
220 | if enable_json: | ||
221 | pkgfilepath = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") | ||
222 | if os.path.exists(pkgfilepath): | ||
223 | with open(pkgfilepath) as j: | ||
224 | data = json.load(j) | ||
225 | cve_check_merge_jsons(json_data, data) | ||
226 | |||
227 | d.setVar("PN", save_pn) | ||
228 | |||
229 | if enable_text: | ||
230 | link_path = os.path.join(deploy_dir, "%s.cve" % link_name) | ||
135 | manifest_name = d.getVar("CVE_CHECK_MANIFEST") | 231 | manifest_name = d.getVar("CVE_CHECK_MANIFEST") |
136 | cve_tmp_file = d.getVar("CVE_CHECK_TMP_FILE") | ||
137 | |||
138 | shutil.copyfile(cve_tmp_file, manifest_name) | ||
139 | 232 | ||
140 | if manifest_name and os.path.exists(manifest_name): | 233 | with open(manifest_name, "w") as f: |
141 | manifest_link = os.path.join(deploy_dir, "%s.cve" % link_name) | 234 | f.write(text_data) |
142 | # If we already have another manifest, update symlinks | ||
143 | if os.path.exists(os.path.realpath(manifest_link)): | ||
144 | os.remove(manifest_link) | ||
145 | os.symlink(os.path.basename(manifest_name), manifest_link) | ||
146 | bb.plain("Image CVE report stored in: %s" % manifest_name) | ||
147 | } | ||
148 | |||
149 | ROOTFS_POSTPROCESS_COMMAND_prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" | ||
150 | do_rootfs[recrdeptask] += "${@'do_cve_check' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" | ||
151 | 235 | ||
152 | def get_patches_cves(d): | 236 | update_symlinks(manifest_name, link_path) |
153 | """ | 237 | bb.plain("Image CVE report stored in: %s" % manifest_name) |
154 | Get patches that solve CVEs using the "CVE: " tag. | ||
155 | """ | ||
156 | 238 | ||
157 | import re | 239 | if enable_json: |
240 | link_path = os.path.join(deploy_dir, "%s.json" % link_name) | ||
241 | manifest_name = d.getVar("CVE_CHECK_MANIFEST_JSON") | ||
158 | 242 | ||
159 | pn = d.getVar("PN") | 243 | with open(manifest_name, "w") as f: |
160 | cve_match = re.compile("CVE:( CVE\-\d{4}\-\d+)+") | 244 | json.dump(json_data, f, indent=2) |
161 | |||
162 | # Matches last CVE-1234-211432 in the file name, also if written | ||
163 | # with small letters. Not supporting multiple CVE id's in a single | ||
164 | # file name. | ||
165 | cve_file_name_match = re.compile(".*([Cc][Vv][Ee]\-\d{4}\-\d+)") | ||
166 | |||
167 | patched_cves = set() | ||
168 | bb.debug(2, "Looking for patches that solves CVEs for %s" % pn) | ||
169 | for url in src_patches(d): | ||
170 | patch_file = bb.fetch.decodeurl(url)[2] | ||
171 | |||
172 | if not os.path.isfile(patch_file): | ||
173 | bb.error("File Not found: %s" % patch_file) | ||
174 | raise FileNotFoundError | ||
175 | |||
176 | # Check patch file name for CVE ID | ||
177 | fname_match = cve_file_name_match.search(patch_file) | ||
178 | if fname_match: | ||
179 | cve = fname_match.group(1).upper() | ||
180 | patched_cves.add(cve) | ||
181 | bb.debug(2, "Found CVE %s from patch file name %s" % (cve, patch_file)) | ||
182 | |||
183 | with open(patch_file, "r", encoding="utf-8") as f: | ||
184 | try: | ||
185 | patch_text = f.read() | ||
186 | except UnicodeDecodeError: | ||
187 | bb.debug(1, "Failed to read patch %s using UTF-8 encoding" | ||
188 | " trying with iso8859-1" % patch_file) | ||
189 | f.close() | ||
190 | with open(patch_file, "r", encoding="iso8859-1") as f: | ||
191 | patch_text = f.read() | ||
192 | |||
193 | # Search for one or more "CVE: " lines | ||
194 | text_match = False | ||
195 | for match in cve_match.finditer(patch_text): | ||
196 | # Get only the CVEs without the "CVE: " tag | ||
197 | cves = patch_text[match.start()+5:match.end()] | ||
198 | for cve in cves.split(): | ||
199 | bb.debug(2, "Patch %s solves %s" % (patch_file, cve)) | ||
200 | patched_cves.add(cve) | ||
201 | text_match = True | ||
202 | 245 | ||
203 | if not fname_match and not text_match: | 246 | update_symlinks(manifest_name, link_path) |
204 | bb.debug(2, "Patch %s doesn't solve CVEs" % patch_file) | 247 | bb.plain("Image CVE JSON report stored in: %s" % manifest_name) |
248 | } | ||
205 | 249 | ||
206 | return patched_cves | 250 | ROOTFS_POSTPROCESS_COMMAND_prepend = "${@'cve_check_write_rootfs_manifest; ' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" |
251 | do_rootfs[recrdeptask] += "${@'do_cve_check' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" | ||
252 | do_populate_sdk[recrdeptask] += "${@'do_cve_check' if d.getVar('CVE_CHECK_CREATE_MANIFEST') == '1' else ''}" | ||
207 | 253 | ||
208 | def check_cves(d, patched_cves): | 254 | def check_cves(d, patched_cves): |
209 | """ | 255 | """ |
210 | Connect to the NVD database and find unpatched cves. | 256 | Connect to the NVD database and find unpatched cves. |
211 | """ | 257 | """ |
212 | from oe.cve_check import Version | 258 | from oe.cve_check import Version, convert_cve_version |
213 | 259 | ||
214 | pn = d.getVar("PN") | 260 | pn = d.getVar("PN") |
215 | real_pv = d.getVar("PV") | 261 | real_pv = d.getVar("PV") |
216 | suffix = d.getVar("CVE_VERSION_SUFFIX") | 262 | suffix = d.getVar("CVE_VERSION_SUFFIX") |
217 | 263 | ||
218 | cves_unpatched = [] | 264 | cves_unpatched = [] |
265 | cves_ignored = [] | ||
266 | cves_status = [] | ||
267 | cves_in_recipe = False | ||
219 | # CVE_PRODUCT can contain more than one product (eg. curl/libcurl) | 268 | # CVE_PRODUCT can contain more than one product (eg. curl/libcurl) |
220 | products = d.getVar("CVE_PRODUCT").split() | 269 | products = d.getVar("CVE_PRODUCT").split() |
221 | # If this has been unset then we're not scanning for CVEs here (for example, image recipes) | 270 | # If this has been unset then we're not scanning for CVEs here (for example, image recipes) |
222 | if not products: | 271 | if not products: |
223 | return ([], [], []) | 272 | return ([], [], [], []) |
224 | pv = d.getVar("CVE_VERSION").split("+git")[0] | 273 | pv = d.getVar("CVE_VERSION").split("+git")[0] |
225 | 274 | ||
226 | # If the recipe has been whitlisted we return empty lists | 275 | # If the recipe has been whitelisted we return empty lists |
227 | if pn in d.getVar("CVE_CHECK_PN_WHITELIST").split(): | 276 | if pn in d.getVar("CVE_CHECK_PN_WHITELIST").split(): |
228 | bb.note("Recipe has been whitelisted, skipping check") | 277 | bb.note("Recipe has been whitelisted, skipping check") |
229 | return ([], [], []) | 278 | return ([], [], [], []) |
230 | 279 | ||
231 | old_cve_whitelist = d.getVar("CVE_CHECK_CVE_WHITELIST") | ||
232 | if old_cve_whitelist: | ||
233 | bb.warn("CVE_CHECK_CVE_WHITELIST is deprecated, please use CVE_CHECK_WHITELIST.") | ||
234 | cve_whitelist = d.getVar("CVE_CHECK_WHITELIST").split() | 280 | cve_whitelist = d.getVar("CVE_CHECK_WHITELIST").split() |
235 | 281 | ||
236 | import sqlite3 | 282 | import sqlite3 |
@@ -239,28 +285,42 @@ def check_cves(d, patched_cves): | |||
239 | 285 | ||
240 | # For each of the known product names (e.g. curl has CPEs using curl and libcurl)... | 286 | # For each of the known product names (e.g. curl has CPEs using curl and libcurl)... |
241 | for product in products: | 287 | for product in products: |
288 | cves_in_product = False | ||
242 | if ":" in product: | 289 | if ":" in product: |
243 | vendor, product = product.split(":", 1) | 290 | vendor, product = product.split(":", 1) |
244 | else: | 291 | else: |
245 | vendor = "%" | 292 | vendor = "%" |
246 | 293 | ||
247 | # Find all relevant CVE IDs. | 294 | # Find all relevant CVE IDs. |
248 | for cverow in conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor)): | 295 | cve_cursor = conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor)) |
296 | for cverow in cve_cursor: | ||
249 | cve = cverow[0] | 297 | cve = cverow[0] |
250 | 298 | ||
251 | if cve in cve_whitelist: | 299 | if cve in cve_whitelist: |
252 | bb.note("%s-%s has been whitelisted for %s" % (product, pv, cve)) | 300 | bb.note("%s-%s has been whitelisted for %s" % (product, pv, cve)) |
253 | # TODO: this should be in the report as 'whitelisted' | 301 | cves_ignored.append(cve) |
254 | patched_cves.add(cve) | ||
255 | continue | 302 | continue |
256 | elif cve in patched_cves: | 303 | elif cve in patched_cves: |
257 | bb.note("%s has been patched" % (cve)) | 304 | bb.note("%s has been patched" % (cve)) |
258 | continue | 305 | continue |
306 | # Write status once only for each product | ||
307 | if not cves_in_product: | ||
308 | cves_status.append([product, True]) | ||
309 | cves_in_product = True | ||
310 | cves_in_recipe = True | ||
259 | 311 | ||
260 | vulnerable = False | 312 | vulnerable = False |
261 | for row in conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)): | 313 | ignored = False |
314 | |||
315 | product_cursor = conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)) | ||
316 | for row in product_cursor: | ||
262 | (_, _, _, version_start, operator_start, version_end, operator_end) = row | 317 | (_, _, _, version_start, operator_start, version_end, operator_end) = row |
263 | #bb.debug(2, "Evaluating row " + str(row)) | 318 | #bb.debug(2, "Evaluating row " + str(row)) |
319 | if cve in cve_whitelist: | ||
320 | ignored = True | ||
321 | |||
322 | version_start = convert_cve_version(version_start) | ||
323 | version_end = convert_cve_version(version_end) | ||
264 | 324 | ||
265 | if (operator_start == '=' and pv == version_start) or version_start == '-': | 325 | if (operator_start == '=' and pv == version_start) or version_start == '-': |
266 | vulnerable = True | 326 | vulnerable = True |
@@ -293,18 +353,27 @@ def check_cves(d, patched_cves): | |||
293 | vulnerable = vulnerable_start or vulnerable_end | 353 | vulnerable = vulnerable_start or vulnerable_end |
294 | 354 | ||
295 | if vulnerable: | 355 | if vulnerable: |
296 | bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve)) | 356 | if ignored: |
297 | cves_unpatched.append(cve) | 357 | bb.note("%s is ignored in %s-%s" % (cve, pn, real_pv)) |
358 | cves_ignored.append(cve) | ||
359 | else: | ||
360 | bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve)) | ||
361 | cves_unpatched.append(cve) | ||
298 | break | 362 | break |
363 | product_cursor.close() | ||
299 | 364 | ||
300 | if not vulnerable: | 365 | if not vulnerable: |
301 | bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve)) | 366 | bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve)) |
302 | # TODO: not patched but not vulnerable | ||
303 | patched_cves.add(cve) | 367 | patched_cves.add(cve) |
368 | cve_cursor.close() | ||
369 | |||
370 | if not cves_in_product: | ||
371 | bb.note("No CVE records found for product %s, pn %s" % (product, pn)) | ||
372 | cves_status.append([product, False]) | ||
304 | 373 | ||
305 | conn.close() | 374 | conn.close() |
306 | 375 | ||
307 | return (list(cve_whitelist), list(patched_cves), cves_unpatched) | 376 | return (list(cves_ignored), list(patched_cves), cves_unpatched, cves_status) |
308 | 377 | ||
309 | def get_cve_info(d, cves): | 378 | def get_cve_info(d, cves): |
310 | """ | 379 | """ |
@@ -314,21 +383,23 @@ def get_cve_info(d, cves): | |||
314 | import sqlite3 | 383 | import sqlite3 |
315 | 384 | ||
316 | cve_data = {} | 385 | cve_data = {} |
317 | conn = sqlite3.connect(d.getVar("CVE_CHECK_DB_FILE")) | 386 | db_file = d.expand("file:${CVE_CHECK_DB_FILE}?mode=ro") |
387 | conn = sqlite3.connect(db_file, uri=True) | ||
318 | 388 | ||
319 | for cve in cves: | 389 | for cve in cves: |
320 | for row in conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)): | 390 | cursor = conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)) |
391 | for row in cursor: | ||
321 | cve_data[row[0]] = {} | 392 | cve_data[row[0]] = {} |
322 | cve_data[row[0]]["summary"] = row[1] | 393 | cve_data[row[0]]["summary"] = row[1] |
323 | cve_data[row[0]]["scorev2"] = row[2] | 394 | cve_data[row[0]]["scorev2"] = row[2] |
324 | cve_data[row[0]]["scorev3"] = row[3] | 395 | cve_data[row[0]]["scorev3"] = row[3] |
325 | cve_data[row[0]]["modified"] = row[4] | 396 | cve_data[row[0]]["modified"] = row[4] |
326 | cve_data[row[0]]["vector"] = row[5] | 397 | cve_data[row[0]]["vector"] = row[5] |
327 | 398 | cursor.close() | |
328 | conn.close() | 399 | conn.close() |
329 | return cve_data | 400 | return cve_data |
330 | 401 | ||
331 | def cve_write_data(d, patched, unpatched, whitelisted, cve_data): | 402 | def cve_write_data_text(d, patched, unpatched, whitelisted, cve_data): |
332 | """ | 403 | """ |
333 | Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and | 404 | Write CVE information in WORKDIR; and to CVE_CHECK_DIR, and |
334 | CVE manifest if enabled. | 405 | CVE manifest if enabled. |
@@ -338,20 +409,38 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data): | |||
338 | fdir_name = d.getVar("FILE_DIRNAME") | 409 | fdir_name = d.getVar("FILE_DIRNAME") |
339 | layer = fdir_name.split("/")[-3] | 410 | layer = fdir_name.split("/")[-3] |
340 | 411 | ||
341 | nvd_link = "https://web.nvd.nist.gov/view/vuln/detail?vulnId=" | 412 | include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split() |
413 | exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split() | ||
414 | |||
415 | report_all = d.getVar("CVE_CHECK_REPORT_PATCHED") == "1" | ||
416 | |||
417 | if exclude_layers and layer in exclude_layers: | ||
418 | return | ||
419 | |||
420 | if include_layers and layer not in include_layers: | ||
421 | return | ||
422 | |||
423 | # Early exit, the text format does not report packages without CVEs | ||
424 | if not patched+unpatched+whitelisted: | ||
425 | return | ||
426 | |||
427 | nvd_link = "https://nvd.nist.gov/vuln/detail/" | ||
342 | write_string = "" | 428 | write_string = "" |
343 | unpatched_cves = [] | 429 | unpatched_cves = [] |
344 | bb.utils.mkdirhier(os.path.dirname(cve_file)) | 430 | bb.utils.mkdirhier(os.path.dirname(cve_file)) |
345 | 431 | ||
346 | for cve in sorted(cve_data): | 432 | for cve in sorted(cve_data): |
347 | is_patched = cve in patched | 433 | is_patched = cve in patched |
348 | if is_patched and (d.getVar("CVE_CHECK_REPORT_PATCHED") != "1"): | 434 | is_ignored = cve in whitelisted |
435 | |||
436 | if (is_patched or is_ignored) and not report_all: | ||
349 | continue | 437 | continue |
438 | |||
350 | write_string += "LAYER: %s\n" % layer | 439 | write_string += "LAYER: %s\n" % layer |
351 | write_string += "PACKAGE NAME: %s\n" % d.getVar("PN") | 440 | write_string += "PACKAGE NAME: %s\n" % d.getVar("PN") |
352 | write_string += "PACKAGE VERSION: %s%s\n" % (d.getVar("EXTENDPE"), d.getVar("PV")) | 441 | write_string += "PACKAGE VERSION: %s%s\n" % (d.getVar("EXTENDPE"), d.getVar("PV")) |
353 | write_string += "CVE: %s\n" % cve | 442 | write_string += "CVE: %s\n" % cve |
354 | if cve in whitelisted: | 443 | if is_ignored: |
355 | write_string += "CVE STATUS: Whitelisted\n" | 444 | write_string += "CVE STATUS: Whitelisted\n" |
356 | elif is_patched: | 445 | elif is_patched: |
357 | write_string += "CVE STATUS: Patched\n" | 446 | write_string += "CVE STATUS: Patched\n" |
@@ -364,23 +453,138 @@ def cve_write_data(d, patched, unpatched, whitelisted, cve_data): | |||
364 | write_string += "VECTOR: %s\n" % cve_data[cve]["vector"] | 453 | write_string += "VECTOR: %s\n" % cve_data[cve]["vector"] |
365 | write_string += "MORE INFORMATION: %s%s\n\n" % (nvd_link, cve) | 454 | write_string += "MORE INFORMATION: %s%s\n\n" % (nvd_link, cve) |
366 | 455 | ||
367 | if unpatched_cves: | 456 | if unpatched_cves and d.getVar("CVE_CHECK_SHOW_WARNINGS") == "1": |
368 | bb.warn("Found unpatched CVE (%s), for more information check %s" % (" ".join(unpatched_cves),cve_file)) | 457 | bb.warn("Found unpatched CVE (%s), for more information check %s" % (" ".join(unpatched_cves),cve_file)) |
369 | 458 | ||
370 | if write_string: | 459 | with open(cve_file, "w") as f: |
371 | with open(cve_file, "w") as f: | 460 | bb.note("Writing file %s with CVE information" % cve_file) |
372 | bb.note("Writing file %s with CVE information" % cve_file) | 461 | f.write(write_string) |
462 | |||
463 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": | ||
464 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE") | ||
465 | bb.utils.mkdirhier(os.path.dirname(deploy_file)) | ||
466 | with open(deploy_file, "w") as f: | ||
467 | f.write(write_string) | ||
468 | |||
469 | if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1": | ||
470 | cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR") | ||
471 | bb.utils.mkdirhier(cvelogpath) | ||
472 | |||
473 | with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f: | ||
474 | f.write("%s" % write_string) | ||
475 | |||
476 | def cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file): | ||
477 | """ | ||
478 | Write CVE information in the JSON format: to WORKDIR; and to | ||
479 | CVE_CHECK_DIR, if CVE manifest if enabled, write fragment | ||
480 | files that will be assembled at the end in cve_check_write_rootfs_manifest. | ||
481 | """ | ||
482 | |||
483 | import json | ||
484 | |||
485 | write_string = json.dumps(output, indent=2) | ||
486 | with open(direct_file, "w") as f: | ||
487 | bb.note("Writing file %s with CVE information" % direct_file) | ||
488 | f.write(write_string) | ||
489 | |||
490 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": | ||
491 | bb.utils.mkdirhier(os.path.dirname(deploy_file)) | ||
492 | with open(deploy_file, "w") as f: | ||
493 | f.write(write_string) | ||
494 | |||
495 | if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1": | ||
496 | cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR") | ||
497 | index_path = d.getVar("CVE_CHECK_SUMMARY_INDEX_PATH") | ||
498 | bb.utils.mkdirhier(cvelogpath) | ||
499 | fragment_file = os.path.basename(deploy_file) | ||
500 | fragment_path = os.path.join(cvelogpath, fragment_file) | ||
501 | with open(fragment_path, "w") as f: | ||
373 | f.write(write_string) | 502 | f.write(write_string) |
503 | with open(index_path, "a+") as f: | ||
504 | f.write("%s\n" % fragment_path) | ||
505 | |||
506 | def cve_write_data_json(d, patched, unpatched, ignored, cve_data, cve_status): | ||
507 | """ | ||
508 | Prepare CVE data for the JSON format, then write it. | ||
509 | """ | ||
510 | |||
511 | output = {"version":"1", "package": []} | ||
512 | nvd_link = "https://nvd.nist.gov/vuln/detail/" | ||
513 | |||
514 | fdir_name = d.getVar("FILE_DIRNAME") | ||
515 | layer = fdir_name.split("/")[-3] | ||
516 | |||
517 | include_layers = d.getVar("CVE_CHECK_LAYER_INCLUDELIST").split() | ||
518 | exclude_layers = d.getVar("CVE_CHECK_LAYER_EXCLUDELIST").split() | ||
519 | |||
520 | report_all = d.getVar("CVE_CHECK_REPORT_PATCHED") == "1" | ||
521 | |||
522 | if exclude_layers and layer in exclude_layers: | ||
523 | return | ||
524 | |||
525 | if include_layers and layer not in include_layers: | ||
526 | return | ||
527 | |||
528 | unpatched_cves = [] | ||
529 | |||
530 | product_data = [] | ||
531 | for s in cve_status: | ||
532 | p = {"product": s[0], "cvesInRecord": "Yes"} | ||
533 | if s[1] == False: | ||
534 | p["cvesInRecord"] = "No" | ||
535 | product_data.append(p) | ||
536 | |||
537 | package_version = "%s%s" % (d.getVar("EXTENDPE"), d.getVar("PV")) | ||
538 | package_data = { | ||
539 | "name" : d.getVar("PN"), | ||
540 | "layer" : layer, | ||
541 | "version" : package_version, | ||
542 | "products": product_data | ||
543 | } | ||
544 | cve_list = [] | ||
545 | |||
546 | for cve in sorted(cve_data): | ||
547 | is_patched = cve in patched | ||
548 | is_ignored = cve in ignored | ||
549 | status = "Unpatched" | ||
550 | if (is_patched or is_ignored) and not report_all: | ||
551 | continue | ||
552 | if is_ignored: | ||
553 | status = "Ignored" | ||
554 | elif is_patched: | ||
555 | status = "Patched" | ||
556 | else: | ||
557 | # default value of status is Unpatched | ||
558 | unpatched_cves.append(cve) | ||
559 | |||
560 | issue_link = "%s%s" % (nvd_link, cve) | ||
374 | 561 | ||
375 | if d.getVar("CVE_CHECK_COPY_FILES") == "1": | 562 | cve_item = { |
376 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE") | 563 | "id" : cve, |
377 | bb.utils.mkdirhier(os.path.dirname(deploy_file)) | 564 | "summary" : cve_data[cve]["summary"], |
378 | with open(deploy_file, "w") as f: | 565 | "scorev2" : cve_data[cve]["scorev2"], |
379 | f.write(write_string) | 566 | "scorev3" : cve_data[cve]["scorev3"], |
567 | "vector" : cve_data[cve]["vector"], | ||
568 | "status" : status, | ||
569 | "link": issue_link | ||
570 | } | ||
571 | cve_list.append(cve_item) | ||
380 | 572 | ||
381 | if d.getVar("CVE_CHECK_CREATE_MANIFEST") == "1": | 573 | package_data["issue"] = cve_list |
382 | cvelogpath = d.getVar("CVE_CHECK_SUMMARY_DIR") | 574 | output["package"].append(package_data) |
383 | bb.utils.mkdirhier(cvelogpath) | 575 | |
576 | direct_file = d.getVar("CVE_CHECK_LOG_JSON") | ||
577 | deploy_file = d.getVar("CVE_CHECK_RECIPE_FILE_JSON") | ||
578 | manifest_file = d.getVar("CVE_CHECK_SUMMARY_FILE_NAME_JSON") | ||
579 | |||
580 | cve_check_write_json_output(d, output, direct_file, deploy_file, manifest_file) | ||
581 | |||
582 | def cve_write_data(d, patched, unpatched, ignored, cve_data, status): | ||
583 | """ | ||
584 | Write CVE data in each enabled format. | ||
585 | """ | ||
384 | 586 | ||
385 | with open(d.getVar("CVE_CHECK_TMP_FILE"), "a") as f: | 587 | if d.getVar("CVE_CHECK_FORMAT_TEXT") == "1": |
386 | f.write("%s" % write_string) | 588 | cve_write_data_text(d, patched, unpatched, ignored, cve_data) |
589 | if d.getVar("CVE_CHECK_FORMAT_JSON") == "1": | ||
590 | cve_write_data_json(d, patched, unpatched, ignored, cve_data, status) | ||