diff options
Diffstat (limited to 'meta/recipes-core/meta/cve-update-nvd2-native.bb')
-rw-r--r-- | meta/recipes-core/meta/cve-update-nvd2-native.bb | 372 |
1 files changed, 372 insertions, 0 deletions
diff --git a/meta/recipes-core/meta/cve-update-nvd2-native.bb b/meta/recipes-core/meta/cve-update-nvd2-native.bb new file mode 100644 index 0000000000..1a3eeba6d0 --- /dev/null +++ b/meta/recipes-core/meta/cve-update-nvd2-native.bb | |||
@@ -0,0 +1,372 @@ | |||
1 | SUMMARY = "Updates the NVD CVE database" | ||
2 | LICENSE = "MIT" | ||
3 | |||
4 | # Important note: | ||
5 | # This product uses the NVD API but is not endorsed or certified by the NVD. | ||
6 | |||
7 | INHIBIT_DEFAULT_DEPS = "1" | ||
8 | |||
9 | inherit native | ||
10 | |||
11 | deltask do_unpack | ||
12 | deltask do_patch | ||
13 | deltask do_configure | ||
14 | deltask do_compile | ||
15 | deltask do_install | ||
16 | deltask do_populate_sysroot | ||
17 | |||
18 | NVDCVE_URL ?= "https://services.nvd.nist.gov/rest/json/cves/2.0" | ||
19 | |||
20 | # If you have a NVD API key (https://nvd.nist.gov/developers/request-an-api-key) | ||
21 | # then setting this to get higher rate limits. | ||
22 | NVDCVE_API_KEY ?= "" | ||
23 | |||
24 | # CVE database update interval, in seconds. By default: once a day (24*60*60). | ||
25 | # Use 0 to force the update | ||
26 | # Use a negative value to skip the update | ||
27 | CVE_DB_UPDATE_INTERVAL ?= "86400" | ||
28 | |||
29 | # CVE database incremental update age threshold, in seconds. If the database is | ||
30 | # older than this threshold, do a full re-download, else, do an incremental | ||
31 | # update. By default: the maximum allowed value from NVD: 120 days (120*24*60*60) | ||
32 | # Use 0 to force a full download. | ||
33 | CVE_DB_INCR_UPDATE_AGE_THRES ?= "10368000" | ||
34 | |||
35 | # Number of attempts for each http query to nvd server before giving up | ||
36 | CVE_DB_UPDATE_ATTEMPTS ?= "5" | ||
37 | |||
38 | CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_2.db" | ||
39 | |||
40 | python () { | ||
41 | if not bb.data.inherits_class("cve-check", d): | ||
42 | raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.") | ||
43 | } | ||
44 | |||
45 | python do_fetch() { | ||
46 | """ | ||
47 | Update NVD database with API 2.0 | ||
48 | """ | ||
49 | import bb.utils | ||
50 | import bb.progress | ||
51 | import shutil | ||
52 | |||
53 | bb.utils.export_proxies(d) | ||
54 | |||
55 | db_file = d.getVar("CVE_CHECK_DB_FILE") | ||
56 | db_dir = os.path.dirname(db_file) | ||
57 | db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") | ||
58 | |||
59 | cleanup_db_download(db_file, db_tmp_file) | ||
60 | # By default let's update the whole database (since time 0) | ||
61 | database_time = 0 | ||
62 | |||
63 | # The NVD database changes once a day, so no need to update more frequently | ||
64 | # Allow the user to force-update | ||
65 | try: | ||
66 | import time | ||
67 | update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL")) | ||
68 | if update_interval < 0: | ||
69 | bb.note("CVE database update skipped") | ||
70 | return | ||
71 | if time.time() - os.path.getmtime(db_file) < update_interval: | ||
72 | bb.note("CVE database recently updated, skipping") | ||
73 | return | ||
74 | database_time = os.path.getmtime(db_file) | ||
75 | |||
76 | except OSError: | ||
77 | pass | ||
78 | |||
79 | bb.utils.mkdirhier(db_dir) | ||
80 | if os.path.exists(db_file): | ||
81 | shutil.copy2(db_file, db_tmp_file) | ||
82 | |||
83 | if update_db_file(db_tmp_file, d, database_time) == True: | ||
84 | # Update downloaded correctly, can swap files | ||
85 | shutil.move(db_tmp_file, db_file) | ||
86 | else: | ||
87 | # Update failed, do not modify the database | ||
88 | bb.warn("CVE database update failed") | ||
89 | os.remove(db_tmp_file) | ||
90 | } | ||
91 | |||
92 | do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" | ||
93 | do_fetch[file-checksums] = "" | ||
94 | do_fetch[vardeps] = "" | ||
95 | |||
96 | def cleanup_db_download(db_file, db_tmp_file): | ||
97 | """ | ||
98 | Cleanup the download space from possible failed downloads | ||
99 | """ | ||
100 | |||
101 | # Clean up the updates done on the main file | ||
102 | # Remove it only if a journal file exists - it means a complete re-download | ||
103 | if os.path.exists("{0}-journal".format(db_file)): | ||
104 | # If a journal is present the last update might have been interrupted. In that case, | ||
105 | # just wipe any leftovers and force the DB to be recreated. | ||
106 | os.remove("{0}-journal".format(db_file)) | ||
107 | |||
108 | if os.path.exists(db_file): | ||
109 | os.remove(db_file) | ||
110 | |||
111 | # Clean-up the temporary file downloads, we can remove both journal | ||
112 | # and the temporary database | ||
113 | if os.path.exists("{0}-journal".format(db_tmp_file)): | ||
114 | # If a journal is present the last update might have been interrupted. In that case, | ||
115 | # just wipe any leftovers and force the DB to be recreated. | ||
116 | os.remove("{0}-journal".format(db_tmp_file)) | ||
117 | |||
118 | if os.path.exists(db_tmp_file): | ||
119 | os.remove(db_tmp_file) | ||
120 | |||
121 | def nvd_request_wait(attempt, min_wait): | ||
122 | return min ( ( (2 * attempt) + min_wait ) , 30) | ||
123 | |||
124 | def nvd_request_next(url, attempts, api_key, args, min_wait): | ||
125 | """ | ||
126 | Request next part of the NVD database | ||
127 | NVD API documentation: https://nvd.nist.gov/developers/vulnerabilities | ||
128 | """ | ||
129 | |||
130 | import urllib.request | ||
131 | import urllib.parse | ||
132 | import gzip | ||
133 | import http | ||
134 | import time | ||
135 | |||
136 | request = urllib.request.Request(url + "?" + urllib.parse.urlencode(args)) | ||
137 | if api_key: | ||
138 | request.add_header("apiKey", api_key) | ||
139 | bb.note("Requesting %s" % request.full_url) | ||
140 | |||
141 | for attempt in range(attempts): | ||
142 | try: | ||
143 | r = urllib.request.urlopen(request) | ||
144 | |||
145 | if (r.headers['content-encoding'] == 'gzip'): | ||
146 | buf = r.read() | ||
147 | raw_data = gzip.decompress(buf).decode("utf-8") | ||
148 | else: | ||
149 | raw_data = r.read().decode("utf-8") | ||
150 | |||
151 | r.close() | ||
152 | |||
153 | except Exception as e: | ||
154 | wait_time = nvd_request_wait(attempt, min_wait) | ||
155 | bb.note("CVE database: received error (%s)" % (e)) | ||
156 | bb.note("CVE database: retrying download after %d seconds. attempted (%d/%d)" % (wait_time, attempt+1, attempts)) | ||
157 | time.sleep(wait_time) | ||
158 | pass | ||
159 | else: | ||
160 | return raw_data | ||
161 | else: | ||
162 | # We failed at all attempts | ||
163 | return None | ||
164 | |||
165 | def update_db_file(db_tmp_file, d, database_time): | ||
166 | """ | ||
167 | Update the given database file | ||
168 | """ | ||
169 | import bb.utils, bb.progress | ||
170 | import datetime | ||
171 | import sqlite3 | ||
172 | import json | ||
173 | |||
174 | # Connect to database | ||
175 | conn = sqlite3.connect(db_tmp_file) | ||
176 | initialize_db(conn) | ||
177 | |||
178 | req_args = {'startIndex' : 0} | ||
179 | |||
180 | incr_update_threshold = int(d.getVar("CVE_DB_INCR_UPDATE_AGE_THRES")) | ||
181 | if database_time != 0: | ||
182 | database_date = datetime.datetime.fromtimestamp(database_time, tz=datetime.timezone.utc) | ||
183 | today_date = datetime.datetime.now(tz=datetime.timezone.utc) | ||
184 | delta = today_date - database_date | ||
185 | if incr_update_threshold == 0: | ||
186 | bb.note("CVE database: forced full update") | ||
187 | elif delta < datetime.timedelta(seconds=incr_update_threshold): | ||
188 | bb.note("CVE database: performing partial update") | ||
189 | # The maximum range for time is 120 days | ||
190 | if delta > datetime.timedelta(days=120): | ||
191 | bb.error("CVE database: Trying to do an incremental update on a larger than supported range") | ||
192 | req_args['lastModStartDate'] = database_date.isoformat() | ||
193 | req_args['lastModEndDate'] = today_date.isoformat() | ||
194 | else: | ||
195 | bb.note("CVE database: file too old, forcing a full update") | ||
196 | else: | ||
197 | bb.note("CVE database: no preexisting database, do a full download") | ||
198 | |||
199 | with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: | ||
200 | |||
201 | bb.note("Updating entries") | ||
202 | index = 0 | ||
203 | url = d.getVar("NVDCVE_URL") | ||
204 | api_key = d.getVar("NVDCVE_API_KEY") or None | ||
205 | attempts = int(d.getVar("CVE_DB_UPDATE_ATTEMPTS")) | ||
206 | |||
207 | # Recommended by NVD | ||
208 | wait_time = 6 | ||
209 | if api_key: | ||
210 | wait_time = 2 | ||
211 | |||
212 | while True: | ||
213 | req_args['startIndex'] = index | ||
214 | raw_data = nvd_request_next(url, attempts, api_key, req_args, wait_time) | ||
215 | if raw_data is None: | ||
216 | # We haven't managed to download data | ||
217 | return False | ||
218 | |||
219 | data = json.loads(raw_data) | ||
220 | |||
221 | index = data["startIndex"] | ||
222 | total = data["totalResults"] | ||
223 | per_page = data["resultsPerPage"] | ||
224 | bb.note("Got %d entries" % per_page) | ||
225 | for cve in data["vulnerabilities"]: | ||
226 | update_db(conn, cve) | ||
227 | |||
228 | index += per_page | ||
229 | ph.update((float(index) / (total+1)) * 100) | ||
230 | if index >= total: | ||
231 | break | ||
232 | |||
233 | # Recommended by NVD | ||
234 | time.sleep(wait_time) | ||
235 | |||
236 | # Update success, set the date to cve_check file. | ||
237 | cve_f.write('CVE database update : %s\n\n' % datetime.date.today()) | ||
238 | |||
239 | conn.commit() | ||
240 | conn.close() | ||
241 | return True | ||
242 | |||
243 | def initialize_db(conn): | ||
244 | with conn: | ||
245 | c = conn.cursor() | ||
246 | |||
247 | c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") | ||
248 | |||
249 | c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ | ||
250 | SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)") | ||
251 | |||
252 | c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ | ||
253 | VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ | ||
254 | VERSION_END TEXT, OPERATOR_END TEXT)") | ||
255 | c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") | ||
256 | |||
257 | c.close() | ||
258 | |||
259 | def parse_node_and_insert(conn, node, cveId): | ||
260 | |||
261 | def cpe_generator(): | ||
262 | for cpe in node.get('cpeMatch', ()): | ||
263 | if not cpe['vulnerable']: | ||
264 | return | ||
265 | cpe23 = cpe.get('criteria') | ||
266 | if not cpe23: | ||
267 | return | ||
268 | cpe23 = cpe23.split(':') | ||
269 | if len(cpe23) < 6: | ||
270 | return | ||
271 | vendor = cpe23[3] | ||
272 | product = cpe23[4] | ||
273 | version = cpe23[5] | ||
274 | |||
275 | if cpe23[6] == '*' or cpe23[6] == '-': | ||
276 | version_suffix = "" | ||
277 | else: | ||
278 | version_suffix = "_" + cpe23[6] | ||
279 | |||
280 | if version != '*' and version != '-': | ||
281 | # Version is defined, this is a '=' match | ||
282 | yield [cveId, vendor, product, version + version_suffix, '=', '', ''] | ||
283 | elif version == '-': | ||
284 | # no version information is available | ||
285 | yield [cveId, vendor, product, version, '', '', ''] | ||
286 | else: | ||
287 | # Parse start version, end version and operators | ||
288 | op_start = '' | ||
289 | op_end = '' | ||
290 | v_start = '' | ||
291 | v_end = '' | ||
292 | |||
293 | if 'versionStartIncluding' in cpe: | ||
294 | op_start = '>=' | ||
295 | v_start = cpe['versionStartIncluding'] | ||
296 | |||
297 | if 'versionStartExcluding' in cpe: | ||
298 | op_start = '>' | ||
299 | v_start = cpe['versionStartExcluding'] | ||
300 | |||
301 | if 'versionEndIncluding' in cpe: | ||
302 | op_end = '<=' | ||
303 | v_end = cpe['versionEndIncluding'] | ||
304 | |||
305 | if 'versionEndExcluding' in cpe: | ||
306 | op_end = '<' | ||
307 | v_end = cpe['versionEndExcluding'] | ||
308 | |||
309 | if op_start or op_end or v_start or v_end: | ||
310 | yield [cveId, vendor, product, v_start, op_start, v_end, op_end] | ||
311 | else: | ||
312 | # This is no version information, expressed differently. | ||
313 | # Save processing by representing as -. | ||
314 | yield [cveId, vendor, product, '-', '', '', ''] | ||
315 | |||
316 | conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close() | ||
317 | |||
318 | def update_db(conn, elt): | ||
319 | """ | ||
320 | Update a single entry in the on-disk database | ||
321 | """ | ||
322 | |||
323 | accessVector = None | ||
324 | cveId = elt['cve']['id'] | ||
325 | if elt['cve']['vulnStatus'] == "Rejected": | ||
326 | c = conn.cursor() | ||
327 | c.execute("delete from PRODUCTS where ID = ?;", [cveId]) | ||
328 | c.execute("delete from NVD where ID = ?;", [cveId]) | ||
329 | c.close() | ||
330 | return | ||
331 | cveDesc = "" | ||
332 | for desc in elt['cve']['descriptions']: | ||
333 | if desc['lang'] == 'en': | ||
334 | cveDesc = desc['value'] | ||
335 | date = elt['cve']['lastModified'] | ||
336 | try: | ||
337 | accessVector = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['accessVector'] | ||
338 | cvssv2 = elt['cve']['metrics']['cvssMetricV2'][0]['cvssData']['baseScore'] | ||
339 | except KeyError: | ||
340 | cvssv2 = 0.0 | ||
341 | cvssv3 = None | ||
342 | try: | ||
343 | accessVector = accessVector or elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['attackVector'] | ||
344 | cvssv3 = elt['cve']['metrics']['cvssMetricV30'][0]['cvssData']['baseScore'] | ||
345 | except KeyError: | ||
346 | pass | ||
347 | try: | ||
348 | accessVector = accessVector or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['attackVector'] | ||
349 | cvssv3 = cvssv3 or elt['cve']['metrics']['cvssMetricV31'][0]['cvssData']['baseScore'] | ||
350 | except KeyError: | ||
351 | pass | ||
352 | accessVector = accessVector or "UNKNOWN" | ||
353 | cvssv3 = cvssv3 or 0.0 | ||
354 | |||
355 | conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)", | ||
356 | [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]).close() | ||
357 | |||
358 | try: | ||
359 | # Remove any pre-existing CVE configuration. Even for partial database | ||
360 | # update, those will be repopulated. This ensures that old | ||
361 | # configuration is not kept for an updated CVE. | ||
362 | conn.execute("delete from PRODUCTS where ID = ?", [cveId]).close() | ||
363 | for config in elt['cve']['configurations']: | ||
364 | # This is suboptimal as it doesn't handle AND/OR and negate, but is better than nothing | ||
365 | for node in config["nodes"]: | ||
366 | parse_node_and_insert(conn, node, cveId) | ||
367 | except KeyError: | ||
368 | bb.note("CVE %s has no configurations" % cveId) | ||
369 | |||
370 | do_fetch[nostamp] = "1" | ||
371 | |||
372 | EXCLUDE_FROM_WORLD = "1" | ||