diff options
Diffstat (limited to 'meta/recipes-core/meta/cve-update-db-native.bb')
-rw-r--r-- | meta/recipes-core/meta/cve-update-db-native.bb | 181 |
1 files changed, 129 insertions, 52 deletions
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb index 9e8e006a32..efc32470d3 100644 --- a/meta/recipes-core/meta/cve-update-db-native.bb +++ b/meta/recipes-core/meta/cve-update-db-native.bb | |||
@@ -12,28 +12,76 @@ deltask do_compile | |||
12 | deltask do_install | 12 | deltask do_install |
13 | deltask do_populate_sysroot | 13 | deltask do_populate_sysroot |
14 | 14 | ||
15 | # CVE database update interval, in seconds. By default: once a day (24*60*60). | ||
16 | # Use 0 to force the update | ||
17 | # Use a negative value to skip the update | ||
18 | CVE_DB_UPDATE_INTERVAL ?= "86400" | ||
19 | |||
20 | # Timeout for blocking socket operations, such as the connection attempt. | ||
21 | CVE_SOCKET_TIMEOUT ?= "60" | ||
22 | NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" | ||
23 | |||
24 | CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db" | ||
25 | |||
15 | python () { | 26 | python () { |
16 | if not bb.data.inherits_class("cve-check", d): | 27 | if not bb.data.inherits_class("cve-check", d): |
17 | raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.") | 28 | raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.") |
18 | } | 29 | } |
19 | 30 | ||
20 | python do_populate_cve_db() { | 31 | python do_fetch() { |
21 | """ | 32 | """ |
22 | Update NVD database with json data feed | 33 | Update NVD database with json data feed |
23 | """ | 34 | """ |
24 | import bb.utils | 35 | import bb.utils |
25 | import bb.progress | 36 | import bb.progress |
26 | import sqlite3, urllib, urllib.parse, shutil, gzip | 37 | import shutil |
27 | from datetime import date | ||
28 | 38 | ||
29 | bb.utils.export_proxies(d) | 39 | bb.utils.export_proxies(d) |
30 | 40 | ||
31 | BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-" | ||
32 | YEAR_START = 2002 | ||
33 | |||
34 | db_file = d.getVar("CVE_CHECK_DB_FILE") | 41 | db_file = d.getVar("CVE_CHECK_DB_FILE") |
35 | db_dir = os.path.dirname(db_file) | 42 | db_dir = os.path.dirname(db_file) |
43 | db_tmp_file = d.getVar("CVE_DB_TEMP_FILE") | ||
44 | |||
45 | cleanup_db_download(db_file, db_tmp_file) | ||
46 | |||
47 | # The NVD database changes once a day, so no need to update more frequently | ||
48 | # Allow the user to force-update | ||
49 | try: | ||
50 | import time | ||
51 | update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL")) | ||
52 | if update_interval < 0: | ||
53 | bb.note("CVE database update skipped") | ||
54 | return | ||
55 | if time.time() - os.path.getmtime(db_file) < update_interval: | ||
56 | return | ||
36 | 57 | ||
58 | except OSError: | ||
59 | pass | ||
60 | |||
61 | bb.utils.mkdirhier(db_dir) | ||
62 | if os.path.exists(db_file): | ||
63 | shutil.copy2(db_file, db_tmp_file) | ||
64 | |||
65 | if update_db_file(db_tmp_file, d) == True: | ||
66 | # Update downloaded correctly, can swap files | ||
67 | shutil.move(db_tmp_file, db_file) | ||
68 | else: | ||
69 | # Update failed, do not modify the database | ||
70 | bb.note("CVE database update failed") | ||
71 | os.remove(db_tmp_file) | ||
72 | } | ||
73 | |||
74 | do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" | ||
75 | do_fetch[file-checksums] = "" | ||
76 | do_fetch[vardeps] = "" | ||
77 | |||
78 | def cleanup_db_download(db_file, db_tmp_file): | ||
79 | """ | ||
80 | Cleanup the download space from possible failed downloads | ||
81 | """ | ||
82 | |||
83 | # Clean up the updates done on the main file | ||
84 | # Remove it only if a journal file exists - it means a complete re-download | ||
37 | if os.path.exists("{0}-journal".format(db_file)): | 85 | if os.path.exists("{0}-journal".format(db_file)): |
38 | # If a journal is present the last update might have been interrupted. In that case, | 86 | # If a journal is present the last update might have been interrupted. In that case, |
39 | # just wipe any leftovers and force the DB to be recreated. | 87 | # just wipe any leftovers and force the DB to be recreated. |
@@ -42,37 +90,50 @@ python do_populate_cve_db() { | |||
42 | if os.path.exists(db_file): | 90 | if os.path.exists(db_file): |
43 | os.remove(db_file) | 91 | os.remove(db_file) |
44 | 92 | ||
45 | # Don't refresh the database more than once an hour | 93 | # Clean-up the temporary file downloads, we can remove both journal |
46 | try: | 94 | # and the temporary database |
47 | import time | 95 | if os.path.exists("{0}-journal".format(db_tmp_file)): |
48 | if time.time() - os.path.getmtime(db_file) < (60*60): | 96 | # If a journal is present the last update might have been interrupted. In that case, |
49 | return | 97 | # just wipe any leftovers and force the DB to be recreated. |
50 | except OSError: | 98 | os.remove("{0}-journal".format(db_tmp_file)) |
51 | pass | ||
52 | 99 | ||
53 | bb.utils.mkdirhier(db_dir) | 100 | if os.path.exists(db_tmp_file): |
101 | os.remove(db_tmp_file) | ||
54 | 102 | ||
55 | # Connect to database | 103 | def update_db_file(db_tmp_file, d): |
56 | conn = sqlite3.connect(db_file) | 104 | """ |
57 | c = conn.cursor() | 105 | Update the given database file |
106 | """ | ||
107 | import bb.utils, bb.progress | ||
108 | from datetime import date | ||
109 | import urllib, gzip, sqlite3 | ||
58 | 110 | ||
59 | initialize_db(c) | 111 | YEAR_START = 2002 |
112 | cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT")) | ||
113 | |||
114 | # Connect to database | ||
115 | conn = sqlite3.connect(db_tmp_file) | ||
116 | initialize_db(conn) | ||
60 | 117 | ||
61 | with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: | 118 | with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: |
62 | total_years = date.today().year + 1 - YEAR_START | 119 | total_years = date.today().year + 1 - YEAR_START |
63 | for i, year in enumerate(range(YEAR_START, date.today().year + 1)): | 120 | for i, year in enumerate(range(YEAR_START, date.today().year + 1)): |
121 | bb.debug(2, "Updating %d" % year) | ||
64 | ph.update((float(i + 1) / total_years) * 100) | 122 | ph.update((float(i + 1) / total_years) * 100) |
65 | year_url = BASE_URL + str(year) | 123 | year_url = (d.getVar('NVDCVE_URL')) + str(year) |
66 | meta_url = year_url + ".meta" | 124 | meta_url = year_url + ".meta" |
67 | json_url = year_url + ".json.gz" | 125 | json_url = year_url + ".json.gz" |
68 | 126 | ||
69 | # Retrieve meta last modified date | 127 | # Retrieve meta last modified date |
70 | try: | 128 | try: |
71 | response = urllib.request.urlopen(meta_url) | 129 | response = urllib.request.urlopen(meta_url, timeout=cve_socket_timeout) |
72 | except urllib.error.URLError as e: | 130 | except urllib.error.URLError as e: |
73 | cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n') | 131 | cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n') |
74 | bb.warn("Failed to fetch CVE data (%s)" % e.reason) | 132 | bb.warn("Failed to fetch CVE data (%s)" % e) |
75 | return | 133 | import socket |
134 | result = socket.getaddrinfo("nvd.nist.gov", 443, proto=socket.IPPROTO_TCP) | ||
135 | bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t in result))) | ||
136 | return False | ||
76 | 137 | ||
77 | if response: | 138 | if response: |
78 | for l in response.read().decode("utf-8").splitlines(): | 139 | for l in response.read().decode("utf-8").splitlines(): |
@@ -82,64 +143,81 @@ python do_populate_cve_db() { | |||
82 | break | 143 | break |
83 | else: | 144 | else: |
84 | bb.warn("Cannot parse CVE metadata, update failed") | 145 | bb.warn("Cannot parse CVE metadata, update failed") |
85 | return | 146 | return False |
86 | 147 | ||
87 | # Compare with current db last modified date | 148 | # Compare with current db last modified date |
88 | c.execute("select DATE from META where YEAR = ?", (year,)) | 149 | cursor = conn.execute("select DATE from META where YEAR = ?", (year,)) |
89 | meta = c.fetchone() | 150 | meta = cursor.fetchone() |
151 | cursor.close() | ||
152 | |||
90 | if not meta or meta[0] != last_modified: | 153 | if not meta or meta[0] != last_modified: |
154 | bb.debug(2, "Updating entries") | ||
91 | # Clear products table entries corresponding to current year | 155 | # Clear products table entries corresponding to current year |
92 | c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,)) | 156 | conn.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,)).close() |
93 | 157 | ||
94 | # Update db with current year json file | 158 | # Update db with current year json file |
95 | try: | 159 | try: |
96 | response = urllib.request.urlopen(json_url) | 160 | response = urllib.request.urlopen(json_url, timeout=cve_socket_timeout) |
97 | if response: | 161 | if response: |
98 | update_db(c, gzip.decompress(response.read()).decode('utf-8')) | 162 | update_db(conn, gzip.decompress(response.read()).decode('utf-8')) |
99 | c.execute("insert or replace into META values (?, ?)", [year, last_modified]) | 163 | conn.execute("insert or replace into META values (?, ?)", [year, last_modified]).close() |
100 | except urllib.error.URLError as e: | 164 | except urllib.error.URLError as e: |
101 | cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') | 165 | cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') |
102 | bb.warn("Cannot parse CVE data (%s), update failed" % e.reason) | 166 | bb.warn("Cannot parse CVE data (%s), update failed" % e.reason) |
103 | return | 167 | return False |
104 | 168 | else: | |
169 | bb.debug(2, "Already up to date (last modified %s)" % last_modified) | ||
105 | # Update success, set the date to cve_check file. | 170 | # Update success, set the date to cve_check file. |
106 | if year == date.today().year: | 171 | if year == date.today().year: |
107 | cve_f.write('CVE database update : %s\n\n' % date.today()) | 172 | cve_f.write('CVE database update : %s\n\n' % date.today()) |
108 | 173 | ||
109 | conn.commit() | 174 | conn.commit() |
110 | conn.close() | 175 | conn.close() |
111 | } | 176 | return True |
112 | 177 | ||
113 | do_populate_cve_db[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" | 178 | def initialize_db(conn): |
179 | with conn: | ||
180 | c = conn.cursor() | ||
114 | 181 | ||
115 | def initialize_db(c): | 182 | c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") |
116 | c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") | ||
117 | 183 | ||
118 | c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ | 184 | c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ |
119 | SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)") | 185 | SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)") |
120 | 186 | ||
121 | c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ | 187 | c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ |
122 | VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ | 188 | VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ |
123 | VERSION_END TEXT, OPERATOR_END TEXT)") | 189 | VERSION_END TEXT, OPERATOR_END TEXT)") |
124 | c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") | 190 | c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") |
125 | 191 | ||
126 | def parse_node_and_insert(c, node, cveId): | 192 | c.close() |
193 | |||
194 | def parse_node_and_insert(conn, node, cveId): | ||
127 | # Parse children node if needed | 195 | # Parse children node if needed |
128 | for child in node.get('children', ()): | 196 | for child in node.get('children', ()): |
129 | parse_node_and_insert(c, child, cveId) | 197 | parse_node_and_insert(conn, child, cveId) |
130 | 198 | ||
131 | def cpe_generator(): | 199 | def cpe_generator(): |
132 | for cpe in node.get('cpe_match', ()): | 200 | for cpe in node.get('cpe_match', ()): |
133 | if not cpe['vulnerable']: | 201 | if not cpe['vulnerable']: |
134 | return | 202 | return |
135 | cpe23 = cpe['cpe23Uri'].split(':') | 203 | cpe23 = cpe.get('cpe23Uri') |
204 | if not cpe23: | ||
205 | return | ||
206 | cpe23 = cpe23.split(':') | ||
207 | if len(cpe23) < 6: | ||
208 | return | ||
136 | vendor = cpe23[3] | 209 | vendor = cpe23[3] |
137 | product = cpe23[4] | 210 | product = cpe23[4] |
138 | version = cpe23[5] | 211 | version = cpe23[5] |
139 | 212 | ||
213 | if cpe23[6] == '*' or cpe23[6] == '-': | ||
214 | version_suffix = "" | ||
215 | else: | ||
216 | version_suffix = "_" + cpe23[6] | ||
217 | |||
140 | if version != '*' and version != '-': | 218 | if version != '*' and version != '-': |
141 | # Version is defined, this is a '=' match | 219 | # Version is defined, this is a '=' match |
142 | yield [cveId, vendor, product, version, '=', '', ''] | 220 | yield [cveId, vendor, product, version + version_suffix, '=', '', ''] |
143 | elif version == '-': | 221 | elif version == '-': |
144 | # no version information is available | 222 | # no version information is available |
145 | yield [cveId, vendor, product, version, '', '', ''] | 223 | yield [cveId, vendor, product, version, '', '', ''] |
@@ -173,9 +251,9 @@ def parse_node_and_insert(c, node, cveId): | |||
173 | # Save processing by representing as -. | 251 | # Save processing by representing as -. |
174 | yield [cveId, vendor, product, '-', '', '', ''] | 252 | yield [cveId, vendor, product, '-', '', '', ''] |
175 | 253 | ||
176 | c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()) | 254 | conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close() |
177 | 255 | ||
178 | def update_db(c, jsondata): | 256 | def update_db(conn, jsondata): |
179 | import json | 257 | import json |
180 | root = json.loads(jsondata) | 258 | root = json.loads(jsondata) |
181 | 259 | ||
@@ -199,15 +277,14 @@ def update_db(c, jsondata): | |||
199 | accessVector = accessVector or "UNKNOWN" | 277 | accessVector = accessVector or "UNKNOWN" |
200 | cvssv3 = 0.0 | 278 | cvssv3 = 0.0 |
201 | 279 | ||
202 | c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)", | 280 | conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)", |
203 | [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]) | 281 | [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]).close() |
204 | 282 | ||
205 | configurations = elt['configurations']['nodes'] | 283 | configurations = elt['configurations']['nodes'] |
206 | for config in configurations: | 284 | for config in configurations: |
207 | parse_node_and_insert(c, config, cveId) | 285 | parse_node_and_insert(conn, config, cveId) |
208 | 286 | ||
209 | 287 | ||
210 | addtask do_populate_cve_db before do_fetch | 288 | do_fetch[nostamp] = "1" |
211 | do_populate_cve_db[nostamp] = "1" | ||
212 | 289 | ||
213 | EXCLUDE_FROM_WORLD = "1" | 290 | EXCLUDE_FROM_WORLD = "1" |