summaryrefslogtreecommitdiffstats
path: root/meta/recipes-core/meta/cve-update-db-native.bb
diff options
context:
space:
mode:
Diffstat (limited to 'meta/recipes-core/meta/cve-update-db-native.bb')
-rw-r--r--meta/recipes-core/meta/cve-update-db-native.bb181
1 files changed, 129 insertions, 52 deletions
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
index 9e8e006a32..efc32470d3 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -12,28 +12,76 @@ deltask do_compile
12deltask do_install 12deltask do_install
13deltask do_populate_sysroot 13deltask do_populate_sysroot
14 14
15# CVE database update interval, in seconds. By default: once a day (24*60*60).
16# Use 0 to force the update
17# Use a negative value to skip the update
18CVE_DB_UPDATE_INTERVAL ?= "86400"
19
20# Timeout for blocking socket operations, such as the connection attempt.
21CVE_SOCKET_TIMEOUT ?= "60"
22NVDCVE_URL ?= "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-"
23
24CVE_DB_TEMP_FILE ?= "${CVE_CHECK_DB_DIR}/temp_nvdcve_1.1.db"
25
15python () { 26python () {
16 if not bb.data.inherits_class("cve-check", d): 27 if not bb.data.inherits_class("cve-check", d):
17 raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.") 28 raise bb.parse.SkipRecipe("Skip recipe when cve-check class is not loaded.")
18} 29}
19 30
20python do_populate_cve_db() { 31python do_fetch() {
21 """ 32 """
22 Update NVD database with json data feed 33 Update NVD database with json data feed
23 """ 34 """
24 import bb.utils 35 import bb.utils
25 import bb.progress 36 import bb.progress
26 import sqlite3, urllib, urllib.parse, shutil, gzip 37 import shutil
27 from datetime import date
28 38
29 bb.utils.export_proxies(d) 39 bb.utils.export_proxies(d)
30 40
31 BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-"
32 YEAR_START = 2002
33
34 db_file = d.getVar("CVE_CHECK_DB_FILE") 41 db_file = d.getVar("CVE_CHECK_DB_FILE")
35 db_dir = os.path.dirname(db_file) 42 db_dir = os.path.dirname(db_file)
43 db_tmp_file = d.getVar("CVE_DB_TEMP_FILE")
44
45 cleanup_db_download(db_file, db_tmp_file)
46
47 # The NVD database changes once a day, so no need to update more frequently
48 # Allow the user to force-update
49 try:
50 import time
51 update_interval = int(d.getVar("CVE_DB_UPDATE_INTERVAL"))
52 if update_interval < 0:
53 bb.note("CVE database update skipped")
54 return
55 if time.time() - os.path.getmtime(db_file) < update_interval:
56 return
36 57
58 except OSError:
59 pass
60
61 bb.utils.mkdirhier(db_dir)
62 if os.path.exists(db_file):
63 shutil.copy2(db_file, db_tmp_file)
64
65 if update_db_file(db_tmp_file, d) == True:
66 # Update downloaded correctly, can swap files
67 shutil.move(db_tmp_file, db_file)
68 else:
69 # Update failed, do not modify the database
70 bb.note("CVE database update failed")
71 os.remove(db_tmp_file)
72}
73
74do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
75do_fetch[file-checksums] = ""
76do_fetch[vardeps] = ""
77
78def cleanup_db_download(db_file, db_tmp_file):
79 """
80 Cleanup the download space from possible failed downloads
81 """
82
83 # Clean up the updates done on the main file
84 # Remove it only if a journal file exists - it means a complete re-download
37 if os.path.exists("{0}-journal".format(db_file)): 85 if os.path.exists("{0}-journal".format(db_file)):
38 # If a journal is present the last update might have been interrupted. In that case, 86 # If a journal is present the last update might have been interrupted. In that case,
39 # just wipe any leftovers and force the DB to be recreated. 87 # just wipe any leftovers and force the DB to be recreated.
@@ -42,37 +90,50 @@ python do_populate_cve_db() {
42 if os.path.exists(db_file): 90 if os.path.exists(db_file):
43 os.remove(db_file) 91 os.remove(db_file)
44 92
45 # Don't refresh the database more than once an hour 93 # Clean-up the temporary file downloads, we can remove both journal
46 try: 94 # and the temporary database
47 import time 95 if os.path.exists("{0}-journal".format(db_tmp_file)):
48 if time.time() - os.path.getmtime(db_file) < (60*60): 96 # If a journal is present the last update might have been interrupted. In that case,
49 return 97 # just wipe any leftovers and force the DB to be recreated.
50 except OSError: 98 os.remove("{0}-journal".format(db_tmp_file))
51 pass
52 99
53 bb.utils.mkdirhier(db_dir) 100 if os.path.exists(db_tmp_file):
101 os.remove(db_tmp_file)
54 102
55 # Connect to database 103def update_db_file(db_tmp_file, d):
56 conn = sqlite3.connect(db_file) 104 """
57 c = conn.cursor() 105 Update the given database file
106 """
107 import bb.utils, bb.progress
108 from datetime import date
109 import urllib, gzip, sqlite3
58 110
59 initialize_db(c) 111 YEAR_START = 2002
112 cve_socket_timeout = int(d.getVar("CVE_SOCKET_TIMEOUT"))
113
114 # Connect to database
115 conn = sqlite3.connect(db_tmp_file)
116 initialize_db(conn)
60 117
61 with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: 118 with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f:
62 total_years = date.today().year + 1 - YEAR_START 119 total_years = date.today().year + 1 - YEAR_START
63 for i, year in enumerate(range(YEAR_START, date.today().year + 1)): 120 for i, year in enumerate(range(YEAR_START, date.today().year + 1)):
121 bb.debug(2, "Updating %d" % year)
64 ph.update((float(i + 1) / total_years) * 100) 122 ph.update((float(i + 1) / total_years) * 100)
65 year_url = BASE_URL + str(year) 123 year_url = (d.getVar('NVDCVE_URL')) + str(year)
66 meta_url = year_url + ".meta" 124 meta_url = year_url + ".meta"
67 json_url = year_url + ".json.gz" 125 json_url = year_url + ".json.gz"
68 126
69 # Retrieve meta last modified date 127 # Retrieve meta last modified date
70 try: 128 try:
71 response = urllib.request.urlopen(meta_url) 129 response = urllib.request.urlopen(meta_url, timeout=cve_socket_timeout)
72 except urllib.error.URLError as e: 130 except urllib.error.URLError as e:
73 cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n') 131 cve_f.write('Warning: CVE db update error, Unable to fetch CVE data.\n\n')
74 bb.warn("Failed to fetch CVE data (%s)" % e.reason) 132 bb.warn("Failed to fetch CVE data (%s)" % e)
75 return 133 import socket
134 result = socket.getaddrinfo("nvd.nist.gov", 443, proto=socket.IPPROTO_TCP)
135 bb.warn("Host IPs are %s" % (", ".join(t[4][0] for t in result)))
136 return False
76 137
77 if response: 138 if response:
78 for l in response.read().decode("utf-8").splitlines(): 139 for l in response.read().decode("utf-8").splitlines():
@@ -82,64 +143,81 @@ python do_populate_cve_db() {
82 break 143 break
83 else: 144 else:
84 bb.warn("Cannot parse CVE metadata, update failed") 145 bb.warn("Cannot parse CVE metadata, update failed")
85 return 146 return False
86 147
87 # Compare with current db last modified date 148 # Compare with current db last modified date
88 c.execute("select DATE from META where YEAR = ?", (year,)) 149 cursor = conn.execute("select DATE from META where YEAR = ?", (year,))
89 meta = c.fetchone() 150 meta = cursor.fetchone()
151 cursor.close()
152
90 if not meta or meta[0] != last_modified: 153 if not meta or meta[0] != last_modified:
154 bb.debug(2, "Updating entries")
91 # Clear products table entries corresponding to current year 155 # Clear products table entries corresponding to current year
92 c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,)) 156 conn.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,)).close()
93 157
94 # Update db with current year json file 158 # Update db with current year json file
95 try: 159 try:
96 response = urllib.request.urlopen(json_url) 160 response = urllib.request.urlopen(json_url, timeout=cve_socket_timeout)
97 if response: 161 if response:
98 update_db(c, gzip.decompress(response.read()).decode('utf-8')) 162 update_db(conn, gzip.decompress(response.read()).decode('utf-8'))
99 c.execute("insert or replace into META values (?, ?)", [year, last_modified]) 163 conn.execute("insert or replace into META values (?, ?)", [year, last_modified]).close()
100 except urllib.error.URLError as e: 164 except urllib.error.URLError as e:
101 cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') 165 cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
102 bb.warn("Cannot parse CVE data (%s), update failed" % e.reason) 166 bb.warn("Cannot parse CVE data (%s), update failed" % e.reason)
103 return 167 return False
104 168 else:
169 bb.debug(2, "Already up to date (last modified %s)" % last_modified)
105 # Update success, set the date to cve_check file. 170 # Update success, set the date to cve_check file.
106 if year == date.today().year: 171 if year == date.today().year:
107 cve_f.write('CVE database update : %s\n\n' % date.today()) 172 cve_f.write('CVE database update : %s\n\n' % date.today())
108 173
109 conn.commit() 174 conn.commit()
110 conn.close() 175 conn.close()
111} 176 return True
112 177
113do_populate_cve_db[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" 178def initialize_db(conn):
179 with conn:
180 c = conn.cursor()
114 181
115def initialize_db(c): 182 c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
116 c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
117 183
118 c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ 184 c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
119 SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)") 185 SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
120 186
121 c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ 187 c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
122 VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ 188 VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
123 VERSION_END TEXT, OPERATOR_END TEXT)") 189 VERSION_END TEXT, OPERATOR_END TEXT)")
124 c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") 190 c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")
125 191
126def parse_node_and_insert(c, node, cveId): 192 c.close()
193
194def parse_node_and_insert(conn, node, cveId):
127 # Parse children node if needed 195 # Parse children node if needed
128 for child in node.get('children', ()): 196 for child in node.get('children', ()):
129 parse_node_and_insert(c, child, cveId) 197 parse_node_and_insert(conn, child, cveId)
130 198
131 def cpe_generator(): 199 def cpe_generator():
132 for cpe in node.get('cpe_match', ()): 200 for cpe in node.get('cpe_match', ()):
133 if not cpe['vulnerable']: 201 if not cpe['vulnerable']:
134 return 202 return
135 cpe23 = cpe['cpe23Uri'].split(':') 203 cpe23 = cpe.get('cpe23Uri')
204 if not cpe23:
205 return
206 cpe23 = cpe23.split(':')
207 if len(cpe23) < 6:
208 return
136 vendor = cpe23[3] 209 vendor = cpe23[3]
137 product = cpe23[4] 210 product = cpe23[4]
138 version = cpe23[5] 211 version = cpe23[5]
139 212
213 if cpe23[6] == '*' or cpe23[6] == '-':
214 version_suffix = ""
215 else:
216 version_suffix = "_" + cpe23[6]
217
140 if version != '*' and version != '-': 218 if version != '*' and version != '-':
141 # Version is defined, this is a '=' match 219 # Version is defined, this is a '=' match
142 yield [cveId, vendor, product, version, '=', '', ''] 220 yield [cveId, vendor, product, version + version_suffix, '=', '', '']
143 elif version == '-': 221 elif version == '-':
144 # no version information is available 222 # no version information is available
145 yield [cveId, vendor, product, version, '', '', ''] 223 yield [cveId, vendor, product, version, '', '', '']
@@ -173,9 +251,9 @@ def parse_node_and_insert(c, node, cveId):
173 # Save processing by representing as -. 251 # Save processing by representing as -.
174 yield [cveId, vendor, product, '-', '', '', ''] 252 yield [cveId, vendor, product, '-', '', '', '']
175 253
176 c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()) 254 conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close()
177 255
178def update_db(c, jsondata): 256def update_db(conn, jsondata):
179 import json 257 import json
180 root = json.loads(jsondata) 258 root = json.loads(jsondata)
181 259
@@ -199,15 +277,14 @@ def update_db(c, jsondata):
199 accessVector = accessVector or "UNKNOWN" 277 accessVector = accessVector or "UNKNOWN"
200 cvssv3 = 0.0 278 cvssv3 = 0.0
201 279
202 c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)", 280 conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
203 [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]) 281 [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]).close()
204 282
205 configurations = elt['configurations']['nodes'] 283 configurations = elt['configurations']['nodes']
206 for config in configurations: 284 for config in configurations:
207 parse_node_and_insert(c, config, cveId) 285 parse_node_and_insert(conn, config, cveId)
208 286
209 287
210addtask do_populate_cve_db before do_fetch 288do_fetch[nostamp] = "1"
211do_populate_cve_db[nostamp] = "1"
212 289
213EXCLUDE_FROM_WORLD = "1" 290EXCLUDE_FROM_WORLD = "1"