1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
SUMMARY = "Updates the NVD CVE database"
LICENSE = "MIT"
INHIBIT_DEFAULT_DEPS = "1"
PACKAGES = ""
inherit nopackages
deltask do_unpack
deltask do_patch
deltask do_configure
deltask do_compile
deltask do_install
deltask do_populate_sysroot
python do_populate_cve_db() {
"""
Update NVD database with json data feed
"""
import sqlite3, urllib, shutil, gzip, re
from datetime import date
BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-"
YEAR_START = 2002
db_dir = d.getVar("DL_DIR") + '/CVE_CHECK'
db_file = db_dir + '/nvdcve_1.0.db'
json_tmpfile = db_dir + '/nvd.json.gz'
proxy = d.getVar("https_proxy")
cve_f = open(d.getVar("TMPDIR") + '/cve_check', 'a')
if not os.path.isdir(db_dir):
os.mkdir(db_dir)
# Connect to database
conn = sqlite3.connect(db_file)
c = conn.cursor()
initialize_db(c)
for year in range(YEAR_START, date.today().year + 1):
year_url = BASE_URL + str(year)
meta_url = year_url + ".meta"
json_url = year_url + ".json.gz"
# Retrieve meta last modified date
req = urllib.request.Request(meta_url)
if proxy:
req.set_proxy(proxy, 'https')
try:
with urllib.request.urlopen(req, timeout=1) as r:
date_line = str(r.read().splitlines()[0])
last_modified = re.search('lastModifiedDate:(.*)', date_line).group(1)
except:
cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
break
# Compare with current db last modified date
c.execute("select DATE from META where YEAR = ?", (year,))
meta = c.fetchone()
if not meta or meta[0] != last_modified:
# Clear products table entries corresponding to current year
cve_year = 'CVE-' + str(year) + '%'
c.execute("delete from PRODUCTS where ID like ?", (cve_year,))
# Update db with current year json file
req = urllib.request.Request(json_url)
if proxy:
req.set_proxy(proxy, 'https')
try:
with urllib.request.urlopen(req, timeout=1) as r, \
open(json_tmpfile, 'wb') as tmpfile:
shutil.copyfileobj(r, tmpfile)
except:
cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
break
with gzip.open(json_tmpfile, 'rt') as jsonfile:
update_db(c, jsonfile)
c.execute("insert or replace into META values (?, ?)",
[year, last_modified])
# Update success, set the date to cve_check file.
if year == date.today().year:
cve_f.write('CVE database update : %s\n\n' % date.today())
cve_f.close()
conn.commit()
conn.close()
}
def initialize_db(c):
c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
VERSION_END TEXT, OPERATOR_END TEXT)")
def insert_elt(c, db_values):
query = "insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)"
c.execute(query, db_values)
def parse_node_and_insert(c, node, cveId):
# Parse children node if needed
try:
for child in node['children']:
parse_node_and_insert(c, child, cveId)
except:
pass
# Exit if the cpe_match node does not exists
try:
cpe_match = node['cpe_match']
except:
return
for cpe in cpe_match:
if not cpe['vulnerable']:
return
cpe23 = cpe['cpe23Uri'].split(':')
vendor = cpe23[3]
product = cpe23[4]
version = cpe23[5]
if version != '*':
# Version is defined, this is a '=' match
db_values = [cveId, vendor, product, version, '=', '', '']
insert_elt(c, db_values)
else:
# Parse start version, end version and operators
op_start = ''
op_end = ''
v_start = ''
v_end = ''
try:
if cpe['versionStartIncluding']:
op_start = '>='
v_start = cpe['versionStartIncluding']
except:
pass
try:
if cpe['versionStartExcluding']:
op_start = '>'
v_start = cpe['versionStartExcluding']
except:
pass
try:
if cpe['versionEndIncluding']:
op_end = '<='
v_end = cpe['versionEndIncluding']
except:
pass
try:
if cpe['versionEndExcluding']:
op_end = '<'
v_end = cpe['versionEndExcluding']
except:
pass
db_values = [cveId, vendor, product, v_start, op_start, v_end, op_end]
insert_elt(c, db_values)
def update_db(c, json_filename):
import json
root = json.load(json_filename)
for elt in root['CVE_Items']:
if not elt['impact']:
continue
cveId = elt['cve']['CVE_data_meta']['ID']
cveDesc = elt['cve']['description']['description_data'][0]['value']
date = elt['lastModifiedDate']
accessVector = elt['impact']['baseMetricV2']['cvssV2']['accessVector']
cvssv2 = elt['impact']['baseMetricV2']['cvssV2']['baseScore']
try:
cvssv3 = elt['impact']['baseMetricV3']['cvssV3']['baseScore']
except:
cvssv3 = 0.0
c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
[cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
configurations = elt['configurations']['nodes']
for config in configurations:
parse_node_and_insert(c, config, cveId)
addtask do_populate_cve_db before do_fetch
do_populate_cve_db[nostamp] = "1"
EXCLUDE_FROM_WORLD = "1"
|
