diff options
author | Ross Burton <ross.burton@arm.com> | 2022-08-26 18:35:47 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-09-02 15:58:25 +0100 |
commit | 3cca59a9bcba1cad84f788e0f6ae09bc00c00ed7 (patch) | |
tree | 51d705eb2ebc4d7a77e2c15b1a1bfe2d024c9cf1 /meta/recipes-core/meta/cve-update-db-native.bb | |
parent | c9342278d7571750b7c0b5b0aced6a117d43c749 (diff) | |
download | poky-3cca59a9bcba1cad84f788e0f6ae09bc00c00ed7.tar.gz |
cve-check: close cursors as soon as possible
We can have multiple processes reading the database at the same time, and
cursors only release their locks when they're garbage collected.
This might be the cause of random sqlite errors on the autobuilder, so
explicitly close the cursors when we're done with them.
(From OE-Core rev: 5d2e90e4a58217a943ec21140bc2ecdd4357a98a)
Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/recipes-core/meta/cve-update-db-native.bb')
-rw-r--r-- | meta/recipes-core/meta/cve-update-db-native.bb | 51 |
1 files changed, 28 insertions, 23 deletions
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb index 18af89b53e..944243fce9 100644 --- a/meta/recipes-core/meta/cve-update-db-native.bb +++ b/meta/recipes-core/meta/cve-update-db-native.bb | |||
@@ -66,9 +66,7 @@ python do_fetch() { | |||
66 | 66 | ||
67 | # Connect to database | 67 | # Connect to database |
68 | conn = sqlite3.connect(db_file) | 68 | conn = sqlite3.connect(db_file) |
69 | c = conn.cursor() | 69 | initialize_db(conn) |
70 | |||
71 | initialize_db(c) | ||
72 | 70 | ||
73 | with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: | 71 | with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f: |
74 | total_years = date.today().year + 1 - YEAR_START | 72 | total_years = date.today().year + 1 - YEAR_START |
@@ -98,19 +96,21 @@ python do_fetch() { | |||
98 | return | 96 | return |
99 | 97 | ||
100 | # Compare with current db last modified date | 98 | # Compare with current db last modified date |
101 | c.execute("select DATE from META where YEAR = ?", (year,)) | 99 | cursor = conn.execute("select DATE from META where YEAR = ?", (year,)) |
102 | meta = c.fetchone() | 100 | meta = cursor.fetchone() |
101 | cursor.close() | ||
102 | |||
103 | if not meta or meta[0] != last_modified: | 103 | if not meta or meta[0] != last_modified: |
104 | bb.debug(2, "Updating entries") | 104 | bb.debug(2, "Updating entries") |
105 | # Clear products table entries corresponding to current year | 105 | # Clear products table entries corresponding to current year |
106 | c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,)) | 106 | conn.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,)).close() |
107 | 107 | ||
108 | # Update db with current year json file | 108 | # Update db with current year json file |
109 | try: | 109 | try: |
110 | response = urllib.request.urlopen(json_url) | 110 | response = urllib.request.urlopen(json_url) |
111 | if response: | 111 | if response: |
112 | update_db(c, gzip.decompress(response.read()).decode('utf-8')) | 112 | update_db(conn, gzip.decompress(response.read()).decode('utf-8')) |
113 | c.execute("insert or replace into META values (?, ?)", [year, last_modified]) | 113 | conn.execute("insert or replace into META values (?, ?)", [year, last_modified]).close() |
114 | except urllib.error.URLError as e: | 114 | except urllib.error.URLError as e: |
115 | cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') | 115 | cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n') |
116 | bb.warn("Cannot parse CVE data (%s), update failed" % e.reason) | 116 | bb.warn("Cannot parse CVE data (%s), update failed" % e.reason) |
@@ -129,21 +129,26 @@ do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}" | |||
129 | do_fetch[file-checksums] = "" | 129 | do_fetch[file-checksums] = "" |
130 | do_fetch[vardeps] = "" | 130 | do_fetch[vardeps] = "" |
131 | 131 | ||
132 | def initialize_db(c): | 132 | def initialize_db(conn): |
133 | c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") | 133 | with conn: |
134 | c = conn.cursor() | ||
135 | |||
136 | c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)") | ||
137 | |||
138 | c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ | ||
139 | SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)") | ||
134 | 140 | ||
135 | c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \ | 141 | c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ |
136 | SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)") | 142 | VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ |
143 | VERSION_END TEXT, OPERATOR_END TEXT)") | ||
144 | c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") | ||
137 | 145 | ||
138 | c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \ | 146 | c.close() |
139 | VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \ | ||
140 | VERSION_END TEXT, OPERATOR_END TEXT)") | ||
141 | c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);") | ||
142 | 147 | ||
143 | def parse_node_and_insert(c, node, cveId): | 148 | def parse_node_and_insert(conn, node, cveId): |
144 | # Parse children node if needed | 149 | # Parse children node if needed |
145 | for child in node.get('children', ()): | 150 | for child in node.get('children', ()): |
146 | parse_node_and_insert(c, child, cveId) | 151 | parse_node_and_insert(conn, child, cveId) |
147 | 152 | ||
148 | def cpe_generator(): | 153 | def cpe_generator(): |
149 | for cpe in node.get('cpe_match', ()): | 154 | for cpe in node.get('cpe_match', ()): |
@@ -200,9 +205,9 @@ def parse_node_and_insert(c, node, cveId): | |||
200 | # Save processing by representing as -. | 205 | # Save processing by representing as -. |
201 | yield [cveId, vendor, product, '-', '', '', ''] | 206 | yield [cveId, vendor, product, '-', '', '', ''] |
202 | 207 | ||
203 | c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()) | 208 | conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close() |
204 | 209 | ||
205 | def update_db(c, jsondata): | 210 | def update_db(conn, jsondata): |
206 | import json | 211 | import json |
207 | root = json.loads(jsondata) | 212 | root = json.loads(jsondata) |
208 | 213 | ||
@@ -226,12 +231,12 @@ def update_db(c, jsondata): | |||
226 | accessVector = accessVector or "UNKNOWN" | 231 | accessVector = accessVector or "UNKNOWN" |
227 | cvssv3 = 0.0 | 232 | cvssv3 = 0.0 |
228 | 233 | ||
229 | c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)", | 234 | conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)", |
230 | [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]) | 235 | [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]).close() |
231 | 236 | ||
232 | configurations = elt['configurations']['nodes'] | 237 | configurations = elt['configurations']['nodes'] |
233 | for config in configurations: | 238 | for config in configurations: |
234 | parse_node_and_insert(c, config, cveId) | 239 | parse_node_and_insert(conn, config, cveId) |
235 | 240 | ||
236 | 241 | ||
237 | do_fetch[nostamp] = "1" | 242 | do_fetch[nostamp] = "1" |