cve-check: close cursors as soon as possible

We can have multiple processes reading the database at the same time, and
cursors only release their locks when they're garbage collected.

This might be the cause of random sqlite errors on the autobuilder, so
explicitly close the cursors when we're done with them.

(From OE-Core rev: 5d2e90e4a58217a943ec21140bc2ecdd4357a98a)

Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 37 insertions, 27 deletions

diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 5c8b512c11..4b4ea7893e 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -297,7 +297,8 @@ def check_cves(d, patched_cves):
             vendor = "%"
 
         # Find all relevant CVE IDs.
-        for cverow in conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor)):
+        cve_cursor = conn.execute("SELECT DISTINCT ID FROM PRODUCTS WHERE PRODUCT IS ? AND VENDOR LIKE ?", (product, vendor))
+        for cverow in cve_cursor:
             cve = cverow[0]
 
             if cve in cve_ignore:
@@ -316,7 +317,8 @@ def check_cves(d, patched_cves):
             vulnerable = False
             ignored = False
 
-            for row in conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor)):
+            product_cursor = conn.execute("SELECT * FROM PRODUCTS WHERE ID IS ? AND PRODUCT IS ? AND VENDOR LIKE ?", (cve, product, vendor))
+            for row in product_cursor:
                 (_, _, _, version_start, operator_start, version_end, operator_end) = row
                 #bb.debug(2, "Evaluating row " + str(row))
                 if cve in cve_ignore:
@@ -360,10 +362,12 @@ def check_cves(d, patched_cves):
                         bb.note("%s-%s is vulnerable to %s" % (pn, real_pv, cve))
                         cves_unpatched.append(cve)
                     break
+            product_cursor.close()
 
             if not vulnerable:
                 bb.note("%s-%s is not vulnerable to %s" % (pn, real_pv, cve))
                 patched_cves.add(cve)
+        cve_cursor.close()
 
         if not cves_in_product:
             bb.note("No CVE records found for product %s, pn %s" % (product, pn))
@@ -388,14 +392,15 @@ def get_cve_info(d, cves):
     conn = sqlite3.connect(db_file, uri=True)
 
     for cve in cves:
-        for row in conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,)):
+        cursor = conn.execute("SELECT * FROM NVD WHERE ID IS ?", (cve,))
+        for row in cursor:
             cve_data[row[0]] = {}
             cve_data[row[0]]["summary"] = row[1]
             cve_data[row[0]]["scorev2"] = row[2]
             cve_data[row[0]]["scorev3"] = row[3]
             cve_data[row[0]]["modified"] = row[4]
             cve_data[row[0]]["vector"] = row[5]
-
+        cursor.close()
     conn.close()
     return cve_data
 
diff --git a/meta/recipes-core/meta/cve-update-db-native.bb b/meta/recipes-core/meta/cve-update-db-native.bb
index 18af89b53e..944243fce9 100644
--- a/meta/recipes-core/meta/cve-update-db-native.bb
+++ b/meta/recipes-core/meta/cve-update-db-native.bb
@@ -66,9 +66,7 @@ python do_fetch() {
 
     # Connect to database
     conn = sqlite3.connect(db_file)
-    c = conn.cursor()
-
-    initialize_db(c)
+    initialize_db(conn)
 
     with bb.progress.ProgressHandler(d) as ph, open(os.path.join(d.getVar("TMPDIR"), 'cve_check'), 'a') as cve_f:
         total_years = date.today().year + 1 - YEAR_START
@@ -98,19 +96,21 @@ python do_fetch() {
                     return
 
             # Compare with current db last modified date
-            c.execute("select DATE from META where YEAR = ?", (year,))
-            meta = c.fetchone()
+            cursor = conn.execute("select DATE from META where YEAR = ?", (year,))
+            meta = cursor.fetchone()
+            cursor.close()
+
             if not meta or meta[0] != last_modified:
                 bb.debug(2, "Updating entries")
                 # Clear products table entries corresponding to current year
-                c.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,))
+                conn.execute("delete from PRODUCTS where ID like ?", ('CVE-%d%%' % year,)).close()
 
                 # Update db with current year json file
                 try:
                     response = urllib.request.urlopen(json_url)
                     if response:
-                        update_db(c, gzip.decompress(response.read()).decode('utf-8'))
-                    c.execute("insert or replace into META values (?, ?)", [year, last_modified])
+                        update_db(conn, gzip.decompress(response.read()).decode('utf-8'))
+                    conn.execute("insert or replace into META values (?, ?)", [year, last_modified]).close()
                 except urllib.error.URLError as e:
                     cve_f.write('Warning: CVE db update error, CVE data is outdated.\n\n')
                     bb.warn("Cannot parse CVE data (%s), update failed" % e.reason)
@@ -129,21 +129,26 @@ do_fetch[lockfiles] += "${CVE_CHECK_DB_FILE_LOCK}"
 do_fetch[file-checksums] = ""
 do_fetch[vardeps] = ""
 
-def initialize_db(c):
-    c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
+def initialize_db(conn):
+    with conn:
+        c = conn.cursor()
+
+        c.execute("CREATE TABLE IF NOT EXISTS META (YEAR INTEGER UNIQUE, DATE TEXT)")
+
+        c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
+            SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
 
-    c.execute("CREATE TABLE IF NOT EXISTS NVD (ID TEXT UNIQUE, SUMMARY TEXT, \
-        SCOREV2 TEXT, SCOREV3 TEXT, MODIFIED INTEGER, VECTOR TEXT)")
+        c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
+            VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
+            VERSION_END TEXT, OPERATOR_END TEXT)")
+        c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")
 
-    c.execute("CREATE TABLE IF NOT EXISTS PRODUCTS (ID TEXT, \
-        VENDOR TEXT, PRODUCT TEXT, VERSION_START TEXT, OPERATOR_START TEXT, \
-        VERSION_END TEXT, OPERATOR_END TEXT)")
-    c.execute("CREATE INDEX IF NOT EXISTS PRODUCT_ID_IDX on PRODUCTS(ID);")
+        c.close()
 
-def parse_node_and_insert(c, node, cveId):
+def parse_node_and_insert(conn, node, cveId):
     # Parse children node if needed
     for child in node.get('children', ()):
-        parse_node_and_insert(c, child, cveId)
+        parse_node_and_insert(conn, child, cveId)
 
     def cpe_generator():
         for cpe in node.get('cpe_match', ()):
@@ -200,9 +205,9 @@ def parse_node_and_insert(c, node, cveId):
                     # Save processing by representing as -.
                     yield [cveId, vendor, product, '-', '', '', '']
 
-    c.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator())
+    conn.executemany("insert into PRODUCTS values (?, ?, ?, ?, ?, ?, ?)", cpe_generator()).close()
 
-def update_db(c, jsondata):
+def update_db(conn, jsondata):
     import json
     root = json.loads(jsondata)
 
@@ -226,12 +231,12 @@ def update_db(c, jsondata):
             accessVector = accessVector or "UNKNOWN"
             cvssv3 = 0.0
 
-        c.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
-                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector])
+        conn.execute("insert or replace into NVD values (?, ?, ?, ?, ?, ?)",
+                [cveId, cveDesc, cvssv2, cvssv3, date, accessVector]).close()
 
         configurations = elt['configurations']['nodes']
         for config in configurations:
-            parse_node_and_insert(c, config, cveId)
+            parse_node_and_insert(conn, config, cveId)
 
 
 do_fetch[nostamp] = "1"