diff options
Diffstat (limited to 'bitbake/lib/prserv')
-rw-r--r-- | bitbake/lib/prserv/__init__.py | 97 | ||||
-rw-r--r-- | bitbake/lib/prserv/client.py | 15 | ||||
-rw-r--r-- | bitbake/lib/prserv/db.py | 452 | ||||
-rw-r--r-- | bitbake/lib/prserv/serv.py | 140 | ||||
-rw-r--r-- | bitbake/lib/prserv/tests.py | 388 |
5 files changed, 793 insertions, 299 deletions
diff --git a/bitbake/lib/prserv/__init__.py b/bitbake/lib/prserv/__init__.py index 0e0aa34d0e..ffc5a40a28 100644 --- a/bitbake/lib/prserv/__init__.py +++ b/bitbake/lib/prserv/__init__.py | |||
@@ -4,17 +4,92 @@ | |||
4 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
5 | # | 5 | # |
6 | 6 | ||
7 | __version__ = "1.0.0" | ||
8 | 7 | ||
9 | import os, time | 8 | __version__ = "2.0.0" |
10 | import sys, logging | ||
11 | 9 | ||
12 | def init_logger(logfile, loglevel): | 10 | import logging |
13 | numeric_level = getattr(logging, loglevel.upper(), None) | 11 | logger = logging.getLogger("BitBake.PRserv") |
14 | if not isinstance(numeric_level, int): | ||
15 | raise ValueError("Invalid log level: %s" % loglevel) | ||
16 | FORMAT = "%(asctime)-15s %(message)s" | ||
17 | logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT) | ||
18 | 12 | ||
19 | class NotFoundError(Exception): | 13 | from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS |
20 | pass | 14 | |
15 | def create_server(addr, dbpath, upstream=None, read_only=False): | ||
16 | from . import serv | ||
17 | |||
18 | s = serv.PRServer(dbpath, upstream=upstream, read_only=read_only) | ||
19 | host, port = addr.split(":") | ||
20 | s.start_tcp_server(host, int(port)) | ||
21 | |||
22 | return s | ||
23 | |||
24 | def increase_revision(ver): | ||
25 | """Take a revision string such as "1" or "1.2.3" or even a number and increase its last number | ||
26 | This fails if the last number is not an integer""" | ||
27 | |||
28 | fields=str(ver).split('.') | ||
29 | last = fields[-1] | ||
30 | |||
31 | try: | ||
32 | val = int(last) | ||
33 | except Exception as e: | ||
34 | logger.critical("Unable to increase revision value %s: %s" % (ver, e)) | ||
35 | raise e | ||
36 | |||
37 | return ".".join(fields[0:-1] + [ str(val + 1) ]) | ||
38 | |||
39 | def _revision_greater_or_equal(rev1, rev2): | ||
40 | """Compares x.y.z revision numbers, using integer comparison | ||
41 | Returns True if rev1 is greater or equal to rev2""" | ||
42 | |||
43 | fields1 = rev1.split(".") | ||
44 | fields2 = rev2.split(".") | ||
45 | l1 = len(fields1) | ||
46 | l2 = len(fields2) | ||
47 | |||
48 | for i in range(l1): | ||
49 | val1 = int(fields1[i]) | ||
50 | if i < l2: | ||
51 | val2 = int(fields2[i]) | ||
52 | if val2 < val1: | ||
53 | return True | ||
54 | elif val2 > val1: | ||
55 | return False | ||
56 | else: | ||
57 | return True | ||
58 | return True | ||
59 | |||
60 | def revision_smaller(rev1, rev2): | ||
61 | """Compares x.y.z revision numbers, using integer comparison | ||
62 | Returns True if rev1 is strictly smaller than rev2""" | ||
63 | return not(_revision_greater_or_equal(rev1, rev2)) | ||
64 | |||
65 | def revision_greater(rev1, rev2): | ||
66 | """Compares x.y.z revision numbers, using integer comparison | ||
67 | Returns True if rev1 is strictly greater than rev2""" | ||
68 | return _revision_greater_or_equal(rev1, rev2) and (rev1 != rev2) | ||
69 | |||
70 | def create_client(addr): | ||
71 | from . import client | ||
72 | |||
73 | c = client.PRClient() | ||
74 | |||
75 | try: | ||
76 | (typ, a) = parse_address(addr) | ||
77 | c.connect_tcp(*a) | ||
78 | return c | ||
79 | except Exception as e: | ||
80 | c.close() | ||
81 | raise e | ||
82 | |||
83 | async def create_async_client(addr): | ||
84 | from . import client | ||
85 | |||
86 | c = client.PRAsyncClient() | ||
87 | |||
88 | try: | ||
89 | (typ, a) = parse_address(addr) | ||
90 | await c.connect_tcp(*a) | ||
91 | return c | ||
92 | |||
93 | except Exception as e: | ||
94 | await c.close() | ||
95 | raise e | ||
diff --git a/bitbake/lib/prserv/client.py b/bitbake/lib/prserv/client.py index 8471ee3046..9f5794c433 100644 --- a/bitbake/lib/prserv/client.py +++ b/bitbake/lib/prserv/client.py | |||
@@ -6,6 +6,7 @@ | |||
6 | 6 | ||
7 | import logging | 7 | import logging |
8 | import bb.asyncrpc | 8 | import bb.asyncrpc |
9 | from . import create_async_client | ||
9 | 10 | ||
10 | logger = logging.getLogger("BitBake.PRserv") | 11 | logger = logging.getLogger("BitBake.PRserv") |
11 | 12 | ||
@@ -13,16 +14,16 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): | |||
13 | def __init__(self): | 14 | def __init__(self): |
14 | super().__init__("PRSERVICE", "1.0", logger) | 15 | super().__init__("PRSERVICE", "1.0", logger) |
15 | 16 | ||
16 | async def getPR(self, version, pkgarch, checksum): | 17 | async def getPR(self, version, pkgarch, checksum, history=False): |
17 | response = await self.invoke( | 18 | response = await self.invoke( |
18 | {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} | 19 | {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}} |
19 | ) | 20 | ) |
20 | if response: | 21 | if response: |
21 | return response["value"] | 22 | return response["value"] |
22 | 23 | ||
23 | async def test_pr(self, version, pkgarch, checksum): | 24 | async def test_pr(self, version, pkgarch, checksum, history=False): |
24 | response = await self.invoke( | 25 | response = await self.invoke( |
25 | {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} | 26 | {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}} |
26 | ) | 27 | ) |
27 | if response: | 28 | if response: |
28 | return response["value"] | 29 | return response["value"] |
@@ -48,9 +49,9 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): | |||
48 | if response: | 49 | if response: |
49 | return response["value"] | 50 | return response["value"] |
50 | 51 | ||
51 | async def export(self, version, pkgarch, checksum, colinfo): | 52 | async def export(self, version, pkgarch, checksum, colinfo, history=False): |
52 | response = await self.invoke( | 53 | response = await self.invoke( |
53 | {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo}} | 54 | {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo, "history": history}} |
54 | ) | 55 | ) |
55 | if response: | 56 | if response: |
56 | return (response["metainfo"], response["datainfo"]) | 57 | return (response["metainfo"], response["datainfo"]) |
@@ -65,7 +66,7 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient): | |||
65 | class PRClient(bb.asyncrpc.Client): | 66 | class PRClient(bb.asyncrpc.Client): |
66 | def __init__(self): | 67 | def __init__(self): |
67 | super().__init__() | 68 | super().__init__() |
68 | self._add_methods("getPR", "test_pr", "test_package", "importone", "export", "is_readonly") | 69 | self._add_methods("getPR", "test_pr", "test_package", "max_package_pr", "importone", "export", "is_readonly") |
69 | 70 | ||
70 | def _get_async_client(self): | 71 | def _get_async_client(self): |
71 | return PRAsyncClient() | 72 | return PRAsyncClient() |
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py index eb41508198..2da493ddf5 100644 --- a/bitbake/lib/prserv/db.py +++ b/bitbake/lib/prserv/db.py | |||
@@ -8,19 +8,13 @@ import logging | |||
8 | import os.path | 8 | import os.path |
9 | import errno | 9 | import errno |
10 | import prserv | 10 | import prserv |
11 | import time | 11 | import sqlite3 |
12 | 12 | ||
13 | try: | 13 | from contextlib import closing |
14 | import sqlite3 | 14 | from . import increase_revision, revision_greater, revision_smaller |
15 | except ImportError: | ||
16 | from pysqlite2 import dbapi2 as sqlite3 | ||
17 | 15 | ||
18 | logger = logging.getLogger("BitBake.PRserv") | 16 | logger = logging.getLogger("BitBake.PRserv") |
19 | 17 | ||
20 | sqlversion = sqlite3.sqlite_version_info | ||
21 | if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): | ||
22 | raise Exception("sqlite3 version 3.3.0 or later is required.") | ||
23 | |||
24 | # | 18 | # |
25 | # "No History" mode - for a given query tuple (version, pkgarch, checksum), | 19 | # "No History" mode - for a given query tuple (version, pkgarch, checksum), |
26 | # the returned value will be the largest among all the values of the same | 20 | # the returned value will be the largest among all the values of the same |
@@ -29,287 +23,232 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3): | |||
29 | # "History" mode - Return a new higher value for previously unseen query | 23 | # "History" mode - Return a new higher value for previously unseen query |
30 | # tuple (version, pkgarch, checksum), otherwise return historical value. | 24 | # tuple (version, pkgarch, checksum), otherwise return historical value. |
31 | # Value can decrement if returning to a previous build. | 25 | # Value can decrement if returning to a previous build. |
32 | # | ||
33 | 26 | ||
34 | class PRTable(object): | 27 | class PRTable(object): |
35 | def __init__(self, conn, table, nohist, read_only): | 28 | def __init__(self, conn, table, read_only): |
36 | self.conn = conn | 29 | self.conn = conn |
37 | self.nohist = nohist | ||
38 | self.read_only = read_only | 30 | self.read_only = read_only |
39 | self.dirty = False | 31 | self.table = table |
40 | if nohist: | 32 | |
41 | self.table = "%s_nohist" % table | 33 | # Creating the table even if the server is read-only. |
42 | else: | 34 | # This avoids a race condition if a shared database |
43 | self.table = "%s_hist" % table | 35 | # is accessed by a read-only server first. |
44 | 36 | ||
45 | if self.read_only: | 37 | with closing(self.conn.cursor()) as cursor: |
46 | table_exists = self._execute( | 38 | cursor.execute("CREATE TABLE IF NOT EXISTS %s \ |
47 | "SELECT count(*) FROM sqlite_master \ | ||
48 | WHERE type='table' AND name='%s'" % (self.table)) | ||
49 | if not table_exists: | ||
50 | raise prserv.NotFoundError | ||
51 | else: | ||
52 | self._execute("CREATE TABLE IF NOT EXISTS %s \ | ||
53 | (version TEXT NOT NULL, \ | 39 | (version TEXT NOT NULL, \ |
54 | pkgarch TEXT NOT NULL, \ | 40 | pkgarch TEXT NOT NULL, \ |
55 | checksum TEXT NOT NULL, \ | 41 | checksum TEXT NOT NULL, \ |
56 | value INTEGER, \ | 42 | value TEXT, \ |
57 | PRIMARY KEY (version, pkgarch, checksum));" % self.table) | 43 | PRIMARY KEY (version, pkgarch, checksum, value));" % self.table) |
58 | |||
59 | def _execute(self, *query): | ||
60 | """Execute a query, waiting to acquire a lock if necessary""" | ||
61 | start = time.time() | ||
62 | end = start + 20 | ||
63 | while True: | ||
64 | try: | ||
65 | return self.conn.execute(*query) | ||
66 | except sqlite3.OperationalError as exc: | ||
67 | if "is locked" in str(exc) and end > time.time(): | ||
68 | continue | ||
69 | raise exc | ||
70 | |||
71 | def sync(self): | ||
72 | if not self.read_only: | ||
73 | self.conn.commit() | 44 | self.conn.commit() |
74 | self._execute("BEGIN EXCLUSIVE TRANSACTION") | ||
75 | 45 | ||
76 | def sync_if_dirty(self): | 46 | def _extremum_value(self, rows, is_max): |
77 | if self.dirty: | 47 | value = None |
78 | self.sync() | 48 | |
79 | self.dirty = False | 49 | for row in rows: |
50 | current_value = row[0] | ||
51 | if value is None: | ||
52 | value = current_value | ||
53 | else: | ||
54 | if is_max: | ||
55 | is_new_extremum = revision_greater(current_value, value) | ||
56 | else: | ||
57 | is_new_extremum = revision_smaller(current_value, value) | ||
58 | if is_new_extremum: | ||
59 | value = current_value | ||
60 | return value | ||
61 | |||
62 | def _max_value(self, rows): | ||
63 | return self._extremum_value(rows, True) | ||
64 | |||
65 | def _min_value(self, rows): | ||
66 | return self._extremum_value(rows, False) | ||
80 | 67 | ||
81 | def test_package(self, version, pkgarch): | 68 | def test_package(self, version, pkgarch): |
82 | """Returns whether the specified package version is found in the database for the specified architecture""" | 69 | """Returns whether the specified package version is found in the database for the specified architecture""" |
83 | 70 | ||
84 | # Just returns the value if found or None otherwise | 71 | # Just returns the value if found or None otherwise |
85 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table, | 72 | with closing(self.conn.cursor()) as cursor: |
86 | (version, pkgarch)) | 73 | data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table, |
87 | row=data.fetchone() | 74 | (version, pkgarch)) |
88 | if row is not None: | 75 | row=data.fetchone() |
89 | return True | 76 | if row is not None: |
90 | else: | 77 | return True |
91 | return False | 78 | else: |
79 | return False | ||
80 | |||
81 | def test_checksum_value(self, version, pkgarch, checksum, value): | ||
82 | """Returns whether the specified value is found in the database for the specified package, architecture and checksum""" | ||
83 | |||
84 | with closing(self.conn.cursor()) as cursor: | ||
85 | data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and checksum=? and value=?;" % self.table, | ||
86 | (version, pkgarch, checksum, value)) | ||
87 | row=data.fetchone() | ||
88 | if row is not None: | ||
89 | return True | ||
90 | else: | ||
91 | return False | ||
92 | 92 | ||
93 | def test_value(self, version, pkgarch, value): | 93 | def test_value(self, version, pkgarch, value): |
94 | """Returns whether the specified value is found in the database for the specified package and architecture""" | 94 | """Returns whether the specified value is found in the database for the specified package and architecture""" |
95 | 95 | ||
96 | # Just returns the value if found or None otherwise | 96 | # Just returns the value if found or None otherwise |
97 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table, | 97 | with closing(self.conn.cursor()) as cursor: |
98 | (version, pkgarch, value)) | 98 | data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table, |
99 | row=data.fetchone() | 99 | (version, pkgarch, value)) |
100 | if row is not None: | 100 | row=data.fetchone() |
101 | return True | 101 | if row is not None: |
102 | else: | 102 | return True |
103 | return False | 103 | else: |
104 | return False | ||
104 | 105 | ||
105 | def find_value(self, version, pkgarch, checksum): | 106 | |
107 | def find_package_max_value(self, version, pkgarch): | ||
108 | """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value""" | ||
109 | |||
110 | with closing(self.conn.cursor()) as cursor: | ||
111 | data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=?;" % (self.table), | ||
112 | (version, pkgarch)) | ||
113 | rows = data.fetchall() | ||
114 | value = self._max_value(rows) | ||
115 | return value | ||
116 | |||
117 | def find_value(self, version, pkgarch, checksum, history=False): | ||
106 | """Returns the value for the specified checksum if found or None otherwise.""" | 118 | """Returns the value for the specified checksum if found or None otherwise.""" |
107 | 119 | ||
108 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | 120 | if history: |
109 | (version, pkgarch, checksum)) | 121 | return self.find_min_value(version, pkgarch, checksum) |
110 | row=data.fetchone() | ||
111 | if row is not None: | ||
112 | return row[0] | ||
113 | else: | 122 | else: |
114 | return None | 123 | return self.find_max_value(version, pkgarch, checksum) |
115 | 124 | ||
116 | def find_max_value(self, version, pkgarch): | ||
117 | """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value""" | ||
118 | 125 | ||
119 | data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), | 126 | def _find_extremum_value(self, version, pkgarch, checksum, is_max): |
120 | (version, pkgarch)) | 127 | """Returns the maximum (if is_max is True) or minimum (if is_max is False) value |
121 | row = data.fetchone() | 128 | for (version, pkgarch, checksum), or None if not found. Doesn't create a new value""" |
122 | if row is not None: | ||
123 | return row[0] | ||
124 | else: | ||
125 | return None | ||
126 | |||
127 | def _get_value_hist(self, version, pkgarch, checksum): | ||
128 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | ||
129 | (version, pkgarch, checksum)) | ||
130 | row=data.fetchone() | ||
131 | if row is not None: | ||
132 | return row[0] | ||
133 | else: | ||
134 | #no value found, try to insert | ||
135 | if self.read_only: | ||
136 | data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), | ||
137 | (version, pkgarch)) | ||
138 | row = data.fetchone() | ||
139 | if row is not None: | ||
140 | return row[0] | ||
141 | else: | ||
142 | return 0 | ||
143 | 129 | ||
144 | try: | 130 | with closing(self.conn.cursor()) as cursor: |
145 | self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" | 131 | data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND checksum=?;" % (self.table), |
146 | % (self.table, self.table), | 132 | (version, pkgarch, checksum)) |
147 | (version, pkgarch, checksum, version, pkgarch)) | 133 | rows = data.fetchall() |
148 | except sqlite3.IntegrityError as exc: | 134 | return self._extremum_value(rows, is_max) |
149 | logger.error(str(exc)) | ||
150 | 135 | ||
151 | self.dirty = True | 136 | def find_max_value(self, version, pkgarch, checksum): |
137 | return self._find_extremum_value(version, pkgarch, checksum, True) | ||
152 | 138 | ||
153 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | 139 | def find_min_value(self, version, pkgarch, checksum): |
154 | (version, pkgarch, checksum)) | 140 | return self._find_extremum_value(version, pkgarch, checksum, False) |
155 | row=data.fetchone() | 141 | |
156 | if row is not None: | 142 | def find_new_subvalue(self, version, pkgarch, base): |
157 | return row[0] | 143 | """Take and increase the greatest "<base>.y" value for (version, pkgarch), or return "<base>.0" if not found. |
158 | else: | 144 | This doesn't store a new value.""" |
159 | raise prserv.NotFoundError | 145 | |
160 | 146 | with closing(self.conn.cursor()) as cursor: | |
161 | def _get_value_no_hist(self, version, pkgarch, checksum): | 147 | data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (self.table, base), |
162 | data=self._execute("SELECT value FROM %s \ | 148 | (version, pkgarch)) |
163 | WHERE version=? AND pkgarch=? AND checksum=? AND \ | 149 | rows = data.fetchall() |
164 | value >= (select max(value) from %s where version=? AND pkgarch=?);" | 150 | value = self._max_value(rows) |
165 | % (self.table, self.table), | 151 | |
166 | (version, pkgarch, checksum, version, pkgarch)) | 152 | if value is not None: |
167 | row=data.fetchone() | 153 | return increase_revision(value) |
168 | if row is not None: | ||
169 | return row[0] | ||
170 | else: | ||
171 | #no value found, try to insert | ||
172 | if self.read_only: | ||
173 | data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table), | ||
174 | (version, pkgarch)) | ||
175 | return data.fetchone()[0] | ||
176 | |||
177 | try: | ||
178 | self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" | ||
179 | % (self.table, self.table), | ||
180 | (version, pkgarch, checksum, version, pkgarch)) | ||
181 | except sqlite3.IntegrityError as exc: | ||
182 | logger.error(str(exc)) | ||
183 | self.conn.rollback() | ||
184 | |||
185 | self.dirty = True | ||
186 | |||
187 | data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | ||
188 | (version, pkgarch, checksum)) | ||
189 | row=data.fetchone() | ||
190 | if row is not None: | ||
191 | return row[0] | ||
192 | else: | 154 | else: |
193 | raise prserv.NotFoundError | 155 | return base + ".0" |
194 | 156 | ||
195 | def get_value(self, version, pkgarch, checksum): | 157 | def store_value(self, version, pkgarch, checksum, value): |
196 | if self.nohist: | 158 | """Store value in the database""" |
197 | return self._get_value_no_hist(version, pkgarch, checksum) | 159 | |
198 | else: | 160 | if not self.read_only and not self.test_checksum_value(version, pkgarch, checksum, value): |
199 | return self._get_value_hist(version, pkgarch, checksum) | 161 | with closing(self.conn.cursor()) as cursor: |
200 | 162 | cursor.execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), | |
201 | def _import_hist(self, version, pkgarch, checksum, value): | ||
202 | if self.read_only: | ||
203 | return None | ||
204 | |||
205 | val = None | ||
206 | data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | ||
207 | (version, pkgarch, checksum)) | ||
208 | row = data.fetchone() | ||
209 | if row is not None: | ||
210 | val=row[0] | ||
211 | else: | ||
212 | #no value found, try to insert | ||
213 | try: | ||
214 | self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), | ||
215 | (version, pkgarch, checksum, value)) | 163 | (version, pkgarch, checksum, value)) |
216 | except sqlite3.IntegrityError as exc: | 164 | self.conn.commit() |
217 | logger.error(str(exc)) | ||
218 | 165 | ||
219 | self.dirty = True | 166 | def _get_value(self, version, pkgarch, checksum, history): |
220 | 167 | ||
221 | data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, | 168 | max_value = self.find_package_max_value(version, pkgarch) |
222 | (version, pkgarch, checksum)) | ||
223 | row = data.fetchone() | ||
224 | if row is not None: | ||
225 | val = row[0] | ||
226 | return val | ||
227 | 169 | ||
228 | def _import_no_hist(self, version, pkgarch, checksum, value): | 170 | if max_value is None: |
229 | if self.read_only: | 171 | # version, pkgarch completely unknown. Return initial value. |
230 | return None | 172 | return "0" |
231 | 173 | ||
232 | try: | 174 | value = self.find_value(version, pkgarch, checksum, history) |
233 | #try to insert | 175 | |
234 | self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), | 176 | if value is None: |
235 | (version, pkgarch, checksum, value)) | 177 | # version, pkgarch found but not checksum. Create a new value from the maximum one |
236 | except sqlite3.IntegrityError as exc: | 178 | return increase_revision(max_value) |
237 | #already have the record, try to update | 179 | |
238 | try: | 180 | if history: |
239 | self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" | 181 | return value |
240 | % (self.table), | 182 | |
241 | (value, version, pkgarch, checksum, value)) | 183 | # "no history" mode - If the value is not the maximum value for the package, need to increase it. |
242 | except sqlite3.IntegrityError as exc: | 184 | if max_value > value: |
243 | logger.error(str(exc)) | 185 | return increase_revision(max_value) |
244 | |||
245 | self.dirty = True | ||
246 | |||
247 | data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table, | ||
248 | (version, pkgarch, checksum, value)) | ||
249 | row=data.fetchone() | ||
250 | if row is not None: | ||
251 | return row[0] | ||
252 | else: | 186 | else: |
253 | return None | 187 | return value |
188 | |||
189 | def get_value(self, version, pkgarch, checksum, history): | ||
190 | value = self._get_value(version, pkgarch, checksum, history) | ||
191 | if not self.read_only: | ||
192 | self.store_value(version, pkgarch, checksum, value) | ||
193 | return value | ||
254 | 194 | ||
255 | def importone(self, version, pkgarch, checksum, value): | 195 | def importone(self, version, pkgarch, checksum, value): |
256 | if self.nohist: | 196 | self.store_value(version, pkgarch, checksum, value) |
257 | return self._import_no_hist(version, pkgarch, checksum, value) | 197 | return value |
258 | else: | ||
259 | return self._import_hist(version, pkgarch, checksum, value) | ||
260 | 198 | ||
261 | def export(self, version, pkgarch, checksum, colinfo): | 199 | def export(self, version, pkgarch, checksum, colinfo, history=False): |
262 | metainfo = {} | 200 | metainfo = {} |
263 | #column info | 201 | with closing(self.conn.cursor()) as cursor: |
264 | if colinfo: | 202 | #column info |
265 | metainfo["tbl_name"] = self.table | 203 | if colinfo: |
266 | metainfo["core_ver"] = prserv.__version__ | 204 | metainfo["tbl_name"] = self.table |
267 | metainfo["col_info"] = [] | 205 | metainfo["core_ver"] = prserv.__version__ |
268 | data = self._execute("PRAGMA table_info(%s);" % self.table) | 206 | metainfo["col_info"] = [] |
207 | data = cursor.execute("PRAGMA table_info(%s);" % self.table) | ||
208 | for row in data: | ||
209 | col = {} | ||
210 | col["name"] = row["name"] | ||
211 | col["type"] = row["type"] | ||
212 | col["notnull"] = row["notnull"] | ||
213 | col["dflt_value"] = row["dflt_value"] | ||
214 | col["pk"] = row["pk"] | ||
215 | metainfo["col_info"].append(col) | ||
216 | |||
217 | #data info | ||
218 | datainfo = [] | ||
219 | |||
220 | if history: | ||
221 | sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table | ||
222 | else: | ||
223 | sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ | ||
224 | (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \ | ||
225 | WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) | ||
226 | sqlarg = [] | ||
227 | where = "" | ||
228 | if version: | ||
229 | where += "AND T1.version=? " | ||
230 | sqlarg.append(str(version)) | ||
231 | if pkgarch: | ||
232 | where += "AND T1.pkgarch=? " | ||
233 | sqlarg.append(str(pkgarch)) | ||
234 | if checksum: | ||
235 | where += "AND T1.checksum=? " | ||
236 | sqlarg.append(str(checksum)) | ||
237 | |||
238 | sqlstmt += where + ";" | ||
239 | |||
240 | if len(sqlarg): | ||
241 | data = cursor.execute(sqlstmt, tuple(sqlarg)) | ||
242 | else: | ||
243 | data = cursor.execute(sqlstmt) | ||
269 | for row in data: | 244 | for row in data: |
270 | col = {} | 245 | if row["version"]: |
271 | col["name"] = row["name"] | 246 | col = {} |
272 | col["type"] = row["type"] | 247 | col["version"] = row["version"] |
273 | col["notnull"] = row["notnull"] | 248 | col["pkgarch"] = row["pkgarch"] |
274 | col["dflt_value"] = row["dflt_value"] | 249 | col["checksum"] = row["checksum"] |
275 | col["pk"] = row["pk"] | 250 | col["value"] = row["value"] |
276 | metainfo["col_info"].append(col) | 251 | datainfo.append(col) |
277 | |||
278 | #data info | ||
279 | datainfo = [] | ||
280 | |||
281 | if self.nohist: | ||
282 | sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \ | ||
283 | (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \ | ||
284 | WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table) | ||
285 | else: | ||
286 | sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table | ||
287 | sqlarg = [] | ||
288 | where = "" | ||
289 | if version: | ||
290 | where += "AND T1.version=? " | ||
291 | sqlarg.append(str(version)) | ||
292 | if pkgarch: | ||
293 | where += "AND T1.pkgarch=? " | ||
294 | sqlarg.append(str(pkgarch)) | ||
295 | if checksum: | ||
296 | where += "AND T1.checksum=? " | ||
297 | sqlarg.append(str(checksum)) | ||
298 | |||
299 | sqlstmt += where + ";" | ||
300 | |||
301 | if len(sqlarg): | ||
302 | data = self._execute(sqlstmt, tuple(sqlarg)) | ||
303 | else: | ||
304 | data = self._execute(sqlstmt) | ||
305 | for row in data: | ||
306 | if row["version"]: | ||
307 | col = {} | ||
308 | col["version"] = row["version"] | ||
309 | col["pkgarch"] = row["pkgarch"] | ||
310 | col["checksum"] = row["checksum"] | ||
311 | col["value"] = row["value"] | ||
312 | datainfo.append(col) | ||
313 | return (metainfo, datainfo) | 252 | return (metainfo, datainfo) |
314 | 253 | ||
315 | def dump_db(self, fd): | 254 | def dump_db(self, fd): |
@@ -322,9 +261,8 @@ class PRTable(object): | |||
322 | 261 | ||
323 | class PRData(object): | 262 | class PRData(object): |
324 | """Object representing the PR database""" | 263 | """Object representing the PR database""" |
325 | def __init__(self, filename, nohist=True, read_only=False): | 264 | def __init__(self, filename, read_only=False): |
326 | self.filename=os.path.abspath(filename) | 265 | self.filename=os.path.abspath(filename) |
327 | self.nohist=nohist | ||
328 | self.read_only = read_only | 266 | self.read_only = read_only |
329 | #build directory hierarchy | 267 | #build directory hierarchy |
330 | try: | 268 | try: |
@@ -334,14 +272,15 @@ class PRData(object): | |||
334 | raise e | 272 | raise e |
335 | uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") | 273 | uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") |
336 | logger.debug("Opening PRServ database '%s'" % (uri)) | 274 | logger.debug("Opening PRServ database '%s'" % (uri)) |
337 | self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) | 275 | self.connection=sqlite3.connect(uri, uri=True) |
338 | self.connection.row_factory=sqlite3.Row | 276 | self.connection.row_factory=sqlite3.Row |
339 | if not self.read_only: | 277 | self.connection.execute("PRAGMA synchronous = OFF;") |
340 | self.connection.execute("pragma synchronous = off;") | 278 | self.connection.execute("PRAGMA journal_mode = WAL;") |
341 | self.connection.execute("PRAGMA journal_mode = MEMORY;") | 279 | self.connection.commit() |
342 | self._tables={} | 280 | self._tables={} |
343 | 281 | ||
344 | def disconnect(self): | 282 | def disconnect(self): |
283 | self.connection.commit() | ||
345 | self.connection.close() | 284 | self.connection.close() |
346 | 285 | ||
347 | def __getitem__(self, tblname): | 286 | def __getitem__(self, tblname): |
@@ -351,7 +290,7 @@ class PRData(object): | |||
351 | if tblname in self._tables: | 290 | if tblname in self._tables: |
352 | return self._tables[tblname] | 291 | return self._tables[tblname] |
353 | else: | 292 | else: |
354 | tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) | 293 | tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.read_only) |
355 | return tableobj | 294 | return tableobj |
356 | 295 | ||
357 | def __delitem__(self, tblname): | 296 | def __delitem__(self, tblname): |
@@ -359,3 +298,4 @@ class PRData(object): | |||
359 | del self._tables[tblname] | 298 | del self._tables[tblname] |
360 | logger.info("drop table %s" % (tblname)) | 299 | logger.info("drop table %s" % (tblname)) |
361 | self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) | 300 | self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) |
301 | self.connection.commit() | ||
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py index dc4be5b620..e175886308 100644 --- a/bitbake/lib/prserv/serv.py +++ b/bitbake/lib/prserv/serv.py | |||
@@ -12,6 +12,7 @@ import sqlite3 | |||
12 | import prserv | 12 | import prserv |
13 | import prserv.db | 13 | import prserv.db |
14 | import errno | 14 | import errno |
15 | from . import create_async_client, revision_smaller, increase_revision | ||
15 | import bb.asyncrpc | 16 | import bb.asyncrpc |
16 | 17 | ||
17 | logger = logging.getLogger("BitBake.PRserv") | 18 | logger = logging.getLogger("BitBake.PRserv") |
@@ -41,18 +42,16 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): | |||
41 | try: | 42 | try: |
42 | return await super().dispatch_message(msg) | 43 | return await super().dispatch_message(msg) |
43 | except: | 44 | except: |
44 | self.server.table.sync() | ||
45 | raise | 45 | raise |
46 | else: | ||
47 | self.server.table.sync_if_dirty() | ||
48 | 46 | ||
49 | async def handle_test_pr(self, request): | 47 | async def handle_test_pr(self, request): |
50 | '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value''' | 48 | '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value''' |
51 | version = request["version"] | 49 | version = request["version"] |
52 | pkgarch = request["pkgarch"] | 50 | pkgarch = request["pkgarch"] |
53 | checksum = request["checksum"] | 51 | checksum = request["checksum"] |
52 | history = request["history"] | ||
54 | 53 | ||
55 | value = self.server.table.find_value(version, pkgarch, checksum) | 54 | value = self.server.table.find_value(version, pkgarch, checksum, history) |
56 | return {"value": value} | 55 | return {"value": value} |
57 | 56 | ||
58 | async def handle_test_package(self, request): | 57 | async def handle_test_package(self, request): |
@@ -68,22 +67,110 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): | |||
68 | version = request["version"] | 67 | version = request["version"] |
69 | pkgarch = request["pkgarch"] | 68 | pkgarch = request["pkgarch"] |
70 | 69 | ||
71 | value = self.server.table.find_max_value(version, pkgarch) | 70 | value = self.server.table.find_package_max_value(version, pkgarch) |
72 | return {"value": value} | 71 | return {"value": value} |
73 | 72 | ||
74 | async def handle_get_pr(self, request): | 73 | async def handle_get_pr(self, request): |
75 | version = request["version"] | 74 | version = request["version"] |
76 | pkgarch = request["pkgarch"] | 75 | pkgarch = request["pkgarch"] |
77 | checksum = request["checksum"] | 76 | checksum = request["checksum"] |
77 | history = request["history"] | ||
78 | 78 | ||
79 | response = None | 79 | if self.upstream_client is None: |
80 | try: | 80 | value = self.server.table.get_value(version, pkgarch, checksum, history) |
81 | value = self.server.table.get_value(version, pkgarch, checksum) | 81 | return {"value": value} |
82 | response = {"value": value} | ||
83 | except prserv.NotFoundError: | ||
84 | self.logger.error("failure storing value in database for (%s, %s)",version, checksum) | ||
85 | 82 | ||
86 | return response | 83 | # We have an upstream server. |
84 | # Check whether the local server already knows the requested configuration. | ||
85 | # If the configuration is a new one, the generated value we will add will | ||
86 | # depend on what's on the upstream server. That's why we're calling find_value() | ||
87 | # instead of get_value() directly. | ||
88 | |||
89 | value = self.server.table.find_value(version, pkgarch, checksum, history) | ||
90 | upstream_max = await self.upstream_client.max_package_pr(version, pkgarch) | ||
91 | |||
92 | if value is not None: | ||
93 | |||
94 | # The configuration is already known locally. | ||
95 | |||
96 | if history: | ||
97 | value = self.server.table.get_value(version, pkgarch, checksum, history) | ||
98 | else: | ||
99 | existing_value = value | ||
100 | # In "no history", we need to make sure the value doesn't decrease | ||
101 | # and is at least greater than the maximum upstream value | ||
102 | # and the maximum local value | ||
103 | |||
104 | local_max = self.server.table.find_package_max_value(version, pkgarch) | ||
105 | if revision_smaller(value, local_max): | ||
106 | value = increase_revision(local_max) | ||
107 | |||
108 | if revision_smaller(value, upstream_max): | ||
109 | # Ask upstream whether it knows the checksum | ||
110 | upstream_value = await self.upstream_client.test_pr(version, pkgarch, checksum) | ||
111 | if upstream_value is None: | ||
112 | # Upstream doesn't have our checksum, let create a new one | ||
113 | value = upstream_max + ".0" | ||
114 | else: | ||
115 | # Fine to take the same value as upstream | ||
116 | value = upstream_max | ||
117 | |||
118 | if not value == existing_value and not self.server.read_only: | ||
119 | self.server.table.store_value(version, pkgarch, checksum, value) | ||
120 | |||
121 | return {"value": value} | ||
122 | |||
123 | # The configuration is a new one for the local server | ||
124 | # Let's ask the upstream server whether it knows it | ||
125 | |||
126 | known_upstream = await self.upstream_client.test_package(version, pkgarch) | ||
127 | |||
128 | if not known_upstream: | ||
129 | |||
130 | # The package is not known upstream, must be a local-only package | ||
131 | # Let's compute the PR number using the local-only method | ||
132 | |||
133 | value = self.server.table.get_value(version, pkgarch, checksum, history) | ||
134 | return {"value": value} | ||
135 | |||
136 | # The package is known upstream, let's ask the upstream server | ||
137 | # whether it knows our new output hash | ||
138 | |||
139 | value = await self.upstream_client.test_pr(version, pkgarch, checksum) | ||
140 | |||
141 | if value is not None: | ||
142 | |||
143 | # Upstream knows this output hash, let's store it and use it too. | ||
144 | |||
145 | if not self.server.read_only: | ||
146 | self.server.table.store_value(version, pkgarch, checksum, value) | ||
147 | # If the local server is read only, won't be able to store the new | ||
148 | # value in the database and will have to keep asking the upstream server | ||
149 | return {"value": value} | ||
150 | |||
151 | # The output hash doesn't exist upstream, get the most recent number from upstream (x) | ||
152 | # Then, we want to have a new PR value for the local server: x.y | ||
153 | |||
154 | upstream_max = await self.upstream_client.max_package_pr(version, pkgarch) | ||
155 | # Here we know that the package is known upstream, so upstream_max can't be None | ||
156 | subvalue = self.server.table.find_new_subvalue(version, pkgarch, upstream_max) | ||
157 | |||
158 | if not self.server.read_only: | ||
159 | self.server.table.store_value(version, pkgarch, checksum, subvalue) | ||
160 | |||
161 | return {"value": subvalue} | ||
162 | |||
163 | async def process_requests(self): | ||
164 | if self.server.upstream is not None: | ||
165 | self.upstream_client = await create_async_client(self.server.upstream) | ||
166 | else: | ||
167 | self.upstream_client = None | ||
168 | |||
169 | try: | ||
170 | await super().process_requests() | ||
171 | finally: | ||
172 | if self.upstream_client is not None: | ||
173 | await self.upstream_client.close() | ||
87 | 174 | ||
88 | async def handle_import_one(self, request): | 175 | async def handle_import_one(self, request): |
89 | response = None | 176 | response = None |
@@ -104,9 +191,10 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): | |||
104 | pkgarch = request["pkgarch"] | 191 | pkgarch = request["pkgarch"] |
105 | checksum = request["checksum"] | 192 | checksum = request["checksum"] |
106 | colinfo = request["colinfo"] | 193 | colinfo = request["colinfo"] |
194 | history = request["history"] | ||
107 | 195 | ||
108 | try: | 196 | try: |
109 | (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) | 197 | (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo, history) |
110 | except sqlite3.Error as exc: | 198 | except sqlite3.Error as exc: |
111 | self.logger.error(str(exc)) | 199 | self.logger.error(str(exc)) |
112 | metainfo = datainfo = None | 200 | metainfo = datainfo = None |
@@ -117,11 +205,12 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection): | |||
117 | return {"readonly": self.server.read_only} | 205 | return {"readonly": self.server.read_only} |
118 | 206 | ||
119 | class PRServer(bb.asyncrpc.AsyncServer): | 207 | class PRServer(bb.asyncrpc.AsyncServer): |
120 | def __init__(self, dbfile, read_only=False): | 208 | def __init__(self, dbfile, read_only=False, upstream=None): |
121 | super().__init__(logger) | 209 | super().__init__(logger) |
122 | self.dbfile = dbfile | 210 | self.dbfile = dbfile |
123 | self.table = None | 211 | self.table = None |
124 | self.read_only = read_only | 212 | self.read_only = read_only |
213 | self.upstream = upstream | ||
125 | 214 | ||
126 | def accept_client(self, socket): | 215 | def accept_client(self, socket): |
127 | return PRServerClient(socket, self) | 216 | return PRServerClient(socket, self) |
@@ -134,27 +223,25 @@ class PRServer(bb.asyncrpc.AsyncServer): | |||
134 | self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % | 223 | self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % |
135 | (self.dbfile, self.address, str(os.getpid()))) | 224 | (self.dbfile, self.address, str(os.getpid()))) |
136 | 225 | ||
226 | if self.upstream is not None: | ||
227 | self.logger.info("And upstream PRServer: %s " % (self.upstream)) | ||
228 | |||
137 | return tasks | 229 | return tasks |
138 | 230 | ||
139 | async def stop(self): | 231 | async def stop(self): |
140 | self.table.sync_if_dirty() | ||
141 | self.db.disconnect() | 232 | self.db.disconnect() |
142 | await super().stop() | 233 | await super().stop() |
143 | 234 | ||
144 | def signal_handler(self): | ||
145 | super().signal_handler() | ||
146 | if self.table: | ||
147 | self.table.sync() | ||
148 | |||
149 | class PRServSingleton(object): | 235 | class PRServSingleton(object): |
150 | def __init__(self, dbfile, logfile, host, port): | 236 | def __init__(self, dbfile, logfile, host, port, upstream): |
151 | self.dbfile = dbfile | 237 | self.dbfile = dbfile |
152 | self.logfile = logfile | 238 | self.logfile = logfile |
153 | self.host = host | 239 | self.host = host |
154 | self.port = port | 240 | self.port = port |
241 | self.upstream = upstream | ||
155 | 242 | ||
156 | def start(self): | 243 | def start(self): |
157 | self.prserv = PRServer(self.dbfile) | 244 | self.prserv = PRServer(self.dbfile, upstream=self.upstream) |
158 | self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port) | 245 | self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port) |
159 | self.process = self.prserv.serve_as_process(log_level=logging.WARNING) | 246 | self.process = self.prserv.serve_as_process(log_level=logging.WARNING) |
160 | 247 | ||
@@ -233,7 +320,7 @@ def run_as_daemon(func, pidfile, logfile): | |||
233 | os.remove(pidfile) | 320 | os.remove(pidfile) |
234 | os._exit(0) | 321 | os._exit(0) |
235 | 322 | ||
236 | def start_daemon(dbfile, host, port, logfile, read_only=False): | 323 | def start_daemon(dbfile, host, port, logfile, read_only=False, upstream=None): |
237 | ip = socket.gethostbyname(host) | 324 | ip = socket.gethostbyname(host) |
238 | pidfile = PIDPREFIX % (ip, port) | 325 | pidfile = PIDPREFIX % (ip, port) |
239 | try: | 326 | try: |
@@ -249,7 +336,7 @@ def start_daemon(dbfile, host, port, logfile, read_only=False): | |||
249 | 336 | ||
250 | dbfile = os.path.abspath(dbfile) | 337 | dbfile = os.path.abspath(dbfile) |
251 | def daemon_main(): | 338 | def daemon_main(): |
252 | server = PRServer(dbfile, read_only=read_only) | 339 | server = PRServer(dbfile, read_only=read_only, upstream=upstream) |
253 | server.start_tcp_server(ip, port) | 340 | server.start_tcp_server(ip, port) |
254 | server.serve_forever() | 341 | server.serve_forever() |
255 | 342 | ||
@@ -336,6 +423,9 @@ def auto_start(d): | |||
336 | 423 | ||
337 | host = host_params[0].strip().lower() | 424 | host = host_params[0].strip().lower() |
338 | port = int(host_params[1]) | 425 | port = int(host_params[1]) |
426 | |||
427 | upstream = d.getVar("PRSERV_UPSTREAM") or None | ||
428 | |||
339 | if is_local_special(host, port): | 429 | if is_local_special(host, port): |
340 | import bb.utils | 430 | import bb.utils |
341 | cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE")) | 431 | cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE")) |
@@ -350,7 +440,7 @@ def auto_start(d): | |||
350 | auto_shutdown() | 440 | auto_shutdown() |
351 | if not singleton: | 441 | if not singleton: |
352 | bb.utils.mkdirhier(cachedir) | 442 | bb.utils.mkdirhier(cachedir) |
353 | singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port) | 443 | singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port, upstream) |
354 | singleton.start() | 444 | singleton.start() |
355 | if singleton: | 445 | if singleton: |
356 | host = singleton.host | 446 | host = singleton.host |
diff --git a/bitbake/lib/prserv/tests.py b/bitbake/lib/prserv/tests.py new file mode 100644 index 0000000000..df0c003003 --- /dev/null +++ b/bitbake/lib/prserv/tests.py | |||
@@ -0,0 +1,388 @@ | |||
1 | #! /usr/bin/env python3 | ||
2 | # | ||
3 | # Copyright (C) 2024 BitBake Contributors | ||
4 | # | ||
5 | # SPDX-License-Identifier: GPL-2.0-only | ||
6 | # | ||
7 | |||
8 | from . import create_server, create_client, increase_revision, revision_greater, revision_smaller, _revision_greater_or_equal | ||
9 | import prserv.db as db | ||
10 | from bb.asyncrpc import InvokeError | ||
11 | import logging | ||
12 | import os | ||
13 | import sys | ||
14 | import tempfile | ||
15 | import unittest | ||
16 | import socket | ||
17 | import subprocess | ||
18 | from pathlib import Path | ||
19 | |||
20 | THIS_DIR = Path(__file__).parent | ||
21 | BIN_DIR = THIS_DIR.parent.parent / "bin" | ||
22 | |||
23 | version = "dummy-1.0-r0" | ||
24 | pkgarch = "core2-64" | ||
25 | other_arch = "aarch64" | ||
26 | |||
27 | checksumX = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4f0" | ||
28 | checksum0 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a0" | ||
29 | checksum1 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a1" | ||
30 | checksum2 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a2" | ||
31 | checksum3 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a3" | ||
32 | checksum4 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a4" | ||
33 | checksum5 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a5" | ||
34 | checksum6 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a6" | ||
35 | checksum7 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a7" | ||
36 | checksum8 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a8" | ||
37 | checksum9 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a9" | ||
38 | checksum10 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4aa" | ||
39 | |||
40 | def server_prefunc(server, name): | ||
41 | logging.basicConfig(level=logging.DEBUG, filename='prserv-%s.log' % name, filemode='w', | ||
42 | format='%(levelname)s %(filename)s:%(lineno)d %(message)s') | ||
43 | server.logger.debug("Running server %s" % name) | ||
44 | sys.stdout = open('prserv-stdout-%s.log' % name, 'w') | ||
45 | sys.stderr = sys.stdout | ||
46 | |||
47 | class PRTestSetup(object): | ||
48 | |||
49 | def start_server(self, name, dbfile, upstream=None, read_only=False, prefunc=server_prefunc): | ||
50 | |||
51 | def cleanup_server(server): | ||
52 | if server.process.exitcode is not None: | ||
53 | return | ||
54 | server.process.terminate() | ||
55 | server.process.join() | ||
56 | |||
57 | server = create_server(socket.gethostbyname("localhost") + ":0", | ||
58 | dbfile, | ||
59 | upstream=upstream, | ||
60 | read_only=read_only) | ||
61 | |||
62 | server.serve_as_process(prefunc=prefunc, args=(name,)) | ||
63 | self.addCleanup(cleanup_server, server) | ||
64 | |||
65 | return server | ||
66 | |||
67 | def start_client(self, server_address): | ||
68 | def cleanup_client(client): | ||
69 | client.close() | ||
70 | |||
71 | client = create_client(server_address) | ||
72 | self.addCleanup(cleanup_client, client) | ||
73 | |||
74 | return client | ||
75 | |||
76 | class FunctionTests(unittest.TestCase): | ||
77 | |||
78 | def setUp(self): | ||
79 | self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv') | ||
80 | self.addCleanup(self.temp_dir.cleanup) | ||
81 | |||
82 | def test_increase_revision(self): | ||
83 | self.assertEqual(increase_revision("1"), "2") | ||
84 | self.assertEqual(increase_revision("1.0"), "1.1") | ||
85 | self.assertEqual(increase_revision("1.1.1"), "1.1.2") | ||
86 | self.assertEqual(increase_revision("1.1.1.3"), "1.1.1.4") | ||
87 | self.assertEqual(increase_revision("9"), "10") | ||
88 | self.assertEqual(increase_revision("1.9"), "1.10") | ||
89 | self.assertRaises(ValueError, increase_revision, "1.a") | ||
90 | self.assertRaises(ValueError, increase_revision, "1.") | ||
91 | self.assertRaises(ValueError, increase_revision, "") | ||
92 | |||
93 | def test_revision_greater_or_equal(self): | ||
94 | self.assertTrue(_revision_greater_or_equal("2", "2")) | ||
95 | self.assertTrue(_revision_greater_or_equal("2", "1")) | ||
96 | self.assertTrue(_revision_greater_or_equal("10", "2")) | ||
97 | self.assertTrue(_revision_greater_or_equal("1.10", "1.2")) | ||
98 | self.assertFalse(_revision_greater_or_equal("1.2", "1.10")) | ||
99 | self.assertTrue(_revision_greater_or_equal("1.10", "1")) | ||
100 | self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10")) | ||
101 | self.assertFalse(_revision_greater_or_equal("1.10.1", "1.10.2")) | ||
102 | self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10.1")) | ||
103 | self.assertTrue(_revision_greater_or_equal("1.10.1", "1")) | ||
104 | self.assertTrue(revision_greater("1.20", "1.3")) | ||
105 | self.assertTrue(revision_smaller("1.3", "1.20")) | ||
106 | |||
107 | # DB tests | ||
108 | |||
109 | def test_db(self): | ||
110 | dbfile = os.path.join(self.temp_dir.name, "testtable.sqlite3") | ||
111 | |||
112 | self.db = db.PRData(dbfile) | ||
113 | self.table = self.db["PRMAIN"] | ||
114 | |||
115 | self.table.store_value(version, pkgarch, checksum0, "0") | ||
116 | self.table.store_value(version, pkgarch, checksum1, "1") | ||
117 | # "No history" mode supports multiple PRs for the same checksum | ||
118 | self.table.store_value(version, pkgarch, checksum0, "2") | ||
119 | self.table.store_value(version, pkgarch, checksum2, "1.0") | ||
120 | |||
121 | self.assertTrue(self.table.test_package(version, pkgarch)) | ||
122 | self.assertFalse(self.table.test_package(version, other_arch)) | ||
123 | |||
124 | self.assertTrue(self.table.test_value(version, pkgarch, "0")) | ||
125 | self.assertTrue(self.table.test_value(version, pkgarch, "1")) | ||
126 | self.assertTrue(self.table.test_value(version, pkgarch, "2")) | ||
127 | |||
128 | self.assertEqual(self.table.find_package_max_value(version, pkgarch), "2") | ||
129 | |||
130 | self.assertEqual(self.table.find_min_value(version, pkgarch, checksum0), "0") | ||
131 | self.assertEqual(self.table.find_max_value(version, pkgarch, checksum0), "2") | ||
132 | |||
133 | # Test history modes | ||
134 | self.assertEqual(self.table.find_value(version, pkgarch, checksum0, True), "0") | ||
135 | self.assertEqual(self.table.find_value(version, pkgarch, checksum0, False), "2") | ||
136 | |||
137 | self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "3"), "3.0") | ||
138 | self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "1"), "1.1") | ||
139 | |||
140 | # Revision comparison tests | ||
141 | self.table.store_value(version, pkgarch, checksum1, "1.3") | ||
142 | self.table.store_value(version, pkgarch, checksum1, "1.20") | ||
143 | self.assertEqual(self.table.find_min_value(version, pkgarch, checksum1), "1") | ||
144 | self.assertEqual(self.table.find_max_value(version, pkgarch, checksum1), "1.20") | ||
145 | |||
146 | class PRBasicTests(PRTestSetup, unittest.TestCase): | ||
147 | |||
148 | def setUp(self): | ||
149 | self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv') | ||
150 | self.addCleanup(self.temp_dir.cleanup) | ||
151 | |||
152 | dbfile = os.path.join(self.temp_dir.name, "prtest-basic.sqlite3") | ||
153 | |||
154 | self.server1 = self.start_server("basic", dbfile) | ||
155 | self.client1 = self.start_client(self.server1.address) | ||
156 | |||
157 | def test_basic(self): | ||
158 | |||
159 | # Checks on non existing configuration | ||
160 | |||
161 | result = self.client1.test_pr(version, pkgarch, checksum0) | ||
162 | self.assertIsNone(result, "test_pr should return 'None' for a non existing PR") | ||
163 | |||
164 | result = self.client1.test_package(version, pkgarch) | ||
165 | self.assertFalse(result, "test_package should return 'False' for a non existing PR") | ||
166 | |||
167 | result = self.client1.max_package_pr(version, pkgarch) | ||
168 | self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR") | ||
169 | |||
170 | # Add a first configuration | ||
171 | |||
172 | result = self.client1.getPR(version, pkgarch, checksum0) | ||
173 | self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'") | ||
174 | |||
175 | result = self.client1.test_pr(version, pkgarch, checksum0) | ||
176 | self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR") | ||
177 | |||
178 | result = self.client1.test_package(version, pkgarch) | ||
179 | self.assertTrue(result, "test_package should return 'True' for an existing PR") | ||
180 | |||
181 | result = self.client1.max_package_pr(version, pkgarch) | ||
182 | self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series") | ||
183 | |||
184 | # Check that the same request gets the same value | ||
185 | |||
186 | result = self.client1.getPR(version, pkgarch, checksum0) | ||
187 | self.assertEqual(result, "0", "getPR: asking for the same PR a second time in a row should return the same value.") | ||
188 | |||
189 | # Add new configurations | ||
190 | |||
191 | result = self.client1.getPR(version, pkgarch, checksum1) | ||
192 | self.assertEqual(result, "1", "getPR: second PR of a package should be '1'") | ||
193 | |||
194 | result = self.client1.test_pr(version, pkgarch, checksum1) | ||
195 | self.assertEqual(result, "1", "test_pr should return '1' here, matching the result of getPR") | ||
196 | |||
197 | result = self.client1.max_package_pr(version, pkgarch) | ||
198 | self.assertEqual(result, "1", "max_package_pr should return '1' in the current test series") | ||
199 | |||
200 | result = self.client1.getPR(version, pkgarch, checksum2) | ||
201 | self.assertEqual(result, "2", "getPR: second PR of a package should be '2'") | ||
202 | |||
203 | result = self.client1.test_pr(version, pkgarch, checksum2) | ||
204 | self.assertEqual(result, "2", "test_pr should return '2' here, matching the result of getPR") | ||
205 | |||
206 | result = self.client1.max_package_pr(version, pkgarch) | ||
207 | self.assertEqual(result, "2", "max_package_pr should return '2' in the current test series") | ||
208 | |||
209 | result = self.client1.getPR(version, pkgarch, checksum3) | ||
210 | self.assertEqual(result, "3", "getPR: second PR of a package should be '3'") | ||
211 | |||
212 | result = self.client1.test_pr(version, pkgarch, checksum3) | ||
213 | self.assertEqual(result, "3", "test_pr should return '3' here, matching the result of getPR") | ||
214 | |||
215 | result = self.client1.max_package_pr(version, pkgarch) | ||
216 | self.assertEqual(result, "3", "max_package_pr should return '3' in the current test series") | ||
217 | |||
218 | # Ask again for the first configuration | ||
219 | |||
220 | result = self.client1.getPR(version, pkgarch, checksum0) | ||
221 | self.assertEqual(result, "4", "getPR: should return '4' in this configuration") | ||
222 | |||
223 | # Ask again with explicit "no history" mode | ||
224 | |||
225 | result = self.client1.getPR(version, pkgarch, checksum0, False) | ||
226 | self.assertEqual(result, "4", "getPR: should return '4' in this configuration") | ||
227 | |||
228 | # Ask again with explicit "history" mode. This should return the first recorded PR for checksum0 | ||
229 | |||
230 | result = self.client1.getPR(version, pkgarch, checksum0, True) | ||
231 | self.assertEqual(result, "0", "getPR: should return '0' in this configuration") | ||
232 | |||
233 | # Check again that another pkgarg resets the counters | ||
234 | |||
235 | result = self.client1.test_pr(version, other_arch, checksum0) | ||
236 | self.assertIsNone(result, "test_pr should return 'None' for a non existing PR") | ||
237 | |||
238 | result = self.client1.test_package(version, other_arch) | ||
239 | self.assertFalse(result, "test_package should return 'False' for a non existing PR") | ||
240 | |||
241 | result = self.client1.max_package_pr(version, other_arch) | ||
242 | self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR") | ||
243 | |||
244 | # Now add the configuration | ||
245 | |||
246 | result = self.client1.getPR(version, other_arch, checksum0) | ||
247 | self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'") | ||
248 | |||
249 | result = self.client1.test_pr(version, other_arch, checksum0) | ||
250 | self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR") | ||
251 | |||
252 | result = self.client1.test_package(version, other_arch) | ||
253 | self.assertTrue(result, "test_package should return 'True' for an existing PR") | ||
254 | |||
255 | result = self.client1.max_package_pr(version, other_arch) | ||
256 | self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series") | ||
257 | |||
258 | result = self.client1.is_readonly() | ||
259 | self.assertFalse(result, "Server should not be described as 'read-only'") | ||
260 | |||
261 | class PRUpstreamTests(PRTestSetup, unittest.TestCase): | ||
262 | |||
263 | def setUp(self): | ||
264 | |||
265 | self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv') | ||
266 | self.addCleanup(self.temp_dir.cleanup) | ||
267 | |||
268 | dbfile2 = os.path.join(self.temp_dir.name, "prtest-upstream2.sqlite3") | ||
269 | self.server2 = self.start_server("upstream2", dbfile2) | ||
270 | self.client2 = self.start_client(self.server2.address) | ||
271 | |||
272 | dbfile1 = os.path.join(self.temp_dir.name, "prtest-upstream1.sqlite3") | ||
273 | self.server1 = self.start_server("upstream1", dbfile1, upstream=self.server2.address) | ||
274 | self.client1 = self.start_client(self.server1.address) | ||
275 | |||
276 | dbfile0 = os.path.join(self.temp_dir.name, "prtest-local.sqlite3") | ||
277 | self.server0 = self.start_server("local", dbfile0, upstream=self.server1.address) | ||
278 | self.client0 = self.start_client(self.server0.address) | ||
279 | self.shared_db = dbfile0 | ||
280 | |||
281 | def test_upstream_and_readonly(self): | ||
282 | |||
283 | # For identical checksums, all servers should return the same PR | ||
284 | |||
285 | result = self.client2.getPR(version, pkgarch, checksum0) | ||
286 | self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'") | ||
287 | |||
288 | result = self.client1.getPR(version, pkgarch, checksum0) | ||
289 | self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)") | ||
290 | |||
291 | result = self.client0.getPR(version, pkgarch, checksum0) | ||
292 | self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)") | ||
293 | |||
294 | # Now introduce new checksums on server1 for, same version | ||
295 | |||
296 | result = self.client1.getPR(version, pkgarch, checksum1) | ||
297 | self.assertEqual(result, "0.0", "getPR: first PR of a package which has a different checksum upstream should be '0.0'") | ||
298 | |||
299 | result = self.client1.getPR(version, pkgarch, checksum2) | ||
300 | self.assertEqual(result, "0.1", "getPR: second PR of a package that has a different checksum upstream should be '0.1'") | ||
301 | |||
302 | # Now introduce checksums on server0 for, same version | ||
303 | |||
304 | result = self.client1.getPR(version, pkgarch, checksum1) | ||
305 | self.assertEqual(result, "0.2", "getPR: can't decrease for known PR") | ||
306 | |||
307 | result = self.client1.getPR(version, pkgarch, checksum2) | ||
308 | self.assertEqual(result, "0.3") | ||
309 | |||
310 | result = self.client1.max_package_pr(version, pkgarch) | ||
311 | self.assertEqual(result, "0.3") | ||
312 | |||
313 | result = self.client0.getPR(version, pkgarch, checksum3) | ||
314 | self.assertEqual(result, "0.3.0", "getPR: first PR of a package that doesn't exist upstream should be '0.3.0'") | ||
315 | |||
316 | result = self.client0.getPR(version, pkgarch, checksum4) | ||
317 | self.assertEqual(result, "0.3.1", "getPR: second PR of a package that doesn't exist upstream should be '0.3.1'") | ||
318 | |||
319 | result = self.client0.getPR(version, pkgarch, checksum3) | ||
320 | self.assertEqual(result, "0.3.2") | ||
321 | |||
322 | # More upstream updates | ||
323 | # Here, we assume no communication between server2 and server0. server2 only impacts server0 | ||
324 | # after impacting server1 | ||
325 | |||
326 | self.assertEqual(self.client2.getPR(version, pkgarch, checksum5), "1") | ||
327 | self.assertEqual(self.client1.getPR(version, pkgarch, checksum6), "1.0") | ||
328 | self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "1.1") | ||
329 | self.assertEqual(self.client0.getPR(version, pkgarch, checksum8), "1.1.0") | ||
330 | self.assertEqual(self.client0.getPR(version, pkgarch, checksum9), "1.1.1") | ||
331 | |||
332 | # "history" mode tests | ||
333 | |||
334 | self.assertEqual(self.client2.getPR(version, pkgarch, checksum0, True), "0") | ||
335 | self.assertEqual(self.client1.getPR(version, pkgarch, checksum2, True), "0.1") | ||
336 | self.assertEqual(self.client0.getPR(version, pkgarch, checksum3, True), "0.3.0") | ||
337 | |||
338 | # More "no history" mode tests | ||
339 | |||
340 | self.assertEqual(self.client2.getPR(version, pkgarch, checksum0), "2") | ||
341 | self.assertEqual(self.client1.getPR(version, pkgarch, checksum0), "2") # Same as upstream | ||
342 | self.assertEqual(self.client0.getPR(version, pkgarch, checksum0), "2") # Same as upstream | ||
343 | self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "3") # This could be surprising, but since the previous revision was "2", increasing it yields "3". | ||
344 | # We don't know how many upstream servers we have | ||
345 | # Start read-only server with server1 as upstream | ||
346 | self.server_ro = self.start_server("local-ro", self.shared_db, upstream=self.server1.address, read_only=True) | ||
347 | self.client_ro = self.start_client(self.server_ro.address) | ||
348 | |||
349 | self.assertTrue(self.client_ro.is_readonly(), "Database should be described as 'read-only'") | ||
350 | |||
351 | # Checks on non existing configurations | ||
352 | self.assertIsNone(self.client_ro.test_pr(version, pkgarch, checksumX)) | ||
353 | self.assertFalse(self.client_ro.test_package("unknown", pkgarch)) | ||
354 | |||
355 | # Look up existing configurations | ||
356 | self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0), "3") # "no history" mode | ||
357 | self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0, True), "0") # "history" mode | ||
358 | self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3), "3") | ||
359 | self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3, True), "0.3.0") | ||
360 | self.assertEqual(self.client_ro.max_package_pr(version, pkgarch), "2") # normal as "3" was never saved | ||
361 | |||
362 | # Try to insert a new value. Here this one is know upstream. | ||
363 | self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum7), "3") | ||
364 | # Try to insert a completely new value. As the max upstream value is already "3", it should be "3.0" | ||
365 | self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum10), "3.0") | ||
366 | # Same with another value which only exists in the upstream upstream server | ||
367 | # This time, as the upstream server doesn't know it, it will ask its upstream server. So that's a known one. | ||
368 | self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum9), "3") | ||
369 | |||
370 | class ScriptTests(unittest.TestCase): | ||
371 | |||
372 | def setUp(self): | ||
373 | |||
374 | self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv') | ||
375 | self.addCleanup(self.temp_dir.cleanup) | ||
376 | self.dbfile = os.path.join(self.temp_dir.name, "prtest.sqlite3") | ||
377 | |||
378 | def test_1_start_bitbake_prserv(self): | ||
379 | try: | ||
380 | subprocess.check_call([BIN_DIR / "bitbake-prserv", "--start", "-f", self.dbfile]) | ||
381 | except subprocess.CalledProcessError as e: | ||
382 | self.fail("Failed to start bitbake-prserv: %s" % e.returncode) | ||
383 | |||
384 | def test_2_stop_bitbake_prserv(self): | ||
385 | try: | ||
386 | subprocess.check_call([BIN_DIR / "bitbake-prserv", "--stop"]) | ||
387 | except subprocess.CalledProcessError as e: | ||
388 | self.fail("Failed to stop bitbake-prserv: %s" % e.returncode) | ||