summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/prserv
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/prserv')
-rw-r--r--bitbake/lib/prserv/__init__.py97
-rw-r--r--bitbake/lib/prserv/client.py15
-rw-r--r--bitbake/lib/prserv/db.py452
-rw-r--r--bitbake/lib/prserv/serv.py140
-rw-r--r--bitbake/lib/prserv/tests.py388
5 files changed, 793 insertions, 299 deletions
diff --git a/bitbake/lib/prserv/__init__.py b/bitbake/lib/prserv/__init__.py
index 0e0aa34d0e..ffc5a40a28 100644
--- a/bitbake/lib/prserv/__init__.py
+++ b/bitbake/lib/prserv/__init__.py
@@ -4,17 +4,92 @@
4# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
5# 5#
6 6
7__version__ = "1.0.0"
8 7
9import os, time 8__version__ = "2.0.0"
10import sys, logging
11 9
12def init_logger(logfile, loglevel): 10import logging
13 numeric_level = getattr(logging, loglevel.upper(), None) 11logger = logging.getLogger("BitBake.PRserv")
14 if not isinstance(numeric_level, int):
15 raise ValueError("Invalid log level: %s" % loglevel)
16 FORMAT = "%(asctime)-15s %(message)s"
17 logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
18 12
19class NotFoundError(Exception): 13from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
20 pass 14
15def create_server(addr, dbpath, upstream=None, read_only=False):
16 from . import serv
17
18 s = serv.PRServer(dbpath, upstream=upstream, read_only=read_only)
19 host, port = addr.split(":")
20 s.start_tcp_server(host, int(port))
21
22 return s
23
24def increase_revision(ver):
25 """Take a revision string such as "1" or "1.2.3" or even a number and increase its last number
26 This fails if the last number is not an integer"""
27
28 fields=str(ver).split('.')
29 last = fields[-1]
30
31 try:
32 val = int(last)
33 except Exception as e:
34 logger.critical("Unable to increase revision value %s: %s" % (ver, e))
35 raise e
36
37 return ".".join(fields[0:-1] + [ str(val + 1) ])
38
39def _revision_greater_or_equal(rev1, rev2):
40 """Compares x.y.z revision numbers, using integer comparison
41 Returns True if rev1 is greater or equal to rev2"""
42
43 fields1 = rev1.split(".")
44 fields2 = rev2.split(".")
45 l1 = len(fields1)
46 l2 = len(fields2)
47
48 for i in range(l1):
49 val1 = int(fields1[i])
50 if i < l2:
51 val2 = int(fields2[i])
52 if val2 < val1:
53 return True
54 elif val2 > val1:
55 return False
56 else:
57 return True
58 return True
59
60def revision_smaller(rev1, rev2):
61 """Compares x.y.z revision numbers, using integer comparison
62 Returns True if rev1 is strictly smaller than rev2"""
63 return not(_revision_greater_or_equal(rev1, rev2))
64
65def revision_greater(rev1, rev2):
66 """Compares x.y.z revision numbers, using integer comparison
67 Returns True if rev1 is strictly greater than rev2"""
68 return _revision_greater_or_equal(rev1, rev2) and (rev1 != rev2)
69
70def create_client(addr):
71 from . import client
72
73 c = client.PRClient()
74
75 try:
76 (typ, a) = parse_address(addr)
77 c.connect_tcp(*a)
78 return c
79 except Exception as e:
80 c.close()
81 raise e
82
83async def create_async_client(addr):
84 from . import client
85
86 c = client.PRAsyncClient()
87
88 try:
89 (typ, a) = parse_address(addr)
90 await c.connect_tcp(*a)
91 return c
92
93 except Exception as e:
94 await c.close()
95 raise e
diff --git a/bitbake/lib/prserv/client.py b/bitbake/lib/prserv/client.py
index 8471ee3046..9f5794c433 100644
--- a/bitbake/lib/prserv/client.py
+++ b/bitbake/lib/prserv/client.py
@@ -6,6 +6,7 @@
6 6
7import logging 7import logging
8import bb.asyncrpc 8import bb.asyncrpc
9from . import create_async_client
9 10
10logger = logging.getLogger("BitBake.PRserv") 11logger = logging.getLogger("BitBake.PRserv")
11 12
@@ -13,16 +14,16 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient):
13 def __init__(self): 14 def __init__(self):
14 super().__init__("PRSERVICE", "1.0", logger) 15 super().__init__("PRSERVICE", "1.0", logger)
15 16
16 async def getPR(self, version, pkgarch, checksum): 17 async def getPR(self, version, pkgarch, checksum, history=False):
17 response = await self.invoke( 18 response = await self.invoke(
18 {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} 19 {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}}
19 ) 20 )
20 if response: 21 if response:
21 return response["value"] 22 return response["value"]
22 23
23 async def test_pr(self, version, pkgarch, checksum): 24 async def test_pr(self, version, pkgarch, checksum, history=False):
24 response = await self.invoke( 25 response = await self.invoke(
25 {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum}} 26 {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}}
26 ) 27 )
27 if response: 28 if response:
28 return response["value"] 29 return response["value"]
@@ -48,9 +49,9 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient):
48 if response: 49 if response:
49 return response["value"] 50 return response["value"]
50 51
51 async def export(self, version, pkgarch, checksum, colinfo): 52 async def export(self, version, pkgarch, checksum, colinfo, history=False):
52 response = await self.invoke( 53 response = await self.invoke(
53 {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo}} 54 {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo, "history": history}}
54 ) 55 )
55 if response: 56 if response:
56 return (response["metainfo"], response["datainfo"]) 57 return (response["metainfo"], response["datainfo"])
@@ -65,7 +66,7 @@ class PRAsyncClient(bb.asyncrpc.AsyncClient):
65class PRClient(bb.asyncrpc.Client): 66class PRClient(bb.asyncrpc.Client):
66 def __init__(self): 67 def __init__(self):
67 super().__init__() 68 super().__init__()
68 self._add_methods("getPR", "test_pr", "test_package", "importone", "export", "is_readonly") 69 self._add_methods("getPR", "test_pr", "test_package", "max_package_pr", "importone", "export", "is_readonly")
69 70
70 def _get_async_client(self): 71 def _get_async_client(self):
71 return PRAsyncClient() 72 return PRAsyncClient()
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
index eb41508198..2da493ddf5 100644
--- a/bitbake/lib/prserv/db.py
+++ b/bitbake/lib/prserv/db.py
@@ -8,19 +8,13 @@ import logging
8import os.path 8import os.path
9import errno 9import errno
10import prserv 10import prserv
11import time 11import sqlite3
12 12
13try: 13from contextlib import closing
14 import sqlite3 14from . import increase_revision, revision_greater, revision_smaller
15except ImportError:
16 from pysqlite2 import dbapi2 as sqlite3
17 15
18logger = logging.getLogger("BitBake.PRserv") 16logger = logging.getLogger("BitBake.PRserv")
19 17
20sqlversion = sqlite3.sqlite_version_info
21if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
22 raise Exception("sqlite3 version 3.3.0 or later is required.")
23
24# 18#
25# "No History" mode - for a given query tuple (version, pkgarch, checksum), 19# "No History" mode - for a given query tuple (version, pkgarch, checksum),
26# the returned value will be the largest among all the values of the same 20# the returned value will be the largest among all the values of the same
@@ -29,287 +23,232 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
29# "History" mode - Return a new higher value for previously unseen query 23# "History" mode - Return a new higher value for previously unseen query
30# tuple (version, pkgarch, checksum), otherwise return historical value. 24# tuple (version, pkgarch, checksum), otherwise return historical value.
31# Value can decrement if returning to a previous build. 25# Value can decrement if returning to a previous build.
32#
33 26
34class PRTable(object): 27class PRTable(object):
35 def __init__(self, conn, table, nohist, read_only): 28 def __init__(self, conn, table, read_only):
36 self.conn = conn 29 self.conn = conn
37 self.nohist = nohist
38 self.read_only = read_only 30 self.read_only = read_only
39 self.dirty = False 31 self.table = table
40 if nohist: 32
41 self.table = "%s_nohist" % table 33 # Creating the table even if the server is read-only.
42 else: 34 # This avoids a race condition if a shared database
43 self.table = "%s_hist" % table 35 # is accessed by a read-only server first.
44 36
45 if self.read_only: 37 with closing(self.conn.cursor()) as cursor:
46 table_exists = self._execute( 38 cursor.execute("CREATE TABLE IF NOT EXISTS %s \
47 "SELECT count(*) FROM sqlite_master \
48 WHERE type='table' AND name='%s'" % (self.table))
49 if not table_exists:
50 raise prserv.NotFoundError
51 else:
52 self._execute("CREATE TABLE IF NOT EXISTS %s \
53 (version TEXT NOT NULL, \ 39 (version TEXT NOT NULL, \
54 pkgarch TEXT NOT NULL, \ 40 pkgarch TEXT NOT NULL, \
55 checksum TEXT NOT NULL, \ 41 checksum TEXT NOT NULL, \
56 value INTEGER, \ 42 value TEXT, \
57 PRIMARY KEY (version, pkgarch, checksum));" % self.table) 43 PRIMARY KEY (version, pkgarch, checksum, value));" % self.table)
58
59 def _execute(self, *query):
60 """Execute a query, waiting to acquire a lock if necessary"""
61 start = time.time()
62 end = start + 20
63 while True:
64 try:
65 return self.conn.execute(*query)
66 except sqlite3.OperationalError as exc:
67 if "is locked" in str(exc) and end > time.time():
68 continue
69 raise exc
70
71 def sync(self):
72 if not self.read_only:
73 self.conn.commit() 44 self.conn.commit()
74 self._execute("BEGIN EXCLUSIVE TRANSACTION")
75 45
76 def sync_if_dirty(self): 46 def _extremum_value(self, rows, is_max):
77 if self.dirty: 47 value = None
78 self.sync() 48
79 self.dirty = False 49 for row in rows:
50 current_value = row[0]
51 if value is None:
52 value = current_value
53 else:
54 if is_max:
55 is_new_extremum = revision_greater(current_value, value)
56 else:
57 is_new_extremum = revision_smaller(current_value, value)
58 if is_new_extremum:
59 value = current_value
60 return value
61
62 def _max_value(self, rows):
63 return self._extremum_value(rows, True)
64
65 def _min_value(self, rows):
66 return self._extremum_value(rows, False)
80 67
81 def test_package(self, version, pkgarch): 68 def test_package(self, version, pkgarch):
82 """Returns whether the specified package version is found in the database for the specified architecture""" 69 """Returns whether the specified package version is found in the database for the specified architecture"""
83 70
84 # Just returns the value if found or None otherwise 71 # Just returns the value if found or None otherwise
85 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table, 72 with closing(self.conn.cursor()) as cursor:
86 (version, pkgarch)) 73 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table,
87 row=data.fetchone() 74 (version, pkgarch))
88 if row is not None: 75 row=data.fetchone()
89 return True 76 if row is not None:
90 else: 77 return True
91 return False 78 else:
79 return False
80
81 def test_checksum_value(self, version, pkgarch, checksum, value):
82 """Returns whether the specified value is found in the database for the specified package, architecture and checksum"""
83
84 with closing(self.conn.cursor()) as cursor:
85 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and checksum=? and value=?;" % self.table,
86 (version, pkgarch, checksum, value))
87 row=data.fetchone()
88 if row is not None:
89 return True
90 else:
91 return False
92 92
93 def test_value(self, version, pkgarch, value): 93 def test_value(self, version, pkgarch, value):
94 """Returns whether the specified value is found in the database for the specified package and architecture""" 94 """Returns whether the specified value is found in the database for the specified package and architecture"""
95 95
96 # Just returns the value if found or None otherwise 96 # Just returns the value if found or None otherwise
97 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table, 97 with closing(self.conn.cursor()) as cursor:
98 (version, pkgarch, value)) 98 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table,
99 row=data.fetchone() 99 (version, pkgarch, value))
100 if row is not None: 100 row=data.fetchone()
101 return True 101 if row is not None:
102 else: 102 return True
103 return False 103 else:
104 return False
104 105
105 def find_value(self, version, pkgarch, checksum): 106
107 def find_package_max_value(self, version, pkgarch):
108 """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
109
110 with closing(self.conn.cursor()) as cursor:
111 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=?;" % (self.table),
112 (version, pkgarch))
113 rows = data.fetchall()
114 value = self._max_value(rows)
115 return value
116
117 def find_value(self, version, pkgarch, checksum, history=False):
106 """Returns the value for the specified checksum if found or None otherwise.""" 118 """Returns the value for the specified checksum if found or None otherwise."""
107 119
108 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 120 if history:
109 (version, pkgarch, checksum)) 121 return self.find_min_value(version, pkgarch, checksum)
110 row=data.fetchone()
111 if row is not None:
112 return row[0]
113 else: 122 else:
114 return None 123 return self.find_max_value(version, pkgarch, checksum)
115 124
116 def find_max_value(self, version, pkgarch):
117 """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
118 125
119 data = self._execute("SELECT max(value) FROM %s where version=? AND pkgarch=?;" % (self.table), 126 def _find_extremum_value(self, version, pkgarch, checksum, is_max):
120 (version, pkgarch)) 127 """Returns the maximum (if is_max is True) or minimum (if is_max is False) value
121 row = data.fetchone() 128 for (version, pkgarch, checksum), or None if not found. Doesn't create a new value"""
122 if row is not None:
123 return row[0]
124 else:
125 return None
126
127 def _get_value_hist(self, version, pkgarch, checksum):
128 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
129 (version, pkgarch, checksum))
130 row=data.fetchone()
131 if row is not None:
132 return row[0]
133 else:
134 #no value found, try to insert
135 if self.read_only:
136 data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
137 (version, pkgarch))
138 row = data.fetchone()
139 if row is not None:
140 return row[0]
141 else:
142 return 0
143 129
144 try: 130 with closing(self.conn.cursor()) as cursor:
145 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));" 131 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND checksum=?;" % (self.table),
146 % (self.table, self.table), 132 (version, pkgarch, checksum))
147 (version, pkgarch, checksum, version, pkgarch)) 133 rows = data.fetchall()
148 except sqlite3.IntegrityError as exc: 134 return self._extremum_value(rows, is_max)
149 logger.error(str(exc))
150 135
151 self.dirty = True 136 def find_max_value(self, version, pkgarch, checksum):
137 return self._find_extremum_value(version, pkgarch, checksum, True)
152 138
153 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 139 def find_min_value(self, version, pkgarch, checksum):
154 (version, pkgarch, checksum)) 140 return self._find_extremum_value(version, pkgarch, checksum, False)
155 row=data.fetchone() 141
156 if row is not None: 142 def find_new_subvalue(self, version, pkgarch, base):
157 return row[0] 143 """Take and increase the greatest "<base>.y" value for (version, pkgarch), or return "<base>.0" if not found.
158 else: 144 This doesn't store a new value."""
159 raise prserv.NotFoundError 145
160 146 with closing(self.conn.cursor()) as cursor:
161 def _get_value_no_hist(self, version, pkgarch, checksum): 147 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (self.table, base),
162 data=self._execute("SELECT value FROM %s \ 148 (version, pkgarch))
163 WHERE version=? AND pkgarch=? AND checksum=? AND \ 149 rows = data.fetchall()
164 value >= (select max(value) from %s where version=? AND pkgarch=?);" 150 value = self._max_value(rows)
165 % (self.table, self.table), 151
166 (version, pkgarch, checksum, version, pkgarch)) 152 if value is not None:
167 row=data.fetchone() 153 return increase_revision(value)
168 if row is not None:
169 return row[0]
170 else:
171 #no value found, try to insert
172 if self.read_only:
173 data = self._execute("SELECT ifnull(max(value)+1, 0) FROM %s where version=? AND pkgarch=?;" % (self.table),
174 (version, pkgarch))
175 return data.fetchone()[0]
176
177 try:
178 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1, 0) from %s where version=? AND pkgarch=?));"
179 % (self.table, self.table),
180 (version, pkgarch, checksum, version, pkgarch))
181 except sqlite3.IntegrityError as exc:
182 logger.error(str(exc))
183 self.conn.rollback()
184
185 self.dirty = True
186
187 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
188 (version, pkgarch, checksum))
189 row=data.fetchone()
190 if row is not None:
191 return row[0]
192 else: 154 else:
193 raise prserv.NotFoundError 155 return base + ".0"
194 156
195 def get_value(self, version, pkgarch, checksum): 157 def store_value(self, version, pkgarch, checksum, value):
196 if self.nohist: 158 """Store value in the database"""
197 return self._get_value_no_hist(version, pkgarch, checksum) 159
198 else: 160 if not self.read_only and not self.test_checksum_value(version, pkgarch, checksum, value):
199 return self._get_value_hist(version, pkgarch, checksum) 161 with closing(self.conn.cursor()) as cursor:
200 162 cursor.execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
201 def _import_hist(self, version, pkgarch, checksum, value):
202 if self.read_only:
203 return None
204
205 val = None
206 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
207 (version, pkgarch, checksum))
208 row = data.fetchone()
209 if row is not None:
210 val=row[0]
211 else:
212 #no value found, try to insert
213 try:
214 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
215 (version, pkgarch, checksum, value)) 163 (version, pkgarch, checksum, value))
216 except sqlite3.IntegrityError as exc: 164 self.conn.commit()
217 logger.error(str(exc))
218 165
219 self.dirty = True 166 def _get_value(self, version, pkgarch, checksum, history):
220 167
221 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 168 max_value = self.find_package_max_value(version, pkgarch)
222 (version, pkgarch, checksum))
223 row = data.fetchone()
224 if row is not None:
225 val = row[0]
226 return val
227 169
228 def _import_no_hist(self, version, pkgarch, checksum, value): 170 if max_value is None:
229 if self.read_only: 171 # version, pkgarch completely unknown. Return initial value.
230 return None 172 return "0"
231 173
232 try: 174 value = self.find_value(version, pkgarch, checksum, history)
233 #try to insert 175
234 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 176 if value is None:
235 (version, pkgarch, checksum, value)) 177 # version, pkgarch found but not checksum. Create a new value from the maximum one
236 except sqlite3.IntegrityError as exc: 178 return increase_revision(max_value)
237 #already have the record, try to update 179
238 try: 180 if history:
239 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" 181 return value
240 % (self.table), 182
241 (value, version, pkgarch, checksum, value)) 183 # "no history" mode - If the value is not the maximum value for the package, need to increase it.
242 except sqlite3.IntegrityError as exc: 184 if max_value > value:
243 logger.error(str(exc)) 185 return increase_revision(max_value)
244
245 self.dirty = True
246
247 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
248 (version, pkgarch, checksum, value))
249 row=data.fetchone()
250 if row is not None:
251 return row[0]
252 else: 186 else:
253 return None 187 return value
188
189 def get_value(self, version, pkgarch, checksum, history):
190 value = self._get_value(version, pkgarch, checksum, history)
191 if not self.read_only:
192 self.store_value(version, pkgarch, checksum, value)
193 return value
254 194
255 def importone(self, version, pkgarch, checksum, value): 195 def importone(self, version, pkgarch, checksum, value):
256 if self.nohist: 196 self.store_value(version, pkgarch, checksum, value)
257 return self._import_no_hist(version, pkgarch, checksum, value) 197 return value
258 else:
259 return self._import_hist(version, pkgarch, checksum, value)
260 198
261 def export(self, version, pkgarch, checksum, colinfo): 199 def export(self, version, pkgarch, checksum, colinfo, history=False):
262 metainfo = {} 200 metainfo = {}
263 #column info 201 with closing(self.conn.cursor()) as cursor:
264 if colinfo: 202 #column info
265 metainfo["tbl_name"] = self.table 203 if colinfo:
266 metainfo["core_ver"] = prserv.__version__ 204 metainfo["tbl_name"] = self.table
267 metainfo["col_info"] = [] 205 metainfo["core_ver"] = prserv.__version__
268 data = self._execute("PRAGMA table_info(%s);" % self.table) 206 metainfo["col_info"] = []
207 data = cursor.execute("PRAGMA table_info(%s);" % self.table)
208 for row in data:
209 col = {}
210 col["name"] = row["name"]
211 col["type"] = row["type"]
212 col["notnull"] = row["notnull"]
213 col["dflt_value"] = row["dflt_value"]
214 col["pk"] = row["pk"]
215 metainfo["col_info"].append(col)
216
217 #data info
218 datainfo = []
219
220 if history:
221 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
222 else:
223 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
224 (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
225 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
226 sqlarg = []
227 where = ""
228 if version:
229 where += "AND T1.version=? "
230 sqlarg.append(str(version))
231 if pkgarch:
232 where += "AND T1.pkgarch=? "
233 sqlarg.append(str(pkgarch))
234 if checksum:
235 where += "AND T1.checksum=? "
236 sqlarg.append(str(checksum))
237
238 sqlstmt += where + ";"
239
240 if len(sqlarg):
241 data = cursor.execute(sqlstmt, tuple(sqlarg))
242 else:
243 data = cursor.execute(sqlstmt)
269 for row in data: 244 for row in data:
270 col = {} 245 if row["version"]:
271 col["name"] = row["name"] 246 col = {}
272 col["type"] = row["type"] 247 col["version"] = row["version"]
273 col["notnull"] = row["notnull"] 248 col["pkgarch"] = row["pkgarch"]
274 col["dflt_value"] = row["dflt_value"] 249 col["checksum"] = row["checksum"]
275 col["pk"] = row["pk"] 250 col["value"] = row["value"]
276 metainfo["col_info"].append(col) 251 datainfo.append(col)
277
278 #data info
279 datainfo = []
280
281 if self.nohist:
282 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
283 (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
284 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
285 else:
286 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
287 sqlarg = []
288 where = ""
289 if version:
290 where += "AND T1.version=? "
291 sqlarg.append(str(version))
292 if pkgarch:
293 where += "AND T1.pkgarch=? "
294 sqlarg.append(str(pkgarch))
295 if checksum:
296 where += "AND T1.checksum=? "
297 sqlarg.append(str(checksum))
298
299 sqlstmt += where + ";"
300
301 if len(sqlarg):
302 data = self._execute(sqlstmt, tuple(sqlarg))
303 else:
304 data = self._execute(sqlstmt)
305 for row in data:
306 if row["version"]:
307 col = {}
308 col["version"] = row["version"]
309 col["pkgarch"] = row["pkgarch"]
310 col["checksum"] = row["checksum"]
311 col["value"] = row["value"]
312 datainfo.append(col)
313 return (metainfo, datainfo) 252 return (metainfo, datainfo)
314 253
315 def dump_db(self, fd): 254 def dump_db(self, fd):
@@ -322,9 +261,8 @@ class PRTable(object):
322 261
323class PRData(object): 262class PRData(object):
324 """Object representing the PR database""" 263 """Object representing the PR database"""
325 def __init__(self, filename, nohist=True, read_only=False): 264 def __init__(self, filename, read_only=False):
326 self.filename=os.path.abspath(filename) 265 self.filename=os.path.abspath(filename)
327 self.nohist=nohist
328 self.read_only = read_only 266 self.read_only = read_only
329 #build directory hierarchy 267 #build directory hierarchy
330 try: 268 try:
@@ -334,14 +272,15 @@ class PRData(object):
334 raise e 272 raise e
335 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") 273 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "")
336 logger.debug("Opening PRServ database '%s'" % (uri)) 274 logger.debug("Opening PRServ database '%s'" % (uri))
337 self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) 275 self.connection=sqlite3.connect(uri, uri=True)
338 self.connection.row_factory=sqlite3.Row 276 self.connection.row_factory=sqlite3.Row
339 if not self.read_only: 277 self.connection.execute("PRAGMA synchronous = OFF;")
340 self.connection.execute("pragma synchronous = off;") 278 self.connection.execute("PRAGMA journal_mode = WAL;")
341 self.connection.execute("PRAGMA journal_mode = MEMORY;") 279 self.connection.commit()
342 self._tables={} 280 self._tables={}
343 281
344 def disconnect(self): 282 def disconnect(self):
283 self.connection.commit()
345 self.connection.close() 284 self.connection.close()
346 285
347 def __getitem__(self, tblname): 286 def __getitem__(self, tblname):
@@ -351,7 +290,7 @@ class PRData(object):
351 if tblname in self._tables: 290 if tblname in self._tables:
352 return self._tables[tblname] 291 return self._tables[tblname]
353 else: 292 else:
354 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) 293 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.read_only)
355 return tableobj 294 return tableobj
356 295
357 def __delitem__(self, tblname): 296 def __delitem__(self, tblname):
@@ -359,3 +298,4 @@ class PRData(object):
359 del self._tables[tblname] 298 del self._tables[tblname]
360 logger.info("drop table %s" % (tblname)) 299 logger.info("drop table %s" % (tblname))
361 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 300 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
301 self.connection.commit()
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index dc4be5b620..e175886308 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -12,6 +12,7 @@ import sqlite3
12import prserv 12import prserv
13import prserv.db 13import prserv.db
14import errno 14import errno
15from . import create_async_client, revision_smaller, increase_revision
15import bb.asyncrpc 16import bb.asyncrpc
16 17
17logger = logging.getLogger("BitBake.PRserv") 18logger = logging.getLogger("BitBake.PRserv")
@@ -41,18 +42,16 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection):
41 try: 42 try:
42 return await super().dispatch_message(msg) 43 return await super().dispatch_message(msg)
43 except: 44 except:
44 self.server.table.sync()
45 raise 45 raise
46 else:
47 self.server.table.sync_if_dirty()
48 46
49 async def handle_test_pr(self, request): 47 async def handle_test_pr(self, request):
50 '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value''' 48 '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value'''
51 version = request["version"] 49 version = request["version"]
52 pkgarch = request["pkgarch"] 50 pkgarch = request["pkgarch"]
53 checksum = request["checksum"] 51 checksum = request["checksum"]
52 history = request["history"]
54 53
55 value = self.server.table.find_value(version, pkgarch, checksum) 54 value = self.server.table.find_value(version, pkgarch, checksum, history)
56 return {"value": value} 55 return {"value": value}
57 56
58 async def handle_test_package(self, request): 57 async def handle_test_package(self, request):
@@ -68,22 +67,110 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection):
68 version = request["version"] 67 version = request["version"]
69 pkgarch = request["pkgarch"] 68 pkgarch = request["pkgarch"]
70 69
71 value = self.server.table.find_max_value(version, pkgarch) 70 value = self.server.table.find_package_max_value(version, pkgarch)
72 return {"value": value} 71 return {"value": value}
73 72
74 async def handle_get_pr(self, request): 73 async def handle_get_pr(self, request):
75 version = request["version"] 74 version = request["version"]
76 pkgarch = request["pkgarch"] 75 pkgarch = request["pkgarch"]
77 checksum = request["checksum"] 76 checksum = request["checksum"]
77 history = request["history"]
78 78
79 response = None 79 if self.upstream_client is None:
80 try: 80 value = self.server.table.get_value(version, pkgarch, checksum, history)
81 value = self.server.table.get_value(version, pkgarch, checksum) 81 return {"value": value}
82 response = {"value": value}
83 except prserv.NotFoundError:
84 self.logger.error("failure storing value in database for (%s, %s)",version, checksum)
85 82
86 return response 83 # We have an upstream server.
84 # Check whether the local server already knows the requested configuration.
85 # If the configuration is a new one, the generated value we will add will
86 # depend on what's on the upstream server. That's why we're calling find_value()
87 # instead of get_value() directly.
88
89 value = self.server.table.find_value(version, pkgarch, checksum, history)
90 upstream_max = await self.upstream_client.max_package_pr(version, pkgarch)
91
92 if value is not None:
93
94 # The configuration is already known locally.
95
96 if history:
97 value = self.server.table.get_value(version, pkgarch, checksum, history)
98 else:
99 existing_value = value
100 # In "no history", we need to make sure the value doesn't decrease
101 # and is at least greater than the maximum upstream value
102 # and the maximum local value
103
104 local_max = self.server.table.find_package_max_value(version, pkgarch)
105 if revision_smaller(value, local_max):
106 value = increase_revision(local_max)
107
108 if revision_smaller(value, upstream_max):
109 # Ask upstream whether it knows the checksum
110 upstream_value = await self.upstream_client.test_pr(version, pkgarch, checksum)
111 if upstream_value is None:
112 # Upstream doesn't have our checksum, let create a new one
113 value = upstream_max + ".0"
114 else:
115 # Fine to take the same value as upstream
116 value = upstream_max
117
118 if not value == existing_value and not self.server.read_only:
119 self.server.table.store_value(version, pkgarch, checksum, value)
120
121 return {"value": value}
122
123 # The configuration is a new one for the local server
124 # Let's ask the upstream server whether it knows it
125
126 known_upstream = await self.upstream_client.test_package(version, pkgarch)
127
128 if not known_upstream:
129
130 # The package is not known upstream, must be a local-only package
131 # Let's compute the PR number using the local-only method
132
133 value = self.server.table.get_value(version, pkgarch, checksum, history)
134 return {"value": value}
135
136 # The package is known upstream, let's ask the upstream server
137 # whether it knows our new output hash
138
139 value = await self.upstream_client.test_pr(version, pkgarch, checksum)
140
141 if value is not None:
142
143 # Upstream knows this output hash, let's store it and use it too.
144
145 if not self.server.read_only:
146 self.server.table.store_value(version, pkgarch, checksum, value)
147 # If the local server is read only, won't be able to store the new
148 # value in the database and will have to keep asking the upstream server
149 return {"value": value}
150
151 # The output hash doesn't exist upstream, get the most recent number from upstream (x)
152 # Then, we want to have a new PR value for the local server: x.y
153
154 upstream_max = await self.upstream_client.max_package_pr(version, pkgarch)
155 # Here we know that the package is known upstream, so upstream_max can't be None
156 subvalue = self.server.table.find_new_subvalue(version, pkgarch, upstream_max)
157
158 if not self.server.read_only:
159 self.server.table.store_value(version, pkgarch, checksum, subvalue)
160
161 return {"value": subvalue}
162
163 async def process_requests(self):
164 if self.server.upstream is not None:
165 self.upstream_client = await create_async_client(self.server.upstream)
166 else:
167 self.upstream_client = None
168
169 try:
170 await super().process_requests()
171 finally:
172 if self.upstream_client is not None:
173 await self.upstream_client.close()
87 174
88 async def handle_import_one(self, request): 175 async def handle_import_one(self, request):
89 response = None 176 response = None
@@ -104,9 +191,10 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection):
104 pkgarch = request["pkgarch"] 191 pkgarch = request["pkgarch"]
105 checksum = request["checksum"] 192 checksum = request["checksum"]
106 colinfo = request["colinfo"] 193 colinfo = request["colinfo"]
194 history = request["history"]
107 195
108 try: 196 try:
109 (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo) 197 (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo, history)
110 except sqlite3.Error as exc: 198 except sqlite3.Error as exc:
111 self.logger.error(str(exc)) 199 self.logger.error(str(exc))
112 metainfo = datainfo = None 200 metainfo = datainfo = None
@@ -117,11 +205,12 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection):
117 return {"readonly": self.server.read_only} 205 return {"readonly": self.server.read_only}
118 206
119class PRServer(bb.asyncrpc.AsyncServer): 207class PRServer(bb.asyncrpc.AsyncServer):
120 def __init__(self, dbfile, read_only=False): 208 def __init__(self, dbfile, read_only=False, upstream=None):
121 super().__init__(logger) 209 super().__init__(logger)
122 self.dbfile = dbfile 210 self.dbfile = dbfile
123 self.table = None 211 self.table = None
124 self.read_only = read_only 212 self.read_only = read_only
213 self.upstream = upstream
125 214
126 def accept_client(self, socket): 215 def accept_client(self, socket):
127 return PRServerClient(socket, self) 216 return PRServerClient(socket, self)
@@ -134,27 +223,25 @@ class PRServer(bb.asyncrpc.AsyncServer):
134 self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % 223 self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" %
135 (self.dbfile, self.address, str(os.getpid()))) 224 (self.dbfile, self.address, str(os.getpid())))
136 225
226 if self.upstream is not None:
227 self.logger.info("And upstream PRServer: %s " % (self.upstream))
228
137 return tasks 229 return tasks
138 230
139 async def stop(self): 231 async def stop(self):
140 self.table.sync_if_dirty()
141 self.db.disconnect() 232 self.db.disconnect()
142 await super().stop() 233 await super().stop()
143 234
144 def signal_handler(self):
145 super().signal_handler()
146 if self.table:
147 self.table.sync()
148
149class PRServSingleton(object): 235class PRServSingleton(object):
150 def __init__(self, dbfile, logfile, host, port): 236 def __init__(self, dbfile, logfile, host, port, upstream):
151 self.dbfile = dbfile 237 self.dbfile = dbfile
152 self.logfile = logfile 238 self.logfile = logfile
153 self.host = host 239 self.host = host
154 self.port = port 240 self.port = port
241 self.upstream = upstream
155 242
156 def start(self): 243 def start(self):
157 self.prserv = PRServer(self.dbfile) 244 self.prserv = PRServer(self.dbfile, upstream=self.upstream)
158 self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port) 245 self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port)
159 self.process = self.prserv.serve_as_process(log_level=logging.WARNING) 246 self.process = self.prserv.serve_as_process(log_level=logging.WARNING)
160 247
@@ -233,7 +320,7 @@ def run_as_daemon(func, pidfile, logfile):
233 os.remove(pidfile) 320 os.remove(pidfile)
234 os._exit(0) 321 os._exit(0)
235 322
236def start_daemon(dbfile, host, port, logfile, read_only=False): 323def start_daemon(dbfile, host, port, logfile, read_only=False, upstream=None):
237 ip = socket.gethostbyname(host) 324 ip = socket.gethostbyname(host)
238 pidfile = PIDPREFIX % (ip, port) 325 pidfile = PIDPREFIX % (ip, port)
239 try: 326 try:
@@ -249,7 +336,7 @@ def start_daemon(dbfile, host, port, logfile, read_only=False):
249 336
250 dbfile = os.path.abspath(dbfile) 337 dbfile = os.path.abspath(dbfile)
251 def daemon_main(): 338 def daemon_main():
252 server = PRServer(dbfile, read_only=read_only) 339 server = PRServer(dbfile, read_only=read_only, upstream=upstream)
253 server.start_tcp_server(ip, port) 340 server.start_tcp_server(ip, port)
254 server.serve_forever() 341 server.serve_forever()
255 342
@@ -336,6 +423,9 @@ def auto_start(d):
336 423
337 host = host_params[0].strip().lower() 424 host = host_params[0].strip().lower()
338 port = int(host_params[1]) 425 port = int(host_params[1])
426
427 upstream = d.getVar("PRSERV_UPSTREAM") or None
428
339 if is_local_special(host, port): 429 if is_local_special(host, port):
340 import bb.utils 430 import bb.utils
341 cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE")) 431 cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE"))
@@ -350,7 +440,7 @@ def auto_start(d):
350 auto_shutdown() 440 auto_shutdown()
351 if not singleton: 441 if not singleton:
352 bb.utils.mkdirhier(cachedir) 442 bb.utils.mkdirhier(cachedir)
353 singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port) 443 singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port, upstream)
354 singleton.start() 444 singleton.start()
355 if singleton: 445 if singleton:
356 host = singleton.host 446 host = singleton.host
diff --git a/bitbake/lib/prserv/tests.py b/bitbake/lib/prserv/tests.py
new file mode 100644
index 0000000000..df0c003003
--- /dev/null
+++ b/bitbake/lib/prserv/tests.py
@@ -0,0 +1,388 @@
1#! /usr/bin/env python3
2#
3# Copyright (C) 2024 BitBake Contributors
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7
8from . import create_server, create_client, increase_revision, revision_greater, revision_smaller, _revision_greater_or_equal
9import prserv.db as db
10from bb.asyncrpc import InvokeError
11import logging
12import os
13import sys
14import tempfile
15import unittest
16import socket
17import subprocess
18from pathlib import Path
19
20THIS_DIR = Path(__file__).parent
21BIN_DIR = THIS_DIR.parent.parent / "bin"
22
23version = "dummy-1.0-r0"
24pkgarch = "core2-64"
25other_arch = "aarch64"
26
27checksumX = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4f0"
28checksum0 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a0"
29checksum1 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a1"
30checksum2 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a2"
31checksum3 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a3"
32checksum4 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a4"
33checksum5 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a5"
34checksum6 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a6"
35checksum7 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a7"
36checksum8 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a8"
37checksum9 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a9"
38checksum10 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4aa"
39
40def server_prefunc(server, name):
41 logging.basicConfig(level=logging.DEBUG, filename='prserv-%s.log' % name, filemode='w',
42 format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
43 server.logger.debug("Running server %s" % name)
44 sys.stdout = open('prserv-stdout-%s.log' % name, 'w')
45 sys.stderr = sys.stdout
46
47class PRTestSetup(object):
48
49 def start_server(self, name, dbfile, upstream=None, read_only=False, prefunc=server_prefunc):
50
51 def cleanup_server(server):
52 if server.process.exitcode is not None:
53 return
54 server.process.terminate()
55 server.process.join()
56
57 server = create_server(socket.gethostbyname("localhost") + ":0",
58 dbfile,
59 upstream=upstream,
60 read_only=read_only)
61
62 server.serve_as_process(prefunc=prefunc, args=(name,))
63 self.addCleanup(cleanup_server, server)
64
65 return server
66
67 def start_client(self, server_address):
68 def cleanup_client(client):
69 client.close()
70
71 client = create_client(server_address)
72 self.addCleanup(cleanup_client, client)
73
74 return client
75
76class FunctionTests(unittest.TestCase):
77
78 def setUp(self):
79 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
80 self.addCleanup(self.temp_dir.cleanup)
81
82 def test_increase_revision(self):
83 self.assertEqual(increase_revision("1"), "2")
84 self.assertEqual(increase_revision("1.0"), "1.1")
85 self.assertEqual(increase_revision("1.1.1"), "1.1.2")
86 self.assertEqual(increase_revision("1.1.1.3"), "1.1.1.4")
87 self.assertEqual(increase_revision("9"), "10")
88 self.assertEqual(increase_revision("1.9"), "1.10")
89 self.assertRaises(ValueError, increase_revision, "1.a")
90 self.assertRaises(ValueError, increase_revision, "1.")
91 self.assertRaises(ValueError, increase_revision, "")
92
93 def test_revision_greater_or_equal(self):
94 self.assertTrue(_revision_greater_or_equal("2", "2"))
95 self.assertTrue(_revision_greater_or_equal("2", "1"))
96 self.assertTrue(_revision_greater_or_equal("10", "2"))
97 self.assertTrue(_revision_greater_or_equal("1.10", "1.2"))
98 self.assertFalse(_revision_greater_or_equal("1.2", "1.10"))
99 self.assertTrue(_revision_greater_or_equal("1.10", "1"))
100 self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10"))
101 self.assertFalse(_revision_greater_or_equal("1.10.1", "1.10.2"))
102 self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10.1"))
103 self.assertTrue(_revision_greater_or_equal("1.10.1", "1"))
104 self.assertTrue(revision_greater("1.20", "1.3"))
105 self.assertTrue(revision_smaller("1.3", "1.20"))
106
107 # DB tests
108
109 def test_db(self):
110 dbfile = os.path.join(self.temp_dir.name, "testtable.sqlite3")
111
112 self.db = db.PRData(dbfile)
113 self.table = self.db["PRMAIN"]
114
115 self.table.store_value(version, pkgarch, checksum0, "0")
116 self.table.store_value(version, pkgarch, checksum1, "1")
117 # "No history" mode supports multiple PRs for the same checksum
118 self.table.store_value(version, pkgarch, checksum0, "2")
119 self.table.store_value(version, pkgarch, checksum2, "1.0")
120
121 self.assertTrue(self.table.test_package(version, pkgarch))
122 self.assertFalse(self.table.test_package(version, other_arch))
123
124 self.assertTrue(self.table.test_value(version, pkgarch, "0"))
125 self.assertTrue(self.table.test_value(version, pkgarch, "1"))
126 self.assertTrue(self.table.test_value(version, pkgarch, "2"))
127
128 self.assertEqual(self.table.find_package_max_value(version, pkgarch), "2")
129
130 self.assertEqual(self.table.find_min_value(version, pkgarch, checksum0), "0")
131 self.assertEqual(self.table.find_max_value(version, pkgarch, checksum0), "2")
132
133 # Test history modes
134 self.assertEqual(self.table.find_value(version, pkgarch, checksum0, True), "0")
135 self.assertEqual(self.table.find_value(version, pkgarch, checksum0, False), "2")
136
137 self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "3"), "3.0")
138 self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "1"), "1.1")
139
140 # Revision comparison tests
141 self.table.store_value(version, pkgarch, checksum1, "1.3")
142 self.table.store_value(version, pkgarch, checksum1, "1.20")
143 self.assertEqual(self.table.find_min_value(version, pkgarch, checksum1), "1")
144 self.assertEqual(self.table.find_max_value(version, pkgarch, checksum1), "1.20")
145
146class PRBasicTests(PRTestSetup, unittest.TestCase):
147
148 def setUp(self):
149 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
150 self.addCleanup(self.temp_dir.cleanup)
151
152 dbfile = os.path.join(self.temp_dir.name, "prtest-basic.sqlite3")
153
154 self.server1 = self.start_server("basic", dbfile)
155 self.client1 = self.start_client(self.server1.address)
156
157 def test_basic(self):
158
159 # Checks on non existing configuration
160
161 result = self.client1.test_pr(version, pkgarch, checksum0)
162 self.assertIsNone(result, "test_pr should return 'None' for a non existing PR")
163
164 result = self.client1.test_package(version, pkgarch)
165 self.assertFalse(result, "test_package should return 'False' for a non existing PR")
166
167 result = self.client1.max_package_pr(version, pkgarch)
168 self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR")
169
170 # Add a first configuration
171
172 result = self.client1.getPR(version, pkgarch, checksum0)
173 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
174
175 result = self.client1.test_pr(version, pkgarch, checksum0)
176 self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR")
177
178 result = self.client1.test_package(version, pkgarch)
179 self.assertTrue(result, "test_package should return 'True' for an existing PR")
180
181 result = self.client1.max_package_pr(version, pkgarch)
182 self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series")
183
184 # Check that the same request gets the same value
185
186 result = self.client1.getPR(version, pkgarch, checksum0)
187 self.assertEqual(result, "0", "getPR: asking for the same PR a second time in a row should return the same value.")
188
189 # Add new configurations
190
191 result = self.client1.getPR(version, pkgarch, checksum1)
192 self.assertEqual(result, "1", "getPR: second PR of a package should be '1'")
193
194 result = self.client1.test_pr(version, pkgarch, checksum1)
195 self.assertEqual(result, "1", "test_pr should return '1' here, matching the result of getPR")
196
197 result = self.client1.max_package_pr(version, pkgarch)
198 self.assertEqual(result, "1", "max_package_pr should return '1' in the current test series")
199
200 result = self.client1.getPR(version, pkgarch, checksum2)
201 self.assertEqual(result, "2", "getPR: second PR of a package should be '2'")
202
203 result = self.client1.test_pr(version, pkgarch, checksum2)
204 self.assertEqual(result, "2", "test_pr should return '2' here, matching the result of getPR")
205
206 result = self.client1.max_package_pr(version, pkgarch)
207 self.assertEqual(result, "2", "max_package_pr should return '2' in the current test series")
208
209 result = self.client1.getPR(version, pkgarch, checksum3)
210 self.assertEqual(result, "3", "getPR: second PR of a package should be '3'")
211
212 result = self.client1.test_pr(version, pkgarch, checksum3)
213 self.assertEqual(result, "3", "test_pr should return '3' here, matching the result of getPR")
214
215 result = self.client1.max_package_pr(version, pkgarch)
216 self.assertEqual(result, "3", "max_package_pr should return '3' in the current test series")
217
218 # Ask again for the first configuration
219
220 result = self.client1.getPR(version, pkgarch, checksum0)
221 self.assertEqual(result, "4", "getPR: should return '4' in this configuration")
222
223 # Ask again with explicit "no history" mode
224
225 result = self.client1.getPR(version, pkgarch, checksum0, False)
226 self.assertEqual(result, "4", "getPR: should return '4' in this configuration")
227
228 # Ask again with explicit "history" mode. This should return the first recorded PR for checksum0
229
230 result = self.client1.getPR(version, pkgarch, checksum0, True)
231 self.assertEqual(result, "0", "getPR: should return '0' in this configuration")
232
233 # Check again that another pkgarg resets the counters
234
235 result = self.client1.test_pr(version, other_arch, checksum0)
236 self.assertIsNone(result, "test_pr should return 'None' for a non existing PR")
237
238 result = self.client1.test_package(version, other_arch)
239 self.assertFalse(result, "test_package should return 'False' for a non existing PR")
240
241 result = self.client1.max_package_pr(version, other_arch)
242 self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR")
243
244 # Now add the configuration
245
246 result = self.client1.getPR(version, other_arch, checksum0)
247 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
248
249 result = self.client1.test_pr(version, other_arch, checksum0)
250 self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR")
251
252 result = self.client1.test_package(version, other_arch)
253 self.assertTrue(result, "test_package should return 'True' for an existing PR")
254
255 result = self.client1.max_package_pr(version, other_arch)
256 self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series")
257
258 result = self.client1.is_readonly()
259 self.assertFalse(result, "Server should not be described as 'read-only'")
260
261class PRUpstreamTests(PRTestSetup, unittest.TestCase):
262
263 def setUp(self):
264
265 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
266 self.addCleanup(self.temp_dir.cleanup)
267
268 dbfile2 = os.path.join(self.temp_dir.name, "prtest-upstream2.sqlite3")
269 self.server2 = self.start_server("upstream2", dbfile2)
270 self.client2 = self.start_client(self.server2.address)
271
272 dbfile1 = os.path.join(self.temp_dir.name, "prtest-upstream1.sqlite3")
273 self.server1 = self.start_server("upstream1", dbfile1, upstream=self.server2.address)
274 self.client1 = self.start_client(self.server1.address)
275
276 dbfile0 = os.path.join(self.temp_dir.name, "prtest-local.sqlite3")
277 self.server0 = self.start_server("local", dbfile0, upstream=self.server1.address)
278 self.client0 = self.start_client(self.server0.address)
279 self.shared_db = dbfile0
280
281 def test_upstream_and_readonly(self):
282
283 # For identical checksums, all servers should return the same PR
284
285 result = self.client2.getPR(version, pkgarch, checksum0)
286 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
287
288 result = self.client1.getPR(version, pkgarch, checksum0)
289 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)")
290
291 result = self.client0.getPR(version, pkgarch, checksum0)
292 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)")
293
294 # Now introduce new checksums on server1 for, same version
295
296 result = self.client1.getPR(version, pkgarch, checksum1)
297 self.assertEqual(result, "0.0", "getPR: first PR of a package which has a different checksum upstream should be '0.0'")
298
299 result = self.client1.getPR(version, pkgarch, checksum2)
300 self.assertEqual(result, "0.1", "getPR: second PR of a package that has a different checksum upstream should be '0.1'")
301
302 # Now introduce checksums on server0 for, same version
303
304 result = self.client1.getPR(version, pkgarch, checksum1)
305 self.assertEqual(result, "0.2", "getPR: can't decrease for known PR")
306
307 result = self.client1.getPR(version, pkgarch, checksum2)
308 self.assertEqual(result, "0.3")
309
310 result = self.client1.max_package_pr(version, pkgarch)
311 self.assertEqual(result, "0.3")
312
313 result = self.client0.getPR(version, pkgarch, checksum3)
314 self.assertEqual(result, "0.3.0", "getPR: first PR of a package that doesn't exist upstream should be '0.3.0'")
315
316 result = self.client0.getPR(version, pkgarch, checksum4)
317 self.assertEqual(result, "0.3.1", "getPR: second PR of a package that doesn't exist upstream should be '0.3.1'")
318
319 result = self.client0.getPR(version, pkgarch, checksum3)
320 self.assertEqual(result, "0.3.2")
321
322 # More upstream updates
323 # Here, we assume no communication between server2 and server0. server2 only impacts server0
324 # after impacting server1
325
326 self.assertEqual(self.client2.getPR(version, pkgarch, checksum5), "1")
327 self.assertEqual(self.client1.getPR(version, pkgarch, checksum6), "1.0")
328 self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "1.1")
329 self.assertEqual(self.client0.getPR(version, pkgarch, checksum8), "1.1.0")
330 self.assertEqual(self.client0.getPR(version, pkgarch, checksum9), "1.1.1")
331
332 # "history" mode tests
333
334 self.assertEqual(self.client2.getPR(version, pkgarch, checksum0, True), "0")
335 self.assertEqual(self.client1.getPR(version, pkgarch, checksum2, True), "0.1")
336 self.assertEqual(self.client0.getPR(version, pkgarch, checksum3, True), "0.3.0")
337
338 # More "no history" mode tests
339
340 self.assertEqual(self.client2.getPR(version, pkgarch, checksum0), "2")
341 self.assertEqual(self.client1.getPR(version, pkgarch, checksum0), "2") # Same as upstream
342 self.assertEqual(self.client0.getPR(version, pkgarch, checksum0), "2") # Same as upstream
343 self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "3") # This could be surprising, but since the previous revision was "2", increasing it yields "3".
344 # We don't know how many upstream servers we have
345 # Start read-only server with server1 as upstream
346 self.server_ro = self.start_server("local-ro", self.shared_db, upstream=self.server1.address, read_only=True)
347 self.client_ro = self.start_client(self.server_ro.address)
348
349 self.assertTrue(self.client_ro.is_readonly(), "Database should be described as 'read-only'")
350
351 # Checks on non existing configurations
352 self.assertIsNone(self.client_ro.test_pr(version, pkgarch, checksumX))
353 self.assertFalse(self.client_ro.test_package("unknown", pkgarch))
354
355 # Look up existing configurations
356 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0), "3") # "no history" mode
357 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0, True), "0") # "history" mode
358 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3), "3")
359 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3, True), "0.3.0")
360 self.assertEqual(self.client_ro.max_package_pr(version, pkgarch), "2") # normal as "3" was never saved
361
362 # Try to insert a new value. Here this one is know upstream.
363 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum7), "3")
364 # Try to insert a completely new value. As the max upstream value is already "3", it should be "3.0"
365 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum10), "3.0")
366 # Same with another value which only exists in the upstream upstream server
367 # This time, as the upstream server doesn't know it, it will ask its upstream server. So that's a known one.
368 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum9), "3")
369
370class ScriptTests(unittest.TestCase):
371
372 def setUp(self):
373
374 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
375 self.addCleanup(self.temp_dir.cleanup)
376 self.dbfile = os.path.join(self.temp_dir.name, "prtest.sqlite3")
377
378 def test_1_start_bitbake_prserv(self):
379 try:
380 subprocess.check_call([BIN_DIR / "bitbake-prserv", "--start", "-f", self.dbfile])
381 except subprocess.CalledProcessError as e:
382 self.fail("Failed to start bitbake-prserv: %s" % e.returncode)
383
384 def test_2_stop_bitbake_prserv(self):
385 try:
386 subprocess.check_call([BIN_DIR / "bitbake-prserv", "--stop"])
387 except subprocess.CalledProcessError as e:
388 self.fail("Failed to stop bitbake-prserv: %s" % e.returncode)