summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/prserv
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/prserv')
-rw-r--r--bitbake/lib/prserv/__init__.py97
-rw-r--r--bitbake/lib/prserv/client.py46
-rw-r--r--bitbake/lib/prserv/db.py430
-rw-r--r--bitbake/lib/prserv/serv.py252
-rw-r--r--bitbake/lib/prserv/tests.py388
5 files changed, 898 insertions, 315 deletions
diff --git a/bitbake/lib/prserv/__init__.py b/bitbake/lib/prserv/__init__.py
index 38ced818ad..ffc5a40a28 100644
--- a/bitbake/lib/prserv/__init__.py
+++ b/bitbake/lib/prserv/__init__.py
@@ -4,17 +4,92 @@
4# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
5# 5#
6 6
7__version__ = "1.0.0"
8 7
9import os, time 8__version__ = "2.0.0"
10import sys,logging
11 9
12def init_logger(logfile, loglevel): 10import logging
13 numeric_level = getattr(logging, loglevel.upper(), None) 11logger = logging.getLogger("BitBake.PRserv")
14 if not isinstance(numeric_level, int):
15 raise ValueError('Invalid log level: %s' % loglevel)
16 FORMAT = '%(asctime)-15s %(message)s'
17 logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
18 12
19class NotFoundError(Exception): 13from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
20 pass 14
15def create_server(addr, dbpath, upstream=None, read_only=False):
16 from . import serv
17
18 s = serv.PRServer(dbpath, upstream=upstream, read_only=read_only)
19 host, port = addr.split(":")
20 s.start_tcp_server(host, int(port))
21
22 return s
23
24def increase_revision(ver):
25 """Take a revision string such as "1" or "1.2.3" or even a number and increase its last number
26 This fails if the last number is not an integer"""
27
28 fields=str(ver).split('.')
29 last = fields[-1]
30
31 try:
32 val = int(last)
33 except Exception as e:
34 logger.critical("Unable to increase revision value %s: %s" % (ver, e))
35 raise e
36
37 return ".".join(fields[0:-1] + [ str(val + 1) ])
38
39def _revision_greater_or_equal(rev1, rev2):
40 """Compares x.y.z revision numbers, using integer comparison
41 Returns True if rev1 is greater or equal to rev2"""
42
43 fields1 = rev1.split(".")
44 fields2 = rev2.split(".")
45 l1 = len(fields1)
46 l2 = len(fields2)
47
48 for i in range(l1):
49 val1 = int(fields1[i])
50 if i < l2:
51 val2 = int(fields2[i])
52 if val2 < val1:
53 return True
54 elif val2 > val1:
55 return False
56 else:
57 return True
58 return True
59
60def revision_smaller(rev1, rev2):
61 """Compares x.y.z revision numbers, using integer comparison
62 Returns True if rev1 is strictly smaller than rev2"""
63 return not(_revision_greater_or_equal(rev1, rev2))
64
65def revision_greater(rev1, rev2):
66 """Compares x.y.z revision numbers, using integer comparison
67 Returns True if rev1 is strictly greater than rev2"""
68 return _revision_greater_or_equal(rev1, rev2) and (rev1 != rev2)
69
70def create_client(addr):
71 from . import client
72
73 c = client.PRClient()
74
75 try:
76 (typ, a) = parse_address(addr)
77 c.connect_tcp(*a)
78 return c
79 except Exception as e:
80 c.close()
81 raise e
82
83async def create_async_client(addr):
84 from . import client
85
86 c = client.PRAsyncClient()
87
88 try:
89 (typ, a) = parse_address(addr)
90 await c.connect_tcp(*a)
91 return c
92
93 except Exception as e:
94 await c.close()
95 raise e
diff --git a/bitbake/lib/prserv/client.py b/bitbake/lib/prserv/client.py
index 6b81356fac..9f5794c433 100644
--- a/bitbake/lib/prserv/client.py
+++ b/bitbake/lib/prserv/client.py
@@ -6,45 +6,67 @@
6 6
7import logging 7import logging
8import bb.asyncrpc 8import bb.asyncrpc
9from . import create_async_client
9 10
10logger = logging.getLogger("BitBake.PRserv") 11logger = logging.getLogger("BitBake.PRserv")
11 12
12class PRAsyncClient(bb.asyncrpc.AsyncClient): 13class PRAsyncClient(bb.asyncrpc.AsyncClient):
13 def __init__(self): 14 def __init__(self):
14 super().__init__('PRSERVICE', '1.0', logger) 15 super().__init__("PRSERVICE", "1.0", logger)
15 16
16 async def getPR(self, version, pkgarch, checksum): 17 async def getPR(self, version, pkgarch, checksum, history=False):
17 response = await self.invoke( 18 response = await self.invoke(
18 {'get-pr': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum}} 19 {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}}
19 ) 20 )
20 if response: 21 if response:
21 return response['value'] 22 return response["value"]
23
24 async def test_pr(self, version, pkgarch, checksum, history=False):
25 response = await self.invoke(
26 {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}}
27 )
28 if response:
29 return response["value"]
30
31 async def test_package(self, version, pkgarch):
32 response = await self.invoke(
33 {"test-package": {"version": version, "pkgarch": pkgarch}}
34 )
35 if response:
36 return response["value"]
37
38 async def max_package_pr(self, version, pkgarch):
39 response = await self.invoke(
40 {"max-package-pr": {"version": version, "pkgarch": pkgarch}}
41 )
42 if response:
43 return response["value"]
22 44
23 async def importone(self, version, pkgarch, checksum, value): 45 async def importone(self, version, pkgarch, checksum, value):
24 response = await self.invoke( 46 response = await self.invoke(
25 {'import-one': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'value': value}} 47 {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}}
26 ) 48 )
27 if response: 49 if response:
28 return response['value'] 50 return response["value"]
29 51
30 async def export(self, version, pkgarch, checksum, colinfo): 52 async def export(self, version, pkgarch, checksum, colinfo, history=False):
31 response = await self.invoke( 53 response = await self.invoke(
32 {'export': {'version': version, 'pkgarch': pkgarch, 'checksum': checksum, 'colinfo': colinfo}} 54 {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo, "history": history}}
33 ) 55 )
34 if response: 56 if response:
35 return (response['metainfo'], response['datainfo']) 57 return (response["metainfo"], response["datainfo"])
36 58
37 async def is_readonly(self): 59 async def is_readonly(self):
38 response = await self.invoke( 60 response = await self.invoke(
39 {'is-readonly': {}} 61 {"is-readonly": {}}
40 ) 62 )
41 if response: 63 if response:
42 return response['readonly'] 64 return response["readonly"]
43 65
44class PRClient(bb.asyncrpc.Client): 66class PRClient(bb.asyncrpc.Client):
45 def __init__(self): 67 def __init__(self):
46 super().__init__() 68 super().__init__()
47 self._add_methods('getPR', 'importone', 'export', 'is_readonly') 69 self._add_methods("getPR", "test_pr", "test_package", "max_package_pr", "importone", "export", "is_readonly")
48 70
49 def _get_async_client(self): 71 def _get_async_client(self):
50 return PRAsyncClient() 72 return PRAsyncClient()
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
index b4bda7078c..2da493ddf5 100644
--- a/bitbake/lib/prserv/db.py
+++ b/bitbake/lib/prserv/db.py
@@ -8,19 +8,13 @@ import logging
8import os.path 8import os.path
9import errno 9import errno
10import prserv 10import prserv
11import time 11import sqlite3
12 12
13try: 13from contextlib import closing
14 import sqlite3 14from . import increase_revision, revision_greater, revision_smaller
15except ImportError:
16 from pysqlite2 import dbapi2 as sqlite3
17 15
18logger = logging.getLogger("BitBake.PRserv") 16logger = logging.getLogger("BitBake.PRserv")
19 17
20sqlversion = sqlite3.sqlite_version_info
21if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
22 raise Exception("sqlite3 version 3.3.0 or later is required.")
23
24# 18#
25# "No History" mode - for a given query tuple (version, pkgarch, checksum), 19# "No History" mode - for a given query tuple (version, pkgarch, checksum),
26# the returned value will be the largest among all the values of the same 20# the returned value will be the largest among all the values of the same
@@ -29,245 +23,232 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
29# "History" mode - Return a new higher value for previously unseen query 23# "History" mode - Return a new higher value for previously unseen query
30# tuple (version, pkgarch, checksum), otherwise return historical value. 24# tuple (version, pkgarch, checksum), otherwise return historical value.
31# Value can decrement if returning to a previous build. 25# Value can decrement if returning to a previous build.
32#
33 26
34class PRTable(object): 27class PRTable(object):
35 def __init__(self, conn, table, nohist, read_only): 28 def __init__(self, conn, table, read_only):
36 self.conn = conn 29 self.conn = conn
37 self.nohist = nohist
38 self.read_only = read_only 30 self.read_only = read_only
39 self.dirty = False 31 self.table = table
40 if nohist: 32
41 self.table = "%s_nohist" % table 33 # Creating the table even if the server is read-only.
42 else: 34 # This avoids a race condition if a shared database
43 self.table = "%s_hist" % table 35 # is accessed by a read-only server first.
44 36
45 if self.read_only: 37 with closing(self.conn.cursor()) as cursor:
46 table_exists = self._execute( 38 cursor.execute("CREATE TABLE IF NOT EXISTS %s \
47 "SELECT count(*) FROM sqlite_master \
48 WHERE type='table' AND name='%s'" % (self.table))
49 if not table_exists:
50 raise prserv.NotFoundError
51 else:
52 self._execute("CREATE TABLE IF NOT EXISTS %s \
53 (version TEXT NOT NULL, \ 39 (version TEXT NOT NULL, \
54 pkgarch TEXT NOT NULL, \ 40 pkgarch TEXT NOT NULL, \
55 checksum TEXT NOT NULL, \ 41 checksum TEXT NOT NULL, \
56 value INTEGER, \ 42 value TEXT, \
57 PRIMARY KEY (version, pkgarch, checksum));" % self.table) 43 PRIMARY KEY (version, pkgarch, checksum, value));" % self.table)
58
59 def _execute(self, *query):
60 """Execute a query, waiting to acquire a lock if necessary"""
61 start = time.time()
62 end = start + 20
63 while True:
64 try:
65 return self.conn.execute(*query)
66 except sqlite3.OperationalError as exc:
67 if 'is locked' in str(exc) and end > time.time():
68 continue
69 raise exc
70
71 def sync(self):
72 if not self.read_only:
73 self.conn.commit() 44 self.conn.commit()
74 self._execute("BEGIN EXCLUSIVE TRANSACTION") 45
75 46 def _extremum_value(self, rows, is_max):
76 def sync_if_dirty(self): 47 value = None
77 if self.dirty: 48
78 self.sync() 49 for row in rows:
79 self.dirty = False 50 current_value = row[0]
80 51 if value is None:
81 def _getValueHist(self, version, pkgarch, checksum): 52 value = current_value
82 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 53 else:
83 (version, pkgarch, checksum)) 54 if is_max:
84 row=data.fetchone() 55 is_new_extremum = revision_greater(current_value, value)
85 if row is not None:
86 return row[0]
87 else:
88 #no value found, try to insert
89 if self.read_only:
90 data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table),
91 (version, pkgarch))
92 row = data.fetchone()
93 if row is not None:
94 return row[0]
95 else: 56 else:
96 return 0 57 is_new_extremum = revision_smaller(current_value, value)
58 if is_new_extremum:
59 value = current_value
60 return value
61
62 def _max_value(self, rows):
63 return self._extremum_value(rows, True)
97 64
98 try: 65 def _min_value(self, rows):
99 self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" 66 return self._extremum_value(rows, False)
100 % (self.table,self.table),
101 (version,pkgarch, checksum,version, pkgarch))
102 except sqlite3.IntegrityError as exc:
103 logger.error(str(exc))
104 67
105 self.dirty = True 68 def test_package(self, version, pkgarch):
69 """Returns whether the specified package version is found in the database for the specified architecture"""
106 70
107 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 71 # Just returns the value if found or None otherwise
108 (version, pkgarch, checksum)) 72 with closing(self.conn.cursor()) as cursor:
73 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table,
74 (version, pkgarch))
109 row=data.fetchone() 75 row=data.fetchone()
110 if row is not None: 76 if row is not None:
111 return row[0] 77 return True
112 else: 78 else:
113 raise prserv.NotFoundError 79 return False
114 80
115 def _getValueNohist(self, version, pkgarch, checksum): 81 def test_checksum_value(self, version, pkgarch, checksum, value):
116 data=self._execute("SELECT value FROM %s \ 82 """Returns whether the specified value is found in the database for the specified package, architecture and checksum"""
117 WHERE version=? AND pkgarch=? AND checksum=? AND \
118 value >= (select max(value) from %s where version=? AND pkgarch=?);"
119 % (self.table, self.table),
120 (version, pkgarch, checksum, version, pkgarch))
121 row=data.fetchone()
122 if row is not None:
123 return row[0]
124 else:
125 #no value found, try to insert
126 if self.read_only:
127 data = self._execute("SELECT ifnull(max(value)+1,0) FROM %s where version=? AND pkgarch=?;" % (self.table),
128 (version, pkgarch))
129 row = data.fetchone()
130 if row is not None:
131 return row[0]
132 else:
133 return 0
134 83
135 try: 84 with closing(self.conn.cursor()) as cursor:
136 self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));" 85 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and checksum=? and value=?;" % self.table,
137 % (self.table,self.table), 86 (version, pkgarch, checksum, value))
138 (version, pkgarch, checksum, version, pkgarch)) 87 row=data.fetchone()
139 except sqlite3.IntegrityError as exc: 88 if row is not None:
140 logger.error(str(exc)) 89 return True
141 self.conn.rollback() 90 else:
91 return False
142 92
143 self.dirty = True 93 def test_value(self, version, pkgarch, value):
94 """Returns whether the specified value is found in the database for the specified package and architecture"""
144 95
145 data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 96 # Just returns the value if found or None otherwise
146 (version, pkgarch, checksum)) 97 with closing(self.conn.cursor()) as cursor:
98 data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table,
99 (version, pkgarch, value))
147 row=data.fetchone() 100 row=data.fetchone()
148 if row is not None: 101 if row is not None:
149 return row[0] 102 return True
150 else: 103 else:
151 raise prserv.NotFoundError 104 return False
152 105
153 def getValue(self, version, pkgarch, checksum): 106
154 if self.nohist: 107 def find_package_max_value(self, version, pkgarch):
155 return self._getValueNohist(version, pkgarch, checksum) 108 """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
156 else: 109
157 return self._getValueHist(version, pkgarch, checksum) 110 with closing(self.conn.cursor()) as cursor:
158 111 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=?;" % (self.table),
159 def _importHist(self, version, pkgarch, checksum, value): 112 (version, pkgarch))
160 if self.read_only: 113 rows = data.fetchall()
161 return None 114 value = self._max_value(rows)
162 115 return value
163 val = None 116
164 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 117 def find_value(self, version, pkgarch, checksum, history=False):
165 (version, pkgarch, checksum)) 118 """Returns the value for the specified checksum if found or None otherwise."""
166 row = data.fetchone() 119
167 if row is not None: 120 if history:
168 val=row[0] 121 return self.find_min_value(version, pkgarch, checksum)
169 else: 122 else:
170 #no value found, try to insert 123 return self.find_max_value(version, pkgarch, checksum)
171 try: 124
172 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 125
126 def _find_extremum_value(self, version, pkgarch, checksum, is_max):
127 """Returns the maximum (if is_max is True) or minimum (if is_max is False) value
128 for (version, pkgarch, checksum), or None if not found. Doesn't create a new value"""
129
130 with closing(self.conn.cursor()) as cursor:
131 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND checksum=?;" % (self.table),
132 (version, pkgarch, checksum))
133 rows = data.fetchall()
134 return self._extremum_value(rows, is_max)
135
136 def find_max_value(self, version, pkgarch, checksum):
137 return self._find_extremum_value(version, pkgarch, checksum, True)
138
139 def find_min_value(self, version, pkgarch, checksum):
140 return self._find_extremum_value(version, pkgarch, checksum, False)
141
142 def find_new_subvalue(self, version, pkgarch, base):
143 """Take and increase the greatest "<base>.y" value for (version, pkgarch), or return "<base>.0" if not found.
144 This doesn't store a new value."""
145
146 with closing(self.conn.cursor()) as cursor:
147 data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (self.table, base),
148 (version, pkgarch))
149 rows = data.fetchall()
150 value = self._max_value(rows)
151
152 if value is not None:
153 return increase_revision(value)
154 else:
155 return base + ".0"
156
157 def store_value(self, version, pkgarch, checksum, value):
158 """Store value in the database"""
159
160 if not self.read_only and not self.test_checksum_value(version, pkgarch, checksum, value):
161 with closing(self.conn.cursor()) as cursor:
162 cursor.execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table),
173 (version, pkgarch, checksum, value)) 163 (version, pkgarch, checksum, value))
174 except sqlite3.IntegrityError as exc: 164 self.conn.commit()
175 logger.error(str(exc))
176 165
177 self.dirty = True 166 def _get_value(self, version, pkgarch, checksum, history):
178 167
179 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table, 168 max_value = self.find_package_max_value(version, pkgarch)
180 (version, pkgarch, checksum))
181 row = data.fetchone()
182 if row is not None:
183 val = row[0]
184 return val
185 169
186 def _importNohist(self, version, pkgarch, checksum, value): 170 if max_value is None:
187 if self.read_only: 171 # version, pkgarch completely unknown. Return initial value.
188 return None 172 return "0"
189 173
190 try: 174 value = self.find_value(version, pkgarch, checksum, history)
191 #try to insert 175
192 self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);" % (self.table), 176 if value is None:
193 (version, pkgarch, checksum,value)) 177 # version, pkgarch found but not checksum. Create a new value from the maximum one
194 except sqlite3.IntegrityError as exc: 178 return increase_revision(max_value)
195 #already have the record, try to update 179
196 try: 180 if history:
197 self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?" 181 return value
198 % (self.table), 182
199 (value,version,pkgarch,checksum,value)) 183 # "no history" mode - If the value is not the maximum value for the package, need to increase it.
200 except sqlite3.IntegrityError as exc: 184 if max_value > value:
201 logger.error(str(exc)) 185 return increase_revision(max_value)
202
203 self.dirty = True
204
205 data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
206 (version,pkgarch,checksum,value))
207 row=data.fetchone()
208 if row is not None:
209 return row[0]
210 else: 186 else:
211 return None 187 return value
188
189 def get_value(self, version, pkgarch, checksum, history):
190 value = self._get_value(version, pkgarch, checksum, history)
191 if not self.read_only:
192 self.store_value(version, pkgarch, checksum, value)
193 return value
212 194
213 def importone(self, version, pkgarch, checksum, value): 195 def importone(self, version, pkgarch, checksum, value):
214 if self.nohist: 196 self.store_value(version, pkgarch, checksum, value)
215 return self._importNohist(version, pkgarch, checksum, value) 197 return value
216 else:
217 return self._importHist(version, pkgarch, checksum, value)
218 198
219 def export(self, version, pkgarch, checksum, colinfo): 199 def export(self, version, pkgarch, checksum, colinfo, history=False):
220 metainfo = {} 200 metainfo = {}
221 #column info 201 with closing(self.conn.cursor()) as cursor:
222 if colinfo: 202 #column info
223 metainfo['tbl_name'] = self.table 203 if colinfo:
224 metainfo['core_ver'] = prserv.__version__ 204 metainfo["tbl_name"] = self.table
225 metainfo['col_info'] = [] 205 metainfo["core_ver"] = prserv.__version__
226 data = self._execute("PRAGMA table_info(%s);" % self.table) 206 metainfo["col_info"] = []
207 data = cursor.execute("PRAGMA table_info(%s);" % self.table)
208 for row in data:
209 col = {}
210 col["name"] = row["name"]
211 col["type"] = row["type"]
212 col["notnull"] = row["notnull"]
213 col["dflt_value"] = row["dflt_value"]
214 col["pk"] = row["pk"]
215 metainfo["col_info"].append(col)
216
217 #data info
218 datainfo = []
219
220 if history:
221 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
222 else:
223 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
224 (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
225 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
226 sqlarg = []
227 where = ""
228 if version:
229 where += "AND T1.version=? "
230 sqlarg.append(str(version))
231 if pkgarch:
232 where += "AND T1.pkgarch=? "
233 sqlarg.append(str(pkgarch))
234 if checksum:
235 where += "AND T1.checksum=? "
236 sqlarg.append(str(checksum))
237
238 sqlstmt += where + ";"
239
240 if len(sqlarg):
241 data = cursor.execute(sqlstmt, tuple(sqlarg))
242 else:
243 data = cursor.execute(sqlstmt)
227 for row in data: 244 for row in data:
228 col = {} 245 if row["version"]:
229 col['name'] = row['name'] 246 col = {}
230 col['type'] = row['type'] 247 col["version"] = row["version"]
231 col['notnull'] = row['notnull'] 248 col["pkgarch"] = row["pkgarch"]
232 col['dflt_value'] = row['dflt_value'] 249 col["checksum"] = row["checksum"]
233 col['pk'] = row['pk'] 250 col["value"] = row["value"]
234 metainfo['col_info'].append(col) 251 datainfo.append(col)
235
236 #data info
237 datainfo = []
238
239 if self.nohist:
240 sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
241 (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
242 WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
243 else:
244 sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
245 sqlarg = []
246 where = ""
247 if version:
248 where += "AND T1.version=? "
249 sqlarg.append(str(version))
250 if pkgarch:
251 where += "AND T1.pkgarch=? "
252 sqlarg.append(str(pkgarch))
253 if checksum:
254 where += "AND T1.checksum=? "
255 sqlarg.append(str(checksum))
256
257 sqlstmt += where + ";"
258
259 if len(sqlarg):
260 data = self._execute(sqlstmt, tuple(sqlarg))
261 else:
262 data = self._execute(sqlstmt)
263 for row in data:
264 if row['version']:
265 col = {}
266 col['version'] = row['version']
267 col['pkgarch'] = row['pkgarch']
268 col['checksum'] = row['checksum']
269 col['value'] = row['value']
270 datainfo.append(col)
271 return (metainfo, datainfo) 252 return (metainfo, datainfo)
272 253
273 def dump_db(self, fd): 254 def dump_db(self, fd):
@@ -275,14 +256,13 @@ class PRTable(object):
275 for line in self.conn.iterdump(): 256 for line in self.conn.iterdump():
276 writeCount = writeCount + len(line) + 1 257 writeCount = writeCount + len(line) + 1
277 fd.write(line) 258 fd.write(line)
278 fd.write('\n') 259 fd.write("\n")
279 return writeCount 260 return writeCount
280 261
281class PRData(object): 262class PRData(object):
282 """Object representing the PR database""" 263 """Object representing the PR database"""
283 def __init__(self, filename, nohist=True, read_only=False): 264 def __init__(self, filename, read_only=False):
284 self.filename=os.path.abspath(filename) 265 self.filename=os.path.abspath(filename)
285 self.nohist=nohist
286 self.read_only = read_only 266 self.read_only = read_only
287 #build directory hierarchy 267 #build directory hierarchy
288 try: 268 try:
@@ -292,28 +272,30 @@ class PRData(object):
292 raise e 272 raise e
293 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "") 273 uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "")
294 logger.debug("Opening PRServ database '%s'" % (uri)) 274 logger.debug("Opening PRServ database '%s'" % (uri))
295 self.connection=sqlite3.connect(uri, uri=True, isolation_level="EXCLUSIVE", check_same_thread = False) 275 self.connection=sqlite3.connect(uri, uri=True)
296 self.connection.row_factory=sqlite3.Row 276 self.connection.row_factory=sqlite3.Row
297 if not self.read_only: 277 self.connection.execute("PRAGMA synchronous = OFF;")
298 self.connection.execute("pragma synchronous = off;") 278 self.connection.execute("PRAGMA journal_mode = WAL;")
299 self.connection.execute("PRAGMA journal_mode = MEMORY;") 279 self.connection.commit()
300 self._tables={} 280 self._tables={}
301 281
302 def disconnect(self): 282 def disconnect(self):
283 self.connection.commit()
303 self.connection.close() 284 self.connection.close()
304 285
305 def __getitem__(self,tblname): 286 def __getitem__(self, tblname):
306 if not isinstance(tblname, str): 287 if not isinstance(tblname, str):
307 raise TypeError("tblname argument must be a string, not '%s'" % 288 raise TypeError("tblname argument must be a string, not '%s'" %
308 type(tblname)) 289 type(tblname))
309 if tblname in self._tables: 290 if tblname in self._tables:
310 return self._tables[tblname] 291 return self._tables[tblname]
311 else: 292 else:
312 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist, self.read_only) 293 tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.read_only)
313 return tableobj 294 return tableobj
314 295
315 def __delitem__(self, tblname): 296 def __delitem__(self, tblname):
316 if tblname in self._tables: 297 if tblname in self._tables:
317 del self._tables[tblname] 298 del self._tables[tblname]
318 logger.info("drop table %s" % (tblname)) 299 logger.info("drop table %s" % (tblname))
319 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 300 self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
301 self.connection.commit()
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index 5fc8863f70..e175886308 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -12,6 +12,7 @@ import sqlite3
12import prserv 12import prserv
13import prserv.db 13import prserv.db
14import errno 14import errno
15from . import create_async_client, revision_smaller, increase_revision
15import bb.asyncrpc 16import bb.asyncrpc
16 17
17logger = logging.getLogger("BitBake.PRserv") 18logger = logging.getLogger("BitBake.PRserv")
@@ -20,16 +21,19 @@ PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
20singleton = None 21singleton = None
21 22
22class PRServerClient(bb.asyncrpc.AsyncServerConnection): 23class PRServerClient(bb.asyncrpc.AsyncServerConnection):
23 def __init__(self, socket, table, read_only): 24 def __init__(self, socket, server):
24 super().__init__(socket, 'PRSERVICE', logger) 25 super().__init__(socket, "PRSERVICE", server.logger)
26 self.server = server
27
25 self.handlers.update({ 28 self.handlers.update({
26 'get-pr': self.handle_get_pr, 29 "get-pr": self.handle_get_pr,
27 'import-one': self.handle_import_one, 30 "test-pr": self.handle_test_pr,
28 'export': self.handle_export, 31 "test-package": self.handle_test_package,
29 'is-readonly': self.handle_is_readonly, 32 "max-package-pr": self.handle_max_package_pr,
33 "import-one": self.handle_import_one,
34 "export": self.handle_export,
35 "is-readonly": self.handle_is_readonly,
30 }) 36 })
31 self.table = table
32 self.read_only = read_only
33 37
34 def validate_proto_version(self): 38 def validate_proto_version(self):
35 return (self.proto_version == (1, 0)) 39 return (self.proto_version == (1, 0))
@@ -38,104 +42,213 @@ class PRServerClient(bb.asyncrpc.AsyncServerConnection):
38 try: 42 try:
39 return await super().dispatch_message(msg) 43 return await super().dispatch_message(msg)
40 except: 44 except:
41 self.table.sync()
42 raise 45 raise
43 else: 46
44 self.table.sync_if_dirty() 47 async def handle_test_pr(self, request):
48 '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value'''
49 version = request["version"]
50 pkgarch = request["pkgarch"]
51 checksum = request["checksum"]
52 history = request["history"]
53
54 value = self.server.table.find_value(version, pkgarch, checksum, history)
55 return {"value": value}
56
57 async def handle_test_package(self, request):
58 '''Tells whether there are entries for (version, pkgarch) in the db. Returns True or False'''
59 version = request["version"]
60 pkgarch = request["pkgarch"]
61
62 value = self.server.table.test_package(version, pkgarch)
63 return {"value": value}
64
65 async def handle_max_package_pr(self, request):
66 '''Finds the greatest PR value for (version, pkgarch) in the db. Returns None if no entry was found'''
67 version = request["version"]
68 pkgarch = request["pkgarch"]
69
70 value = self.server.table.find_package_max_value(version, pkgarch)
71 return {"value": value}
45 72
46 async def handle_get_pr(self, request): 73 async def handle_get_pr(self, request):
47 version = request['version'] 74 version = request["version"]
48 pkgarch = request['pkgarch'] 75 pkgarch = request["pkgarch"]
49 checksum = request['checksum'] 76 checksum = request["checksum"]
77 history = request["history"]
50 78
51 response = None 79 if self.upstream_client is None:
52 try: 80 value = self.server.table.get_value(version, pkgarch, checksum, history)
53 value = self.table.getValue(version, pkgarch, checksum) 81 return {"value": value}
54 response = {'value': value}
55 except prserv.NotFoundError:
56 logger.error("can not find value for (%s, %s)",version, checksum)
57 except sqlite3.Error as exc:
58 logger.error(str(exc))
59 82
60 return response 83 # We have an upstream server.
84 # Check whether the local server already knows the requested configuration.
85 # If the configuration is a new one, the generated value we will add will
86 # depend on what's on the upstream server. That's why we're calling find_value()
87 # instead of get_value() directly.
88
89 value = self.server.table.find_value(version, pkgarch, checksum, history)
90 upstream_max = await self.upstream_client.max_package_pr(version, pkgarch)
91
92 if value is not None:
93
94 # The configuration is already known locally.
95
96 if history:
97 value = self.server.table.get_value(version, pkgarch, checksum, history)
98 else:
99 existing_value = value
100 # In "no history", we need to make sure the value doesn't decrease
101 # and is at least greater than the maximum upstream value
102 # and the maximum local value
103
104 local_max = self.server.table.find_package_max_value(version, pkgarch)
105 if revision_smaller(value, local_max):
106 value = increase_revision(local_max)
107
108 if revision_smaller(value, upstream_max):
109 # Ask upstream whether it knows the checksum
110 upstream_value = await self.upstream_client.test_pr(version, pkgarch, checksum)
111 if upstream_value is None:
112 # Upstream doesn't have our checksum, let create a new one
113 value = upstream_max + ".0"
114 else:
115 # Fine to take the same value as upstream
116 value = upstream_max
117
118 if not value == existing_value and not self.server.read_only:
119 self.server.table.store_value(version, pkgarch, checksum, value)
120
121 return {"value": value}
122
123 # The configuration is a new one for the local server
124 # Let's ask the upstream server whether it knows it
125
126 known_upstream = await self.upstream_client.test_package(version, pkgarch)
127
128 if not known_upstream:
129
130 # The package is not known upstream, must be a local-only package
131 # Let's compute the PR number using the local-only method
132
133 value = self.server.table.get_value(version, pkgarch, checksum, history)
134 return {"value": value}
135
136 # The package is known upstream, let's ask the upstream server
137 # whether it knows our new output hash
138
139 value = await self.upstream_client.test_pr(version, pkgarch, checksum)
140
141 if value is not None:
142
143 # Upstream knows this output hash, let's store it and use it too.
144
145 if not self.server.read_only:
146 self.server.table.store_value(version, pkgarch, checksum, value)
147 # If the local server is read only, won't be able to store the new
148 # value in the database and will have to keep asking the upstream server
149 return {"value": value}
150
151 # The output hash doesn't exist upstream, get the most recent number from upstream (x)
152 # Then, we want to have a new PR value for the local server: x.y
153
154 upstream_max = await self.upstream_client.max_package_pr(version, pkgarch)
155 # Here we know that the package is known upstream, so upstream_max can't be None
156 subvalue = self.server.table.find_new_subvalue(version, pkgarch, upstream_max)
157
158 if not self.server.read_only:
159 self.server.table.store_value(version, pkgarch, checksum, subvalue)
160
161 return {"value": subvalue}
162
163 async def process_requests(self):
164 if self.server.upstream is not None:
165 self.upstream_client = await create_async_client(self.server.upstream)
166 else:
167 self.upstream_client = None
168
169 try:
170 await super().process_requests()
171 finally:
172 if self.upstream_client is not None:
173 await self.upstream_client.close()
61 174
62 async def handle_import_one(self, request): 175 async def handle_import_one(self, request):
63 response = None 176 response = None
64 if not self.read_only: 177 if not self.server.read_only:
65 version = request['version'] 178 version = request["version"]
66 pkgarch = request['pkgarch'] 179 pkgarch = request["pkgarch"]
67 checksum = request['checksum'] 180 checksum = request["checksum"]
68 value = request['value'] 181 value = request["value"]
69 182
70 value = self.table.importone(version, pkgarch, checksum, value) 183 value = self.server.table.importone(version, pkgarch, checksum, value)
71 if value is not None: 184 if value is not None:
72 response = {'value': value} 185 response = {"value": value}
73 186
74 return response 187 return response
75 188
76 async def handle_export(self, request): 189 async def handle_export(self, request):
77 version = request['version'] 190 version = request["version"]
78 pkgarch = request['pkgarch'] 191 pkgarch = request["pkgarch"]
79 checksum = request['checksum'] 192 checksum = request["checksum"]
80 colinfo = request['colinfo'] 193 colinfo = request["colinfo"]
194 history = request["history"]
81 195
82 try: 196 try:
83 (metainfo, datainfo) = self.table.export(version, pkgarch, checksum, colinfo) 197 (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo, history)
84 except sqlite3.Error as exc: 198 except sqlite3.Error as exc:
85 logger.error(str(exc)) 199 self.logger.error(str(exc))
86 metainfo = datainfo = None 200 metainfo = datainfo = None
87 201
88 return {'metainfo': metainfo, 'datainfo': datainfo} 202 return {"metainfo": metainfo, "datainfo": datainfo}
89 203
90 async def handle_is_readonly(self, request): 204 async def handle_is_readonly(self, request):
91 return {'readonly': self.read_only} 205 return {"readonly": self.server.read_only}
92 206
93class PRServer(bb.asyncrpc.AsyncServer): 207class PRServer(bb.asyncrpc.AsyncServer):
94 def __init__(self, dbfile, read_only=False): 208 def __init__(self, dbfile, read_only=False, upstream=None):
95 super().__init__(logger) 209 super().__init__(logger)
96 self.dbfile = dbfile 210 self.dbfile = dbfile
97 self.table = None 211 self.table = None
98 self.read_only = read_only 212 self.read_only = read_only
213 self.upstream = upstream
99 214
100 def accept_client(self, socket): 215 def accept_client(self, socket):
101 return PRServerClient(socket, self.table, self.read_only) 216 return PRServerClient(socket, self)
102 217
103 def start(self): 218 def start(self):
104 tasks = super().start() 219 tasks = super().start()
105 self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only) 220 self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only)
106 self.table = self.db["PRMAIN"] 221 self.table = self.db["PRMAIN"]
107 222
108 logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" % 223 self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" %
109 (self.dbfile, self.address, str(os.getpid()))) 224 (self.dbfile, self.address, str(os.getpid())))
110 225
226 if self.upstream is not None:
227 self.logger.info("And upstream PRServer: %s " % (self.upstream))
228
111 return tasks 229 return tasks
112 230
113 async def stop(self): 231 async def stop(self):
114 self.table.sync_if_dirty()
115 self.db.disconnect() 232 self.db.disconnect()
116 await super().stop() 233 await super().stop()
117 234
118 def signal_handler(self):
119 super().signal_handler()
120 if self.table:
121 self.table.sync()
122
123class PRServSingleton(object): 235class PRServSingleton(object):
124 def __init__(self, dbfile, logfile, host, port): 236 def __init__(self, dbfile, logfile, host, port, upstream):
125 self.dbfile = dbfile 237 self.dbfile = dbfile
126 self.logfile = logfile 238 self.logfile = logfile
127 self.host = host 239 self.host = host
128 self.port = port 240 self.port = port
241 self.upstream = upstream
129 242
130 def start(self): 243 def start(self):
131 self.prserv = PRServer(self.dbfile) 244 self.prserv = PRServer(self.dbfile, upstream=self.upstream)
132 self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port) 245 self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port)
133 self.process = self.prserv.serve_as_process(log_level=logging.WARNING) 246 self.process = self.prserv.serve_as_process(log_level=logging.WARNING)
134 247
135 if not self.prserv.address: 248 if not self.prserv.address:
136 raise PRServiceConfigError 249 raise PRServiceConfigError
137 if not self.port: 250 if not self.port:
138 self.port = int(self.prserv.address.rsplit(':', 1)[1]) 251 self.port = int(self.prserv.address.rsplit(":", 1)[1])
139 252
140def run_as_daemon(func, pidfile, logfile): 253def run_as_daemon(func, pidfile, logfile):
141 """ 254 """
@@ -171,18 +284,18 @@ def run_as_daemon(func, pidfile, logfile):
171 # stdout/stderr or it could be 'real' unix fd forking where we need 284 # stdout/stderr or it could be 'real' unix fd forking where we need
172 # to physically close the fds to prevent the program launching us from 285 # to physically close the fds to prevent the program launching us from
173 # potentially hanging on a pipe. Handle both cases. 286 # potentially hanging on a pipe. Handle both cases.
174 si = open('/dev/null', 'r') 287 si = open("/dev/null", "r")
175 try: 288 try:
176 os.dup2(si.fileno(),sys.stdin.fileno()) 289 os.dup2(si.fileno(), sys.stdin.fileno())
177 except (AttributeError, io.UnsupportedOperation): 290 except (AttributeError, io.UnsupportedOperation):
178 sys.stdin = si 291 sys.stdin = si
179 so = open(logfile, 'a+') 292 so = open(logfile, "a+")
180 try: 293 try:
181 os.dup2(so.fileno(),sys.stdout.fileno()) 294 os.dup2(so.fileno(), sys.stdout.fileno())
182 except (AttributeError, io.UnsupportedOperation): 295 except (AttributeError, io.UnsupportedOperation):
183 sys.stdout = so 296 sys.stdout = so
184 try: 297 try:
185 os.dup2(so.fileno(),sys.stderr.fileno()) 298 os.dup2(so.fileno(), sys.stderr.fileno())
186 except (AttributeError, io.UnsupportedOperation): 299 except (AttributeError, io.UnsupportedOperation):
187 sys.stderr = so 300 sys.stderr = so
188 301
@@ -200,14 +313,14 @@ def run_as_daemon(func, pidfile, logfile):
200 313
201 # write pidfile 314 # write pidfile
202 pid = str(os.getpid()) 315 pid = str(os.getpid())
203 with open(pidfile, 'w') as pf: 316 with open(pidfile, "w") as pf:
204 pf.write("%s\n" % pid) 317 pf.write("%s\n" % pid)
205 318
206 func() 319 func()
207 os.remove(pidfile) 320 os.remove(pidfile)
208 os._exit(0) 321 os._exit(0)
209 322
210def start_daemon(dbfile, host, port, logfile, read_only=False): 323def start_daemon(dbfile, host, port, logfile, read_only=False, upstream=None):
211 ip = socket.gethostbyname(host) 324 ip = socket.gethostbyname(host)
212 pidfile = PIDPREFIX % (ip, port) 325 pidfile = PIDPREFIX % (ip, port)
213 try: 326 try:
@@ -223,7 +336,7 @@ def start_daemon(dbfile, host, port, logfile, read_only=False):
223 336
224 dbfile = os.path.abspath(dbfile) 337 dbfile = os.path.abspath(dbfile)
225 def daemon_main(): 338 def daemon_main():
226 server = PRServer(dbfile, read_only=read_only) 339 server = PRServer(dbfile, read_only=read_only, upstream=upstream)
227 server.start_tcp_server(ip, port) 340 server.start_tcp_server(ip, port)
228 server.serve_forever() 341 server.serve_forever()
229 342
@@ -245,15 +358,15 @@ def stop_daemon(host, port):
245 # so at least advise the user which ports the corresponding server is listening 358 # so at least advise the user which ports the corresponding server is listening
246 ports = [] 359 ports = []
247 portstr = "" 360 portstr = ""
248 for pf in glob.glob(PIDPREFIX % (ip,'*')): 361 for pf in glob.glob(PIDPREFIX % (ip, "*")):
249 bn = os.path.basename(pf) 362 bn = os.path.basename(pf)
250 root, _ = os.path.splitext(bn) 363 root, _ = os.path.splitext(bn)
251 ports.append(root.split('_')[-1]) 364 ports.append(root.split("_")[-1])
252 if len(ports): 365 if len(ports):
253 portstr = "Wrong port? Other ports listening at %s: %s" % (host, ' '.join(ports)) 366 portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports))
254 367
255 sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n" 368 sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n"
256 % (pidfile,portstr)) 369 % (pidfile, portstr))
257 return 1 370 return 1
258 371
259 try: 372 try:
@@ -284,7 +397,7 @@ def is_running(pid):
284 return True 397 return True
285 398
286def is_local_special(host, port): 399def is_local_special(host, port):
287 if (host == 'localhost' or host == '127.0.0.1') and not port: 400 if (host == "localhost" or host == "127.0.0.1") and not port:
288 return True 401 return True
289 else: 402 else:
290 return False 403 return False
@@ -295,7 +408,7 @@ class PRServiceConfigError(Exception):
295def auto_start(d): 408def auto_start(d):
296 global singleton 409 global singleton
297 410
298 host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':'))) 411 host_params = list(filter(None, (d.getVar("PRSERV_HOST") or "").split(":")))
299 if not host_params: 412 if not host_params:
300 # Shutdown any existing PR Server 413 # Shutdown any existing PR Server
301 auto_shutdown() 414 auto_shutdown()
@@ -304,12 +417,15 @@ def auto_start(d):
304 if len(host_params) != 2: 417 if len(host_params) != 2:
305 # Shutdown any existing PR Server 418 # Shutdown any existing PR Server
306 auto_shutdown() 419 auto_shutdown()
307 logger.critical('\n'.join(['PRSERV_HOST: incorrect format', 420 logger.critical("\n".join(["PRSERV_HOST: incorrect format",
308 'Usage: PRSERV_HOST = "<hostname>:<port>"'])) 421 'Usage: PRSERV_HOST = "<hostname>:<port>"']))
309 raise PRServiceConfigError 422 raise PRServiceConfigError
310 423
311 host = host_params[0].strip().lower() 424 host = host_params[0].strip().lower()
312 port = int(host_params[1]) 425 port = int(host_params[1])
426
427 upstream = d.getVar("PRSERV_UPSTREAM") or None
428
313 if is_local_special(host, port): 429 if is_local_special(host, port):
314 import bb.utils 430 import bb.utils
315 cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE")) 431 cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE"))
@@ -324,7 +440,7 @@ def auto_start(d):
324 auto_shutdown() 440 auto_shutdown()
325 if not singleton: 441 if not singleton:
326 bb.utils.mkdirhier(cachedir) 442 bb.utils.mkdirhier(cachedir)
327 singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port) 443 singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port, upstream)
328 singleton.start() 444 singleton.start()
329 if singleton: 445 if singleton:
330 host = singleton.host 446 host = singleton.host
@@ -357,8 +473,8 @@ def connect(host, port):
357 473
358 global singleton 474 global singleton
359 475
360 if host.strip().lower() == 'localhost' and not port: 476 if host.strip().lower() == "localhost" and not port:
361 host = 'localhost' 477 host = "localhost"
362 port = singleton.port 478 port = singleton.port
363 479
364 conn = client.PRClient() 480 conn = client.PRClient()
diff --git a/bitbake/lib/prserv/tests.py b/bitbake/lib/prserv/tests.py
new file mode 100644
index 0000000000..df0c003003
--- /dev/null
+++ b/bitbake/lib/prserv/tests.py
@@ -0,0 +1,388 @@
1#! /usr/bin/env python3
2#
3# Copyright (C) 2024 BitBake Contributors
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7
8from . import create_server, create_client, increase_revision, revision_greater, revision_smaller, _revision_greater_or_equal
9import prserv.db as db
10from bb.asyncrpc import InvokeError
11import logging
12import os
13import sys
14import tempfile
15import unittest
16import socket
17import subprocess
18from pathlib import Path
19
20THIS_DIR = Path(__file__).parent
21BIN_DIR = THIS_DIR.parent.parent / "bin"
22
23version = "dummy-1.0-r0"
24pkgarch = "core2-64"
25other_arch = "aarch64"
26
27checksumX = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4f0"
28checksum0 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a0"
29checksum1 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a1"
30checksum2 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a2"
31checksum3 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a3"
32checksum4 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a4"
33checksum5 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a5"
34checksum6 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a6"
35checksum7 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a7"
36checksum8 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a8"
37checksum9 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a9"
38checksum10 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4aa"
39
40def server_prefunc(server, name):
41 logging.basicConfig(level=logging.DEBUG, filename='prserv-%s.log' % name, filemode='w',
42 format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
43 server.logger.debug("Running server %s" % name)
44 sys.stdout = open('prserv-stdout-%s.log' % name, 'w')
45 sys.stderr = sys.stdout
46
47class PRTestSetup(object):
48
49 def start_server(self, name, dbfile, upstream=None, read_only=False, prefunc=server_prefunc):
50
51 def cleanup_server(server):
52 if server.process.exitcode is not None:
53 return
54 server.process.terminate()
55 server.process.join()
56
57 server = create_server(socket.gethostbyname("localhost") + ":0",
58 dbfile,
59 upstream=upstream,
60 read_only=read_only)
61
62 server.serve_as_process(prefunc=prefunc, args=(name,))
63 self.addCleanup(cleanup_server, server)
64
65 return server
66
67 def start_client(self, server_address):
68 def cleanup_client(client):
69 client.close()
70
71 client = create_client(server_address)
72 self.addCleanup(cleanup_client, client)
73
74 return client
75
76class FunctionTests(unittest.TestCase):
77
78 def setUp(self):
79 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
80 self.addCleanup(self.temp_dir.cleanup)
81
82 def test_increase_revision(self):
83 self.assertEqual(increase_revision("1"), "2")
84 self.assertEqual(increase_revision("1.0"), "1.1")
85 self.assertEqual(increase_revision("1.1.1"), "1.1.2")
86 self.assertEqual(increase_revision("1.1.1.3"), "1.1.1.4")
87 self.assertEqual(increase_revision("9"), "10")
88 self.assertEqual(increase_revision("1.9"), "1.10")
89 self.assertRaises(ValueError, increase_revision, "1.a")
90 self.assertRaises(ValueError, increase_revision, "1.")
91 self.assertRaises(ValueError, increase_revision, "")
92
93 def test_revision_greater_or_equal(self):
94 self.assertTrue(_revision_greater_or_equal("2", "2"))
95 self.assertTrue(_revision_greater_or_equal("2", "1"))
96 self.assertTrue(_revision_greater_or_equal("10", "2"))
97 self.assertTrue(_revision_greater_or_equal("1.10", "1.2"))
98 self.assertFalse(_revision_greater_or_equal("1.2", "1.10"))
99 self.assertTrue(_revision_greater_or_equal("1.10", "1"))
100 self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10"))
101 self.assertFalse(_revision_greater_or_equal("1.10.1", "1.10.2"))
102 self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10.1"))
103 self.assertTrue(_revision_greater_or_equal("1.10.1", "1"))
104 self.assertTrue(revision_greater("1.20", "1.3"))
105 self.assertTrue(revision_smaller("1.3", "1.20"))
106
107 # DB tests
108
109 def test_db(self):
110 dbfile = os.path.join(self.temp_dir.name, "testtable.sqlite3")
111
112 self.db = db.PRData(dbfile)
113 self.table = self.db["PRMAIN"]
114
115 self.table.store_value(version, pkgarch, checksum0, "0")
116 self.table.store_value(version, pkgarch, checksum1, "1")
117 # "No history" mode supports multiple PRs for the same checksum
118 self.table.store_value(version, pkgarch, checksum0, "2")
119 self.table.store_value(version, pkgarch, checksum2, "1.0")
120
121 self.assertTrue(self.table.test_package(version, pkgarch))
122 self.assertFalse(self.table.test_package(version, other_arch))
123
124 self.assertTrue(self.table.test_value(version, pkgarch, "0"))
125 self.assertTrue(self.table.test_value(version, pkgarch, "1"))
126 self.assertTrue(self.table.test_value(version, pkgarch, "2"))
127
128 self.assertEqual(self.table.find_package_max_value(version, pkgarch), "2")
129
130 self.assertEqual(self.table.find_min_value(version, pkgarch, checksum0), "0")
131 self.assertEqual(self.table.find_max_value(version, pkgarch, checksum0), "2")
132
133 # Test history modes
134 self.assertEqual(self.table.find_value(version, pkgarch, checksum0, True), "0")
135 self.assertEqual(self.table.find_value(version, pkgarch, checksum0, False), "2")
136
137 self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "3"), "3.0")
138 self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "1"), "1.1")
139
140 # Revision comparison tests
141 self.table.store_value(version, pkgarch, checksum1, "1.3")
142 self.table.store_value(version, pkgarch, checksum1, "1.20")
143 self.assertEqual(self.table.find_min_value(version, pkgarch, checksum1), "1")
144 self.assertEqual(self.table.find_max_value(version, pkgarch, checksum1), "1.20")
145
146class PRBasicTests(PRTestSetup, unittest.TestCase):
147
148 def setUp(self):
149 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
150 self.addCleanup(self.temp_dir.cleanup)
151
152 dbfile = os.path.join(self.temp_dir.name, "prtest-basic.sqlite3")
153
154 self.server1 = self.start_server("basic", dbfile)
155 self.client1 = self.start_client(self.server1.address)
156
157 def test_basic(self):
158
159 # Checks on non existing configuration
160
161 result = self.client1.test_pr(version, pkgarch, checksum0)
162 self.assertIsNone(result, "test_pr should return 'None' for a non existing PR")
163
164 result = self.client1.test_package(version, pkgarch)
165 self.assertFalse(result, "test_package should return 'False' for a non existing PR")
166
167 result = self.client1.max_package_pr(version, pkgarch)
168 self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR")
169
170 # Add a first configuration
171
172 result = self.client1.getPR(version, pkgarch, checksum0)
173 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
174
175 result = self.client1.test_pr(version, pkgarch, checksum0)
176 self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR")
177
178 result = self.client1.test_package(version, pkgarch)
179 self.assertTrue(result, "test_package should return 'True' for an existing PR")
180
181 result = self.client1.max_package_pr(version, pkgarch)
182 self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series")
183
184 # Check that the same request gets the same value
185
186 result = self.client1.getPR(version, pkgarch, checksum0)
187 self.assertEqual(result, "0", "getPR: asking for the same PR a second time in a row should return the same value.")
188
189 # Add new configurations
190
191 result = self.client1.getPR(version, pkgarch, checksum1)
192 self.assertEqual(result, "1", "getPR: second PR of a package should be '1'")
193
194 result = self.client1.test_pr(version, pkgarch, checksum1)
195 self.assertEqual(result, "1", "test_pr should return '1' here, matching the result of getPR")
196
197 result = self.client1.max_package_pr(version, pkgarch)
198 self.assertEqual(result, "1", "max_package_pr should return '1' in the current test series")
199
200 result = self.client1.getPR(version, pkgarch, checksum2)
201 self.assertEqual(result, "2", "getPR: second PR of a package should be '2'")
202
203 result = self.client1.test_pr(version, pkgarch, checksum2)
204 self.assertEqual(result, "2", "test_pr should return '2' here, matching the result of getPR")
205
206 result = self.client1.max_package_pr(version, pkgarch)
207 self.assertEqual(result, "2", "max_package_pr should return '2' in the current test series")
208
209 result = self.client1.getPR(version, pkgarch, checksum3)
210 self.assertEqual(result, "3", "getPR: second PR of a package should be '3'")
211
212 result = self.client1.test_pr(version, pkgarch, checksum3)
213 self.assertEqual(result, "3", "test_pr should return '3' here, matching the result of getPR")
214
215 result = self.client1.max_package_pr(version, pkgarch)
216 self.assertEqual(result, "3", "max_package_pr should return '3' in the current test series")
217
218 # Ask again for the first configuration
219
220 result = self.client1.getPR(version, pkgarch, checksum0)
221 self.assertEqual(result, "4", "getPR: should return '4' in this configuration")
222
223 # Ask again with explicit "no history" mode
224
225 result = self.client1.getPR(version, pkgarch, checksum0, False)
226 self.assertEqual(result, "4", "getPR: should return '4' in this configuration")
227
228 # Ask again with explicit "history" mode. This should return the first recorded PR for checksum0
229
230 result = self.client1.getPR(version, pkgarch, checksum0, True)
231 self.assertEqual(result, "0", "getPR: should return '0' in this configuration")
232
233 # Check again that another pkgarg resets the counters
234
235 result = self.client1.test_pr(version, other_arch, checksum0)
236 self.assertIsNone(result, "test_pr should return 'None' for a non existing PR")
237
238 result = self.client1.test_package(version, other_arch)
239 self.assertFalse(result, "test_package should return 'False' for a non existing PR")
240
241 result = self.client1.max_package_pr(version, other_arch)
242 self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR")
243
244 # Now add the configuration
245
246 result = self.client1.getPR(version, other_arch, checksum0)
247 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
248
249 result = self.client1.test_pr(version, other_arch, checksum0)
250 self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR")
251
252 result = self.client1.test_package(version, other_arch)
253 self.assertTrue(result, "test_package should return 'True' for an existing PR")
254
255 result = self.client1.max_package_pr(version, other_arch)
256 self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series")
257
258 result = self.client1.is_readonly()
259 self.assertFalse(result, "Server should not be described as 'read-only'")
260
261class PRUpstreamTests(PRTestSetup, unittest.TestCase):
262
263 def setUp(self):
264
265 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
266 self.addCleanup(self.temp_dir.cleanup)
267
268 dbfile2 = os.path.join(self.temp_dir.name, "prtest-upstream2.sqlite3")
269 self.server2 = self.start_server("upstream2", dbfile2)
270 self.client2 = self.start_client(self.server2.address)
271
272 dbfile1 = os.path.join(self.temp_dir.name, "prtest-upstream1.sqlite3")
273 self.server1 = self.start_server("upstream1", dbfile1, upstream=self.server2.address)
274 self.client1 = self.start_client(self.server1.address)
275
276 dbfile0 = os.path.join(self.temp_dir.name, "prtest-local.sqlite3")
277 self.server0 = self.start_server("local", dbfile0, upstream=self.server1.address)
278 self.client0 = self.start_client(self.server0.address)
279 self.shared_db = dbfile0
280
281 def test_upstream_and_readonly(self):
282
283 # For identical checksums, all servers should return the same PR
284
285 result = self.client2.getPR(version, pkgarch, checksum0)
286 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
287
288 result = self.client1.getPR(version, pkgarch, checksum0)
289 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)")
290
291 result = self.client0.getPR(version, pkgarch, checksum0)
292 self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)")
293
294 # Now introduce new checksums on server1 for, same version
295
296 result = self.client1.getPR(version, pkgarch, checksum1)
297 self.assertEqual(result, "0.0", "getPR: first PR of a package which has a different checksum upstream should be '0.0'")
298
299 result = self.client1.getPR(version, pkgarch, checksum2)
300 self.assertEqual(result, "0.1", "getPR: second PR of a package that has a different checksum upstream should be '0.1'")
301
302 # Now introduce checksums on server0 for, same version
303
304 result = self.client1.getPR(version, pkgarch, checksum1)
305 self.assertEqual(result, "0.2", "getPR: can't decrease for known PR")
306
307 result = self.client1.getPR(version, pkgarch, checksum2)
308 self.assertEqual(result, "0.3")
309
310 result = self.client1.max_package_pr(version, pkgarch)
311 self.assertEqual(result, "0.3")
312
313 result = self.client0.getPR(version, pkgarch, checksum3)
314 self.assertEqual(result, "0.3.0", "getPR: first PR of a package that doesn't exist upstream should be '0.3.0'")
315
316 result = self.client0.getPR(version, pkgarch, checksum4)
317 self.assertEqual(result, "0.3.1", "getPR: second PR of a package that doesn't exist upstream should be '0.3.1'")
318
319 result = self.client0.getPR(version, pkgarch, checksum3)
320 self.assertEqual(result, "0.3.2")
321
322 # More upstream updates
323 # Here, we assume no communication between server2 and server0. server2 only impacts server0
324 # after impacting server1
325
326 self.assertEqual(self.client2.getPR(version, pkgarch, checksum5), "1")
327 self.assertEqual(self.client1.getPR(version, pkgarch, checksum6), "1.0")
328 self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "1.1")
329 self.assertEqual(self.client0.getPR(version, pkgarch, checksum8), "1.1.0")
330 self.assertEqual(self.client0.getPR(version, pkgarch, checksum9), "1.1.1")
331
332 # "history" mode tests
333
334 self.assertEqual(self.client2.getPR(version, pkgarch, checksum0, True), "0")
335 self.assertEqual(self.client1.getPR(version, pkgarch, checksum2, True), "0.1")
336 self.assertEqual(self.client0.getPR(version, pkgarch, checksum3, True), "0.3.0")
337
338 # More "no history" mode tests
339
340 self.assertEqual(self.client2.getPR(version, pkgarch, checksum0), "2")
341 self.assertEqual(self.client1.getPR(version, pkgarch, checksum0), "2") # Same as upstream
342 self.assertEqual(self.client0.getPR(version, pkgarch, checksum0), "2") # Same as upstream
343 self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "3") # This could be surprising, but since the previous revision was "2", increasing it yields "3".
344 # We don't know how many upstream servers we have
345 # Start read-only server with server1 as upstream
346 self.server_ro = self.start_server("local-ro", self.shared_db, upstream=self.server1.address, read_only=True)
347 self.client_ro = self.start_client(self.server_ro.address)
348
349 self.assertTrue(self.client_ro.is_readonly(), "Database should be described as 'read-only'")
350
351 # Checks on non existing configurations
352 self.assertIsNone(self.client_ro.test_pr(version, pkgarch, checksumX))
353 self.assertFalse(self.client_ro.test_package("unknown", pkgarch))
354
355 # Look up existing configurations
356 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0), "3") # "no history" mode
357 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0, True), "0") # "history" mode
358 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3), "3")
359 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3, True), "0.3.0")
360 self.assertEqual(self.client_ro.max_package_pr(version, pkgarch), "2") # normal as "3" was never saved
361
362 # Try to insert a new value. Here this one is know upstream.
363 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum7), "3")
364 # Try to insert a completely new value. As the max upstream value is already "3", it should be "3.0"
365 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum10), "3.0")
366 # Same with another value which only exists in the upstream upstream server
367 # This time, as the upstream server doesn't know it, it will ask its upstream server. So that's a known one.
368 self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum9), "3")
369
370class ScriptTests(unittest.TestCase):
371
372 def setUp(self):
373
374 self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
375 self.addCleanup(self.temp_dir.cleanup)
376 self.dbfile = os.path.join(self.temp_dir.name, "prtest.sqlite3")
377
378 def test_1_start_bitbake_prserv(self):
379 try:
380 subprocess.check_call([BIN_DIR / "bitbake-prserv", "--start", "-f", self.dbfile])
381 except subprocess.CalledProcessError as e:
382 self.fail("Failed to start bitbake-prserv: %s" % e.returncode)
383
384 def test_2_stop_bitbake_prserv(self):
385 try:
386 subprocess.check_call([BIN_DIR / "bitbake-prserv", "--stop"])
387 except subprocess.CalledProcessError as e:
388 self.fail("Failed to stop bitbake-prserv: %s" % e.returncode)