diff options
-rwxr-xr-x | bitbake/bin/bitbake-hashclient | 84 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-hashserv | 37 | ||||
-rw-r--r-- | bitbake/lib/hashserv/__init__.py | 69 | ||||
-rw-r--r-- | bitbake/lib/hashserv/client.py | 62 | ||||
-rw-r--r-- | bitbake/lib/hashserv/server.py | 357 | ||||
-rw-r--r-- | bitbake/lib/hashserv/sqlalchemy.py | 111 | ||||
-rw-r--r-- | bitbake/lib/hashserv/sqlite.py | 105 | ||||
-rw-r--r-- | bitbake/lib/hashserv/tests.py | 276 |
8 files changed, 1054 insertions, 47 deletions
diff --git a/bitbake/bin/bitbake-hashclient b/bitbake/bin/bitbake-hashclient index a02a65b937..328c15cdec 100755 --- a/bitbake/bin/bitbake-hashclient +++ b/bitbake/bin/bitbake-hashclient | |||
@@ -14,6 +14,7 @@ import sys | |||
14 | import threading | 14 | import threading |
15 | import time | 15 | import time |
16 | import warnings | 16 | import warnings |
17 | import netrc | ||
17 | warnings.simplefilter("default") | 18 | warnings.simplefilter("default") |
18 | 19 | ||
19 | try: | 20 | try: |
@@ -36,10 +37,18 @@ except ImportError: | |||
36 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) | 37 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) |
37 | 38 | ||
38 | import hashserv | 39 | import hashserv |
40 | import bb.asyncrpc | ||
39 | 41 | ||
40 | DEFAULT_ADDRESS = 'unix://./hashserve.sock' | 42 | DEFAULT_ADDRESS = 'unix://./hashserve.sock' |
41 | METHOD = 'stress.test.method' | 43 | METHOD = 'stress.test.method' |
42 | 44 | ||
45 | def print_user(u): | ||
46 | print(f"Username: {u['username']}") | ||
47 | if "permissions" in u: | ||
48 | print("Permissions: " + " ".join(u["permissions"])) | ||
49 | if "token" in u: | ||
50 | print(f"Token: {u['token']}") | ||
51 | |||
43 | 52 | ||
44 | def main(): | 53 | def main(): |
45 | def handle_stats(args, client): | 54 | def handle_stats(args, client): |
@@ -125,9 +134,39 @@ def main(): | |||
125 | print("Removed %d rows" % (result["count"])) | 134 | print("Removed %d rows" % (result["count"])) |
126 | return 0 | 135 | return 0 |
127 | 136 | ||
137 | def handle_refresh_token(args, client): | ||
138 | r = client.refresh_token(args.username) | ||
139 | print_user(r) | ||
140 | |||
141 | def handle_set_user_permissions(args, client): | ||
142 | r = client.set_user_perms(args.username, args.permissions) | ||
143 | print_user(r) | ||
144 | |||
145 | def handle_get_user(args, client): | ||
146 | r = client.get_user(args.username) | ||
147 | print_user(r) | ||
148 | |||
149 | def handle_get_all_users(args, client): | ||
150 | users = client.get_all_users() | ||
151 | print("{username:20}| {permissions}".format(username="Username", permissions="Permissions")) | ||
152 | print(("-" * 20) + "+" + ("-" * 20)) | ||
153 | for u in users: | ||
154 | print("{username:20}| {permissions}".format(username=u["username"], permissions=" ".join(u["permissions"]))) | ||
155 | |||
156 | def handle_new_user(args, client): | ||
157 | r = client.new_user(args.username, args.permissions) | ||
158 | print_user(r) | ||
159 | |||
160 | def handle_delete_user(args, client): | ||
161 | r = client.delete_user(args.username) | ||
162 | print_user(r) | ||
163 | |||
128 | parser = argparse.ArgumentParser(description='Hash Equivalence Client') | 164 | parser = argparse.ArgumentParser(description='Hash Equivalence Client') |
129 | parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")') | 165 | parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")') |
130 | parser.add_argument('--log', default='WARNING', help='Set logging level') | 166 | parser.add_argument('--log', default='WARNING', help='Set logging level') |
167 | parser.add_argument('--login', '-l', metavar="USERNAME", help="Authenticate as USERNAME") | ||
168 | parser.add_argument('--password', '-p', metavar="TOKEN", help="Authenticate using token TOKEN") | ||
169 | parser.add_argument('--no-netrc', '-n', action="store_false", dest="netrc", help="Do not use .netrc") | ||
131 | 170 | ||
132 | subparsers = parser.add_subparsers() | 171 | subparsers = parser.add_subparsers() |
133 | 172 | ||
@@ -158,6 +197,31 @@ def main(): | |||
158 | clean_unused_parser.add_argument("max_age", metavar="SECONDS", type=int, help="Remove unused entries older than SECONDS old") | 197 | clean_unused_parser.add_argument("max_age", metavar="SECONDS", type=int, help="Remove unused entries older than SECONDS old") |
159 | clean_unused_parser.set_defaults(func=handle_clean_unused) | 198 | clean_unused_parser.set_defaults(func=handle_clean_unused) |
160 | 199 | ||
200 | refresh_token_parser = subparsers.add_parser('refresh-token', help="Refresh auth token") | ||
201 | refresh_token_parser.add_argument("--username", "-u", help="Refresh the token for another user (if authorized)") | ||
202 | refresh_token_parser.set_defaults(func=handle_refresh_token) | ||
203 | |||
204 | set_user_perms_parser = subparsers.add_parser('set-user-perms', help="Set new permissions for user") | ||
205 | set_user_perms_parser.add_argument("--username", "-u", help="Username", required=True) | ||
206 | set_user_perms_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions") | ||
207 | set_user_perms_parser.set_defaults(func=handle_set_user_permissions) | ||
208 | |||
209 | get_user_parser = subparsers.add_parser('get-user', help="Get user") | ||
210 | get_user_parser.add_argument("--username", "-u", help="Username") | ||
211 | get_user_parser.set_defaults(func=handle_get_user) | ||
212 | |||
213 | get_all_users_parser = subparsers.add_parser('get-all-users', help="List all users") | ||
214 | get_all_users_parser.set_defaults(func=handle_get_all_users) | ||
215 | |||
216 | new_user_parser = subparsers.add_parser('new-user', help="Create new user") | ||
217 | new_user_parser.add_argument("--username", "-u", help="Username", required=True) | ||
218 | new_user_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions") | ||
219 | new_user_parser.set_defaults(func=handle_new_user) | ||
220 | |||
221 | delete_user_parser = subparsers.add_parser('delete-user', help="Delete user") | ||
222 | delete_user_parser.add_argument("--username", "-u", help="Username", required=True) | ||
223 | delete_user_parser.set_defaults(func=handle_delete_user) | ||
224 | |||
161 | args = parser.parse_args() | 225 | args = parser.parse_args() |
162 | 226 | ||
163 | logger = logging.getLogger('hashserv') | 227 | logger = logging.getLogger('hashserv') |
@@ -171,10 +235,26 @@ def main(): | |||
171 | console.setLevel(level) | 235 | console.setLevel(level) |
172 | logger.addHandler(console) | 236 | logger.addHandler(console) |
173 | 237 | ||
238 | login = args.login | ||
239 | password = args.password | ||
240 | |||
241 | if login is None and args.netrc: | ||
242 | try: | ||
243 | n = netrc.netrc() | ||
244 | auth = n.authenticators(args.address) | ||
245 | if auth is not None: | ||
246 | login, _, password = auth | ||
247 | except FileNotFoundError: | ||
248 | pass | ||
249 | |||
174 | func = getattr(args, 'func', None) | 250 | func = getattr(args, 'func', None) |
175 | if func: | 251 | if func: |
176 | with hashserv.create_client(args.address) as client: | 252 | try: |
177 | return func(args, client) | 253 | with hashserv.create_client(args.address, login, password) as client: |
254 | return func(args, client) | ||
255 | except bb.asyncrpc.InvokeError as e: | ||
256 | print(f"ERROR: {e}") | ||
257 | return 1 | ||
178 | 258 | ||
179 | return 0 | 259 | return 0 |
180 | 260 | ||
diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv index 59b8b07f59..1085d0584e 100755 --- a/bitbake/bin/bitbake-hashserv +++ b/bitbake/bin/bitbake-hashserv | |||
@@ -17,6 +17,7 @@ warnings.simplefilter("default") | |||
17 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib")) | 17 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib")) |
18 | 18 | ||
19 | import hashserv | 19 | import hashserv |
20 | from hashserv.server import DEFAULT_ANON_PERMS | ||
20 | 21 | ||
21 | VERSION = "1.0.0" | 22 | VERSION = "1.0.0" |
22 | 23 | ||
@@ -36,6 +37,22 @@ The bind address may take one of the following formats: | |||
36 | To bind to all addresses, leave the ADDRESS empty, e.g. "--bind :8686" or | 37 | To bind to all addresses, leave the ADDRESS empty, e.g. "--bind :8686" or |
37 | "--bind ws://:8686". To bind to a specific IPv6 address, enclose the address in | 38 | "--bind ws://:8686". To bind to a specific IPv6 address, enclose the address in |
38 | "[]", e.g. "--bind [::1]:8686" or "--bind ws://[::1]:8686" | 39 | "[]", e.g. "--bind [::1]:8686" or "--bind ws://[::1]:8686" |
40 | |||
41 | Note that the default Anonymous permissions are designed to not break existing | ||
42 | server instances when upgrading, but are not particularly secure defaults. If | ||
43 | you want to use authentication, it is recommended that you use "--anon-perms | ||
44 | @read" to only give anonymous users read access, or "--anon-perms @none" to | ||
45 | give un-authenticated users no access at all. | ||
46 | |||
47 | Setting "--anon-perms @all" or "--anon-perms @user-admin" is not allowed, since | ||
48 | this would allow anonymous users to manage all users accounts, which is a bad | ||
49 | idea. | ||
50 | |||
51 | If you are using user authentication, you should run your server in websockets | ||
52 | mode with an SSL terminating load balancer in front of it (as this server does | ||
53 | not implement SSL). Otherwise all usernames and passwords will be transmitted | ||
54 | in the clear. When configured this way, clients can connect using a secure | ||
55 | websocket, as in "wss://SERVER:PORT" | ||
39 | """, | 56 | """, |
40 | ) | 57 | ) |
41 | 58 | ||
@@ -79,6 +96,22 @@ To bind to all addresses, leave the ADDRESS empty, e.g. "--bind :8686" or | |||
79 | default=os.environ.get("HASHSERVER_DB_PASSWORD", None), | 96 | default=os.environ.get("HASHSERVER_DB_PASSWORD", None), |
80 | help="Database password ($HASHSERVER_DB_PASSWORD)", | 97 | help="Database password ($HASHSERVER_DB_PASSWORD)", |
81 | ) | 98 | ) |
99 | parser.add_argument( | ||
100 | "--anon-perms", | ||
101 | metavar="PERM[,PERM[,...]]", | ||
102 | default=os.environ.get("HASHSERVER_ANON_PERMS", ",".join(DEFAULT_ANON_PERMS)), | ||
103 | help='Permissions to give anonymous users (default $HASHSERVER_ANON_PERMS, "%(default)s")', | ||
104 | ) | ||
105 | parser.add_argument( | ||
106 | "--admin-user", | ||
107 | default=os.environ.get("HASHSERVER_ADMIN_USER", None), | ||
108 | help="Create default admin user with name ADMIN_USER ($HASHSERVER_ADMIN_USER)", | ||
109 | ) | ||
110 | parser.add_argument( | ||
111 | "--admin-password", | ||
112 | default=os.environ.get("HASHSERVER_ADMIN_PASSWORD", None), | ||
113 | help="Create default admin user with password ADMIN_PASSWORD ($HASHSERVER_ADMIN_PASSWORD)", | ||
114 | ) | ||
82 | 115 | ||
83 | args = parser.parse_args() | 116 | args = parser.parse_args() |
84 | 117 | ||
@@ -94,6 +127,7 @@ To bind to all addresses, leave the ADDRESS empty, e.g. "--bind :8686" or | |||
94 | logger.addHandler(console) | 127 | logger.addHandler(console) |
95 | 128 | ||
96 | read_only = (os.environ.get("HASHSERVER_READ_ONLY", "0") == "1") or args.read_only | 129 | read_only = (os.environ.get("HASHSERVER_READ_ONLY", "0") == "1") or args.read_only |
130 | anon_perms = args.anon_perms.split(",") | ||
97 | 131 | ||
98 | server = hashserv.create_server( | 132 | server = hashserv.create_server( |
99 | args.bind, | 133 | args.bind, |
@@ -102,6 +136,9 @@ To bind to all addresses, leave the ADDRESS empty, e.g. "--bind :8686" or | |||
102 | read_only=read_only, | 136 | read_only=read_only, |
103 | db_username=args.db_username, | 137 | db_username=args.db_username, |
104 | db_password=args.db_password, | 138 | db_password=args.db_password, |
139 | anon_perms=anon_perms, | ||
140 | admin_username=args.admin_user, | ||
141 | admin_password=args.admin_password, | ||
105 | ) | 142 | ) |
106 | server.serve_forever() | 143 | server.serve_forever() |
107 | return 0 | 144 | return 0 |
diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py index 9a8ee4e88b..552a33278f 100644 --- a/bitbake/lib/hashserv/__init__.py +++ b/bitbake/lib/hashserv/__init__.py | |||
@@ -8,6 +8,7 @@ from contextlib import closing | |||
8 | import re | 8 | import re |
9 | import itertools | 9 | import itertools |
10 | import json | 10 | import json |
11 | from collections import namedtuple | ||
11 | from urllib.parse import urlparse | 12 | from urllib.parse import urlparse |
12 | 13 | ||
13 | UNIX_PREFIX = "unix://" | 14 | UNIX_PREFIX = "unix://" |
@@ -18,6 +19,8 @@ ADDR_TYPE_UNIX = 0 | |||
18 | ADDR_TYPE_TCP = 1 | 19 | ADDR_TYPE_TCP = 1 |
19 | ADDR_TYPE_WS = 2 | 20 | ADDR_TYPE_WS = 2 |
20 | 21 | ||
22 | User = namedtuple("User", ("username", "permissions")) | ||
23 | |||
21 | 24 | ||
22 | def parse_address(addr): | 25 | def parse_address(addr): |
23 | if addr.startswith(UNIX_PREFIX): | 26 | if addr.startswith(UNIX_PREFIX): |
@@ -43,7 +46,10 @@ def create_server( | |||
43 | upstream=None, | 46 | upstream=None, |
44 | read_only=False, | 47 | read_only=False, |
45 | db_username=None, | 48 | db_username=None, |
46 | db_password=None | 49 | db_password=None, |
50 | anon_perms=None, | ||
51 | admin_username=None, | ||
52 | admin_password=None, | ||
47 | ): | 53 | ): |
48 | def sqlite_engine(): | 54 | def sqlite_engine(): |
49 | from .sqlite import DatabaseEngine | 55 | from .sqlite import DatabaseEngine |
@@ -62,7 +68,17 @@ def create_server( | |||
62 | else: | 68 | else: |
63 | db_engine = sqlite_engine() | 69 | db_engine = sqlite_engine() |
64 | 70 | ||
65 | s = server.Server(db_engine, upstream=upstream, read_only=read_only) | 71 | if anon_perms is None: |
72 | anon_perms = server.DEFAULT_ANON_PERMS | ||
73 | |||
74 | s = server.Server( | ||
75 | db_engine, | ||
76 | upstream=upstream, | ||
77 | read_only=read_only, | ||
78 | anon_perms=anon_perms, | ||
79 | admin_username=admin_username, | ||
80 | admin_password=admin_password, | ||
81 | ) | ||
66 | 82 | ||
67 | (typ, a) = parse_address(addr) | 83 | (typ, a) = parse_address(addr) |
68 | if typ == ADDR_TYPE_UNIX: | 84 | if typ == ADDR_TYPE_UNIX: |
@@ -76,33 +92,40 @@ def create_server( | |||
76 | return s | 92 | return s |
77 | 93 | ||
78 | 94 | ||
79 | def create_client(addr): | 95 | def create_client(addr, username=None, password=None): |
80 | from . import client | 96 | from . import client |
81 | 97 | ||
82 | c = client.Client() | 98 | c = client.Client(username, password) |
83 | |||
84 | (typ, a) = parse_address(addr) | ||
85 | if typ == ADDR_TYPE_UNIX: | ||
86 | c.connect_unix(*a) | ||
87 | elif typ == ADDR_TYPE_WS: | ||
88 | c.connect_websocket(*a) | ||
89 | else: | ||
90 | c.connect_tcp(*a) | ||
91 | 99 | ||
92 | return c | 100 | try: |
101 | (typ, a) = parse_address(addr) | ||
102 | if typ == ADDR_TYPE_UNIX: | ||
103 | c.connect_unix(*a) | ||
104 | elif typ == ADDR_TYPE_WS: | ||
105 | c.connect_websocket(*a) | ||
106 | else: | ||
107 | c.connect_tcp(*a) | ||
108 | return c | ||
109 | except Exception as e: | ||
110 | c.close() | ||
111 | raise e | ||
93 | 112 | ||
94 | 113 | ||
95 | async def create_async_client(addr): | 114 | async def create_async_client(addr, username=None, password=None): |
96 | from . import client | 115 | from . import client |
97 | 116 | ||
98 | c = client.AsyncClient() | 117 | c = client.AsyncClient(username, password) |
99 | 118 | ||
100 | (typ, a) = parse_address(addr) | 119 | try: |
101 | if typ == ADDR_TYPE_UNIX: | 120 | (typ, a) = parse_address(addr) |
102 | await c.connect_unix(*a) | 121 | if typ == ADDR_TYPE_UNIX: |
103 | elif typ == ADDR_TYPE_WS: | 122 | await c.connect_unix(*a) |
104 | await c.connect_websocket(*a) | 123 | elif typ == ADDR_TYPE_WS: |
105 | else: | 124 | await c.connect_websocket(*a) |
106 | await c.connect_tcp(*a) | 125 | else: |
126 | await c.connect_tcp(*a) | ||
107 | 127 | ||
108 | return c | 128 | return c |
129 | except Exception as e: | ||
130 | await c.close() | ||
131 | raise e | ||
diff --git a/bitbake/lib/hashserv/client.py b/bitbake/lib/hashserv/client.py index 9542d72f6c..82400fe5aa 100644 --- a/bitbake/lib/hashserv/client.py +++ b/bitbake/lib/hashserv/client.py | |||
@@ -6,6 +6,7 @@ | |||
6 | import logging | 6 | import logging |
7 | import socket | 7 | import socket |
8 | import bb.asyncrpc | 8 | import bb.asyncrpc |
9 | import json | ||
9 | from . import create_async_client | 10 | from . import create_async_client |
10 | 11 | ||
11 | 12 | ||
@@ -16,15 +17,19 @@ class AsyncClient(bb.asyncrpc.AsyncClient): | |||
16 | MODE_NORMAL = 0 | 17 | MODE_NORMAL = 0 |
17 | MODE_GET_STREAM = 1 | 18 | MODE_GET_STREAM = 1 |
18 | 19 | ||
19 | def __init__(self): | 20 | def __init__(self, username=None, password=None): |
20 | super().__init__('OEHASHEQUIV', '1.1', logger) | 21 | super().__init__('OEHASHEQUIV', '1.1', logger) |
21 | self.mode = self.MODE_NORMAL | 22 | self.mode = self.MODE_NORMAL |
23 | self.username = username | ||
24 | self.password = password | ||
22 | 25 | ||
23 | async def setup_connection(self): | 26 | async def setup_connection(self): |
24 | await super().setup_connection() | 27 | await super().setup_connection() |
25 | cur_mode = self.mode | 28 | cur_mode = self.mode |
26 | self.mode = self.MODE_NORMAL | 29 | self.mode = self.MODE_NORMAL |
27 | await self._set_mode(cur_mode) | 30 | await self._set_mode(cur_mode) |
31 | if self.username: | ||
32 | await self.auth(self.username, self.password) | ||
28 | 33 | ||
29 | async def send_stream(self, msg): | 34 | async def send_stream(self, msg): |
30 | async def proc(): | 35 | async def proc(): |
@@ -41,6 +46,7 @@ class AsyncClient(bb.asyncrpc.AsyncClient): | |||
41 | if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM: | 46 | if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM: |
42 | r = await self._send_wrapper(stream_to_normal) | 47 | r = await self._send_wrapper(stream_to_normal) |
43 | if r != "ok": | 48 | if r != "ok": |
49 | self.check_invoke_error(r) | ||
44 | raise ConnectionError("Unable to transition to normal mode: Bad response from server %r" % r) | 50 | raise ConnectionError("Unable to transition to normal mode: Bad response from server %r" % r) |
45 | elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL: | 51 | elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL: |
46 | r = await self.invoke({"get-stream": None}) | 52 | r = await self.invoke({"get-stream": None}) |
@@ -109,9 +115,52 @@ class AsyncClient(bb.asyncrpc.AsyncClient): | |||
109 | await self._set_mode(self.MODE_NORMAL) | 115 | await self._set_mode(self.MODE_NORMAL) |
110 | return await self.invoke({"clean-unused": {"max_age_seconds": max_age}}) | 116 | return await self.invoke({"clean-unused": {"max_age_seconds": max_age}}) |
111 | 117 | ||
118 | async def auth(self, username, token): | ||
119 | await self._set_mode(self.MODE_NORMAL) | ||
120 | result = await self.invoke({"auth": {"username": username, "token": token}}) | ||
121 | self.username = username | ||
122 | self.password = token | ||
123 | return result | ||
124 | |||
125 | async def refresh_token(self, username=None): | ||
126 | await self._set_mode(self.MODE_NORMAL) | ||
127 | m = {} | ||
128 | if username: | ||
129 | m["username"] = username | ||
130 | result = await self.invoke({"refresh-token": m}) | ||
131 | if self.username and result["username"] == self.username: | ||
132 | self.password = result["token"] | ||
133 | return result | ||
134 | |||
135 | async def set_user_perms(self, username, permissions): | ||
136 | await self._set_mode(self.MODE_NORMAL) | ||
137 | return await self.invoke({"set-user-perms": {"username": username, "permissions": permissions}}) | ||
138 | |||
139 | async def get_user(self, username=None): | ||
140 | await self._set_mode(self.MODE_NORMAL) | ||
141 | m = {} | ||
142 | if username: | ||
143 | m["username"] = username | ||
144 | return await self.invoke({"get-user": m}) | ||
145 | |||
146 | async def get_all_users(self): | ||
147 | await self._set_mode(self.MODE_NORMAL) | ||
148 | return (await self.invoke({"get-all-users": {}}))["users"] | ||
149 | |||
150 | async def new_user(self, username, permissions): | ||
151 | await self._set_mode(self.MODE_NORMAL) | ||
152 | return await self.invoke({"new-user": {"username": username, "permissions": permissions}}) | ||
153 | |||
154 | async def delete_user(self, username): | ||
155 | await self._set_mode(self.MODE_NORMAL) | ||
156 | return await self.invoke({"delete-user": {"username": username}}) | ||
157 | |||
112 | 158 | ||
113 | class Client(bb.asyncrpc.Client): | 159 | class Client(bb.asyncrpc.Client): |
114 | def __init__(self): | 160 | def __init__(self, username=None, password=None): |
161 | self.username = username | ||
162 | self.password = password | ||
163 | |||
115 | super().__init__() | 164 | super().__init__() |
116 | self._add_methods( | 165 | self._add_methods( |
117 | "connect_tcp", | 166 | "connect_tcp", |
@@ -126,7 +175,14 @@ class Client(bb.asyncrpc.Client): | |||
126 | "backfill_wait", | 175 | "backfill_wait", |
127 | "remove", | 176 | "remove", |
128 | "clean_unused", | 177 | "clean_unused", |
178 | "auth", | ||
179 | "refresh_token", | ||
180 | "set_user_perms", | ||
181 | "get_user", | ||
182 | "get_all_users", | ||
183 | "new_user", | ||
184 | "delete_user", | ||
129 | ) | 185 | ) |
130 | 186 | ||
131 | def _get_async_client(self): | 187 | def _get_async_client(self): |
132 | return AsyncClient() | 188 | return AsyncClient(self.username, self.password) |
diff --git a/bitbake/lib/hashserv/server.py b/bitbake/lib/hashserv/server.py index c691df7618..f5baa6be78 100644 --- a/bitbake/lib/hashserv/server.py +++ b/bitbake/lib/hashserv/server.py | |||
@@ -8,13 +8,48 @@ import asyncio | |||
8 | import logging | 8 | import logging |
9 | import math | 9 | import math |
10 | import time | 10 | import time |
11 | import os | ||
12 | import base64 | ||
13 | import hashlib | ||
11 | from . import create_async_client | 14 | from . import create_async_client |
12 | import bb.asyncrpc | 15 | import bb.asyncrpc |
13 | 16 | ||
14 | |||
15 | logger = logging.getLogger("hashserv.server") | 17 | logger = logging.getLogger("hashserv.server") |
16 | 18 | ||
17 | 19 | ||
20 | # This permission only exists to match nothing | ||
21 | NONE_PERM = "@none" | ||
22 | |||
23 | READ_PERM = "@read" | ||
24 | REPORT_PERM = "@report" | ||
25 | DB_ADMIN_PERM = "@db-admin" | ||
26 | USER_ADMIN_PERM = "@user-admin" | ||
27 | ALL_PERM = "@all" | ||
28 | |||
29 | ALL_PERMISSIONS = { | ||
30 | READ_PERM, | ||
31 | REPORT_PERM, | ||
32 | DB_ADMIN_PERM, | ||
33 | USER_ADMIN_PERM, | ||
34 | ALL_PERM, | ||
35 | } | ||
36 | |||
37 | DEFAULT_ANON_PERMS = ( | ||
38 | READ_PERM, | ||
39 | REPORT_PERM, | ||
40 | DB_ADMIN_PERM, | ||
41 | ) | ||
42 | |||
43 | TOKEN_ALGORITHM = "sha256" | ||
44 | |||
45 | # 48 bytes of random data will result in 64 characters when base64 | ||
46 | # encoded. This number also ensures that the base64 encoding won't have any | ||
47 | # trailing '=' characters. | ||
48 | TOKEN_SIZE = 48 | ||
49 | |||
50 | SALT_SIZE = 8 | ||
51 | |||
52 | |||
18 | class Measurement(object): | 53 | class Measurement(object): |
19 | def __init__(self, sample): | 54 | def __init__(self, sample): |
20 | self.sample = sample | 55 | self.sample = sample |
@@ -108,6 +143,85 @@ class Stats(object): | |||
108 | } | 143 | } |
109 | 144 | ||
110 | 145 | ||
146 | token_refresh_semaphore = asyncio.Lock() | ||
147 | |||
148 | |||
149 | async def new_token(): | ||
150 | # Prevent malicious users from using this API to deduce the entropy | ||
151 | # pool on the server and thus be able to guess a token. *All* token | ||
152 | # refresh requests lock the same global semaphore and then sleep for a | ||
153 | # short time. The effectively rate limits the total number of requests | ||
154 | # than can be made across all clients to 10/second, which should be enough | ||
155 | # since you have to be an authenticated users to make the request in the | ||
156 | # first place | ||
157 | async with token_refresh_semaphore: | ||
158 | await asyncio.sleep(0.1) | ||
159 | raw = os.getrandom(TOKEN_SIZE, os.GRND_NONBLOCK) | ||
160 | |||
161 | return base64.b64encode(raw, b"._").decode("utf-8") | ||
162 | |||
163 | |||
164 | def new_salt(): | ||
165 | return os.getrandom(SALT_SIZE, os.GRND_NONBLOCK).hex() | ||
166 | |||
167 | |||
168 | def hash_token(algo, salt, token): | ||
169 | h = hashlib.new(algo) | ||
170 | h.update(salt.encode("utf-8")) | ||
171 | h.update(token.encode("utf-8")) | ||
172 | return ":".join([algo, salt, h.hexdigest()]) | ||
173 | |||
174 | |||
175 | def permissions(*permissions, allow_anon=True, allow_self_service=False): | ||
176 | """ | ||
177 | Function decorator that can be used to decorate an RPC function call and | ||
178 | check that the current users permissions match the require permissions. | ||
179 | |||
180 | If allow_anon is True, the user will also be allowed to make the RPC call | ||
181 | if the anonymous user permissions match the permissions. | ||
182 | |||
183 | If allow_self_service is True, and the "username" property in the request | ||
184 | is the currently logged in user, or not specified, the user will also be | ||
185 | allowed to make the request. This allows users to access normal privileged | ||
186 | API, as long as they are only modifying their own user properties (e.g. | ||
187 | users can be allowed to reset their own token without @user-admin | ||
188 | permissions, but not the token for any other user. | ||
189 | """ | ||
190 | |||
191 | def wrapper(func): | ||
192 | async def wrap(self, request): | ||
193 | if allow_self_service and self.user is not None: | ||
194 | username = request.get("username", self.user.username) | ||
195 | if username == self.user.username: | ||
196 | request["username"] = self.user.username | ||
197 | return await func(self, request) | ||
198 | |||
199 | if not self.user_has_permissions(*permissions, allow_anon=allow_anon): | ||
200 | if not self.user: | ||
201 | username = "Anonymous user" | ||
202 | user_perms = self.anon_perms | ||
203 | else: | ||
204 | username = self.user.username | ||
205 | user_perms = self.user.permissions | ||
206 | |||
207 | self.logger.info( | ||
208 | "User %s with permissions %r denied from calling %s. Missing permissions(s) %r", | ||
209 | username, | ||
210 | ", ".join(user_perms), | ||
211 | func.__name__, | ||
212 | ", ".join(permissions), | ||
213 | ) | ||
214 | raise bb.asyncrpc.InvokeError( | ||
215 | f"{username} is not allowed to access permissions(s) {', '.join(permissions)}" | ||
216 | ) | ||
217 | |||
218 | return await func(self, request) | ||
219 | |||
220 | return wrap | ||
221 | |||
222 | return wrapper | ||
223 | |||
224 | |||
111 | class ServerClient(bb.asyncrpc.AsyncServerConnection): | 225 | class ServerClient(bb.asyncrpc.AsyncServerConnection): |
112 | def __init__( | 226 | def __init__( |
113 | self, | 227 | self, |
@@ -117,6 +231,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
117 | backfill_queue, | 231 | backfill_queue, |
118 | upstream, | 232 | upstream, |
119 | read_only, | 233 | read_only, |
234 | anon_perms, | ||
120 | ): | 235 | ): |
121 | super().__init__(socket, "OEHASHEQUIV", logger) | 236 | super().__init__(socket, "OEHASHEQUIV", logger) |
122 | self.db_engine = db_engine | 237 | self.db_engine = db_engine |
@@ -125,6 +240,8 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
125 | self.backfill_queue = backfill_queue | 240 | self.backfill_queue = backfill_queue |
126 | self.upstream = upstream | 241 | self.upstream = upstream |
127 | self.read_only = read_only | 242 | self.read_only = read_only |
243 | self.user = None | ||
244 | self.anon_perms = anon_perms | ||
128 | 245 | ||
129 | self.handlers.update( | 246 | self.handlers.update( |
130 | { | 247 | { |
@@ -135,6 +252,9 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
135 | # Not always read-only, but internally checks if the server is | 252 | # Not always read-only, but internally checks if the server is |
136 | # read-only | 253 | # read-only |
137 | "report": self.handle_report, | 254 | "report": self.handle_report, |
255 | "auth": self.handle_auth, | ||
256 | "get-user": self.handle_get_user, | ||
257 | "get-all-users": self.handle_get_all_users, | ||
138 | } | 258 | } |
139 | ) | 259 | ) |
140 | 260 | ||
@@ -146,9 +266,36 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
146 | "backfill-wait": self.handle_backfill_wait, | 266 | "backfill-wait": self.handle_backfill_wait, |
147 | "remove": self.handle_remove, | 267 | "remove": self.handle_remove, |
148 | "clean-unused": self.handle_clean_unused, | 268 | "clean-unused": self.handle_clean_unused, |
269 | "refresh-token": self.handle_refresh_token, | ||
270 | "set-user-perms": self.handle_set_perms, | ||
271 | "new-user": self.handle_new_user, | ||
272 | "delete-user": self.handle_delete_user, | ||
149 | } | 273 | } |
150 | ) | 274 | ) |
151 | 275 | ||
276 | def raise_no_user_error(self, username): | ||
277 | raise bb.asyncrpc.InvokeError(f"No user named '{username}' exists") | ||
278 | |||
279 | def user_has_permissions(self, *permissions, allow_anon=True): | ||
280 | permissions = set(permissions) | ||
281 | if allow_anon: | ||
282 | if ALL_PERM in self.anon_perms: | ||
283 | return True | ||
284 | |||
285 | if not permissions - self.anon_perms: | ||
286 | return True | ||
287 | |||
288 | if self.user is None: | ||
289 | return False | ||
290 | |||
291 | if ALL_PERM in self.user.permissions: | ||
292 | return True | ||
293 | |||
294 | if not permissions - self.user.permissions: | ||
295 | return True | ||
296 | |||
297 | return False | ||
298 | |||
152 | def validate_proto_version(self): | 299 | def validate_proto_version(self): |
153 | return self.proto_version > (1, 0) and self.proto_version <= (1, 1) | 300 | return self.proto_version > (1, 0) and self.proto_version <= (1, 1) |
154 | 301 | ||
@@ -178,6 +325,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
178 | 325 | ||
179 | raise bb.asyncrpc.ClientError("Unrecognized command %r" % msg) | 326 | raise bb.asyncrpc.ClientError("Unrecognized command %r" % msg) |
180 | 327 | ||
328 | @permissions(READ_PERM) | ||
181 | async def handle_get(self, request): | 329 | async def handle_get(self, request): |
182 | method = request["method"] | 330 | method = request["method"] |
183 | taskhash = request["taskhash"] | 331 | taskhash = request["taskhash"] |
@@ -206,6 +354,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
206 | 354 | ||
207 | return d | 355 | return d |
208 | 356 | ||
357 | @permissions(READ_PERM) | ||
209 | async def handle_get_outhash(self, request): | 358 | async def handle_get_outhash(self, request): |
210 | method = request["method"] | 359 | method = request["method"] |
211 | outhash = request["outhash"] | 360 | outhash = request["outhash"] |
@@ -236,6 +385,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
236 | await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"]) | 385 | await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"]) |
237 | await self.db.insert_outhash(data) | 386 | await self.db.insert_outhash(data) |
238 | 387 | ||
388 | @permissions(READ_PERM) | ||
239 | async def handle_get_stream(self, request): | 389 | async def handle_get_stream(self, request): |
240 | await self.socket.send_message("ok") | 390 | await self.socket.send_message("ok") |
241 | 391 | ||
@@ -304,8 +454,11 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
304 | "unihash": unihash, | 454 | "unihash": unihash, |
305 | } | 455 | } |
306 | 456 | ||
457 | # Since this can be called either read only or to report, the check to | ||
458 | # report is made inside the function | ||
459 | @permissions(READ_PERM) | ||
307 | async def handle_report(self, data): | 460 | async def handle_report(self, data): |
308 | if self.read_only: | 461 | if self.read_only or not self.user_has_permissions(REPORT_PERM): |
309 | return await self.report_readonly(data) | 462 | return await self.report_readonly(data) |
310 | 463 | ||
311 | outhash_data = { | 464 | outhash_data = { |
@@ -358,6 +511,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
358 | "unihash": unihash, | 511 | "unihash": unihash, |
359 | } | 512 | } |
360 | 513 | ||
514 | @permissions(READ_PERM, REPORT_PERM) | ||
361 | async def handle_equivreport(self, data): | 515 | async def handle_equivreport(self, data): |
362 | await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"]) | 516 | await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"]) |
363 | 517 | ||
@@ -375,11 +529,13 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
375 | 529 | ||
376 | return {k: row[k] for k in ("taskhash", "method", "unihash")} | 530 | return {k: row[k] for k in ("taskhash", "method", "unihash")} |
377 | 531 | ||
532 | @permissions(READ_PERM) | ||
378 | async def handle_get_stats(self, request): | 533 | async def handle_get_stats(self, request): |
379 | return { | 534 | return { |
380 | "requests": self.request_stats.todict(), | 535 | "requests": self.request_stats.todict(), |
381 | } | 536 | } |
382 | 537 | ||
538 | @permissions(DB_ADMIN_PERM) | ||
383 | async def handle_reset_stats(self, request): | 539 | async def handle_reset_stats(self, request): |
384 | d = { | 540 | d = { |
385 | "requests": self.request_stats.todict(), | 541 | "requests": self.request_stats.todict(), |
@@ -388,6 +544,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
388 | self.request_stats.reset() | 544 | self.request_stats.reset() |
389 | return d | 545 | return d |
390 | 546 | ||
547 | @permissions(READ_PERM) | ||
391 | async def handle_backfill_wait(self, request): | 548 | async def handle_backfill_wait(self, request): |
392 | d = { | 549 | d = { |
393 | "tasks": self.backfill_queue.qsize(), | 550 | "tasks": self.backfill_queue.qsize(), |
@@ -395,6 +552,7 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
395 | await self.backfill_queue.join() | 552 | await self.backfill_queue.join() |
396 | return d | 553 | return d |
397 | 554 | ||
555 | @permissions(DB_ADMIN_PERM) | ||
398 | async def handle_remove(self, request): | 556 | async def handle_remove(self, request): |
399 | condition = request["where"] | 557 | condition = request["where"] |
400 | if not isinstance(condition, dict): | 558 | if not isinstance(condition, dict): |
@@ -402,19 +560,178 @@ class ServerClient(bb.asyncrpc.AsyncServerConnection): | |||
402 | 560 | ||
403 | return {"count": await self.db.remove(condition)} | 561 | return {"count": await self.db.remove(condition)} |
404 | 562 | ||
563 | @permissions(DB_ADMIN_PERM) | ||
405 | async def handle_clean_unused(self, request): | 564 | async def handle_clean_unused(self, request): |
406 | max_age = request["max_age_seconds"] | 565 | max_age = request["max_age_seconds"] |
407 | oldest = datetime.now() - timedelta(seconds=-max_age) | 566 | oldest = datetime.now() - timedelta(seconds=-max_age) |
408 | return {"count": await self.db.clean_unused(oldest)} | 567 | return {"count": await self.db.clean_unused(oldest)} |
409 | 568 | ||
569 | # The authentication API is always allowed | ||
570 | async def handle_auth(self, request): | ||
571 | username = str(request["username"]) | ||
572 | token = str(request["token"]) | ||
573 | |||
574 | async def fail_auth(): | ||
575 | nonlocal username | ||
576 | # Rate limit bad login attempts | ||
577 | await asyncio.sleep(1) | ||
578 | raise bb.asyncrpc.InvokeError(f"Unable to authenticate as {username}") | ||
579 | |||
580 | user, db_token = await self.db.lookup_user_token(username) | ||
581 | |||
582 | if not user or not db_token: | ||
583 | await fail_auth() | ||
584 | |||
585 | try: | ||
586 | algo, salt, _ = db_token.split(":") | ||
587 | except ValueError: | ||
588 | await fail_auth() | ||
589 | |||
590 | if hash_token(algo, salt, token) != db_token: | ||
591 | await fail_auth() | ||
592 | |||
593 | self.user = user | ||
594 | |||
595 | self.logger.info("Authenticated as %s", username) | ||
596 | |||
597 | return { | ||
598 | "result": True, | ||
599 | "username": self.user.username, | ||
600 | "permissions": sorted(list(self.user.permissions)), | ||
601 | } | ||
602 | |||
603 | @permissions(USER_ADMIN_PERM, allow_self_service=True, allow_anon=False) | ||
604 | async def handle_refresh_token(self, request): | ||
605 | username = str(request["username"]) | ||
606 | |||
607 | token = await new_token() | ||
608 | |||
609 | updated = await self.db.set_user_token( | ||
610 | username, | ||
611 | hash_token(TOKEN_ALGORITHM, new_salt(), token), | ||
612 | ) | ||
613 | if not updated: | ||
614 | self.raise_no_user_error(username) | ||
615 | |||
616 | return {"username": username, "token": token} | ||
617 | |||
618 | def get_perm_arg(self, arg): | ||
619 | if not isinstance(arg, list): | ||
620 | raise bb.asyncrpc.InvokeError("Unexpected type for permissions") | ||
621 | |||
622 | arg = set(arg) | ||
623 | try: | ||
624 | arg.remove(NONE_PERM) | ||
625 | except KeyError: | ||
626 | pass | ||
627 | |||
628 | unknown_perms = arg - ALL_PERMISSIONS | ||
629 | if unknown_perms: | ||
630 | raise bb.asyncrpc.InvokeError( | ||
631 | "Unknown permissions %s" % ", ".join(sorted(list(unknown_perms))) | ||
632 | ) | ||
633 | |||
634 | return sorted(list(arg)) | ||
635 | |||
636 | def return_perms(self, permissions): | ||
637 | if ALL_PERM in permissions: | ||
638 | return sorted(list(ALL_PERMISSIONS)) | ||
639 | return sorted(list(permissions)) | ||
640 | |||
641 | @permissions(USER_ADMIN_PERM, allow_anon=False) | ||
642 | async def handle_set_perms(self, request): | ||
643 | username = str(request["username"]) | ||
644 | permissions = self.get_perm_arg(request["permissions"]) | ||
645 | |||
646 | if not await self.db.set_user_perms(username, permissions): | ||
647 | self.raise_no_user_error(username) | ||
648 | |||
649 | return { | ||
650 | "username": username, | ||
651 | "permissions": self.return_perms(permissions), | ||
652 | } | ||
653 | |||
654 | @permissions(USER_ADMIN_PERM, allow_self_service=True, allow_anon=False) | ||
655 | async def handle_get_user(self, request): | ||
656 | username = str(request["username"]) | ||
657 | |||
658 | user = await self.db.lookup_user(username) | ||
659 | if user is None: | ||
660 | return None | ||
661 | |||
662 | return { | ||
663 | "username": user.username, | ||
664 | "permissions": self.return_perms(user.permissions), | ||
665 | } | ||
666 | |||
667 | @permissions(USER_ADMIN_PERM, allow_anon=False) | ||
668 | async def handle_get_all_users(self, request): | ||
669 | users = await self.db.get_all_users() | ||
670 | return { | ||
671 | "users": [ | ||
672 | { | ||
673 | "username": u.username, | ||
674 | "permissions": self.return_perms(u.permissions), | ||
675 | } | ||
676 | for u in users | ||
677 | ] | ||
678 | } | ||
679 | |||
680 | @permissions(USER_ADMIN_PERM, allow_anon=False) | ||
681 | async def handle_new_user(self, request): | ||
682 | username = str(request["username"]) | ||
683 | permissions = self.get_perm_arg(request["permissions"]) | ||
684 | |||
685 | token = await new_token() | ||
686 | |||
687 | inserted = await self.db.new_user( | ||
688 | username, | ||
689 | permissions, | ||
690 | hash_token(TOKEN_ALGORITHM, new_salt(), token), | ||
691 | ) | ||
692 | if not inserted: | ||
693 | raise bb.asyncrpc.InvokeError(f"Cannot create new user '{username}'") | ||
694 | |||
695 | return { | ||
696 | "username": username, | ||
697 | "permissions": self.return_perms(permissions), | ||
698 | "token": token, | ||
699 | } | ||
700 | |||
701 | @permissions(USER_ADMIN_PERM, allow_anon=False) | ||
702 | async def handle_delete_user(self, request): | ||
703 | username = str(request["username"]) | ||
704 | |||
705 | if not await self.db.delete_user(username): | ||
706 | self.raise_no_user_error(username) | ||
707 | |||
708 | return {"username": username} | ||
709 | |||
410 | 710 | ||
411 | class Server(bb.asyncrpc.AsyncServer): | 711 | class Server(bb.asyncrpc.AsyncServer): |
412 | def __init__(self, db_engine, upstream=None, read_only=False): | 712 | def __init__( |
713 | self, | ||
714 | db_engine, | ||
715 | upstream=None, | ||
716 | read_only=False, | ||
717 | anon_perms=DEFAULT_ANON_PERMS, | ||
718 | admin_username=None, | ||
719 | admin_password=None, | ||
720 | ): | ||
413 | if upstream and read_only: | 721 | if upstream and read_only: |
414 | raise bb.asyncrpc.ServerError( | 722 | raise bb.asyncrpc.ServerError( |
415 | "Read-only hashserv cannot pull from an upstream server" | 723 | "Read-only hashserv cannot pull from an upstream server" |
416 | ) | 724 | ) |
417 | 725 | ||
726 | disallowed_perms = set(anon_perms) - set( | ||
727 | [NONE_PERM, READ_PERM, REPORT_PERM, DB_ADMIN_PERM] | ||
728 | ) | ||
729 | |||
730 | if disallowed_perms: | ||
731 | raise bb.asyncrpc.ServerError( | ||
732 | f"Permission(s) {' '.join(disallowed_perms)} are not allowed for anonymous users" | ||
733 | ) | ||
734 | |||
418 | super().__init__(logger) | 735 | super().__init__(logger) |
419 | 736 | ||
420 | self.request_stats = Stats() | 737 | self.request_stats = Stats() |
@@ -422,6 +739,13 @@ class Server(bb.asyncrpc.AsyncServer): | |||
422 | self.upstream = upstream | 739 | self.upstream = upstream |
423 | self.read_only = read_only | 740 | self.read_only = read_only |
424 | self.backfill_queue = None | 741 | self.backfill_queue = None |
742 | self.anon_perms = set(anon_perms) | ||
743 | self.admin_username = admin_username | ||
744 | self.admin_password = admin_password | ||
745 | |||
746 | self.logger.info( | ||
747 | "Anonymous user permissions are: %s", ", ".join(self.anon_perms) | ||
748 | ) | ||
425 | 749 | ||
426 | def accept_client(self, socket): | 750 | def accept_client(self, socket): |
427 | return ServerClient( | 751 | return ServerClient( |
@@ -431,12 +755,34 @@ class Server(bb.asyncrpc.AsyncServer): | |||
431 | self.backfill_queue, | 755 | self.backfill_queue, |
432 | self.upstream, | 756 | self.upstream, |
433 | self.read_only, | 757 | self.read_only, |
758 | self.anon_perms, | ||
434 | ) | 759 | ) |
435 | 760 | ||
761 | async def create_admin_user(self): | ||
762 | admin_permissions = (ALL_PERM,) | ||
763 | async with self.db_engine.connect(self.logger) as db: | ||
764 | added = await db.new_user( | ||
765 | self.admin_username, | ||
766 | admin_permissions, | ||
767 | hash_token(TOKEN_ALGORITHM, new_salt(), self.admin_password), | ||
768 | ) | ||
769 | if added: | ||
770 | self.logger.info("Created admin user '%s'", self.admin_username) | ||
771 | else: | ||
772 | await db.set_user_perms( | ||
773 | self.admin_username, | ||
774 | admin_permissions, | ||
775 | ) | ||
776 | await db.set_user_token( | ||
777 | self.admin_username, | ||
778 | hash_token(TOKEN_ALGORITHM, new_salt(), self.admin_password), | ||
779 | ) | ||
780 | self.logger.info("Admin user '%s' updated", self.admin_username) | ||
781 | |||
436 | async def backfill_worker_task(self): | 782 | async def backfill_worker_task(self): |
437 | async with await create_async_client( | 783 | async with await create_async_client( |
438 | self.upstream | 784 | self.upstream |
439 | ) as client, self.db_engine.connect(logger) as db: | 785 | ) as client, self.db_engine.connect(self.logger) as db: |
440 | while True: | 786 | while True: |
441 | item = await self.backfill_queue.get() | 787 | item = await self.backfill_queue.get() |
442 | if item is None: | 788 | if item is None: |
@@ -457,6 +803,9 @@ class Server(bb.asyncrpc.AsyncServer): | |||
457 | 803 | ||
458 | self.loop.run_until_complete(self.db_engine.create()) | 804 | self.loop.run_until_complete(self.db_engine.create()) |
459 | 805 | ||
806 | if self.admin_username: | ||
807 | self.loop.run_until_complete(self.create_admin_user()) | ||
808 | |||
460 | return tasks | 809 | return tasks |
461 | 810 | ||
462 | async def stop(self): | 811 | async def stop(self): |
diff --git a/bitbake/lib/hashserv/sqlalchemy.py b/bitbake/lib/hashserv/sqlalchemy.py index 3216621f9d..bfd8a8446e 100644 --- a/bitbake/lib/hashserv/sqlalchemy.py +++ b/bitbake/lib/hashserv/sqlalchemy.py | |||
@@ -7,6 +7,7 @@ | |||
7 | 7 | ||
8 | import logging | 8 | import logging |
9 | from datetime import datetime | 9 | from datetime import datetime |
10 | from . import User | ||
10 | 11 | ||
11 | from sqlalchemy.ext.asyncio import create_async_engine | 12 | from sqlalchemy.ext.asyncio import create_async_engine |
12 | from sqlalchemy.pool import NullPool | 13 | from sqlalchemy.pool import NullPool |
@@ -25,13 +26,12 @@ from sqlalchemy import ( | |||
25 | literal, | 26 | literal, |
26 | and_, | 27 | and_, |
27 | delete, | 28 | delete, |
29 | update, | ||
28 | ) | 30 | ) |
29 | import sqlalchemy.engine | 31 | import sqlalchemy.engine |
30 | from sqlalchemy.orm import declarative_base | 32 | from sqlalchemy.orm import declarative_base |
31 | from sqlalchemy.exc import IntegrityError | 33 | from sqlalchemy.exc import IntegrityError |
32 | 34 | ||
33 | logger = logging.getLogger("hashserv.sqlalchemy") | ||
34 | |||
35 | Base = declarative_base() | 35 | Base = declarative_base() |
36 | 36 | ||
37 | 37 | ||
@@ -68,9 +68,19 @@ class OuthashesV2(Base): | |||
68 | ) | 68 | ) |
69 | 69 | ||
70 | 70 | ||
71 | class Users(Base): | ||
72 | __tablename__ = "users" | ||
73 | id = Column(Integer, primary_key=True, autoincrement=True) | ||
74 | username = Column(Text, nullable=False) | ||
75 | token = Column(Text, nullable=False) | ||
76 | permissions = Column(Text) | ||
77 | |||
78 | __table_args__ = (UniqueConstraint("username"),) | ||
79 | |||
80 | |||
71 | class DatabaseEngine(object): | 81 | class DatabaseEngine(object): |
72 | def __init__(self, url, username=None, password=None): | 82 | def __init__(self, url, username=None, password=None): |
73 | self.logger = logger | 83 | self.logger = logging.getLogger("hashserv.sqlalchemy") |
74 | self.url = sqlalchemy.engine.make_url(url) | 84 | self.url = sqlalchemy.engine.make_url(url) |
75 | 85 | ||
76 | if username is not None: | 86 | if username is not None: |
@@ -85,7 +95,7 @@ class DatabaseEngine(object): | |||
85 | 95 | ||
86 | async with self.engine.begin() as conn: | 96 | async with self.engine.begin() as conn: |
87 | # Create tables | 97 | # Create tables |
88 | logger.info("Creating tables...") | 98 | self.logger.info("Creating tables...") |
89 | await conn.run_sync(Base.metadata.create_all) | 99 | await conn.run_sync(Base.metadata.create_all) |
90 | 100 | ||
91 | def connect(self, logger): | 101 | def connect(self, logger): |
@@ -98,6 +108,15 @@ def map_row(row): | |||
98 | return dict(**row._mapping) | 108 | return dict(**row._mapping) |
99 | 109 | ||
100 | 110 | ||
111 | def map_user(row): | ||
112 | if row is None: | ||
113 | return None | ||
114 | return User( | ||
115 | username=row.username, | ||
116 | permissions=set(row.permissions.split()), | ||
117 | ) | ||
118 | |||
119 | |||
101 | class Database(object): | 120 | class Database(object): |
102 | def __init__(self, engine, logger): | 121 | def __init__(self, engine, logger): |
103 | self.engine = engine | 122 | self.engine = engine |
@@ -278,7 +297,7 @@ class Database(object): | |||
278 | await self.db.execute(statement) | 297 | await self.db.execute(statement) |
279 | return True | 298 | return True |
280 | except IntegrityError: | 299 | except IntegrityError: |
281 | logger.debug( | 300 | self.logger.debug( |
282 | "%s, %s, %s already in unihash database", method, taskhash, unihash | 301 | "%s, %s, %s already in unihash database", method, taskhash, unihash |
283 | ) | 302 | ) |
284 | return False | 303 | return False |
@@ -298,7 +317,87 @@ class Database(object): | |||
298 | await self.db.execute(statement) | 317 | await self.db.execute(statement) |
299 | return True | 318 | return True |
300 | except IntegrityError: | 319 | except IntegrityError: |
301 | logger.debug( | 320 | self.logger.debug( |
302 | "%s, %s already in outhash database", data["method"], data["outhash"] | 321 | "%s, %s already in outhash database", data["method"], data["outhash"] |
303 | ) | 322 | ) |
304 | return False | 323 | return False |
324 | |||
325 | async def _get_user(self, username): | ||
326 | statement = select( | ||
327 | Users.username, | ||
328 | Users.permissions, | ||
329 | Users.token, | ||
330 | ).where( | ||
331 | Users.username == username, | ||
332 | ) | ||
333 | self.logger.debug("%s", statement) | ||
334 | async with self.db.begin(): | ||
335 | result = await self.db.execute(statement) | ||
336 | return result.first() | ||
337 | |||
338 | async def lookup_user_token(self, username): | ||
339 | row = await self._get_user(username) | ||
340 | if not row: | ||
341 | return None, None | ||
342 | return map_user(row), row.token | ||
343 | |||
344 | async def lookup_user(self, username): | ||
345 | return map_user(await self._get_user(username)) | ||
346 | |||
347 | async def set_user_token(self, username, token): | ||
348 | statement = ( | ||
349 | update(Users) | ||
350 | .where( | ||
351 | Users.username == username, | ||
352 | ) | ||
353 | .values( | ||
354 | token=token, | ||
355 | ) | ||
356 | ) | ||
357 | self.logger.debug("%s", statement) | ||
358 | async with self.db.begin(): | ||
359 | result = await self.db.execute(statement) | ||
360 | return result.rowcount != 0 | ||
361 | |||
362 | async def set_user_perms(self, username, permissions): | ||
363 | statement = ( | ||
364 | update(Users) | ||
365 | .where(Users.username == username) | ||
366 | .values(permissions=" ".join(permissions)) | ||
367 | ) | ||
368 | self.logger.debug("%s", statement) | ||
369 | async with self.db.begin(): | ||
370 | result = await self.db.execute(statement) | ||
371 | return result.rowcount != 0 | ||
372 | |||
373 | async def get_all_users(self): | ||
374 | statement = select( | ||
375 | Users.username, | ||
376 | Users.permissions, | ||
377 | ) | ||
378 | self.logger.debug("%s", statement) | ||
379 | async with self.db.begin(): | ||
380 | result = await self.db.execute(statement) | ||
381 | return [map_user(row) for row in result] | ||
382 | |||
383 | async def new_user(self, username, permissions, token): | ||
384 | statement = insert(Users).values( | ||
385 | username=username, | ||
386 | permissions=" ".join(permissions), | ||
387 | token=token, | ||
388 | ) | ||
389 | self.logger.debug("%s", statement) | ||
390 | try: | ||
391 | async with self.db.begin(): | ||
392 | await self.db.execute(statement) | ||
393 | return True | ||
394 | except IntegrityError as e: | ||
395 | self.logger.debug("Cannot create new user %s: %s", username, e) | ||
396 | return False | ||
397 | |||
398 | async def delete_user(self, username): | ||
399 | statement = delete(Users).where(Users.username == username) | ||
400 | self.logger.debug("%s", statement) | ||
401 | async with self.db.begin(): | ||
402 | result = await self.db.execute(statement) | ||
403 | return result.rowcount != 0 | ||
diff --git a/bitbake/lib/hashserv/sqlite.py b/bitbake/lib/hashserv/sqlite.py index 6809c53706..414ee8ffb8 100644 --- a/bitbake/lib/hashserv/sqlite.py +++ b/bitbake/lib/hashserv/sqlite.py | |||
@@ -7,6 +7,7 @@ | |||
7 | import sqlite3 | 7 | import sqlite3 |
8 | import logging | 8 | import logging |
9 | from contextlib import closing | 9 | from contextlib import closing |
10 | from . import User | ||
10 | 11 | ||
11 | logger = logging.getLogger("hashserv.sqlite") | 12 | logger = logging.getLogger("hashserv.sqlite") |
12 | 13 | ||
@@ -34,6 +35,14 @@ OUTHASH_TABLE_DEFINITION = ( | |||
34 | 35 | ||
35 | OUTHASH_TABLE_COLUMNS = tuple(name for name, _, _ in OUTHASH_TABLE_DEFINITION) | 36 | OUTHASH_TABLE_COLUMNS = tuple(name for name, _, _ in OUTHASH_TABLE_DEFINITION) |
36 | 37 | ||
38 | USERS_TABLE_DEFINITION = ( | ||
39 | ("username", "TEXT NOT NULL", "UNIQUE"), | ||
40 | ("token", "TEXT NOT NULL", ""), | ||
41 | ("permissions", "TEXT NOT NULL", ""), | ||
42 | ) | ||
43 | |||
44 | USERS_TABLE_COLUMNS = tuple(name for name, _, _ in USERS_TABLE_DEFINITION) | ||
45 | |||
37 | 46 | ||
38 | def _make_table(cursor, name, definition): | 47 | def _make_table(cursor, name, definition): |
39 | cursor.execute( | 48 | cursor.execute( |
@@ -53,6 +62,15 @@ def _make_table(cursor, name, definition): | |||
53 | ) | 62 | ) |
54 | 63 | ||
55 | 64 | ||
65 | def map_user(row): | ||
66 | if row is None: | ||
67 | return None | ||
68 | return User( | ||
69 | username=row["username"], | ||
70 | permissions=set(row["permissions"].split()), | ||
71 | ) | ||
72 | |||
73 | |||
56 | class DatabaseEngine(object): | 74 | class DatabaseEngine(object): |
57 | def __init__(self, dbname, sync): | 75 | def __init__(self, dbname, sync): |
58 | self.dbname = dbname | 76 | self.dbname = dbname |
@@ -66,6 +84,7 @@ class DatabaseEngine(object): | |||
66 | with closing(db.cursor()) as cursor: | 84 | with closing(db.cursor()) as cursor: |
67 | _make_table(cursor, "unihashes_v2", UNIHASH_TABLE_DEFINITION) | 85 | _make_table(cursor, "unihashes_v2", UNIHASH_TABLE_DEFINITION) |
68 | _make_table(cursor, "outhashes_v2", OUTHASH_TABLE_DEFINITION) | 86 | _make_table(cursor, "outhashes_v2", OUTHASH_TABLE_DEFINITION) |
87 | _make_table(cursor, "users", USERS_TABLE_DEFINITION) | ||
69 | 88 | ||
70 | cursor.execute("PRAGMA journal_mode = WAL") | 89 | cursor.execute("PRAGMA journal_mode = WAL") |
71 | cursor.execute( | 90 | cursor.execute( |
@@ -227,6 +246,7 @@ class Database(object): | |||
227 | "oldest": oldest, | 246 | "oldest": oldest, |
228 | }, | 247 | }, |
229 | ) | 248 | ) |
249 | self.db.commit() | ||
230 | return cursor.rowcount | 250 | return cursor.rowcount |
231 | 251 | ||
232 | async def insert_unihash(self, method, taskhash, unihash): | 252 | async def insert_unihash(self, method, taskhash, unihash): |
@@ -257,3 +277,88 @@ class Database(object): | |||
257 | cursor.execute(query, data) | 277 | cursor.execute(query, data) |
258 | self.db.commit() | 278 | self.db.commit() |
259 | return cursor.lastrowid != prevrowid | 279 | return cursor.lastrowid != prevrowid |
280 | |||
281 | def _get_user(self, username): | ||
282 | with closing(self.db.cursor()) as cursor: | ||
283 | cursor.execute( | ||
284 | """ | ||
285 | SELECT username, permissions, token FROM users WHERE username=:username | ||
286 | """, | ||
287 | { | ||
288 | "username": username, | ||
289 | }, | ||
290 | ) | ||
291 | return cursor.fetchone() | ||
292 | |||
293 | async def lookup_user_token(self, username): | ||
294 | row = self._get_user(username) | ||
295 | if row is None: | ||
296 | return None, None | ||
297 | return map_user(row), row["token"] | ||
298 | |||
299 | async def lookup_user(self, username): | ||
300 | return map_user(self._get_user(username)) | ||
301 | |||
302 | async def set_user_token(self, username, token): | ||
303 | with closing(self.db.cursor()) as cursor: | ||
304 | cursor.execute( | ||
305 | """ | ||
306 | UPDATE users SET token=:token WHERE username=:username | ||
307 | """, | ||
308 | { | ||
309 | "username": username, | ||
310 | "token": token, | ||
311 | }, | ||
312 | ) | ||
313 | self.db.commit() | ||
314 | return cursor.rowcount != 0 | ||
315 | |||
316 | async def set_user_perms(self, username, permissions): | ||
317 | with closing(self.db.cursor()) as cursor: | ||
318 | cursor.execute( | ||
319 | """ | ||
320 | UPDATE users SET permissions=:permissions WHERE username=:username | ||
321 | """, | ||
322 | { | ||
323 | "username": username, | ||
324 | "permissions": " ".join(permissions), | ||
325 | }, | ||
326 | ) | ||
327 | self.db.commit() | ||
328 | return cursor.rowcount != 0 | ||
329 | |||
330 | async def get_all_users(self): | ||
331 | with closing(self.db.cursor()) as cursor: | ||
332 | cursor.execute("SELECT username, permissions FROM users") | ||
333 | return [map_user(r) for r in cursor.fetchall()] | ||
334 | |||
335 | async def new_user(self, username, permissions, token): | ||
336 | with closing(self.db.cursor()) as cursor: | ||
337 | try: | ||
338 | cursor.execute( | ||
339 | """ | ||
340 | INSERT INTO users (username, token, permissions) VALUES (:username, :token, :permissions) | ||
341 | """, | ||
342 | { | ||
343 | "username": username, | ||
344 | "token": token, | ||
345 | "permissions": " ".join(permissions), | ||
346 | }, | ||
347 | ) | ||
348 | self.db.commit() | ||
349 | return True | ||
350 | except sqlite3.IntegrityError: | ||
351 | return False | ||
352 | |||
353 | async def delete_user(self, username): | ||
354 | with closing(self.db.cursor()) as cursor: | ||
355 | cursor.execute( | ||
356 | """ | ||
357 | DELETE FROM users WHERE username=:username | ||
358 | """, | ||
359 | { | ||
360 | "username": username, | ||
361 | }, | ||
362 | ) | ||
363 | self.db.commit() | ||
364 | return cursor.rowcount != 0 | ||
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py index e9a361dc4b..f92f37c459 100644 --- a/bitbake/lib/hashserv/tests.py +++ b/bitbake/lib/hashserv/tests.py | |||
@@ -6,6 +6,8 @@ | |||
6 | # | 6 | # |
7 | 7 | ||
8 | from . import create_server, create_client | 8 | from . import create_server, create_client |
9 | from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS | ||
10 | from bb.asyncrpc import InvokeError | ||
9 | import hashlib | 11 | import hashlib |
10 | import logging | 12 | import logging |
11 | import multiprocessing | 13 | import multiprocessing |
@@ -29,8 +31,9 @@ class HashEquivalenceTestSetup(object): | |||
29 | METHOD = 'TestMethod' | 31 | METHOD = 'TestMethod' |
30 | 32 | ||
31 | server_index = 0 | 33 | server_index = 0 |
34 | client_index = 0 | ||
32 | 35 | ||
33 | def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc): | 36 | def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc, anon_perms=DEFAULT_ANON_PERMS, admin_username=None, admin_password=None): |
34 | self.server_index += 1 | 37 | self.server_index += 1 |
35 | if dbpath is None: | 38 | if dbpath is None: |
36 | dbpath = self.make_dbpath() | 39 | dbpath = self.make_dbpath() |
@@ -45,7 +48,10 @@ class HashEquivalenceTestSetup(object): | |||
45 | server = create_server(self.get_server_addr(self.server_index), | 48 | server = create_server(self.get_server_addr(self.server_index), |
46 | dbpath, | 49 | dbpath, |
47 | upstream=upstream, | 50 | upstream=upstream, |
48 | read_only=read_only) | 51 | read_only=read_only, |
52 | anon_perms=anon_perms, | ||
53 | admin_username=admin_username, | ||
54 | admin_password=admin_password) | ||
49 | server.dbpath = dbpath | 55 | server.dbpath = dbpath |
50 | 56 | ||
51 | server.serve_as_process(prefunc=prefunc, args=(self.server_index,)) | 57 | server.serve_as_process(prefunc=prefunc, args=(self.server_index,)) |
@@ -56,18 +62,31 @@ class HashEquivalenceTestSetup(object): | |||
56 | def make_dbpath(self): | 62 | def make_dbpath(self): |
57 | return os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index) | 63 | return os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index) |
58 | 64 | ||
59 | def start_client(self, server_address): | 65 | def start_client(self, server_address, username=None, password=None): |
60 | def cleanup_client(client): | 66 | def cleanup_client(client): |
61 | client.close() | 67 | client.close() |
62 | 68 | ||
63 | client = create_client(server_address) | 69 | client = create_client(server_address, username=username, password=password) |
64 | self.addCleanup(cleanup_client, client) | 70 | self.addCleanup(cleanup_client, client) |
65 | 71 | ||
66 | return client | 72 | return client |
67 | 73 | ||
68 | def start_test_server(self): | 74 | def start_test_server(self): |
69 | server = self.start_server() | 75 | self.server = self.start_server() |
70 | return server.address | 76 | return self.server.address |
77 | |||
78 | def start_auth_server(self): | ||
79 | self.auth_server = self.start_server(self.server.dbpath, anon_perms=[], admin_username="admin", admin_password="password") | ||
80 | self.admin_client = self.start_client(self.auth_server.address, username="admin", password="password") | ||
81 | return self.admin_client | ||
82 | |||
83 | def auth_client(self, user): | ||
84 | return self.start_client(self.auth_server.address, user["username"], user["token"]) | ||
85 | |||
86 | def auth_perms(self, *permissions): | ||
87 | self.client_index += 1 | ||
88 | user = self.admin_client.new_user(f"user-{self.client_index}", permissions) | ||
89 | return self.auth_client(user) | ||
71 | 90 | ||
72 | def setUp(self): | 91 | def setUp(self): |
73 | if sys.version_info < (3, 5, 0): | 92 | if sys.version_info < (3, 5, 0): |
@@ -86,18 +105,21 @@ class HashEquivalenceTestSetup(object): | |||
86 | 105 | ||
87 | 106 | ||
88 | class HashEquivalenceCommonTests(object): | 107 | class HashEquivalenceCommonTests(object): |
89 | def test_create_hash(self): | 108 | def create_test_hash(self, client): |
90 | # Simple test that hashes can be created | 109 | # Simple test that hashes can be created |
91 | taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' | 110 | taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9' |
92 | outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' | 111 | outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f' |
93 | unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' | 112 | unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd' |
94 | 113 | ||
95 | self.assertClientGetHash(self.client, taskhash, None) | 114 | self.assertClientGetHash(client, taskhash, None) |
96 | 115 | ||
97 | result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash) | 116 | result = client.report_unihash(taskhash, self.METHOD, outhash, unihash) |
98 | self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') | 117 | self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash') |
99 | return taskhash, outhash, unihash | 118 | return taskhash, outhash, unihash |
100 | 119 | ||
120 | def test_create_hash(self): | ||
121 | return self.create_test_hash(self.client) | ||
122 | |||
101 | def test_create_equivalent(self): | 123 | def test_create_equivalent(self): |
102 | # Tests that a second reported task with the same outhash will be | 124 | # Tests that a second reported task with the same outhash will be |
103 | # assigned the same unihash | 125 | # assigned the same unihash |
@@ -471,6 +493,242 @@ class HashEquivalenceCommonTests(object): | |||
471 | # shares a taskhash with Task 2 | 493 | # shares a taskhash with Task 2 |
472 | self.assertClientGetHash(self.client, taskhash2, unihash2) | 494 | self.assertClientGetHash(self.client, taskhash2, unihash2) |
473 | 495 | ||
496 | def test_auth_read_perms(self): | ||
497 | admin_client = self.start_auth_server() | ||
498 | |||
499 | # Create hashes with non-authenticated server | ||
500 | taskhash, outhash, unihash = self.test_create_hash() | ||
501 | |||
502 | # Validate hash can be retrieved using authenticated client | ||
503 | with self.auth_perms("@read") as client: | ||
504 | self.assertClientGetHash(client, taskhash, unihash) | ||
505 | |||
506 | with self.auth_perms() as client, self.assertRaises(InvokeError): | ||
507 | self.assertClientGetHash(client, taskhash, unihash) | ||
508 | |||
509 | def test_auth_report_perms(self): | ||
510 | admin_client = self.start_auth_server() | ||
511 | |||
512 | # Without read permission, the user is completely denied | ||
513 | with self.auth_perms() as client, self.assertRaises(InvokeError): | ||
514 | self.create_test_hash(client) | ||
515 | |||
516 | # Read permission allows the call to succeed, but it doesn't record | ||
517 | # anythin in the database | ||
518 | with self.auth_perms("@read") as client: | ||
519 | taskhash, outhash, unihash = self.create_test_hash(client) | ||
520 | self.assertClientGetHash(client, taskhash, None) | ||
521 | |||
522 | # Report permission alone is insufficient | ||
523 | with self.auth_perms("@report") as client, self.assertRaises(InvokeError): | ||
524 | self.create_test_hash(client) | ||
525 | |||
526 | # Read and report permission actually modify the database | ||
527 | with self.auth_perms("@read", "@report") as client: | ||
528 | taskhash, outhash, unihash = self.create_test_hash(client) | ||
529 | self.assertClientGetHash(client, taskhash, unihash) | ||
530 | |||
531 | def test_auth_no_token_refresh_from_anon_user(self): | ||
532 | self.start_auth_server() | ||
533 | |||
534 | with self.start_client(self.auth_server.address) as client, self.assertRaises(InvokeError): | ||
535 | client.refresh_token() | ||
536 | |||
537 | def assertUserCanAuth(self, user): | ||
538 | with self.start_client(self.auth_server.address) as client: | ||
539 | client.auth(user["username"], user["token"]) | ||
540 | |||
541 | def assertUserCannotAuth(self, user): | ||
542 | with self.start_client(self.auth_server.address) as client, self.assertRaises(InvokeError): | ||
543 | client.auth(user["username"], user["token"]) | ||
544 | |||
545 | def test_auth_self_token_refresh(self): | ||
546 | admin_client = self.start_auth_server() | ||
547 | |||
548 | # Create a new user with no permissions | ||
549 | user = admin_client.new_user("test-user", []) | ||
550 | |||
551 | with self.auth_client(user) as client: | ||
552 | new_user = client.refresh_token() | ||
553 | |||
554 | self.assertEqual(user["username"], new_user["username"]) | ||
555 | self.assertNotEqual(user["token"], new_user["token"]) | ||
556 | self.assertUserCanAuth(new_user) | ||
557 | self.assertUserCannotAuth(user) | ||
558 | |||
559 | # Explicitly specifying with your own username is fine also | ||
560 | with self.auth_client(new_user) as client: | ||
561 | new_user2 = client.refresh_token(user["username"]) | ||
562 | |||
563 | self.assertEqual(user["username"], new_user2["username"]) | ||
564 | self.assertNotEqual(user["token"], new_user2["token"]) | ||
565 | self.assertUserCanAuth(new_user2) | ||
566 | self.assertUserCannotAuth(new_user) | ||
567 | self.assertUserCannotAuth(user) | ||
568 | |||
569 | def test_auth_token_refresh(self): | ||
570 | admin_client = self.start_auth_server() | ||
571 | |||
572 | user = admin_client.new_user("test-user", []) | ||
573 | |||
574 | with self.auth_perms() as client, self.assertRaises(InvokeError): | ||
575 | client.refresh_token(user["username"]) | ||
576 | |||
577 | with self.auth_perms("@user-admin") as client: | ||
578 | new_user = client.refresh_token(user["username"]) | ||
579 | |||
580 | self.assertEqual(user["username"], new_user["username"]) | ||
581 | self.assertNotEqual(user["token"], new_user["token"]) | ||
582 | self.assertUserCanAuth(new_user) | ||
583 | self.assertUserCannotAuth(user) | ||
584 | |||
585 | def test_auth_self_get_user(self): | ||
586 | admin_client = self.start_auth_server() | ||
587 | |||
588 | user = admin_client.new_user("test-user", []) | ||
589 | user_info = user.copy() | ||
590 | del user_info["token"] | ||
591 | |||
592 | with self.auth_client(user) as client: | ||
593 | info = client.get_user() | ||
594 | self.assertEqual(info, user_info) | ||
595 | |||
596 | # Explicitly asking for your own username is fine also | ||
597 | info = client.get_user(user["username"]) | ||
598 | self.assertEqual(info, user_info) | ||
599 | |||
600 | def test_auth_get_user(self): | ||
601 | admin_client = self.start_auth_server() | ||
602 | |||
603 | user = admin_client.new_user("test-user", []) | ||
604 | user_info = user.copy() | ||
605 | del user_info["token"] | ||
606 | |||
607 | with self.auth_perms() as client, self.assertRaises(InvokeError): | ||
608 | client.get_user(user["username"]) | ||
609 | |||
610 | with self.auth_perms("@user-admin") as client: | ||
611 | info = client.get_user(user["username"]) | ||
612 | self.assertEqual(info, user_info) | ||
613 | |||
614 | info = client.get_user("nonexist-user") | ||
615 | self.assertIsNone(info) | ||
616 | |||
617 | def test_auth_reconnect(self): | ||
618 | admin_client = self.start_auth_server() | ||
619 | |||
620 | user = admin_client.new_user("test-user", []) | ||
621 | user_info = user.copy() | ||
622 | del user_info["token"] | ||
623 | |||
624 | with self.auth_client(user) as client: | ||
625 | info = client.get_user() | ||
626 | self.assertEqual(info, user_info) | ||
627 | |||
628 | client.disconnect() | ||
629 | |||
630 | info = client.get_user() | ||
631 | self.assertEqual(info, user_info) | ||
632 | |||
633 | def test_auth_delete_user(self): | ||
634 | admin_client = self.start_auth_server() | ||
635 | |||
636 | user = admin_client.new_user("test-user", []) | ||
637 | |||
638 | # No self service | ||
639 | with self.auth_client(user) as client, self.assertRaises(InvokeError): | ||
640 | client.delete_user(user["username"]) | ||
641 | |||
642 | with self.auth_perms() as client, self.assertRaises(InvokeError): | ||
643 | client.delete_user(user["username"]) | ||
644 | |||
645 | with self.auth_perms("@user-admin") as client: | ||
646 | client.delete_user(user["username"]) | ||
647 | |||
648 | # User doesn't exist, so even though the permission is correct, it's an | ||
649 | # error | ||
650 | with self.auth_perms("@user-admin") as client, self.assertRaises(InvokeError): | ||
651 | client.delete_user(user["username"]) | ||
652 | |||
653 | def assertUserPerms(self, user, permissions): | ||
654 | with self.auth_client(user) as client: | ||
655 | info = client.get_user() | ||
656 | self.assertEqual(info, { | ||
657 | "username": user["username"], | ||
658 | "permissions": permissions, | ||
659 | }) | ||
660 | |||
661 | def test_auth_set_user_perms(self): | ||
662 | admin_client = self.start_auth_server() | ||
663 | |||
664 | user = admin_client.new_user("test-user", []) | ||
665 | |||
666 | self.assertUserPerms(user, []) | ||
667 | |||
668 | # No self service to change permissions | ||
669 | with self.auth_client(user) as client, self.assertRaises(InvokeError): | ||
670 | client.set_user_perms(user["username"], ["@all"]) | ||
671 | self.assertUserPerms(user, []) | ||
672 | |||
673 | with self.auth_perms() as client, self.assertRaises(InvokeError): | ||
674 | client.set_user_perms(user["username"], ["@all"]) | ||
675 | self.assertUserPerms(user, []) | ||
676 | |||
677 | with self.auth_perms("@user-admin") as client: | ||
678 | client.set_user_perms(user["username"], ["@all"]) | ||
679 | self.assertUserPerms(user, sorted(list(ALL_PERMISSIONS))) | ||
680 | |||
681 | # Bad permissions | ||
682 | with self.auth_perms("@user-admin") as client, self.assertRaises(InvokeError): | ||
683 | client.set_user_perms(user["username"], ["@this-is-not-a-permission"]) | ||
684 | self.assertUserPerms(user, sorted(list(ALL_PERMISSIONS))) | ||
685 | |||
686 | def test_auth_get_all_users(self): | ||
687 | admin_client = self.start_auth_server() | ||
688 | |||
689 | user = admin_client.new_user("test-user", []) | ||
690 | |||
691 | with self.auth_client(user) as client, self.assertRaises(InvokeError): | ||
692 | client.get_all_users() | ||
693 | |||
694 | # Give the test user the correct permission | ||
695 | admin_client.set_user_perms(user["username"], ["@user-admin"]) | ||
696 | |||
697 | with self.auth_client(user) as client: | ||
698 | all_users = client.get_all_users() | ||
699 | |||
700 | # Convert to a dictionary for easier comparison | ||
701 | all_users = {u["username"]: u for u in all_users} | ||
702 | |||
703 | self.assertEqual(all_users, | ||
704 | { | ||
705 | "admin": { | ||
706 | "username": "admin", | ||
707 | "permissions": sorted(list(ALL_PERMISSIONS)), | ||
708 | }, | ||
709 | "test-user": { | ||
710 | "username": "test-user", | ||
711 | "permissions": ["@user-admin"], | ||
712 | } | ||
713 | } | ||
714 | ) | ||
715 | |||
716 | def test_auth_new_user(self): | ||
717 | self.start_auth_server() | ||
718 | |||
719 | permissions = ["@read", "@report", "@db-admin", "@user-admin"] | ||
720 | permissions.sort() | ||
721 | |||
722 | with self.auth_perms() as client, self.assertRaises(InvokeError): | ||
723 | client.new_user("test-user", permissions) | ||
724 | |||
725 | with self.auth_perms("@user-admin") as client: | ||
726 | user = client.new_user("test-user", permissions) | ||
727 | self.assertIn("token", user) | ||
728 | self.assertEqual(user["username"], "test-user") | ||
729 | self.assertEqual(user["permissions"], permissions) | ||
730 | |||
731 | |||
474 | class TestHashEquivalenceUnixServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): | 732 | class TestHashEquivalenceUnixServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase): |
475 | def get_server_addr(self, server_idx): | 733 | def get_server_addr(self, server_idx): |
476 | return "unix://" + os.path.join(self.temp_dir.name, 'sock%d' % server_idx) | 734 | return "unix://" + os.path.join(self.temp_dir.name, 'sock%d' % server_idx) |