diff options
-rw-r--r-- | hooks.py | 431 | ||||
-rw-r--r-- | project.py | 404 | ||||
-rw-r--r-- | subcmds/upload.py | 2 | ||||
-rw-r--r-- | tests/test_hooks.py | 60 | ||||
-rw-r--r-- | tests/test_project.py | 39 |
5 files changed, 493 insertions, 443 deletions
diff --git a/hooks.py b/hooks.py new file mode 100644 index 00000000..177bc88b --- /dev/null +++ b/hooks.py | |||
@@ -0,0 +1,431 @@ | |||
1 | # -*- coding:utf-8 -*- | ||
2 | # | ||
3 | # Copyright (C) 2008 The Android Open Source Project | ||
4 | # | ||
5 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
6 | # you may not use this file except in compliance with the License. | ||
7 | # You may obtain a copy of the License at | ||
8 | # | ||
9 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
10 | # | ||
11 | # Unless required by applicable law or agreed to in writing, software | ||
12 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
14 | # See the License for the specific language governing permissions and | ||
15 | # limitations under the License. | ||
16 | |||
17 | import json | ||
18 | import os | ||
19 | import re | ||
20 | import sys | ||
21 | import traceback | ||
22 | |||
23 | from error import HookError | ||
24 | from git_refs import HEAD | ||
25 | |||
26 | from pyversion import is_python3 | ||
27 | if is_python3(): | ||
28 | import urllib.parse | ||
29 | else: | ||
30 | import imp | ||
31 | import urlparse | ||
32 | urllib = imp.new_module('urllib') | ||
33 | urllib.parse = urlparse | ||
34 | input = raw_input # noqa: F821 | ||
35 | |||
36 | class RepoHook(object): | ||
37 | """A RepoHook contains information about a script to run as a hook. | ||
38 | |||
39 | Hooks are used to run a python script before running an upload (for instance, | ||
40 | to run presubmit checks). Eventually, we may have hooks for other actions. | ||
41 | |||
42 | This shouldn't be confused with files in the 'repo/hooks' directory. Those | ||
43 | files are copied into each '.git/hooks' folder for each project. Repo-level | ||
44 | hooks are associated instead with repo actions. | ||
45 | |||
46 | Hooks are always python. When a hook is run, we will load the hook into the | ||
47 | interpreter and execute its main() function. | ||
48 | """ | ||
49 | |||
50 | def __init__(self, | ||
51 | hook_type, | ||
52 | hooks_project, | ||
53 | topdir, | ||
54 | manifest_url, | ||
55 | abort_if_user_denies=False): | ||
56 | """RepoHook constructor. | ||
57 | |||
58 | Params: | ||
59 | hook_type: A string representing the type of hook. This is also used | ||
60 | to figure out the name of the file containing the hook. For | ||
61 | example: 'pre-upload'. | ||
62 | hooks_project: The project containing the repo hooks. If you have a | ||
63 | manifest, this is manifest.repo_hooks_project. OK if this is None, | ||
64 | which will make the hook a no-op. | ||
65 | topdir: Repo's top directory (the one containing the .repo directory). | ||
66 | Scripts will run with CWD as this directory. If you have a manifest, | ||
67 | this is manifest.topdir | ||
68 | manifest_url: The URL to the manifest git repo. | ||
69 | abort_if_user_denies: If True, we'll throw a HookError() if the user | ||
70 | doesn't allow us to run the hook. | ||
71 | """ | ||
72 | self._hook_type = hook_type | ||
73 | self._hooks_project = hooks_project | ||
74 | self._manifest_url = manifest_url | ||
75 | self._topdir = topdir | ||
76 | self._abort_if_user_denies = abort_if_user_denies | ||
77 | |||
78 | # Store the full path to the script for convenience. | ||
79 | if self._hooks_project: | ||
80 | self._script_fullpath = os.path.join(self._hooks_project.worktree, | ||
81 | self._hook_type + '.py') | ||
82 | else: | ||
83 | self._script_fullpath = None | ||
84 | |||
85 | def _GetHash(self): | ||
86 | """Return a hash of the contents of the hooks directory. | ||
87 | |||
88 | We'll just use git to do this. This hash has the property that if anything | ||
89 | changes in the directory we will return a different has. | ||
90 | |||
91 | SECURITY CONSIDERATION: | ||
92 | This hash only represents the contents of files in the hook directory, not | ||
93 | any other files imported or called by hooks. Changes to imported files | ||
94 | can change the script behavior without affecting the hash. | ||
95 | |||
96 | Returns: | ||
97 | A string representing the hash. This will always be ASCII so that it can | ||
98 | be printed to the user easily. | ||
99 | """ | ||
100 | assert self._hooks_project, "Must have hooks to calculate their hash." | ||
101 | |||
102 | # We will use the work_git object rather than just calling GetRevisionId(). | ||
103 | # That gives us a hash of the latest checked in version of the files that | ||
104 | # the user will actually be executing. Specifically, GetRevisionId() | ||
105 | # doesn't appear to change even if a user checks out a different version | ||
106 | # of the hooks repo (via git checkout) nor if a user commits their own revs. | ||
107 | # | ||
108 | # NOTE: Local (non-committed) changes will not be factored into this hash. | ||
109 | # I think this is OK, since we're really only worried about warning the user | ||
110 | # about upstream changes. | ||
111 | return self._hooks_project.work_git.rev_parse('HEAD') | ||
112 | |||
113 | def _GetMustVerb(self): | ||
114 | """Return 'must' if the hook is required; 'should' if not.""" | ||
115 | if self._abort_if_user_denies: | ||
116 | return 'must' | ||
117 | else: | ||
118 | return 'should' | ||
119 | |||
120 | def _CheckForHookApproval(self): | ||
121 | """Check to see whether this hook has been approved. | ||
122 | |||
123 | We'll accept approval of manifest URLs if they're using secure transports. | ||
124 | This way the user can say they trust the manifest hoster. For insecure | ||
125 | hosts, we fall back to checking the hash of the hooks repo. | ||
126 | |||
127 | Note that we ask permission for each individual hook even though we use | ||
128 | the hash of all hooks when detecting changes. We'd like the user to be | ||
129 | able to approve / deny each hook individually. We only use the hash of all | ||
130 | hooks because there is no other easy way to detect changes to local imports. | ||
131 | |||
132 | Returns: | ||
133 | True if this hook is approved to run; False otherwise. | ||
134 | |||
135 | Raises: | ||
136 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
137 | was passed to the consturctor. | ||
138 | """ | ||
139 | if self._ManifestUrlHasSecureScheme(): | ||
140 | return self._CheckForHookApprovalManifest() | ||
141 | else: | ||
142 | return self._CheckForHookApprovalHash() | ||
143 | |||
144 | def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt, | ||
145 | changed_prompt): | ||
146 | """Check for approval for a particular attribute and hook. | ||
147 | |||
148 | Args: | ||
149 | subkey: The git config key under [repo.hooks.<hook_type>] to store the | ||
150 | last approved string. | ||
151 | new_val: The new value to compare against the last approved one. | ||
152 | main_prompt: Message to display to the user to ask for approval. | ||
153 | changed_prompt: Message explaining why we're re-asking for approval. | ||
154 | |||
155 | Returns: | ||
156 | True if this hook is approved to run; False otherwise. | ||
157 | |||
158 | Raises: | ||
159 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
160 | was passed to the consturctor. | ||
161 | """ | ||
162 | hooks_config = self._hooks_project.config | ||
163 | git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey) | ||
164 | |||
165 | # Get the last value that the user approved for this hook; may be None. | ||
166 | old_val = hooks_config.GetString(git_approval_key) | ||
167 | |||
168 | if old_val is not None: | ||
169 | # User previously approved hook and asked not to be prompted again. | ||
170 | if new_val == old_val: | ||
171 | # Approval matched. We're done. | ||
172 | return True | ||
173 | else: | ||
174 | # Give the user a reason why we're prompting, since they last told | ||
175 | # us to "never ask again". | ||
176 | prompt = 'WARNING: %s\n\n' % (changed_prompt,) | ||
177 | else: | ||
178 | prompt = '' | ||
179 | |||
180 | # Prompt the user if we're not on a tty; on a tty we'll assume "no". | ||
181 | if sys.stdout.isatty(): | ||
182 | prompt += main_prompt + ' (yes/always/NO)? ' | ||
183 | response = input(prompt).lower() | ||
184 | print() | ||
185 | |||
186 | # User is doing a one-time approval. | ||
187 | if response in ('y', 'yes'): | ||
188 | return True | ||
189 | elif response == 'always': | ||
190 | hooks_config.SetString(git_approval_key, new_val) | ||
191 | return True | ||
192 | |||
193 | # For anything else, we'll assume no approval. | ||
194 | if self._abort_if_user_denies: | ||
195 | raise HookError('You must allow the %s hook or use --no-verify.' % | ||
196 | self._hook_type) | ||
197 | |||
198 | return False | ||
199 | |||
200 | def _ManifestUrlHasSecureScheme(self): | ||
201 | """Check if the URI for the manifest is a secure transport.""" | ||
202 | secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc') | ||
203 | parse_results = urllib.parse.urlparse(self._manifest_url) | ||
204 | return parse_results.scheme in secure_schemes | ||
205 | |||
206 | def _CheckForHookApprovalManifest(self): | ||
207 | """Check whether the user has approved this manifest host. | ||
208 | |||
209 | Returns: | ||
210 | True if this hook is approved to run; False otherwise. | ||
211 | """ | ||
212 | return self._CheckForHookApprovalHelper( | ||
213 | 'approvedmanifest', | ||
214 | self._manifest_url, | ||
215 | 'Run hook scripts from %s' % (self._manifest_url,), | ||
216 | 'Manifest URL has changed since %s was allowed.' % (self._hook_type,)) | ||
217 | |||
218 | def _CheckForHookApprovalHash(self): | ||
219 | """Check whether the user has approved the hooks repo. | ||
220 | |||
221 | Returns: | ||
222 | True if this hook is approved to run; False otherwise. | ||
223 | """ | ||
224 | prompt = ('Repo %s run the script:\n' | ||
225 | ' %s\n' | ||
226 | '\n' | ||
227 | 'Do you want to allow this script to run') | ||
228 | return self._CheckForHookApprovalHelper( | ||
229 | 'approvedhash', | ||
230 | self._GetHash(), | ||
231 | prompt % (self._GetMustVerb(), self._script_fullpath), | ||
232 | 'Scripts have changed since %s was allowed.' % (self._hook_type,)) | ||
233 | |||
234 | @staticmethod | ||
235 | def _ExtractInterpFromShebang(data): | ||
236 | """Extract the interpreter used in the shebang. | ||
237 | |||
238 | Try to locate the interpreter the script is using (ignoring `env`). | ||
239 | |||
240 | Args: | ||
241 | data: The file content of the script. | ||
242 | |||
243 | Returns: | ||
244 | The basename of the main script interpreter, or None if a shebang is not | ||
245 | used or could not be parsed out. | ||
246 | """ | ||
247 | firstline = data.splitlines()[:1] | ||
248 | if not firstline: | ||
249 | return None | ||
250 | |||
251 | # The format here can be tricky. | ||
252 | shebang = firstline[0].strip() | ||
253 | m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang) | ||
254 | if not m: | ||
255 | return None | ||
256 | |||
257 | # If the using `env`, find the target program. | ||
258 | interp = m.group(1) | ||
259 | if os.path.basename(interp) == 'env': | ||
260 | interp = m.group(2) | ||
261 | |||
262 | return interp | ||
263 | |||
264 | def _ExecuteHookViaReexec(self, interp, context, **kwargs): | ||
265 | """Execute the hook script through |interp|. | ||
266 | |||
267 | Note: Support for this feature should be dropped ~Jun 2021. | ||
268 | |||
269 | Args: | ||
270 | interp: The Python program to run. | ||
271 | context: Basic Python context to execute the hook inside. | ||
272 | kwargs: Arbitrary arguments to pass to the hook script. | ||
273 | |||
274 | Raises: | ||
275 | HookError: When the hooks failed for any reason. | ||
276 | """ | ||
277 | # This logic needs to be kept in sync with _ExecuteHookViaImport below. | ||
278 | script = """ | ||
279 | import json, os, sys | ||
280 | path = '''%(path)s''' | ||
281 | kwargs = json.loads('''%(kwargs)s''') | ||
282 | context = json.loads('''%(context)s''') | ||
283 | sys.path.insert(0, os.path.dirname(path)) | ||
284 | data = open(path).read() | ||
285 | exec(compile(data, path, 'exec'), context) | ||
286 | context['main'](**kwargs) | ||
287 | """ % { | ||
288 | 'path': self._script_fullpath, | ||
289 | 'kwargs': json.dumps(kwargs), | ||
290 | 'context': json.dumps(context), | ||
291 | } | ||
292 | |||
293 | # We pass the script via stdin to avoid OS argv limits. It also makes | ||
294 | # unhandled exception tracebacks less verbose/confusing for users. | ||
295 | cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())'] | ||
296 | proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) | ||
297 | proc.communicate(input=script.encode('utf-8')) | ||
298 | if proc.returncode: | ||
299 | raise HookError('Failed to run %s hook.' % (self._hook_type,)) | ||
300 | |||
301 | def _ExecuteHookViaImport(self, data, context, **kwargs): | ||
302 | """Execute the hook code in |data| directly. | ||
303 | |||
304 | Args: | ||
305 | data: The code of the hook to execute. | ||
306 | context: Basic Python context to execute the hook inside. | ||
307 | kwargs: Arbitrary arguments to pass to the hook script. | ||
308 | |||
309 | Raises: | ||
310 | HookError: When the hooks failed for any reason. | ||
311 | """ | ||
312 | # Exec, storing global context in the context dict. We catch exceptions | ||
313 | # and convert to a HookError w/ just the failing traceback. | ||
314 | try: | ||
315 | exec(compile(data, self._script_fullpath, 'exec'), context) | ||
316 | except Exception: | ||
317 | raise HookError('%s\nFailed to import %s hook; see traceback above.' % | ||
318 | (traceback.format_exc(), self._hook_type)) | ||
319 | |||
320 | # Running the script should have defined a main() function. | ||
321 | if 'main' not in context: | ||
322 | raise HookError('Missing main() in: "%s"' % self._script_fullpath) | ||
323 | |||
324 | # Call the main function in the hook. If the hook should cause the | ||
325 | # build to fail, it will raise an Exception. We'll catch that convert | ||
326 | # to a HookError w/ just the failing traceback. | ||
327 | try: | ||
328 | context['main'](**kwargs) | ||
329 | except Exception: | ||
330 | raise HookError('%s\nFailed to run main() for %s hook; see traceback ' | ||
331 | 'above.' % (traceback.format_exc(), self._hook_type)) | ||
332 | |||
333 | def _ExecuteHook(self, **kwargs): | ||
334 | """Actually execute the given hook. | ||
335 | |||
336 | This will run the hook's 'main' function in our python interpreter. | ||
337 | |||
338 | Args: | ||
339 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
340 | to the hook type. For instance, pre-upload hooks will contain | ||
341 | a project_list. | ||
342 | """ | ||
343 | # Keep sys.path and CWD stashed away so that we can always restore them | ||
344 | # upon function exit. | ||
345 | orig_path = os.getcwd() | ||
346 | orig_syspath = sys.path | ||
347 | |||
348 | try: | ||
349 | # Always run hooks with CWD as topdir. | ||
350 | os.chdir(self._topdir) | ||
351 | |||
352 | # Put the hook dir as the first item of sys.path so hooks can do | ||
353 | # relative imports. We want to replace the repo dir as [0] so | ||
354 | # hooks can't import repo files. | ||
355 | sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:] | ||
356 | |||
357 | # Initial global context for the hook to run within. | ||
358 | context = {'__file__': self._script_fullpath} | ||
359 | |||
360 | # Add 'hook_should_take_kwargs' to the arguments to be passed to main. | ||
361 | # We don't actually want hooks to define their main with this argument-- | ||
362 | # it's there to remind them that their hook should always take **kwargs. | ||
363 | # For instance, a pre-upload hook should be defined like: | ||
364 | # def main(project_list, **kwargs): | ||
365 | # | ||
366 | # This allows us to later expand the API without breaking old hooks. | ||
367 | kwargs = kwargs.copy() | ||
368 | kwargs['hook_should_take_kwargs'] = True | ||
369 | |||
370 | # See what version of python the hook has been written against. | ||
371 | data = open(self._script_fullpath).read() | ||
372 | interp = self._ExtractInterpFromShebang(data) | ||
373 | reexec = False | ||
374 | if interp: | ||
375 | prog = os.path.basename(interp) | ||
376 | if prog.startswith('python2') and sys.version_info.major != 2: | ||
377 | reexec = True | ||
378 | elif prog.startswith('python3') and sys.version_info.major == 2: | ||
379 | reexec = True | ||
380 | |||
381 | # Attempt to execute the hooks through the requested version of Python. | ||
382 | if reexec: | ||
383 | try: | ||
384 | self._ExecuteHookViaReexec(interp, context, **kwargs) | ||
385 | except OSError as e: | ||
386 | if e.errno == errno.ENOENT: | ||
387 | # We couldn't find the interpreter, so fallback to importing. | ||
388 | reexec = False | ||
389 | else: | ||
390 | raise | ||
391 | |||
392 | # Run the hook by importing directly. | ||
393 | if not reexec: | ||
394 | self._ExecuteHookViaImport(data, context, **kwargs) | ||
395 | finally: | ||
396 | # Restore sys.path and CWD. | ||
397 | sys.path = orig_syspath | ||
398 | os.chdir(orig_path) | ||
399 | |||
400 | def Run(self, user_allows_all_hooks, **kwargs): | ||
401 | """Run the hook. | ||
402 | |||
403 | If the hook doesn't exist (because there is no hooks project or because | ||
404 | this particular hook is not enabled), this is a no-op. | ||
405 | |||
406 | Args: | ||
407 | user_allows_all_hooks: If True, we will never prompt about running the | ||
408 | hook--we'll just assume it's OK to run it. | ||
409 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
410 | to the hook type. For instance, pre-upload hooks will contain | ||
411 | a project_list. | ||
412 | |||
413 | Raises: | ||
414 | HookError: If there was a problem finding the hook or the user declined | ||
415 | to run a required hook (from _CheckForHookApproval). | ||
416 | """ | ||
417 | # No-op if there is no hooks project or if hook is disabled. | ||
418 | if ((not self._hooks_project) or (self._hook_type not in | ||
419 | self._hooks_project.enabled_repo_hooks)): | ||
420 | return | ||
421 | |||
422 | # Bail with a nice error if we can't find the hook. | ||
423 | if not os.path.isfile(self._script_fullpath): | ||
424 | raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath) | ||
425 | |||
426 | # Make sure the user is OK with running the hook. | ||
427 | if (not user_allows_all_hooks) and (not self._CheckForHookApproval()): | ||
428 | return | ||
429 | |||
430 | # Run the hook with the same version of python we're using. | ||
431 | self._ExecuteHook(**kwargs) | ||
@@ -18,7 +18,6 @@ from __future__ import print_function | |||
18 | import errno | 18 | import errno |
19 | import filecmp | 19 | import filecmp |
20 | import glob | 20 | import glob |
21 | import json | ||
22 | import os | 21 | import os |
23 | import random | 22 | import random |
24 | import re | 23 | import re |
@@ -29,13 +28,12 @@ import sys | |||
29 | import tarfile | 28 | import tarfile |
30 | import tempfile | 29 | import tempfile |
31 | import time | 30 | import time |
32 | import traceback | ||
33 | 31 | ||
34 | from color import Coloring | 32 | from color import Coloring |
35 | from git_command import GitCommand, git_require | 33 | from git_command import GitCommand, git_require |
36 | from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ | 34 | from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ |
37 | ID_RE | 35 | ID_RE |
38 | from error import GitError, HookError, UploadError, DownloadError | 36 | from error import GitError, UploadError, DownloadError |
39 | from error import ManifestInvalidRevisionError, ManifestInvalidPathError | 37 | from error import ManifestInvalidRevisionError, ManifestInvalidPathError |
40 | from error import NoManifestException | 38 | from error import NoManifestException |
41 | import platform_utils | 39 | import platform_utils |
@@ -451,406 +449,6 @@ class RemoteSpec(object): | |||
451 | self.orig_name = orig_name | 449 | self.orig_name = orig_name |
452 | self.fetchUrl = fetchUrl | 450 | self.fetchUrl = fetchUrl |
453 | 451 | ||
454 | |||
455 | class RepoHook(object): | ||
456 | |||
457 | """A RepoHook contains information about a script to run as a hook. | ||
458 | |||
459 | Hooks are used to run a python script before running an upload (for instance, | ||
460 | to run presubmit checks). Eventually, we may have hooks for other actions. | ||
461 | |||
462 | This shouldn't be confused with files in the 'repo/hooks' directory. Those | ||
463 | files are copied into each '.git/hooks' folder for each project. Repo-level | ||
464 | hooks are associated instead with repo actions. | ||
465 | |||
466 | Hooks are always python. When a hook is run, we will load the hook into the | ||
467 | interpreter and execute its main() function. | ||
468 | """ | ||
469 | |||
470 | def __init__(self, | ||
471 | hook_type, | ||
472 | hooks_project, | ||
473 | topdir, | ||
474 | manifest_url, | ||
475 | abort_if_user_denies=False): | ||
476 | """RepoHook constructor. | ||
477 | |||
478 | Params: | ||
479 | hook_type: A string representing the type of hook. This is also used | ||
480 | to figure out the name of the file containing the hook. For | ||
481 | example: 'pre-upload'. | ||
482 | hooks_project: The project containing the repo hooks. If you have a | ||
483 | manifest, this is manifest.repo_hooks_project. OK if this is None, | ||
484 | which will make the hook a no-op. | ||
485 | topdir: Repo's top directory (the one containing the .repo directory). | ||
486 | Scripts will run with CWD as this directory. If you have a manifest, | ||
487 | this is manifest.topdir | ||
488 | manifest_url: The URL to the manifest git repo. | ||
489 | abort_if_user_denies: If True, we'll throw a HookError() if the user | ||
490 | doesn't allow us to run the hook. | ||
491 | """ | ||
492 | self._hook_type = hook_type | ||
493 | self._hooks_project = hooks_project | ||
494 | self._manifest_url = manifest_url | ||
495 | self._topdir = topdir | ||
496 | self._abort_if_user_denies = abort_if_user_denies | ||
497 | |||
498 | # Store the full path to the script for convenience. | ||
499 | if self._hooks_project: | ||
500 | self._script_fullpath = os.path.join(self._hooks_project.worktree, | ||
501 | self._hook_type + '.py') | ||
502 | else: | ||
503 | self._script_fullpath = None | ||
504 | |||
505 | def _GetHash(self): | ||
506 | """Return a hash of the contents of the hooks directory. | ||
507 | |||
508 | We'll just use git to do this. This hash has the property that if anything | ||
509 | changes in the directory we will return a different has. | ||
510 | |||
511 | SECURITY CONSIDERATION: | ||
512 | This hash only represents the contents of files in the hook directory, not | ||
513 | any other files imported or called by hooks. Changes to imported files | ||
514 | can change the script behavior without affecting the hash. | ||
515 | |||
516 | Returns: | ||
517 | A string representing the hash. This will always be ASCII so that it can | ||
518 | be printed to the user easily. | ||
519 | """ | ||
520 | assert self._hooks_project, "Must have hooks to calculate their hash." | ||
521 | |||
522 | # We will use the work_git object rather than just calling GetRevisionId(). | ||
523 | # That gives us a hash of the latest checked in version of the files that | ||
524 | # the user will actually be executing. Specifically, GetRevisionId() | ||
525 | # doesn't appear to change even if a user checks out a different version | ||
526 | # of the hooks repo (via git checkout) nor if a user commits their own revs. | ||
527 | # | ||
528 | # NOTE: Local (non-committed) changes will not be factored into this hash. | ||
529 | # I think this is OK, since we're really only worried about warning the user | ||
530 | # about upstream changes. | ||
531 | return self._hooks_project.work_git.rev_parse('HEAD') | ||
532 | |||
533 | def _GetMustVerb(self): | ||
534 | """Return 'must' if the hook is required; 'should' if not.""" | ||
535 | if self._abort_if_user_denies: | ||
536 | return 'must' | ||
537 | else: | ||
538 | return 'should' | ||
539 | |||
540 | def _CheckForHookApproval(self): | ||
541 | """Check to see whether this hook has been approved. | ||
542 | |||
543 | We'll accept approval of manifest URLs if they're using secure transports. | ||
544 | This way the user can say they trust the manifest hoster. For insecure | ||
545 | hosts, we fall back to checking the hash of the hooks repo. | ||
546 | |||
547 | Note that we ask permission for each individual hook even though we use | ||
548 | the hash of all hooks when detecting changes. We'd like the user to be | ||
549 | able to approve / deny each hook individually. We only use the hash of all | ||
550 | hooks because there is no other easy way to detect changes to local imports. | ||
551 | |||
552 | Returns: | ||
553 | True if this hook is approved to run; False otherwise. | ||
554 | |||
555 | Raises: | ||
556 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
557 | was passed to the consturctor. | ||
558 | """ | ||
559 | if self._ManifestUrlHasSecureScheme(): | ||
560 | return self._CheckForHookApprovalManifest() | ||
561 | else: | ||
562 | return self._CheckForHookApprovalHash() | ||
563 | |||
564 | def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt, | ||
565 | changed_prompt): | ||
566 | """Check for approval for a particular attribute and hook. | ||
567 | |||
568 | Args: | ||
569 | subkey: The git config key under [repo.hooks.<hook_type>] to store the | ||
570 | last approved string. | ||
571 | new_val: The new value to compare against the last approved one. | ||
572 | main_prompt: Message to display to the user to ask for approval. | ||
573 | changed_prompt: Message explaining why we're re-asking for approval. | ||
574 | |||
575 | Returns: | ||
576 | True if this hook is approved to run; False otherwise. | ||
577 | |||
578 | Raises: | ||
579 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
580 | was passed to the consturctor. | ||
581 | """ | ||
582 | hooks_config = self._hooks_project.config | ||
583 | git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey) | ||
584 | |||
585 | # Get the last value that the user approved for this hook; may be None. | ||
586 | old_val = hooks_config.GetString(git_approval_key) | ||
587 | |||
588 | if old_val is not None: | ||
589 | # User previously approved hook and asked not to be prompted again. | ||
590 | if new_val == old_val: | ||
591 | # Approval matched. We're done. | ||
592 | return True | ||
593 | else: | ||
594 | # Give the user a reason why we're prompting, since they last told | ||
595 | # us to "never ask again". | ||
596 | prompt = 'WARNING: %s\n\n' % (changed_prompt,) | ||
597 | else: | ||
598 | prompt = '' | ||
599 | |||
600 | # Prompt the user if we're not on a tty; on a tty we'll assume "no". | ||
601 | if sys.stdout.isatty(): | ||
602 | prompt += main_prompt + ' (yes/always/NO)? ' | ||
603 | response = input(prompt).lower() | ||
604 | print() | ||
605 | |||
606 | # User is doing a one-time approval. | ||
607 | if response in ('y', 'yes'): | ||
608 | return True | ||
609 | elif response == 'always': | ||
610 | hooks_config.SetString(git_approval_key, new_val) | ||
611 | return True | ||
612 | |||
613 | # For anything else, we'll assume no approval. | ||
614 | if self._abort_if_user_denies: | ||
615 | raise HookError('You must allow the %s hook or use --no-verify.' % | ||
616 | self._hook_type) | ||
617 | |||
618 | return False | ||
619 | |||
620 | def _ManifestUrlHasSecureScheme(self): | ||
621 | """Check if the URI for the manifest is a secure transport.""" | ||
622 | secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc') | ||
623 | parse_results = urllib.parse.urlparse(self._manifest_url) | ||
624 | return parse_results.scheme in secure_schemes | ||
625 | |||
626 | def _CheckForHookApprovalManifest(self): | ||
627 | """Check whether the user has approved this manifest host. | ||
628 | |||
629 | Returns: | ||
630 | True if this hook is approved to run; False otherwise. | ||
631 | """ | ||
632 | return self._CheckForHookApprovalHelper( | ||
633 | 'approvedmanifest', | ||
634 | self._manifest_url, | ||
635 | 'Run hook scripts from %s' % (self._manifest_url,), | ||
636 | 'Manifest URL has changed since %s was allowed.' % (self._hook_type,)) | ||
637 | |||
638 | def _CheckForHookApprovalHash(self): | ||
639 | """Check whether the user has approved the hooks repo. | ||
640 | |||
641 | Returns: | ||
642 | True if this hook is approved to run; False otherwise. | ||
643 | """ | ||
644 | prompt = ('Repo %s run the script:\n' | ||
645 | ' %s\n' | ||
646 | '\n' | ||
647 | 'Do you want to allow this script to run') | ||
648 | return self._CheckForHookApprovalHelper( | ||
649 | 'approvedhash', | ||
650 | self._GetHash(), | ||
651 | prompt % (self._GetMustVerb(), self._script_fullpath), | ||
652 | 'Scripts have changed since %s was allowed.' % (self._hook_type,)) | ||
653 | |||
654 | @staticmethod | ||
655 | def _ExtractInterpFromShebang(data): | ||
656 | """Extract the interpreter used in the shebang. | ||
657 | |||
658 | Try to locate the interpreter the script is using (ignoring `env`). | ||
659 | |||
660 | Args: | ||
661 | data: The file content of the script. | ||
662 | |||
663 | Returns: | ||
664 | The basename of the main script interpreter, or None if a shebang is not | ||
665 | used or could not be parsed out. | ||
666 | """ | ||
667 | firstline = data.splitlines()[:1] | ||
668 | if not firstline: | ||
669 | return None | ||
670 | |||
671 | # The format here can be tricky. | ||
672 | shebang = firstline[0].strip() | ||
673 | m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang) | ||
674 | if not m: | ||
675 | return None | ||
676 | |||
677 | # If the using `env`, find the target program. | ||
678 | interp = m.group(1) | ||
679 | if os.path.basename(interp) == 'env': | ||
680 | interp = m.group(2) | ||
681 | |||
682 | return interp | ||
683 | |||
684 | def _ExecuteHookViaReexec(self, interp, context, **kwargs): | ||
685 | """Execute the hook script through |interp|. | ||
686 | |||
687 | Note: Support for this feature should be dropped ~Jun 2021. | ||
688 | |||
689 | Args: | ||
690 | interp: The Python program to run. | ||
691 | context: Basic Python context to execute the hook inside. | ||
692 | kwargs: Arbitrary arguments to pass to the hook script. | ||
693 | |||
694 | Raises: | ||
695 | HookError: When the hooks failed for any reason. | ||
696 | """ | ||
697 | # This logic needs to be kept in sync with _ExecuteHookViaImport below. | ||
698 | script = """ | ||
699 | import json, os, sys | ||
700 | path = '''%(path)s''' | ||
701 | kwargs = json.loads('''%(kwargs)s''') | ||
702 | context = json.loads('''%(context)s''') | ||
703 | sys.path.insert(0, os.path.dirname(path)) | ||
704 | data = open(path).read() | ||
705 | exec(compile(data, path, 'exec'), context) | ||
706 | context['main'](**kwargs) | ||
707 | """ % { | ||
708 | 'path': self._script_fullpath, | ||
709 | 'kwargs': json.dumps(kwargs), | ||
710 | 'context': json.dumps(context), | ||
711 | } | ||
712 | |||
713 | # We pass the script via stdin to avoid OS argv limits. It also makes | ||
714 | # unhandled exception tracebacks less verbose/confusing for users. | ||
715 | cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())'] | ||
716 | proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) | ||
717 | proc.communicate(input=script.encode('utf-8')) | ||
718 | if proc.returncode: | ||
719 | raise HookError('Failed to run %s hook.' % (self._hook_type,)) | ||
720 | |||
721 | def _ExecuteHookViaImport(self, data, context, **kwargs): | ||
722 | """Execute the hook code in |data| directly. | ||
723 | |||
724 | Args: | ||
725 | data: The code of the hook to execute. | ||
726 | context: Basic Python context to execute the hook inside. | ||
727 | kwargs: Arbitrary arguments to pass to the hook script. | ||
728 | |||
729 | Raises: | ||
730 | HookError: When the hooks failed for any reason. | ||
731 | """ | ||
732 | # Exec, storing global context in the context dict. We catch exceptions | ||
733 | # and convert to a HookError w/ just the failing traceback. | ||
734 | try: | ||
735 | exec(compile(data, self._script_fullpath, 'exec'), context) | ||
736 | except Exception: | ||
737 | raise HookError('%s\nFailed to import %s hook; see traceback above.' % | ||
738 | (traceback.format_exc(), self._hook_type)) | ||
739 | |||
740 | # Running the script should have defined a main() function. | ||
741 | if 'main' not in context: | ||
742 | raise HookError('Missing main() in: "%s"' % self._script_fullpath) | ||
743 | |||
744 | # Call the main function in the hook. If the hook should cause the | ||
745 | # build to fail, it will raise an Exception. We'll catch that convert | ||
746 | # to a HookError w/ just the failing traceback. | ||
747 | try: | ||
748 | context['main'](**kwargs) | ||
749 | except Exception: | ||
750 | raise HookError('%s\nFailed to run main() for %s hook; see traceback ' | ||
751 | 'above.' % (traceback.format_exc(), self._hook_type)) | ||
752 | |||
753 | def _ExecuteHook(self, **kwargs): | ||
754 | """Actually execute the given hook. | ||
755 | |||
756 | This will run the hook's 'main' function in our python interpreter. | ||
757 | |||
758 | Args: | ||
759 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
760 | to the hook type. For instance, pre-upload hooks will contain | ||
761 | a project_list. | ||
762 | """ | ||
763 | # Keep sys.path and CWD stashed away so that we can always restore them | ||
764 | # upon function exit. | ||
765 | orig_path = os.getcwd() | ||
766 | orig_syspath = sys.path | ||
767 | |||
768 | try: | ||
769 | # Always run hooks with CWD as topdir. | ||
770 | os.chdir(self._topdir) | ||
771 | |||
772 | # Put the hook dir as the first item of sys.path so hooks can do | ||
773 | # relative imports. We want to replace the repo dir as [0] so | ||
774 | # hooks can't import repo files. | ||
775 | sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:] | ||
776 | |||
777 | # Initial global context for the hook to run within. | ||
778 | context = {'__file__': self._script_fullpath} | ||
779 | |||
780 | # Add 'hook_should_take_kwargs' to the arguments to be passed to main. | ||
781 | # We don't actually want hooks to define their main with this argument-- | ||
782 | # it's there to remind them that their hook should always take **kwargs. | ||
783 | # For instance, a pre-upload hook should be defined like: | ||
784 | # def main(project_list, **kwargs): | ||
785 | # | ||
786 | # This allows us to later expand the API without breaking old hooks. | ||
787 | kwargs = kwargs.copy() | ||
788 | kwargs['hook_should_take_kwargs'] = True | ||
789 | |||
790 | # See what version of python the hook has been written against. | ||
791 | data = open(self._script_fullpath).read() | ||
792 | interp = self._ExtractInterpFromShebang(data) | ||
793 | reexec = False | ||
794 | if interp: | ||
795 | prog = os.path.basename(interp) | ||
796 | if prog.startswith('python2') and sys.version_info.major != 2: | ||
797 | reexec = True | ||
798 | elif prog.startswith('python3') and sys.version_info.major == 2: | ||
799 | reexec = True | ||
800 | |||
801 | # Attempt to execute the hooks through the requested version of Python. | ||
802 | if reexec: | ||
803 | try: | ||
804 | self._ExecuteHookViaReexec(interp, context, **kwargs) | ||
805 | except OSError as e: | ||
806 | if e.errno == errno.ENOENT: | ||
807 | # We couldn't find the interpreter, so fallback to importing. | ||
808 | reexec = False | ||
809 | else: | ||
810 | raise | ||
811 | |||
812 | # Run the hook by importing directly. | ||
813 | if not reexec: | ||
814 | self._ExecuteHookViaImport(data, context, **kwargs) | ||
815 | finally: | ||
816 | # Restore sys.path and CWD. | ||
817 | sys.path = orig_syspath | ||
818 | os.chdir(orig_path) | ||
819 | |||
820 | def Run(self, user_allows_all_hooks, **kwargs): | ||
821 | """Run the hook. | ||
822 | |||
823 | If the hook doesn't exist (because there is no hooks project or because | ||
824 | this particular hook is not enabled), this is a no-op. | ||
825 | |||
826 | Args: | ||
827 | user_allows_all_hooks: If True, we will never prompt about running the | ||
828 | hook--we'll just assume it's OK to run it. | ||
829 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
830 | to the hook type. For instance, pre-upload hooks will contain | ||
831 | a project_list. | ||
832 | |||
833 | Raises: | ||
834 | HookError: If there was a problem finding the hook or the user declined | ||
835 | to run a required hook (from _CheckForHookApproval). | ||
836 | """ | ||
837 | # No-op if there is no hooks project or if hook is disabled. | ||
838 | if ((not self._hooks_project) or (self._hook_type not in | ||
839 | self._hooks_project.enabled_repo_hooks)): | ||
840 | return | ||
841 | |||
842 | # Bail with a nice error if we can't find the hook. | ||
843 | if not os.path.isfile(self._script_fullpath): | ||
844 | raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath) | ||
845 | |||
846 | # Make sure the user is OK with running the hook. | ||
847 | if (not user_allows_all_hooks) and (not self._CheckForHookApproval()): | ||
848 | return | ||
849 | |||
850 | # Run the hook with the same version of python we're using. | ||
851 | self._ExecuteHook(**kwargs) | ||
852 | |||
853 | |||
854 | class Project(object): | 452 | class Project(object): |
855 | # These objects can be shared between several working trees. | 453 | # These objects can be shared between several working trees. |
856 | shareable_files = ['description', 'info'] | 454 | shareable_files = ['description', 'info'] |
diff --git a/subcmds/upload.py b/subcmds/upload.py index a886af93..cc6ccf74 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py | |||
@@ -24,7 +24,7 @@ from editor import Editor | |||
24 | from error import HookError, UploadError | 24 | from error import HookError, UploadError |
25 | from git_command import GitCommand | 25 | from git_command import GitCommand |
26 | from git_refs import R_HEADS | 26 | from git_refs import R_HEADS |
27 | from project import RepoHook | 27 | from hooks import RepoHook |
28 | 28 | ||
29 | from pyversion import is_python3 | 29 | from pyversion import is_python3 |
30 | if not is_python3(): | 30 | if not is_python3(): |
diff --git a/tests/test_hooks.py b/tests/test_hooks.py new file mode 100644 index 00000000..ed8268df --- /dev/null +++ b/tests/test_hooks.py | |||
@@ -0,0 +1,60 @@ | |||
1 | # -*- coding:utf-8 -*- | ||
2 | # | ||
3 | # Copyright (C) 2019 The Android Open Source Project | ||
4 | # | ||
5 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
6 | # you may not use this file except in compliance with the License. | ||
7 | # You may obtain a copy of the License at | ||
8 | # | ||
9 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
10 | # | ||
11 | # Unless required by applicable law or agreed to in writing, software | ||
12 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
14 | # See the License for the specific language governing permissions and | ||
15 | # limitations under the License. | ||
16 | |||
17 | """Unittests for the hooks.py module.""" | ||
18 | |||
19 | from __future__ import print_function | ||
20 | |||
21 | import hooks | ||
22 | import unittest | ||
23 | |||
24 | class RepoHookShebang(unittest.TestCase): | ||
25 | """Check shebang parsing in RepoHook.""" | ||
26 | |||
27 | def test_no_shebang(self): | ||
28 | """Lines w/out shebangs should be rejected.""" | ||
29 | DATA = ( | ||
30 | '', | ||
31 | '# -*- coding:utf-8 -*-\n', | ||
32 | '#\n# foo\n', | ||
33 | '# Bad shebang in script\n#!/foo\n' | ||
34 | ) | ||
35 | for data in DATA: | ||
36 | self.assertIsNone(hooks.RepoHook._ExtractInterpFromShebang(data)) | ||
37 | |||
38 | def test_direct_interp(self): | ||
39 | """Lines whose shebang points directly to the interpreter.""" | ||
40 | DATA = ( | ||
41 | ('#!/foo', '/foo'), | ||
42 | ('#! /foo', '/foo'), | ||
43 | ('#!/bin/foo ', '/bin/foo'), | ||
44 | ('#! /usr/foo ', '/usr/foo'), | ||
45 | ('#! /usr/foo -args', '/usr/foo'), | ||
46 | ) | ||
47 | for shebang, interp in DATA: | ||
48 | self.assertEqual(hooks.RepoHook._ExtractInterpFromShebang(shebang), | ||
49 | interp) | ||
50 | |||
51 | def test_env_interp(self): | ||
52 | """Lines whose shebang launches through `env`.""" | ||
53 | DATA = ( | ||
54 | ('#!/usr/bin/env foo', 'foo'), | ||
55 | ('#!/bin/env foo', 'foo'), | ||
56 | ('#! /bin/env /bin/foo ', '/bin/foo'), | ||
57 | ) | ||
58 | for shebang, interp in DATA: | ||
59 | self.assertEqual(hooks.RepoHook._ExtractInterpFromShebang(shebang), | ||
60 | interp) | ||
diff --git a/tests/test_project.py b/tests/test_project.py index 67574cb8..4e710ae5 100644 --- a/tests/test_project.py +++ b/tests/test_project.py | |||
@@ -44,45 +44,6 @@ def TempGitTree(): | |||
44 | platform_utils.rmtree(tempdir) | 44 | platform_utils.rmtree(tempdir) |
45 | 45 | ||
46 | 46 | ||
47 | class RepoHookShebang(unittest.TestCase): | ||
48 | """Check shebang parsing in RepoHook.""" | ||
49 | |||
50 | def test_no_shebang(self): | ||
51 | """Lines w/out shebangs should be rejected.""" | ||
52 | DATA = ( | ||
53 | '', | ||
54 | '# -*- coding:utf-8 -*-\n', | ||
55 | '#\n# foo\n', | ||
56 | '# Bad shebang in script\n#!/foo\n' | ||
57 | ) | ||
58 | for data in DATA: | ||
59 | self.assertIsNone(project.RepoHook._ExtractInterpFromShebang(data)) | ||
60 | |||
61 | def test_direct_interp(self): | ||
62 | """Lines whose shebang points directly to the interpreter.""" | ||
63 | DATA = ( | ||
64 | ('#!/foo', '/foo'), | ||
65 | ('#! /foo', '/foo'), | ||
66 | ('#!/bin/foo ', '/bin/foo'), | ||
67 | ('#! /usr/foo ', '/usr/foo'), | ||
68 | ('#! /usr/foo -args', '/usr/foo'), | ||
69 | ) | ||
70 | for shebang, interp in DATA: | ||
71 | self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang), | ||
72 | interp) | ||
73 | |||
74 | def test_env_interp(self): | ||
75 | """Lines whose shebang launches through `env`.""" | ||
76 | DATA = ( | ||
77 | ('#!/usr/bin/env foo', 'foo'), | ||
78 | ('#!/bin/env foo', 'foo'), | ||
79 | ('#! /bin/env /bin/foo ', '/bin/foo'), | ||
80 | ) | ||
81 | for shebang, interp in DATA: | ||
82 | self.assertEqual(project.RepoHook._ExtractInterpFromShebang(shebang), | ||
83 | interp) | ||
84 | |||
85 | |||
86 | class FakeProject(object): | 47 | class FakeProject(object): |
87 | """A fake for Project for basic functionality.""" | 48 | """A fake for Project for basic functionality.""" |
88 | 49 | ||