diff options
Diffstat (limited to 'project.py')
| -rw-r--r-- | project.py | 404 |
1 files changed, 1 insertions, 403 deletions
| @@ -18,7 +18,6 @@ from __future__ import print_function | |||
| 18 | import errno | 18 | import errno |
| 19 | import filecmp | 19 | import filecmp |
| 20 | import glob | 20 | import glob |
| 21 | import json | ||
| 22 | import os | 21 | import os |
| 23 | import random | 22 | import random |
| 24 | import re | 23 | import re |
| @@ -29,13 +28,12 @@ import sys | |||
| 29 | import tarfile | 28 | import tarfile |
| 30 | import tempfile | 29 | import tempfile |
| 31 | import time | 30 | import time |
| 32 | import traceback | ||
| 33 | 31 | ||
| 34 | from color import Coloring | 32 | from color import Coloring |
| 35 | from git_command import GitCommand, git_require | 33 | from git_command import GitCommand, git_require |
| 36 | from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ | 34 | from git_config import GitConfig, IsId, GetSchemeFromUrl, GetUrlCookieFile, \ |
| 37 | ID_RE | 35 | ID_RE |
| 38 | from error import GitError, HookError, UploadError, DownloadError | 36 | from error import GitError, UploadError, DownloadError |
| 39 | from error import ManifestInvalidRevisionError, ManifestInvalidPathError | 37 | from error import ManifestInvalidRevisionError, ManifestInvalidPathError |
| 40 | from error import NoManifestException | 38 | from error import NoManifestException |
| 41 | import platform_utils | 39 | import platform_utils |
| @@ -451,406 +449,6 @@ class RemoteSpec(object): | |||
| 451 | self.orig_name = orig_name | 449 | self.orig_name = orig_name |
| 452 | self.fetchUrl = fetchUrl | 450 | self.fetchUrl = fetchUrl |
| 453 | 451 | ||
| 454 | |||
| 455 | class RepoHook(object): | ||
| 456 | |||
| 457 | """A RepoHook contains information about a script to run as a hook. | ||
| 458 | |||
| 459 | Hooks are used to run a python script before running an upload (for instance, | ||
| 460 | to run presubmit checks). Eventually, we may have hooks for other actions. | ||
| 461 | |||
| 462 | This shouldn't be confused with files in the 'repo/hooks' directory. Those | ||
| 463 | files are copied into each '.git/hooks' folder for each project. Repo-level | ||
| 464 | hooks are associated instead with repo actions. | ||
| 465 | |||
| 466 | Hooks are always python. When a hook is run, we will load the hook into the | ||
| 467 | interpreter and execute its main() function. | ||
| 468 | """ | ||
| 469 | |||
| 470 | def __init__(self, | ||
| 471 | hook_type, | ||
| 472 | hooks_project, | ||
| 473 | topdir, | ||
| 474 | manifest_url, | ||
| 475 | abort_if_user_denies=False): | ||
| 476 | """RepoHook constructor. | ||
| 477 | |||
| 478 | Params: | ||
| 479 | hook_type: A string representing the type of hook. This is also used | ||
| 480 | to figure out the name of the file containing the hook. For | ||
| 481 | example: 'pre-upload'. | ||
| 482 | hooks_project: The project containing the repo hooks. If you have a | ||
| 483 | manifest, this is manifest.repo_hooks_project. OK if this is None, | ||
| 484 | which will make the hook a no-op. | ||
| 485 | topdir: Repo's top directory (the one containing the .repo directory). | ||
| 486 | Scripts will run with CWD as this directory. If you have a manifest, | ||
| 487 | this is manifest.topdir | ||
| 488 | manifest_url: The URL to the manifest git repo. | ||
| 489 | abort_if_user_denies: If True, we'll throw a HookError() if the user | ||
| 490 | doesn't allow us to run the hook. | ||
| 491 | """ | ||
| 492 | self._hook_type = hook_type | ||
| 493 | self._hooks_project = hooks_project | ||
| 494 | self._manifest_url = manifest_url | ||
| 495 | self._topdir = topdir | ||
| 496 | self._abort_if_user_denies = abort_if_user_denies | ||
| 497 | |||
| 498 | # Store the full path to the script for convenience. | ||
| 499 | if self._hooks_project: | ||
| 500 | self._script_fullpath = os.path.join(self._hooks_project.worktree, | ||
| 501 | self._hook_type + '.py') | ||
| 502 | else: | ||
| 503 | self._script_fullpath = None | ||
| 504 | |||
| 505 | def _GetHash(self): | ||
| 506 | """Return a hash of the contents of the hooks directory. | ||
| 507 | |||
| 508 | We'll just use git to do this. This hash has the property that if anything | ||
| 509 | changes in the directory we will return a different has. | ||
| 510 | |||
| 511 | SECURITY CONSIDERATION: | ||
| 512 | This hash only represents the contents of files in the hook directory, not | ||
| 513 | any other files imported or called by hooks. Changes to imported files | ||
| 514 | can change the script behavior without affecting the hash. | ||
| 515 | |||
| 516 | Returns: | ||
| 517 | A string representing the hash. This will always be ASCII so that it can | ||
| 518 | be printed to the user easily. | ||
| 519 | """ | ||
| 520 | assert self._hooks_project, "Must have hooks to calculate their hash." | ||
| 521 | |||
| 522 | # We will use the work_git object rather than just calling GetRevisionId(). | ||
| 523 | # That gives us a hash of the latest checked in version of the files that | ||
| 524 | # the user will actually be executing. Specifically, GetRevisionId() | ||
| 525 | # doesn't appear to change even if a user checks out a different version | ||
| 526 | # of the hooks repo (via git checkout) nor if a user commits their own revs. | ||
| 527 | # | ||
| 528 | # NOTE: Local (non-committed) changes will not be factored into this hash. | ||
| 529 | # I think this is OK, since we're really only worried about warning the user | ||
| 530 | # about upstream changes. | ||
| 531 | return self._hooks_project.work_git.rev_parse('HEAD') | ||
| 532 | |||
| 533 | def _GetMustVerb(self): | ||
| 534 | """Return 'must' if the hook is required; 'should' if not.""" | ||
| 535 | if self._abort_if_user_denies: | ||
| 536 | return 'must' | ||
| 537 | else: | ||
| 538 | return 'should' | ||
| 539 | |||
| 540 | def _CheckForHookApproval(self): | ||
| 541 | """Check to see whether this hook has been approved. | ||
| 542 | |||
| 543 | We'll accept approval of manifest URLs if they're using secure transports. | ||
| 544 | This way the user can say they trust the manifest hoster. For insecure | ||
| 545 | hosts, we fall back to checking the hash of the hooks repo. | ||
| 546 | |||
| 547 | Note that we ask permission for each individual hook even though we use | ||
| 548 | the hash of all hooks when detecting changes. We'd like the user to be | ||
| 549 | able to approve / deny each hook individually. We only use the hash of all | ||
| 550 | hooks because there is no other easy way to detect changes to local imports. | ||
| 551 | |||
| 552 | Returns: | ||
| 553 | True if this hook is approved to run; False otherwise. | ||
| 554 | |||
| 555 | Raises: | ||
| 556 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
| 557 | was passed to the consturctor. | ||
| 558 | """ | ||
| 559 | if self._ManifestUrlHasSecureScheme(): | ||
| 560 | return self._CheckForHookApprovalManifest() | ||
| 561 | else: | ||
| 562 | return self._CheckForHookApprovalHash() | ||
| 563 | |||
| 564 | def _CheckForHookApprovalHelper(self, subkey, new_val, main_prompt, | ||
| 565 | changed_prompt): | ||
| 566 | """Check for approval for a particular attribute and hook. | ||
| 567 | |||
| 568 | Args: | ||
| 569 | subkey: The git config key under [repo.hooks.<hook_type>] to store the | ||
| 570 | last approved string. | ||
| 571 | new_val: The new value to compare against the last approved one. | ||
| 572 | main_prompt: Message to display to the user to ask for approval. | ||
| 573 | changed_prompt: Message explaining why we're re-asking for approval. | ||
| 574 | |||
| 575 | Returns: | ||
| 576 | True if this hook is approved to run; False otherwise. | ||
| 577 | |||
| 578 | Raises: | ||
| 579 | HookError: Raised if the user doesn't approve and abort_if_user_denies | ||
| 580 | was passed to the consturctor. | ||
| 581 | """ | ||
| 582 | hooks_config = self._hooks_project.config | ||
| 583 | git_approval_key = 'repo.hooks.%s.%s' % (self._hook_type, subkey) | ||
| 584 | |||
| 585 | # Get the last value that the user approved for this hook; may be None. | ||
| 586 | old_val = hooks_config.GetString(git_approval_key) | ||
| 587 | |||
| 588 | if old_val is not None: | ||
| 589 | # User previously approved hook and asked not to be prompted again. | ||
| 590 | if new_val == old_val: | ||
| 591 | # Approval matched. We're done. | ||
| 592 | return True | ||
| 593 | else: | ||
| 594 | # Give the user a reason why we're prompting, since they last told | ||
| 595 | # us to "never ask again". | ||
| 596 | prompt = 'WARNING: %s\n\n' % (changed_prompt,) | ||
| 597 | else: | ||
| 598 | prompt = '' | ||
| 599 | |||
| 600 | # Prompt the user if we're not on a tty; on a tty we'll assume "no". | ||
| 601 | if sys.stdout.isatty(): | ||
| 602 | prompt += main_prompt + ' (yes/always/NO)? ' | ||
| 603 | response = input(prompt).lower() | ||
| 604 | print() | ||
| 605 | |||
| 606 | # User is doing a one-time approval. | ||
| 607 | if response in ('y', 'yes'): | ||
| 608 | return True | ||
| 609 | elif response == 'always': | ||
| 610 | hooks_config.SetString(git_approval_key, new_val) | ||
| 611 | return True | ||
| 612 | |||
| 613 | # For anything else, we'll assume no approval. | ||
| 614 | if self._abort_if_user_denies: | ||
| 615 | raise HookError('You must allow the %s hook or use --no-verify.' % | ||
| 616 | self._hook_type) | ||
| 617 | |||
| 618 | return False | ||
| 619 | |||
| 620 | def _ManifestUrlHasSecureScheme(self): | ||
| 621 | """Check if the URI for the manifest is a secure transport.""" | ||
| 622 | secure_schemes = ('file', 'https', 'ssh', 'persistent-https', 'sso', 'rpc') | ||
| 623 | parse_results = urllib.parse.urlparse(self._manifest_url) | ||
| 624 | return parse_results.scheme in secure_schemes | ||
| 625 | |||
| 626 | def _CheckForHookApprovalManifest(self): | ||
| 627 | """Check whether the user has approved this manifest host. | ||
| 628 | |||
| 629 | Returns: | ||
| 630 | True if this hook is approved to run; False otherwise. | ||
| 631 | """ | ||
| 632 | return self._CheckForHookApprovalHelper( | ||
| 633 | 'approvedmanifest', | ||
| 634 | self._manifest_url, | ||
| 635 | 'Run hook scripts from %s' % (self._manifest_url,), | ||
| 636 | 'Manifest URL has changed since %s was allowed.' % (self._hook_type,)) | ||
| 637 | |||
| 638 | def _CheckForHookApprovalHash(self): | ||
| 639 | """Check whether the user has approved the hooks repo. | ||
| 640 | |||
| 641 | Returns: | ||
| 642 | True if this hook is approved to run; False otherwise. | ||
| 643 | """ | ||
| 644 | prompt = ('Repo %s run the script:\n' | ||
| 645 | ' %s\n' | ||
| 646 | '\n' | ||
| 647 | 'Do you want to allow this script to run') | ||
| 648 | return self._CheckForHookApprovalHelper( | ||
| 649 | 'approvedhash', | ||
| 650 | self._GetHash(), | ||
| 651 | prompt % (self._GetMustVerb(), self._script_fullpath), | ||
| 652 | 'Scripts have changed since %s was allowed.' % (self._hook_type,)) | ||
| 653 | |||
| 654 | @staticmethod | ||
| 655 | def _ExtractInterpFromShebang(data): | ||
| 656 | """Extract the interpreter used in the shebang. | ||
| 657 | |||
| 658 | Try to locate the interpreter the script is using (ignoring `env`). | ||
| 659 | |||
| 660 | Args: | ||
| 661 | data: The file content of the script. | ||
| 662 | |||
| 663 | Returns: | ||
| 664 | The basename of the main script interpreter, or None if a shebang is not | ||
| 665 | used or could not be parsed out. | ||
| 666 | """ | ||
| 667 | firstline = data.splitlines()[:1] | ||
| 668 | if not firstline: | ||
| 669 | return None | ||
| 670 | |||
| 671 | # The format here can be tricky. | ||
| 672 | shebang = firstline[0].strip() | ||
| 673 | m = re.match(r'^#!\s*([^\s]+)(?:\s+([^\s]+))?', shebang) | ||
| 674 | if not m: | ||
| 675 | return None | ||
| 676 | |||
| 677 | # If the using `env`, find the target program. | ||
| 678 | interp = m.group(1) | ||
| 679 | if os.path.basename(interp) == 'env': | ||
| 680 | interp = m.group(2) | ||
| 681 | |||
| 682 | return interp | ||
| 683 | |||
| 684 | def _ExecuteHookViaReexec(self, interp, context, **kwargs): | ||
| 685 | """Execute the hook script through |interp|. | ||
| 686 | |||
| 687 | Note: Support for this feature should be dropped ~Jun 2021. | ||
| 688 | |||
| 689 | Args: | ||
| 690 | interp: The Python program to run. | ||
| 691 | context: Basic Python context to execute the hook inside. | ||
| 692 | kwargs: Arbitrary arguments to pass to the hook script. | ||
| 693 | |||
| 694 | Raises: | ||
| 695 | HookError: When the hooks failed for any reason. | ||
| 696 | """ | ||
| 697 | # This logic needs to be kept in sync with _ExecuteHookViaImport below. | ||
| 698 | script = """ | ||
| 699 | import json, os, sys | ||
| 700 | path = '''%(path)s''' | ||
| 701 | kwargs = json.loads('''%(kwargs)s''') | ||
| 702 | context = json.loads('''%(context)s''') | ||
| 703 | sys.path.insert(0, os.path.dirname(path)) | ||
| 704 | data = open(path).read() | ||
| 705 | exec(compile(data, path, 'exec'), context) | ||
| 706 | context['main'](**kwargs) | ||
| 707 | """ % { | ||
| 708 | 'path': self._script_fullpath, | ||
| 709 | 'kwargs': json.dumps(kwargs), | ||
| 710 | 'context': json.dumps(context), | ||
| 711 | } | ||
| 712 | |||
| 713 | # We pass the script via stdin to avoid OS argv limits. It also makes | ||
| 714 | # unhandled exception tracebacks less verbose/confusing for users. | ||
| 715 | cmd = [interp, '-c', 'import sys; exec(sys.stdin.read())'] | ||
| 716 | proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) | ||
| 717 | proc.communicate(input=script.encode('utf-8')) | ||
| 718 | if proc.returncode: | ||
| 719 | raise HookError('Failed to run %s hook.' % (self._hook_type,)) | ||
| 720 | |||
| 721 | def _ExecuteHookViaImport(self, data, context, **kwargs): | ||
| 722 | """Execute the hook code in |data| directly. | ||
| 723 | |||
| 724 | Args: | ||
| 725 | data: The code of the hook to execute. | ||
| 726 | context: Basic Python context to execute the hook inside. | ||
| 727 | kwargs: Arbitrary arguments to pass to the hook script. | ||
| 728 | |||
| 729 | Raises: | ||
| 730 | HookError: When the hooks failed for any reason. | ||
| 731 | """ | ||
| 732 | # Exec, storing global context in the context dict. We catch exceptions | ||
| 733 | # and convert to a HookError w/ just the failing traceback. | ||
| 734 | try: | ||
| 735 | exec(compile(data, self._script_fullpath, 'exec'), context) | ||
| 736 | except Exception: | ||
| 737 | raise HookError('%s\nFailed to import %s hook; see traceback above.' % | ||
| 738 | (traceback.format_exc(), self._hook_type)) | ||
| 739 | |||
| 740 | # Running the script should have defined a main() function. | ||
| 741 | if 'main' not in context: | ||
| 742 | raise HookError('Missing main() in: "%s"' % self._script_fullpath) | ||
| 743 | |||
| 744 | # Call the main function in the hook. If the hook should cause the | ||
| 745 | # build to fail, it will raise an Exception. We'll catch that convert | ||
| 746 | # to a HookError w/ just the failing traceback. | ||
| 747 | try: | ||
| 748 | context['main'](**kwargs) | ||
| 749 | except Exception: | ||
| 750 | raise HookError('%s\nFailed to run main() for %s hook; see traceback ' | ||
| 751 | 'above.' % (traceback.format_exc(), self._hook_type)) | ||
| 752 | |||
| 753 | def _ExecuteHook(self, **kwargs): | ||
| 754 | """Actually execute the given hook. | ||
| 755 | |||
| 756 | This will run the hook's 'main' function in our python interpreter. | ||
| 757 | |||
| 758 | Args: | ||
| 759 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
| 760 | to the hook type. For instance, pre-upload hooks will contain | ||
| 761 | a project_list. | ||
| 762 | """ | ||
| 763 | # Keep sys.path and CWD stashed away so that we can always restore them | ||
| 764 | # upon function exit. | ||
| 765 | orig_path = os.getcwd() | ||
| 766 | orig_syspath = sys.path | ||
| 767 | |||
| 768 | try: | ||
| 769 | # Always run hooks with CWD as topdir. | ||
| 770 | os.chdir(self._topdir) | ||
| 771 | |||
| 772 | # Put the hook dir as the first item of sys.path so hooks can do | ||
| 773 | # relative imports. We want to replace the repo dir as [0] so | ||
| 774 | # hooks can't import repo files. | ||
| 775 | sys.path = [os.path.dirname(self._script_fullpath)] + sys.path[1:] | ||
| 776 | |||
| 777 | # Initial global context for the hook to run within. | ||
| 778 | context = {'__file__': self._script_fullpath} | ||
| 779 | |||
| 780 | # Add 'hook_should_take_kwargs' to the arguments to be passed to main. | ||
| 781 | # We don't actually want hooks to define their main with this argument-- | ||
| 782 | # it's there to remind them that their hook should always take **kwargs. | ||
| 783 | # For instance, a pre-upload hook should be defined like: | ||
| 784 | # def main(project_list, **kwargs): | ||
| 785 | # | ||
| 786 | # This allows us to later expand the API without breaking old hooks. | ||
| 787 | kwargs = kwargs.copy() | ||
| 788 | kwargs['hook_should_take_kwargs'] = True | ||
| 789 | |||
| 790 | # See what version of python the hook has been written against. | ||
| 791 | data = open(self._script_fullpath).read() | ||
| 792 | interp = self._ExtractInterpFromShebang(data) | ||
| 793 | reexec = False | ||
| 794 | if interp: | ||
| 795 | prog = os.path.basename(interp) | ||
| 796 | if prog.startswith('python2') and sys.version_info.major != 2: | ||
| 797 | reexec = True | ||
| 798 | elif prog.startswith('python3') and sys.version_info.major == 2: | ||
| 799 | reexec = True | ||
| 800 | |||
| 801 | # Attempt to execute the hooks through the requested version of Python. | ||
| 802 | if reexec: | ||
| 803 | try: | ||
| 804 | self._ExecuteHookViaReexec(interp, context, **kwargs) | ||
| 805 | except OSError as e: | ||
| 806 | if e.errno == errno.ENOENT: | ||
| 807 | # We couldn't find the interpreter, so fallback to importing. | ||
| 808 | reexec = False | ||
| 809 | else: | ||
| 810 | raise | ||
| 811 | |||
| 812 | # Run the hook by importing directly. | ||
| 813 | if not reexec: | ||
| 814 | self._ExecuteHookViaImport(data, context, **kwargs) | ||
| 815 | finally: | ||
| 816 | # Restore sys.path and CWD. | ||
| 817 | sys.path = orig_syspath | ||
| 818 | os.chdir(orig_path) | ||
| 819 | |||
| 820 | def Run(self, user_allows_all_hooks, **kwargs): | ||
| 821 | """Run the hook. | ||
| 822 | |||
| 823 | If the hook doesn't exist (because there is no hooks project or because | ||
| 824 | this particular hook is not enabled), this is a no-op. | ||
| 825 | |||
| 826 | Args: | ||
| 827 | user_allows_all_hooks: If True, we will never prompt about running the | ||
| 828 | hook--we'll just assume it's OK to run it. | ||
| 829 | kwargs: Keyword arguments to pass to the hook. These are often specific | ||
| 830 | to the hook type. For instance, pre-upload hooks will contain | ||
| 831 | a project_list. | ||
| 832 | |||
| 833 | Raises: | ||
| 834 | HookError: If there was a problem finding the hook or the user declined | ||
| 835 | to run a required hook (from _CheckForHookApproval). | ||
| 836 | """ | ||
| 837 | # No-op if there is no hooks project or if hook is disabled. | ||
| 838 | if ((not self._hooks_project) or (self._hook_type not in | ||
| 839 | self._hooks_project.enabled_repo_hooks)): | ||
| 840 | return | ||
| 841 | |||
| 842 | # Bail with a nice error if we can't find the hook. | ||
| 843 | if not os.path.isfile(self._script_fullpath): | ||
| 844 | raise HookError('Couldn\'t find repo hook: "%s"' % self._script_fullpath) | ||
| 845 | |||
| 846 | # Make sure the user is OK with running the hook. | ||
| 847 | if (not user_allows_all_hooks) and (not self._CheckForHookApproval()): | ||
| 848 | return | ||
| 849 | |||
| 850 | # Run the hook with the same version of python we're using. | ||
| 851 | self._ExecuteHook(**kwargs) | ||
| 852 | |||
| 853 | |||
| 854 | class Project(object): | 452 | class Project(object): |
| 855 | # These objects can be shared between several working trees. | 453 | # These objects can be shared between several working trees. |
| 856 | shareable_files = ['description', 'info'] | 454 | shareable_files = ['description', 'info'] |
