diff options
-rw-r--r-- | meta/lib/oeqa/utils/gitarchive.py | 173 | ||||
-rwxr-xr-x | scripts/oe-git-archive | 166 |
2 files changed, 179 insertions, 160 deletions
diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py new file mode 100644 index 0000000000..fddd608593 --- /dev/null +++ b/meta/lib/oeqa/utils/gitarchive.py | |||
@@ -0,0 +1,173 @@ | |||
1 | # | ||
2 | # Helper functions for committing data to git and pushing upstream | ||
3 | # | ||
4 | # Copyright (c) 2017, Intel Corporation. | ||
5 | # Copyright (c) 2019, Linux Foundation | ||
6 | # | ||
7 | # This program is free software; you can redistribute it and/or modify it | ||
8 | # under the terms and conditions of the GNU General Public License, | ||
9 | # version 2, as published by the Free Software Foundation. | ||
10 | # | ||
11 | # This program is distributed in the hope it will be useful, but WITHOUT | ||
12 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
13 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
14 | # more details. | ||
15 | # | ||
16 | |||
17 | import os | ||
18 | import re | ||
19 | import sys | ||
20 | from oeqa.utils.git import GitRepo, GitError | ||
21 | |||
22 | class ArchiveError(Exception): | ||
23 | """Internal error handling of this script""" | ||
24 | |||
25 | def format_str(string, fields): | ||
26 | """Format string using the given fields (dict)""" | ||
27 | try: | ||
28 | return string.format(**fields) | ||
29 | except KeyError as err: | ||
30 | raise ArchiveError("Unable to expand string '{}': unknown field {} " | ||
31 | "(valid fields are: {})".format( | ||
32 | string, err, ', '.join(sorted(fields.keys())))) | ||
33 | |||
34 | |||
35 | def init_git_repo(path, no_create, bare, log): | ||
36 | """Initialize local Git repository""" | ||
37 | path = os.path.abspath(path) | ||
38 | if os.path.isfile(path): | ||
39 | raise ArchiveError("Invalid Git repo at {}: path exists but is not a " | ||
40 | "directory".format(path)) | ||
41 | if not os.path.isdir(path) or not os.listdir(path): | ||
42 | if no_create: | ||
43 | raise ArchiveError("No git repo at {}, refusing to create " | ||
44 | "one".format(path)) | ||
45 | if not os.path.isdir(path): | ||
46 | try: | ||
47 | os.mkdir(path) | ||
48 | except (FileNotFoundError, PermissionError) as err: | ||
49 | raise ArchiveError("Failed to mkdir {}: {}".format(path, err)) | ||
50 | if not os.listdir(path): | ||
51 | log.info("Initializing a new Git repo at %s", path) | ||
52 | repo = GitRepo.init(path, bare) | ||
53 | try: | ||
54 | repo = GitRepo(path, is_topdir=True) | ||
55 | except GitError: | ||
56 | raise ArchiveError("Non-empty directory that is not a Git repository " | ||
57 | "at {}\nPlease specify an existing Git repository, " | ||
58 | "an empty directory or a non-existing directory " | ||
59 | "path.".format(path)) | ||
60 | return repo | ||
61 | |||
62 | |||
63 | def git_commit_data(repo, data_dir, branch, message, exclude, notes, log): | ||
64 | """Commit data into a Git repository""" | ||
65 | log.info("Committing data into to branch %s", branch) | ||
66 | tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive') | ||
67 | try: | ||
68 | # Create new tree object from the data | ||
69 | env_update = {'GIT_INDEX_FILE': tmp_index, | ||
70 | 'GIT_WORK_TREE': os.path.abspath(data_dir)} | ||
71 | repo.run_cmd('add .', env_update) | ||
72 | |||
73 | # Remove files that are excluded | ||
74 | if exclude: | ||
75 | repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update) | ||
76 | |||
77 | tree = repo.run_cmd('write-tree', env_update) | ||
78 | |||
79 | # Create new commit object from the tree | ||
80 | parent = repo.rev_parse(branch) | ||
81 | git_cmd = ['commit-tree', tree, '-m', message] | ||
82 | if parent: | ||
83 | git_cmd += ['-p', parent] | ||
84 | commit = repo.run_cmd(git_cmd, env_update) | ||
85 | |||
86 | # Create git notes | ||
87 | for ref, filename in notes: | ||
88 | ref = ref.format(branch_name=branch) | ||
89 | repo.run_cmd(['notes', '--ref', ref, 'add', | ||
90 | '-F', os.path.abspath(filename), commit]) | ||
91 | |||
92 | # Update branch head | ||
93 | git_cmd = ['update-ref', 'refs/heads/' + branch, commit] | ||
94 | if parent: | ||
95 | git_cmd.append(parent) | ||
96 | repo.run_cmd(git_cmd) | ||
97 | |||
98 | # Update current HEAD, if we're on branch 'branch' | ||
99 | if not repo.bare and repo.get_current_branch() == branch: | ||
100 | log.info("Updating %s HEAD to latest commit", repo.top_dir) | ||
101 | repo.run_cmd('reset --hard') | ||
102 | |||
103 | return commit | ||
104 | finally: | ||
105 | if os.path.exists(tmp_index): | ||
106 | os.unlink(tmp_index) | ||
107 | |||
108 | |||
109 | def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, | ||
110 | keywords): | ||
111 | """Generate tag name and message, with support for running id number""" | ||
112 | keyws = keywords.copy() | ||
113 | # Tag number is handled specially: if not defined, we autoincrement it | ||
114 | if 'tag_number' not in keyws: | ||
115 | # Fill in all other fields than 'tag_number' | ||
116 | keyws['tag_number'] = '{tag_number}' | ||
117 | tag_re = format_str(name_pattern, keyws) | ||
118 | # Replace parentheses for proper regex matching | ||
119 | tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$' | ||
120 | # Inject regex group pattern for 'tag_number' | ||
121 | tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') | ||
122 | |||
123 | keyws['tag_number'] = 0 | ||
124 | for existing_tag in repo.run_cmd('tag').splitlines(): | ||
125 | match = re.match(tag_re, existing_tag) | ||
126 | |||
127 | if match and int(match.group('tag_number')) >= keyws['tag_number']: | ||
128 | keyws['tag_number'] = int(match.group('tag_number')) + 1 | ||
129 | |||
130 | tag_name = format_str(name_pattern, keyws) | ||
131 | msg_subj= format_str(msg_subj_pattern.strip(), keyws) | ||
132 | msg_body = format_str(msg_body_pattern, keyws) | ||
133 | return tag_name, msg_subj + '\n\n' + msg_body | ||
134 | |||
135 | def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_msg_body, branch_name, no_tag, tagname, tag_msg_subject, tag_msg_body, exclude, notes, push, keywords, log): | ||
136 | |||
137 | if not os.path.isdir(data_dir): | ||
138 | raise ArchiveError("Not a directory: {}".format(data_dir)) | ||
139 | |||
140 | data_repo = init_git_repo(git_dir, no_create, bare, log) | ||
141 | |||
142 | # Expand strings early in order to avoid getting into inconsistent | ||
143 | # state (e.g. no tag even if data was committed) | ||
144 | commit_msg = format_str(commit_msg_subject.strip(), keywords) | ||
145 | commit_msg += '\n\n' + format_str(commit_msg_body, keywords) | ||
146 | branch_name = format_str(branch_name, keywords) | ||
147 | tag_name = None | ||
148 | if not no_tag and tagname: | ||
149 | tag_name, tag_msg = expand_tag_strings(data_repo, tagname, | ||
150 | tag_msg_subject, | ||
151 | tag_msg_body, keywords) | ||
152 | |||
153 | # Commit data | ||
154 | commit = git_commit_data(data_repo, data_dir, branch_name, | ||
155 | commit_msg, exclude, notes, log) | ||
156 | |||
157 | # Create tag | ||
158 | if tag_name: | ||
159 | log.info("Creating tag %s", tag_name) | ||
160 | data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit]) | ||
161 | |||
162 | # Push data to remote | ||
163 | if push: | ||
164 | cmd = ['push', '--tags'] | ||
165 | # If no remote is given we push with the default settings from | ||
166 | # gitconfig | ||
167 | if push is not True: | ||
168 | notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name) | ||
169 | for ref, _ in notes] | ||
170 | cmd.extend([push, branch_name] + notes_refs) | ||
171 | log.info("Pushing data to remote") | ||
172 | data_repo.run_cmd(cmd) | ||
173 | |||
diff --git a/scripts/oe-git-archive b/scripts/oe-git-archive index 913291a99c..ab1c2b9ad4 100755 --- a/scripts/oe-git-archive +++ b/scripts/oe-git-archive | |||
@@ -14,16 +14,10 @@ | |||
14 | # more details. | 14 | # more details. |
15 | # | 15 | # |
16 | import argparse | 16 | import argparse |
17 | import glob | ||
18 | import json | ||
19 | import logging | 17 | import logging |
20 | import math | ||
21 | import os | 18 | import os |
22 | import re | 19 | import re |
23 | import sys | 20 | import sys |
24 | from collections import namedtuple, OrderedDict | ||
25 | from datetime import datetime, timedelta, tzinfo | ||
26 | from operator import attrgetter | ||
27 | 21 | ||
28 | # Import oe and bitbake libs | 22 | # Import oe and bitbake libs |
29 | scripts_path = os.path.dirname(os.path.realpath(__file__)) | 23 | scripts_path = os.path.dirname(os.path.realpath(__file__)) |
@@ -34,128 +28,13 @@ scriptpath.add_oe_lib_path() | |||
34 | 28 | ||
35 | from oeqa.utils.git import GitRepo, GitError | 29 | from oeqa.utils.git import GitRepo, GitError |
36 | from oeqa.utils.metadata import metadata_from_bb | 30 | from oeqa.utils.metadata import metadata_from_bb |
37 | 31 | import oeqa.utils.gitarchive as gitarchive | |
38 | 32 | ||
39 | # Setup logging | 33 | # Setup logging |
40 | logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") | 34 | logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") |
41 | log = logging.getLogger() | 35 | log = logging.getLogger() |
42 | 36 | ||
43 | 37 | ||
44 | class ArchiveError(Exception): | ||
45 | """Internal error handling of this script""" | ||
46 | |||
47 | |||
48 | def format_str(string, fields): | ||
49 | """Format string using the given fields (dict)""" | ||
50 | try: | ||
51 | return string.format(**fields) | ||
52 | except KeyError as err: | ||
53 | raise ArchiveError("Unable to expand string '{}': unknown field {} " | ||
54 | "(valid fields are: {})".format( | ||
55 | string, err, ', '.join(sorted(fields.keys())))) | ||
56 | |||
57 | |||
58 | def init_git_repo(path, no_create, bare): | ||
59 | """Initialize local Git repository""" | ||
60 | path = os.path.abspath(path) | ||
61 | if os.path.isfile(path): | ||
62 | raise ArchiveError("Invalid Git repo at {}: path exists but is not a " | ||
63 | "directory".format(path)) | ||
64 | if not os.path.isdir(path) or not os.listdir(path): | ||
65 | if no_create: | ||
66 | raise ArchiveError("No git repo at {}, refusing to create " | ||
67 | "one".format(path)) | ||
68 | if not os.path.isdir(path): | ||
69 | try: | ||
70 | os.mkdir(path) | ||
71 | except (FileNotFoundError, PermissionError) as err: | ||
72 | raise ArchiveError("Failed to mkdir {}: {}".format(path, err)) | ||
73 | if not os.listdir(path): | ||
74 | log.info("Initializing a new Git repo at %s", path) | ||
75 | repo = GitRepo.init(path, bare) | ||
76 | try: | ||
77 | repo = GitRepo(path, is_topdir=True) | ||
78 | except GitError: | ||
79 | raise ArchiveError("Non-empty directory that is not a Git repository " | ||
80 | "at {}\nPlease specify an existing Git repository, " | ||
81 | "an empty directory or a non-existing directory " | ||
82 | "path.".format(path)) | ||
83 | return repo | ||
84 | |||
85 | |||
86 | def git_commit_data(repo, data_dir, branch, message, exclude, notes): | ||
87 | """Commit data into a Git repository""" | ||
88 | log.info("Committing data into to branch %s", branch) | ||
89 | tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive') | ||
90 | try: | ||
91 | # Create new tree object from the data | ||
92 | env_update = {'GIT_INDEX_FILE': tmp_index, | ||
93 | 'GIT_WORK_TREE': os.path.abspath(data_dir)} | ||
94 | repo.run_cmd('add .', env_update) | ||
95 | |||
96 | # Remove files that are excluded | ||
97 | if exclude: | ||
98 | repo.run_cmd(['rm', '--cached'] + [f for f in exclude], env_update) | ||
99 | |||
100 | tree = repo.run_cmd('write-tree', env_update) | ||
101 | |||
102 | # Create new commit object from the tree | ||
103 | parent = repo.rev_parse(branch) | ||
104 | git_cmd = ['commit-tree', tree, '-m', message] | ||
105 | if parent: | ||
106 | git_cmd += ['-p', parent] | ||
107 | commit = repo.run_cmd(git_cmd, env_update) | ||
108 | |||
109 | # Create git notes | ||
110 | for ref, filename in notes: | ||
111 | ref = ref.format(branch_name=branch) | ||
112 | repo.run_cmd(['notes', '--ref', ref, 'add', | ||
113 | '-F', os.path.abspath(filename), commit]) | ||
114 | |||
115 | # Update branch head | ||
116 | git_cmd = ['update-ref', 'refs/heads/' + branch, commit] | ||
117 | if parent: | ||
118 | git_cmd.append(parent) | ||
119 | repo.run_cmd(git_cmd) | ||
120 | |||
121 | # Update current HEAD, if we're on branch 'branch' | ||
122 | if not repo.bare and repo.get_current_branch() == branch: | ||
123 | log.info("Updating %s HEAD to latest commit", repo.top_dir) | ||
124 | repo.run_cmd('reset --hard') | ||
125 | |||
126 | return commit | ||
127 | finally: | ||
128 | if os.path.exists(tmp_index): | ||
129 | os.unlink(tmp_index) | ||
130 | |||
131 | |||
132 | def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern, | ||
133 | keywords): | ||
134 | """Generate tag name and message, with support for running id number""" | ||
135 | keyws = keywords.copy() | ||
136 | # Tag number is handled specially: if not defined, we autoincrement it | ||
137 | if 'tag_number' not in keyws: | ||
138 | # Fill in all other fields than 'tag_number' | ||
139 | keyws['tag_number'] = '{tag_number}' | ||
140 | tag_re = format_str(name_pattern, keyws) | ||
141 | # Replace parentheses for proper regex matching | ||
142 | tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$' | ||
143 | # Inject regex group pattern for 'tag_number' | ||
144 | tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})') | ||
145 | |||
146 | keyws['tag_number'] = 0 | ||
147 | for existing_tag in repo.run_cmd('tag').splitlines(): | ||
148 | match = re.match(tag_re, existing_tag) | ||
149 | |||
150 | if match and int(match.group('tag_number')) >= keyws['tag_number']: | ||
151 | keyws['tag_number'] = int(match.group('tag_number')) + 1 | ||
152 | |||
153 | tag_name = format_str(name_pattern, keyws) | ||
154 | msg_subj= format_str(msg_subj_pattern.strip(), keyws) | ||
155 | msg_body = format_str(msg_body_pattern, keyws) | ||
156 | return tag_name, msg_subj + '\n\n' + msg_body | ||
157 | |||
158 | |||
159 | def parse_args(argv): | 38 | def parse_args(argv): |
160 | """Parse command line arguments""" | 39 | """Parse command line arguments""" |
161 | parser = argparse.ArgumentParser( | 40 | parser = argparse.ArgumentParser( |
@@ -217,17 +96,11 @@ def get_nested(d, list_of_keys): | |||
217 | return "" | 96 | return "" |
218 | 97 | ||
219 | def main(argv=None): | 98 | def main(argv=None): |
220 | """Script entry point""" | ||
221 | args = parse_args(argv) | 99 | args = parse_args(argv) |
222 | if args.debug: | 100 | if args.debug: |
223 | log.setLevel(logging.DEBUG) | 101 | log.setLevel(logging.DEBUG) |
224 | 102 | ||
225 | try: | 103 | try: |
226 | if not os.path.isdir(args.data_dir): | ||
227 | raise ArchiveError("Not a directory: {}".format(args.data_dir)) | ||
228 | |||
229 | data_repo = init_git_repo(args.git_dir, args.no_create, args.bare) | ||
230 | |||
231 | # Get keywords to be used in tag and branch names and messages | 104 | # Get keywords to be used in tag and branch names and messages |
232 | metadata = metadata_from_bb() | 105 | metadata = metadata_from_bb() |
233 | keywords = {'hostname': get_nested(metadata, ['hostname']), | 106 | keywords = {'hostname': get_nested(metadata, ['hostname']), |
@@ -236,39 +109,12 @@ def main(argv=None): | |||
236 | 'commit_count': get_nested(metadata, ['layers', 'meta', 'commit_count']), | 109 | 'commit_count': get_nested(metadata, ['layers', 'meta', 'commit_count']), |
237 | 'machine': get_nested(metadata, ['config', 'MACHINE'])} | 110 | 'machine': get_nested(metadata, ['config', 'MACHINE'])} |
238 | 111 | ||
239 | # Expand strings early in order to avoid getting into inconsistent | 112 | gitarchive.gitarchive(args.data_dir, args.git_dir, args.no_create, args.bare, |
240 | # state (e.g. no tag even if data was committed) | 113 | args.commit_msg_subject.strip(), args.commit_msg_body, args.branch_name, |
241 | commit_msg = format_str(args.commit_msg_subject.strip(), keywords) | 114 | args.no_tag, args.tag_name, args.tag_msg_subject, args.tag_msg_body, |
242 | commit_msg += '\n\n' + format_str(args.commit_msg_body, keywords) | 115 | args.exclude, args.notes, args.push, keywords, log) |
243 | branch_name = format_str(args.branch_name, keywords) | ||
244 | tag_name = None | ||
245 | if not args.no_tag and args.tag_name: | ||
246 | tag_name, tag_msg = expand_tag_strings(data_repo, args.tag_name, | ||
247 | args.tag_msg_subject, | ||
248 | args.tag_msg_body, keywords) | ||
249 | |||
250 | # Commit data | ||
251 | commit = git_commit_data(data_repo, args.data_dir, branch_name, | ||
252 | commit_msg, args.exclude, args.notes) | ||
253 | |||
254 | # Create tag | ||
255 | if tag_name: | ||
256 | log.info("Creating tag %s", tag_name) | ||
257 | data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit]) | ||
258 | |||
259 | # Push data to remote | ||
260 | if args.push: | ||
261 | cmd = ['push', '--tags'] | ||
262 | # If no remote is given we push with the default settings from | ||
263 | # gitconfig | ||
264 | if args.push is not True: | ||
265 | notes_refs = ['refs/notes/' + ref.format(branch_name=branch_name) | ||
266 | for ref, _ in args.notes] | ||
267 | cmd.extend([args.push, branch_name] + notes_refs) | ||
268 | log.info("Pushing data to remote") | ||
269 | data_repo.run_cmd(cmd) | ||
270 | 116 | ||
271 | except ArchiveError as err: | 117 | except gitarchive.ArchiveError as err: |
272 | log.error(str(err)) | 118 | log.error(str(err)) |
273 | return 1 | 119 | return 1 |
274 | 120 | ||