summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorMarkus Lehtonen <markus.lehtonen@linux.intel.com>2017-02-02 16:30:57 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-02-15 20:06:41 -0800
commit69c7907bd9c8ea49d1499a8ef044a89d71c84885 (patch)
treeea048ba91b3fcd128e916098111b87f5da544c4a /scripts
parent5c907bde11be1af6e3d57b0c68cd040e04b4000c (diff)
downloadpoky-69c7907bd9c8ea49d1499a8ef044a89d71c84885.tar.gz
scripts: Implement oe-git-archive
A helper script for committing data to git and pushing it upstream. The motivation for the script stems from the need to archiving QA test results in git, but it could be used elsewhere, of course. The script needs to be run under an initialized build environment because it utilizes bitbake configuration information. [YOCTO #10582] (From OE-Core rev: 801e612b137b9d5366639d5b1635151347da5983) Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/oe-git-archive244
1 files changed, 244 insertions, 0 deletions
diff --git a/scripts/oe-git-archive b/scripts/oe-git-archive
new file mode 100755
index 0000000000..419332ded1
--- /dev/null
+++ b/scripts/oe-git-archive
@@ -0,0 +1,244 @@
1#!/usr/bin/python3
2#
3# Helper script for committing data to git and pushing upstream
4#
5# Copyright (c) 2017, Intel Corporation.
6#
7# This program is free software; you can redistribute it and/or modify it
8# under the terms and conditions of the GNU General Public License,
9# version 2, as published by the Free Software Foundation.
10#
11# This program is distributed in the hope it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14# more details.
15#
16import argparse
17import glob
18import json
19import logging
20import math
21import os
22import re
23import sys
24from collections import namedtuple, OrderedDict
25from datetime import datetime, timedelta, tzinfo
26from operator import attrgetter
27
28# Import oe and bitbake libs
29scripts_path = os.path.dirname(os.path.realpath(__file__))
30sys.path.append(os.path.join(scripts_path, 'lib'))
31import scriptpath
32scriptpath.add_bitbake_lib_path()
33scriptpath.add_oe_lib_path()
34
35from oeqa.utils.git import GitRepo, GitError
36from oeqa.utils.metadata import metadata_from_bb
37
38
39# Setup logging
40logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
41log = logging.getLogger()
42
43
44class ArchiveError(Exception):
45 """Internal error handling of this script"""
46
47
48def format_str(string, fields):
49 """Format string using the given fields (dict)"""
50 try:
51 return string.format(**fields)
52 except KeyError as err:
53 raise ArchiveError("Unable to expand string '{}': unknown field {} "
54 "(valid fields are: {})".format(
55 string, err, ', '.join(sorted(fields.keys()))))
56
57
58def init_git_repo(path, no_create):
59 """Initialize local Git repository"""
60 path = os.path.abspath(path)
61 if os.path.isfile(path):
62 raise ArchiveError("Invalid Git repo at {}: path exists but is not a "
63 "directory".format(path))
64 if not os.path.isdir(path) or not os.listdir(path):
65 if no_create:
66 raise ArchiveError("No git repo at {}, refusing to create "
67 "one".format(path))
68 if not os.path.isdir(path):
69 try:
70 os.mkdir(path)
71 except (FileNotFoundError, PermissionError) as err:
72 raise ArchiveError("Failed to mkdir {}: {}".format(path, err))
73 if not os.listdir(path):
74 log.info("Initializing a new Git repo at %s", path)
75 repo = GitRepo.init(path)
76 try:
77 repo = GitRepo(path, is_topdir=True)
78 except GitError:
79 raise ArchiveError("Non-empty directory that is not a Git repository "
80 "at {}\nPlease specify an existing Git repository, "
81 "an empty directory or a non-existing directory "
82 "path.".format(path))
83 return repo
84
85
86def git_commit_data(repo, data_dir, branch, message):
87 """Commit data into a Git repository"""
88 log.info("Committing data into to branch %s", branch)
89 tmp_index = os.path.join(repo.git_dir, 'index.oe-git-archive')
90 try:
91 # Create new tree object from the data
92 env_update = {'GIT_INDEX_FILE': tmp_index,
93 'GIT_WORK_TREE': os.path.abspath(data_dir)}
94 repo.run_cmd('add .', env_update)
95 tree = repo.run_cmd('write-tree', env_update)
96
97 # Create new commit object from the tree
98 parent = repo.rev_parse(branch)
99 git_cmd = ['commit-tree', tree, '-m', message]
100 if parent:
101 git_cmd += ['-p', parent]
102 commit = repo.run_cmd(git_cmd, env_update)
103
104 # Update branch head
105 git_cmd = ['update-ref', 'refs/heads/' + branch, commit]
106 if parent:
107 git_cmd.append(parent)
108 repo.run_cmd(git_cmd)
109
110 # Update current HEAD, if we're on branch 'branch'
111 if repo.get_current_branch() == branch:
112 log.info("Updating %s HEAD to latest commit", repo.top_dir)
113 repo.run_cmd('reset --hard')
114
115 return commit
116 finally:
117 if os.path.exists(tmp_index):
118 os.unlink(tmp_index)
119
120
121def expand_tag_strings(repo, name_pattern, msg_subj_pattern, msg_body_pattern,
122 keywords):
123 """Generate tag name and message, with support for running id number"""
124 keyws = keywords.copy()
125 # Tag number is handled specially: if not defined, we autoincrement it
126 if 'tag_number' not in keyws:
127 # Fill in all other fields than 'tag_number'
128 keyws['tag_number'] = '{tag_number}'
129 tag_re = format_str(name_pattern, keyws)
130 # Replace parentheses for proper regex matching
131 tag_re = tag_re.replace('(', '\(').replace(')', '\)') + '$'
132 # Inject regex group pattern for 'tag_number'
133 tag_re = tag_re.format(tag_number='(?P<tag_number>[0-9]{1,5})')
134
135 keyws['tag_number'] = 0
136 for existing_tag in repo.run_cmd('tag').splitlines():
137 match = re.match(tag_re, existing_tag)
138
139 if match and int(match.group('tag_number')) >= keyws['tag_number']:
140 keyws['tag_number'] = int(match.group('tag_number')) + 1
141
142 tag_name = format_str(name_pattern, keyws)
143 msg_subj= format_str(msg_subj_pattern.strip(), keyws)
144 msg_body = format_str(msg_body_pattern, keyws)
145 return tag_name, msg_subj + '\n\n' + msg_body
146
147
148def parse_args(argv):
149 """Parse command line arguments"""
150 parser = argparse.ArgumentParser(
151 description="Commit data to git and push upstream",
152 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
153
154 parser.add_argument('--debug', '-D', action='store_true',
155 help="Verbose logging")
156 parser.add_argument('--git-dir', '-g', required=True,
157 help="Local git directory to use")
158 parser.add_argument('--no-create', action='store_true',
159 help="If GIT_DIR is not a valid Git repository, do not "
160 "try to create one")
161 parser.add_argument('--push', '-p', nargs='?', default=False, const=True,
162 help="Push to remote")
163 parser.add_argument('--branch-name', '-b',
164 default='{hostname}/{branch}/{machine}',
165 help="Git branch name (pattern) to use")
166 parser.add_argument('--no-tag', action='store_true',
167 help="Do not create Git tag")
168 parser.add_argument('--tag-name', '-t',
169 default='{hostname}/{branch}/{machine}/{commit_count}-g{commit}/{tag_number}',
170 help="Tag name (pattern) to use")
171 parser.add_argument('--commit-msg-subject',
172 default='Results of {branch}:{commit} on {hostname}',
173 help="Subject line (pattern) to use in the commit message")
174 parser.add_argument('--commit-msg-body',
175 default='branch: {branch}\ncommit: {commit}\nhostname: {hostname}',
176 help="Commit message body (pattern)")
177 parser.add_argument('--tag-msg-subject',
178 default='Test run #{tag_number} of {branch}:{commit} on {hostname}',
179 help="Subject line (pattern) of the tag message")
180 parser.add_argument('--tag-msg-body',
181 default='',
182 help="Tag message body (pattern)")
183 parser.add_argument('data_dir', metavar='DATA_DIR',
184 help="Data to commit")
185 return parser.parse_args(argv)
186
187
188def main(argv=None):
189 """Script entry point"""
190 args = parse_args(argv)
191 if args.debug:
192 log.setLevel(logging.DEBUG)
193
194 try:
195 if not os.path.isdir(args.data_dir):
196 raise ArchiveError("Not a directory: {}".format(args.data_dir))
197
198 data_repo = init_git_repo(args.git_dir, args.no_create)
199
200 # Get keywords to be used in tag and branch names and messages
201 metadata = metadata_from_bb()
202 keywords = {'hostname': metadata['hostname'],
203 'branch': metadata['layers']['meta']['branch'],
204 'commit': metadata['layers']['meta']['commit'],
205 'commit_count': metadata['layers']['meta']['commit_count'],
206 'machine': metadata['config']['MACHINE']}
207
208 # Expand strings early in order to avoid getting into inconsistent
209 # state (e.g. no tag even if data was committed)
210 commit_msg = format_str(args.commit_msg_subject.strip(), keywords)
211 commit_msg += '\n\n' + format_str(args.commit_msg_body, keywords)
212 branch_name = format_str(args.branch_name, keywords)
213 tag_name = None
214 if not args.no_tag and args.tag_name:
215 tag_name, tag_msg = expand_tag_strings(data_repo, args.tag_name,
216 args.tag_msg_subject,
217 args.tag_msg_body, keywords)
218
219 # Commit data
220 commit = git_commit_data(data_repo, args.data_dir, branch_name,
221 commit_msg)
222
223 # Create tag
224 if tag_name:
225 log.info("Creating tag %s", tag_name)
226 data_repo.run_cmd(['tag', '-a', '-m', tag_msg, tag_name, commit])
227
228 # Push data to remote
229 if args.push:
230 cmd = ['push', '--tags']
231 if args.push is not True:
232 cmd.extend(['--repo', args.push])
233 cmd.append(branch_name)
234 log.info("Pushing data to remote")
235 data_repo.run_cmd(cmd)
236
237 except ArchiveError as err:
238 log.error(str(err))
239 return 1
240
241 return 0
242
243if __name__ == "__main__":
244 sys.exit(main())