summaryrefslogtreecommitdiffstats
path: root/scripts/combo-layer
diff options
context:
space:
mode:
authorYu Ke <ke.yu@intel.com>2011-06-13 12:20:53 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-07-08 16:52:27 (GMT)
commitb9ff62a0d0096138cb762ec5dbee9fd056999b70 (patch)
treed9dedb438d6fe596472958c7d83e7741433f0045 /scripts/combo-layer
parent4fadc30b92d42523fee013c168d47bc93dfe4fbd (diff)
downloadpoky-b9ff62a0d0096138cb762ec5dbee9fd056999b70.tar.gz
combo-layer-tool: add tool to manipulate combo layers
This patch adds the script "combo-layer" to manipulate combo layer repos. A combo layer repo is a repo containing multiple component repos, e.g. oe-core, bitbake, BSP repos. The combo layer repo needs to be updated by syncing with the component repo upstream. This script is written to assist the combo layer handling. The combo layer tool provides three functionalities: - init: when the combo layer repo and component repo does not exist, init will "git init" the combo layer repo, and also "git clone" the component repos - update: combo layer tool will pull the latest commit from component repo upstream, and apply the commits since last update commit to the combo repo. If the user specifies interactive mode(--interactive), they can edit the patch list to select which commits to apply. - splitpatch: split the combo repo commit into separate patches per component repo, to facilitate upstream submission. Combo layer tool uses a config file to define the component repo info. Please check the combo-layer.conf.example for a detailed explanation of the config file fields. (From OE-Core rev: 68394476748386e58f40173643967f5a248173b1) Signed-off-by: Yu Ke <ke.yu@intel.com> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/combo-layer')
-rwxr-xr-xscripts/combo-layer366
1 files changed, 366 insertions, 0 deletions
diff --git a/scripts/combo-layer b/scripts/combo-layer
new file mode 100755
index 0000000..84cc48f
--- /dev/null
+++ b/scripts/combo-layer
@@ -0,0 +1,366 @@
1#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# Copyright 2011 Intel Corporation
6# Authored-by: Yu Ke <ke.yu@intel.com>
7# Paul Eggleton <paul.eggleton@intel.com>
8# Richard Purdie <richard.purdie@intel.com>
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License version 2 as
12# published by the Free Software Foundation.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License along
20# with this program; if not, write to the Free Software Foundation, Inc.,
21# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22
23import os, sys
24import optparse
25import logging
26import subprocess
27import ConfigParser
28
29__version__ = "0.1.0"
30
31def logger_create():
32 logger = logging.getLogger("")
33 loggerhandler = logging.StreamHandler()
34 loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
35 logger.addHandler(loggerhandler)
36 logger.setLevel(logging.INFO)
37 return logger
38
39logger = logger_create()
40
41class Configuration(object):
42 """
43 Manages the configuration
44
45 A valid conf looks like:
46
47# component name
48[bitbake]
49
50# mandatory options
51
52# git upstream uri
53src_uri = git://git.openembedded.org/bitbake
54
55# the directory to clone the component repo
56local_repo_dir = ~/src/bitbake
57
58# the relative dir to commit the repo patch
59# use "." if it is root dir
60dest_dir = bitbake
61
62# the updated revision last time.
63# leave it empty if no commit updated yet, and then the tool
64# will start from the first commit
65last_revision =
66
67# optional options
68
69# file_filter: only include the interested file
70# file_filter = [path] [path] ...
71# example:
72# file_filter = src/ : only include the subdir src
73# file_filter = src/*.c : only include the src *.c file
74# file_filter = src/main.c src/Makefile.am : only include these two files
75
76[oe-core]
77src_uri = git://git.openembedded.org/openembedded-core
78local_repo_dir = ~/src/oecore
79dest_dir = .
80last_revision =
81
82# more components ...
83
84 """
85 def __init__(self, options):
86 for key, val in options.__dict__.items():
87 setattr(self, key, val)
88 self.parser = ConfigParser.ConfigParser()
89 self.parser.readfp(open(self.conffile))
90 self.repos = {}
91 for repo in self.parser.sections():
92 self.repos[repo] = {}
93 for (name, value) in self.parser.items(repo):
94 self.repos[repo][name] = value
95
96 def update(self, repo, option, value):
97 self.parser.set(repo, option, value)
98 self.parser.write(open(self.conffile, "w"))
99
100 def sanity_check(self):
101 required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
102 msg = ""
103 for name in self.repos:
104 for option in required_options:
105 if option not in self.repos[name]:
106 msg = "%s\nOption %s is not defined for component %s" %(msg, option, name)
107 if msg != "":
108 logger.error("configuration file %s has the following error:%s" % (self.conffile,msg))
109 sys.exit(1)
110
111 # filterdiff is required by action_splitpatch, so check its availability
112 if subprocess.call("which filterdiff &>/dev/null", shell=True) != 0:
113 logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
114 sys.exit(1)
115
116def runcmd(cmd,destdir=None):
117 """
118 execute command, raise CalledProcessError if fail
119 return output if succeed
120 """
121 logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
122 out = os.tmpfile()
123 try:
124 subprocess.check_call(cmd, stdout=out, stderr=out, cwd=destdir, shell=True)
125 except subprocess.CalledProcessError,e:
126 out.seek(0)
127 logger.error("%s" % out.read())
128 raise e
129
130 out.seek(0)
131 output = out.read()
132 logger.debug("output: %s" % output )
133 return output
134
135def action_init(conf, args):
136 """
137 Clone component repositories
138 Check git initialised and working tree is clean
139 """
140 for name in conf.repos:
141 ldir = conf.repos[name]['local_repo_dir']
142 if not os.path.exists(ldir):
143 logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
144 subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
145 if not os.path.exists(".git"):
146 runcmd("git init")
147
148def check_repo_clean(repodir):
149 """
150 check if the repo is clean
151 exit if repo is dirty
152 """
153 try:
154 runcmd("git diff --quiet", repodir)
155 #TODO: also check the index using "git diff --cached"
156 # but this will fail in just initialized git repo
157 # so need figure out a way
158 except:
159 logger.error("git repo %s is dirty, please fix it first", repodir)
160 sys.exit(1)
161
162def action_update(conf, args):
163 """
164 update the component repo
165 generate the patch list
166 apply the generated patches
167 """
168 # make sure all repos are clean
169 for name in conf.repos:
170 check_repo_clean(conf.repos[name]['local_repo_dir'])
171 check_repo_clean(os.getcwd())
172
173 import uuid
174 patch_dir = "patch-%s" % uuid.uuid4()
175 os.mkdir(patch_dir)
176
177 for name in conf.repos:
178 repo = conf.repos[name]
179 ldir = repo['local_repo_dir']
180 dest_dir = repo['dest_dir']
181 repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
182
183 # Step 1: update the component repo
184 logger.info("git pull for component repo %s in %s ..." % (name, ldir))
185 output=runcmd("git pull", ldir)
186 logger.info(output)
187
188 # Step 2: generate the patch list and store to patch dir
189 logger.info("generating patches for %s" % name)
190 if dest_dir != ".":
191 prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
192 else:
193 prefix = ""
194 if repo['last_revision'] == "":
195 logger.info("Warning: last_revision of component %s is not set, so start from the first commit" % name)
196 patch_cmd_range = "--root master"
197 rev_cmd_range = "master"
198 else:
199 patch_cmd_range = "%s..master" % repo['last_revision']
200 rev_cmd_range = "%s..master" % repo['last_revision']
201
202 file_filter = repo.get('file_filter',"")
203
204 patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
205 (prefix,repo_patch_dir, patch_cmd_range, file_filter)
206 output = runcmd(patch_cmd, ldir)
207 logger.debug("generated patch set:\n%s" % output)
208 patchlist = output.splitlines()
209
210 rev_cmd = 'git log --pretty=format:"%H" ' + rev_cmd_range
211 revlist = runcmd(rev_cmd, ldir).splitlines()
212
213 # Step 3: Call repo specific hook to adjust patch
214 if 'hook' in repo:
215 # hook parameter is: ./hook patchpath revision reponame
216 count=len(revlist)-1
217 for patch in patchlist:
218 runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
219 count=count-1
220
221 # Step 4: write patch list and revision list to file, for user to edit later
222 patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
223 repo['patchlist'] = patchlist_file
224 f = open(patchlist_file, 'w')
225 count=len(revlist)-1
226 for patch in patchlist:
227 f.write("%s %s\n" % (patch, revlist[count]))
228 count=count-1
229 f.close()
230
231 # Step 5: invoke bash for user to edit patch and patch list
232 if conf.interactive:
233 print 'Edit the patch and patch list in %s\n' \
234 'For example, remove the unwanted patch entry from patchlist-*, so that it will be not applied later\n' \
235 'After finish, press following command to continue\n' \
236 ' exit 0 -- exit and continue to apply the patch\n' \
237 ' exit 1 -- abort and not apply patch\n' % patch_dir
238 ret = subprocess.call(["bash"], cwd=patch_dir)
239 if ret != 0:
240 print "Abort without applying patch"
241 sys.exit(0)
242
243 # Step 6: apply the generated and revised patch
244 action_apply_patch(conf, args)
245 runcmd("rm -rf %s" % patch_dir)
246
247def action_apply_patch(conf, args):
248 """
249 apply the generated patch list to combo repo
250 """
251 for name in conf.repos:
252 repo = conf.repos[name]
253 lastrev = repo["last_revision"]
254 for line in open(repo['patchlist']):
255 patchfile = line.split()[0]
256 lastrev = line.split()[1]
257 cmd = "git am -s -p1 %s" % patchfile
258 logger.info("Apply %s" % patchfile )
259 try:
260 runcmd(cmd)
261 except subprocess.CalledProcessError:
262 logger.info('"git am --abort" is executed to cleanup repo')
263 runcmd("git am --abort")
264 logger.error('"%s" failed' % cmd)
265 logger.info("please manually apply patch %s" % patchfile)
266 logger.info("After applying, run this tool again to apply the rest patches")
267 conf.update(name, "last_revision", lastrev)
268 sys.exit(0)
269 conf.update(name, "last_revision", lastrev)
270
271def action_splitpatch(conf, args):
272 """
273 generate the commit patch and
274 split the patch per repo
275 """
276 logger.debug("action_splitpatch")
277 if len(args) > 1:
278 commit = args[1]
279 else:
280 commit = "HEAD"
281 patchdir = "splitpatch-%s" % commit
282 if not os.path.exists(patchdir):
283 os.mkdir(patchdir)
284
285 # filerange_root is for the repo whose dest_dir is root "."
286 # and it should be specified by excluding all other repo dest dir
287 # like "-x repo1 -x repo2 -x repo3 ..."
288 filerange_root = ""
289 for name in conf.repos:
290 dest_dir = conf.repos[name]['dest_dir']
291 if dest_dir != ".":
292 filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
293
294 for name in conf.repos:
295 dest_dir = conf.repos[name]['dest_dir']
296 patch_filename = "%s/%s.patch" % (patchdir, name)
297 if dest_dir == ".":
298 cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename)
299 else:
300 cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename)
301 runcmd(cmd)
302 # Detect empty patches (including those produced by filterdiff above
303 # that contain only preamble text)
304 if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "":
305 os.remove(patch_filename)
306 logger.info("(skipping %s - no changes)", name)
307 else:
308 logger.info(patch_filename)
309
310def action_error(conf, args):
311 logger.info("invalid action %s" % args[0])
312
313actions = {
314 "init": action_init,
315 "update": action_update,
316 "splitpatch": action_splitpatch,
317}
318
319def main():
320 parser = optparse.OptionParser(
321 version = "Combo Layer Repo Tool version %s" % __version__,
322 usage = """%prog [options] action
323
324Create and update a combination layer repository from multiple component repositories.
325
326Action:
327 init initialise the combo layer repo
328 update get patches from component repos and apply them to the combo repo
329 splitpatch [commit] generate commit patch and split per component, default commit is HEAD""")
330
331 parser.add_option("-c", "--conf", help = "specify the config file. default is conf/combolayer.conf",
332 action = "store", dest = "conffile", default = "combo-layer.conf")
333
334 parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches",
335 action = "store_true", dest = "interactive", default = False)
336
337 parser.add_option("-D", "--debug", help = "output debug information",
338 action = "store_true", dest = "debug", default = False)
339
340 options, args = parser.parse_args(sys.argv)
341
342 # Dispatch to action handler
343 if len(args) == 1:
344 logger.error("No action specified, exiting")
345 parser.print_help()
346 elif args[1] not in actions:
347 logger.error("Unsupported action %s, exiting\n" % (args[1]))
348 parser.print_help()
349 elif not os.path.exists(options.conffile):
350 logger.error("No valid config file, exiting\n")
351 parser.print_help()
352 else:
353 if options.debug:
354 logger.setLevel(logging.DEBUG)
355 confdata = Configuration(options)
356 confdata.sanity_check()
357 actions.get(args[1], action_error)(confdata, args[1:])
358
359if __name__ == "__main__":
360 try:
361 ret = main()
362 except Exception:
363 ret = 1
364 import traceback
365 traceback.print_exc(5)
366 sys.exit(ret)