summaryrefslogtreecommitdiffstats
path: root/scripts/combo-layer
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/combo-layer')
-rwxr-xr-xscripts/combo-layer1384
1 files changed, 0 insertions, 1384 deletions
diff --git a/scripts/combo-layer b/scripts/combo-layer
deleted file mode 100755
index 4a715914af..0000000000
--- a/scripts/combo-layer
+++ /dev/null
@@ -1,1384 +0,0 @@
1#!/usr/bin/env python3
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# Copyright 2011 Intel Corporation
6# Authored-by: Yu Ke <ke.yu@intel.com>
7# Paul Eggleton <paul.eggleton@intel.com>
8# Richard Purdie <richard.purdie@intel.com>
9#
10# SPDX-License-Identifier: GPL-2.0-only
11#
12
13import fnmatch
14import os, sys
15import optparse
16import logging
17import subprocess
18import tempfile
19import configparser
20import re
21import copy
22import shlex
23import shutil
24from string import Template
25from functools import reduce
26
27__version__ = "0.2.1"
28
29def logger_create():
30 logger = logging.getLogger("")
31 loggerhandler = logging.StreamHandler()
32 loggerhandler.setFormatter(logging.Formatter("[%(asctime)s] %(message)s","%H:%M:%S"))
33 logger.addHandler(loggerhandler)
34 logger.setLevel(logging.INFO)
35 return logger
36
37logger = logger_create()
38
39def get_current_branch(repodir=None):
40 try:
41 if not os.path.exists(os.path.join(repodir if repodir else '', ".git")):
42 # Repo not created yet (i.e. during init) so just assume master
43 return "master"
44 branchname = runcmd("git symbolic-ref HEAD 2>/dev/null", repodir).strip()
45 if branchname.startswith("refs/heads/"):
46 branchname = branchname[11:]
47 return branchname
48 except subprocess.CalledProcessError:
49 return ""
50
51class Configuration(object):
52 """
53 Manages the configuration
54
55 For an example config file, see combo-layer.conf.example
56
57 """
58 def __init__(self, options):
59 for key, val in options.__dict__.items():
60 setattr(self, key, val)
61
62 def readsection(parser, section, repo):
63 for (name, value) in parser.items(section):
64 if value.startswith("@"):
65 self.repos[repo][name] = eval(value.strip("@"))
66 else:
67 # Apply special type transformations for some properties.
68 # Type matches the RawConfigParser.get*() methods.
69 types = {'signoff': 'boolean', 'update': 'boolean', 'history': 'boolean'}
70 if name in types:
71 value = getattr(parser, 'get' + types[name])(section, name)
72 self.repos[repo][name] = value
73
74 def readglobalsection(parser, section):
75 for (name, value) in parser.items(section):
76 if name == "commit_msg":
77 self.commit_msg_template = value
78
79 logger.debug("Loading config file %s" % self.conffile)
80 self.parser = configparser.ConfigParser()
81 with open(self.conffile) as f:
82 self.parser.read_file(f)
83
84 # initialize default values
85 self.commit_msg_template = "Automatic commit to update last_revision"
86
87 self.repos = {}
88 for repo in self.parser.sections():
89 if repo == "combo-layer-settings":
90 # special handling for global settings
91 readglobalsection(self.parser, repo)
92 else:
93 self.repos[repo] = {}
94 readsection(self.parser, repo, repo)
95
96 # Load local configuration, if available
97 self.localconffile = None
98 self.localparser = None
99 self.combobranch = None
100 if self.conffile.endswith('.conf'):
101 lcfile = self.conffile.replace('.conf', '-local.conf')
102 if os.path.exists(lcfile):
103 # Read combo layer branch
104 self.combobranch = get_current_branch()
105 logger.debug("Combo layer branch is %s" % self.combobranch)
106
107 self.localconffile = lcfile
108 logger.debug("Loading local config file %s" % self.localconffile)
109 self.localparser = configparser.ConfigParser()
110 with open(self.localconffile) as f:
111 self.localparser.readfp(f)
112
113 for section in self.localparser.sections():
114 if '|' in section:
115 sectionvals = section.split('|')
116 repo = sectionvals[0]
117 if sectionvals[1] != self.combobranch:
118 continue
119 else:
120 repo = section
121 if repo in self.repos:
122 readsection(self.localparser, section, repo)
123
124 def update(self, repo, option, value, initmode=False):
125 # If the main config has the option already, that is what we
126 # are expected to modify.
127 if self.localparser and not self.parser.has_option(repo, option):
128 parser = self.localparser
129 section = "%s|%s" % (repo, self.combobranch)
130 conffile = self.localconffile
131 if initmode and not parser.has_section(section):
132 parser.add_section(section)
133 else:
134 parser = self.parser
135 section = repo
136 conffile = self.conffile
137 parser.set(section, option, value)
138 with open(conffile, "w") as f:
139 parser.write(f)
140 self.repos[repo][option] = value
141
142 def sanity_check(self, initmode=False):
143 required_options=["src_uri", "local_repo_dir", "dest_dir", "last_revision"]
144 if initmode:
145 required_options.remove("last_revision")
146 msg = ""
147 missing_options = []
148 for name in self.repos:
149 for option in required_options:
150 if option not in self.repos[name]:
151 msg = "%s\nOption %s is not defined for component %s" %(msg, option, name)
152 missing_options.append(option)
153 # Sanitize dest_dir so that we do not have to deal with edge cases
154 # (unset, empty string, double slashes) in the rest of the code.
155 # It not being set will still be flagged as error because it is
156 # listed as required option above; that could be changed now.
157 dest_dir = os.path.normpath(self.repos[name].get("dest_dir", "."))
158 self.repos[name]["dest_dir"] = "." if not dest_dir else dest_dir
159 if msg != "":
160 logger.error("configuration file %s has the following error: %s" % (self.conffile,msg))
161 if self.localconffile and 'last_revision' in missing_options:
162 logger.error("local configuration file %s may be missing configuration for combo branch %s" % (self.localconffile, self.combobranch))
163 sys.exit(1)
164
165 # filterdiff is required by action_splitpatch, so check its availability
166 if subprocess.call("which filterdiff > /dev/null 2>&1", shell=True) != 0:
167 logger.error("ERROR: patchutils package is missing, please install it (e.g. # apt-get install patchutils)")
168 sys.exit(1)
169
170def runcmd(cmd,destdir=None,printerr=True,out=None,env=None):
171 """
172 execute command, raise CalledProcessError if fail
173 return output if succeed
174 """
175 logger.debug("run cmd '%s' in %s" % (cmd, os.getcwd() if destdir is None else destdir))
176 if not out:
177 out = tempfile.TemporaryFile()
178 err = out
179 else:
180 err = tempfile.TemporaryFile()
181 try:
182 subprocess.check_call(cmd, stdout=out, stderr=err, cwd=destdir, shell=isinstance(cmd, str), env=env or os.environ)
183 except subprocess.CalledProcessError as e:
184 err.seek(0)
185 if printerr:
186 logger.error("%s" % err.read())
187 raise e
188
189 err.seek(0)
190 output = err.read().decode('utf-8')
191 logger.debug("output: %s" % output.replace(chr(0), '\\0'))
192 return output
193
194def action_sync_revs(conf, args):
195 """
196 Update the last_revision config option for each repo with the latest
197 revision in the remote's branch. Useful if multiple people are using
198 combo-layer.
199 """
200 repos = get_repos(conf, args[1:])
201
202 for name in repos:
203 repo = conf.repos[name]
204 ldir = repo['local_repo_dir']
205 branch = repo.get('branch', "master")
206 runcmd("git fetch", ldir)
207 lastrev = runcmd('git rev-parse origin/%s' % branch, ldir).strip()
208 print("Updating %s to %s" % (name, lastrev))
209 conf.update(name, "last_revision", lastrev)
210
211def action_init(conf, args):
212 """
213 Clone component repositories
214 Check git is initialised; if not, copy initial data from component repos
215 """
216 for name in conf.repos:
217 ldir = conf.repos[name]['local_repo_dir']
218 if not os.path.exists(ldir):
219 logger.info("cloning %s to %s" %(conf.repos[name]['src_uri'], ldir))
220 subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
221 if not os.path.exists(".git"):
222 runcmd("git init")
223 if conf.history:
224 # Need a common ref for all trees.
225 runcmd('git commit -m "initial empty commit" --allow-empty')
226 startrev = runcmd('git rev-parse master').strip()
227
228 for name in conf.repos:
229 repo = conf.repos[name]
230 ldir = repo['local_repo_dir']
231 branch = repo.get('branch', "master")
232 lastrev = repo.get('last_revision', None)
233 if lastrev and lastrev != "HEAD":
234 initialrev = lastrev
235 if branch:
236 if not check_rev_branch(name, ldir, lastrev, branch):
237 sys.exit(1)
238 logger.info("Copying data from %s at specified revision %s..." % (name, lastrev))
239 else:
240 lastrev = None
241 initialrev = branch
242 logger.info("Copying data from %s..." % name)
243 # Sanity check initialrev and turn it into hash (required for copying history,
244 # because resolving a name ref only works in the component repo).
245 rev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
246 if rev != initialrev:
247 try:
248 refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n')
249 if len(set(refs)) > 1:
250 # Happens for example when configured to track
251 # "master" and there is a refs/heads/master. The
252 # traditional behavior from "git archive" (preserved
253 # here) it to choose the first one. This might not be
254 # intended, so at least warn about it.
255 logger.warning("%s: initial revision '%s' not unique, picking result of rev-parse = %s" %
256 (name, initialrev, refs[0]))
257 initialrev = rev
258 except:
259 # show-ref fails for hashes. Skip the sanity warning in that case.
260 pass
261 initialrev = rev
262 dest_dir = repo['dest_dir']
263 if dest_dir != ".":
264 extract_dir = os.path.join(os.getcwd(), dest_dir)
265 if not os.path.exists(extract_dir):
266 os.makedirs(extract_dir)
267 else:
268 extract_dir = os.getcwd()
269 file_filter = repo.get('file_filter', "")
270 exclude_patterns = repo.get('file_exclude', '').split()
271 def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir,
272 subdir=""):
273 # When working inside a filtered branch which had the
274 # files already moved, we need to prepend the
275 # subdirectory to all filters, otherwise they would
276 # not match.
277 if subdir == '.':
278 subdir = ''
279 elif subdir:
280 subdir = os.path.normpath(subdir)
281 file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()])
282 exclude_patterns = [subdir + '/' + x for x in exclude_patterns]
283 # To handle both cases, we cd into the target
284 # directory and optionally tell tar to strip the path
285 # prefix when the files were already moved.
286 subdir_components = len(subdir.split(os.path.sep)) if subdir else 0
287 strip=('--strip-components=%d' % subdir_components) if subdir else ''
288 # TODO: file_filter wild cards do not work (and haven't worked before either), because
289 # a) GNU tar requires a --wildcards parameter before turning on wild card matching.
290 # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c,
291 # in contrast to the other use of file_filter as parameter of "git archive"
292 # where it only matches .c files directly in src).
293 files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" %
294 (initialrev, subdir,
295 strip, extract_dir, file_filter),
296 ldir)
297 if exclude_patterns:
298 # Implement file removal by letting tar create the
299 # file and then deleting it in the file system
300 # again. Uses the list of files created by tar (easier
301 # than walking the tree).
302 for file in files.split('\n'):
303 if file.endswith(os.path.sep):
304 continue
305 for pattern in exclude_patterns:
306 if fnmatch.fnmatch(file, pattern):
307 os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file])))
308 break
309
310 if not conf.history:
311 copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir)
312 else:
313 # First fetch remote history into local repository.
314 # We need a ref for that, so ensure that there is one.
315 refname = "combo-layer-init-%s" % name
316 runcmd("git branch -f %s %s" % (refname, initialrev), ldir)
317 runcmd("git fetch %s %s" % (ldir, refname))
318 runcmd("git branch -D %s" % refname, ldir)
319 # Make that the head revision.
320 runcmd("git checkout -b %s %s" % (name, initialrev))
321 # Optional: cut the history by replacing the given
322 # start point(s) with commits providing the same
323 # content (aka tree), but with commit information that
324 # makes it clear that this is an artifically created
325 # commit and nothing the original authors had anything
326 # to do with.
327 since_rev = repo.get('since_revision', '')
328 if since_rev:
329 committer = runcmd('git var GIT_AUTHOR_IDENT').strip()
330 # Same time stamp, no name.
331 author = re.sub('.* (\d+ [+-]\d+)', r'unknown <unknown> \1', committer)
332 logger.info('author %s' % author)
333 for rev in since_rev.split():
334 # Resolve in component repo...
335 rev = runcmd('git log --oneline --no-abbrev-commit -n1 %s' % rev, ldir).split()[0]
336 # ... and then get the tree in current
337 # one. The commit should be in both repos with
338 # the same tree, but better check here.
339 tree = runcmd('git show -s --pretty=format:%%T %s' % rev).strip()
340 with tempfile.NamedTemporaryFile(mode='wt') as editor:
341 editor.write('''cat >$1 <<EOF
342tree %s
343author %s
344committer %s
345
346%s: squashed import of component
347
348This commit copies the entire set of files as found in
349%s %s
350
351For more information about previous commits, see the
352upstream repository.
353
354Commit created by combo-layer.
355EOF
356''' % (tree, author, committer, name, name, since_rev))
357 editor.flush()
358 os.environ['GIT_EDITOR'] = 'sh %s' % editor.name
359 runcmd('git replace --edit %s' % rev)
360
361 # Optional: rewrite history to change commit messages or to move files.
362 if 'hook' in repo or dest_dir != ".":
363 filter_branch = ['git', 'filter-branch', '--force']
364 with tempfile.NamedTemporaryFile(mode='wt') as hookwrapper:
365 if 'hook' in repo:
366 # Create a shell script wrapper around the original hook that
367 # can be used by git filter-branch. Hook may or may not have
368 # an absolute path.
369 hook = repo['hook']
370 hook = os.path.join(os.path.dirname(conf.conffile), '..', hook)
371 # The wrappers turns the commit message
372 # from stdin into a fake patch header.
373 # This is good enough for changing Subject
374 # and commit msg body with normal
375 # combo-layer hooks.
376 hookwrapper.write('''set -e
377tmpname=$(mktemp)
378trap "rm $tmpname" EXIT
379echo -n 'Subject: [PATCH] ' >>$tmpname
380cat >>$tmpname
381if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then
382 echo >>$tmpname
383fi
384echo '---' >>$tmpname
385%s $tmpname $GIT_COMMIT %s
386tail -c +18 $tmpname | head -c -4
387''' % (hook, name))
388 hookwrapper.flush()
389 filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name])
390 if dest_dir != ".":
391 parent = os.path.dirname(dest_dir)
392 if not parent:
393 parent = '.'
394 # May run outside of the current directory, so do not assume that .git exists.
395 filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && find . -mindepth 1 -maxdepth 1 ! -name .git -print0 | xargs -0 -I SOURCE mv SOURCE .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)])
396 filter_branch.append('HEAD')
397 runcmd(filter_branch)
398 runcmd('git update-ref -d refs/original/refs/heads/%s' % name)
399 repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip()
400 repo['stripped_revision'] = repo['rewritten_revision']
401 # Optional filter files: remove everything and re-populate using the normal filtering code.
402 # Override any potential .gitignore.
403 if file_filter or exclude_patterns:
404 runcmd('git rm -rf .')
405 if not os.path.exists(extract_dir):
406 os.makedirs(extract_dir)
407 copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.',
408 subdir=dest_dir)
409 runcmd('git add --all --force .')
410 if runcmd('git status --porcelain'):
411 # Something to commit.
412 runcmd(['git', 'commit', '-m',
413 '''%s: select file subset
414
415Files from the component repository were chosen based on
416the following filters:
417file_filter = %s
418file_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))])
419 repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip()
420
421 if not lastrev:
422 lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
423 conf.update(name, "last_revision", lastrev, initmode=True)
424
425 if not conf.history:
426 runcmd("git add .")
427 else:
428 # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies
429 runcmd('git checkout master')
430 merge = ['git', 'merge', '--no-commit']
431 for name in conf.repos:
432 repo = conf.repos[name]
433 # Use branch created earlier.
434 merge.append(name)
435 # Root all commits which have no parent in the common
436 # ancestor in the new repository.
437 for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s --' % name).split('\n'):
438 runcmd('git replace --graft %s %s' % (start, startrev))
439 try:
440 runcmd(merge)
441 except Exception as error:
442 logger.info('''Merging component repository history failed, perhaps because of merge conflicts.
443It may be possible to commit anyway after resolving these conflicts.
444
445%s''' % error)
446 # Create MERGE_HEAD and MERGE_MSG. "git merge" itself
447 # does not create MERGE_HEAD in case of a (harmless) failure,
448 # and we want certain auto-generated information in the
449 # commit message for future reference and/or automation.
450 with open('.git/MERGE_HEAD', 'w') as head:
451 with open('.git/MERGE_MSG', 'w') as msg:
452 msg.write('repo: initial import of components\n\n')
453 # head.write('%s\n' % startrev)
454 for name in conf.repos:
455 repo = conf.repos[name]
456 # <upstream ref> <rewritten ref> <rewritten + files removed>
457 msg.write('combo-layer-%s: %s %s %s\n' % (name,
458 repo['last_revision'],
459 repo['rewritten_revision'],
460 repo['stripped_revision']))
461 rev = runcmd('git rev-parse %s' % name).strip()
462 head.write('%s\n' % rev)
463
464 if conf.localconffile:
465 localadded = True
466 try:
467 runcmd("git rm --cached %s" % conf.localconffile, printerr=False)
468 except subprocess.CalledProcessError:
469 localadded = False
470 if localadded:
471 localrelpath = os.path.relpath(conf.localconffile)
472 runcmd("grep -q %s .gitignore || echo %s >> .gitignore" % (localrelpath, localrelpath))
473 runcmd("git add .gitignore")
474 logger.info("Added local configuration file %s to .gitignore", localrelpath)
475 logger.info("Initial combo layer repository data has been created; please make any changes if desired and then use 'git commit' to make the initial commit.")
476 else:
477 logger.info("Repository already initialised, nothing to do.")
478
479
480def check_repo_clean(repodir):
481 """
482 check if the repo is clean
483 exit if repo is dirty
484 """
485 output=runcmd("git status --porcelain", repodir)
486 r = re.compile(r'\?\? patch-.*/')
487 dirtyout = [item for item in output.splitlines() if not r.match(item)]
488 if dirtyout:
489 logger.error("git repo %s is dirty, please fix it first", repodir)
490 sys.exit(1)
491
492def check_patch(patchfile):
493 f = open(patchfile, 'rb')
494 ln = f.readline()
495 of = None
496 in_patch = False
497 beyond_msg = False
498 pre_buf = b''
499 while ln:
500 if not beyond_msg:
501 if ln == b'---\n':
502 if not of:
503 break
504 in_patch = False
505 beyond_msg = True
506 elif ln.startswith(b'--- '):
507 # We have a diff in the commit message
508 in_patch = True
509 if not of:
510 print('WARNING: %s contains a diff in its commit message, indenting to avoid failure during apply' % patchfile)
511 of = open(patchfile + '.tmp', 'wb')
512 of.write(pre_buf)
513 pre_buf = b''
514 elif in_patch and not ln[0] in b'+-@ \n\r':
515 in_patch = False
516 if of:
517 if in_patch:
518 of.write(b' ' + ln)
519 else:
520 of.write(ln)
521 else:
522 pre_buf += ln
523 ln = f.readline()
524 f.close()
525 if of:
526 of.close()
527 os.rename(of.name, patchfile)
528
529def drop_to_shell(workdir=None):
530 if not sys.stdin.isatty():
531 print("Not a TTY so can't drop to shell for resolution, exiting.")
532 return False
533
534 shell = os.environ.get('SHELL', 'bash')
535 print('Dropping to shell "%s"\n' \
536 'When you are finished, run the following to continue:\n' \
537 ' exit -- continue to apply the patches\n' \
538 ' exit 1 -- abort\n' % shell);
539 ret = subprocess.call([shell], cwd=workdir)
540 if ret != 0:
541 print("Aborting")
542 return False
543 else:
544 return True
545
546def check_rev_branch(component, repodir, rev, branch):
547 try:
548 actualbranch = runcmd("git branch --contains %s" % rev, repodir, printerr=False)
549 except subprocess.CalledProcessError as e:
550 if e.returncode == 129:
551 actualbranch = ""
552 else:
553 raise
554
555 if not actualbranch:
556 logger.error("%s: specified revision %s is invalid!" % (component, rev))
557 return False
558
559 branches = []
560 branchlist = actualbranch.split("\n")
561 for b in branchlist:
562 branches.append(b.strip().split(' ')[-1])
563
564 if branch not in branches:
565 logger.error("%s: specified revision %s is not on specified branch %s!" % (component, rev, branch))
566 return False
567 return True
568
569def get_repos(conf, repo_names):
570 repos = []
571 for name in repo_names:
572 if name.startswith('-'):
573 break
574 else:
575 repos.append(name)
576 for repo in repos:
577 if not repo in conf.repos:
578 logger.error("Specified component '%s' not found in configuration" % repo)
579 sys.exit(1)
580
581 if not repos:
582 repos = [ repo for repo in conf.repos if conf.repos[repo].get("update", True) ]
583
584 return repos
585
586def action_pull(conf, args):
587 """
588 update the component repos only
589 """
590 repos = get_repos(conf, args[1:])
591
592 # make sure all repos are clean
593 for name in repos:
594 check_repo_clean(conf.repos[name]['local_repo_dir'])
595
596 for name in repos:
597 repo = conf.repos[name]
598 ldir = repo['local_repo_dir']
599 branch = repo.get('branch', "master")
600 logger.info("update branch %s of component repo %s in %s ..." % (branch, name, ldir))
601 if not conf.hard_reset:
602 # Try to pull only the configured branch. Beware that this may fail
603 # when the branch is currently unknown (for example, after reconfiguring
604 # combo-layer). In that case we need to fetch everything and try the check out
605 # and pull again.
606 try:
607 runcmd("git checkout %s" % branch, ldir, printerr=False)
608 except subprocess.CalledProcessError:
609 output=runcmd("git fetch", ldir)
610 logger.info(output)
611 runcmd("git checkout %s" % branch, ldir)
612 runcmd("git pull --ff-only", ldir)
613 else:
614 output=runcmd("git pull --ff-only", ldir)
615 logger.info(output)
616 else:
617 output=runcmd("git fetch", ldir)
618 logger.info(output)
619 runcmd("git checkout %s" % branch, ldir)
620 runcmd("git reset --hard FETCH_HEAD", ldir)
621
622def action_update(conf, args):
623 """
624 update the component repos
625 either:
626 generate the patch list
627 apply the generated patches
628 or:
629 re-creates the entire component history and merges them
630 into the current branch with a merge commit
631 """
632 components = [arg.split(':')[0] for arg in args[1:]]
633 revisions = {}
634 for arg in args[1:]:
635 if ':' in arg:
636 a = arg.split(':', 1)
637 revisions[a[0]] = a[1]
638 repos = get_repos(conf, components)
639
640 # make sure combo repo is clean
641 check_repo_clean(os.getcwd())
642
643 # Check whether we keep the component histories. Must be
644 # set either via --history command line parameter or consistently
645 # in combo-layer.conf. Mixing modes is (currently, and probably
646 # permanently because it would be complicated) not supported.
647 if conf.history:
648 history = True
649 else:
650 history = None
651 for name in repos:
652 repo = conf.repos[name]
653 repo_history = repo.get('history', False)
654 if history is None:
655 history = repo_history
656 elif history != repo_history:
657 logger.error("'history' property is set inconsistently")
658 sys.exit(1)
659
660 # Step 1: update the component repos
661 if conf.nopull:
662 logger.info("Skipping pull (-n)")
663 else:
664 action_pull(conf, ['arg0'] + components)
665
666 if history:
667 update_with_history(conf, components, revisions, repos)
668 else:
669 update_with_patches(conf, components, revisions, repos)
670
671def update_with_patches(conf, components, revisions, repos):
672 import uuid
673 patch_dir = "patch-%s" % uuid.uuid4()
674 if not os.path.exists(patch_dir):
675 os.mkdir(patch_dir)
676
677 for name in repos:
678 revision = revisions.get(name, None)
679 repo = conf.repos[name]
680 ldir = repo['local_repo_dir']
681 dest_dir = repo['dest_dir']
682 branch = repo.get('branch', "master")
683 repo_patch_dir = os.path.join(os.getcwd(), patch_dir, name)
684
685 # Step 2: generate the patch list and store to patch dir
686 logger.info("Generating patches from %s..." % name)
687 top_revision = revision or branch
688 if not check_rev_branch(name, ldir, top_revision, branch):
689 sys.exit(1)
690 if dest_dir != ".":
691 prefix = "--src-prefix=a/%s/ --dst-prefix=b/%s/" % (dest_dir, dest_dir)
692 else:
693 prefix = ""
694 if repo['last_revision'] == "":
695 logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
696 patch_cmd_range = "--root %s" % top_revision
697 rev_cmd_range = top_revision
698 else:
699 if not check_rev_branch(name, ldir, repo['last_revision'], branch):
700 sys.exit(1)
701 patch_cmd_range = "%s..%s" % (repo['last_revision'], top_revision)
702 rev_cmd_range = patch_cmd_range
703
704 file_filter = repo.get('file_filter',".")
705
706 # Filter out unwanted files
707 exclude = repo.get('file_exclude', '')
708 if exclude:
709 for path in exclude.split():
710 p = "%s/%s" % (dest_dir, path) if dest_dir != '.' else path
711 file_filter += " ':!%s'" % p
712
713 patch_cmd = "git format-patch -N %s --output-directory %s %s -- %s" % \
714 (prefix,repo_patch_dir, patch_cmd_range, file_filter)
715 output = runcmd(patch_cmd, ldir)
716 logger.debug("generated patch set:\n%s" % output)
717 patchlist = output.splitlines()
718
719 rev_cmd = "git rev-list --no-merges %s -- %s" % (rev_cmd_range, file_filter)
720 revlist = runcmd(rev_cmd, ldir).splitlines()
721
722 # Step 3: Call repo specific hook to adjust patch
723 if 'hook' in repo:
724 # hook parameter is: ./hook patchpath revision reponame
725 count=len(revlist)-1
726 for patch in patchlist:
727 runcmd("%s %s %s %s" % (repo['hook'], patch, revlist[count], name))
728 count=count-1
729
730 # Step 4: write patch list and revision list to file, for user to edit later
731 patchlist_file = os.path.join(os.getcwd(), patch_dir, "patchlist-%s" % name)
732 repo['patchlist'] = patchlist_file
733 f = open(patchlist_file, 'w')
734 count=len(revlist)-1
735 for patch in patchlist:
736 f.write("%s %s\n" % (patch, revlist[count]))
737 check_patch(os.path.join(patch_dir, patch))
738 count=count-1
739 f.close()
740
741 # Step 5: invoke bash for user to edit patch and patch list
742 if conf.interactive:
743 print('You may now edit the patch and patch list in %s\n' \
744 'For example, you can remove unwanted patch entries from patchlist-*, so that they will be not applied later' % patch_dir);
745 if not drop_to_shell(patch_dir):
746 sys.exit(1)
747
748 # Step 6: apply the generated and revised patch
749 apply_patchlist(conf, repos)
750 runcmd("rm -rf %s" % patch_dir)
751
752 # Step 7: commit the updated config file if it's being tracked
753 commit_conf_file(conf, components)
754
755def conf_commit_msg(conf, components):
756 # create the "components" string
757 component_str = "all components"
758 if len(components) > 0:
759 # otherwise tell which components were actually changed
760 component_str = ", ".join(components)
761
762 # expand the template with known values
763 template = Template(conf.commit_msg_template)
764 msg = template.substitute(components = component_str)
765 return msg
766
767def commit_conf_file(conf, components, commit=True):
768 relpath = os.path.relpath(conf.conffile)
769 try:
770 output = runcmd("git status --porcelain %s" % relpath, printerr=False)
771 except:
772 # Outside the repository
773 output = None
774 if output:
775 if output.lstrip().startswith("M"):
776 logger.info("Committing updated configuration file")
777 if commit:
778 msg = conf_commit_msg(conf, components)
779 runcmd('git commit -m'.split() + [msg, relpath])
780 else:
781 runcmd('git add %s' % relpath)
782 return True
783 return False
784
785def apply_patchlist(conf, repos):
786 """
787 apply the generated patch list to combo repo
788 """
789 for name in repos:
790 repo = conf.repos[name]
791 lastrev = repo["last_revision"]
792 prevrev = lastrev
793
794 # Get non-blank lines from patch list file
795 patchlist = []
796 if os.path.exists(repo['patchlist']) or not conf.interactive:
797 # Note: we want this to fail here if the file doesn't exist and we're not in
798 # interactive mode since the file should exist in this case
799 with open(repo['patchlist']) as f:
800 for line in f:
801 line = line.rstrip()
802 if line:
803 patchlist.append(line)
804
805 ldir = conf.repos[name]['local_repo_dir']
806 branch = conf.repos[name].get('branch', "master")
807 branchrev = runcmd("git rev-parse %s" % branch, ldir).strip()
808
809 if patchlist:
810 logger.info("Applying patches from %s..." % name)
811 linecount = len(patchlist)
812 i = 1
813 for line in patchlist:
814 patchfile = line.split()[0]
815 lastrev = line.split()[1]
816 patchdisp = os.path.relpath(patchfile)
817 if os.path.getsize(patchfile) == 0:
818 logger.info("(skipping %d/%d %s - no changes)" % (i, linecount, patchdisp))
819 else:
820 cmd = "git am --keep-cr %s-p1 %s" % ('-s ' if repo.get('signoff', True) else '', patchfile)
821 logger.info("Applying %d/%d: %s" % (i, linecount, patchdisp))
822 try:
823 runcmd(cmd)
824 except subprocess.CalledProcessError:
825 logger.info('Running "git am --abort" to cleanup repo')
826 runcmd("git am --abort")
827 logger.error('"%s" failed' % cmd)
828 logger.info("Please manually apply patch %s" % patchdisp)
829 logger.info("Note: if you exit and continue applying without manually applying the patch, it will be skipped")
830 if not drop_to_shell():
831 if prevrev != repo['last_revision']:
832 conf.update(name, "last_revision", prevrev)
833 sys.exit(1)
834 prevrev = lastrev
835 i += 1
836 # Once all patches are applied, we should update
837 # last_revision to the branch head instead of the last
838 # applied patch. The two are not necessarily the same when
839 # the last commit is a merge commit or when the patches at
840 # the branch head were intentionally excluded.
841 #
842 # If we do not do that for a merge commit, the next
843 # combo-layer run will only exclude patches reachable from
844 # one of the merged branches and try to re-apply patches
845 # from other branches even though they were already
846 # copied.
847 #
848 # If patches were intentionally excluded, the next run will
849 # present them again instead of skipping over them. This
850 # may or may not be intended, so the code here is conservative
851 # and only addresses the "head is merge commit" case.
852 if lastrev != branchrev and \
853 len(runcmd("git show --pretty=format:%%P --no-patch %s" % branch, ldir).split()) > 1:
854 lastrev = branchrev
855 else:
856 logger.info("No patches to apply from %s" % name)
857 lastrev = branchrev
858
859 if lastrev != repo['last_revision']:
860 conf.update(name, "last_revision", lastrev)
861
862def action_splitpatch(conf, args):
863 """
864 generate the commit patch and
865 split the patch per repo
866 """
867 logger.debug("action_splitpatch")
868 if len(args) > 1:
869 commit = args[1]
870 else:
871 commit = "HEAD"
872 patchdir = "splitpatch-%s" % commit
873 if not os.path.exists(patchdir):
874 os.mkdir(patchdir)
875
876 # filerange_root is for the repo whose dest_dir is root "."
877 # and it should be specified by excluding all other repo dest dir
878 # like "-x repo1 -x repo2 -x repo3 ..."
879 filerange_root = ""
880 for name in conf.repos:
881 dest_dir = conf.repos[name]['dest_dir']
882 if dest_dir != ".":
883 filerange_root = '%s -x "%s/*"' % (filerange_root, dest_dir)
884
885 for name in conf.repos:
886 dest_dir = conf.repos[name]['dest_dir']
887 patch_filename = "%s/%s.patch" % (patchdir, name)
888 if dest_dir == ".":
889 cmd = "git format-patch -n1 --stdout %s^..%s | filterdiff -p1 %s > %s" % (commit, commit, filerange_root, patch_filename)
890 else:
891 cmd = "git format-patch --no-prefix -n1 --stdout %s^..%s -- %s > %s" % (commit, commit, dest_dir, patch_filename)
892 runcmd(cmd)
893 # Detect empty patches (including those produced by filterdiff above
894 # that contain only preamble text)
895 if os.path.getsize(patch_filename) == 0 or runcmd("filterdiff %s" % patch_filename) == "":
896 os.remove(patch_filename)
897 logger.info("(skipping %s - no changes)", name)
898 else:
899 logger.info(patch_filename)
900
901def update_with_history(conf, components, revisions, repos):
902 '''Update all components with full history.
903
904 Works by importing all commits reachable from a component's
905 current head revision. If those commits are rooted in an already
906 imported commit, their content gets mixed with the content of the
907 combined repo of that commit (new or modified files overwritten,
908 removed files removed).
909
910 The last commit is an artificial merge commit that merges all the
911 updated components into the combined repository.
912
913 The HEAD ref only gets updated at the very end. All intermediate work
914 happens in a worktree which will get garbage collected by git eventually
915 after a failure.
916 '''
917 # Remember current HEAD and what we need to add to it.
918 head = runcmd("git rev-parse HEAD").strip()
919 additional_heads = {}
920
921 # Track the mapping between original commit and commit in the
922 # combined repo. We do not have to distinguish between components,
923 # because commit hashes are different anyway. Often we can
924 # skip find_revs() entirely (for example, when all new commits
925 # are derived from the last imported revision).
926 #
927 # Using "head" (typically the merge commit) instead of the actual
928 # commit for the component leads to a nicer history in the combined
929 # repo.
930 old2new_revs = {}
931 for name in repos:
932 repo = conf.repos[name]
933 revision = repo['last_revision']
934 if revision:
935 old2new_revs[revision] = head
936
937 def add_p(parents):
938 '''Insert -p before each entry.'''
939 parameters = []
940 for p in parents:
941 parameters.append('-p')
942 parameters.append(p)
943 return parameters
944
945 # Do all intermediate work with a separate work dir and index,
946 # chosen via env variables (can't use "git worktree", it is too
947 # new). This is useful (no changes to current work tree unless the
948 # update succeeds) and required (otherwise we end up temporarily
949 # removing the combo-layer hooks that we currently use when
950 # importing a new component).
951 #
952 # Not cleaned up after a failure at the moment.
953 wdir = os.path.join(os.getcwd(), ".git", "combo-layer")
954 windex = wdir + ".index"
955 if os.path.isdir(wdir):
956 shutil.rmtree(wdir)
957 os.mkdir(wdir)
958 wenv = copy.deepcopy(os.environ)
959 wenv["GIT_WORK_TREE"] = wdir
960 wenv["GIT_INDEX_FILE"] = windex
961 # This one turned out to be needed in practice.
962 wenv["GIT_OBJECT_DIRECTORY"] = os.path.join(os.getcwd(), ".git", "objects")
963 wargs = {"destdir": wdir, "env": wenv}
964
965 for name in repos:
966 revision = revisions.get(name, None)
967 repo = conf.repos[name]
968 ldir = repo['local_repo_dir']
969 dest_dir = repo['dest_dir']
970 branch = repo.get('branch', "master")
971 hook = repo.get('hook', None)
972 largs = {"destdir": ldir, "env": None}
973 file_include = repo.get('file_filter', '').split()
974 file_include.sort() # make sure that short entries like '.' come first.
975 file_exclude = repo.get('file_exclude', '').split()
976
977 def include_file(file):
978 if not file_include:
979 # No explicit filter set, include file.
980 return True
981 for filter in file_include:
982 if filter == '.':
983 # Another special case: include current directory and thus all files.
984 return True
985 if os.path.commonprefix((filter, file)) == filter:
986 # Included in directory or direct file match.
987 return True
988 # Check for wildcard match *with* allowing * to match /, i.e.
989 # src/*.c does match src/foobar/*.c. That's not how it is done elsewhere
990 # when passing the filtering to "git archive", but it is unclear what
991 # the intended semantic is (the comment on file_exclude that "append a * wildcard
992 # at the end" to match the full content of a directories implies that
993 # slashes are indeed not special), so here we simply do what's easy to
994 # implement in Python.
995 logger.debug('fnmatch(%s, %s)' % (file, filter))
996 if fnmatch.fnmatchcase(file, filter):
997 return True
998 return False
999
1000 def exclude_file(file):
1001 for filter in file_exclude:
1002 if fnmatch.fnmatchcase(file, filter):
1003 return True
1004 return False
1005
1006 def file_filter(files):
1007 '''Clean up file list so that only included files remain.'''
1008 index = 0
1009 while index < len(files):
1010 file = files[index]
1011 if not include_file(file) or exclude_file(file):
1012 del files[index]
1013 else:
1014 index += 1
1015
1016
1017 # Generate the revision list.
1018 logger.info("Analyzing commits from %s..." % name)
1019 top_revision = revision or branch
1020 if not check_rev_branch(name, ldir, top_revision, branch):
1021 sys.exit(1)
1022
1023 last_revision = repo['last_revision']
1024 rev_list_args = "--full-history --sparse --topo-order --reverse"
1025 if not last_revision:
1026 logger.info("Warning: last_revision of component %s is not set, starting from the first commit" % name)
1027 rev_list_args = rev_list_args + ' ' + top_revision
1028 else:
1029 if not check_rev_branch(name, ldir, last_revision, branch):
1030 sys.exit(1)
1031 rev_list_args = "%s %s..%s" % (rev_list_args, last_revision, top_revision)
1032
1033 # By definition, the current HEAD contains the latest imported
1034 # commit of each component. We use that as initial mapping even
1035 # though the commits do not match exactly because
1036 # a) it always works (in contrast to find_revs, which relies on special
1037 # commit messages)
1038 # b) it is faster than find_revs, which will only be called on demand
1039 # and can be skipped entirely in most cases
1040 # c) last but not least, the combined history looks nicer when all
1041 # new commits are rooted in the same merge commit
1042 old2new_revs[last_revision] = head
1043
1044 # We care about all commits (--full-history and --sparse) and
1045 # we want reconstruct the topology and thus do not care
1046 # about ordering by time (--topo-order). We ask for the ones
1047 # we need to import first to be listed first (--reverse).
1048 revs = runcmd("git rev-list %s" % rev_list_args, **largs).split()
1049 logger.debug("To be imported: %s" % revs)
1050 # Now 'revs' contains all revisions reachable from the top revision.
1051 # All revisions derived from the 'last_revision' definitely are new,
1052 # whereas the others may or may not have been imported before. For
1053 # a linear history in the component, that second set will be empty.
1054 # To distinguish between them, we also get the shorter list
1055 # of revisions starting at the ancestor.
1056 if last_revision:
1057 ancestor_revs = runcmd("git rev-list --ancestry-path %s" % rev_list_args, **largs).split()
1058 else:
1059 ancestor_revs = []
1060 logger.debug("Ancestors: %s" % ancestor_revs)
1061
1062 # Now import each revision.
1063 logger.info("Importing commits from %s..." % name)
1064 def import_rev(rev):
1065 global scanned_revs
1066
1067 # If it is part of the new commits, we definitely need
1068 # to import it. Otherwise we need to check, we might have
1069 # imported it before. If it was imported and we merely
1070 # fail to find it because commit messages did not track
1071 # the mapping, then we end up importing it again. So
1072 # combined repos using "updating with history" really should
1073 # enable the "From ... rev:" commit header modifications.
1074 if rev not in ancestor_revs and rev not in old2new_revs and not scanned_revs:
1075 logger.debug("Revision %s triggers log analysis." % rev)
1076 find_revs(old2new_revs, head)
1077 scanned_revs = True
1078 new_rev = old2new_revs.get(rev, None)
1079 if new_rev:
1080 return new_rev
1081
1082 # If the commit is not in the original list of revisions
1083 # to be imported, then it must be a parent of one of those
1084 # commits and it was skipped during earlier imports or not
1085 # found. Importing such merge commits leads to very ugly
1086 # history (long cascade of merge commits which all point
1087 # to to older commits) when switching from "update via
1088 # patches" to "update with history".
1089 #
1090 # We can avoid importing merge commits if all non-merge commits
1091 # reachable from it were already imported. In that case we
1092 # can root the new commits in the current head revision.
1093 def is_imported(prev):
1094 parents = runcmd("git show --no-patch --pretty=format:%P " + prev, **largs).split()
1095 if len(parents) > 1:
1096 for p in parents:
1097 if not is_imported(p):
1098 logger.debug("Must import %s because %s is not imported." % (rev, p))
1099 return False
1100 return True
1101 elif prev in old2new_revs:
1102 return True
1103 else:
1104 logger.debug("Must import %s because %s is not imported." % (rev, prev))
1105 return False
1106 if rev not in revs and is_imported(rev):
1107 old2new_revs[rev] = head
1108 return head
1109
1110 # Need to import rev. Collect some information about it.
1111 logger.debug("Importing %s" % rev)
1112 (parents, author_name, author_email, author_timestamp, body) = \
1113 runcmd("git show --no-patch --pretty=format:%P%x00%an%x00%ae%x00%at%x00%B " + rev, **largs).split(chr(0))
1114 parents = parents.split()
1115 if parents:
1116 # Arbitrarily pick the first parent as base. It may or may not have
1117 # been imported before. For example, if the parent is a merge commit
1118 # and previously the combined repository used patching as update
1119 # method, then the actual merge commit parent never was imported.
1120 # To cover this, We recursively import parents.
1121 parent = parents[0]
1122 new_parent = import_rev(parent)
1123 # Clean index and working tree. TODO: can we combine this and the
1124 # next into one command with less file IO?
1125 # "git reset --hard" does not work, it changes HEAD of the parent
1126 # repo, which we wanted to avoid. Probably need to keep
1127 # track of the rev that corresponds to the index and use apply_commit().
1128 runcmd("git rm -q --ignore-unmatch -rf .", **wargs)
1129 # Update index and working tree to match the parent.
1130 runcmd("git checkout -q -f %s ." % new_parent, **wargs)
1131 else:
1132 parent = None
1133 # Clean index and working tree.
1134 runcmd("git rm -q --ignore-unmatch -rf .", **wargs)
1135
1136 # Modify index and working tree such that it mirrors the commit.
1137 apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=file_filter)
1138
1139 # Now commit.
1140 new_tree = runcmd("git write-tree", **wargs).strip()
1141 env = copy.deepcopy(wenv)
1142 env['GIT_AUTHOR_NAME'] = author_name
1143 env['GIT_AUTHOR_EMAIL'] = author_email
1144 env['GIT_AUTHOR_DATE'] = author_timestamp
1145 if hook:
1146 # Need to turn the verbatim commit message into something resembling a patch header
1147 # for the hook.
1148 with tempfile.NamedTemporaryFile(mode='wt', delete=False) as patch:
1149 patch.write('Subject: [PATCH] ')
1150 patch.write(body)
1151 patch.write('\n---\n')
1152 patch.close()
1153 runcmd([hook, patch.name, rev, name])
1154 with open(patch.name) as f:
1155 body = f.read()[len('Subject: [PATCH] '):][:-len('\n---\n')]
1156
1157 # We can skip non-merge commits that did not change any files. Those are typically
1158 # the result of file filtering, although they could also have been introduced
1159 # intentionally upstream, in which case we drop some information here.
1160 if len(parents) == 1:
1161 parent_rev = import_rev(parents[0])
1162 old_tree = runcmd("git show -s --pretty=format:%T " + parent_rev, **wargs).strip()
1163 commit = old_tree != new_tree
1164 if not commit:
1165 new_rev = parent_rev
1166 else:
1167 commit = True
1168 if commit:
1169 new_rev = runcmd("git commit-tree".split() + add_p([import_rev(p) for p in parents]) +
1170 ["-m", body, new_tree],
1171 env=env).strip()
1172 old2new_revs[rev] = new_rev
1173
1174 return new_rev
1175
1176 if revs:
1177 for rev in revs:
1178 import_rev(rev)
1179 # Remember how to update our current head. New components get added,
1180 # updated components get the delta between current head and the updated component
1181 # applied.
1182 additional_heads[old2new_revs[revs[-1]]] = head if repo['last_revision'] else None
1183 repo['last_revision'] = revs[-1]
1184
1185 # Now construct the final merge commit. We create the tree by
1186 # starting with the head and applying the changes from each
1187 # components imported head revision.
1188 if additional_heads:
1189 runcmd("git reset --hard", **wargs)
1190 for rev, base in additional_heads.items():
1191 apply_commit(base, rev, wargs, wargs, None)
1192
1193 # Commit with all component branches as parents as well as the previous head.
1194 logger.info("Writing final merge commit...")
1195 msg = conf_commit_msg(conf, components)
1196 new_tree = runcmd("git write-tree", **wargs).strip()
1197 new_rev = runcmd("git commit-tree".split() +
1198 add_p([head] + list(additional_heads.keys())) +
1199 ["-m", msg, new_tree],
1200 **wargs).strip()
1201 # And done! This is the first time we change the HEAD in the actual work tree.
1202 runcmd("git reset --hard %s" % new_rev)
1203
1204 # Update and stage the (potentially modified)
1205 # combo-layer.conf, but do not commit separately.
1206 for name in repos:
1207 repo = conf.repos[name]
1208 rev = repo['last_revision']
1209 conf.update(name, "last_revision", rev)
1210 if commit_conf_file(conf, components, False):
1211 # Must augment the previous commit.
1212 runcmd("git commit --amend -C HEAD")
1213
1214
1215scanned_revs = False
1216def find_revs(old2new, head):
1217 '''Construct mapping from original commit hash to commit hash in
1218 combined repo by looking at the commit messages. Depends on the
1219 "From ... rev: ..." convention.'''
1220 logger.info("Analyzing log messages to find previously imported commits...")
1221 num_known = len(old2new)
1222 log = runcmd("git log --grep='From .* rev: [a-fA-F0-9][a-fA-F0-9]*' --pretty=format:%H%x00%B%x00 " + head).split(chr(0))
1223 regex = re.compile(r'From .* rev: ([a-fA-F0-9]+)')
1224 for new_rev, body in zip(*[iter(log)]* 2):
1225 # Use the last one, in the unlikely case there are more than one.
1226 rev = regex.findall(body)[-1]
1227 if rev not in old2new:
1228 old2new[rev] = new_rev.strip()
1229 logger.info("Found %d additional commits, leading to: %s" % (len(old2new) - num_known, old2new))
1230
1231
1232def apply_commit(parent, rev, largs, wargs, dest_dir, file_filter=None):
1233 '''Compare revision against parent, remove files deleted in the
1234 commit, re-write new or modified ones. Moves them into dest_dir.
1235 Optionally filters files.
1236 '''
1237 if not dest_dir:
1238 dest_dir = "."
1239 # -r recurses into sub-directories, given is the full overview of
1240 # what changed. We do not care about copy/edits or renames, so we
1241 # can disable those with --no-renames (but we still parse them,
1242 # because it was not clear from git documentation whether C and M
1243 # lines can still occur).
1244 logger.debug("Applying changes between %s and %s in %s" % (parent, rev, largs["destdir"]))
1245 delete = []
1246 update = []
1247 if parent:
1248 # Apply delta.
1249 changes = runcmd("git diff-tree --no-commit-id --no-renames --name-status -r --raw -z %s %s" % (parent, rev), **largs).split(chr(0))
1250 for status, name in zip(*[iter(changes)]*2):
1251 if status[0] in "ACMRT":
1252 update.append(name)
1253 elif status[0] in "D":
1254 delete.append(name)
1255 else:
1256 logger.error("Unknown status %s of file %s in revision %s" % (status, name, rev))
1257 sys.exit(1)
1258 else:
1259 # Copy all files.
1260 update.extend(runcmd("git ls-tree -r --name-only -z %s" % rev, **largs).split(chr(0)))
1261
1262 # Include/exclude files as define in the component config.
1263 # Both updated and deleted file lists get filtered, because it might happen
1264 # that a file gets excluded, pulled from a different component, and then the
1265 # excluded file gets deleted. In that case we must keep the copy.
1266 if file_filter:
1267 file_filter(update)
1268 file_filter(delete)
1269
1270 # We export into a tar archive here and extract with tar because it is simple (no
1271 # need to implement file and symlink writing ourselves) and gives us some degree
1272 # of parallel IO. The downside is that we have to pass the list of files via
1273 # command line parameters - hopefully there will never be too many at once.
1274 if update:
1275 target = os.path.join(wargs["destdir"], dest_dir)
1276 if not os.path.isdir(target):
1277 os.makedirs(target)
1278 quoted_target = shlex.quote(target)
1279 # os.sysconf('SC_ARG_MAX') is lying: running a command with
1280 # string length 629343 already failed with "Argument list too
1281 # long" although SC_ARG_MAX = 2097152. "man execve" explains
1282 # the limitations, but those are pretty complicated. So here
1283 # we just hard-code a fixed value which is more likely to work.
1284 max_cmdsize = 64 * 1024
1285 while update:
1286 quoted_args = []
1287 unquoted_args = []
1288 cmdsize = 100 + len(quoted_target)
1289 while update:
1290 quoted_next = shlex.quote(update[0])
1291 size_next = len(quoted_next) + len(dest_dir) + 1
1292 logger.debug('cmdline length %d + %d < %d?' % (cmdsize, size_next, os.sysconf('SC_ARG_MAX')))
1293 if cmdsize + size_next < max_cmdsize:
1294 quoted_args.append(quoted_next)
1295 unquoted_args.append(update.pop(0))
1296 cmdsize += size_next
1297 else:
1298 logger.debug('Breaking the cmdline at length %d' % cmdsize)
1299 break
1300 logger.debug('Final cmdline length %d / %d' % (cmdsize, os.sysconf('SC_ARG_MAX')))
1301 cmd = "git archive %s %s | tar -C %s -xf -" % (rev, ' '.join(quoted_args), quoted_target)
1302 logger.debug('First cmdline length %d' % len(cmd))
1303 runcmd(cmd, **largs)
1304 cmd = "git add -f".split() + [os.path.join(dest_dir, x) for x in unquoted_args]
1305 logger.debug('Second cmdline length %d' % reduce(lambda x, y: x + len(y), cmd, 0))
1306 runcmd(cmd, **wargs)
1307 if delete:
1308 for path in delete:
1309 if dest_dir:
1310 path = os.path.join(dest_dir, path)
1311 runcmd("git rm -f --ignore-unmatch".split() + [os.path.join(dest_dir, x) for x in delete], **wargs)
1312
1313def action_error(conf, args):
1314 logger.info("invalid action %s" % args[0])
1315
1316actions = {
1317 "init": action_init,
1318 "update": action_update,
1319 "pull": action_pull,
1320 "splitpatch": action_splitpatch,
1321 "sync-revs": action_sync_revs,
1322}
1323
1324def main():
1325 parser = optparse.OptionParser(
1326 version = "Combo Layer Repo Tool version %s" % __version__,
1327 usage = """%prog [options] action
1328
1329Create and update a combination layer repository from multiple component repositories.
1330
1331Action:
1332 init initialise the combo layer repo
1333 update [components] get patches from component repos and apply them to the combo repo
1334 pull [components] just pull component repos only
1335 sync-revs [components] update the config file's last_revision for each repository
1336 splitpatch [commit] generate commit patch and split per component, default commit is HEAD""")
1337
1338 parser.add_option("-c", "--conf", help = "specify the config file (conf/combo-layer.conf is the default).",
1339 action = "store", dest = "conffile", default = "conf/combo-layer.conf")
1340
1341 parser.add_option("-i", "--interactive", help = "interactive mode, user can edit the patch list and patches",
1342 action = "store_true", dest = "interactive", default = False)
1343
1344 parser.add_option("-D", "--debug", help = "output debug information",
1345 action = "store_true", dest = "debug", default = False)
1346
1347 parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update",
1348 action = "store_true", dest = "nopull", default = False)
1349
1350 parser.add_option("--hard-reset",
1351 help = "instead of pull do fetch and hard-reset in component repos",
1352 action = "store_true", dest = "hard_reset", default = False)
1353
1354 parser.add_option("-H", "--history", help = "import full history of components during init",
1355 action = "store_true", default = False)
1356
1357 options, args = parser.parse_args(sys.argv)
1358
1359 # Dispatch to action handler
1360 if len(args) == 1:
1361 logger.error("No action specified, exiting")
1362 parser.print_help()
1363 elif args[1] not in actions:
1364 logger.error("Unsupported action %s, exiting\n" % (args[1]))
1365 parser.print_help()
1366 elif not os.path.exists(options.conffile):
1367 logger.error("No valid config file, exiting\n")
1368 parser.print_help()
1369 else:
1370 if options.debug:
1371 logger.setLevel(logging.DEBUG)
1372 confdata = Configuration(options)
1373 initmode = (args[1] == 'init')
1374 confdata.sanity_check(initmode)
1375 actions.get(args[1], action_error)(confdata, args[1:])
1376
1377if __name__ == "__main__":
1378 try:
1379 ret = main()
1380 except Exception:
1381 ret = 1
1382 import traceback
1383 traceback.print_exc()
1384 sys.exit(ret)
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2025-12-17 19:39:46 +0000