summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xscripts/combo-layer187
1 files changed, 174 insertions, 13 deletions
diff --git a/scripts/combo-layer b/scripts/combo-layer
index 8ed9be8f37..d11274e245 100755
--- a/scripts/combo-layer
+++ b/scripts/combo-layer
@@ -25,6 +25,7 @@ import os, sys
25import optparse 25import optparse
26import logging 26import logging
27import subprocess 27import subprocess
28import tempfile
28import ConfigParser 29import ConfigParser
29import re 30import re
30from collections import OrderedDict 31from collections import OrderedDict
@@ -190,6 +191,11 @@ def action_init(conf, args):
190 subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True) 191 subprocess.check_call("git clone %s %s" % (conf.repos[name]['src_uri'], ldir), shell=True)
191 if not os.path.exists(".git"): 192 if not os.path.exists(".git"):
192 runcmd("git init") 193 runcmd("git init")
194 if conf.history:
195 # Need a common ref for all trees.
196 runcmd('git commit -m "initial empty commit" --allow-empty')
197 startrev = runcmd('git rev-parse master').strip()
198
193 for name in conf.repos: 199 for name in conf.repos:
194 repo = conf.repos[name] 200 repo = conf.repos[name]
195 ldir = repo['local_repo_dir'] 201 ldir = repo['local_repo_dir']
@@ -205,6 +211,25 @@ def action_init(conf, args):
205 lastrev = None 211 lastrev = None
206 initialrev = branch 212 initialrev = branch
207 logger.info("Copying data from %s..." % name) 213 logger.info("Copying data from %s..." % name)
214 # Sanity check initialrev and turn it into hash (required for copying history,
215 # because resolving a name ref only works in the component repo).
216 rev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
217 if rev != initialrev:
218 try:
219 refs = runcmd('git show-ref -s %s' % initialrev, ldir).split('\n')
220 if len(set(refs)) > 1:
221 # Happens for example when configured to track
222 # "master" and there is a refs/heads/master. The
223 # traditional behavior from "git archive" (preserved
224 # here) it to choose the first one. This might not be
225 # intended, so at least warn about it.
226 logger.warn("%s: initial revision '%s' not unique, picking result of rev-parse = %s" %
227 (name, initialrev, refs[0]))
228 initialrev = rev
229 except:
230 # show-ref fails for hashes. Skip the sanity warning in that case.
231 pass
232 initialrev = rev
208 dest_dir = repo['dest_dir'] 233 dest_dir = repo['dest_dir']
209 if dest_dir and dest_dir != ".": 234 if dest_dir and dest_dir != ".":
210 extract_dir = os.path.join(os.getcwd(), dest_dir) 235 extract_dir = os.path.join(os.getcwd(), dest_dir)
@@ -213,22 +238,155 @@ def action_init(conf, args):
213 else: 238 else:
214 extract_dir = os.getcwd() 239 extract_dir = os.getcwd()
215 file_filter = repo.get('file_filter', "") 240 file_filter = repo.get('file_filter', "")
216 files = runcmd("git archive %s | tar -x -v -C %s %s" % (initialrev, extract_dir, file_filter), ldir)
217 exclude_patterns = repo.get('file_exclude', '').split() 241 exclude_patterns = repo.get('file_exclude', '').split()
218 if exclude_patterns: 242 def copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir,
219 # Implement file removal by letting tar create the 243 subdir=""):
220 # file and then deleting it in the file system 244 # When working inside a filtered branch which had the
221 # again. Uses the list of files created by tar (easier 245 # files already moved, we need to prepend the
222 # than walking the tree). 246 # subdirectory to all filters, otherwise they would
223 for file in files.split('\n'): 247 # not match.
224 for pattern in exclude_patterns: 248 if subdir:
225 if fnmatch.fnmatch(file, pattern): 249 file_filter = ' '.join([subdir + '/' + x for x in file_filter.split()])
226 os.unlink(os.path.join(extract_dir, file)) 250 exclude_patterns = [subdir + '/' + x for x in exclude_patterns]
227 break 251 # To handle both cases, we cd into the target
252 # directory and optionally tell tar to strip the path
253 # prefix when the files were already moved.
254 subdir_components = len(os.path.normpath(subdir).split(os.path.sep)) if subdir else 0
255 strip=('--strip-components=%d' % subdir_components) if subdir else ''
256 # TODO: file_filter wild cards do not work (and haven't worked before either), because
257 # a) GNU tar requires a --wildcards parameter before turning on wild card matching.
258 # b) The semantic is not as intendend (src/*.c also matches src/foo/bar.c,
259 # in contrast to the other use of file_filter as parameter of "git archive"
260 # where it only matches .c files directly in src).
261 files = runcmd("git archive %s %s | tar -x -v %s -C %s %s" %
262 (initialrev, subdir,
263 strip, extract_dir, file_filter),
264 ldir)
265 if exclude_patterns:
266 # Implement file removal by letting tar create the
267 # file and then deleting it in the file system
268 # again. Uses the list of files created by tar (easier
269 # than walking the tree).
270 for file in files.split('\n'):
271 for pattern in exclude_patterns:
272 if fnmatch.fnmatch(file, pattern):
273 os.unlink(os.path.join(*([extract_dir] + ['..'] * subdir_components + [file])))
274 break
275
276 if not conf.history:
277 copy_selected_files(initialrev, extract_dir, file_filter, exclude_patterns, ldir)
278 else:
279 # First fetch remote history into local repository.
280 # We need a ref for that, so ensure that there is one.
281 refname = "combo-layer-init-%s" % name
282 runcmd("git branch -f %s %s" % (refname, initialrev), ldir)
283 runcmd("git fetch %s %s" % (ldir, refname))
284 runcmd("git branch -D %s" % refname, ldir)
285 # Make that the head revision.
286 runcmd("git checkout -b %s %s" % (name, initialrev))
287 # Optional: rewrite history to change commit messages or to move files.
288 if 'hook' in repo or dest_dir and dest_dir != ".":
289 filter_branch = ['git', 'filter-branch', '--force']
290 with tempfile.NamedTemporaryFile() as hookwrapper:
291 if 'hook' in repo:
292 # Create a shell script wrapper around the original hook that
293 # can be used by git filter-branch. Hook may or may not have
294 # an absolute path.
295 hook = repo['hook']
296 hook = os.path.join(os.path.dirname(conf.conffile), '..', hook)
297 # The wrappers turns the commit message
298 # from stdin into a fake patch header.
299 # This is good enough for changing Subject
300 # and commit msg body with normal
301 # combo-layer hooks.
302 hookwrapper.write('''set -e
303tmpname=$(mktemp)
304trap "rm $tmpname" EXIT
305echo -n 'Subject: [PATCH] ' >>$tmpname
306cat >>$tmpname
307if ! [ $(tail -c 1 $tmpname | od -A n -t x1) == '0a' ]; then
308 echo >>$tmpname
309fi
310echo '---' >>$tmpname
311%s $tmpname $GIT_COMMIT %s
312tail -c +18 $tmpname | head -c -4
313''' % (hook, name))
314 hookwrapper.flush()
315 filter_branch.extend(['--msg-filter', 'bash %s' % hookwrapper.name])
316 if dest_dir and dest_dir != ".":
317 parent = os.path.dirname(dest_dir)
318 if not parent:
319 parent = '.'
320 # May run outside of the current directory, so do not assume that .git exists.
321 filter_branch.extend(['--tree-filter', 'mkdir -p .git/tmptree && mv $(ls -1 -a | grep -v -e ^.git$ -e ^.$ -e ^..$) .git/tmptree && mkdir -p %s && mv .git/tmptree %s' % (parent, dest_dir)])
322 filter_branch.append('HEAD')
323 runcmd(filter_branch)
324 runcmd('git update-ref -d refs/original/refs/heads/%s' % name)
325 repo['rewritten_revision'] = runcmd('git rev-parse HEAD').strip()
326 repo['stripped_revision'] = repo['rewritten_revision']
327 # Optional filter files: remove everything and re-populate using the normal filtering code.
328 # Override any potential .gitignore.
329 if file_filter or exclude_patterns:
330 runcmd('git rm -rf .')
331 if not os.path.exists(extract_dir):
332 os.makedirs(extract_dir)
333 copy_selected_files('HEAD', extract_dir, file_filter, exclude_patterns, '.',
334 subdir=dest_dir if dest_dir and dest_dir != '.' else '')
335 runcmd('git add --all --force .')
336 if runcmd('git status --porcelain'):
337 # Something to commit.
338 runcmd(['git', 'commit', '-m',
339 '''%s: select file subset
340
341Files from the component repository were chosen based on
342the following filters:
343file_filter = %s
344file_exclude = %s''' % (name, file_filter or '<empty>', repo.get('file_exclude', '<empty>'))])
345 repo['stripped_revision'] = runcmd('git rev-parse HEAD').strip()
346
228 if not lastrev: 347 if not lastrev:
229 lastrev = runcmd("git rev-parse %s" % initialrev, ldir).strip() 348 lastrev = runcmd('git rev-parse %s' % initialrev, ldir).strip()
230 conf.update(name, "last_revision", lastrev, initmode=True) 349 conf.update(name, "last_revision", lastrev, initmode=True)
231 runcmd("git add .") 350
351 if not conf.history:
352 runcmd("git add .")
353 else:
354 # Create Octopus merge commit according to http://stackoverflow.com/questions/10874149/git-octopus-merge-with-unrelated-repositoies
355 runcmd('git checkout master')
356 merge = ['git', 'merge', '--no-commit']
357 with open('.git/info/grafts', 'w') as grafts:
358 grafts.write('%s\n' % startrev)
359 for name in conf.repos:
360 repo = conf.repos[name]
361 # Use branch created earlier.
362 merge.append(name)
363 for start in runcmd('git log --pretty=format:%%H --max-parents=0 %s' % name).split('\n'):
364 grafts.write('%s %s\n' % (start, startrev))
365 try:
366 runcmd(merge)
367 except Exception, error:
368 logger.info('''Merging component repository history failed, perhaps because of merge conflicts.
369It may be possible to commit anyway after resolving these conflicts.
370
371%s''' % error)
372 # Create MERGE_HEAD and MERGE_MSG. "git merge" itself
373 # does not create MERGE_HEAD in case of a (harmless) failure,
374 # and we want certain auto-generated information in the
375 # commit message for future reference and/or automation.
376 with open('.git/MERGE_HEAD', 'w') as head:
377 with open('.git/MERGE_MSG', 'w') as msg:
378 msg.write('repo: initial import of components\n\n')
379 # head.write('%s\n' % startrev)
380 for name in conf.repos:
381 repo = conf.repos[name]
382 # <upstream ref> <rewritten ref> <rewritten + files removed>
383 msg.write('combo-layer-%s: %s %s %s\n' % (name,
384 repo['last_revision'],
385 repo['rewritten_revision'],
386 repo['stripped_revision']))
387 rev = runcmd('git rev-parse %s' % name).strip()
388 head.write('%s\n' % rev)
389
232 if conf.localconffile: 390 if conf.localconffile:
233 localadded = True 391 localadded = True
234 try: 392 try:
@@ -631,6 +789,9 @@ Action:
631 parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update", 789 parser.add_option("-n", "--no-pull", help = "skip pulling component repos during update",
632 action = "store_true", dest = "nopull", default = False) 790 action = "store_true", dest = "nopull", default = False)
633 791
792 parser.add_option("-H", "--history", help = "import full history of components during init",
793 action = "store_true", default = False)
794
634 options, args = parser.parse_args(sys.argv) 795 options, args = parser.parse_args(sys.argv)
635 796
636 # Dispatch to action handler 797 # Dispatch to action handler