2 files changed, 153 insertions, 26 deletions

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index d55d504a14..26187a0c41 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -35,6 +35,8 @@ from devtool import parse_recipe
 
 logger = logging.getLogger('devtool')
 
+override_branch_prefix = 'devtool-override-'
+
 
 def add(args, config, basepath, workspace):
     """Entry point for the devtool 'add' subcommand"""
@@ -425,7 +427,7 @@ def extract(args, config, basepath, workspace):
     """Entry point for the devtool 'extract' subcommand"""
     import bb
 
-    tinfoil = setup_tinfoil(basepath=basepath)
+    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
     if not tinfoil:
         # Error already shown
         return 1
@@ -435,7 +437,7 @@ def extract(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
         logger.info('Source tree extracted to %s' % srctree)
 
         if initial_rev:
@@ -449,7 +451,7 @@ def sync(args, config, basepath, workspace):
     """Entry point for the devtool 'sync' subcommand"""
     import bb
 
-    tinfoil = setup_tinfoil(basepath=basepath)
+    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
     if not tinfoil:
         # Error already shown
         return 1
@@ -459,7 +461,7 @@ def sync(args, config, basepath, workspace):
             return 1
 
         srctree = os.path.abspath(args.srctree)
-        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+        initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True)
         logger.info('Source tree %s synchronized' % srctree)
 
         if initial_rev:
@@ -470,7 +472,7 @@ def sync(args, config, basepath, workspace):
         tinfoil.shutdown()
 
 
-def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil):
+def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
     """Extract sources of a recipe"""
     import oe.recipeutils
     import oe.patch
@@ -500,6 +502,16 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
         bb.utils.mkdirhier(srctree)
         os.rmdir(srctree)
 
+    extra_overrides = []
+    if not no_overrides:
+        history = d.varhistory.variable('SRC_URI')
+        for event in history:
+            if not 'flag' in event:
+                if event['op'].startswith(('_append[', '_prepend[')):
+                    extra_overrides.append(event['op'].split('[')[1].split(']')[0])
+        if extra_overrides:
+            logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these')
+
     initial_rev = None
 
     appendexisted = False
@@ -542,6 +554,8 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
             if not is_kernel_yocto:
                 f.write('PATCHTOOL = "git"\n')
                 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n')
+            if extra_overrides:
+                f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides))
             f.write('inherit devtool-source\n')
             f.write('###--- _extract_source\n')
 
@@ -702,12 +716,13 @@ def modify(args, config, basepath, workspace):
     """Entry point for the devtool 'modify' subcommand"""
     import bb
     import oe.recipeutils
+    import oe.patch
 
     if args.recipename in workspace:
         raise DevtoolError("recipe %s is already in your workspace" %
                            args.recipename)
 
-    tinfoil = setup_tinfoil(basepath=basepath)
+    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
     try:
         rd = parse_recipe(config, tinfoil, args.recipename, True)
         if not rd:
@@ -742,14 +757,16 @@ def modify(args, config, basepath, workspace):
 
         initial_rev = None
         commits = []
+        check_commits = False
         if not args.no_extract:
-            initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+            initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
             if not initial_rev:
                 return 1
             logger.info('Source tree extracted to %s' % srctree)
             # Get list of commits since this revision
             (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
             commits = stdout.split()
+            check_commits = True
         else:
             if os.path.exists(os.path.join(srctree, '.git')):
                 # Check if it's a tree previously extracted by us
@@ -757,6 +774,8 @@ def modify(args, config, basepath, workspace):
                     (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
                 except bb.process.ExecutionError:
                     stdout = ''
+                if stdout:
+                    check_commits = True
                 for line in stdout.splitlines():
                     if line.startswith('*'):
                         (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
@@ -766,6 +785,30 @@ def modify(args, config, basepath, workspace):
                     (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
                     initial_rev = stdout.rstrip()
 
+        branch_patches = {}
+        if check_commits:
+            # Check if there are override branches
+            (stdout, _) = bb.process.run('git branch', cwd=srctree)
+            branches = []
+            for line in stdout.rstrip().splitlines():
+                branchname = line[2:].rstrip()
+                if branchname.startswith(override_branch_prefix):
+                    branches.append(branchname)
+            if branches:
+                logger.warn('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix))
+            branches.insert(0, args.branch)
+            seen_patches = []
+            for branch in branches:
+                branch_patches[branch] = []
+                (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree)
+                for line in stdout.splitlines():
+                    line = line.strip()
+                    if line.startswith(oe.patch.GitApplyTree.patch_line_prefix):
+                        origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip()
+                        if not origpatch in seen_patches:
+                            seen_patches.append(origpatch)
+                            branch_patches[branch].append(origpatch)
+
         # Need to grab this here in case the source is within a subdirectory
         srctreebase = srctree
 
@@ -808,6 +851,11 @@ def modify(args, config, basepath, workspace):
                 f.write('\n# initial_rev: %s\n' % initial_rev)
                 for commit in commits:
                     f.write('# commit: %s\n' % commit)
+            if branch_patches:
+                for branch in branch_patches:
+                    if branch == args.branch:
+                        continue
+                    f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch])))
 
         update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
 
@@ -1019,8 +1067,14 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
     """
     import bb
 
+    # Get current branch
+    stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
+                               cwd=srctree)
+    branchname = stdout.rstrip()
+
     # Parse initial rev from recipe if not specified
     commits = []
+    patches = []
     with open(recipe_path, 'r') as f:
         for line in f:
             if line.startswith('# initial_rev:'):
@@ -1028,6 +1082,8 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
                     initial_rev = line.split(':')[-1].strip()
             elif line.startswith('# commit:'):
                 commits.append(line.split(':')[-1].strip())
+            elif line.startswith('# patches_%s:' % branchname):
+                patches = line.split(':')[-1].strip().split(',')
 
     update_rev = initial_rev
     changed_revs = None
@@ -1054,7 +1110,7 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None):
                     if rev in newcommits:
                         changed_revs.append(rev)
 
-    return initial_rev, update_rev, changed_revs
+    return initial_rev, update_rev, changed_revs, patches
 
 def _remove_file_entries(srcuri, filelist):
     """Remove file:// entries from SRC_URI"""
@@ -1126,6 +1182,7 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
 
     existing_patches = dict((os.path.basename(path), path) for path in
                             oe.recipeutils.get_recipe_patches(rd))
+    logger.debug('Existing patches: %s' % existing_patches)
 
     # Generate patches from Git, exclude local files directory
     patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
@@ -1414,7 +1471,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
         raise DevtoolError('unable to find workspace bbappend for recipe %s' %
                            recipename)
 
-    initial_rev, update_rev, changed_revs = _get_patchset_revs(srctree, append, initial_rev)
+    initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev)
     if not initial_rev:
         raise DevtoolError('Unable to find initial revision - please specify '
                            'it with --initial-rev')
@@ -1429,22 +1486,32 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)
-        srctreebase = workspace[recipename]['srctreebase']
-        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
+        if filter_patches:
+            upd_f = {}
+            new_f = {}
+            del_f = {}
+        else:
+            srctreebase = workspace[recipename]['srctreebase']
+            upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
 
         remove_files = []
         if not no_remove:
             # Get all patches from source tree and check if any should be removed
             all_patches_dir = tempfile.mkdtemp(dir=tempdir)
-            upd_p, new_p, del_p = _export_patches(srctree, rd, initial_rev,
-                                                  all_patches_dir)
+            _, _, del_p = _export_patches(srctree, rd, initial_rev,
+                                          all_patches_dir)
             # Remove deleted local files and  patches
             remove_files = list(del_f.values()) + list(del_p.values())
 
         # Get updated patches from source tree
         patches_dir = tempfile.mkdtemp(dir=tempdir)
-        upd_p, new_p, del_p = _export_patches(srctree, rd, update_rev,
-                                              patches_dir, changed_revs)
+        upd_p, new_p, _ = _export_patches(srctree, rd, update_rev,
+                                          patches_dir, changed_revs)
+        logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
+        if filter_patches:
+            new_p = {}
+            upd_p = {k:v for k,v in upd_p.items() if k in filter_patches}
+            remove_files = [f for f in remove_files if f in filter_patches]
         updatefiles = False
         updaterecipe = False
         destpath = None
@@ -1556,18 +1623,73 @@ def _guess_recipe_update_mode(srctree, rdata):
 
     return 'patch'
 
-def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None):
+def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False):
     srctree = workspace[recipename]['srctree']
     if mode == 'auto':
         mode = _guess_recipe_update_mode(srctree, rd)
 
-    if mode == 'srcrev':
-        updated, appendfile, removed = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
-    elif mode == 'patch':
-        updated, appendfile, removed = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir)
-    else:
-        raise DevtoolError('update_recipe: invalid mode %s' % mode)
-    return updated, appendfile, removed
+    override_branches = []
+    mainbranch = None
+    startbranch = None
+    if not no_overrides:
+        stdout, _ = bb.process.run('git branch', cwd=srctree)
+        other_branches = []
+        for line in stdout.splitlines():
+            branchname = line[2:]
+            if line.startswith('* '):
+                startbranch = branchname
+            if branchname.startswith(override_branch_prefix):
+                override_branches.append(branchname)
+            else:
+                other_branches.append(branchname)
+
+        if override_branches:
+            logger.debug('_update_recipe: override branches: %s' % override_branches)
+            logger.debug('_update_recipe: other branches: %s' % other_branches)
+            if startbranch.startswith(override_branch_prefix):
+                if len(other_branches) == 1:
+                    mainbranch = other_branches[1]
+                else:
+                    raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first')
+            else:
+                mainbranch = startbranch
+
+    checkedout = None
+    anyupdated = False
+    appendfile = None
+    allremoved = []
+    if override_branches:
+        logger.info('Handling main branch (%s)...' % mainbranch)
+        if startbranch != mainbranch:
+            bb.process.run('git checkout %s' % mainbranch, cwd=srctree)
+        checkedout = mainbranch
+    try:
+        branchlist = [mainbranch] + override_branches
+        for branch in branchlist:
+            crd = bb.data.createCopy(rd)
+            if branch != mainbranch:
+                logger.info('Handling branch %s...' % branch)
+                override = branch[len(override_branch_prefix):]
+                crd.appendVar('OVERRIDES', ':%s' % override)
+                bb.process.run('git checkout %s' % branch, cwd=srctree)
+                checkedout = branch
+
+            if mode == 'srcrev':
+                updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
+            elif mode == 'patch':
+                updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir)
+            else:
+                raise DevtoolError('update_recipe: invalid mode %s' % mode)
+            if updated:
+                anyupdated = True
+            if appendf:
+                appendfile = appendf
+            allremoved.extend(removed)
+    finally:
+        if startbranch and checkedout != startbranch:
+            bb.process.run('git checkout %s' % startbranch, cwd=srctree)
+
+    return anyupdated, appendfile, allremoved
 
 def update_recipe(args, config, basepath, workspace):
     """Entry point for the devtool 'update-recipe' subcommand"""
@@ -1593,7 +1715,7 @@ def update_recipe(args, config, basepath, workspace):
         if args.dry_run:
             dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
             dry_run_outdir = dry_run_output.name
-        updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir)
+        updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides)
 
         if updated:
             rf = rd.getVar('FILE')
@@ -1816,7 +1938,7 @@ def finish(args, config, basepath, workspace):
         if args.dry_run:
             dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
             dry_run_outdir = dry_run_output.name
-        updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir)
+        updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides)
         removed = [os.path.relpath(pth, recipedir) for pth in removed]
 
         # Remove any old files in the case of an upgrade
@@ -1959,6 +2081,7 @@ def register_commands(subparsers, context):
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
     parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
+    parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
     parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
     parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup)
 
@@ -1968,6 +2091,7 @@ def register_commands(subparsers, context):
     parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
     parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
     parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
+    parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
     parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
     parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup)
 
@@ -1999,6 +2123,7 @@ def register_commands(subparsers, context):
     parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
     parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
     parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
+    parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
     parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
     parser_update_recipe.set_defaults(func=update_recipe)
 
@@ -2023,5 +2148,6 @@ def register_commands(subparsers, context):
     parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
     parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
     parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
+    parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
     parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
     parser_finish.set_defaults(func=finish)
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 3cb523c2f4..f6141bfdc3 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -474,7 +474,7 @@ def upgrade(args, config, basepath, workspace):
         rf = None
         try:
             logger.info('Extracting current version source...')
-            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil)
+            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
             logger.info('Extracting upgraded version source...')
             rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
                                                     args.srcrev, args.srcbranch, args.branch, args.keep_temp,
@@ -513,6 +513,7 @@ def register_commands(subparsers, context):
     parser_upgrade.add_argument('--srcbranch', '-B', help='Branch in source repository containing the revision to use (if fetching from an SCM such as git)')
     parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
     parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
+    parser_upgrade.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
     group = parser_upgrade.add_mutually_exclusive_group()
     group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
     group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")