1 files changed, 199 insertions, 71 deletions

diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 9d9031c2b3..d55d504a14 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -341,19 +341,45 @@ def _check_compatible_recipe(pn, d):
                            "from working. You will need to disable this "
                            "first." % pn)
 
-def _move_file(src, dst):
-    """Move a file. Creates all the directory components of destination path."""
+def _dry_run_copy(src, dst, dry_run_outdir, base_outdir):
+    """Common function for copying a file to the dry run output directory"""
+    relpath = os.path.relpath(dst, base_outdir)
+    if relpath.startswith('..'):
+        raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst))
+    dst = os.path.join(dry_run_outdir, relpath)
     dst_d = os.path.dirname(dst)
     if dst_d:
         bb.utils.mkdirhier(dst_d)
-    shutil.move(src, dst)
+    # Don't overwrite existing files, otherwise in the case of an upgrade
+    # the dry-run written out recipe will be overwritten with an unmodified
+    # version
+    if not os.path.exists(dst):
+        shutil.copy(src, dst)
+
+def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
+    """Move a file. Creates all the directory components of destination path."""
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+    logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix))
+    if dry_run_outdir:
+        # We want to copy here, not move
+        _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
+    else:
+        dst_d = os.path.dirname(dst)
+        if dst_d:
+            bb.utils.mkdirhier(dst_d)
+        shutil.move(src, dst)
 
-def _copy_file(src, dst):
+def _copy_file(src, dst, dry_run_outdir=None):
     """Copy a file. Creates all the directory components of destination path."""
-    dst_d = os.path.dirname(dst)
-    if dst_d:
-        bb.utils.mkdirhier(dst_d)
-    shutil.copy(src, dst)
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+    logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
+    if dry_run_outdir:
+        _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
+    else:
+        dst_d = os.path.dirname(dst)
+        if dst_d:
+            bb.utils.mkdirhier(dst_d)
+        shutil.copy(src, dst)
 
 def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
     """List contents of a git treeish"""
@@ -1054,8 +1080,11 @@ def _replace_srcuri_entry(srcuri, filename, newentry):
             srcuri.insert(i, newentry)
             break
 
-def _remove_source_files(append, files, destpath, no_report_remove=False):
+def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False):
     """Unlink existing patch files"""
+
+    dry_run_suffix = ' (dry-run)' if dry_run else ''
+
     for path in files:
         if append:
             if not destpath:
@@ -1064,19 +1093,20 @@ def _remove_source_files(append, files, destpath, no_report_remove=False):
 
         if os.path.exists(path):
             if not no_report_remove:
-                logger.info('Removing file %s' % path)
-            # FIXME "git rm" here would be nice if the file in question is
-            #       tracked
-            # FIXME there's a chance that this file is referred to by
-            #       another recipe, in which case deleting wouldn't be the
-            #       right thing to do
-            os.remove(path)
-            # Remove directory if empty
-            try:
-                os.rmdir(os.path.dirname(path))
-            except OSError as ose:
-                if ose.errno != errno.ENOTEMPTY:
-                    raise
+                logger.info('Removing file %s%s' % (path, dry_run_suffix))
+            if not dry_run:
+                # FIXME "git rm" here would be nice if the file in question is
+                #       tracked
+                # FIXME there's a chance that this file is referred to by
+                #       another recipe, in which case deleting wouldn't be the
+                #       right thing to do
+                os.remove(path)
+                # Remove directory if empty
+                try:
+                    os.rmdir(os.path.dirname(path))
+                except OSError as ose:
+                    if ose.errno != errno.ENOTEMPTY:
+                        raise
 
 
 def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
@@ -1277,13 +1307,16 @@ def _determine_files_dir(rd):
     return os.path.join(recipedir, rd.getVar('BPN'))
 
 
-def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove):
+def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None):
     """Implement the 'srcrev' mode of update-recipe"""
     import bb
     import oe.recipeutils
 
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+
     recipefile = rd.getVar('FILE')
-    logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile))
+    recipedir = os.path.basename(recipefile)
+    logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
 
     # Get HEAD revision
     try:
@@ -1303,6 +1336,7 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
     srcuri = orig_src_uri.split()
     tempdir = tempfile.mkdtemp(prefix='devtool')
     update_srcuri = False
+    appendfile = None
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)
         srctreebase = workspace[recipename]['srctreebase']
@@ -1328,29 +1362,36 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
             if update_srcuri:
                 removevalues  = {'SRC_URI': removedentries}
                 patchfields['SRC_URI'] = '\\\n    '.join(srcuri)
-            _, destpath = oe.recipeutils.bbappend_recipe(
-                    rd, appendlayerdir, files, wildcardver=wildcard_version,
-                    extralines=patchfields, removevalues=removevalues)
+            if dry_run_outdir:
+                logger.info('Creating bbappend (dry-run)')
+            else:
+                appendfile, destpath = oe.recipeutils.bbappend_recipe(
+                        rd, appendlayerdir, files, wildcardver=wildcard_version,
+                        extralines=patchfields, removevalues=removevalues,
+                        redirect_output=dry_run_outdir)
         else:
             files_dir = _determine_files_dir(rd)
             for basepath, path in upd_f.items():
-                logger.info('Updating file %s' % basepath)
+                logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
                 if os.path.isabs(basepath):
                     # Original file (probably with subdir pointing inside source tree)
                     # so we do not want to move it, just copy
-                    _copy_file(basepath, path)
+                    _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 else:
-                    _move_file(os.path.join(local_files_dir, basepath), path)
+                    _move_file(os.path.join(local_files_dir, basepath), path,
+                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 update_srcuri= True
             for basepath, path in new_f.items():
-                logger.info('Adding new file %s' % basepath)
+                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(local_files_dir, basepath),
-                           os.path.join(files_dir, basepath))
+                           os.path.join(files_dir, basepath),
+                           dry_run_outdir=dry_run_outdir,
+                           base_outdir=recipedir)
                 srcuri.append('file://%s' % basepath)
                 update_srcuri = True
             if update_srcuri:
                 patchfields['SRC_URI'] = ' '.join(srcuri)
-            oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
+            ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir)
     finally:
         shutil.rmtree(tempdir)
     if not 'git://' in orig_src_uri:
@@ -1358,15 +1399,16 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
                     'point to a git repository where you have pushed your '
                     'changes')
 
-    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove)
-    return True
+    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
+    return True, appendfile, remove_files
 
-def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev):
+def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None):
     """Implement the 'patch' mode of update-recipe"""
     import bb
     import oe.recipeutils
 
     recipefile = rd.getVar('FILE')
+    recipedir = os.path.dirname(recipefile)
     append = workspace[recipename]['bbappend']
     if not os.path.exists(append):
         raise DevtoolError('unable to find workspace bbappend for recipe %s' %
@@ -1377,10 +1419,13 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
         raise DevtoolError('Unable to find initial revision - please specify '
                            'it with --initial-rev')
 
+    appendfile = None
     dl_dir = rd.getVar('DL_DIR')
     if not dl_dir.endswith('/'):
         dl_dir += '/'
 
+    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
+
     tempdir = tempfile.mkdtemp(prefix='devtool')
     try:
         local_files_dir = tempfile.mkdtemp(dir=tempdir)
@@ -1418,10 +1463,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                         remaining = ['file://' + os.path.basename(item) for
                                      item in remaining]
                         removevalues = {'SRC_URI': removedentries + remaining}
-                _, destpath = oe.recipeutils.bbappend_recipe(
+                appendfile, destpath = oe.recipeutils.bbappend_recipe(
                                 rd, appendlayerdir, files,
                                 wildcardver=wildcard_version,
-                                removevalues=removevalues)
+                                removevalues=removevalues,
+                                redirect_output=dry_run_outdir)
             else:
                 logger.info('No patches or local source files needed updating')
         else:
@@ -1432,9 +1478,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                 if os.path.isabs(basepath):
                     # Original file (probably with subdir pointing inside source tree)
                     # so we do not want to move it, just copy
-                    _copy_file(basepath, path)
+                    _copy_file(basepath, path,
+                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 else:
-                    _move_file(os.path.join(local_files_dir, basepath), path)
+                    _move_file(os.path.join(local_files_dir, basepath), path,
+                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 updatefiles = True
             for basepath, path in upd_p.items():
                 patchfn = os.path.join(patches_dir, basepath)
@@ -1446,39 +1494,45 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                     _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
                     updaterecipe = True
                 else:
-                    logger.info('Updating patch %s' % basepath)
-                logger.debug('Moving new patch %s to %s' % (patchfn, path))
-                _move_file(patchfn, path)
+                    logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
+                _move_file(patchfn, path,
+                           dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 updatefiles = True
             # Add any new files
             for basepath, path in new_f.items():
-                logger.info('Adding new file %s' % basepath)
+                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(local_files_dir, basepath),
-                           os.path.join(files_dir, basepath))
+                           os.path.join(files_dir, basepath),
+                           dry_run_outdir=dry_run_outdir,
+                           base_outdir=recipedir)
                 srcuri.append('file://%s' % basepath)
                 updaterecipe = True
             for basepath, path in new_p.items():
-                logger.info('Adding new patch %s' % basepath)
+                logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(patches_dir, basepath),
-                           os.path.join(files_dir, basepath))
+                           os.path.join(files_dir, basepath),
+                           dry_run_outdir=dry_run_outdir,
+                           base_outdir=recipedir)
                 srcuri.append('file://%s' % basepath)
                 updaterecipe = True
             # Update recipe, if needed
             if _remove_file_entries(srcuri, remove_files)[0]:
                 updaterecipe = True
             if updaterecipe:
-                logger.info('Updating recipe %s' % os.path.basename(recipefile))
-                oe.recipeutils.patch_recipe(rd, recipefile,
-                                            {'SRC_URI': ' '.join(srcuri)})
+                if not dry_run_outdir:
+                    logger.info('Updating recipe %s' % os.path.basename(recipefile))
+                ret = oe.recipeutils.patch_recipe(rd, recipefile,
+                                                  {'SRC_URI': ' '.join(srcuri)},
+                                                  redirect_output=dry_run_outdir)
             elif not updatefiles:
                 # Neither patches nor recipe were updated
                 logger.info('No patches or files need updating')
-                return False
+                return False, None, []
     finally:
         shutil.rmtree(tempdir)
 
-    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove)
-    return True
+    _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
+    return True, appendfile, remove_files
 
 def _guess_recipe_update_mode(srctree, rdata):
     """Guess the recipe update mode to use"""
@@ -1502,18 +1556,18 @@ def _guess_recipe_update_mode(srctree, rdata):
 
     return 'patch'
 
-def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False):
+def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None):
     srctree = workspace[recipename]['srctree']
     if mode == 'auto':
         mode = _guess_recipe_update_mode(srctree, rd)
 
     if mode == 'srcrev':
-        updated = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove)
+        updated, appendfile, removed = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
     elif mode == 'patch':
-        updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev)
+        updated, appendfile, removed = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir)
     else:
         raise DevtoolError('update_recipe: invalid mode %s' % mode)
-    return updated
+    return updated, appendfile, removed
 
 def update_recipe(args, config, basepath, workspace):
     """Entry point for the devtool 'update-recipe' subcommand"""
@@ -1534,7 +1588,12 @@ def update_recipe(args, config, basepath, workspace):
         if not rd:
             return 1
 
-        updated = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev)
+        dry_run_output = None
+        dry_run_outdir = None
+        if args.dry_run:
+            dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
+            dry_run_outdir = dry_run_output.name
+        updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir)
 
         if updated:
             rf = rd.getVar('FILE')
@@ -1680,6 +1739,8 @@ def finish(args, config, basepath, workspace):
 
     check_workspace_recipe(workspace, args.recipename)
 
+    dry_run_suffix = ' (dry-run)' if args.dry_run else ''
+
     # Grab the equivalent of COREBASE without having to initialise tinfoil
     corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
 
@@ -1700,7 +1761,9 @@ def finish(args, config, basepath, workspace):
             return 1
 
         destlayerdir = _get_layer(args.destination, tinfoil.config_data)
-        origlayerdir = oe.recipeutils.find_layerdir(rd.getVar('FILE'))
+        recipefile = rd.getVar('FILE')
+        recipedir = os.path.dirname(recipefile)
+        origlayerdir = oe.recipeutils.find_layerdir(recipefile)
 
         if not os.path.isdir(destlayerdir):
             raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
@@ -1748,38 +1811,101 @@ def finish(args, config, basepath, workspace):
 
         # Actually update the recipe / bbappend
         removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir)
-        _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev, no_report_remove=removing_original)
+        dry_run_output = None
+        dry_run_outdir = None
+        if args.dry_run:
+            dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
+            dry_run_outdir = dry_run_output.name
+        updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir)
+        removed = [os.path.relpath(pth, recipedir) for pth in removed]
 
         # Remove any old files in the case of an upgrade
-        recipedir = os.path.dirname(rd.getVar('FILE'))
         if removing_original:
             for fn in origfilelist:
                 fnp = os.path.join(origpath, fn)
-                if not os.path.exists(os.path.join(recipedir, fn)):
-                    logger.info('Removing file %s' % fnp)
-                try:
-                    os.remove(fnp)
-                except FileNotFoundError:
-                    pass
+                if fn in removed or not os.path.exists(os.path.join(recipedir, fn)):
+                    logger.info('Removing file %s%s' % (fnp, dry_run_suffix))
+                if not args.dry_run:
+                    try:
+                        os.remove(fnp)
+                    except FileNotFoundError:
+                        pass
 
         if origlayerdir == config.workspace_path and destpath:
             # Recipe file itself is in the workspace - need to move it and any
             # associated files to the specified layer
             no_clean = True
-            logger.info('Moving recipe file to %s' % destpath)
+            logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix))
             for root, _, files in os.walk(recipedir):
                 for fn in files:
                     srcpath = os.path.join(root, fn)
                     relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
                     destdir = os.path.abspath(os.path.join(destpath, relpth))
-                    bb.utils.mkdirhier(destdir)
-                    shutil.move(srcpath, os.path.join(destdir, fn))
+                    destfp = os.path.join(destdir, fn)
+                    _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath)
 
+        if dry_run_outdir:
+            import difflib
+            comparelist = []
+            for root, _, files in os.walk(dry_run_outdir):
+                for fn in files:
+                    outf = os.path.join(root, fn)
+                    relf = os.path.relpath(outf, dry_run_outdir)
+                    logger.debug('dry-run: output file %s' % relf)
+                    if fn.endswith('.bb'):
+                        if origfilelist and origpath and destpath:
+                            # Need to match this up with the pre-upgrade recipe file
+                            for origf in origfilelist:
+                                if origf.endswith('.bb'):
+                                    comparelist.append((os.path.abspath(os.path.join(origpath, origf)),
+                                                        outf,
+                                                        os.path.abspath(os.path.join(destpath, relf))))
+                                    break
+                        else:
+                            # Compare to the existing recipe
+                            comparelist.append((recipefile, outf, recipefile))
+                    elif fn.endswith('.bbappend'):
+                        if appendfile:
+                            if os.path.exists(appendfile):
+                                comparelist.append((appendfile, outf, appendfile))
+                            else:
+                                comparelist.append((None, outf, appendfile))
+                    else:
+                        if destpath:
+                            recipedest = destpath
+                        elif appendfile:
+                            recipedest = os.path.dirname(appendfile)
+                        else:
+                            recipedest = os.path.dirname(recipefile)
+                        destfp = os.path.join(recipedest, relf)
+                        if os.path.exists(destfp):
+                            comparelist.append((destfp, outf, destfp))
+            output = ''
+            for oldfile, newfile, newfileshow in comparelist:
+                if oldfile:
+                    with open(oldfile, 'r') as f:
+                        oldlines = f.readlines()
+                else:
+                    oldfile = '/dev/null'
+                    oldlines = []
+                with open(newfile, 'r') as f:
+                    newlines = f.readlines()
+                if not newfileshow:
+                    newfileshow = newfile
+                diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow)
+                difflines = list(diff)
+                if difflines:
+                    output += ''.join(difflines)
+            if output:
+                logger.info('Diff of changed files:\n%s' % output)
     finally:
         tinfoil.shutdown()
 
     # Everything else has succeeded, we can now reset
-    _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace)
+    if args.dry_run:
+        logger.info('Resetting recipe (dry-run)')
+    else:
+        _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace)
 
     return 0
 
@@ -1873,6 +1999,7 @@ def register_commands(subparsers, context):
     parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
     parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
     parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
+    parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
     parser_update_recipe.set_defaults(func=update_recipe)
 
     parser_status = subparsers.add_parser('status', help='Show workspace status',
@@ -1896,4 +2023,5 @@ def register_commands(subparsers, context):
     parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
     parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
     parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
+    parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
     parser_finish.set_defaults(func=finish)