summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2017-11-01 11:56:18 +1300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-11-11 12:14:27 +0000
commit9a80078e4b8e01d9bb49288f214f40e8497aa3ac (patch)
treead8387c4352dd3e0d81cda000ad9452e0449d4e7
parentd906d7cea814de917baf70eb3ecb568862fbc363 (diff)
downloadpoky-9a80078e4b8e01d9bb49288f214f40e8497aa3ac.tar.gz
devtool: finish: add dry-run option
If you're not sure what changes devtool finish is going to make, or you're not sure you're finished with your modifications, it is useful to be able to see what devtool finish is going to do beforehand, so add a -N/--dry-run option to make that possible. (It's also very useful for debugging devtool finish itself.) (From OE-Core rev: 05f2d5d2ce00c53825ccea5cd9c2262f9d27a638) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oe/recipeutils.py48
-rw-r--r--scripts/lib/devtool/standard.py270
2 files changed, 235 insertions, 83 deletions
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index cab94b1350..a1e191afc7 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -251,7 +251,7 @@ def patch_recipe_lines(fromlines, values, trailing_newline=True):
251 return changed, tolines 251 return changed, tolines
252 252
253 253
254def patch_recipe_file(fn, values, patch=False, relpath=''): 254def patch_recipe_file(fn, values, patch=False, relpath='', redirect_output=None):
255 """Update or insert variable values into a recipe file (assuming you 255 """Update or insert variable values into a recipe file (assuming you
256 have already identified the exact file you want to update.) 256 have already identified the exact file you want to update.)
257 Note that some manual inspection/intervention may be required 257 Note that some manual inspection/intervention may be required
@@ -263,7 +263,11 @@ def patch_recipe_file(fn, values, patch=False, relpath=''):
263 263
264 _, tolines = patch_recipe_lines(fromlines, values) 264 _, tolines = patch_recipe_lines(fromlines, values)
265 265
266 if patch: 266 if redirect_output:
267 with open(os.path.join(redirect_output, os.path.basename(fn)), 'w') as f:
268 f.writelines(tolines)
269 return None
270 elif patch:
267 relfn = os.path.relpath(fn, relpath) 271 relfn = os.path.relpath(fn, relpath)
268 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn) 272 diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
269 return diff 273 return diff
@@ -313,7 +317,7 @@ def localise_file_vars(fn, varfiles, varlist):
313 317
314 return filevars 318 return filevars
315 319
316def patch_recipe(d, fn, varvalues, patch=False, relpath=''): 320def patch_recipe(d, fn, varvalues, patch=False, relpath='', redirect_output=None):
317 """Modify a list of variable values in the specified recipe. Handles inc files if 321 """Modify a list of variable values in the specified recipe. Handles inc files if
318 used by the recipe. 322 used by the recipe.
319 """ 323 """
@@ -323,7 +327,7 @@ def patch_recipe(d, fn, varvalues, patch=False, relpath=''):
323 patches = [] 327 patches = []
324 for f,v in locs.items(): 328 for f,v in locs.items():
325 vals = {k: varvalues[k] for k in v} 329 vals = {k: varvalues[k] for k in v}
326 patchdata = patch_recipe_file(f, vals, patch, relpath) 330 patchdata = patch_recipe_file(f, vals, patch, relpath, redirect_output)
327 if patch: 331 if patch:
328 patches.append(patchdata) 332 patches.append(patchdata)
329 333
@@ -584,7 +588,7 @@ def get_bbappend_path(d, destlayerdir, wildcardver=False):
584 return (appendpath, pathok) 588 return (appendpath, pathok)
585 589
586 590
587def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None): 591def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None, redirect_output=None):
588 """ 592 """
589 Writes a bbappend file for a recipe 593 Writes a bbappend file for a recipe
590 Parameters: 594 Parameters:
@@ -611,6 +615,9 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
611 value pairs, or simply a list of the lines. 615 value pairs, or simply a list of the lines.
612 removevalues: 616 removevalues:
613 Variable values to remove - a dict of names/values. 617 Variable values to remove - a dict of names/values.
618 redirect_output:
619 If specified, redirects writing the output file to the
620 specified directory (for dry-run purposes)
614 """ 621 """
615 622
616 if not removevalues: 623 if not removevalues:
@@ -625,7 +632,8 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
625 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath))) 632 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
626 633
627 appenddir = os.path.dirname(appendpath) 634 appenddir = os.path.dirname(appendpath)
628 bb.utils.mkdirhier(appenddir) 635 if not redirect_output:
636 bb.utils.mkdirhier(appenddir)
629 637
630 # FIXME check if the bbappend doesn't get overridden by a higher priority layer? 638 # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
631 639
@@ -702,9 +710,18 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
702 if instfunclines: 710 if instfunclines:
703 bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines)) 711 bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines))
704 712
705 bb.note('Writing append file %s' % appendpath) 713 if redirect_output:
714 bb.note('Writing append file %s (dry-run)' % appendpath)
715 outfile = os.path.join(redirect_output, os.path.basename(appendpath))
716 # Only take a copy if the file isn't already there (this function may be called
717 # multiple times per operation when we're handling overrides)
718 if os.path.exists(appendpath) and not os.path.exists(outfile):
719 shutil.copy2(appendpath, outfile)
720 else:
721 bb.note('Writing append file %s' % appendpath)
722 outfile = appendpath
706 723
707 if os.path.exists(appendpath): 724 if os.path.exists(outfile):
708 # Work around lack of nonlocal in python 2 725 # Work around lack of nonlocal in python 2
709 extvars = {'destsubdir': destsubdir} 726 extvars = {'destsubdir': destsubdir}
710 727
@@ -776,7 +793,7 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
776 if removevalues: 793 if removevalues:
777 varnames.extend(list(removevalues.keys())) 794 varnames.extend(list(removevalues.keys()))
778 795
779 with open(appendpath, 'r') as f: 796 with open(outfile, 'r') as f:
780 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc) 797 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
781 798
782 destsubdir = extvars['destsubdir'] 799 destsubdir = extvars['destsubdir']
@@ -793,20 +810,27 @@ def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False,
793 updated = True 810 updated = True
794 811
795 if updated: 812 if updated:
796 with open(appendpath, 'w') as f: 813 with open(outfile, 'w') as f:
797 f.writelines(newlines) 814 f.writelines(newlines)
798 815
799 if copyfiles: 816 if copyfiles:
800 if machine: 817 if machine:
801 destsubdir = os.path.join(destsubdir, machine) 818 destsubdir = os.path.join(destsubdir, machine)
819 if redirect_output:
820 outdir = redirect_output
821 else:
822 outdir = appenddir
802 for newfile, srcfile in copyfiles.items(): 823 for newfile, srcfile in copyfiles.items():
803 filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile)) 824 filedest = os.path.join(outdir, destsubdir, os.path.basename(srcfile))
804 if os.path.abspath(newfile) != os.path.abspath(filedest): 825 if os.path.abspath(newfile) != os.path.abspath(filedest):
805 if newfile.startswith(tempfile.gettempdir()): 826 if newfile.startswith(tempfile.gettempdir()):
806 newfiledisp = os.path.basename(newfile) 827 newfiledisp = os.path.basename(newfile)
807 else: 828 else:
808 newfiledisp = newfile 829 newfiledisp = newfile
809 bb.note('Copying %s to %s' % (newfiledisp, filedest)) 830 if redirect_output:
831 bb.note('Copying %s to %s (dry-run)' % (newfiledisp, os.path.join(appenddir, destsubdir, os.path.basename(srcfile))))
832 else:
833 bb.note('Copying %s to %s' % (newfiledisp, filedest))
810 bb.utils.mkdirhier(os.path.dirname(filedest)) 834 bb.utils.mkdirhier(os.path.dirname(filedest))
811 shutil.copyfile(newfile, filedest) 835 shutil.copyfile(newfile, filedest)
812 836
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 9d9031c2b3..d55d504a14 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -341,19 +341,45 @@ def _check_compatible_recipe(pn, d):
341 "from working. You will need to disable this " 341 "from working. You will need to disable this "
342 "first." % pn) 342 "first." % pn)
343 343
344def _move_file(src, dst): 344def _dry_run_copy(src, dst, dry_run_outdir, base_outdir):
345 """Move a file. Creates all the directory components of destination path.""" 345 """Common function for copying a file to the dry run output directory"""
346 relpath = os.path.relpath(dst, base_outdir)
347 if relpath.startswith('..'):
348 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst))
349 dst = os.path.join(dry_run_outdir, relpath)
346 dst_d = os.path.dirname(dst) 350 dst_d = os.path.dirname(dst)
347 if dst_d: 351 if dst_d:
348 bb.utils.mkdirhier(dst_d) 352 bb.utils.mkdirhier(dst_d)
349 shutil.move(src, dst) 353 # Don't overwrite existing files, otherwise in the case of an upgrade
354 # the dry-run written out recipe will be overwritten with an unmodified
355 # version
356 if not os.path.exists(dst):
357 shutil.copy(src, dst)
358
359def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
360 """Move a file. Creates all the directory components of destination path."""
361 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
362 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix))
363 if dry_run_outdir:
364 # We want to copy here, not move
365 _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
366 else:
367 dst_d = os.path.dirname(dst)
368 if dst_d:
369 bb.utils.mkdirhier(dst_d)
370 shutil.move(src, dst)
350 371
351def _copy_file(src, dst): 372def _copy_file(src, dst, dry_run_outdir=None):
352 """Copy a file. Creates all the directory components of destination path.""" 373 """Copy a file. Creates all the directory components of destination path."""
353 dst_d = os.path.dirname(dst) 374 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
354 if dst_d: 375 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
355 bb.utils.mkdirhier(dst_d) 376 if dry_run_outdir:
356 shutil.copy(src, dst) 377 _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
378 else:
379 dst_d = os.path.dirname(dst)
380 if dst_d:
381 bb.utils.mkdirhier(dst_d)
382 shutil.copy(src, dst)
357 383
358def _git_ls_tree(repodir, treeish='HEAD', recursive=False): 384def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
359 """List contents of a git treeish""" 385 """List contents of a git treeish"""
@@ -1054,8 +1080,11 @@ def _replace_srcuri_entry(srcuri, filename, newentry):
1054 srcuri.insert(i, newentry) 1080 srcuri.insert(i, newentry)
1055 break 1081 break
1056 1082
1057def _remove_source_files(append, files, destpath, no_report_remove=False): 1083def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False):
1058 """Unlink existing patch files""" 1084 """Unlink existing patch files"""
1085
1086 dry_run_suffix = ' (dry-run)' if dry_run else ''
1087
1059 for path in files: 1088 for path in files:
1060 if append: 1089 if append:
1061 if not destpath: 1090 if not destpath:
@@ -1064,19 +1093,20 @@ def _remove_source_files(append, files, destpath, no_report_remove=False):
1064 1093
1065 if os.path.exists(path): 1094 if os.path.exists(path):
1066 if not no_report_remove: 1095 if not no_report_remove:
1067 logger.info('Removing file %s' % path) 1096 logger.info('Removing file %s%s' % (path, dry_run_suffix))
1068 # FIXME "git rm" here would be nice if the file in question is 1097 if not dry_run:
1069 # tracked 1098 # FIXME "git rm" here would be nice if the file in question is
1070 # FIXME there's a chance that this file is referred to by 1099 # tracked
1071 # another recipe, in which case deleting wouldn't be the 1100 # FIXME there's a chance that this file is referred to by
1072 # right thing to do 1101 # another recipe, in which case deleting wouldn't be the
1073 os.remove(path) 1102 # right thing to do
1074 # Remove directory if empty 1103 os.remove(path)
1075 try: 1104 # Remove directory if empty
1076 os.rmdir(os.path.dirname(path)) 1105 try:
1077 except OSError as ose: 1106 os.rmdir(os.path.dirname(path))
1078 if ose.errno != errno.ENOTEMPTY: 1107 except OSError as ose:
1079 raise 1108 if ose.errno != errno.ENOTEMPTY:
1109 raise
1080 1110
1081 1111
1082def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1112def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
@@ -1277,13 +1307,16 @@ def _determine_files_dir(rd):
1277 return os.path.join(recipedir, rd.getVar('BPN')) 1307 return os.path.join(recipedir, rd.getVar('BPN'))
1278 1308
1279 1309
1280def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove): 1310def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None):
1281 """Implement the 'srcrev' mode of update-recipe""" 1311 """Implement the 'srcrev' mode of update-recipe"""
1282 import bb 1312 import bb
1283 import oe.recipeutils 1313 import oe.recipeutils
1284 1314
1315 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1316
1285 recipefile = rd.getVar('FILE') 1317 recipefile = rd.getVar('FILE')
1286 logger.info('Updating SRCREV in recipe %s' % os.path.basename(recipefile)) 1318 recipedir = os.path.basename(recipefile)
1319 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
1287 1320
1288 # Get HEAD revision 1321 # Get HEAD revision
1289 try: 1322 try:
@@ -1303,6 +1336,7 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
1303 srcuri = orig_src_uri.split() 1336 srcuri = orig_src_uri.split()
1304 tempdir = tempfile.mkdtemp(prefix='devtool') 1337 tempdir = tempfile.mkdtemp(prefix='devtool')
1305 update_srcuri = False 1338 update_srcuri = False
1339 appendfile = None
1306 try: 1340 try:
1307 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1341 local_files_dir = tempfile.mkdtemp(dir=tempdir)
1308 srctreebase = workspace[recipename]['srctreebase'] 1342 srctreebase = workspace[recipename]['srctreebase']
@@ -1328,29 +1362,36 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
1328 if update_srcuri: 1362 if update_srcuri:
1329 removevalues = {'SRC_URI': removedentries} 1363 removevalues = {'SRC_URI': removedentries}
1330 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1364 patchfields['SRC_URI'] = '\\\n '.join(srcuri)
1331 _, destpath = oe.recipeutils.bbappend_recipe( 1365 if dry_run_outdir:
1332 rd, appendlayerdir, files, wildcardver=wildcard_version, 1366 logger.info('Creating bbappend (dry-run)')
1333 extralines=patchfields, removevalues=removevalues) 1367 else:
1368 appendfile, destpath = oe.recipeutils.bbappend_recipe(
1369 rd, appendlayerdir, files, wildcardver=wildcard_version,
1370 extralines=patchfields, removevalues=removevalues,
1371 redirect_output=dry_run_outdir)
1334 else: 1372 else:
1335 files_dir = _determine_files_dir(rd) 1373 files_dir = _determine_files_dir(rd)
1336 for basepath, path in upd_f.items(): 1374 for basepath, path in upd_f.items():
1337 logger.info('Updating file %s' % basepath) 1375 logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
1338 if os.path.isabs(basepath): 1376 if os.path.isabs(basepath):
1339 # Original file (probably with subdir pointing inside source tree) 1377 # Original file (probably with subdir pointing inside source tree)
1340 # so we do not want to move it, just copy 1378 # so we do not want to move it, just copy
1341 _copy_file(basepath, path) 1379 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1342 else: 1380 else:
1343 _move_file(os.path.join(local_files_dir, basepath), path) 1381 _move_file(os.path.join(local_files_dir, basepath), path,
1382 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1344 update_srcuri= True 1383 update_srcuri= True
1345 for basepath, path in new_f.items(): 1384 for basepath, path in new_f.items():
1346 logger.info('Adding new file %s' % basepath) 1385 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1347 _move_file(os.path.join(local_files_dir, basepath), 1386 _move_file(os.path.join(local_files_dir, basepath),
1348 os.path.join(files_dir, basepath)) 1387 os.path.join(files_dir, basepath),
1388 dry_run_outdir=dry_run_outdir,
1389 base_outdir=recipedir)
1349 srcuri.append('file://%s' % basepath) 1390 srcuri.append('file://%s' % basepath)
1350 update_srcuri = True 1391 update_srcuri = True
1351 if update_srcuri: 1392 if update_srcuri:
1352 patchfields['SRC_URI'] = ' '.join(srcuri) 1393 patchfields['SRC_URI'] = ' '.join(srcuri)
1353 oe.recipeutils.patch_recipe(rd, recipefile, patchfields) 1394 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir)
1354 finally: 1395 finally:
1355 shutil.rmtree(tempdir) 1396 shutil.rmtree(tempdir)
1356 if not 'git://' in orig_src_uri: 1397 if not 'git://' in orig_src_uri:
@@ -1358,15 +1399,16 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
1358 'point to a git repository where you have pushed your ' 1399 'point to a git repository where you have pushed your '
1359 'changes') 1400 'changes')
1360 1401
1361 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove) 1402 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1362 return True 1403 return True, appendfile, remove_files
1363 1404
1364def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev): 1405def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None):
1365 """Implement the 'patch' mode of update-recipe""" 1406 """Implement the 'patch' mode of update-recipe"""
1366 import bb 1407 import bb
1367 import oe.recipeutils 1408 import oe.recipeutils
1368 1409
1369 recipefile = rd.getVar('FILE') 1410 recipefile = rd.getVar('FILE')
1411 recipedir = os.path.dirname(recipefile)
1370 append = workspace[recipename]['bbappend'] 1412 append = workspace[recipename]['bbappend']
1371 if not os.path.exists(append): 1413 if not os.path.exists(append):
1372 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1414 raise DevtoolError('unable to find workspace bbappend for recipe %s' %
@@ -1377,10 +1419,13 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1377 raise DevtoolError('Unable to find initial revision - please specify ' 1419 raise DevtoolError('Unable to find initial revision - please specify '
1378 'it with --initial-rev') 1420 'it with --initial-rev')
1379 1421
1422 appendfile = None
1380 dl_dir = rd.getVar('DL_DIR') 1423 dl_dir = rd.getVar('DL_DIR')
1381 if not dl_dir.endswith('/'): 1424 if not dl_dir.endswith('/'):
1382 dl_dir += '/' 1425 dl_dir += '/'
1383 1426
1427 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1428
1384 tempdir = tempfile.mkdtemp(prefix='devtool') 1429 tempdir = tempfile.mkdtemp(prefix='devtool')
1385 try: 1430 try:
1386 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1431 local_files_dir = tempfile.mkdtemp(dir=tempdir)
@@ -1418,10 +1463,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1418 remaining = ['file://' + os.path.basename(item) for 1463 remaining = ['file://' + os.path.basename(item) for
1419 item in remaining] 1464 item in remaining]
1420 removevalues = {'SRC_URI': removedentries + remaining} 1465 removevalues = {'SRC_URI': removedentries + remaining}
1421 _, destpath = oe.recipeutils.bbappend_recipe( 1466 appendfile, destpath = oe.recipeutils.bbappend_recipe(
1422 rd, appendlayerdir, files, 1467 rd, appendlayerdir, files,
1423 wildcardver=wildcard_version, 1468 wildcardver=wildcard_version,
1424 removevalues=removevalues) 1469 removevalues=removevalues,
1470 redirect_output=dry_run_outdir)
1425 else: 1471 else:
1426 logger.info('No patches or local source files needed updating') 1472 logger.info('No patches or local source files needed updating')
1427 else: 1473 else:
@@ -1432,9 +1478,11 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1432 if os.path.isabs(basepath): 1478 if os.path.isabs(basepath):
1433 # Original file (probably with subdir pointing inside source tree) 1479 # Original file (probably with subdir pointing inside source tree)
1434 # so we do not want to move it, just copy 1480 # so we do not want to move it, just copy
1435 _copy_file(basepath, path) 1481 _copy_file(basepath, path,
1482 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1436 else: 1483 else:
1437 _move_file(os.path.join(local_files_dir, basepath), path) 1484 _move_file(os.path.join(local_files_dir, basepath), path,
1485 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1438 updatefiles = True 1486 updatefiles = True
1439 for basepath, path in upd_p.items(): 1487 for basepath, path in upd_p.items():
1440 patchfn = os.path.join(patches_dir, basepath) 1488 patchfn = os.path.join(patches_dir, basepath)
@@ -1446,39 +1494,45 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1446 _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) 1494 _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
1447 updaterecipe = True 1495 updaterecipe = True
1448 else: 1496 else:
1449 logger.info('Updating patch %s' % basepath) 1497 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
1450 logger.debug('Moving new patch %s to %s' % (patchfn, path)) 1498 _move_file(patchfn, path,
1451 _move_file(patchfn, path) 1499 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1452 updatefiles = True 1500 updatefiles = True
1453 # Add any new files 1501 # Add any new files
1454 for basepath, path in new_f.items(): 1502 for basepath, path in new_f.items():
1455 logger.info('Adding new file %s' % basepath) 1503 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1456 _move_file(os.path.join(local_files_dir, basepath), 1504 _move_file(os.path.join(local_files_dir, basepath),
1457 os.path.join(files_dir, basepath)) 1505 os.path.join(files_dir, basepath),
1506 dry_run_outdir=dry_run_outdir,
1507 base_outdir=recipedir)
1458 srcuri.append('file://%s' % basepath) 1508 srcuri.append('file://%s' % basepath)
1459 updaterecipe = True 1509 updaterecipe = True
1460 for basepath, path in new_p.items(): 1510 for basepath, path in new_p.items():
1461 logger.info('Adding new patch %s' % basepath) 1511 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
1462 _move_file(os.path.join(patches_dir, basepath), 1512 _move_file(os.path.join(patches_dir, basepath),
1463 os.path.join(files_dir, basepath)) 1513 os.path.join(files_dir, basepath),
1514 dry_run_outdir=dry_run_outdir,
1515 base_outdir=recipedir)
1464 srcuri.append('file://%s' % basepath) 1516 srcuri.append('file://%s' % basepath)
1465 updaterecipe = True 1517 updaterecipe = True
1466 # Update recipe, if needed 1518 # Update recipe, if needed
1467 if _remove_file_entries(srcuri, remove_files)[0]: 1519 if _remove_file_entries(srcuri, remove_files)[0]:
1468 updaterecipe = True 1520 updaterecipe = True
1469 if updaterecipe: 1521 if updaterecipe:
1470 logger.info('Updating recipe %s' % os.path.basename(recipefile)) 1522 if not dry_run_outdir:
1471 oe.recipeutils.patch_recipe(rd, recipefile, 1523 logger.info('Updating recipe %s' % os.path.basename(recipefile))
1472 {'SRC_URI': ' '.join(srcuri)}) 1524 ret = oe.recipeutils.patch_recipe(rd, recipefile,
1525 {'SRC_URI': ' '.join(srcuri)},
1526 redirect_output=dry_run_outdir)
1473 elif not updatefiles: 1527 elif not updatefiles:
1474 # Neither patches nor recipe were updated 1528 # Neither patches nor recipe were updated
1475 logger.info('No patches or files need updating') 1529 logger.info('No patches or files need updating')
1476 return False 1530 return False, None, []
1477 finally: 1531 finally:
1478 shutil.rmtree(tempdir) 1532 shutil.rmtree(tempdir)
1479 1533
1480 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove) 1534 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1481 return True 1535 return True, appendfile, remove_files
1482 1536
1483def _guess_recipe_update_mode(srctree, rdata): 1537def _guess_recipe_update_mode(srctree, rdata):
1484 """Guess the recipe update mode to use""" 1538 """Guess the recipe update mode to use"""
@@ -1502,18 +1556,18 @@ def _guess_recipe_update_mode(srctree, rdata):
1502 1556
1503 return 'patch' 1557 return 'patch'
1504 1558
1505def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False): 1559def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None):
1506 srctree = workspace[recipename]['srctree'] 1560 srctree = workspace[recipename]['srctree']
1507 if mode == 'auto': 1561 if mode == 'auto':
1508 mode = _guess_recipe_update_mode(srctree, rd) 1562 mode = _guess_recipe_update_mode(srctree, rd)
1509 1563
1510 if mode == 'srcrev': 1564 if mode == 'srcrev':
1511 updated = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove) 1565 updated, appendfile, removed = _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
1512 elif mode == 'patch': 1566 elif mode == 'patch':
1513 updated = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev) 1567 updated, appendfile, removed = _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir)
1514 else: 1568 else:
1515 raise DevtoolError('update_recipe: invalid mode %s' % mode) 1569 raise DevtoolError('update_recipe: invalid mode %s' % mode)
1516 return updated 1570 return updated, appendfile, removed
1517 1571
1518def update_recipe(args, config, basepath, workspace): 1572def update_recipe(args, config, basepath, workspace):
1519 """Entry point for the devtool 'update-recipe' subcommand""" 1573 """Entry point for the devtool 'update-recipe' subcommand"""
@@ -1534,7 +1588,12 @@ def update_recipe(args, config, basepath, workspace):
1534 if not rd: 1588 if not rd:
1535 return 1 1589 return 1
1536 1590
1537 updated = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev) 1591 dry_run_output = None
1592 dry_run_outdir = None
1593 if args.dry_run:
1594 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
1595 dry_run_outdir = dry_run_output.name
1596 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir)
1538 1597
1539 if updated: 1598 if updated:
1540 rf = rd.getVar('FILE') 1599 rf = rd.getVar('FILE')
@@ -1680,6 +1739,8 @@ def finish(args, config, basepath, workspace):
1680 1739
1681 check_workspace_recipe(workspace, args.recipename) 1740 check_workspace_recipe(workspace, args.recipename)
1682 1741
1742 dry_run_suffix = ' (dry-run)' if args.dry_run else ''
1743
1683 # Grab the equivalent of COREBASE without having to initialise tinfoil 1744 # Grab the equivalent of COREBASE without having to initialise tinfoil
1684 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) 1745 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
1685 1746
@@ -1700,7 +1761,9 @@ def finish(args, config, basepath, workspace):
1700 return 1 1761 return 1
1701 1762
1702 destlayerdir = _get_layer(args.destination, tinfoil.config_data) 1763 destlayerdir = _get_layer(args.destination, tinfoil.config_data)
1703 origlayerdir = oe.recipeutils.find_layerdir(rd.getVar('FILE')) 1764 recipefile = rd.getVar('FILE')
1765 recipedir = os.path.dirname(recipefile)
1766 origlayerdir = oe.recipeutils.find_layerdir(recipefile)
1704 1767
1705 if not os.path.isdir(destlayerdir): 1768 if not os.path.isdir(destlayerdir):
1706 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination) 1769 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
@@ -1748,38 +1811,101 @@ def finish(args, config, basepath, workspace):
1748 1811
1749 # Actually update the recipe / bbappend 1812 # Actually update the recipe / bbappend
1750 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir) 1813 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir)
1751 _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, initial_rev=args.initial_rev, no_report_remove=removing_original) 1814 dry_run_output = None
1815 dry_run_outdir = None
1816 if args.dry_run:
1817 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
1818 dry_run_outdir = dry_run_output.name
1819 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir)
1820 removed = [os.path.relpath(pth, recipedir) for pth in removed]
1752 1821
1753 # Remove any old files in the case of an upgrade 1822 # Remove any old files in the case of an upgrade
1754 recipedir = os.path.dirname(rd.getVar('FILE'))
1755 if removing_original: 1823 if removing_original:
1756 for fn in origfilelist: 1824 for fn in origfilelist:
1757 fnp = os.path.join(origpath, fn) 1825 fnp = os.path.join(origpath, fn)
1758 if not os.path.exists(os.path.join(recipedir, fn)): 1826 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)):
1759 logger.info('Removing file %s' % fnp) 1827 logger.info('Removing file %s%s' % (fnp, dry_run_suffix))
1760 try: 1828 if not args.dry_run:
1761 os.remove(fnp) 1829 try:
1762 except FileNotFoundError: 1830 os.remove(fnp)
1763 pass 1831 except FileNotFoundError:
1832 pass
1764 1833
1765 if origlayerdir == config.workspace_path and destpath: 1834 if origlayerdir == config.workspace_path and destpath:
1766 # Recipe file itself is in the workspace - need to move it and any 1835 # Recipe file itself is in the workspace - need to move it and any
1767 # associated files to the specified layer 1836 # associated files to the specified layer
1768 no_clean = True 1837 no_clean = True
1769 logger.info('Moving recipe file to %s' % destpath) 1838 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix))
1770 for root, _, files in os.walk(recipedir): 1839 for root, _, files in os.walk(recipedir):
1771 for fn in files: 1840 for fn in files:
1772 srcpath = os.path.join(root, fn) 1841 srcpath = os.path.join(root, fn)
1773 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir) 1842 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
1774 destdir = os.path.abspath(os.path.join(destpath, relpth)) 1843 destdir = os.path.abspath(os.path.join(destpath, relpth))
1775 bb.utils.mkdirhier(destdir) 1844 destfp = os.path.join(destdir, fn)
1776 shutil.move(srcpath, os.path.join(destdir, fn)) 1845 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath)
1777 1846
1847 if dry_run_outdir:
1848 import difflib
1849 comparelist = []
1850 for root, _, files in os.walk(dry_run_outdir):
1851 for fn in files:
1852 outf = os.path.join(root, fn)
1853 relf = os.path.relpath(outf, dry_run_outdir)
1854 logger.debug('dry-run: output file %s' % relf)
1855 if fn.endswith('.bb'):
1856 if origfilelist and origpath and destpath:
1857 # Need to match this up with the pre-upgrade recipe file
1858 for origf in origfilelist:
1859 if origf.endswith('.bb'):
1860 comparelist.append((os.path.abspath(os.path.join(origpath, origf)),
1861 outf,
1862 os.path.abspath(os.path.join(destpath, relf))))
1863 break
1864 else:
1865 # Compare to the existing recipe
1866 comparelist.append((recipefile, outf, recipefile))
1867 elif fn.endswith('.bbappend'):
1868 if appendfile:
1869 if os.path.exists(appendfile):
1870 comparelist.append((appendfile, outf, appendfile))
1871 else:
1872 comparelist.append((None, outf, appendfile))
1873 else:
1874 if destpath:
1875 recipedest = destpath
1876 elif appendfile:
1877 recipedest = os.path.dirname(appendfile)
1878 else:
1879 recipedest = os.path.dirname(recipefile)
1880 destfp = os.path.join(recipedest, relf)
1881 if os.path.exists(destfp):
1882 comparelist.append((destfp, outf, destfp))
1883 output = ''
1884 for oldfile, newfile, newfileshow in comparelist:
1885 if oldfile:
1886 with open(oldfile, 'r') as f:
1887 oldlines = f.readlines()
1888 else:
1889 oldfile = '/dev/null'
1890 oldlines = []
1891 with open(newfile, 'r') as f:
1892 newlines = f.readlines()
1893 if not newfileshow:
1894 newfileshow = newfile
1895 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow)
1896 difflines = list(diff)
1897 if difflines:
1898 output += ''.join(difflines)
1899 if output:
1900 logger.info('Diff of changed files:\n%s' % output)
1778 finally: 1901 finally:
1779 tinfoil.shutdown() 1902 tinfoil.shutdown()
1780 1903
1781 # Everything else has succeeded, we can now reset 1904 # Everything else has succeeded, we can now reset
1782 _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace) 1905 if args.dry_run:
1906 logger.info('Resetting recipe (dry-run)')
1907 else:
1908 _reset([args.recipename], no_clean=no_clean, config=config, basepath=basepath, workspace=workspace)
1783 1909
1784 return 0 1910 return 0
1785 1911
@@ -1873,6 +1999,7 @@ def register_commands(subparsers, context):
1873 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR') 1999 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
1874 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true') 2000 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
1875 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update') 2001 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
2002 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
1876 parser_update_recipe.set_defaults(func=update_recipe) 2003 parser_update_recipe.set_defaults(func=update_recipe)
1877 2004
1878 parser_status = subparsers.add_parser('status', help='Show workspace status', 2005 parser_status = subparsers.add_parser('status', help='Show workspace status',
@@ -1896,4 +2023,5 @@ def register_commands(subparsers, context):
1896 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE') 2023 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
1897 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches') 2024 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
1898 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository') 2025 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
2026 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
1899 parser_finish.set_defaults(func=finish) 2027 parser_finish.set_defaults(func=finish)