summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-02-19 22:38:59 +1300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-02-21 09:32:43 +0000
commita8e0e5ecd329b5a9f3019854ee6faa1cff62c69e (patch)
tree91c882d7baf2767d873a60ff049cd3901447570d /scripts
parent2059a344b60837b83261a713eb8cbcd34657a73d (diff)
downloadpoky-a8e0e5ecd329b5a9f3019854ee6faa1cff62c69e.tar.gz
devtool: deploy-target: preserve existing files
If files would be overwritten by the deployment, preserve them in a separate location on the target so that they can be restored if you later run devtool undeploy-target. At the same time, also check for sufficient space before starting the operation so that we avoid potentially failing part way through. Fixes [YOCTO #8978]. (From OE-Core rev: a2da55712691bcf1066c53d813107d75213d6b10) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/lib/devtool/deploy.py87
1 files changed, 78 insertions, 9 deletions
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index d54f6ba2e1..66644ccb6a 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -28,7 +28,7 @@ logger = logging.getLogger('devtool')
28 28
29deploylist_path = '/.devtool' 29deploylist_path = '/.devtool'
30 30
31def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False): 31def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=False, nopreserve=False, nocheckspace=False):
32 """ 32 """
33 Prepare a shell script for running on the target to 33 Prepare a shell script for running on the target to
34 deploy/undeploy files. We have to be careful what we put in this 34 deploy/undeploy files. We have to be careful what we put in this
@@ -48,6 +48,7 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
48 if not deploy: 48 if not deploy:
49 lines.append('echo "Previously deployed files for $1:"') 49 lines.append('echo "Previously deployed files for $1:"')
50 lines.append('manifest="%s/$1.list"' % deploylist_path) 50 lines.append('manifest="%s/$1.list"' % deploylist_path)
51 lines.append('preservedir="%s/$1.preserve"' % deploylist_path)
51 lines.append('if [ -f $manifest ] ; then') 52 lines.append('if [ -f $manifest ] ; then')
52 # Read manifest in reverse and delete files / remove empty dirs 53 # Read manifest in reverse and delete files / remove empty dirs
53 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file') 54 lines.append(' sed \'1!G;h;$!d\' $manifest | while read file')
@@ -58,7 +59,10 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
58 lines.append(' fi') 59 lines.append(' fi')
59 else: 60 else:
60 lines.append(' if [ -d $file ] ; then') 61 lines.append(' if [ -d $file ] ; then')
61 lines.append(' rmdir $file > /dev/null 2>&1 || true') 62 # Avoid deleting a preserved directory in case it has special perms
63 lines.append(' if [ ! -d $preservedir/$file ] ; then')
64 lines.append(' rmdir $file > /dev/null 2>&1 || true')
65 lines.append(' fi')
62 lines.append(' else') 66 lines.append(' else')
63 lines.append(' rm $file') 67 lines.append(' rm $file')
64 lines.append(' fi') 68 lines.append(' fi')
@@ -71,6 +75,39 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
71 lines.append('fi') 75 lines.append('fi')
72 76
73 if deploy: 77 if deploy:
78 if not nocheckspace:
79 # Check for available space
80 # FIXME This doesn't take into account files spread across multiple
81 # partitions, but doing that is non-trivial
82 # Find the part of the destination path that exists
83 lines.append('checkpath="$2"')
84 lines.append('while [ "$checkpath" != "/" ] && [ ! -e $checkpath ]')
85 lines.append('do')
86 lines.append(' checkpath=`dirname "$checkpath"`')
87 lines.append('done')
88 lines.append('freespace=`df -P $checkpath | sed "1d" | awk \'{ print $4 }\'`')
89 # First line of the file is the total space
90 lines.append('total=`head -n1 $3`')
91 lines.append('if [ $total -gt $freespace ] ; then')
92 lines.append(' echo "ERROR: insufficient space on target (available ${freespace}, needed ${total})"')
93 lines.append(' exit 1')
94 lines.append('fi')
95 if not nopreserve:
96 # Preserve any files that exist. Note that this will add to the
97 # preserved list with successive deployments if the list of files
98 # deployed changes, but because we've deleted any previously
99 # deployed files at this point it will never preserve anything
100 # that was deployed, only files that existed prior to any deploying
101 # (which makes the most sense)
102 lines.append('cat $3 | sed "1d" | while read file fsize')
103 lines.append('do')
104 lines.append(' if [ -e $file ] ; then')
105 lines.append(' dest="$preservedir/$file"')
106 lines.append(' mkdir -p `dirname $dest`')
107 lines.append(' mv $file $dest')
108 lines.append(' fi')
109 lines.append('done')
110 lines.append('rm $3')
74 lines.append('mkdir -p `dirname $manifest`') 111 lines.append('mkdir -p `dirname $manifest`')
75 lines.append('mkdir -p $2') 112 lines.append('mkdir -p $2')
76 if verbose: 113 if verbose:
@@ -78,6 +115,14 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
78 else: 115 else:
79 lines.append(' tar xv -C $2 -f - > $manifest') 116 lines.append(' tar xv -C $2 -f - > $manifest')
80 lines.append('sed -i "s!^./!$2!" $manifest') 117 lines.append('sed -i "s!^./!$2!" $manifest')
118 elif not dryrun:
119 # Put any preserved files back
120 lines.append('if [ -d $preservedir ] ; then')
121 lines.append(' cd $preservedir')
122 lines.append(' find . -type f -exec mv {} /{} \;')
123 lines.append(' cd /')
124 lines.append(' rm -rf $preservedir')
125 lines.append('fi')
81 126
82 if undeployall: 127 if undeployall:
83 if not dryrun: 128 if not dryrun:
@@ -94,6 +139,7 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
94def deploy(args, config, basepath, workspace): 139def deploy(args, config, basepath, workspace):
95 """Entry point for the devtool 'deploy' subcommand""" 140 """Entry point for the devtool 'deploy' subcommand"""
96 import re 141 import re
142 import math
97 import oe.recipeutils 143 import oe.recipeutils
98 144
99 check_workspace_recipe(workspace, args.recipename, checksrc=False) 145 check_workspace_recipe(workspace, args.recipename, checksrc=False)
@@ -119,11 +165,23 @@ def deploy(args, config, basepath, workspace):
119 'recipe? If so, the install step has not installed ' 165 'recipe? If so, the install step has not installed '
120 'any files.' % args.recipename) 166 'any files.' % args.recipename)
121 167
168 filelist = []
169 ftotalsize = 0
170 for root, _, files in os.walk(recipe_outdir):
171 for fn in files:
172 # Get the size in kiB (since we'll be comparing it to the output of du -k)
173 # MUST use lstat() here not stat() or getfilesize() since we don't want to
174 # dereference symlinks
175 fsize = int(math.ceil(float(os.lstat(os.path.join(root, fn)).st_size)/1024))
176 ftotalsize += fsize
177 # The path as it would appear on the target
178 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
179 filelist.append((fpath, fsize))
180
122 if args.dry_run: 181 if args.dry_run:
123 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) 182 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
124 for root, _, files in os.walk(recipe_outdir): 183 for item, _ in filelist:
125 for fn in files: 184 print(' %s' % item)
126 print(' %s' % os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn))
127 return 0 185 return 0
128 186
129 187
@@ -140,11 +198,20 @@ def deploy(args, config, basepath, workspace):
140 tmpdir = tempfile.mkdtemp(prefix='devtool') 198 tmpdir = tempfile.mkdtemp(prefix='devtool')
141 try: 199 try:
142 tmpscript = '/tmp/devtool_deploy.sh' 200 tmpscript = '/tmp/devtool_deploy.sh'
143 shellscript = _prepare_remote_script(deploy=True, verbose=args.show_status) 201 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
202 shellscript = _prepare_remote_script(deploy=True,
203 verbose=args.show_status,
204 nopreserve=args.no_preserve,
205 nocheckspace=args.no_check_space)
144 # Write out the script to a file 206 # Write out the script to a file
145 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: 207 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
146 f.write(shellscript) 208 f.write(shellscript)
147 # Copy it to the target 209 # Write out the file list
210 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
211 f.write('%d\n' % ftotalsize)
212 for fpath, fsize in filelist:
213 f.write('%s %d\n' % (fpath, fsize))
214 # Copy them to the target
148 ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) 215 ret = subprocess.call("scp %s %s/* %s:%s" % (extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
149 if ret != 0: 216 if ret != 0:
150 raise DevtoolError('Failed to copy script to %s - rerun with -s to ' 217 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
@@ -153,7 +220,7 @@ def deploy(args, config, basepath, workspace):
153 shutil.rmtree(tmpdir) 220 shutil.rmtree(tmpdir)
154 221
155 # Now run the script 222 # Now run the script
156 ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'sh %s %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename, destdir), cwd=recipe_outdir, shell=True) 223 ret = exec_fakeroot(rd, 'tar cf - . | ssh %s %s \'sh %s %s %s %s\'' % (extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
157 if ret != 0: 224 if ret != 0:
158 raise DevtoolError('Deploy failed - rerun with -s to get a complete ' 225 raise DevtoolError('Deploy failed - rerun with -s to get a complete '
159 'error message') 226 'error message')
@@ -213,13 +280,15 @@ def register_commands(subparsers, context):
213 """Register devtool subcommands from the deploy plugin""" 280 """Register devtool subcommands from the deploy plugin"""
214 parser_deploy = subparsers.add_parser('deploy-target', 281 parser_deploy = subparsers.add_parser('deploy-target',
215 help='Deploy recipe output files to live target machine', 282 help='Deploy recipe output files to live target machine',
216 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.', 283 description='Deploys a recipe\'s build output (i.e. the output of the do_install task) to a live target machine over ssh. By default, any existing files will be preserved instead of being overwritten and will be restored if you run devtool undeploy-target. Note: this only deploys the recipe itself and not any runtime dependencies, so it is assumed that those have been installed on the target beforehand.',
217 group='testbuild') 284 group='testbuild')
218 parser_deploy.add_argument('recipename', help='Recipe to deploy') 285 parser_deploy.add_argument('recipename', help='Recipe to deploy')
219 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]') 286 parser_deploy.add_argument('target', help='Live target machine running an ssh server: user@hostname[:destdir]')
220 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true') 287 parser_deploy.add_argument('-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
221 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true') 288 parser_deploy.add_argument('-s', '--show-status', help='Show progress/status output', action='store_true')
222 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true') 289 parser_deploy.add_argument('-n', '--dry-run', help='List files to be deployed only', action='store_true')
290 parser_deploy.add_argument('-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
291 parser_deploy.add_argument('--no-check-space', help='Do not check for available space before deploying', action='store_true')
223 parser_deploy.set_defaults(func=deploy) 292 parser_deploy.set_defaults(func=deploy)
224 293
225 parser_undeploy = subparsers.add_parser('undeploy-target', 294 parser_undeploy = subparsers.add_parser('undeploy-target',