diff options
author | Adrian Freihofer <adrian.freihofer@gmail.com> | 2024-01-22 14:58:25 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2024-02-02 10:38:28 +0000 |
commit | c2813a4c69bf18784c01e9181bbd070bd4b844ce (patch) | |
tree | c4fcdc8e8801e7e44a7ea04b355cc4357d58a38d /scripts | |
parent | 50618e59ddf6603a198df36f2c12cd66539b3667 (diff) | |
download | poky-c2813a4c69bf18784c01e9181bbd070bd4b844ce.tar.gz |
devtool: refactor deploy-target
Make the deploy function independent from d. This allows to call the
function also from Python code not running in bitbake.
This is needed to for the devtool ide plugin which will call the
do_install task and the code from devtool deploy-target independently
from a bitbake server. This allows a much quicker workflow.
(From OE-Core rev: c8697d1132cbd4b2a2502b4c48e7d91fc18de786)
Signed-off-by: Adrian Freihofer <adrian.freihofer@siemens.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/lib/devtool/deploy.py | 236 |
1 files changed, 122 insertions, 114 deletions
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py index b71d55e231..b5ca8f2c2f 100644 --- a/scripts/lib/devtool/deploy.py +++ b/scripts/lib/devtool/deploy.py | |||
@@ -133,17 +133,38 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals | |||
133 | 133 | ||
134 | return '\n'.join(lines) | 134 | return '\n'.join(lines) |
135 | 135 | ||
136 | |||
137 | |||
138 | def deploy(args, config, basepath, workspace): | 136 | def deploy(args, config, basepath, workspace): |
139 | """Entry point for the devtool 'deploy' subcommand""" | 137 | """Entry point for the devtool 'deploy' subcommand""" |
140 | import math | ||
141 | import oe.recipeutils | ||
142 | import oe.package | ||
143 | import oe.utils | 138 | import oe.utils |
144 | 139 | ||
145 | check_workspace_recipe(workspace, args.recipename, checksrc=False) | 140 | check_workspace_recipe(workspace, args.recipename, checksrc=False) |
146 | 141 | ||
142 | tinfoil = setup_tinfoil(basepath=basepath) | ||
143 | try: | ||
144 | try: | ||
145 | rd = tinfoil.parse_recipe(args.recipename) | ||
146 | except Exception as e: | ||
147 | raise DevtoolError('Exception parsing recipe %s: %s' % | ||
148 | (args.recipename, e)) | ||
149 | |||
150 | srcdir = rd.getVar('D') | ||
151 | workdir = rd.getVar('WORKDIR') | ||
152 | path = rd.getVar('PATH') | ||
153 | strip_cmd = rd.getVar('STRIP') | ||
154 | libdir = rd.getVar('libdir') | ||
155 | base_libdir = rd.getVar('base_libdir') | ||
156 | max_process = oe.utils.get_bb_number_threads(rd) | ||
157 | fakerootcmd = rd.getVar('FAKEROOTCMD') | ||
158 | fakerootenv = rd.getVar('FAKEROOTENV') | ||
159 | finally: | ||
160 | tinfoil.shutdown() | ||
161 | |||
162 | return deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args) | ||
163 | |||
164 | def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args): | ||
165 | import math | ||
166 | import oe.package | ||
167 | |||
147 | try: | 168 | try: |
148 | host, destdir = args.target.split(':') | 169 | host, destdir = args.target.split(':') |
149 | except ValueError: | 170 | except ValueError: |
@@ -153,121 +174,108 @@ def deploy(args, config, basepath, workspace): | |||
153 | if not destdir.endswith('/'): | 174 | if not destdir.endswith('/'): |
154 | destdir += '/' | 175 | destdir += '/' |
155 | 176 | ||
156 | tinfoil = setup_tinfoil(basepath=basepath) | 177 | recipe_outdir = srcdir |
157 | try: | 178 | if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): |
158 | try: | 179 | raise DevtoolError('No files to deploy - have you built the %s ' |
159 | rd = tinfoil.parse_recipe(args.recipename) | 180 | 'recipe? If so, the install step has not installed ' |
160 | except Exception as e: | 181 | 'any files.' % args.recipename) |
161 | raise DevtoolError('Exception parsing recipe %s: %s' % | 182 | |
162 | (args.recipename, e)) | 183 | if args.strip and not args.dry_run: |
184 | # Fakeroot copy to new destination | ||
185 | srcdir = recipe_outdir | ||
186 | recipe_outdir = os.path.join(workdir, 'devtool-deploy-target-stripped') | ||
187 | if os.path.isdir(recipe_outdir): | ||
188 | exec_fakeroot_no_d(fakerootcmd, fakerootenv, "rm -rf %s" % recipe_outdir, shell=True) | ||
189 | exec_fakeroot_no_d(fakerootcmd, fakerootenv, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) | ||
190 | os.environ['PATH'] = ':'.join([os.environ['PATH'], path or '']) | ||
191 | oe.package.strip_execs(args.recipename, recipe_outdir, strip_cmd, libdir, base_libdir, max_process) | ||
192 | |||
193 | filelist = [] | ||
194 | inodes = set({}) | ||
195 | ftotalsize = 0 | ||
196 | for root, _, files in os.walk(recipe_outdir): | ||
197 | for fn in files: | ||
198 | fstat = os.lstat(os.path.join(root, fn)) | ||
199 | # Get the size in kiB (since we'll be comparing it to the output of du -k) | ||
200 | # MUST use lstat() here not stat() or getfilesize() since we don't want to | ||
201 | # dereference symlinks | ||
202 | if fstat.st_ino in inodes: | ||
203 | fsize = 0 | ||
204 | else: | ||
205 | fsize = int(math.ceil(float(fstat.st_size)/1024)) | ||
206 | inodes.add(fstat.st_ino) | ||
207 | ftotalsize += fsize | ||
208 | # The path as it would appear on the target | ||
209 | fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) | ||
210 | filelist.append((fpath, fsize)) | ||
211 | |||
212 | if args.dry_run: | ||
213 | print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) | ||
214 | for item, _ in filelist: | ||
215 | print(' %s' % item) | ||
216 | return 0 | ||
163 | 217 | ||
164 | recipe_outdir = rd.getVar('D') | 218 | extraoptions = '' |
165 | if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): | 219 | if args.no_host_check: |
166 | raise DevtoolError('No files to deploy - have you built the %s ' | 220 | extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' |
167 | 'recipe? If so, the install step has not installed ' | 221 | if not args.show_status: |
168 | 'any files.' % args.recipename) | 222 | extraoptions += ' -q' |
169 | 223 | ||
170 | fakerootcmd = rd.getVar('FAKEROOTCMD') | 224 | scp_sshexec = '' |
171 | fakerootenv = rd.getVar('FAKEROOTENV') | 225 | ssh_sshexec = 'ssh' |
172 | if args.strip and not args.dry_run: | 226 | if args.ssh_exec: |
173 | # Fakeroot copy to new destination | 227 | scp_sshexec = "-S %s" % args.ssh_exec |
174 | srcdir = recipe_outdir | 228 | ssh_sshexec = args.ssh_exec |
175 | recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'devtool-deploy-target-stripped') | 229 | scp_port = '' |
176 | if os.path.isdir(recipe_outdir): | 230 | ssh_port = '' |
177 | exec_fakeroot_no_d(fakerootcmd, fakerootenv, "rm -rf %s" % recipe_outdir, shell=True) | 231 | if args.port: |
178 | exec_fakeroot_no_d(fakerootcmd, fakerootenv, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) | 232 | scp_port = "-P %s" % args.port |
179 | os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) | 233 | ssh_port = "-p %s" % args.port |
180 | oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), | ||
181 | rd.getVar('base_libdir'), oe.utils.get_bb_number_threads(rd), rd) | ||
182 | |||
183 | filelist = [] | ||
184 | inodes = set({}) | ||
185 | ftotalsize = 0 | ||
186 | for root, _, files in os.walk(recipe_outdir): | ||
187 | for fn in files: | ||
188 | fstat = os.lstat(os.path.join(root, fn)) | ||
189 | # Get the size in kiB (since we'll be comparing it to the output of du -k) | ||
190 | # MUST use lstat() here not stat() or getfilesize() since we don't want to | ||
191 | # dereference symlinks | ||
192 | if fstat.st_ino in inodes: | ||
193 | fsize = 0 | ||
194 | else: | ||
195 | fsize = int(math.ceil(float(fstat.st_size)/1024)) | ||
196 | inodes.add(fstat.st_ino) | ||
197 | ftotalsize += fsize | ||
198 | # The path as it would appear on the target | ||
199 | fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn) | ||
200 | filelist.append((fpath, fsize)) | ||
201 | |||
202 | if args.dry_run: | ||
203 | print('Files to be deployed for %s on target %s:' % (args.recipename, args.target)) | ||
204 | for item, _ in filelist: | ||
205 | print(' %s' % item) | ||
206 | return 0 | ||
207 | |||
208 | extraoptions = '' | ||
209 | if args.no_host_check: | ||
210 | extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' | ||
211 | if not args.show_status: | ||
212 | extraoptions += ' -q' | ||
213 | |||
214 | scp_sshexec = '' | ||
215 | ssh_sshexec = 'ssh' | ||
216 | if args.ssh_exec: | ||
217 | scp_sshexec = "-S %s" % args.ssh_exec | ||
218 | ssh_sshexec = args.ssh_exec | ||
219 | scp_port = '' | ||
220 | ssh_port = '' | ||
221 | if args.port: | ||
222 | scp_port = "-P %s" % args.port | ||
223 | ssh_port = "-p %s" % args.port | ||
224 | |||
225 | if args.key: | ||
226 | extraoptions += ' -i %s' % args.key | ||
227 | |||
228 | # In order to delete previously deployed files and have the manifest file on | ||
229 | # the target, we write out a shell script and then copy it to the target | ||
230 | # so we can then run it (piping tar output to it). | ||
231 | # (We cannot use scp here, because it doesn't preserve symlinks.) | ||
232 | tmpdir = tempfile.mkdtemp(prefix='devtool') | ||
233 | try: | ||
234 | tmpscript = '/tmp/devtool_deploy.sh' | ||
235 | tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') | ||
236 | shellscript = _prepare_remote_script(deploy=True, | ||
237 | verbose=args.show_status, | ||
238 | nopreserve=args.no_preserve, | ||
239 | nocheckspace=args.no_check_space) | ||
240 | # Write out the script to a file | ||
241 | with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: | ||
242 | f.write(shellscript) | ||
243 | # Write out the file list | ||
244 | with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: | ||
245 | f.write('%d\n' % ftotalsize) | ||
246 | for fpath, fsize in filelist: | ||
247 | f.write('%s %d\n' % (fpath, fsize)) | ||
248 | # Copy them to the target | ||
249 | ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) | ||
250 | if ret != 0: | ||
251 | raise DevtoolError('Failed to copy script to %s - rerun with -s to ' | ||
252 | 'get a complete error message' % args.target) | ||
253 | finally: | ||
254 | shutil.rmtree(tmpdir) | ||
255 | 234 | ||
256 | # Now run the script | 235 | if args.key: |
257 | ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) | 236 | extraoptions += ' -i %s' % args.key |
237 | |||
238 | # In order to delete previously deployed files and have the manifest file on | ||
239 | # the target, we write out a shell script and then copy it to the target | ||
240 | # so we can then run it (piping tar output to it). | ||
241 | # (We cannot use scp here, because it doesn't preserve symlinks.) | ||
242 | tmpdir = tempfile.mkdtemp(prefix='devtool') | ||
243 | try: | ||
244 | tmpscript = '/tmp/devtool_deploy.sh' | ||
245 | tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list') | ||
246 | shellscript = _prepare_remote_script(deploy=True, | ||
247 | verbose=args.show_status, | ||
248 | nopreserve=args.no_preserve, | ||
249 | nocheckspace=args.no_check_space) | ||
250 | # Write out the script to a file | ||
251 | with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f: | ||
252 | f.write(shellscript) | ||
253 | # Write out the file list | ||
254 | with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f: | ||
255 | f.write('%d\n' % ftotalsize) | ||
256 | for fpath, fsize in filelist: | ||
257 | f.write('%s %d\n' % (fpath, fsize)) | ||
258 | # Copy them to the target | ||
259 | ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True) | ||
258 | if ret != 0: | 260 | if ret != 0: |
259 | raise DevtoolError('Deploy failed - rerun with -s to get a complete ' | 261 | raise DevtoolError('Failed to copy script to %s - rerun with -s to ' |
260 | 'error message') | 262 | 'get a complete error message' % args.target) |
263 | finally: | ||
264 | shutil.rmtree(tmpdir) | ||
261 | 265 | ||
262 | logger.info('Successfully deployed %s' % recipe_outdir) | 266 | # Now run the script |
267 | ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) | ||
268 | if ret != 0: | ||
269 | raise DevtoolError('Deploy failed - rerun with -s to get a complete ' | ||
270 | 'error message') | ||
263 | 271 | ||
264 | files_list = [] | 272 | logger.info('Successfully deployed %s' % recipe_outdir) |
265 | for root, _, files in os.walk(recipe_outdir): | 273 | |
266 | for filename in files: | 274 | files_list = [] |
267 | filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) | 275 | for root, _, files in os.walk(recipe_outdir): |
268 | files_list.append(os.path.join(destdir, filename)) | 276 | for filename in files: |
269 | finally: | 277 | filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) |
270 | tinfoil.shutdown() | 278 | files_list.append(os.path.join(destdir, filename)) |
271 | 279 | ||
272 | return 0 | 280 | return 0 |
273 | 281 | ||