summaryrefslogtreecommitdiffstats
path: root/scripts/lib/devtool/standard.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
commit8c22ff0d8b70d9b12f0487ef696a7e915b9e3173 (patch)
treeefdc32587159d0050a69009bdf2330a531727d95 /scripts/lib/devtool/standard.py
parentd412d2747595c1cc4a5e3ca975e3adc31b2f7891 (diff)
downloadpoky-8c22ff0d8b70d9b12f0487ef696a7e915b9e3173.tar.gz
The poky repository master branch is no longer being updated.
You can either: a) switch to individual clones of bitbake, openembedded-core, meta-yocto and yocto-docs b) use the new bitbake-setup You can find information about either approach in our documentation: https://docs.yoctoproject.org/ Note that "poky" the distro setting is still available in meta-yocto as before and we continue to use and maintain that. Long live Poky! Some further information on the background of this change can be found in: https://lists.openembedded.org/g/openembedded-architecture/message/2179 Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/devtool/standard.py')
-rw-r--r--scripts/lib/devtool/standard.py2396
1 files changed, 0 insertions, 2396 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
deleted file mode 100644
index 1fd5947c41..0000000000
--- a/scripts/lib/devtool/standard.py
+++ /dev/null
@@ -1,2396 +0,0 @@
1# Development tool - standard commands plugin
2#
3# Copyright (C) 2014-2017 Intel Corporation
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7"""Devtool standard plugins"""
8
9import os
10import sys
11import re
12import shutil
13import subprocess
14import tempfile
15import logging
16import argparse
17import argparse_oe
18import scriptutils
19import errno
20import glob
21from collections import OrderedDict
22
23from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError
24from devtool import parse_recipe
25
26import bb.utils
27
28logger = logging.getLogger('devtool')
29
30override_branch_prefix = 'devtool-override-'
31
32
33def add(args, config, basepath, workspace):
34 """Entry point for the devtool 'add' subcommand"""
35 import bb.data
36 import bb.process
37 import oe.recipeutils
38
39 if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri:
40 raise argparse_oe.ArgumentUsageError('At least one of recipename, srctree, fetchuri or -f/--fetch must be specified', 'add')
41
42 # These are positional arguments, but because we're nice, allow
43 # specifying e.g. source tree without name, or fetch URI without name or
44 # source tree (if we can detect that that is what the user meant)
45 if scriptutils.is_src_url(args.recipename):
46 if not args.fetchuri:
47 if args.fetch:
48 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
49 args.fetchuri = args.recipename
50 args.recipename = ''
51 elif scriptutils.is_src_url(args.srctree):
52 if not args.fetchuri:
53 if args.fetch:
54 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
55 args.fetchuri = args.srctree
56 args.srctree = ''
57 elif args.recipename and not args.srctree:
58 if os.sep in args.recipename:
59 args.srctree = args.recipename
60 args.recipename = None
61 elif os.path.isdir(args.recipename):
62 logger.warning('Ambiguous argument "%s" - assuming you mean it to be the recipe name' % args.recipename)
63
64 if not args.fetchuri:
65 if args.srcrev:
66 raise DevtoolError('The -S/--srcrev option is only valid when fetching from an SCM repository')
67 if args.srcbranch:
68 raise DevtoolError('The -B/--srcbranch option is only valid when fetching from an SCM repository')
69
70 if args.srctree and os.path.isfile(args.srctree):
71 args.fetchuri = 'file://' + os.path.abspath(args.srctree)
72 args.srctree = ''
73
74 if args.fetch:
75 if args.fetchuri:
76 raise DevtoolError('URI specified as positional argument as well as -f/--fetch')
77 else:
78 logger.warning('-f/--fetch option is deprecated - you can now simply specify the URL to fetch as a positional argument instead')
79 args.fetchuri = args.fetch
80
81 if args.recipename:
82 if args.recipename in workspace:
83 raise DevtoolError("recipe %s is already in your workspace" %
84 args.recipename)
85 reason = oe.recipeutils.validate_pn(args.recipename)
86 if reason:
87 raise DevtoolError(reason)
88
89 if args.srctree:
90 srctree = os.path.abspath(args.srctree)
91 srctreeparent = None
92 tmpsrcdir = None
93 else:
94 srctree = None
95 srctreeparent = get_default_srctree(config)
96 bb.utils.mkdirhier(srctreeparent)
97 tmpsrcdir = tempfile.mkdtemp(prefix='devtoolsrc', dir=srctreeparent)
98
99 if srctree and os.path.exists(srctree):
100 if args.fetchuri:
101 if not os.path.isdir(srctree):
102 raise DevtoolError("Cannot fetch into source tree path %s as "
103 "it exists and is not a directory" %
104 srctree)
105 elif os.listdir(srctree):
106 raise DevtoolError("Cannot fetch into source tree path %s as "
107 "it already exists and is non-empty" %
108 srctree)
109 elif not args.fetchuri:
110 if args.srctree:
111 raise DevtoolError("Specified source tree %s could not be found" %
112 args.srctree)
113 elif srctree:
114 raise DevtoolError("No source tree exists at default path %s - "
115 "either create and populate this directory, "
116 "or specify a path to a source tree, or a "
117 "URI to fetch source from" % srctree)
118 else:
119 raise DevtoolError("You must either specify a source tree "
120 "or a URI to fetch source from")
121
122 if args.version:
123 if '_' in args.version or ' ' in args.version:
124 raise DevtoolError('Invalid version string "%s"' % args.version)
125
126 if args.color == 'auto' and sys.stdout.isatty():
127 color = 'always'
128 else:
129 color = args.color
130 extracmdopts = ''
131 if args.fetchuri:
132 source = args.fetchuri
133 if srctree:
134 extracmdopts += ' -x %s' % srctree
135 else:
136 extracmdopts += ' -x %s' % tmpsrcdir
137 else:
138 source = srctree
139 if args.recipename:
140 extracmdopts += ' -N %s' % args.recipename
141 if args.version:
142 extracmdopts += ' -V %s' % args.version
143 if args.binary:
144 extracmdopts += ' -b'
145 if args.also_native:
146 extracmdopts += ' --also-native'
147 if args.src_subdir:
148 extracmdopts += ' --src-subdir "%s"' % args.src_subdir
149 if args.autorev:
150 extracmdopts += ' -a'
151 if args.npm_dev:
152 extracmdopts += ' --npm-dev'
153 if args.no_pypi:
154 extracmdopts += ' --no-pypi'
155 if args.mirrors:
156 extracmdopts += ' --mirrors'
157 if args.srcrev:
158 extracmdopts += ' --srcrev %s' % args.srcrev
159 if args.srcbranch:
160 extracmdopts += ' --srcbranch %s' % args.srcbranch
161 if args.provides:
162 extracmdopts += ' --provides %s' % args.provides
163
164 tempdir = tempfile.mkdtemp(prefix='devtool')
165 try:
166 try:
167 stdout, _ = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create --devtool -o %s \'%s\' %s' % (color, tempdir, source, extracmdopts), watch=True)
168 except bb.process.ExecutionError as e:
169 if e.exitcode == 15:
170 raise DevtoolError('Could not auto-determine recipe name, please specify it on the command line')
171 else:
172 raise DevtoolError('Command \'%s\' failed' % e.command)
173
174 recipes = glob.glob(os.path.join(tempdir, '*.bb'))
175 if recipes:
176 recipename = os.path.splitext(os.path.basename(recipes[0]))[0].split('_')[0]
177 if recipename in workspace:
178 raise DevtoolError('A recipe with the same name as the one being created (%s) already exists in your workspace' % recipename)
179 recipedir = os.path.join(config.workspace_path, 'recipes', recipename)
180 bb.utils.mkdirhier(recipedir)
181 recipefile = os.path.join(recipedir, os.path.basename(recipes[0]))
182 appendfile = recipe_to_append(recipefile, config)
183 if os.path.exists(appendfile):
184 # This shouldn't be possible, but just in case
185 raise DevtoolError('A recipe with the same name as the one being created already exists in your workspace')
186 if os.path.exists(recipefile):
187 raise DevtoolError('A recipe file %s already exists in your workspace; this shouldn\'t be there - please delete it before continuing' % recipefile)
188 if tmpsrcdir:
189 srctree = os.path.join(srctreeparent, recipename)
190 if os.path.exists(tmpsrcdir):
191 if os.path.exists(srctree):
192 if os.path.isdir(srctree):
193 try:
194 os.rmdir(srctree)
195 except OSError as e:
196 if e.errno == errno.ENOTEMPTY:
197 raise DevtoolError('Source tree path %s already exists and is not empty' % srctree)
198 else:
199 raise
200 else:
201 raise DevtoolError('Source tree path %s already exists and is not a directory' % srctree)
202 logger.info('Using default source tree path %s' % srctree)
203 shutil.move(tmpsrcdir, srctree)
204 else:
205 raise DevtoolError('Couldn\'t find source tree created by recipetool')
206 bb.utils.mkdirhier(recipedir)
207 shutil.move(recipes[0], recipefile)
208 # Move any additional files created by recipetool
209 for fn in os.listdir(tempdir):
210 shutil.move(os.path.join(tempdir, fn), recipedir)
211 else:
212 raise DevtoolError(f'Failed to create a recipe file for source {source}')
213 attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
214 if os.path.exists(attic_recipe):
215 logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe)
216 finally:
217 if tmpsrcdir and os.path.exists(tmpsrcdir):
218 shutil.rmtree(tmpsrcdir)
219 shutil.rmtree(tempdir)
220
221 for fn in os.listdir(recipedir):
222 _add_md5(config, recipename, os.path.join(recipedir, fn))
223
224 tinfoil = setup_tinfoil(config_only=True, basepath=basepath)
225 try:
226 try:
227 rd = tinfoil.parse_recipe_file(recipefile, False)
228 except Exception as e:
229 logger.error(str(e))
230 rd = None
231 if not rd:
232 # Parsing failed. We just created this recipe and we shouldn't
233 # leave it in the workdir or it'll prevent bitbake from starting
234 movefn = '%s.parsefailed' % recipefile
235 logger.error('Parsing newly created recipe failed, moving recipe to %s for reference. If this looks to be caused by the recipe itself, please report this error.' % movefn)
236 shutil.move(recipefile, movefn)
237 return 1
238
239 if args.fetchuri and not args.no_git:
240 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data)
241
242 initial_rev = {}
243 if os.path.exists(os.path.join(srctree, '.git')):
244 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
245 initial_rev["."] = stdout.rstrip()
246 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree)
247 for line in stdout.splitlines():
248 (rev, submodule) = line.split()
249 initial_rev[os.path.relpath(submodule, srctree)] = rev
250
251 if args.src_subdir:
252 srctree = os.path.join(srctree, args.src_subdir)
253
254 bb.utils.mkdirhier(os.path.dirname(appendfile))
255 with open(appendfile, 'w') as f:
256 f.write('inherit externalsrc\n')
257 f.write('EXTERNALSRC = "%s"\n' % srctree)
258
259 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
260 if b_is_s:
261 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
262 if initial_rev:
263 for key, value in initial_rev.items():
264 f.write('\n# initial_rev %s: %s\n' % (key, value))
265
266 if args.binary:
267 f.write('do_install:append() {\n')
268 f.write(' rm -rf ${D}/.git\n')
269 f.write(' rm -f ${D}/singletask.lock\n')
270 f.write('}\n')
271
272 if bb.data.inherits_class('npm', rd):
273 f.write('python do_configure:append() {\n')
274 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n')
275 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n')
276 f.write(' bb.utils.remove(lockfile)\n')
277 f.write('}\n')
278
279 # Check if the new layer provides recipes whose priorities have been
280 # overriden by PREFERRED_PROVIDER.
281 recipe_name = rd.getVar('PN')
282 provides = rd.getVar('PROVIDES')
283 # Search every item defined in PROVIDES
284 for recipe_provided in provides.split():
285 preferred_provider = 'PREFERRED_PROVIDER_' + recipe_provided
286 current_pprovider = rd.getVar(preferred_provider)
287 if current_pprovider and current_pprovider != recipe_name:
288 if args.fixed_setup:
289 #if we are inside the eSDK add the new PREFERRED_PROVIDER in the workspace layer.conf
290 layerconf_file = os.path.join(config.workspace_path, "conf", "layer.conf")
291 with open(layerconf_file, 'a') as f:
292 f.write('%s = "%s"\n' % (preferred_provider, recipe_name))
293 else:
294 logger.warning('Set \'%s\' in order to use the recipe' % preferred_provider)
295 break
296
297 _add_md5(config, recipename, appendfile)
298
299 check_prerelease_version(rd.getVar('PV'), 'devtool add')
300
301 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
302
303 finally:
304 tinfoil.shutdown()
305
306 return 0
307
308
309def _check_compatible_recipe(pn, d):
310 """Check if the recipe is supported by devtool"""
311 import bb.data
312 if pn == 'perf':
313 raise DevtoolError("The perf recipe does not actually check out "
314 "source and thus cannot be supported by this tool",
315 4)
316
317 if pn in ['kernel-devsrc', 'package-index'] or pn.startswith('gcc-source'):
318 raise DevtoolError("The %s recipe is not supported by this tool" % pn, 4)
319
320 if bb.data.inherits_class('image', d):
321 raise DevtoolError("The %s recipe is an image, and therefore is not "
322 "supported by this tool" % pn, 4)
323
324 if bb.data.inherits_class('populate_sdk', d):
325 raise DevtoolError("The %s recipe is an SDK, and therefore is not "
326 "supported by this tool" % pn, 4)
327
328 if bb.data.inherits_class('packagegroup', d):
329 raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
330 "not supported by this tool" % pn, 4)
331
332 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
333 # Not an incompatibility error per se, so we don't pass the error code
334 raise DevtoolError("externalsrc is currently enabled for the %s "
335 "recipe. This prevents the normal do_patch task "
336 "from working. You will need to disable this "
337 "first." % pn)
338
339def _dry_run_copy(src, dst, dry_run_outdir, base_outdir):
340 """Common function for copying a file to the dry run output directory"""
341 relpath = os.path.relpath(dst, base_outdir)
342 if relpath.startswith('..'):
343 raise Exception('Incorrect base path %s for path %s' % (base_outdir, dst))
344 dst = os.path.join(dry_run_outdir, relpath)
345 dst_d = os.path.dirname(dst)
346 if dst_d:
347 bb.utils.mkdirhier(dst_d)
348 # Don't overwrite existing files, otherwise in the case of an upgrade
349 # the dry-run written out recipe will be overwritten with an unmodified
350 # version
351 if not os.path.exists(dst):
352 shutil.copy(src, dst)
353
354def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
355 """Move a file. Creates all the directory components of destination path."""
356 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
357 logger.debug('Moving %s to %s%s' % (src, dst, dry_run_suffix))
358 if dry_run_outdir:
359 # We want to copy here, not move
360 _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
361 else:
362 dst_d = os.path.dirname(dst)
363 if dst_d:
364 bb.utils.mkdirhier(dst_d)
365 shutil.move(src, dst)
366
367def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None):
368 """Copy a file. Creates all the directory components of destination path."""
369 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
370 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
371 if dry_run_outdir:
372 _dry_run_copy(src, dst, dry_run_outdir, base_outdir)
373 else:
374 dst_d = os.path.dirname(dst)
375 if dst_d:
376 bb.utils.mkdirhier(dst_d)
377 shutil.copy(src, dst)
378
379def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
380 """List contents of a git treeish"""
381 import bb.process
382 cmd = ['git', 'ls-tree', '-z', treeish]
383 if recursive:
384 cmd.append('-r')
385 out, _ = bb.process.run(cmd, cwd=repodir)
386 ret = {}
387 if out:
388 for line in out.split('\0'):
389 if line:
390 split = line.split(None, 4)
391 ret[split[3]] = split[0:3]
392 return ret
393
394def _git_modified(repodir):
395 """List the difference between HEAD and the index"""
396 import bb.process
397 cmd = ['git', 'status', '--porcelain']
398 out, _ = bb.process.run(cmd, cwd=repodir)
399 ret = []
400 if out:
401 for line in out.split("\n"):
402 if line and not line.startswith('??'):
403 ret.append(line[3:])
404 return ret
405
406
407def _git_exclude_path(srctree, path):
408 """Return pathspec (list of paths) that excludes certain path"""
409 # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
410 # we don't catch files that are deleted, for example. A more reliable way
411 # to implement this would be to use "negative pathspecs" which were
412 # introduced in Git v1.9.0. Revisit this when/if the required Git version
413 # becomes greater than that.
414 path = os.path.normpath(path)
415 recurse = True if len(path.split(os.path.sep)) > 1 else False
416 git_files = list(_git_ls_tree(srctree, 'HEAD', recurse).keys())
417 if path in git_files:
418 git_files.remove(path)
419 return git_files
420 else:
421 return ['.']
422
423def _ls_tree(directory):
424 """Recursive listing of files in a directory"""
425 ret = []
426 for root, dirs, files in os.walk(directory):
427 ret.extend([os.path.relpath(os.path.join(root, fname), directory) for
428 fname in files])
429 return ret
430
431
432def extract(args, config, basepath, workspace):
433 """Entry point for the devtool 'extract' subcommand"""
434 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
435 if not tinfoil:
436 # Error already shown
437 return 1
438 try:
439 rd = parse_recipe(config, tinfoil, args.recipename, True)
440 if not rd:
441 return 1
442
443 srctree = os.path.abspath(args.srctree)
444 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
445 logger.info('Source tree extracted to %s' % srctree)
446
447 if initial_rev:
448 return 0
449 else:
450 return 1
451 finally:
452 tinfoil.shutdown()
453
454def sync(args, config, basepath, workspace):
455 """Entry point for the devtool 'sync' subcommand"""
456 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
457 if not tinfoil:
458 # Error already shown
459 return 1
460 try:
461 rd = parse_recipe(config, tinfoil, args.recipename, True)
462 if not rd:
463 return 1
464
465 srctree = os.path.abspath(args.srctree)
466 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, True, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=True)
467 logger.info('Source tree %s synchronized' % srctree)
468
469 if initial_rev:
470 return 0
471 else:
472 return 1
473 finally:
474 tinfoil.shutdown()
475
476def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
477 """Extract sources of a recipe"""
478 import oe.path
479 import bb.data
480 import bb.process
481
482 pn = d.getVar('PN')
483
484 _check_compatible_recipe(pn, d)
485
486 if sync:
487 if not os.path.exists(srctree):
488 raise DevtoolError("output path %s does not exist" % srctree)
489 else:
490 if os.path.exists(srctree):
491 if not os.path.isdir(srctree):
492 raise DevtoolError("output path %s exists and is not a directory" %
493 srctree)
494 elif os.listdir(srctree):
495 raise DevtoolError("output path %s already exists and is "
496 "non-empty" % srctree)
497
498 if 'noexec' in (d.getVarFlags('do_unpack', False) or []):
499 raise DevtoolError("The %s recipe has do_unpack disabled, unable to "
500 "extract source" % pn, 4)
501
502 if not sync:
503 # Prepare for shutil.move later on
504 bb.utils.mkdirhier(srctree)
505 os.rmdir(srctree)
506
507 extra_overrides = []
508 if not no_overrides:
509 history = d.varhistory.variable('SRC_URI')
510 for event in history:
511 if not 'flag' in event:
512 if event['op'].startswith((':append[', ':prepend[')):
513 override = event['op'].split('[')[1].split(']')[0]
514 if not override.startswith('pn-'):
515 extra_overrides.append(override)
516 # We want to remove duplicate overrides. If a recipe had multiple
517 # SRC_URI_override += values it would cause mulitple instances of
518 # overrides. This doesn't play nicely with things like creating a
519 # branch for every instance of DEVTOOL_EXTRA_OVERRIDES.
520 extra_overrides = list(set(extra_overrides))
521 if extra_overrides:
522 logger.info('SRC_URI contains some conditional appends/prepends - will create branches to represent these')
523
524 initial_rev = None
525
526 recipefile = d.getVar('FILE')
527 appendfile = recipe_to_append(recipefile, config)
528 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
529
530 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary
531 # directory so that:
532 # (a) we pick up all files that get unpacked to the WORKDIR, and
533 # (b) we don't disturb the existing build
534 # However, with recipe-specific sysroots the sysroots for the recipe
535 # will be prepared under WORKDIR, and if we used the system temporary
536 # directory (i.e. usually /tmp) as used by mkdtemp by default, then
537 # our attempts to hardlink files into the recipe-specific sysroots
538 # will fail on systems where /tmp is a different filesystem, and it
539 # would have to fall back to copying the files which is a waste of
540 # time. Put the temp directory under the WORKDIR to prevent that from
541 # being a problem.
542 tempbasedir = d.getVar('WORKDIR')
543 bb.utils.mkdirhier(tempbasedir)
544 tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir)
545 appendbackup = None
546 try:
547 tinfoil.logger.setLevel(logging.WARNING)
548
549 # FIXME this results in a cache reload under control of tinfoil, which is fine
550 # except we don't get the knotty progress bar
551
552 if os.path.exists(appendfile):
553 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak')
554 shutil.copyfile(appendfile, appendbackup)
555 else:
556 bb.utils.mkdirhier(os.path.dirname(appendfile))
557 logger.debug('writing append file %s' % appendfile)
558 with open(appendfile, 'a') as f:
559 f.write('###--- _extract_source\n')
560 f.write('deltask do_recipe_qa\n')
561 f.write('deltask do_recipe_qa_setscene\n')
562 f.write('ERROR_QA:remove = "patch-fuzz"\n')
563 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir)
564 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch)
565 if not is_kernel_yocto:
566 f.write('PATCHTOOL = "git"\n')
567 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n')
568 if extra_overrides:
569 f.write('DEVTOOL_EXTRA_OVERRIDES = "%s"\n' % ':'.join(extra_overrides))
570 f.write('inherit devtool-source\n')
571 f.write('###--- _extract_source\n')
572
573 update_unlockedsigs(basepath, workspace, fixed_setup, [pn])
574
575 sstate_manifests = d.getVar('SSTATE_MANIFESTS')
576 bb.utils.mkdirhier(sstate_manifests)
577 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps')
578 with open(preservestampfile, 'w') as f:
579 f.write(d.getVar('STAMP'))
580 tinfoil.modified_files()
581 try:
582 if is_kernel_yocto:
583 # We need to generate the kernel config
584 task = 'do_configure'
585 else:
586 task = 'do_patch'
587
588 if 'noexec' in (d.getVarFlags(task, False) or []) or 'task' not in (d.getVarFlags(task, False) or []):
589 logger.info('The %s recipe has %s disabled. Running only '
590 'do_configure task dependencies' % (pn, task))
591
592 if 'depends' in d.getVarFlags('do_configure', False):
593 pn = d.getVarFlags('do_configure', False)['depends']
594 pn = pn.replace('${PV}', d.getVar('PV'))
595 pn = pn.replace('${COMPILERDEP}', d.getVar('COMPILERDEP'))
596 task = None
597
598 # Run the fetch + unpack tasks
599 res = tinfoil.build_targets(pn,
600 task,
601 handle_events=True)
602 finally:
603 if os.path.exists(preservestampfile):
604 os.remove(preservestampfile)
605
606 if not res:
607 raise DevtoolError('Extracting source for %s failed' % pn)
608
609 if not is_kernel_yocto and ('noexec' in (d.getVarFlags('do_patch', False) or []) or 'task' not in (d.getVarFlags('do_patch', False) or [])):
610 workshareddir = d.getVar('S')
611 if os.path.islink(srctree):
612 os.unlink(srctree)
613
614 os.symlink(workshareddir, srctree)
615
616 # The initial_rev file is created in devtool_post_unpack function that will not be executed if
617 # do_unpack/do_patch tasks are disabled so we have to directly say that source extraction was successful
618 return True, True
619
620 try:
621 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
622 initial_rev = f.read()
623
624 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
625 srcsubdir = f.read()
626 except FileNotFoundError as e:
627 raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e))
628 srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir', os.path.relpath(d.getVar('UNPACKDIR'), d.getVar('WORKDIR'))))
629
630 # Check if work-shared is empty, if yes
631 # find source and copy to work-shared
632 if is_kernel_yocto:
633 workshareddir = d.getVar('STAGING_KERNEL_DIR')
634 staging_kerVer = get_staging_kver(workshareddir)
635 kernelVersion = d.getVar('LINUX_VERSION')
636
637 # handle dangling symbolic link in work-shared:
638 if os.path.islink(workshareddir):
639 os.unlink(workshareddir)
640
641 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer):
642 shutil.rmtree(workshareddir)
643 oe.path.copyhardlinktree(srcsubdir, workshareddir)
644 elif not os.path.exists(workshareddir):
645 oe.path.copyhardlinktree(srcsubdir, workshareddir)
646
647 if sync:
648 try:
649 logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch))
650 bb.process.run('git branch -f ' + devbranch + '.bak', cwd=srctree)
651
652 # Use git fetch to update the source with the current recipe
653 # To be able to update the currently checked out branch with
654 # possibly new history (no fast-forward) git needs to be told
655 # that's ok
656 logger.info('Syncing source files including patches to git branch: %s' % devbranch)
657 bb.process.run('git fetch --update-head-ok --force file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
658 except bb.process.ExecutionError as e:
659 raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e))
660
661 else:
662 shutil.move(srcsubdir, srctree)
663
664 if is_kernel_yocto:
665 logger.info('Copying kernel config to srctree')
666 shutil.copy2(os.path.join(tempdir, '.config'), srctree)
667
668 finally:
669 if appendbackup:
670 shutil.copyfile(appendbackup, appendfile)
671 elif os.path.exists(appendfile):
672 os.remove(appendfile)
673 if keep_temp:
674 logger.info('Preserving temporary directory %s' % tempdir)
675 else:
676 shutil.rmtree(tempdir)
677 return initial_rev, srcsubdir_rel
678
679def _add_md5(config, recipename, filename):
680 """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""
681 def addfile(fn):
682 md5 = bb.utils.md5_file(fn)
683 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f:
684 md5_str = '%s|%s|%s\n' % (recipename, os.path.relpath(fn, config.workspace_path), md5)
685 f.seek(0, os.SEEK_SET)
686 if not md5_str in f.read():
687 f.write(md5_str)
688
689 if os.path.isdir(filename):
690 for root, _, files in os.walk(filename):
691 for f in files:
692 addfile(os.path.join(root, f))
693 else:
694 addfile(filename)
695
696def _check_preserve(config, recipename):
697 """Check if a file was manually changed and needs to be saved in 'attic'
698 directory"""
699 origfile = os.path.join(config.workspace_path, '.devtool_md5')
700 newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
701 preservepath = os.path.join(config.workspace_path, 'attic', recipename)
702 with open(origfile, 'r') as f:
703 with open(newfile, 'w') as tf:
704 for line in f.readlines():
705 splitline = line.rstrip().split('|')
706 if splitline[0] == recipename:
707 removefile = os.path.join(config.workspace_path, splitline[1])
708 try:
709 md5 = bb.utils.md5_file(removefile)
710 except IOError as err:
711 if err.errno == 2:
712 # File no longer exists, skip it
713 continue
714 else:
715 raise
716 if splitline[2] != md5:
717 bb.utils.mkdirhier(preservepath)
718 preservefile = os.path.basename(removefile)
719 logger.warning('File %s modified since it was written, preserving in %s' % (preservefile, preservepath))
720 shutil.move(removefile, os.path.join(preservepath, preservefile))
721 else:
722 os.remove(removefile)
723 else:
724 tf.write(line)
725 bb.utils.rename(newfile, origfile)
726
727def get_staging_kver(srcdir):
728 # Kernel version from work-shared
729 import itertools
730 try:
731 with open(os.path.join(srcdir, "Makefile")) as f:
732 # Take VERSION, PATCHLEVEL, SUBLEVEL from lines 1, 2, 3
733 return ".".join(line.rstrip().split('= ')[1] for line in itertools.islice(f, 1, 4))
734 except FileNotFoundError:
735 return ""
736
737def get_staging_kbranch(srcdir):
738 import bb.process
739 staging_kbranch = ""
740 if os.path.exists(srcdir) and os.listdir(srcdir):
741 (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir)
742 staging_kbranch = "".join(branch.split('\n')[0])
743 return staging_kbranch
744
745def get_real_srctree(srctree, s, unpackdir):
746 # Check that recipe isn't using a shared workdir
747 s = os.path.abspath(s)
748 unpackdir = os.path.abspath(unpackdir)
749 if s.startswith(unpackdir) and s != unpackdir and os.path.dirname(s) != unpackdir:
750 # Handle if S is set to a subdirectory of the source
751 srcsubdir = os.path.relpath(s, unpackdir).split(os.sep, 1)[1]
752 srctree = os.path.join(srctree, srcsubdir)
753 return srctree
754
755def modify(args, config, basepath, workspace):
756 """Entry point for the devtool 'modify' subcommand"""
757 import bb.data
758 import bb.process
759 import oe.recipeutils
760 import oe.patch
761 import oe.path
762
763 if args.recipename in workspace:
764 raise DevtoolError("recipe %s is already in your workspace" %
765 args.recipename)
766
767 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
768 try:
769 rd = parse_recipe(config, tinfoil, args.recipename, True)
770 if not rd:
771 return 1
772
773 pn = rd.getVar('PN')
774 if pn != args.recipename:
775 logger.info('Mapping %s to %s' % (args.recipename, pn))
776 if pn in workspace:
777 raise DevtoolError("recipe %s is already in your workspace" %
778 pn)
779
780 if args.srctree:
781 srctree = os.path.abspath(args.srctree)
782 else:
783 srctree = get_default_srctree(config, pn)
784
785 if args.no_extract and not os.path.isdir(srctree):
786 raise DevtoolError("--no-extract specified and source path %s does "
787 "not exist or is not a directory" %
788 srctree)
789
790 recipefile = rd.getVar('FILE')
791 appendfile = recipe_to_append(recipefile, config, args.wildcard)
792 if os.path.exists(appendfile):
793 raise DevtoolError("Another variant of recipe %s is already in your "
794 "workspace (only one variant of a recipe can "
795 "currently be worked on at once)"
796 % pn)
797
798 _check_compatible_recipe(pn, rd)
799
800 initial_revs = {}
801 commits = {}
802 check_commits = False
803
804 if bb.data.inherits_class('kernel-yocto', rd):
805 # Current set kernel version
806 kernelVersion = rd.getVar('LINUX_VERSION')
807 srcdir = rd.getVar('STAGING_KERNEL_DIR')
808 kbranch = rd.getVar('KBRANCH')
809
810 staging_kerVer = get_staging_kver(srcdir)
811 staging_kbranch = get_staging_kbranch(srcdir)
812 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch):
813 oe.path.copyhardlinktree(srcdir, srctree)
814 unpackdir = rd.getVar('UNPACKDIR')
815 srcsubdir = rd.getVar('S')
816
817 # Add locally copied files to gitignore as we add back to the metadata directly
818 local_files = oe.recipeutils.get_recipe_local_files(rd)
819 srcabspath = os.path.abspath(srcsubdir)
820 local_files = [fname for fname in local_files if
821 os.path.exists(os.path.join(unpackdir, fname)) and
822 srcabspath == unpackdir]
823 if local_files:
824 with open(os.path.join(srctree, '.gitignore'), 'a+') as f:
825 f.write('# Ignore local files, by default. Remove following lines'
826 'if you want to commit the directory to Git\n')
827 for fname in local_files:
828 f.write('%s\n' % fname)
829
830 task = 'do_configure'
831 res = tinfoil.build_targets(pn, task, handle_events=True)
832
833 # Copy .config to workspace
834 kconfpath = rd.getVar('B')
835 logger.info('Copying kernel config to workspace')
836 shutil.copy2(os.path.join(kconfpath, '.config'), srctree)
837
838 # Set this to true, we still need to get initial_rev
839 # by parsing the git repo
840 args.no_extract = True
841
842 if not args.no_extract:
843 initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
844 if not initial_revs["."]:
845 return 1
846 logger.info('Source tree extracted to %s' % srctree)
847
848 if os.path.exists(os.path.join(srctree, '.git')):
849 # Get list of commits since this revision
850 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree)
851 commits["."] = stdout.split()
852 check_commits = True
853 try:
854 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree)
855 except bb.process.ExecutionError:
856 stdout = ""
857 for line in stdout.splitlines():
858 (rev, submodule_path) = line.split()
859 submodule = os.path.relpath(submodule_path, srctree)
860 initial_revs[submodule] = rev
861 (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path)
862 commits[submodule] = stdout.split()
863 else:
864 if os.path.exists(os.path.join(srctree, '.git')):
865 # Check if it's a tree previously extracted by us. This is done
866 # by ensuring that devtool-base and args.branch (devtool) exist.
867 # The check_commits logic will cause an exception if either one
868 # of these doesn't exist
869 try:
870 (stdout, _) = bb.process.run('git branch --contains devtool-base', cwd=srctree)
871 bb.process.run('git rev-parse %s' % args.branch, cwd=srctree)
872 except bb.process.ExecutionError:
873 stdout = ''
874 if stdout:
875 check_commits = True
876 for line in stdout.splitlines():
877 if line.startswith('*'):
878 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
879 initial_revs["."] = stdout.rstrip()
880 if "." not in initial_revs:
881 # Otherwise, just grab the head revision
882 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
883 initial_revs["."] = stdout.rstrip()
884
885 branch_patches = {}
886 if check_commits:
887 # Check if there are override branches
888 (stdout, _) = bb.process.run('git branch', cwd=srctree)
889 branches = []
890 for line in stdout.rstrip().splitlines():
891 branchname = line[2:].rstrip()
892 if branchname.startswith(override_branch_prefix):
893 branches.append(branchname)
894 if branches:
895 logger.warning('SRC_URI is conditionally overridden in this recipe, thus several %s* branches have been created, one for each override that makes changes to SRC_URI. It is recommended that you make changes to the %s branch first, then checkout and rebase each %s* branch and update any unique patches there (duplicates on those branches will be ignored by devtool finish/update-recipe)' % (override_branch_prefix, args.branch, override_branch_prefix))
896 branches.insert(0, args.branch)
897 seen_patches = []
898 for branch in branches:
899 branch_patches[branch] = []
900 (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree)
901 for sha1 in stdout.splitlines():
902 notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip())
903 origpatch = notes.get(oe.patch.GitApplyTree.original_patch)
904 if origpatch and origpatch not in seen_patches:
905 seen_patches.append(origpatch)
906 branch_patches[branch].append(origpatch)
907
908 # Need to grab this here in case the source is within a subdirectory
909 srctreebase = srctree
910 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('UNPACKDIR'))
911
912 bb.utils.mkdirhier(os.path.dirname(appendfile))
913 with open(appendfile, 'w') as f:
914 # if not present, add type=git-dependency to the secondary sources
915 # (non local files) so they can be extracted correctly when building a recipe after
916 # doing a devtool modify on it
917 src_uri = rd.getVar('SRC_URI').split()
918 src_uri_append = []
919 src_uri_remove = []
920
921 # Assume first entry is main source extracted in ${S} so skip it
922 src_uri = src_uri[1::]
923
924 # Add "type=git-dependency" to all non local sources
925 for url in src_uri:
926 if not url.startswith('file://') and not 'type=' in url:
927 src_uri_remove.append(url)
928 src_uri_append.append('%s;type=git-dependency' % url)
929
930 if src_uri_remove:
931 f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove))
932 f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append))
933
934 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n')
935 # Local files can be modified/tracked in separate subdir under srctree
936 # Mostly useful for packages with S != WORKDIR
937 f.write('FILESPATH:prepend := "%s:"\n' %
938 os.path.join(srctreebase, 'oe-local-files'))
939 f.write('# srctreebase: %s\n' % srctreebase)
940
941 f.write('\ninherit externalsrc\n')
942 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
943 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
944
945 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
946 if b_is_s:
947 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
948
949 if bb.data.inherits_class('kernel', rd):
950 f.write('\ndo_kernel_configme:prepend() {\n'
951 ' if [ -e ${S}/.config ]; then\n'
952 ' mv ${S}/.config ${S}/.config.old\n'
953 ' fi\n'
954 '}\n')
955 if rd.getVarFlag('do_menuconfig', 'task'):
956 f.write('\ndo_configure:append() {\n'
957 ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n'
958 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n'
959 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n'
960 ' fi\n'
961 '}\n')
962 if initial_revs:
963 for name, rev in initial_revs.items():
964 f.write('\n# initial_rev %s: %s\n' % (name, rev))
965 if name in commits:
966 for commit in commits[name]:
967 f.write('# commit %s: %s\n' % (name, commit))
968 if branch_patches:
969 for branch in branch_patches:
970 if branch == args.branch:
971 continue
972 f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch])))
973 if args.debug_build:
974 f.write('\nDEBUG_BUILD = "1"\n')
975
976 update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
977
978 _add_md5(config, pn, appendfile)
979
980 logger.info('Recipe %s now set up to build from %s' % (pn, srctree))
981
982 finally:
983 tinfoil.shutdown()
984
985 return 0
986
987
988def rename(args, config, basepath, workspace):
989 """Entry point for the devtool 'rename' subcommand"""
990 import bb
991 import oe.recipeutils
992
993 check_workspace_recipe(workspace, args.recipename)
994
995 if not (args.newname or args.version):
996 raise DevtoolError('You must specify a new name, a version with -V/--version, or both')
997
998 recipefile = workspace[args.recipename]['recipefile']
999 if not recipefile:
1000 raise DevtoolError('devtool rename can only be used where the recipe file itself is in the workspace (e.g. after devtool add)')
1001
1002 if args.newname and args.newname != args.recipename:
1003 reason = oe.recipeutils.validate_pn(args.newname)
1004 if reason:
1005 raise DevtoolError(reason)
1006 newname = args.newname
1007 else:
1008 newname = args.recipename
1009
1010 append = workspace[args.recipename]['bbappend']
1011 appendfn = os.path.splitext(os.path.basename(append))[0]
1012 splitfn = appendfn.split('_')
1013 if len(splitfn) > 1:
1014 origfnver = appendfn.split('_')[1]
1015 else:
1016 origfnver = ''
1017
1018 recipefilemd5 = None
1019 newrecipefilemd5 = None
1020 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
1021 try:
1022 rd = parse_recipe(config, tinfoil, args.recipename, True)
1023 if not rd:
1024 return 1
1025
1026 bp = rd.getVar('BP')
1027 bpn = rd.getVar('BPN')
1028 if newname != args.recipename:
1029 localdata = rd.createCopy()
1030 localdata.setVar('PN', newname)
1031 newbpn = localdata.getVar('BPN')
1032 else:
1033 newbpn = bpn
1034 s = rd.getVar('S', False)
1035 src_uri = rd.getVar('SRC_URI', False)
1036 pv = rd.getVar('PV')
1037
1038 # Correct variable values that refer to the upstream source - these
1039 # values must stay the same, so if the name/version are changing then
1040 # we need to fix them up
1041 new_s = s
1042 new_src_uri = src_uri
1043 if newbpn != bpn:
1044 # ${PN} here is technically almost always incorrect, but people do use it
1045 new_s = new_s.replace('${BPN}', bpn)
1046 new_s = new_s.replace('${PN}', bpn)
1047 new_s = new_s.replace('${BP}', '%s-${PV}' % bpn)
1048 new_src_uri = new_src_uri.replace('${BPN}', bpn)
1049 new_src_uri = new_src_uri.replace('${PN}', bpn)
1050 new_src_uri = new_src_uri.replace('${BP}', '%s-${PV}' % bpn)
1051 if args.version and origfnver == pv:
1052 new_s = new_s.replace('${PV}', pv)
1053 new_s = new_s.replace('${BP}', '${BPN}-%s' % pv)
1054 new_src_uri = new_src_uri.replace('${PV}', pv)
1055 new_src_uri = new_src_uri.replace('${BP}', '${BPN}-%s' % pv)
1056 patchfields = {}
1057 if new_s != s:
1058 patchfields['S'] = new_s
1059 if new_src_uri != src_uri:
1060 patchfields['SRC_URI'] = new_src_uri
1061 if patchfields:
1062 recipefilemd5 = bb.utils.md5_file(recipefile)
1063 oe.recipeutils.patch_recipe(rd, recipefile, patchfields)
1064 newrecipefilemd5 = bb.utils.md5_file(recipefile)
1065 finally:
1066 tinfoil.shutdown()
1067
1068 if args.version:
1069 newver = args.version
1070 else:
1071 newver = origfnver
1072
1073 if newver:
1074 newappend = '%s_%s.bbappend' % (newname, newver)
1075 newfile = '%s_%s.bb' % (newname, newver)
1076 else:
1077 newappend = '%s.bbappend' % newname
1078 newfile = '%s.bb' % newname
1079
1080 oldrecipedir = os.path.dirname(recipefile)
1081 newrecipedir = os.path.join(config.workspace_path, 'recipes', newname)
1082 if oldrecipedir != newrecipedir:
1083 bb.utils.mkdirhier(newrecipedir)
1084
1085 newappend = os.path.join(os.path.dirname(append), newappend)
1086 newfile = os.path.join(newrecipedir, newfile)
1087
1088 # Rename bbappend
1089 logger.info('Renaming %s to %s' % (append, newappend))
1090 bb.utils.rename(append, newappend)
1091 # Rename recipe file
1092 logger.info('Renaming %s to %s' % (recipefile, newfile))
1093 bb.utils.rename(recipefile, newfile)
1094
1095 # Rename source tree if it's the default path
1096 appendmd5 = None
1097 newappendmd5 = None
1098 if not args.no_srctree:
1099 srctree = workspace[args.recipename]['srctree']
1100 if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename):
1101 newsrctree = os.path.join(config.workspace_path, 'sources', newname)
1102 logger.info('Renaming %s to %s' % (srctree, newsrctree))
1103 shutil.move(srctree, newsrctree)
1104 # Correct any references (basically EXTERNALSRC*) in the .bbappend
1105 appendmd5 = bb.utils.md5_file(newappend)
1106 appendlines = []
1107 with open(newappend, 'r') as f:
1108 for line in f:
1109 appendlines.append(line)
1110 with open(newappend, 'w') as f:
1111 for line in appendlines:
1112 if srctree in line:
1113 line = line.replace(srctree, newsrctree)
1114 f.write(line)
1115 newappendmd5 = bb.utils.md5_file(newappend)
1116
1117 bpndir = None
1118 newbpndir = None
1119 if newbpn != bpn:
1120 bpndir = os.path.join(oldrecipedir, bpn)
1121 if os.path.exists(bpndir):
1122 newbpndir = os.path.join(newrecipedir, newbpn)
1123 logger.info('Renaming %s to %s' % (bpndir, newbpndir))
1124 shutil.move(bpndir, newbpndir)
1125
1126 bpdir = None
1127 newbpdir = None
1128 if newver != origfnver or newbpn != bpn:
1129 bpdir = os.path.join(oldrecipedir, bp)
1130 if os.path.exists(bpdir):
1131 newbpdir = os.path.join(newrecipedir, '%s-%s' % (newbpn, newver))
1132 logger.info('Renaming %s to %s' % (bpdir, newbpdir))
1133 shutil.move(bpdir, newbpdir)
1134
1135 if oldrecipedir != newrecipedir:
1136 # Move any stray files and delete the old recipe directory
1137 for entry in os.listdir(oldrecipedir):
1138 oldpath = os.path.join(oldrecipedir, entry)
1139 newpath = os.path.join(newrecipedir, entry)
1140 logger.info('Renaming %s to %s' % (oldpath, newpath))
1141 shutil.move(oldpath, newpath)
1142 os.rmdir(oldrecipedir)
1143
1144 # Now take care of entries in .devtool_md5
1145 md5entries = []
1146 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'r') as f:
1147 for line in f:
1148 md5entries.append(line)
1149
1150 if bpndir and newbpndir:
1151 relbpndir = os.path.relpath(bpndir, config.workspace_path) + '/'
1152 else:
1153 relbpndir = None
1154 if bpdir and newbpdir:
1155 relbpdir = os.path.relpath(bpdir, config.workspace_path) + '/'
1156 else:
1157 relbpdir = None
1158
1159 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'w') as f:
1160 for entry in md5entries:
1161 splitentry = entry.rstrip().split('|')
1162 if len(splitentry) > 2:
1163 if splitentry[0] == args.recipename:
1164 splitentry[0] = newname
1165 if splitentry[1] == os.path.relpath(append, config.workspace_path):
1166 splitentry[1] = os.path.relpath(newappend, config.workspace_path)
1167 if appendmd5 and splitentry[2] == appendmd5:
1168 splitentry[2] = newappendmd5
1169 elif splitentry[1] == os.path.relpath(recipefile, config.workspace_path):
1170 splitentry[1] = os.path.relpath(newfile, config.workspace_path)
1171 if recipefilemd5 and splitentry[2] == recipefilemd5:
1172 splitentry[2] = newrecipefilemd5
1173 elif relbpndir and splitentry[1].startswith(relbpndir):
1174 splitentry[1] = os.path.relpath(os.path.join(newbpndir, splitentry[1][len(relbpndir):]), config.workspace_path)
1175 elif relbpdir and splitentry[1].startswith(relbpdir):
1176 splitentry[1] = os.path.relpath(os.path.join(newbpdir, splitentry[1][len(relbpdir):]), config.workspace_path)
1177 entry = '|'.join(splitentry) + '\n'
1178 f.write(entry)
1179 return 0
1180
1181
1182def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refresh=False):
1183 """Get initial and update rev of a recipe. These are the start point of the
1184 whole patchset and start point for the patches to be re-generated/updated.
1185 """
1186 import bb.process
1187
1188 # Get current branch
1189 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
1190 cwd=srctree)
1191 branchname = stdout.rstrip()
1192
1193 # Parse initial rev from recipe if not specified
1194 commits = {}
1195 patches = []
1196 initial_revs = {}
1197 with open(recipe_path, 'r') as f:
1198 for line in f:
1199 pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$'
1200 match = re.search(pattern, line)
1201 if match:
1202 name = match.group(1)
1203 rev = match.group(2)
1204 if line.startswith('# initial_rev'):
1205 if not (name == "." and initial_rev):
1206 initial_revs[name] = rev
1207 elif line.startswith('# commit') and not force_patch_refresh:
1208 if name not in commits:
1209 commits[name] = [rev]
1210 else:
1211 commits[name].append(rev)
1212 elif line.startswith('# patches_%s:' % branchname):
1213 patches = line.split(':')[-1].strip().split(',')
1214
1215 update_revs = dict(initial_revs)
1216 changed_revs = {}
1217 for name, rev in initial_revs.items():
1218 # Find first actually changed revision
1219 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' %
1220 rev, cwd=os.path.join(srctree, name))
1221 newcommits = stdout.split()
1222 if name in commits:
1223 for i in range(min(len(commits[name]), len(newcommits))):
1224 if newcommits[i] == commits[name][i]:
1225 update_revs[name] = commits[name][i]
1226
1227 try:
1228 stdout, _ = bb.process.run('git cherry devtool-patched',
1229 cwd=os.path.join(srctree, name))
1230 except bb.process.ExecutionError as err:
1231 stdout = None
1232
1233 if stdout is not None and not force_patch_refresh:
1234 for line in stdout.splitlines():
1235 if line.startswith('+ '):
1236 rev = line.split()[1]
1237 if rev in newcommits:
1238 if name not in changed_revs:
1239 changed_revs[name] = [rev]
1240 else:
1241 changed_revs[name].append(rev)
1242
1243 return initial_revs, update_revs, changed_revs, patches
1244
1245def _remove_file_entries(srcuri, filelist):
1246 """Remove file:// entries from SRC_URI"""
1247 remaining = filelist[:]
1248 entries = []
1249 for fname in filelist:
1250 basename = os.path.basename(fname)
1251 for i in range(len(srcuri)):
1252 if (srcuri[i].startswith('file://') and
1253 os.path.basename(srcuri[i].split(';')[0]) == basename):
1254 entries.append(srcuri[i])
1255 remaining.remove(fname)
1256 srcuri.pop(i)
1257 break
1258 return entries, remaining
1259
1260def _replace_srcuri_entry(srcuri, filename, newentry):
1261 """Replace entry corresponding to specified file with a new entry"""
1262 basename = os.path.basename(filename)
1263 for i in range(len(srcuri)):
1264 if os.path.basename(srcuri[i].split(';')[0]) == basename:
1265 srcuri.pop(i)
1266 srcuri.insert(i, newentry)
1267 break
1268
1269def _remove_source_files(append, files, destpath, no_report_remove=False, dry_run=False):
1270 """Unlink existing patch files"""
1271
1272 dry_run_suffix = ' (dry-run)' if dry_run else ''
1273
1274 for path in files:
1275 if append:
1276 if not destpath:
1277 raise Exception('destpath should be set here')
1278 path = os.path.join(destpath, os.path.basename(path))
1279
1280 if os.path.exists(path):
1281 if not no_report_remove:
1282 logger.info('Removing file %s%s' % (path, dry_run_suffix))
1283 if not dry_run:
1284 # FIXME "git rm" here would be nice if the file in question is
1285 # tracked
1286 # FIXME there's a chance that this file is referred to by
1287 # another recipe, in which case deleting wouldn't be the
1288 # right thing to do
1289 os.remove(path)
1290 # Remove directory if empty
1291 try:
1292 os.rmdir(os.path.dirname(path))
1293 except OSError as ose:
1294 if ose.errno != errno.ENOTEMPTY:
1295 raise
1296
1297
1298def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None):
1299 """Export patches from srctree to given location.
1300 Returns three-tuple of dicts:
1301 1. updated - patches that already exist in SRCURI
1302 2. added - new patches that don't exist in SRCURI
1303 3 removed - patches that exist in SRCURI but not in exported patches
1304 In each dict the key is the 'basepath' of the URI and value is:
1305 - for updated and added dicts, a dict with 2 optionnal keys:
1306 - 'path': the absolute path to the existing file in recipe space (if any)
1307 - 'patchdir': the directory in wich the patch should be applied (if any)
1308 - for removed dict, the absolute path to the existing file in recipe space
1309 """
1310 import oe.recipeutils
1311 from oe.patch import GitApplyTree
1312 import bb.process
1313 updated = OrderedDict()
1314 added = OrderedDict()
1315 seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
1316
1317 existing_patches = dict((os.path.basename(path), path) for path in
1318 oe.recipeutils.get_recipe_patches(rd))
1319 logger.debug('Existing patches: %s' % existing_patches)
1320
1321 # Generate patches from Git, exclude local files directory
1322 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
1323 GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec)
1324 for dirpath, dirnames, filenames in os.walk(destdir):
1325 new_patches = filenames
1326 reldirpath = os.path.relpath(dirpath, destdir)
1327 for new_patch in new_patches:
1328 # Strip numbering from patch names. If it's a git sequence named patch,
1329 # the numbers might not match up since we are starting from a different
1330 # revision This does assume that people are using unique shortlog
1331 # values, but they ought to be anyway...
1332 new_basename = seqpatch_re.match(new_patch).group(2)
1333 match_name = None
1334 old_patch = None
1335 for old_patch in existing_patches:
1336 old_basename = seqpatch_re.match(old_patch).group(2)
1337 old_basename_splitext = os.path.splitext(old_basename)
1338 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
1339 old_patch_noext = os.path.splitext(old_patch)[0]
1340 match_name = old_patch_noext
1341 break
1342 elif new_basename == old_basename:
1343 match_name = old_patch
1344 break
1345 if match_name:
1346 # Rename patch files
1347 if new_patch != match_name:
1348 bb.utils.rename(os.path.join(destdir, new_patch),
1349 os.path.join(destdir, match_name))
1350 # Need to pop it off the list now before checking changed_revs
1351 oldpath = existing_patches.pop(old_patch)
1352 if changed_revs is not None and dirpath in changed_revs:
1353 # Avoid updating patches that have not actually changed
1354 with open(os.path.join(dirpath, match_name), 'r') as f:
1355 firstlineitems = f.readline().split()
1356 # Looking for "From <hash>" line
1357 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
1358 if not firstlineitems[1] in changed_revs[dirpath]:
1359 continue
1360 # Recompress if necessary
1361 if oldpath.endswith(('.gz', '.Z')):
1362 bb.process.run(['gzip', match_name], cwd=destdir)
1363 if oldpath.endswith('.gz'):
1364 match_name += '.gz'
1365 else:
1366 match_name += '.Z'
1367 elif oldpath.endswith('.bz2'):
1368 bb.process.run(['bzip2', match_name], cwd=destdir)
1369 match_name += '.bz2'
1370 updated[match_name] = {'path' : oldpath}
1371 if reldirpath != ".":
1372 updated[match_name]['patchdir'] = reldirpath
1373 else:
1374 added[new_patch] = {}
1375 if reldirpath != ".":
1376 added[new_patch]['patchdir'] = reldirpath
1377
1378 return (updated, added, existing_patches)
1379
1380
1381def _create_kconfig_diff(srctree, rd, outfile):
1382 """Create a kconfig fragment"""
1383 import bb.process
1384 # Only update config fragment if both config files exist
1385 orig_config = os.path.join(srctree, '.config.baseline')
1386 new_config = os.path.join(srctree, '.config.new')
1387 if os.path.exists(orig_config) and os.path.exists(new_config):
1388 cmd = ['diff', '--new-line-format=%L', '--old-line-format=',
1389 '--unchanged-line-format=', orig_config, new_config]
1390 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE,
1391 stderr=subprocess.PIPE)
1392 stdout, stderr = pipe.communicate()
1393 if pipe.returncode == 1:
1394 logger.info("Updating config fragment %s" % outfile)
1395 with open(outfile, 'wb') as fobj:
1396 fobj.write(stdout)
1397 elif pipe.returncode == 0:
1398 logger.info("Would remove config fragment %s" % outfile)
1399 if os.path.exists(outfile):
1400 # Remove fragment file in case of empty diff
1401 logger.info("Removing config fragment %s" % outfile)
1402 os.unlink(outfile)
1403 else:
1404 raise bb.process.ExecutionError(cmd, pipe.returncode, stdout, stderr)
1405 return True
1406 return False
1407
1408
1409def _export_local_files(srctree, rd, destdir, srctreebase):
1410 """Copy local files from srctree to given location.
1411 Returns three-tuple of dicts:
1412 1. updated - files that already exist in SRCURI
1413 2. added - new files files that don't exist in SRCURI
1414 3 removed - files that exist in SRCURI but not in exported files
1415 In each dict the key is the 'basepath' of the URI and value is:
1416 - for updated and added dicts, a dict with 1 optionnal key:
1417 - 'path': the absolute path to the existing file in recipe space (if any)
1418 - for removed dict, the absolute path to the existing file in recipe space
1419 """
1420 import oe.recipeutils
1421 import bb.data
1422 import bb.process
1423
1424 # Find out local files (SRC_URI files that exist in the "recipe space").
1425 # Local files that reside in srctree are not included in patch generation.
1426 # Instead they are directly copied over the original source files (in
1427 # recipe space).
1428 existing_files = oe.recipeutils.get_recipe_local_files(rd)
1429
1430 new_set = None
1431 updated = OrderedDict()
1432 added = OrderedDict()
1433 removed = OrderedDict()
1434
1435 # Get current branch and return early with empty lists
1436 # if on one of the override branches
1437 # (local files are provided only for the main branch and processing
1438 # them against lists from recipe overrides will result in mismatches
1439 # and broken modifications to recipes).
1440 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
1441 cwd=srctree)
1442 branchname = stdout.rstrip()
1443 if branchname.startswith(override_branch_prefix):
1444 return (updated, added, removed)
1445
1446 files = _git_modified(srctree)
1447 #if not files:
1448 # files = _ls_tree(srctree)
1449 for f in files:
1450 fullfile = os.path.join(srctree, f)
1451 if os.path.exists(os.path.join(fullfile, ".git")):
1452 # submodules handled elsewhere
1453 continue
1454 if f not in existing_files:
1455 added[f] = {}
1456 if os.path.isdir(os.path.join(srctree, f)):
1457 shutil.copytree(fullfile, os.path.join(destdir, f))
1458 else:
1459 shutil.copy2(fullfile, os.path.join(destdir, f))
1460 elif not os.path.exists(fullfile):
1461 removed[f] = existing_files[f]
1462 elif f in existing_files:
1463 updated[f] = {'path' : existing_files[f]}
1464 if os.path.isdir(os.path.join(srctree, f)):
1465 shutil.copytree(fullfile, os.path.join(destdir, f))
1466 else:
1467 shutil.copy2(fullfile, os.path.join(destdir, f))
1468
1469 # Special handling for kernel config
1470 if bb.data.inherits_class('kernel-yocto', rd):
1471 fragment_fn = 'devtool-fragment.cfg'
1472 fragment_path = os.path.join(destdir, fragment_fn)
1473 if _create_kconfig_diff(srctree, rd, fragment_path):
1474 if os.path.exists(fragment_path):
1475 if fragment_fn in removed:
1476 del removed[fragment_fn]
1477 if fragment_fn not in updated and fragment_fn not in added:
1478 added[fragment_fn] = {}
1479 else:
1480 if fragment_fn in updated:
1481 removed[fragment_fn] = updated[fragment_fn]
1482 del updated[fragment_fn]
1483
1484 # Special handling for cml1, ccmake, etc bbclasses that generated
1485 # configuration fragment files that are consumed as source files
1486 for frag_class, frag_name in [("cml1", "fragment.cfg"), ("ccmake", "site-file.cmake")]:
1487 if bb.data.inherits_class(frag_class, rd):
1488 srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name)
1489 if os.path.exists(srcpath):
1490 if frag_name in removed:
1491 del removed[frag_name]
1492 if frag_name not in updated:
1493 added[frag_name] = {}
1494 # copy fragment into destdir
1495 shutil.copy2(srcpath, destdir)
1496
1497 return (updated, added, removed)
1498
1499
1500def _determine_files_dir(rd):
1501 """Determine the appropriate files directory for a recipe"""
1502 recipedir = rd.getVar('FILE_DIRNAME')
1503 for entry in rd.getVar('FILESPATH').split(':'):
1504 relpth = os.path.relpath(entry, recipedir)
1505 if not os.sep in relpth:
1506 # One (or zero) levels below only, so we don't put anything in machine-specific directories
1507 if os.path.isdir(entry):
1508 return entry
1509 return os.path.join(recipedir, rd.getVar('BPN'))
1510
1511
1512def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None):
1513 """Implement the 'srcrev' mode of update-recipe"""
1514 import bb.process
1515 import oe.recipeutils
1516
1517 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1518
1519 recipefile = rd.getVar('FILE')
1520 recipedir = os.path.basename(recipefile)
1521 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
1522
1523 # Get original SRCREV
1524 old_srcrev = rd.getVar('SRCREV') or ''
1525 if old_srcrev == "INVALID":
1526 raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository')
1527 old_srcrev = {'.': old_srcrev}
1528
1529 # Get HEAD revision
1530 try:
1531 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1532 except bb.process.ExecutionError as err:
1533 raise DevtoolError('Failed to get HEAD revision in %s: %s' %
1534 (srctree, err))
1535 srcrev = stdout.strip()
1536 if len(srcrev) != 40:
1537 raise DevtoolError('Invalid hash returned by git: %s' % stdout)
1538
1539 destpath = None
1540 remove_files = []
1541 patchfields = {}
1542 patchfields['SRCREV'] = srcrev
1543 orig_src_uri = rd.getVar('SRC_URI', False) or ''
1544 srcuri = orig_src_uri.split()
1545 tempdir = tempfile.mkdtemp(prefix='devtool')
1546 update_srcuri = False
1547 appendfile = None
1548 try:
1549 local_files_dir = tempfile.mkdtemp(dir=tempdir)
1550 srctreebase = workspace[recipename]['srctreebase']
1551 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
1552 removedentries = {}
1553 if not no_remove:
1554 # Find list of existing patches in recipe file
1555 patches_dir = tempfile.mkdtemp(dir=tempdir)
1556 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
1557 patches_dir)
1558 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p)))
1559
1560 # Remove deleted local files and "overlapping" patches
1561 remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value]
1562 if remove_files:
1563 removedentries = _remove_file_entries(srcuri, remove_files)[0]
1564 update_srcuri = True
1565
1566 if appendlayerdir:
1567 files = dict((os.path.join(local_files_dir, key), val) for
1568 key, val in list(upd_f.items()) + list(new_f.items()))
1569 removevalues = {}
1570 if update_srcuri:
1571 removevalues = {'SRC_URI': removedentries}
1572 patchfields['SRC_URI'] = '\\\n '.join(srcuri)
1573 if dry_run_outdir:
1574 logger.info('Creating bbappend (dry-run)')
1575 appendfile, destpath = oe.recipeutils.bbappend_recipe(
1576 rd, appendlayerdir, files, wildcardver=wildcard_version,
1577 extralines=patchfields, removevalues=removevalues,
1578 redirect_output=dry_run_outdir)
1579 else:
1580 files_dir = _determine_files_dir(rd)
1581 for basepath, param in upd_f.items():
1582 path = param['path']
1583 logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
1584 if os.path.isabs(basepath):
1585 # Original file (probably with subdir pointing inside source tree)
1586 # so we do not want to move it, just copy
1587 _copy_file(basepath, path, dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1588 else:
1589 _move_file(os.path.join(local_files_dir, basepath), path,
1590 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1591 update_srcuri= True
1592 for basepath, param in new_f.items():
1593 path = param['path']
1594 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1595 _move_file(os.path.join(local_files_dir, basepath),
1596 os.path.join(files_dir, basepath),
1597 dry_run_outdir=dry_run_outdir,
1598 base_outdir=recipedir)
1599 srcuri.append('file://%s' % basepath)
1600 update_srcuri = True
1601 if update_srcuri:
1602 patchfields['SRC_URI'] = ' '.join(srcuri)
1603 ret = oe.recipeutils.patch_recipe(rd, recipefile, patchfields, redirect_output=dry_run_outdir)
1604 finally:
1605 shutil.rmtree(tempdir)
1606 if not 'git://' in orig_src_uri:
1607 logger.info('You will need to update SRC_URI within the recipe to '
1608 'point to a git repository where you have pushed your '
1609 'changes')
1610
1611 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1612 return True, appendfile, remove_files
1613
1614def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False):
1615 """Implement the 'patch' mode of update-recipe"""
1616 import oe.recipeutils
1617
1618 recipefile = rd.getVar('FILE')
1619 recipedir = os.path.dirname(recipefile)
1620 append = workspace[recipename]['bbappend']
1621 if not os.path.exists(append):
1622 raise DevtoolError('unable to find workspace bbappend for recipe %s' %
1623 recipename)
1624 srctreebase = workspace[recipename]['srctreebase']
1625 relpatchdir = os.path.relpath(srctreebase, srctree)
1626 if relpatchdir == '.':
1627 patchdir_params = {}
1628 else:
1629 patchdir_params = {'patchdir': relpatchdir}
1630
1631 def srcuri_entry(basepath, patchdir_params):
1632 if patchdir_params:
1633 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items())
1634 else:
1635 paramstr = ''
1636 return 'file://%s%s' % (basepath, paramstr)
1637
1638 initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
1639 if not initial_revs:
1640 raise DevtoolError('Unable to find initial revision - please specify '
1641 'it with --initial-rev')
1642
1643 appendfile = None
1644 dl_dir = rd.getVar('DL_DIR')
1645 if not dl_dir.endswith('/'):
1646 dl_dir += '/'
1647
1648 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
1649
1650 tempdir = tempfile.mkdtemp(prefix='devtool')
1651 try:
1652 local_files_dir = tempfile.mkdtemp(dir=tempdir)
1653 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
1654
1655 # Get updated patches from source tree
1656 patches_dir = tempfile.mkdtemp(dir=tempdir)
1657 upd_p, new_p, _ = _export_patches(srctree, rd, update_revs,
1658 patches_dir, changed_revs)
1659 # Get all patches from source tree and check if any should be removed
1660 all_patches_dir = tempfile.mkdtemp(dir=tempdir)
1661 _, _, del_p = _export_patches(srctree, rd, initial_revs,
1662 all_patches_dir)
1663 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
1664 if filter_patches:
1665 new_p = OrderedDict()
1666 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches)
1667 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches)
1668 remove_files = []
1669 if not no_remove:
1670 # Remove deleted local files and patches
1671 remove_files = list(del_f.values()) + list(del_p.values())
1672 updatefiles = False
1673 updaterecipe = False
1674 destpath = None
1675 srcuri = (rd.getVar('SRC_URI', False) or '').split()
1676
1677 if appendlayerdir:
1678 files = OrderedDict((os.path.join(local_files_dir, key), val) for
1679 key, val in list(upd_f.items()) + list(new_f.items()))
1680 files.update(OrderedDict((os.path.join(patches_dir, key), val) for
1681 key, val in list(upd_p.items()) + list(new_p.items())))
1682
1683 params = []
1684 for file, param in files.items():
1685 patchdir_param = dict(patchdir_params)
1686 patchdir = param.get('patchdir', ".")
1687 if patchdir != "." :
1688 if patchdir_param:
1689 patchdir_param['patchdir'] += patchdir
1690 else:
1691 patchdir_param['patchdir'] = patchdir
1692 params.append(patchdir_param)
1693
1694 if files or remove_files:
1695 removevalues = None
1696 if remove_files:
1697 removedentries, remaining = _remove_file_entries(
1698 srcuri, remove_files)
1699 if removedentries or remaining:
1700 remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for
1701 item in remaining]
1702 removevalues = {'SRC_URI': removedentries + remaining}
1703 appendfile, destpath = oe.recipeutils.bbappend_recipe(
1704 rd, appendlayerdir, files,
1705 wildcardver=wildcard_version,
1706 removevalues=removevalues,
1707 redirect_output=dry_run_outdir,
1708 params=params)
1709 else:
1710 logger.info('No patches or local source files needed updating')
1711 else:
1712 # Update existing files
1713 files_dir = _determine_files_dir(rd)
1714 for basepath, param in upd_f.items():
1715 path = param['path']
1716 logger.info('Updating file %s' % basepath)
1717 if os.path.isabs(basepath):
1718 # Original file (probably with subdir pointing inside source tree)
1719 # so we do not want to move it, just copy
1720 _copy_file(basepath, path,
1721 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1722 else:
1723 _move_file(os.path.join(local_files_dir, basepath), path,
1724 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1725 updatefiles = True
1726 for basepath, param in upd_p.items():
1727 path = param['path']
1728 patchdir = param.get('patchdir', ".")
1729 patchdir_param = {}
1730 if patchdir != "." :
1731 patchdir_param = dict(patchdir_params)
1732 if patchdir_param:
1733 patchdir_param['patchdir'] += patchdir
1734 else:
1735 patchdir_param['patchdir'] = patchdir
1736 patchfn = os.path.join(patches_dir, patchdir, basepath)
1737 if os.path.dirname(path) + '/' == dl_dir:
1738 # This is a a downloaded patch file - we now need to
1739 # replace the entry in SRC_URI with our local version
1740 logger.info('Replacing remote patch %s with updated local version' % basepath)
1741 path = os.path.join(files_dir, basepath)
1742 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param))
1743 updaterecipe = True
1744 else:
1745 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
1746 _move_file(patchfn, path,
1747 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1748 updatefiles = True
1749 # Add any new files
1750 for basepath, param in new_f.items():
1751 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1752 _move_file(os.path.join(local_files_dir, basepath),
1753 os.path.join(files_dir, basepath),
1754 dry_run_outdir=dry_run_outdir,
1755 base_outdir=recipedir)
1756 srcuri.append(srcuri_entry(basepath, patchdir_params))
1757 updaterecipe = True
1758 for basepath, param in new_p.items():
1759 patchdir = param.get('patchdir', ".")
1760 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
1761 _move_file(os.path.join(patches_dir, patchdir, basepath),
1762 os.path.join(files_dir, basepath),
1763 dry_run_outdir=dry_run_outdir,
1764 base_outdir=recipedir)
1765 params = dict(patchdir_params)
1766 if patchdir != "." :
1767 if params:
1768 params['patchdir'] += patchdir
1769 else:
1770 params['patchdir'] = patchdir
1771
1772 srcuri.append(srcuri_entry(basepath, params))
1773 updaterecipe = True
1774 # Update recipe, if needed
1775 if _remove_file_entries(srcuri, remove_files)[0]:
1776 updaterecipe = True
1777 if updaterecipe:
1778 if not dry_run_outdir:
1779 logger.info('Updating recipe %s' % os.path.basename(recipefile))
1780 ret = oe.recipeutils.patch_recipe(rd, recipefile,
1781 {'SRC_URI': ' '.join(srcuri)},
1782 redirect_output=dry_run_outdir)
1783 elif not updatefiles:
1784 # Neither patches nor recipe were updated
1785 logger.info('No patches or files need updating')
1786 return False, None, []
1787 finally:
1788 shutil.rmtree(tempdir)
1789
1790 _remove_source_files(appendlayerdir, remove_files, destpath, no_report_remove, dry_run=dry_run_outdir)
1791 return True, appendfile, remove_files
1792
1793def _guess_recipe_update_mode(srctree, rdata):
1794 """Guess the recipe update mode to use"""
1795 import bb.process
1796 src_uri = (rdata.getVar('SRC_URI') or '').split()
1797 git_uris = [uri for uri in src_uri if uri.startswith('git://')]
1798 if not git_uris:
1799 return 'patch'
1800 # Just use the first URI for now
1801 uri = git_uris[0]
1802 # Check remote branch
1803 params = bb.fetch.decodeurl(uri)[5]
1804 upstr_branch = params['branch'] if 'branch' in params else 'master'
1805 # Check if current branch HEAD is found in upstream branch
1806 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
1807 head_rev = stdout.rstrip()
1808 stdout, _ = bb.process.run('git branch -r --contains %s' % head_rev,
1809 cwd=srctree)
1810 remote_brs = [branch.strip() for branch in stdout.splitlines()]
1811 if 'origin/' + upstr_branch in remote_brs:
1812 return 'srcrev'
1813
1814 return 'patch'
1815
1816def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False):
1817 import bb.data
1818 import bb.process
1819 srctree = workspace[recipename]['srctree']
1820 if mode == 'auto':
1821 mode = _guess_recipe_update_mode(srctree, rd)
1822
1823 override_branches = []
1824 mainbranch = None
1825 startbranch = None
1826 if not no_overrides:
1827 stdout, _ = bb.process.run('git branch', cwd=srctree)
1828 other_branches = []
1829 for line in stdout.splitlines():
1830 branchname = line[2:]
1831 if line.startswith('* '):
1832 if 'HEAD' in line:
1833 raise DevtoolError('Detached HEAD - please check out a branch, e.g., "devtool"')
1834 startbranch = branchname
1835 if branchname.startswith(override_branch_prefix):
1836 override_branches.append(branchname)
1837 else:
1838 other_branches.append(branchname)
1839
1840 if override_branches:
1841 logger.debug('_update_recipe: override branches: %s' % override_branches)
1842 logger.debug('_update_recipe: other branches: %s' % other_branches)
1843 if startbranch.startswith(override_branch_prefix):
1844 if len(other_branches) == 1:
1845 mainbranch = other_branches[1]
1846 else:
1847 raise DevtoolError('Unable to determine main branch - please check out the main branch in source tree first')
1848 else:
1849 mainbranch = startbranch
1850
1851 checkedout = None
1852 anyupdated = False
1853 appendfile = None
1854 allremoved = []
1855 if override_branches:
1856 logger.info('Handling main branch (%s)...' % mainbranch)
1857 if startbranch != mainbranch:
1858 bb.process.run('git checkout %s' % mainbranch, cwd=srctree)
1859 checkedout = mainbranch
1860 try:
1861 branchlist = [mainbranch] + override_branches
1862 for branch in branchlist:
1863 crd = bb.data.createCopy(rd)
1864 if branch != mainbranch:
1865 logger.info('Handling branch %s...' % branch)
1866 override = branch[len(override_branch_prefix):]
1867 crd.appendVar('OVERRIDES', ':%s' % override)
1868 bb.process.run('git checkout %s' % branch, cwd=srctree)
1869 checkedout = branch
1870
1871 if mode == 'srcrev':
1872 updated, appendf, removed = _update_recipe_srcrev(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir)
1873 elif mode == 'patch':
1874 updated, appendf, removed = _update_recipe_patch(recipename, workspace, srctree, crd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir, force_patch_refresh)
1875 else:
1876 raise DevtoolError('update_recipe: invalid mode %s' % mode)
1877 if updated:
1878 anyupdated = True
1879 if appendf:
1880 appendfile = appendf
1881 allremoved.extend(removed)
1882 finally:
1883 if startbranch and checkedout != startbranch:
1884 bb.process.run('git checkout %s' % startbranch, cwd=srctree)
1885
1886 return anyupdated, appendfile, allremoved
1887
1888def update_recipe(args, config, basepath, workspace):
1889 """Entry point for the devtool 'update-recipe' subcommand"""
1890 check_workspace_recipe(workspace, args.recipename)
1891
1892 if args.append:
1893 if not os.path.exists(args.append):
1894 raise DevtoolError('bbappend destination layer directory "%s" '
1895 'does not exist' % args.append)
1896 if not os.path.exists(os.path.join(args.append, 'conf', 'layer.conf')):
1897 raise DevtoolError('conf/layer.conf not found in bbappend '
1898 'destination layer "%s"' % args.append)
1899
1900 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
1901 try:
1902
1903 rd = parse_recipe(config, tinfoil, args.recipename, True)
1904 if not rd:
1905 return 1
1906
1907 dry_run_output = None
1908 dry_run_outdir = None
1909 if args.dry_run:
1910 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
1911 dry_run_outdir = dry_run_output.name
1912 updated, _, _ = _update_recipe(args.recipename, workspace, rd, args.mode, args.append, args.wildcard_version, args.no_remove, args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh)
1913
1914 if updated:
1915 rf = rd.getVar('FILE')
1916 if rf.startswith(config.workspace_path):
1917 logger.warning('Recipe file %s has been updated but is inside the workspace - you will need to move it (and any associated files next to it) out to the desired layer before using "devtool reset" in order to keep any changes' % rf)
1918 finally:
1919 tinfoil.shutdown()
1920
1921 return 0
1922
1923
1924def status(args, config, basepath, workspace):
1925 """Entry point for the devtool 'status' subcommand"""
1926 if workspace:
1927 for recipe, value in sorted(workspace.items()):
1928 recipefile = value['recipefile']
1929 if recipefile:
1930 recipestr = ' (%s)' % recipefile
1931 else:
1932 recipestr = ''
1933 print("%s: %s%s" % (recipe, value['srctree'], recipestr))
1934 else:
1935 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one')
1936 return 0
1937
1938
1939def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
1940 """Reset one or more recipes"""
1941 import bb.process
1942 import oe.path
1943
1944 def clean_preferred_provider(pn, layerconf_path):
1945 """Remove PREFERRED_PROVIDER from layer.conf'"""
1946 import re
1947 layerconf_file = os.path.join(layerconf_path, 'conf', 'layer.conf')
1948 new_layerconf_file = os.path.join(layerconf_path, 'conf', '.layer.conf')
1949 pprovider_found = False
1950 with open(layerconf_file, 'r') as f:
1951 lines = f.readlines()
1952 with open(new_layerconf_file, 'a') as nf:
1953 for line in lines:
1954 pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + re.escape(pn) + r'"$'
1955 if not re.match(pprovider_exp, line):
1956 nf.write(line)
1957 else:
1958 pprovider_found = True
1959 if pprovider_found:
1960 shutil.move(new_layerconf_file, layerconf_file)
1961 else:
1962 os.remove(new_layerconf_file)
1963
1964 if recipes and not no_clean:
1965 if len(recipes) == 1:
1966 logger.info('Cleaning sysroot for recipe %s...' % recipes[0])
1967 else:
1968 logger.info('Cleaning sysroot for recipes %s...' % ', '.join(recipes))
1969 # If the recipe file itself was created in the workspace, and
1970 # it uses BBCLASSEXTEND, then we need to also clean the other
1971 # variants
1972 targets = []
1973 for recipe in recipes:
1974 targets.append(recipe)
1975 recipefile = workspace[recipe]['recipefile']
1976 if recipefile and os.path.exists(recipefile):
1977 targets.extend(get_bbclassextend_targets(recipefile, recipe))
1978 try:
1979 exec_build_env_command(config.init_path, basepath, 'bitbake -c clean %s' % ' '.join(targets))
1980 except bb.process.ExecutionError as e:
1981 raise DevtoolError('Command \'%s\' failed, output:\n%s\nIf you '
1982 'wish, you may specify -n/--no-clean to '
1983 'skip running this command when resetting' %
1984 (e.command, e.stdout))
1985
1986 for pn in recipes:
1987 _check_preserve(config, pn)
1988
1989 appendfile = workspace[pn]['bbappend']
1990 if os.path.exists(appendfile):
1991 # This shouldn't happen, but is possible if devtool errored out prior to
1992 # writing the md5 file. We need to delete this here or the recipe won't
1993 # actually be reset
1994 os.remove(appendfile)
1995
1996 preservepath = os.path.join(config.workspace_path, 'attic', pn, pn)
1997 def preservedir(origdir):
1998 if os.path.exists(origdir):
1999 for root, dirs, files in os.walk(origdir):
2000 for fn in files:
2001 logger.warning('Preserving %s in %s' % (fn, preservepath))
2002 _move_file(os.path.join(origdir, fn),
2003 os.path.join(preservepath, fn))
2004 for dn in dirs:
2005 preservedir(os.path.join(root, dn))
2006 os.rmdir(origdir)
2007
2008 recipefile = workspace[pn]['recipefile']
2009 if recipefile and oe.path.is_path_parent(config.workspace_path, recipefile):
2010 # This should always be true if recipefile is set, but just in case
2011 preservedir(os.path.dirname(recipefile))
2012 # We don't automatically create this dir next to appends, but the user can
2013 preservedir(os.path.join(config.workspace_path, 'appends', pn))
2014
2015 srctreebase = workspace[pn]['srctreebase']
2016 if os.path.isdir(srctreebase):
2017 if os.listdir(srctreebase):
2018 if remove_work:
2019 logger.info('-r argument used on %s, removing source tree.'
2020 ' You will lose any unsaved work' %pn)
2021 shutil.rmtree(srctreebase)
2022 else:
2023 # We don't want to risk wiping out any work in progress
2024 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')):
2025 from datetime import datetime
2026 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S")))
2027 logger.info('Preserving source tree in %s\nIf you no '
2028 'longer need it then please delete it manually.\n'
2029 'It is also possible to reuse it via devtool source tree argument.'
2030 % preservesrc)
2031 bb.utils.mkdirhier(os.path.dirname(preservesrc))
2032 shutil.move(srctreebase, preservesrc)
2033 else:
2034 logger.info('Leaving source tree %s as-is; if you no '
2035 'longer need it then please delete it manually'
2036 % srctreebase)
2037 else:
2038 # This is unlikely, but if it's empty we can just remove it
2039 os.rmdir(srctreebase)
2040
2041 clean_preferred_provider(pn, config.workspace_path)
2042
2043def reset(args, config, basepath, workspace):
2044 """Entry point for the devtool 'reset' subcommand"""
2045
2046 recipes = ""
2047
2048 if args.recipename:
2049 if args.all:
2050 raise DevtoolError("Recipe cannot be specified if -a/--all is used")
2051 else:
2052 for recipe in args.recipename:
2053 check_workspace_recipe(workspace, recipe, checksrc=False)
2054 elif not args.all:
2055 raise DevtoolError("Recipe must be specified, or specify -a/--all to "
2056 "reset all recipes")
2057 if args.all:
2058 recipes = list(workspace.keys())
2059 else:
2060 recipes = args.recipename
2061
2062 _reset(recipes, args.no_clean, args.remove_work, config, basepath, workspace)
2063
2064 return 0
2065
2066
2067def _get_layer(layername, d):
2068 """Determine the base layer path for the specified layer name/path"""
2069 layerdirs = d.getVar('BBLAYERS').split()
2070 layers = {} # {basename: layer_paths}
2071 for p in layerdirs:
2072 bn = os.path.basename(p)
2073 if bn not in layers:
2074 layers[bn] = [p]
2075 else:
2076 layers[bn].append(p)
2077 # Provide some shortcuts
2078 if layername.lower() in ['oe-core', 'openembedded-core']:
2079 layername = 'meta'
2080 layer_paths = layers.get(layername, None)
2081 if not layer_paths:
2082 return os.path.abspath(layername)
2083 elif len(layer_paths) == 1:
2084 return os.path.abspath(layer_paths[0])
2085 else:
2086 # multiple layers having the same base name
2087 logger.warning("Multiple layers have the same base name '%s', use the first one '%s'." % (layername, layer_paths[0]))
2088 logger.warning("Consider using path instead of base name to specify layer:\n\t\t%s" % '\n\t\t'.join(layer_paths))
2089 return os.path.abspath(layer_paths[0])
2090
2091
2092def finish(args, config, basepath, workspace):
2093 """Entry point for the devtool 'finish' subcommand"""
2094 import bb
2095 import oe.recipeutils
2096
2097 check_workspace_recipe(workspace, args.recipename)
2098
2099 dry_run_suffix = ' (dry-run)' if args.dry_run else ''
2100
2101 # Grab the equivalent of COREBASE without having to initialise tinfoil
2102 corebasedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))
2103
2104 srctree = workspace[args.recipename]['srctree']
2105 check_git_repo_op(srctree, [corebasedir])
2106 dirty = check_git_repo_dirty(srctree)
2107 if dirty:
2108 if args.force:
2109 logger.warning('Source tree is not clean, continuing as requested by -f/--force')
2110 else:
2111 raise DevtoolError('Source tree is not clean:\n\n%s\nEnsure you have committed your changes or use -f/--force if you are sure there\'s nothing that needs to be committed' % dirty)
2112
2113 no_clean = args.no_clean
2114 remove_work=args.remove_work
2115 tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
2116 try:
2117 rd = parse_recipe(config, tinfoil, args.recipename, True)
2118 if not rd:
2119 return 1
2120
2121 destlayerdir = _get_layer(args.destination, tinfoil.config_data)
2122 recipefile = rd.getVar('FILE')
2123 recipedir = os.path.dirname(recipefile)
2124 origlayerdir = oe.recipeutils.find_layerdir(recipefile)
2125
2126 if not os.path.isdir(destlayerdir):
2127 raise DevtoolError('Unable to find layer or directory matching "%s"' % args.destination)
2128
2129 if os.path.abspath(destlayerdir) == config.workspace_path:
2130 raise DevtoolError('"%s" specifies the workspace layer - that is not a valid destination' % args.destination)
2131
2132 # If it's an upgrade, grab the original path
2133 origpath = None
2134 origfilelist = None
2135 append = workspace[args.recipename]['bbappend']
2136 with open(append, 'r') as f:
2137 for line in f:
2138 if line.startswith('# original_path:'):
2139 origpath = line.split(':')[1].strip()
2140 elif line.startswith('# original_files:'):
2141 origfilelist = line.split(':')[1].split()
2142
2143 destlayerbasedir = oe.recipeutils.find_layerdir(destlayerdir)
2144
2145 if origlayerdir == config.workspace_path:
2146 # Recipe file itself is in workspace, update it there first
2147 appendlayerdir = None
2148 origrelpath = None
2149 if origpath:
2150 origlayerpath = oe.recipeutils.find_layerdir(origpath)
2151 if origlayerpath:
2152 origrelpath = os.path.relpath(origpath, origlayerpath)
2153 destpath = oe.recipeutils.get_bbfile_path(rd, destlayerdir, origrelpath)
2154 if not destpath:
2155 raise DevtoolError("Unable to determine destination layer path - check that %s specifies an actual layer and %s/conf/layer.conf specifies BBFILES. You may also need to specify a more complete path." % (args.destination, destlayerdir))
2156 # Warn if the layer isn't in bblayers.conf (the code to create a bbappend will do this in other cases)
2157 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS').split()]
2158 if not os.path.abspath(destlayerbasedir) in layerdirs:
2159 bb.warn('Specified destination layer is not currently enabled in bblayers.conf, so the %s recipe will now be unavailable in your current configuration until you add the layer there' % args.recipename)
2160
2161 elif destlayerdir == origlayerdir:
2162 # Same layer, update the original recipe
2163 appendlayerdir = None
2164 destpath = None
2165 else:
2166 # Create/update a bbappend in the specified layer
2167 appendlayerdir = destlayerdir
2168 destpath = None
2169
2170 # Actually update the recipe / bbappend
2171 removing_original = (origpath and origfilelist and oe.recipeutils.find_layerdir(origpath) == destlayerbasedir)
2172 dry_run_output = None
2173 dry_run_outdir = None
2174 if args.dry_run:
2175 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
2176 dry_run_outdir = dry_run_output.name
2177 updated, appendfile, removed = _update_recipe(args.recipename, workspace, rd, args.mode, appendlayerdir, wildcard_version=True, no_remove=False, no_report_remove=removing_original, initial_rev=args.initial_rev, dry_run_outdir=dry_run_outdir, no_overrides=args.no_overrides, force_patch_refresh=args.force_patch_refresh)
2178 removed = [os.path.relpath(pth, recipedir) for pth in removed]
2179
2180 # Remove any old files in the case of an upgrade
2181 if removing_original:
2182 for fn in origfilelist:
2183 fnp = os.path.join(origpath, fn)
2184 if fn in removed or not os.path.exists(os.path.join(recipedir, fn)):
2185 logger.info('Removing file %s%s' % (fnp, dry_run_suffix))
2186 if not args.dry_run:
2187 try:
2188 os.remove(fnp)
2189 except FileNotFoundError:
2190 pass
2191
2192 if origlayerdir == config.workspace_path and destpath:
2193 # Recipe file itself is in the workspace - need to move it and any
2194 # associated files to the specified layer
2195 no_clean = True
2196 logger.info('Moving recipe file to %s%s' % (destpath, dry_run_suffix))
2197 for root, _, files in os.walk(recipedir):
2198 for fn in files:
2199 srcpath = os.path.join(root, fn)
2200 relpth = os.path.relpath(os.path.dirname(srcpath), recipedir)
2201 destdir = os.path.abspath(os.path.join(destpath, relpth))
2202 destfp = os.path.join(destdir, fn)
2203 _move_file(srcpath, destfp, dry_run_outdir=dry_run_outdir, base_outdir=destpath)
2204
2205 if dry_run_outdir:
2206 import difflib
2207 comparelist = []
2208 for root, _, files in os.walk(dry_run_outdir):
2209 for fn in files:
2210 outf = os.path.join(root, fn)
2211 relf = os.path.relpath(outf, dry_run_outdir)
2212 logger.debug('dry-run: output file %s' % relf)
2213 if fn.endswith('.bb'):
2214 if origfilelist and origpath and destpath:
2215 # Need to match this up with the pre-upgrade recipe file
2216 for origf in origfilelist:
2217 if origf.endswith('.bb'):
2218 comparelist.append((os.path.abspath(os.path.join(origpath, origf)),
2219 outf,
2220 os.path.abspath(os.path.join(destpath, relf))))
2221 break
2222 else:
2223 # Compare to the existing recipe
2224 comparelist.append((recipefile, outf, recipefile))
2225 elif fn.endswith('.bbappend'):
2226 if appendfile:
2227 if os.path.exists(appendfile):
2228 comparelist.append((appendfile, outf, appendfile))
2229 else:
2230 comparelist.append((None, outf, appendfile))
2231 else:
2232 if destpath:
2233 recipedest = destpath
2234 elif appendfile:
2235 recipedest = os.path.dirname(appendfile)
2236 else:
2237 recipedest = os.path.dirname(recipefile)
2238 destfp = os.path.join(recipedest, relf)
2239 if os.path.exists(destfp):
2240 comparelist.append((destfp, outf, destfp))
2241 output = ''
2242 for oldfile, newfile, newfileshow in comparelist:
2243 if oldfile:
2244 with open(oldfile, 'r') as f:
2245 oldlines = f.readlines()
2246 else:
2247 oldfile = '/dev/null'
2248 oldlines = []
2249 with open(newfile, 'r') as f:
2250 newlines = f.readlines()
2251 if not newfileshow:
2252 newfileshow = newfile
2253 diff = difflib.unified_diff(oldlines, newlines, oldfile, newfileshow)
2254 difflines = list(diff)
2255 if difflines:
2256 output += ''.join(difflines)
2257 if output:
2258 logger.info('Diff of changed files:\n%s' % output)
2259 finally:
2260 tinfoil.shutdown()
2261
2262 # Everything else has succeeded, we can now reset
2263 if args.dry_run:
2264 logger.info('Resetting recipe (dry-run)')
2265 else:
2266 _reset([args.recipename], no_clean=no_clean, remove_work=remove_work, config=config, basepath=basepath, workspace=workspace)
2267
2268 return 0
2269
2270
2271def get_default_srctree(config, recipename=''):
2272 """Get the default srctree path"""
2273 srctreeparent = config.get('General', 'default_source_parent_dir', config.workspace_path)
2274 if recipename:
2275 return os.path.join(srctreeparent, 'sources', recipename)
2276 else:
2277 return os.path.join(srctreeparent, 'sources')
2278
2279def register_commands(subparsers, context):
2280 """Register devtool subcommands from this plugin"""
2281
2282 defsrctree = get_default_srctree(context.config)
2283 parser_add = subparsers.add_parser('add', help='Add a new recipe',
2284 description='Adds a new recipe to the workspace to build a specified source tree. Can optionally fetch a remote URI and unpack it to create the source tree.',
2285 group='starting', order=100)
2286 parser_add.add_argument('recipename', nargs='?', help='Name for new recipe to add (just name - no version, path or extension). If not specified, will attempt to auto-detect it.')
2287 parser_add.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
2288 parser_add.add_argument('fetchuri', nargs='?', help='Fetch the specified URI and extract it to create the source tree')
2289 group = parser_add.add_mutually_exclusive_group()
2290 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
2291 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
2292 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
2293 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true")
2294 parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true")
2295 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
2296 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
2297 group = parser_add.add_mutually_exclusive_group()
2298 group.add_argument('--srcrev', '-S', help='Source revision to fetch if fetching from an SCM such as git (default latest)')
2299 group.add_argument('--autorev', '-a', help='When fetching from a git repository, set SRCREV in the recipe to a floating revision instead of fixed', action="store_true")
2300 parser_add.add_argument('--srcbranch', '-B', help='Branch in source repository if fetching from an SCM such as git (default master)')
2301 parser_add.add_argument('--binary', '-b', help='Treat the source tree as something that should be installed verbatim (no compilation, same directory structure). Useful with binary packages e.g. RPMs.', action='store_true')
2302 parser_add.add_argument('--also-native', help='Also add native variant (i.e. support building recipe for the build host as well as the target machine)', action='store_true')
2303 parser_add.add_argument('--src-subdir', help='Specify subdirectory within source tree to use', metavar='SUBDIR')
2304 parser_add.add_argument('--mirrors', help='Enable PREMIRRORS and MIRRORS for source tree fetching (disable by default).', action="store_true")
2305 parser_add.add_argument('--provides', '-p', help='Specify an alias for the item provided by the recipe. E.g. virtual/libgl')
2306 parser_add.set_defaults(func=add, fixed_setup=context.fixed_setup)
2307
2308 parser_modify = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
2309 description='Sets up the build environment to modify the source for an existing recipe. The default behaviour is to extract the source being fetched by the recipe into a git tree so you can work on it; alternatively if you already have your own pre-prepared source tree you can specify -n/--no-extract.',
2310 group='starting', order=90)
2311 parser_modify.add_argument('recipename', help='Name of existing recipe to edit (just name - no version, path or extension)')
2312 parser_modify.add_argument('srctree', nargs='?', help='Path to external source tree. If not specified, a subdirectory of %s will be used.' % defsrctree)
2313 parser_modify.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
2314 group = parser_modify.add_mutually_exclusive_group()
2315 group.add_argument('--extract', '-x', action="store_true", help='Extract source for recipe (default)')
2316 group.add_argument('--no-extract', '-n', action="store_true", help='Do not extract source, expect it to exist')
2317 group = parser_modify.add_mutually_exclusive_group()
2318 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
2319 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
2320 parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
2321 parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
2322 parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
2323 parser_modify.add_argument('--debug-build', action="store_true", help='Add DEBUG_BUILD = "1" to the modified recipe')
2324 parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup)
2325
2326 parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
2327 description='Extracts the source for an existing recipe',
2328 group='advanced')
2329 parser_extract.add_argument('recipename', help='Name of recipe to extract the source for')
2330 parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
2331 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
2332 parser_extract.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
2333 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
2334 parser_extract.set_defaults(func=extract, fixed_setup=context.fixed_setup)
2335
2336 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
2337 description='Synchronize the previously extracted source tree for an existing recipe',
2338 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
2339 group='advanced')
2340 parser_sync.add_argument('recipename', help='Name of recipe to sync the source for')
2341 parser_sync.add_argument('srctree', help='Path to the source tree')
2342 parser_sync.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
2343 parser_sync.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
2344 parser_sync.set_defaults(func=sync, fixed_setup=context.fixed_setup)
2345
2346 parser_rename = subparsers.add_parser('rename', help='Rename a recipe file in the workspace',
2347 description='Renames the recipe file for a recipe in the workspace, changing the name or version part or both, ensuring that all references within the workspace are updated at the same time. Only works when the recipe file itself is in the workspace, e.g. after devtool add. Particularly useful when devtool add did not automatically determine the correct name.',
2348 group='working', order=10)
2349 parser_rename.add_argument('recipename', help='Current name of recipe to rename')
2350 parser_rename.add_argument('newname', nargs='?', help='New name for recipe (optional, not needed if you only want to change the version)')
2351 parser_rename.add_argument('--version', '-V', help='Change the version (NOTE: this does not change the version fetched by the recipe, just the version in the recipe file name)')
2352 parser_rename.add_argument('--no-srctree', '-s', action='store_true', help='Do not rename the source tree directory (if the default source tree path has been used) - keeping the old name may be desirable if there are internal/other external references to this path')
2353 parser_rename.set_defaults(func=rename)
2354
2355 parser_update_recipe = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
2356 description='Applies changes from external source tree to a recipe (updating/adding/removing patches as necessary, or by updating SRCREV). Note that these changes need to have been committed to the git repository in order to be recognised.',
2357 group='working', order=-90)
2358 parser_update_recipe.add_argument('recipename', help='Name of recipe to update')
2359 parser_update_recipe.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
2360 parser_update_recipe.add_argument('--initial-rev', help='Override starting revision for patches')
2361 parser_update_recipe.add_argument('--append', '-a', help='Write changes to a bbappend in the specified layer instead of the recipe', metavar='LAYERDIR')
2362 parser_update_recipe.add_argument('--wildcard-version', '-w', help='In conjunction with -a/--append, use a wildcard to make the bbappend apply to any recipe version', action='store_true')
2363 parser_update_recipe.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
2364 parser_update_recipe.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
2365 parser_update_recipe.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
2366 parser_update_recipe.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)')
2367 parser_update_recipe.set_defaults(func=update_recipe)
2368
2369 parser_status = subparsers.add_parser('status', help='Show workspace status',
2370 description='Lists recipes currently in your workspace and the paths to their respective external source trees',
2371 group='info', order=100)
2372 parser_status.set_defaults(func=status)
2373
2374 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
2375 description='Removes the specified recipe(s) from your workspace (resetting its state back to that defined by the metadata).',
2376 group='working', order=-100)
2377 parser_reset.add_argument('recipename', nargs='*', help='Recipe to reset')
2378 parser_reset.add_argument('--all', '-a', action="store_true", help='Reset all recipes (clear workspace)')
2379 parser_reset.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
2380 parser_reset.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory along with append')
2381 parser_reset.set_defaults(func=reset)
2382
2383 parser_finish = subparsers.add_parser('finish', help='Finish working on a recipe in your workspace',
2384 description='Pushes any committed changes to the specified recipe to the specified layer and removes it from your workspace. Roughly equivalent to an update-recipe followed by reset, except the update-recipe step will do the "right thing" depending on the recipe and the destination layer specified. Note that your changes must have been committed to the git repository in order to be recognised.',
2385 group='working', order=-100)
2386 parser_finish.add_argument('recipename', help='Recipe to finish')
2387 parser_finish.add_argument('destination', help='Layer/path to put recipe into. Can be the name of a layer configured in your bblayers.conf, the path to the base of a layer, or a partial path inside a layer. %(prog)s will attempt to complete the path based on the layer\'s structure.')
2388 parser_finish.add_argument('--mode', '-m', choices=['patch', 'srcrev', 'auto'], default='auto', help='Update mode (where %(metavar)s is %(choices)s; default is %(default)s)', metavar='MODE')
2389 parser_finish.add_argument('--initial-rev', help='Override starting revision for patches')
2390 parser_finish.add_argument('--force', '-f', action="store_true", help='Force continuing even if there are uncommitted changes in the source tree repository')
2391 parser_finish.add_argument('--remove-work', '-r', action="store_true", help='Clean the sources directory under workspace')
2392 parser_finish.add_argument('--no-clean', '-n', action="store_true", help='Don\'t clean the sysroot to remove recipe output')
2393 parser_finish.add_argument('--no-overrides', '-O', action="store_true", help='Do not handle other override branches (if they exist)')
2394 parser_finish.add_argument('--dry-run', '-N', action="store_true", help='Dry-run (just report changes instead of writing them)')
2395 parser_finish.add_argument('--force-patch-refresh', action="store_true", help='Update patches in the layer even if they have not been modified (useful for refreshing patch context)')
2396 parser_finish.set_defaults(func=finish)
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2025-11-28 04:27:20 +0000