summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2017-09-05 10:56:18 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-09-18 11:07:29 +0100
commit10af6d86b3effc523cfa0ec49741c5b02ee2cf86 (patch)
tree64974ae0b7f5fbfaa5647618bf401b98b1d93228 /scripts
parent4842db00b7aaac96f7bbdfbea5d8bcdb2ebac81b (diff)
downloadpoky-10af6d86b3effc523cfa0ec49741c5b02ee2cf86.tar.gz
devtool: rework source extraction so that dependencies are handled
Since it was first implemented, devtool's source extraction (as used by the devtool modify, extract and upgrade subcommands) ignored other recipe dependencies - so for example if you ran devtool modify on a recipe that fetches from svn or is compressed using xz then it would fail if those dependencies hadn't been built first. Now that we can execute tasks in the normal way (i.e. tinfoil.build_targets()) then we can rework it to use that. This is slightly tricky in that the source extraction needs to insert some logic in between tasks; luckily we can use a helper class that conditionally adds prefuncs to make that possible. Some side-effects / aspects of this change worth noting: * Operations are a little slower because we have to go through the task dependency graph generation and other startup processing. There's not really any way to avoid this though. * devtool extract didn't used to require a workspace, now it does because it needs to create a temporary bbappend for the recipe. (As with other commands the workspace be created on the fly if it doesn't already exist.) * I want any existing sysroot files and stamps to be left alone during extraction since we are running the tasks off to the side, and especially devtool extract should be able to be used without touching these. However, this was hampered by the automatic removal process in sstate.bbclass triggered by bb.event.ReachableStamps when the task signatures change, thus I had to introduce a way to disable this removal on a per-recipe basis (we still want it to function for any dependencies that we aren't working on). To implement this I elected to use a file written to tmp/sstate-control which gets deleted automatically after reading so that there's less chance of stale files affecting future sessions. I could have used a variable but this would have needed to be whitelisted and I'd have to have poked its value in using the setVariable command. Fixes [YOCTO #11198]. (From OE-Core rev: 830dbd66992cbb9e731b48d56fddf8f220349666) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/lib/devtool/standard.py209
-rw-r--r--scripts/lib/devtool/upgrade.py2
2 files changed, 64 insertions, 147 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 4b489cfd79..b7f278f394 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -407,7 +407,7 @@ def extract(args, config, basepath, workspace):
407 return 1 407 return 1
408 408
409 srctree = os.path.abspath(args.srctree) 409 srctree = os.path.abspath(args.srctree)
410 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) 410 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil)
411 logger.info('Source tree extracted to %s' % srctree) 411 logger.info('Source tree extracted to %s' % srctree)
412 412
413 if initial_rev: 413 if initial_rev:
@@ -431,7 +431,7 @@ def sync(args, config, basepath, workspace):
431 return 1 431 return 1
432 432
433 srctree = os.path.abspath(args.srctree) 433 srctree = os.path.abspath(args.srctree)
434 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil) 434 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, config, rd, tinfoil)
435 logger.info('Source tree %s synchronized' % srctree) 435 logger.info('Source tree %s synchronized' % srctree)
436 436
437 if initial_rev: 437 if initial_rev:
@@ -442,9 +442,10 @@ def sync(args, config, basepath, workspace):
442 tinfoil.shutdown() 442 tinfoil.shutdown()
443 443
444 444
445def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): 445def _extract_source(srctree, keep_temp, devbranch, sync, config, d, tinfoil):
446 """Extract sources of a recipe""" 446 """Extract sources of a recipe"""
447 import oe.recipeutils 447 import oe.recipeutils
448 import oe.patch
448 449
449 pn = d.getVar('PN') 450 pn = d.getVar('PN')
450 451
@@ -466,15 +467,18 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
466 raise DevtoolError("The %s recipe has do_unpack disabled, unable to " 467 raise DevtoolError("The %s recipe has do_unpack disabled, unable to "
467 "extract source" % pn, 4) 468 "extract source" % pn, 4)
468 469
469 if bb.data.inherits_class('kernel-yocto', d):
470 tinfoil.build_targets('kern-tools-native')
471
472 if not sync: 470 if not sync:
473 # Prepare for shutil.move later on 471 # Prepare for shutil.move later on
474 bb.utils.mkdirhier(srctree) 472 bb.utils.mkdirhier(srctree)
475 os.rmdir(srctree) 473 os.rmdir(srctree)
476 474
477 initial_rev = None 475 initial_rev = None
476
477 appendexisted = False
478 recipefile = d.getVar('FILE')
479 appendfile = recipe_to_append(recipefile, config)
480 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
481
478 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary 482 # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary
479 # directory so that: 483 # directory so that:
480 # (a) we pick up all files that get unpacked to the WORKDIR, and 484 # (a) we pick up all files that get unpacked to the WORKDIR, and
@@ -493,147 +497,56 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
493 try: 497 try:
494 tinfoil.logger.setLevel(logging.WARNING) 498 tinfoil.logger.setLevel(logging.WARNING)
495 499
496 crd = d.createCopy() 500 # FIXME this results in a cache reload under control of tinfoil, which is fine
497 # Make a subdir so we guard against WORKDIR==S 501 # except we don't get the knotty progress bar
498 workdir = os.path.join(tempdir, 'workdir')
499 crd.setVar('WORKDIR', workdir)
500 if not crd.getVar('S').startswith(workdir):
501 # Usually a shared workdir recipe (kernel, gcc)
502 # Try to set a reasonable default
503 if bb.data.inherits_class('kernel', d):
504 crd.setVar('S', '${WORKDIR}/source')
505 else:
506 crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S')))
507 if bb.data.inherits_class('kernel', d):
508 # We don't want to move the source to STAGING_KERNEL_DIR here
509 crd.setVar('STAGING_KERNEL_DIR', '${S}')
510
511 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
512 if not is_kernel_yocto:
513 crd.setVar('PATCHTOOL', 'git')
514 crd.setVar('PATCH_COMMIT_FUNCTIONS', '1')
515
516 # Apply our changes to the datastore to the server's datastore
517 for key in crd.localkeys():
518 tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False))
519
520 tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
521 tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp'))
522 tinfoil.config_data.setVar('BUILDCFG_FUNCS', '')
523 tinfoil.config_data.setVar('BUILDCFG_HEADER', '')
524 tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1')
525
526 tinfoil.set_event_mask(['bb.event.BuildStarted',
527 'bb.event.BuildCompleted',
528 'logging.LogRecord',
529 'bb.command.CommandCompleted',
530 'bb.command.CommandFailed',
531 'bb.cooker.CookerExit',
532 'bb.build.TaskStarted',
533 'bb.build.TaskSucceeded',
534 'bb.build.TaskFailed',
535 'bb.build.TaskFailedSilent'])
536
537 def runtask(target, task):
538 error = False
539 if tinfoil.build_file(target, task):
540 while True:
541 event = tinfoil.wait_event(0.25)
542 if event:
543 if isinstance(event, bb.command.CommandCompleted):
544 break
545 elif isinstance(event, bb.cooker.CookerExit):
546 # The server is going away, so drop the connection
547 tinfoil.server_connection = None
548 break
549 elif isinstance(event, bb.command.CommandFailed):
550 raise DevtoolError('Task do_%s failed: %s' % (task, event.error))
551 elif isinstance(event, bb.build.TaskFailed):
552 raise DevtoolError('Task do_%s failed' % task)
553 elif isinstance(event, bb.build.TaskStarted):
554 logger.info('Executing %s...' % event._task)
555 elif isinstance(event, logging.LogRecord):
556 if event.levelno <= logging.INFO:
557 continue
558 if event.levelno >= logging.ERROR:
559 error = True
560 logger.handle(event)
561 if error:
562 raise DevtoolError('An error occurred during do_%s, exiting' % task)
563
564 # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND
565 fn = tinfoil.get_recipe_file(pn)
566 runtask(fn, 'unpack')
567
568 if bb.data.inherits_class('kernel-yocto', d):
569 # Extra step for kernel to populate the source directory
570 runtask(fn, 'kernel_checkout')
571
572 srcsubdir = crd.getVar('S')
573
574 # Move local source files into separate subdir
575 recipe_patches = [os.path.basename(patch) for patch in
576 oe.recipeutils.get_recipe_patches(crd)]
577 local_files = oe.recipeutils.get_recipe_local_files(crd)
578
579 # Ignore local files with subdir={BP}
580 srcabspath = os.path.abspath(srcsubdir)
581 local_files = [fname for fname in local_files if
582 os.path.exists(os.path.join(workdir, fname)) and
583 (srcabspath == workdir or not
584 os.path.join(workdir, fname).startswith(srcabspath +
585 os.sep))]
586 if local_files:
587 for fname in local_files:
588 _move_file(os.path.join(workdir, fname),
589 os.path.join(tempdir, 'oe-local-files', fname))
590 with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'),
591 'w') as f:
592 f.write('# Ignore local files, by default. Remove this file '
593 'if you want to commit the directory to Git\n*\n')
594
595 if srcsubdir == workdir:
596 # Find non-patch non-local sources that were "unpacked" to srctree
597 # directory
598 src_files = [fname for fname in _ls_tree(workdir) if
599 os.path.basename(fname) not in recipe_patches]
600 # Force separate S so that patch files can be left out from srctree
601 srcsubdir = tempfile.mkdtemp(dir=workdir)
602 tinfoil.config_data.setVar('S_task-patch', srcsubdir)
603 # Move source files to S
604 for path in src_files:
605 _move_file(os.path.join(workdir, path),
606 os.path.join(srcsubdir, path))
607 elif os.path.dirname(srcsubdir) != workdir:
608 # Handle if S is set to a subdirectory of the source
609 srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
610
611 scriptutils.git_convert_standalone_clone(srcsubdir)
612 502
613 # Make sure that srcsubdir exists 503 if os.path.exists(appendfile):
614 bb.utils.mkdirhier(srcsubdir) 504 appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak')
615 if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir): 505 shutil.copyfile(appendfile, appendbackup)
616 logger.warning("no source unpacked to S, either the %s recipe " 506 else:
617 "doesn't use any source or the correct source " 507 appendbackup = None
618 "directory could not be determined" % pn) 508 bb.utils.mkdirhier(os.path.dirname(appendfile))
619 509 logger.debug('writing append file %s' % appendfile)
620 setup_git_repo(srcsubdir, crd.getVar('PV'), devbranch, d=d) 510 with open(appendfile, 'a') as f:
621 511 f.write('###--- _extract_source\n')
622 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) 512 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir)
623 initial_rev = stdout.rstrip() 513 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch)
514 if not is_kernel_yocto:
515 f.write('PATCHTOOL = "git"\n')
516 f.write('PATCH_COMMIT_FUNCTIONS = "1"\n')
517 f.write('inherit devtool-source\n')
518 f.write('###--- _extract_source\n')
519
520 update_unlockedsigs(basepath, workspace, fixed_setup, [pn])
521
522 sstate_manifests = d.getVar('SSTATE_MANIFESTS')
523 bb.utils.mkdirhier(sstate_manifests)
524 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps')
525 with open(preservestampfile, 'w') as f:
526 f.write(d.getVar('STAMP'))
527 try:
528 if bb.data.inherits_class('kernel-yocto', d):
529 # We need to generate the kernel config
530 task = 'do_configure'
531 else:
532 task = 'do_patch'
624 533
625 logger.info('Patching...') 534 # Run the fetch + unpack tasks
626 runtask(fn, 'patch') 535 res = tinfoil.build_targets(pn,
536 task,
537 handle_events=True)
538 finally:
539 if os.path.exists(preservestampfile):
540 os.remove(preservestampfile)
627 541
628 bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) 542 if not res:
543 raise DevtoolError('Extracting source for %s failed' % pn)
629 544
630 kconfig = None 545 with open(os.path.join(tempdir, 'initial_rev'), 'r') as f:
631 if bb.data.inherits_class('kernel-yocto', d): 546 initial_rev = f.read()
632 # Store generate and store kernel config
633 logger.info('Generating kernel config')
634 runtask(fn, 'configure')
635 kconfig = os.path.join(crd.getVar('B'), '.config')
636 547
548 with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f:
549 srcsubdir = f.read()
637 550
638 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 551 tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
639 srctree_localdir = os.path.join(srctree, 'oe-local-files') 552 srctree_localdir = os.path.join(srctree, 'oe-local-files')
@@ -687,11 +600,15 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
687 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) 600 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d)
688 bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) 601 bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
689 602
690 if kconfig: 603 if is_kernel_yocto:
691 logger.info('Copying kernel config to srctree') 604 logger.info('Copying kernel config to srctree')
692 shutil.copy2(kconfig, srctree) 605 shutil.copy2(os.path.join(tempdir, '.config'), srctree)
693 606
694 finally: 607 finally:
608 if appendbackup:
609 shutil.copyfile(appendbackup, appendfile)
610 elif os.path.exists(appendfile):
611 os.remove(appendfile)
695 if keep_temp: 612 if keep_temp:
696 logger.info('Preserving temporary directory %s' % tempdir) 613 logger.info('Preserving temporary directory %s' % tempdir)
697 else: 614 else:
@@ -794,7 +711,7 @@ def modify(args, config, basepath, workspace):
794 initial_rev = None 711 initial_rev = None
795 commits = [] 712 commits = []
796 if not args.no_extract: 713 if not args.no_extract:
797 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) 714 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil)
798 if not initial_rev: 715 if not initial_rev:
799 return 1 716 return 1
800 logger.info('Source tree extracted to %s' % srctree) 717 logger.info('Source tree extracted to %s' % srctree)
@@ -1894,7 +1811,7 @@ def register_commands(subparsers, context):
1894 parser_extract.add_argument('srctree', help='Path to where to extract the source tree') 1811 parser_extract.add_argument('srctree', help='Path to where to extract the source tree')
1895 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') 1812 parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")')
1896 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 1813 parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
1897 parser_extract.set_defaults(func=extract, no_workspace=True) 1814 parser_extract.set_defaults(func=extract)
1898 1815
1899 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', 1816 parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe',
1900 description='Synchronize the previously extracted source tree for an existing recipe', 1817 description='Synchronize the previously extracted source tree for an existing recipe',
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 41bd34e61a..39d0bd72db 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -418,7 +418,7 @@ def upgrade(args, config, basepath, workspace):
418 418
419 rf = None 419 rf = None
420 try: 420 try:
421 rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd, tinfoil) 421 rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, rd, tinfoil)
422 rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch, 422 rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch,
423 args.srcrev, args.srcbranch, args.branch, args.keep_temp, 423 args.srcrev, args.srcbranch, args.branch, args.keep_temp,
424 tinfoil, rd) 424 tinfoil, rd)