diff options
author | Paul Eggleton <paul.eggleton@linux.intel.com> | 2017-09-05 10:56:18 +1200 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2017-09-18 11:07:29 +0100 |
commit | 10af6d86b3effc523cfa0ec49741c5b02ee2cf86 (patch) | |
tree | 64974ae0b7f5fbfaa5647618bf401b98b1d93228 /scripts | |
parent | 4842db00b7aaac96f7bbdfbea5d8bcdb2ebac81b (diff) | |
download | poky-10af6d86b3effc523cfa0ec49741c5b02ee2cf86.tar.gz |
devtool: rework source extraction so that dependencies are handled
Since it was first implemented, devtool's source extraction (as used by
the devtool modify, extract and upgrade subcommands) ignored other recipe
dependencies - so for example if you ran devtool modify on a recipe that
fetches from svn or is compressed using xz then it would fail if those
dependencies hadn't been built first. Now that we can execute tasks in
the normal way (i.e. tinfoil.build_targets()) then we can rework it to
use that. This is slightly tricky in that the source extraction needs to
insert some logic in between tasks; luckily we can use a helper class
that conditionally adds prefuncs to make that possible.
Some side-effects / aspects of this change worth noting:
* Operations are a little slower because we have to go through the task
dependency graph generation and other startup processing. There's not
really any way to avoid this though.
* devtool extract didn't used to require a workspace, now it does
because it needs to create a temporary bbappend for the recipe. (As
with other commands the workspace be created on the fly if it doesn't
already exist.)
* I want any existing sysroot files and stamps to be left alone during
extraction since we are running the tasks off to the side, and
especially devtool extract should be able to be used without touching
these. However, this was hampered by the automatic removal process in
sstate.bbclass triggered by bb.event.ReachableStamps when the task
signatures change, thus I had to introduce a way to disable this
removal on a per-recipe basis (we still want it to function for any
dependencies that we aren't working on). To implement this I elected
to use a file written to tmp/sstate-control which gets deleted
automatically after reading so that there's less chance of stale files
affecting future sessions. I could have used a variable but this would
have needed to be whitelisted and I'd have to have poked its value in
using the setVariable command.
Fixes [YOCTO #11198].
(From OE-Core rev: 830dbd66992cbb9e731b48d56fddf8f220349666)
Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/lib/devtool/standard.py | 209 | ||||
-rw-r--r-- | scripts/lib/devtool/upgrade.py | 2 |
2 files changed, 64 insertions, 147 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index 4b489cfd79..b7f278f394 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py | |||
@@ -407,7 +407,7 @@ def extract(args, config, basepath, workspace): | |||
407 | return 1 | 407 | return 1 |
408 | 408 | ||
409 | srctree = os.path.abspath(args.srctree) | 409 | srctree = os.path.abspath(args.srctree) |
410 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) | 410 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil) |
411 | logger.info('Source tree extracted to %s' % srctree) | 411 | logger.info('Source tree extracted to %s' % srctree) |
412 | 412 | ||
413 | if initial_rev: | 413 | if initial_rev: |
@@ -431,7 +431,7 @@ def sync(args, config, basepath, workspace): | |||
431 | return 1 | 431 | return 1 |
432 | 432 | ||
433 | srctree = os.path.abspath(args.srctree) | 433 | srctree = os.path.abspath(args.srctree) |
434 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil) | 434 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, config, rd, tinfoil) |
435 | logger.info('Source tree %s synchronized' % srctree) | 435 | logger.info('Source tree %s synchronized' % srctree) |
436 | 436 | ||
437 | if initial_rev: | 437 | if initial_rev: |
@@ -442,9 +442,10 @@ def sync(args, config, basepath, workspace): | |||
442 | tinfoil.shutdown() | 442 | tinfoil.shutdown() |
443 | 443 | ||
444 | 444 | ||
445 | def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | 445 | def _extract_source(srctree, keep_temp, devbranch, sync, config, d, tinfoil): |
446 | """Extract sources of a recipe""" | 446 | """Extract sources of a recipe""" |
447 | import oe.recipeutils | 447 | import oe.recipeutils |
448 | import oe.patch | ||
448 | 449 | ||
449 | pn = d.getVar('PN') | 450 | pn = d.getVar('PN') |
450 | 451 | ||
@@ -466,15 +467,18 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | |||
466 | raise DevtoolError("The %s recipe has do_unpack disabled, unable to " | 467 | raise DevtoolError("The %s recipe has do_unpack disabled, unable to " |
467 | "extract source" % pn, 4) | 468 | "extract source" % pn, 4) |
468 | 469 | ||
469 | if bb.data.inherits_class('kernel-yocto', d): | ||
470 | tinfoil.build_targets('kern-tools-native') | ||
471 | |||
472 | if not sync: | 470 | if not sync: |
473 | # Prepare for shutil.move later on | 471 | # Prepare for shutil.move later on |
474 | bb.utils.mkdirhier(srctree) | 472 | bb.utils.mkdirhier(srctree) |
475 | os.rmdir(srctree) | 473 | os.rmdir(srctree) |
476 | 474 | ||
477 | initial_rev = None | 475 | initial_rev = None |
476 | |||
477 | appendexisted = False | ||
478 | recipefile = d.getVar('FILE') | ||
479 | appendfile = recipe_to_append(recipefile, config) | ||
480 | is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) | ||
481 | |||
478 | # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary | 482 | # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary |
479 | # directory so that: | 483 | # directory so that: |
480 | # (a) we pick up all files that get unpacked to the WORKDIR, and | 484 | # (a) we pick up all files that get unpacked to the WORKDIR, and |
@@ -493,147 +497,56 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | |||
493 | try: | 497 | try: |
494 | tinfoil.logger.setLevel(logging.WARNING) | 498 | tinfoil.logger.setLevel(logging.WARNING) |
495 | 499 | ||
496 | crd = d.createCopy() | 500 | # FIXME this results in a cache reload under control of tinfoil, which is fine |
497 | # Make a subdir so we guard against WORKDIR==S | 501 | # except we don't get the knotty progress bar |
498 | workdir = os.path.join(tempdir, 'workdir') | ||
499 | crd.setVar('WORKDIR', workdir) | ||
500 | if not crd.getVar('S').startswith(workdir): | ||
501 | # Usually a shared workdir recipe (kernel, gcc) | ||
502 | # Try to set a reasonable default | ||
503 | if bb.data.inherits_class('kernel', d): | ||
504 | crd.setVar('S', '${WORKDIR}/source') | ||
505 | else: | ||
506 | crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S'))) | ||
507 | if bb.data.inherits_class('kernel', d): | ||
508 | # We don't want to move the source to STAGING_KERNEL_DIR here | ||
509 | crd.setVar('STAGING_KERNEL_DIR', '${S}') | ||
510 | |||
511 | is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) | ||
512 | if not is_kernel_yocto: | ||
513 | crd.setVar('PATCHTOOL', 'git') | ||
514 | crd.setVar('PATCH_COMMIT_FUNCTIONS', '1') | ||
515 | |||
516 | # Apply our changes to the datastore to the server's datastore | ||
517 | for key in crd.localkeys(): | ||
518 | tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False)) | ||
519 | |||
520 | tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps')) | ||
521 | tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp')) | ||
522 | tinfoil.config_data.setVar('BUILDCFG_FUNCS', '') | ||
523 | tinfoil.config_data.setVar('BUILDCFG_HEADER', '') | ||
524 | tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1') | ||
525 | |||
526 | tinfoil.set_event_mask(['bb.event.BuildStarted', | ||
527 | 'bb.event.BuildCompleted', | ||
528 | 'logging.LogRecord', | ||
529 | 'bb.command.CommandCompleted', | ||
530 | 'bb.command.CommandFailed', | ||
531 | 'bb.cooker.CookerExit', | ||
532 | 'bb.build.TaskStarted', | ||
533 | 'bb.build.TaskSucceeded', | ||
534 | 'bb.build.TaskFailed', | ||
535 | 'bb.build.TaskFailedSilent']) | ||
536 | |||
537 | def runtask(target, task): | ||
538 | error = False | ||
539 | if tinfoil.build_file(target, task): | ||
540 | while True: | ||
541 | event = tinfoil.wait_event(0.25) | ||
542 | if event: | ||
543 | if isinstance(event, bb.command.CommandCompleted): | ||
544 | break | ||
545 | elif isinstance(event, bb.cooker.CookerExit): | ||
546 | # The server is going away, so drop the connection | ||
547 | tinfoil.server_connection = None | ||
548 | break | ||
549 | elif isinstance(event, bb.command.CommandFailed): | ||
550 | raise DevtoolError('Task do_%s failed: %s' % (task, event.error)) | ||
551 | elif isinstance(event, bb.build.TaskFailed): | ||
552 | raise DevtoolError('Task do_%s failed' % task) | ||
553 | elif isinstance(event, bb.build.TaskStarted): | ||
554 | logger.info('Executing %s...' % event._task) | ||
555 | elif isinstance(event, logging.LogRecord): | ||
556 | if event.levelno <= logging.INFO: | ||
557 | continue | ||
558 | if event.levelno >= logging.ERROR: | ||
559 | error = True | ||
560 | logger.handle(event) | ||
561 | if error: | ||
562 | raise DevtoolError('An error occurred during do_%s, exiting' % task) | ||
563 | |||
564 | # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND | ||
565 | fn = tinfoil.get_recipe_file(pn) | ||
566 | runtask(fn, 'unpack') | ||
567 | |||
568 | if bb.data.inherits_class('kernel-yocto', d): | ||
569 | # Extra step for kernel to populate the source directory | ||
570 | runtask(fn, 'kernel_checkout') | ||
571 | |||
572 | srcsubdir = crd.getVar('S') | ||
573 | |||
574 | # Move local source files into separate subdir | ||
575 | recipe_patches = [os.path.basename(patch) for patch in | ||
576 | oe.recipeutils.get_recipe_patches(crd)] | ||
577 | local_files = oe.recipeutils.get_recipe_local_files(crd) | ||
578 | |||
579 | # Ignore local files with subdir={BP} | ||
580 | srcabspath = os.path.abspath(srcsubdir) | ||
581 | local_files = [fname for fname in local_files if | ||
582 | os.path.exists(os.path.join(workdir, fname)) and | ||
583 | (srcabspath == workdir or not | ||
584 | os.path.join(workdir, fname).startswith(srcabspath + | ||
585 | os.sep))] | ||
586 | if local_files: | ||
587 | for fname in local_files: | ||
588 | _move_file(os.path.join(workdir, fname), | ||
589 | os.path.join(tempdir, 'oe-local-files', fname)) | ||
590 | with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'), | ||
591 | 'w') as f: | ||
592 | f.write('# Ignore local files, by default. Remove this file ' | ||
593 | 'if you want to commit the directory to Git\n*\n') | ||
594 | |||
595 | if srcsubdir == workdir: | ||
596 | # Find non-patch non-local sources that were "unpacked" to srctree | ||
597 | # directory | ||
598 | src_files = [fname for fname in _ls_tree(workdir) if | ||
599 | os.path.basename(fname) not in recipe_patches] | ||
600 | # Force separate S so that patch files can be left out from srctree | ||
601 | srcsubdir = tempfile.mkdtemp(dir=workdir) | ||
602 | tinfoil.config_data.setVar('S_task-patch', srcsubdir) | ||
603 | # Move source files to S | ||
604 | for path in src_files: | ||
605 | _move_file(os.path.join(workdir, path), | ||
606 | os.path.join(srcsubdir, path)) | ||
607 | elif os.path.dirname(srcsubdir) != workdir: | ||
608 | # Handle if S is set to a subdirectory of the source | ||
609 | srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) | ||
610 | |||
611 | scriptutils.git_convert_standalone_clone(srcsubdir) | ||
612 | 502 | ||
613 | # Make sure that srcsubdir exists | 503 | if os.path.exists(appendfile): |
614 | bb.utils.mkdirhier(srcsubdir) | 504 | appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') |
615 | if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir): | 505 | shutil.copyfile(appendfile, appendbackup) |
616 | logger.warning("no source unpacked to S, either the %s recipe " | 506 | else: |
617 | "doesn't use any source or the correct source " | 507 | appendbackup = None |
618 | "directory could not be determined" % pn) | 508 | bb.utils.mkdirhier(os.path.dirname(appendfile)) |
619 | 509 | logger.debug('writing append file %s' % appendfile) | |
620 | setup_git_repo(srcsubdir, crd.getVar('PV'), devbranch, d=d) | 510 | with open(appendfile, 'a') as f: |
621 | 511 | f.write('###--- _extract_source\n') | |
622 | (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) | 512 | f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) |
623 | initial_rev = stdout.rstrip() | 513 | f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) |
514 | if not is_kernel_yocto: | ||
515 | f.write('PATCHTOOL = "git"\n') | ||
516 | f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') | ||
517 | f.write('inherit devtool-source\n') | ||
518 | f.write('###--- _extract_source\n') | ||
519 | |||
520 | update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) | ||
521 | |||
522 | sstate_manifests = d.getVar('SSTATE_MANIFESTS') | ||
523 | bb.utils.mkdirhier(sstate_manifests) | ||
524 | preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') | ||
525 | with open(preservestampfile, 'w') as f: | ||
526 | f.write(d.getVar('STAMP')) | ||
527 | try: | ||
528 | if bb.data.inherits_class('kernel-yocto', d): | ||
529 | # We need to generate the kernel config | ||
530 | task = 'do_configure' | ||
531 | else: | ||
532 | task = 'do_patch' | ||
624 | 533 | ||
625 | logger.info('Patching...') | 534 | # Run the fetch + unpack tasks |
626 | runtask(fn, 'patch') | 535 | res = tinfoil.build_targets(pn, |
536 | task, | ||
537 | handle_events=True) | ||
538 | finally: | ||
539 | if os.path.exists(preservestampfile): | ||
540 | os.remove(preservestampfile) | ||
627 | 541 | ||
628 | bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) | 542 | if not res: |
543 | raise DevtoolError('Extracting source for %s failed' % pn) | ||
629 | 544 | ||
630 | kconfig = None | 545 | with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: |
631 | if bb.data.inherits_class('kernel-yocto', d): | 546 | initial_rev = f.read() |
632 | # Store generate and store kernel config | ||
633 | logger.info('Generating kernel config') | ||
634 | runtask(fn, 'configure') | ||
635 | kconfig = os.path.join(crd.getVar('B'), '.config') | ||
636 | 547 | ||
548 | with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: | ||
549 | srcsubdir = f.read() | ||
637 | 550 | ||
638 | tempdir_localdir = os.path.join(tempdir, 'oe-local-files') | 551 | tempdir_localdir = os.path.join(tempdir, 'oe-local-files') |
639 | srctree_localdir = os.path.join(srctree, 'oe-local-files') | 552 | srctree_localdir = os.path.join(srctree, 'oe-local-files') |
@@ -687,11 +600,15 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | |||
687 | oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) | 600 | oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) |
688 | bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) | 601 | bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) |
689 | 602 | ||
690 | if kconfig: | 603 | if is_kernel_yocto: |
691 | logger.info('Copying kernel config to srctree') | 604 | logger.info('Copying kernel config to srctree') |
692 | shutil.copy2(kconfig, srctree) | 605 | shutil.copy2(os.path.join(tempdir, '.config'), srctree) |
693 | 606 | ||
694 | finally: | 607 | finally: |
608 | if appendbackup: | ||
609 | shutil.copyfile(appendbackup, appendfile) | ||
610 | elif os.path.exists(appendfile): | ||
611 | os.remove(appendfile) | ||
695 | if keep_temp: | 612 | if keep_temp: |
696 | logger.info('Preserving temporary directory %s' % tempdir) | 613 | logger.info('Preserving temporary directory %s' % tempdir) |
697 | else: | 614 | else: |
@@ -794,7 +711,7 @@ def modify(args, config, basepath, workspace): | |||
794 | initial_rev = None | 711 | initial_rev = None |
795 | commits = [] | 712 | commits = [] |
796 | if not args.no_extract: | 713 | if not args.no_extract: |
797 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) | 714 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil) |
798 | if not initial_rev: | 715 | if not initial_rev: |
799 | return 1 | 716 | return 1 |
800 | logger.info('Source tree extracted to %s' % srctree) | 717 | logger.info('Source tree extracted to %s' % srctree) |
@@ -1894,7 +1811,7 @@ def register_commands(subparsers, context): | |||
1894 | parser_extract.add_argument('srctree', help='Path to where to extract the source tree') | 1811 | parser_extract.add_argument('srctree', help='Path to where to extract the source tree') |
1895 | parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') | 1812 | parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') |
1896 | parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') | 1813 | parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') |
1897 | parser_extract.set_defaults(func=extract, no_workspace=True) | 1814 | parser_extract.set_defaults(func=extract) |
1898 | 1815 | ||
1899 | parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', | 1816 | parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', |
1900 | description='Synchronize the previously extracted source tree for an existing recipe', | 1817 | description='Synchronize the previously extracted source tree for an existing recipe', |
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py index 41bd34e61a..39d0bd72db 100644 --- a/scripts/lib/devtool/upgrade.py +++ b/scripts/lib/devtool/upgrade.py | |||
@@ -418,7 +418,7 @@ def upgrade(args, config, basepath, workspace): | |||
418 | 418 | ||
419 | rf = None | 419 | rf = None |
420 | try: | 420 | try: |
421 | rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd, tinfoil) | 421 | rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, rd, tinfoil) |
422 | rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch, | 422 | rev2, md5, sha256, srcbranch = _extract_new_source(args.version, srctree, args.no_patch, |
423 | args.srcrev, args.srcbranch, args.branch, args.keep_temp, | 423 | args.srcrev, args.srcbranch, args.branch, args.keep_temp, |
424 | tinfoil, rd) | 424 | tinfoil, rd) |