diff options
Diffstat (limited to 'scripts/lib/devtool/standard.py')
-rw-r--r-- | scripts/lib/devtool/standard.py | 209 |
1 files changed, 63 insertions, 146 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index 4b489cfd79..b7f278f394 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py | |||
@@ -407,7 +407,7 @@ def extract(args, config, basepath, workspace): | |||
407 | return 1 | 407 | return 1 |
408 | 408 | ||
409 | srctree = os.path.abspath(args.srctree) | 409 | srctree = os.path.abspath(args.srctree) |
410 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) | 410 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil) |
411 | logger.info('Source tree extracted to %s' % srctree) | 411 | logger.info('Source tree extracted to %s' % srctree) |
412 | 412 | ||
413 | if initial_rev: | 413 | if initial_rev: |
@@ -431,7 +431,7 @@ def sync(args, config, basepath, workspace): | |||
431 | return 1 | 431 | return 1 |
432 | 432 | ||
433 | srctree = os.path.abspath(args.srctree) | 433 | srctree = os.path.abspath(args.srctree) |
434 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil) | 434 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, config, rd, tinfoil) |
435 | logger.info('Source tree %s synchronized' % srctree) | 435 | logger.info('Source tree %s synchronized' % srctree) |
436 | 436 | ||
437 | if initial_rev: | 437 | if initial_rev: |
@@ -442,9 +442,10 @@ def sync(args, config, basepath, workspace): | |||
442 | tinfoil.shutdown() | 442 | tinfoil.shutdown() |
443 | 443 | ||
444 | 444 | ||
445 | def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | 445 | def _extract_source(srctree, keep_temp, devbranch, sync, config, d, tinfoil): |
446 | """Extract sources of a recipe""" | 446 | """Extract sources of a recipe""" |
447 | import oe.recipeutils | 447 | import oe.recipeutils |
448 | import oe.patch | ||
448 | 449 | ||
449 | pn = d.getVar('PN') | 450 | pn = d.getVar('PN') |
450 | 451 | ||
@@ -466,15 +467,18 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | |||
466 | raise DevtoolError("The %s recipe has do_unpack disabled, unable to " | 467 | raise DevtoolError("The %s recipe has do_unpack disabled, unable to " |
467 | "extract source" % pn, 4) | 468 | "extract source" % pn, 4) |
468 | 469 | ||
469 | if bb.data.inherits_class('kernel-yocto', d): | ||
470 | tinfoil.build_targets('kern-tools-native') | ||
471 | |||
472 | if not sync: | 470 | if not sync: |
473 | # Prepare for shutil.move later on | 471 | # Prepare for shutil.move later on |
474 | bb.utils.mkdirhier(srctree) | 472 | bb.utils.mkdirhier(srctree) |
475 | os.rmdir(srctree) | 473 | os.rmdir(srctree) |
476 | 474 | ||
477 | initial_rev = None | 475 | initial_rev = None |
476 | |||
477 | appendexisted = False | ||
478 | recipefile = d.getVar('FILE') | ||
479 | appendfile = recipe_to_append(recipefile, config) | ||
480 | is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) | ||
481 | |||
478 | # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary | 482 | # We need to redirect WORKDIR, STAMPS_DIR etc. under a temporary |
479 | # directory so that: | 483 | # directory so that: |
480 | # (a) we pick up all files that get unpacked to the WORKDIR, and | 484 | # (a) we pick up all files that get unpacked to the WORKDIR, and |
@@ -493,147 +497,56 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | |||
493 | try: | 497 | try: |
494 | tinfoil.logger.setLevel(logging.WARNING) | 498 | tinfoil.logger.setLevel(logging.WARNING) |
495 | 499 | ||
496 | crd = d.createCopy() | 500 | # FIXME this results in a cache reload under control of tinfoil, which is fine |
497 | # Make a subdir so we guard against WORKDIR==S | 501 | # except we don't get the knotty progress bar |
498 | workdir = os.path.join(tempdir, 'workdir') | ||
499 | crd.setVar('WORKDIR', workdir) | ||
500 | if not crd.getVar('S').startswith(workdir): | ||
501 | # Usually a shared workdir recipe (kernel, gcc) | ||
502 | # Try to set a reasonable default | ||
503 | if bb.data.inherits_class('kernel', d): | ||
504 | crd.setVar('S', '${WORKDIR}/source') | ||
505 | else: | ||
506 | crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S'))) | ||
507 | if bb.data.inherits_class('kernel', d): | ||
508 | # We don't want to move the source to STAGING_KERNEL_DIR here | ||
509 | crd.setVar('STAGING_KERNEL_DIR', '${S}') | ||
510 | |||
511 | is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d) | ||
512 | if not is_kernel_yocto: | ||
513 | crd.setVar('PATCHTOOL', 'git') | ||
514 | crd.setVar('PATCH_COMMIT_FUNCTIONS', '1') | ||
515 | |||
516 | # Apply our changes to the datastore to the server's datastore | ||
517 | for key in crd.localkeys(): | ||
518 | tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False)) | ||
519 | |||
520 | tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps')) | ||
521 | tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp')) | ||
522 | tinfoil.config_data.setVar('BUILDCFG_FUNCS', '') | ||
523 | tinfoil.config_data.setVar('BUILDCFG_HEADER', '') | ||
524 | tinfoil.config_data.setVar('BB_HASH_IGNORE_MISMATCH', '1') | ||
525 | |||
526 | tinfoil.set_event_mask(['bb.event.BuildStarted', | ||
527 | 'bb.event.BuildCompleted', | ||
528 | 'logging.LogRecord', | ||
529 | 'bb.command.CommandCompleted', | ||
530 | 'bb.command.CommandFailed', | ||
531 | 'bb.cooker.CookerExit', | ||
532 | 'bb.build.TaskStarted', | ||
533 | 'bb.build.TaskSucceeded', | ||
534 | 'bb.build.TaskFailed', | ||
535 | 'bb.build.TaskFailedSilent']) | ||
536 | |||
537 | def runtask(target, task): | ||
538 | error = False | ||
539 | if tinfoil.build_file(target, task): | ||
540 | while True: | ||
541 | event = tinfoil.wait_event(0.25) | ||
542 | if event: | ||
543 | if isinstance(event, bb.command.CommandCompleted): | ||
544 | break | ||
545 | elif isinstance(event, bb.cooker.CookerExit): | ||
546 | # The server is going away, so drop the connection | ||
547 | tinfoil.server_connection = None | ||
548 | break | ||
549 | elif isinstance(event, bb.command.CommandFailed): | ||
550 | raise DevtoolError('Task do_%s failed: %s' % (task, event.error)) | ||
551 | elif isinstance(event, bb.build.TaskFailed): | ||
552 | raise DevtoolError('Task do_%s failed' % task) | ||
553 | elif isinstance(event, bb.build.TaskStarted): | ||
554 | logger.info('Executing %s...' % event._task) | ||
555 | elif isinstance(event, logging.LogRecord): | ||
556 | if event.levelno <= logging.INFO: | ||
557 | continue | ||
558 | if event.levelno >= logging.ERROR: | ||
559 | error = True | ||
560 | logger.handle(event) | ||
561 | if error: | ||
562 | raise DevtoolError('An error occurred during do_%s, exiting' % task) | ||
563 | |||
564 | # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND | ||
565 | fn = tinfoil.get_recipe_file(pn) | ||
566 | runtask(fn, 'unpack') | ||
567 | |||
568 | if bb.data.inherits_class('kernel-yocto', d): | ||
569 | # Extra step for kernel to populate the source directory | ||
570 | runtask(fn, 'kernel_checkout') | ||
571 | |||
572 | srcsubdir = crd.getVar('S') | ||
573 | |||
574 | # Move local source files into separate subdir | ||
575 | recipe_patches = [os.path.basename(patch) for patch in | ||
576 | oe.recipeutils.get_recipe_patches(crd)] | ||
577 | local_files = oe.recipeutils.get_recipe_local_files(crd) | ||
578 | |||
579 | # Ignore local files with subdir={BP} | ||
580 | srcabspath = os.path.abspath(srcsubdir) | ||
581 | local_files = [fname for fname in local_files if | ||
582 | os.path.exists(os.path.join(workdir, fname)) and | ||
583 | (srcabspath == workdir or not | ||
584 | os.path.join(workdir, fname).startswith(srcabspath + | ||
585 | os.sep))] | ||
586 | if local_files: | ||
587 | for fname in local_files: | ||
588 | _move_file(os.path.join(workdir, fname), | ||
589 | os.path.join(tempdir, 'oe-local-files', fname)) | ||
590 | with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'), | ||
591 | 'w') as f: | ||
592 | f.write('# Ignore local files, by default. Remove this file ' | ||
593 | 'if you want to commit the directory to Git\n*\n') | ||
594 | |||
595 | if srcsubdir == workdir: | ||
596 | # Find non-patch non-local sources that were "unpacked" to srctree | ||
597 | # directory | ||
598 | src_files = [fname for fname in _ls_tree(workdir) if | ||
599 | os.path.basename(fname) not in recipe_patches] | ||
600 | # Force separate S so that patch files can be left out from srctree | ||
601 | srcsubdir = tempfile.mkdtemp(dir=workdir) | ||
602 | tinfoil.config_data.setVar('S_task-patch', srcsubdir) | ||
603 | # Move source files to S | ||
604 | for path in src_files: | ||
605 | _move_file(os.path.join(workdir, path), | ||
606 | os.path.join(srcsubdir, path)) | ||
607 | elif os.path.dirname(srcsubdir) != workdir: | ||
608 | # Handle if S is set to a subdirectory of the source | ||
609 | srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) | ||
610 | |||
611 | scriptutils.git_convert_standalone_clone(srcsubdir) | ||
612 | 502 | ||
613 | # Make sure that srcsubdir exists | 503 | if os.path.exists(appendfile): |
614 | bb.utils.mkdirhier(srcsubdir) | 504 | appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak') |
615 | if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir): | 505 | shutil.copyfile(appendfile, appendbackup) |
616 | logger.warning("no source unpacked to S, either the %s recipe " | 506 | else: |
617 | "doesn't use any source or the correct source " | 507 | appendbackup = None |
618 | "directory could not be determined" % pn) | 508 | bb.utils.mkdirhier(os.path.dirname(appendfile)) |
619 | 509 | logger.debug('writing append file %s' % appendfile) | |
620 | setup_git_repo(srcsubdir, crd.getVar('PV'), devbranch, d=d) | 510 | with open(appendfile, 'a') as f: |
621 | 511 | f.write('###--- _extract_source\n') | |
622 | (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) | 512 | f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) |
623 | initial_rev = stdout.rstrip() | 513 | f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) |
514 | if not is_kernel_yocto: | ||
515 | f.write('PATCHTOOL = "git"\n') | ||
516 | f.write('PATCH_COMMIT_FUNCTIONS = "1"\n') | ||
517 | f.write('inherit devtool-source\n') | ||
518 | f.write('###--- _extract_source\n') | ||
519 | |||
520 | update_unlockedsigs(basepath, workspace, fixed_setup, [pn]) | ||
521 | |||
522 | sstate_manifests = d.getVar('SSTATE_MANIFESTS') | ||
523 | bb.utils.mkdirhier(sstate_manifests) | ||
524 | preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') | ||
525 | with open(preservestampfile, 'w') as f: | ||
526 | f.write(d.getVar('STAMP')) | ||
527 | try: | ||
528 | if bb.data.inherits_class('kernel-yocto', d): | ||
529 | # We need to generate the kernel config | ||
530 | task = 'do_configure' | ||
531 | else: | ||
532 | task = 'do_patch' | ||
624 | 533 | ||
625 | logger.info('Patching...') | 534 | # Run the fetch + unpack tasks |
626 | runtask(fn, 'patch') | 535 | res = tinfoil.build_targets(pn, |
536 | task, | ||
537 | handle_events=True) | ||
538 | finally: | ||
539 | if os.path.exists(preservestampfile): | ||
540 | os.remove(preservestampfile) | ||
627 | 541 | ||
628 | bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) | 542 | if not res: |
543 | raise DevtoolError('Extracting source for %s failed' % pn) | ||
629 | 544 | ||
630 | kconfig = None | 545 | with open(os.path.join(tempdir, 'initial_rev'), 'r') as f: |
631 | if bb.data.inherits_class('kernel-yocto', d): | 546 | initial_rev = f.read() |
632 | # Store generate and store kernel config | ||
633 | logger.info('Generating kernel config') | ||
634 | runtask(fn, 'configure') | ||
635 | kconfig = os.path.join(crd.getVar('B'), '.config') | ||
636 | 547 | ||
548 | with open(os.path.join(tempdir, 'srcsubdir'), 'r') as f: | ||
549 | srcsubdir = f.read() | ||
637 | 550 | ||
638 | tempdir_localdir = os.path.join(tempdir, 'oe-local-files') | 551 | tempdir_localdir = os.path.join(tempdir, 'oe-local-files') |
639 | srctree_localdir = os.path.join(srctree, 'oe-local-files') | 552 | srctree_localdir = os.path.join(srctree, 'oe-local-files') |
@@ -687,11 +600,15 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil): | |||
687 | oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) | 600 | oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=d) |
688 | bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) | 601 | bb.process.run('git %s commit -a -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree) |
689 | 602 | ||
690 | if kconfig: | 603 | if is_kernel_yocto: |
691 | logger.info('Copying kernel config to srctree') | 604 | logger.info('Copying kernel config to srctree') |
692 | shutil.copy2(kconfig, srctree) | 605 | shutil.copy2(os.path.join(tempdir, '.config'), srctree) |
693 | 606 | ||
694 | finally: | 607 | finally: |
608 | if appendbackup: | ||
609 | shutil.copyfile(appendbackup, appendfile) | ||
610 | elif os.path.exists(appendfile): | ||
611 | os.remove(appendfile) | ||
695 | if keep_temp: | 612 | if keep_temp: |
696 | logger.info('Preserving temporary directory %s' % tempdir) | 613 | logger.info('Preserving temporary directory %s' % tempdir) |
697 | else: | 614 | else: |
@@ -794,7 +711,7 @@ def modify(args, config, basepath, workspace): | |||
794 | initial_rev = None | 711 | initial_rev = None |
795 | commits = [] | 712 | commits = [] |
796 | if not args.no_extract: | 713 | if not args.no_extract: |
797 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil) | 714 | initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, config, rd, tinfoil) |
798 | if not initial_rev: | 715 | if not initial_rev: |
799 | return 1 | 716 | return 1 |
800 | logger.info('Source tree extracted to %s' % srctree) | 717 | logger.info('Source tree extracted to %s' % srctree) |
@@ -1894,7 +1811,7 @@ def register_commands(subparsers, context): | |||
1894 | parser_extract.add_argument('srctree', help='Path to where to extract the source tree') | 1811 | parser_extract.add_argument('srctree', help='Path to where to extract the source tree') |
1895 | parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') | 1812 | parser_extract.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (default "%(default)s")') |
1896 | parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') | 1813 | parser_extract.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') |
1897 | parser_extract.set_defaults(func=extract, no_workspace=True) | 1814 | parser_extract.set_defaults(func=extract) |
1898 | 1815 | ||
1899 | parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', | 1816 | parser_sync = subparsers.add_parser('sync', help='Synchronize the source tree for an existing recipe', |
1900 | description='Synchronize the previously extracted source tree for an existing recipe', | 1817 | description='Synchronize the previously extracted source tree for an existing recipe', |