summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-12-13 20:09:41 +1300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-12-14 12:30:50 +0000
commit85de43fccda7718876ffefd64457c40a2fdc38fd (patch)
tree0a4103ffb6f2edef22dd258aa72f2c8601313bc9 /scripts/lib
parent86b55e6026840b3042745a3c4d2f857bf4808d18 (diff)
downloadpoky-85de43fccda7718876ffefd64457c40a2fdc38fd.tar.gz
devtool: fix extraction of source to work in memres mode
Extracting the source for a recipe (as used by devtool's extract, modify and upgrade subcommands) requires us to run do_fetch, do_unpack, do_patch and any tasks that the recipe has inserted inbetween, and do so with a modified datastore primarily so that we can redirect WORKDIR and STAMPS_DIR in order to have the files written out to a place of our choosing and avoid stamping the tasks as having executed in a real build context respectively. However, this all gets much more difficult when in memres mode since we can't call internal functions such as bb.build.exec_func() directly - instead we need to execute the tasks on the server. To do this we use the buildFile command which already exists for the purpose of supporting bitbake -b, and setVariable commands to set up the appropriate datastore. (I did look at passing the modified datastore to the buildFile command instead of using setVar() on the main datastore, however its use of databuilder makes that very difficult, and we'd also need a different method of getting the changes in the datastore over to the worker as well.) (From OE-Core rev: eb63b5339014fc72ba4829714e0a96a98e135ee2) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/devtool/standard.py122
-rw-r--r--scripts/lib/devtool/upgrade.py2
2 files changed, 55 insertions, 69 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 06c508c838..fbd8a710b2 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -378,7 +378,7 @@ def extract(args, config, basepath, workspace):
378 return 1 378 return 1
379 379
380 srctree = os.path.abspath(args.srctree) 380 srctree = os.path.abspath(args.srctree)
381 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd) 381 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, False, rd, tinfoil)
382 logger.info('Source tree extracted to %s' % srctree) 382 logger.info('Source tree extracted to %s' % srctree)
383 383
384 if initial_rev: 384 if initial_rev:
@@ -402,7 +402,7 @@ def sync(args, config, basepath, workspace):
402 return 1 402 return 1
403 403
404 srctree = os.path.abspath(args.srctree) 404 srctree = os.path.abspath(args.srctree)
405 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd) 405 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, True, rd, tinfoil)
406 logger.info('Source tree %s synchronized' % srctree) 406 logger.info('Source tree %s synchronized' % srctree)
407 407
408 if initial_rev: 408 if initial_rev:
@@ -412,35 +412,6 @@ def sync(args, config, basepath, workspace):
412 finally: 412 finally:
413 tinfoil.shutdown() 413 tinfoil.shutdown()
414 414
415class BbTaskExecutor(object):
416 """Class for executing bitbake tasks for a recipe
417
418 FIXME: This is very awkward. Unfortunately it's not currently easy to
419 properly execute tasks outside of bitbake itself, until then this has to
420 suffice if we are to handle e.g. linux-yocto's extra tasks
421 """
422
423 def __init__(self, rdata):
424 self.rdata = rdata
425 self.executed = []
426
427 def exec_func(self, func, report):
428 """Run bitbake task function"""
429 if not func in self.executed:
430 deps = self.rdata.getVarFlag(func, 'deps', False)
431 if deps:
432 for taskdepfunc in deps:
433 self.exec_func(taskdepfunc, True)
434 if report:
435 logger.info('Executing %s...' % func)
436 fn = self.rdata.getVar('FILE', True)
437 localdata = bb.build._task_data(fn, func, self.rdata)
438 try:
439 bb.build.exec_func(func, localdata)
440 except bb.build.FuncFailed as e:
441 raise DevtoolError(str(e))
442 self.executed.append(func)
443
444 415
445def _prep_extract_operation(config, basepath, recipename, tinfoil=None): 416def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
446 """HACK: Ugly workaround for making sure that requirements are met when 417 """HACK: Ugly workaround for making sure that requirements are met when
@@ -464,21 +435,10 @@ def _prep_extract_operation(config, basepath, recipename, tinfoil=None):
464 return tinfoil 435 return tinfoil
465 436
466 437
467def _extract_source(srctree, keep_temp, devbranch, sync, d): 438def _extract_source(srctree, keep_temp, devbranch, sync, d, tinfoil):
468 """Extract sources of a recipe""" 439 """Extract sources of a recipe"""
469 import bb.event
470 import oe.recipeutils 440 import oe.recipeutils
471 441
472 def eventfilter(name, handler, event, d):
473 """Bitbake event filter for devtool extract operation"""
474 if name == 'base_eventhandler':
475 return True
476 else:
477 return False
478
479 if hasattr(bb.event, 'set_eventfilter'):
480 bb.event.set_eventfilter(eventfilter)
481
482 pn = d.getVar('PN', True) 442 pn = d.getVar('PN', True)
483 443
484 _check_compatible_recipe(pn, d) 444 _check_compatible_recipe(pn, d)
@@ -504,19 +464,15 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
504 bb.utils.mkdirhier(srctree) 464 bb.utils.mkdirhier(srctree)
505 os.rmdir(srctree) 465 os.rmdir(srctree)
506 466
507 # We don't want notes to be printed, they are too verbose
508 origlevel = bb.logger.getEffectiveLevel()
509 if logger.getEffectiveLevel() > logging.DEBUG:
510 bb.logger.setLevel(logging.WARNING)
511
512 initial_rev = None 467 initial_rev = None
513 tempdir = tempfile.mkdtemp(prefix='devtool') 468 tempdir = tempfile.mkdtemp(prefix='devtool')
514 try: 469 try:
470 tinfoil.logger.setLevel(logging.WARNING)
471
515 crd = d.createCopy() 472 crd = d.createCopy()
516 # Make a subdir so we guard against WORKDIR==S 473 # Make a subdir so we guard against WORKDIR==S
517 workdir = os.path.join(tempdir, 'workdir') 474 workdir = os.path.join(tempdir, 'workdir')
518 crd.setVar('WORKDIR', workdir) 475 crd.setVar('WORKDIR', workdir)
519 crd.setVar('T', os.path.join(tempdir, 'temp'))
520 if not crd.getVar('S', True).startswith(workdir): 476 if not crd.getVar('S', True).startswith(workdir):
521 # Usually a shared workdir recipe (kernel, gcc) 477 # Usually a shared workdir recipe (kernel, gcc)
522 # Try to set a reasonable default 478 # Try to set a reasonable default
@@ -528,21 +484,55 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
528 # We don't want to move the source to STAGING_KERNEL_DIR here 484 # We don't want to move the source to STAGING_KERNEL_DIR here
529 crd.setVar('STAGING_KERNEL_DIR', '${S}') 485 crd.setVar('STAGING_KERNEL_DIR', '${S}')
530 486
531 task_executor = BbTaskExecutor(crd) 487 is_kernel_yocto = bb.data.inherits_class('kernel-yocto', d)
488 if not is_kernel_yocto:
489 crd.setVar('PATCHTOOL', 'git')
490 crd.setVar('PATCH_COMMIT_FUNCTIONS', '1')
491
492 # Apply our changes to the datastore to the server's datastore
493 for key in crd.localkeys():
494 tinfoil.config_data.setVar('%s_pn-%s' % (key, pn), crd.getVar(key, False))
495
496 tinfoil.config_data.setVar('STAMPS_DIR', os.path.join(tempdir, 'stamps'))
497 tinfoil.config_data.setVar('T', os.path.join(tempdir, 'temp'))
498 tinfoil.config_data.setVar('BUILDCFG_FUNCS', '')
499 tinfoil.config_data.setVar('BUILDCFG_HEADER', '')
500
501 tinfoil.set_event_mask(['bb.event.BuildStarted',
502 'bb.event.BuildCompleted',
503 'bb.event.TaskStarted',
504 'logging.LogRecord',
505 'bb.command.CommandCompleted',
506 'bb.command.CommandFailed',
507 'bb.build.TaskStarted',
508 'bb.build.TaskSucceeded',
509 'bb.build.TaskFailedSilent'])
510
511 def runtask(target, task):
512 if tinfoil.build_file(target, task):
513 while True:
514 event = tinfoil.wait_event(0.25)
515 if event:
516 if isinstance(event, bb.command.CommandCompleted):
517 break
518 elif isinstance(event, bb.command.CommandFailed):
519 raise DevtoolError('Task do_%s failed: %s' % (task, event.error))
520 elif isinstance(event, bb.build.TaskStarted):
521 logger.info('Executing %s...' % event._task)
522 elif isinstance(event, logging.LogRecord):
523 if event.levelno <= logging.INFO:
524 continue
525 logger.handle(event)
526
527 # we need virtual:native:/path/to/recipe if it's a BBCLASSEXTEND
528 fn = tinfoil.get_recipe_file(pn)
529 runtask(fn, 'unpack')
532 530
533 crd.setVar('EXTERNALSRC_forcevariable', '')
534
535 logger.info('Fetching %s...' % pn)
536 task_executor.exec_func('do_fetch', False)
537 logger.info('Unpacking...')
538 task_executor.exec_func('do_unpack', False)
539 if bb.data.inherits_class('kernel-yocto', d): 531 if bb.data.inherits_class('kernel-yocto', d):
540 # Extra step for kernel to populate the source directory 532 # Extra step for kernel to populate the source directory
541 logger.info('Doing kernel checkout...') 533 runtask(fn, 'kernel_checkout')
542 task_executor.exec_func('do_kernel_checkout', False)
543 srcsubdir = crd.getVar('S', True)
544 534
545 task_executor.check_git = True 535 srcsubdir = crd.getVar('S', True)
546 536
547 # Move local source files into separate subdir 537 # Move local source files into separate subdir
548 recipe_patches = [os.path.basename(patch) for patch in 538 recipe_patches = [os.path.basename(patch) for patch in
@@ -572,7 +562,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
572 os.path.basename(fname) not in recipe_patches] 562 os.path.basename(fname) not in recipe_patches]
573 # Force separate S so that patch files can be left out from srctree 563 # Force separate S so that patch files can be left out from srctree
574 srcsubdir = tempfile.mkdtemp(dir=workdir) 564 srcsubdir = tempfile.mkdtemp(dir=workdir)
575 crd.setVar('S', srcsubdir) 565 tinfoil.config_data.setVar('S_task-patch', srcsubdir)
576 # Move source files to S 566 # Move source files to S
577 for path in src_files: 567 for path in src_files:
578 _move_file(os.path.join(workdir, path), 568 _move_file(os.path.join(workdir, path),
@@ -595,10 +585,8 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
595 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) 585 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
596 initial_rev = stdout.rstrip() 586 initial_rev = stdout.rstrip()
597 587
598 crd.setVar('PATCHTOOL', 'git')
599
600 logger.info('Patching...') 588 logger.info('Patching...')
601 task_executor.exec_func('do_patch', False) 589 runtask(fn, 'patch')
602 590
603 bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) 591 bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
604 592
@@ -606,7 +594,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
606 if bb.data.inherits_class('kernel-yocto', d): 594 if bb.data.inherits_class('kernel-yocto', d):
607 # Store generate and store kernel config 595 # Store generate and store kernel config
608 logger.info('Generating kernel config') 596 logger.info('Generating kernel config')
609 task_executor.exec_func('do_configure', False) 597 runtask(fn, 'configure')
610 kconfig = os.path.join(crd.getVar('B', True), '.config') 598 kconfig = os.path.join(crd.getVar('B', True), '.config')
611 599
612 600
@@ -667,8 +655,6 @@ def _extract_source(srctree, keep_temp, devbranch, sync, d):
667 shutil.copy2(kconfig, srctree) 655 shutil.copy2(kconfig, srctree)
668 656
669 finally: 657 finally:
670 bb.logger.setLevel(origlevel)
671
672 if keep_temp: 658 if keep_temp:
673 logger.info('Preserving temporary directory %s' % tempdir) 659 logger.info('Preserving temporary directory %s' % tempdir)
674 else: 660 else:
@@ -773,7 +759,7 @@ def modify(args, config, basepath, workspace):
773 initial_rev = None 759 initial_rev = None
774 commits = [] 760 commits = []
775 if not args.no_extract: 761 if not args.no_extract:
776 initial_rev = _extract_source(srctree, False, args.branch, False, rd) 762 initial_rev = _extract_source(srctree, False, args.branch, False, rd, tinfoil)
777 if not initial_rev: 763 if not initial_rev:
778 return 1 764 return 1
779 logger.info('Source tree extracted to %s' % srctree) 765 logger.info('Source tree extracted to %s' % srctree)
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 52f9ab1a01..d89e9a23ac 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -363,7 +363,7 @@ def upgrade(args, config, basepath, workspace):
363 363
364 rf = None 364 rf = None
365 try: 365 try:
366 rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd) 366 rev1 = standard._extract_source(srctree, False, 'devtool-orig', False, rd, tinfoil)
367 rev2, md5, sha256 = _extract_new_source(args.version, srctree, args.no_patch, 367 rev2, md5, sha256 = _extract_new_source(args.version, srctree, args.no_patch,
368 args.srcrev, args.branch, args.keep_temp, 368 args.srcrev, args.branch, args.keep_temp,
369 tinfoil, rd) 369 tinfoil, rd)