summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-01-25 14:06:35 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-02-06 11:06:30 +0000
commit0899c332f76fac042bd368ab5d030aadfa327f58 (patch)
tree92eb12f306ef22d2f11f4c04733c36fa22319db6
parent110f56dfa5acf8424ed8b9cff73d36a1b92f6bbb (diff)
downloadpoky-0899c332f76fac042bd368ab5d030aadfa327f58.tar.gz
bitbake: runqueue: Fix recidepends handling
Currently we only run through the recidepends/recrdepends code once. This means that we can miss some expansions of dependency trees where one rec{r,i}depends tasks depends on another rec{r,i}depends task. In reality we need to iterate over the data until we stop adding dependencies. In doing this we can't show quite so granular progress information since we don't know how many times we'll need to do this. This does slow down the runqueue prepare phase however some optimisations are possible and can be handled in subsequent patches. This fix means some missing dependencies, such as: <image>:do_fetchall -> <image>:do_rootfs -> <pkgs>:do_package_write_X -> <ca-certs>:do_package_write_X -> debianutils-native (via PAKAGE_WRITE_DEPS) are now found/added. [YOCTO #12510] (Bitbake rev: aec2f07d56a19b97b6515897532b113cdead8338) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/runqueue.py92
1 files changed, 51 insertions, 41 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index b7be102a8a..2543d4ebcc 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -681,49 +681,59 @@ class RunQueueData:
681 # e.g. do_sometask[recrdeptask] = "do_someothertask" 681 # e.g. do_sometask[recrdeptask] = "do_someothertask"
682 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) 682 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
683 # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed 683 # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
684 self.init_progress_reporter.next_stage(len(recursivetasks)) 684 self.init_progress_reporter.next_stage()
685 extradeps = {} 685 extradeps = True
686 for taskcounter, tid in enumerate(recursivetasks): 686 # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to
687 extradeps[tid] = set(self.runtaskentries[tid].depends) 687 # resolve these recursively until we aren't adding any further extra dependencies
688 688 while extradeps:
689 tasknames = recursivetasks[tid] 689 extradeps = {}
690 seendeps = set() 690
691 691 for taskcounter, tid in enumerate(recursivetasks):
692 def generate_recdeps(t): 692 extradeps[tid] = set(self.runtaskentries[tid].depends)
693 newdeps = set() 693
694 (mc, fn, taskname, _) = split_tid_mcfn(t) 694 tasknames = recursivetasks[tid]
695 add_resolved_dependencies(mc, fn, tasknames, newdeps) 695 seendeps = set()
696 extradeps[tid].update(newdeps) 696
697 seendeps.add(t) 697 def generate_recdeps(t):
698 newdeps.add(t) 698 newdeps = set()
699 for i in newdeps: 699 (mc, fn, taskname, _) = split_tid_mcfn(t)
700 if i not in self.runtaskentries: 700 add_resolved_dependencies(mc, fn, tasknames, newdeps)
701 # Not all recipes might have the recrdeptask task as a task 701 extradeps[tid].update(newdeps)
702 continue 702 seendeps.add(t)
703 task = self.runtaskentries[i].task 703 newdeps.add(t)
704 for n in self.runtaskentries[i].depends: 704 for i in newdeps:
705 if n not in seendeps: 705 if i not in self.runtaskentries:
706 generate_recdeps(n) 706 # Not all recipes might have the recrdeptask task as a task
707 generate_recdeps(tid) 707 continue
708 708 task = self.runtaskentries[i].task
709 if tid in recursiveitasks: 709 for n in self.runtaskentries[i].depends:
710 for dep in recursiveitasks[tid]: 710 if n not in seendeps:
711 generate_recdeps(dep) 711 generate_recdeps(n)
712 self.init_progress_reporter.update(taskcounter) 712 generate_recdeps(tid)
713 713
714 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work 714 if tid in recursiveitasks:
715 for tid in recursivetasks: 715 for dep in recursiveitasks[tid]:
716 extradeps[tid].difference_update(recursivetasksselfref) 716 generate_recdeps(dep)
717 717
718 for tid in self.runtaskentries: 718 for tid in self.runtaskentries:
719 task = self.runtaskentries[tid].task 719 task = self.runtaskentries[tid].task
720 # Add in extra dependencies 720 # Add in extra dependencies
721 if tid in extradeps: 721 if tid in extradeps:
722 self.runtaskentries[tid].depends = extradeps[tid] 722 extradeps[tid].difference_update(self.runtaskentries[tid].depends)
723 # Remove all self references 723 self.runtaskentries[tid].depends.update(extradeps[tid])
724 if tid in self.runtaskentries[tid].depends: 724 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
725 logger.debug(2, "Task %s contains self reference!", tid) 725 self.runtaskentries[tid].depends.difference_update(recursivetasksselfref)
726 self.runtaskentries[tid].depends.remove(tid) 726 extradeps[tid].difference_update(recursivetasksselfref)
727 if not len(extradeps[tid]):
728 del extradeps[tid]
729 if tid not in recursivetasks:
730 bb.warn(tid)
731 # Remove all self references
732 if tid in self.runtaskentries[tid].depends:
733 logger.debug(2, "Task %s contains self reference!", tid)
734 self.runtaskentries[tid].depends.remove(tid)
735
736 bb.debug(1, "Added %s recursive dependencies in this loop" % len(extradeps))
727 737
728 self.init_progress_reporter.next_stage() 738 self.init_progress_reporter.next_stage()
729 739