summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorLiping Ke <liping.ke@intel.com>2011-06-03 08:17:23 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-06-07 22:38:32 +0100
commit8df355140c5afb683cb97fed82f679366e60320b (patch)
tree1b1107c50de4e7f20990bb8ea959415293e7712e /bitbake
parent911e8bb56abac938c6a6bf042844fdaa126970a3 (diff)
downloadpoky-8df355140c5afb683cb97fed82f679366e60320b.tar.gz
cache.py: Refactory Current Cache implementation
This patch is for refactorying current cache implementation, the main reason is for introducing extra cache fields requests for image creator as well as other users. The refactory parts include: Move cache data retrieve methods out of Cache Data Fields Definition. Since this retrieve methods will be shared for both CoreRecipeInfo as well as the new introduced extra RecipeInfo in the following patches. (Bitbake rev: f0f53506926a3f79181796dde177f11f0a396b75) Signed-off-by: Liping Ke <liping.ke@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/cache.py358
1 files changed, 165 insertions, 193 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index d083c515ed..09691d98cc 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -30,7 +30,7 @@
30 30
31import os 31import os
32import logging 32import logging
33from collections import defaultdict, namedtuple 33from collections import defaultdict
34import bb.data 34import bb.data
35import bb.utils 35import bb.utils
36 36
@@ -45,46 +45,11 @@ except ImportError:
45 45
46__cache_version__ = "138" 46__cache_version__ = "138"
47 47
48recipe_fields = ( 48
49 'pn', 49# RecipeInfoCommon defines common data retrieving methods
50 'pv', 50# from meta data for caches. CoreRecipeInfo as well as other
51 'pr', 51# Extra RecipeInfo needs to inherit this class
52 'pe', 52class RecipeInfoCommon(object):
53 'defaultpref',
54 'depends',
55 'provides',
56 'task_deps',
57 'stamp',
58 'stamp_extrainfo',
59 'broken',
60 'not_world',
61 'skipped',
62 'timestamp',
63 'packages',
64 'packages_dynamic',
65 'rdepends',
66 'rdepends_pkg',
67 'rprovides',
68 'rprovides_pkg',
69 'rrecommends',
70 'rrecommends_pkg',
71 'nocache',
72 'variants',
73 'file_depends',
74 'tasks',
75 'basetaskhashes',
76 'hashfilename',
77 'inherits',
78 'summary',
79 'license',
80 'section',
81 'fakerootenv',
82 'fakerootdirs'
83)
84
85
86class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
87 __slots__ = ()
88 53
89 @classmethod 54 @classmethod
90 def listvar(cls, var, metadata): 55 def listvar(cls, var, metadata):
@@ -117,69 +82,162 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
117 def getvar(cls, var, metadata): 82 def getvar(cls, var, metadata):
118 return metadata.getVar(var, True) or '' 83 return metadata.getVar(var, True) or ''
119 84
120 @classmethod 85
121 def make_optional(cls, default=None, **kwargs): 86class CoreRecipeInfo(RecipeInfoCommon):
122 """Construct the namedtuple from the specified keyword arguments, 87 __slots__ = ()
123 with every value considered optional, using the default value if 88
124 it was not specified.""" 89 def __init__(self, filename, metadata):
125 for field in cls._fields: 90 self.name = "base"
126 kwargs[field] = kwargs.get(field, default) 91 # please override this member with the correct data cache file
127 return cls(**kwargs) 92 # such as (bb_cache.dat, bb_extracache_hob.dat)
93 self.cachefile = "bb_cache.dat"
94
95 self.file_depends = metadata.getVar('__depends', False)
96 self.timestamp = bb.parse.cached_mtime(filename)
97 self.variants = self.listvar('__VARIANTS', metadata) + ['']
98 self.nocache = self.getvar('__BB_DONT_CACHE', metadata)
99
100 if self.getvar('__SKIPPED', metadata):
101 self.skipped = True
102 return
103
104 self.tasks = metadata.getVar('__BBTASKS', False)
105
106 self.pn = self.getvar('PN', metadata)
107 self.packages = self.listvar('PACKAGES', metadata)
108 if not self.pn in self.packages:
109 self.packages.append(self.pn)
110
111 self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
112 self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
113
114 self.file_depends = metadata.getVar('__depends', False)
115 self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
116
117 self.skipped = False
118 self.pe = self.getvar('PE', metadata)
119 self.pv = self.getvar('PV', metadata)
120 self.pr = self.getvar('PR', metadata)
121 self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata)
122 self.broken = self.getvar('BROKEN', metadata)
123 self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata)
124 self.stamp = self.getvar('STAMP', metadata)
125 self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
126 self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
127 self.depends = self.depvar('DEPENDS', metadata)
128 self.provides = self.depvar('PROVIDES', metadata)
129 self.rdepends = self.depvar('RDEPENDS', metadata)
130 self.rprovides = self.depvar('RPROVIDES', metadata)
131 self.rrecommends = self.depvar('RRECOMMENDS', metadata)
132 self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
133 self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata)
134 self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata)
135 self.inherits = self.getvar('__inherit_cache', metadata)
136 self.summary = self.getvar('SUMMARY', metadata)
137 self.license = self.getvar('LICENSE', metadata)
138 self.section = self.getvar('SECTION', metadata)
139 self.fakerootenv = self.getvar('FAKEROOTENV', metadata)
140 self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata)
128 141
129 @classmethod 142 @classmethod
130 def from_metadata(cls, filename, metadata): 143 def init_cacheData(cls, cachedata):
131 if cls.getvar('__SKIPPED', metadata): 144 # CacheData in Core RecipeInfo Class
132 return cls.make_optional(skipped=True, 145 cachedata.task_deps = {}
133 file_depends=metadata.getVar('__depends', False), 146 cachedata.pkg_fn = {}
134 timestamp=bb.parse.cached_mtime(filename), 147 cachedata.pkg_pn = defaultdict(list)
135 variants=cls.listvar('__VARIANTS', metadata) + ['']) 148 cachedata.pkg_pepvpr = {}
136 149 cachedata.pkg_dp = {}
137 tasks = metadata.getVar('__BBTASKS', False) 150
138 151 cachedata.stamp = {}
139 pn = cls.getvar('PN', metadata) 152 cachedata.stamp_extrainfo = {}
140 packages = cls.listvar('PACKAGES', metadata) 153 cachedata.fn_provides = {}
141 if not pn in packages: 154 cachedata.pn_provides = defaultdict(list)
142 packages.append(pn) 155 cachedata.all_depends = []
143 156
144 return RecipeInfo( 157 cachedata.deps = defaultdict(list)
145 tasks = tasks, 158 cachedata.packages = defaultdict(list)
146 basetaskhashes = cls.taskvar('BB_BASEHASH', tasks, metadata), 159 cachedata.providers = defaultdict(list)
147 hashfilename = cls.getvar('BB_HASHFILENAME', metadata), 160 cachedata.rproviders = defaultdict(list)
148 161 cachedata.packages_dynamic = defaultdict(list)
149 file_depends = metadata.getVar('__depends', False), 162
150 task_deps = metadata.getVar('_task_deps', False) or 163 cachedata.rundeps = defaultdict(lambda: defaultdict(list))
151 {'tasks': [], 'parents': {}}, 164 cachedata.runrecs = defaultdict(lambda: defaultdict(list))
152 variants = cls.listvar('__VARIANTS', metadata) + [''], 165 cachedata.possible_world = []
153 166 cachedata.universe_target = []
154 skipped = False, 167 cachedata.hashfn = {}
155 timestamp = bb.parse.cached_mtime(filename), 168
156 packages = cls.listvar('PACKAGES', metadata), 169 cachedata.basetaskhash = {}
157 pn = pn, 170 cachedata.inherits = {}
158 pe = cls.getvar('PE', metadata), 171 cachedata.summary = {}
159 pv = cls.getvar('PV', metadata), 172 cachedata.license = {}
160 pr = cls.getvar('PR', metadata), 173 cachedata.section = {}
161 nocache = cls.getvar('__BB_DONT_CACHE', metadata), 174 cachedata.fakerootenv = {}
162 defaultpref = cls.intvar('DEFAULT_PREFERENCE', metadata), 175 cachedata.fakerootdirs = {}
163 broken = cls.getvar('BROKEN', metadata), 176
164 not_world = cls.getvar('EXCLUDE_FROM_WORLD', metadata), 177 def add_cacheData(self, cachedata, fn):
165 stamp = cls.getvar('STAMP', metadata), 178 cachedata.task_deps[fn] = self.task_deps
166 stamp_extrainfo = cls.flaglist('stamp-extra-info', tasks, metadata), 179 cachedata.pkg_fn[fn] = self.pn
167 packages_dynamic = cls.listvar('PACKAGES_DYNAMIC', metadata), 180 cachedata.pkg_pn[self].append(fn)
168 depends = cls.depvar('DEPENDS', metadata), 181 cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr)
169 provides = cls.depvar('PROVIDES', metadata), 182 cachedata.pkg_dp[fn] = self.defaultpref
170 rdepends = cls.depvar('RDEPENDS', metadata), 183 cachedata.stamp[fn] = self.stamp
171 rprovides = cls.depvar('RPROVIDES', metadata), 184 cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo
172 rrecommends = cls.depvar('RRECOMMENDS', metadata), 185
173 rprovides_pkg = cls.pkgvar('RPROVIDES', packages, metadata), 186 provides = [self.pn]
174 rdepends_pkg = cls.pkgvar('RDEPENDS', packages, metadata), 187 for provide in self.provides:
175 rrecommends_pkg = cls.pkgvar('RRECOMMENDS', packages, metadata), 188 if provide not in provides:
176 inherits = cls.getvar('__inherit_cache', metadata), 189 provides.append(provide)
177 summary = cls.getvar('SUMMARY', metadata), 190 cachedata.fn_provides[fn] = provides
178 license = cls.getvar('LICENSE', metadata), 191
179 section = cls.getvar('SECTION', metadata), 192 for provide in provides:
180 fakerootenv = cls.getvar('FAKEROOTENV', metadata), 193 cachedata.providers[provide].append(fn)
181 fakerootdirs = cls.getvar('FAKEROOTDIRS', metadata), 194 if provide not in cachedata.pn_provides[self.pn]:
182 ) 195 cachedata.pn_provides[self.pn].append(provide)
196
197 for dep in self.depends:
198 if dep not in cachedata.deps[fn]:
199 cachedata.deps[fn].append(dep)
200 if dep not in cachedata.all_depends:
201 cachedata.all_depends.append(dep)
202
203 rprovides = self.rprovides
204 for package in self.packages:
205 cachedata.packages[package].append(fn)
206 rprovides += self.rprovides_pkg[package]
207
208 for rprovide in rprovides:
209 cachedata.rproviders[rprovide].append(fn)
210
211 for package in self.packages_dynamic:
212 cachedata.packages_dynamic[package].append(fn)
213
214 # Build hash of runtime depends and rececommends
215 for package in self.packages + [self.pn]:
216 cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package]
217 cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package]
218
219 # Collect files we may need for possible world-dep
220 # calculations
221 if not self.broken and not self.not_world:
222 cachedata.possible_world.append(fn)
223
224 # create a collection of all targets for sanity checking
225 # tasks, such as upstream versions, license, and tools for
226 # task and image creation.
227 cachedata.universe_target.append(self.pn)
228
229 cachedata.hashfn[fn] = self.hashfilename
230 for task, taskhash in self.basetaskhashes.iteritems():
231 identifier = '%s.%s' % (fn, task)
232 cachedata.basetaskhash[identifier] = taskhash
233
234 cachedata.inherits[fn] = self.inherits
235 cachedata.summary[fn] = self.summary
236 cachedata.license[fn] = self.license
237 cachedata.section[fn] = self.section
238 cachedata.fakerootenv[fn] = self.fakerootenv
239 cachedata.fakerootdirs[fn] = self.fakerootdirs
240
183 241
184 242
185class Cache(object): 243class Cache(object):
@@ -314,7 +372,7 @@ class Cache(object):
314 depends |= (data.getVar("__depends", False) or set()) 372 depends |= (data.getVar("__depends", False) or set())
315 if depends and not variant: 373 if depends and not variant:
316 data.setVar("__depends", depends) 374 data.setVar("__depends", depends)
317 info = RecipeInfo.from_metadata(filename, data) 375 info = CoreRecipeInfo(filename, data)
318 infos.append((virtualfn, info)) 376 infos.append((virtualfn, info))
319 return infos 377 return infos
320 378
@@ -500,7 +558,7 @@ class Cache(object):
500 """ 558 """
501 559
502 realfn = self.virtualfn2realfn(file_name)[0] 560 realfn = self.virtualfn2realfn(file_name)[0]
503 info = RecipeInfo.from_metadata(realfn, data) 561 info = CoreRecipeInfo(realfn, data)
504 self.add_info(file_name, info, cacheData, parsed) 562 self.add_info(file_name, info, cacheData, parsed)
505 563
506 @staticmethod 564 @staticmethod
@@ -566,38 +624,11 @@ class CacheData(object):
566 """ 624 """
567 625
568 def __init__(self): 626 def __init__(self):
627 CoreRecipeInfo.init_cacheData(self)
569 # Direct cache variables 628 # Direct cache variables
570 self.providers = defaultdict(list)
571 self.rproviders = defaultdict(list)
572 self.packages = defaultdict(list)
573 self.packages_dynamic = defaultdict(list)
574 self.possible_world = []
575 self.universe_target = []
576 self.pkg_pn = defaultdict(list)
577 self.pkg_fn = {}
578 self.pkg_pepvpr = {}
579 self.pkg_dp = {}
580 self.pn_provides = defaultdict(list)
581 self.fn_provides = {}
582 self.all_depends = []
583 self.deps = defaultdict(list)
584 self.rundeps = defaultdict(lambda: defaultdict(list))
585 self.runrecs = defaultdict(lambda: defaultdict(list))
586 self.task_queues = {} 629 self.task_queues = {}
587 self.task_deps = {}
588 self.stamp = {}
589 self.stamp_extrainfo = {}
590 self.preferred = {} 630 self.preferred = {}
591 self.tasks = {} 631 self.tasks = {}
592 self.basetaskhash = {}
593 self.hashfn = {}
594 self.inherits = {}
595 self.summary = {}
596 self.license = {}
597 self.section = {}
598 self.fakerootenv = {}
599 self.fakerootdirs = {}
600
601 # Indirect Cache variables (set elsewhere) 632 # Indirect Cache variables (set elsewhere)
602 self.ignored_dependencies = [] 633 self.ignored_dependencies = []
603 self.world_target = set() 634 self.world_target = set()
@@ -605,65 +636,6 @@ class CacheData(object):
605 self.bbfile_config_priorities = [] 636 self.bbfile_config_priorities = []
606 637
607 def add_from_recipeinfo(self, fn, info): 638 def add_from_recipeinfo(self, fn, info):
608 self.task_deps[fn] = info.task_deps 639 info.add_cacheData(self, fn)
609 self.pkg_fn[fn] = info.pn
610 self.pkg_pn[info.pn].append(fn)
611 self.pkg_pepvpr[fn] = (info.pe, info.pv, info.pr)
612 self.pkg_dp[fn] = info.defaultpref
613 self.stamp[fn] = info.stamp
614 self.stamp_extrainfo[fn] = info.stamp_extrainfo
615
616 provides = [info.pn]
617 for provide in info.provides:
618 if provide not in provides:
619 provides.append(provide)
620 self.fn_provides[fn] = provides
621
622 for provide in provides:
623 self.providers[provide].append(fn)
624 if provide not in self.pn_provides[info.pn]:
625 self.pn_provides[info.pn].append(provide)
626
627 for dep in info.depends:
628 if dep not in self.deps[fn]:
629 self.deps[fn].append(dep)
630 if dep not in self.all_depends:
631 self.all_depends.append(dep)
632
633 rprovides = info.rprovides
634 for package in info.packages:
635 self.packages[package].append(fn)
636 rprovides += info.rprovides_pkg[package]
637 640
638 for rprovide in rprovides: 641
639 self.rproviders[rprovide].append(fn)
640
641 for package in info.packages_dynamic:
642 self.packages_dynamic[package].append(fn)
643
644 # Build hash of runtime depends and rececommends
645 for package in info.packages + [info.pn]:
646 self.rundeps[fn][package] = list(info.rdepends) + info.rdepends_pkg[package]
647 self.runrecs[fn][package] = list(info.rrecommends) + info.rrecommends_pkg[package]
648
649 # Collect files we may need for possible world-dep
650 # calculations
651 if not info.broken and not info.not_world:
652 self.possible_world.append(fn)
653
654 # create a collection of all targets for sanity checking
655 # tasks, such as upstream versions, license, and tools for
656 # task and image creation.
657 self.universe_target.append(info.pn)
658
659 self.hashfn[fn] = info.hashfilename
660 for task, taskhash in info.basetaskhashes.iteritems():
661 identifier = '%s.%s' % (fn, task)
662 self.basetaskhash[identifier] = taskhash
663
664 self.inherits[fn] = info.inherits
665 self.summary[fn] = info.summary
666 self.license[fn] = info.license
667 self.section[fn] = info.section
668 self.fakerootenv[fn] = info.fakerootenv
669 self.fakerootdirs[fn] = info.fakerootdirs