diff options
author | Chris Larson <chris_larson@mentor.com> | 2010-11-18 20:21:54 -0700 |
---|---|---|
committer | Richard Purdie <rpurdie@linux.intel.com> | 2011-01-04 14:46:42 +0000 |
commit | 32ea7668712a50d8f8b67d5e4558039e5092a485 (patch) | |
tree | 2473f8b1aade6131c7a37fbad2cc4d23998a3a56 /bitbake/lib/bb/cache.py | |
parent | 570bec37a898fb502d166a22f20bdb1da8c21c38 (diff) | |
download | poky-32ea7668712a50d8f8b67d5e4558039e5092a485.tar.gz |
Implement parallel parsing support
This utilizes python's multiprocessing module. The default number of threads
to be used is the same as the number of available processor cores, however,
you can manually set this with the BB_NUMBER_PARSE_THREADS variable.
(Bitbake rev: c7b3ec819549e51e438d293969e205883fee725f)
Signed-off-by: Chris Larson <chris_larson@mentor.com>
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
Diffstat (limited to 'bitbake/lib/bb/cache.py')
-rw-r--r-- | bitbake/lib/bb/cache.py | 98 |
1 files changed, 59 insertions, 39 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 23845bc07b..93dccf21f1 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
@@ -232,48 +232,57 @@ class Cache(object): | |||
232 | bb_data = cls.load_bbfile(fn, appends, cfgData) | 232 | bb_data = cls.load_bbfile(fn, appends, cfgData) |
233 | return bb_data[virtual] | 233 | return bb_data[virtual] |
234 | 234 | ||
235 | def loadData(self, fn, appends, cfgData, cacheData): | 235 | @classmethod |
236 | """ | 236 | def parse(cls, filename, appends, configdata): |
237 | Load a subset of data for fn. | 237 | """Parse the specified filename, returning the recipe information""" |
238 | If the cached data is valid we do nothing, | 238 | infos = [] |
239 | To do this, we need to parse the file and set the system | 239 | datastores = cls.load_bbfile(filename, appends, configdata) |
240 | to record the variables accessed. | 240 | depends = set() |
241 | Return the cache status and whether the file was skipped when parsed | 241 | for variant, data in sorted(datastores.iteritems(), |
242 | """ | 242 | key=lambda i: i[0], |
243 | skipped, virtuals = 0, 0 | 243 | reverse=True): |
244 | virtualfn = cls.realfn2virtual(filename, variant) | ||
245 | depends |= (data.getVar("__depends", False) or set()) | ||
246 | if depends and not variant: | ||
247 | data.setVar("__depends", depends) | ||
248 | info = RecipeInfo.from_metadata(filename, data) | ||
249 | infos.append((virtualfn, info)) | ||
250 | return infos | ||
251 | |||
252 | def load(self, filename, appends, configdata): | ||
253 | """Obtain the recipe information for the specified filename, | ||
254 | using cached values if available, otherwise parsing. | ||
255 | |||
256 | Note that if it does parse to obtain the info, it will not | ||
257 | automatically add the information to the cache or to your | ||
258 | CacheData. Use the add or add_info method to do so after | ||
259 | running this, or use loadData instead.""" | ||
260 | cached = self.cacheValid(filename) | ||
261 | if cached: | ||
262 | infos = [] | ||
263 | info = self.depends_cache[filename] | ||
264 | for variant in info.variants: | ||
265 | virtualfn = self.realfn2virtual(filename, variant) | ||
266 | infos.append((virtualfn, self.depends_cache[virtualfn])) | ||
267 | else: | ||
268 | logger.debug(1, "Parsing %s", filename) | ||
269 | return self.parse(filename, appends, configdata) | ||
244 | 270 | ||
245 | if fn not in self.checked: | 271 | return cached, infos |
246 | self.cacheValidUpdate(fn) | ||
247 | 272 | ||
248 | cached = self.cacheValid(fn) | 273 | def loadData(self, fn, appends, cfgData, cacheData): |
249 | if not cached: | 274 | """Load the recipe info for the specified filename, |
250 | logger.debug(1, "Parsing %s", fn) | 275 | parsing and adding to the cache if necessary, and adding |
251 | datastores = self.load_bbfile(fn, appends, cfgData) | 276 | the recipe information to the supplied CacheData instance.""" |
252 | depends = set() | 277 | skipped, virtuals = 0, 0 |
253 | for variant, data in sorted(datastores.iteritems(), | ||
254 | key=lambda i: i[0], | ||
255 | reverse=True): | ||
256 | virtualfn = self.realfn2virtual(fn, variant) | ||
257 | depends |= (data.getVar("__depends", False) or set()) | ||
258 | if depends and not variant: | ||
259 | data.setVar("__depends", depends) | ||
260 | info = RecipeInfo.from_metadata(fn, data) | ||
261 | if not info.nocache: | ||
262 | # The recipe was parsed, and is not marked as being | ||
263 | # uncacheable, so we need to ensure that we write out the | ||
264 | # new cache data. | ||
265 | self.cacheclean = False | ||
266 | self.depends_cache[virtualfn] = info | ||
267 | 278 | ||
268 | info = self.depends_cache[fn] | 279 | cached, infos = self.load(fn, appends, cfgData) |
269 | for variant in info.variants: | 280 | for virtualfn, info in infos: |
270 | virtualfn = self.realfn2virtual(fn, variant) | 281 | if info.skipped: |
271 | vinfo = self.depends_cache[virtualfn] | ||
272 | if vinfo.skipped: | ||
273 | logger.debug(1, "Skipping %s", virtualfn) | 282 | logger.debug(1, "Skipping %s", virtualfn) |
274 | skipped += 1 | 283 | skipped += 1 |
275 | else: | 284 | else: |
276 | cacheData.add_from_recipeinfo(virtualfn, vinfo) | 285 | self.add_info(virtualfn, info, cacheData, not cached) |
277 | virtuals += 1 | 286 | virtuals += 1 |
278 | 287 | ||
279 | return cached, skipped, virtuals | 288 | return cached, skipped, virtuals |
@@ -283,6 +292,9 @@ class Cache(object): | |||
283 | Is the cache valid for fn? | 292 | Is the cache valid for fn? |
284 | Fast version, no timestamps checked. | 293 | Fast version, no timestamps checked. |
285 | """ | 294 | """ |
295 | if fn not in self.checked: | ||
296 | self.cacheValidUpdate(fn) | ||
297 | |||
286 | # Is cache enabled? | 298 | # Is cache enabled? |
287 | if not self.has_cache: | 299 | if not self.has_cache: |
288 | return False | 300 | return False |
@@ -412,14 +424,22 @@ class Cache(object): | |||
412 | def mtime(cachefile): | 424 | def mtime(cachefile): |
413 | return bb.parse.cached_mtime_noerror(cachefile) | 425 | return bb.parse.cached_mtime_noerror(cachefile) |
414 | 426 | ||
415 | def add(self, file_name, data, cacheData): | 427 | def add_info(self, filename, info, cacheData, parsed=None): |
428 | self.depends_cache[filename] = info | ||
429 | cacheData.add_from_recipeinfo(filename, info) | ||
430 | if parsed and not info.nocache: | ||
431 | # The recipe was parsed, and is not marked as being | ||
432 | # uncacheable, so we need to ensure that we write out the | ||
433 | # new cache data. | ||
434 | self.cacheclean = False | ||
435 | |||
436 | def add(self, file_name, data, cacheData, parsed=None): | ||
416 | """ | 437 | """ |
417 | Save data we need into the cache | 438 | Save data we need into the cache |
418 | """ | 439 | """ |
419 | realfn = self.virtualfn2realfn(file_name)[0] | 440 | realfn = self.virtualfn2realfn(file_name)[0] |
420 | info = RecipeInfo.from_metadata(realfn, data) | 441 | info = RecipeInfo.from_metadata(realfn, data) |
421 | self.depends_cache[file_name] = info | 442 | self.add_info(file_name, info, cacheData, parsed) |
422 | cacheData.add_from_recipeinfo(file_name, info) | ||
423 | 443 | ||
424 | @staticmethod | 444 | @staticmethod |
425 | def load_bbfile(bbfile, appends, config): | 445 | def load_bbfile(bbfile, appends, config): |