diff options
Diffstat (limited to 'bitbake/lib/bb/cache.py')
-rw-r--r-- | bitbake/lib/bb/cache.py | 99 |
1 files changed, 50 insertions, 49 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 1f180012e0..da4546640a 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
@@ -28,7 +28,7 @@ | |||
28 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | 28 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
29 | 29 | ||
30 | 30 | ||
31 | import os, re | 31 | import os |
32 | import bb.data | 32 | import bb.data |
33 | import bb.utils | 33 | import bb.utils |
34 | 34 | ||
@@ -44,10 +44,10 @@ class Cache: | |||
44 | """ | 44 | """ |
45 | BitBake Cache implementation | 45 | BitBake Cache implementation |
46 | """ | 46 | """ |
47 | def __init__(self, cooker): | 47 | def __init__(self, data): |
48 | 48 | ||
49 | 49 | ||
50 | self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True) | 50 | self.cachedir = bb.data.getVar("CACHE", data, True) |
51 | self.clean = {} | 51 | self.clean = {} |
52 | self.checked = {} | 52 | self.checked = {} |
53 | self.depends_cache = {} | 53 | self.depends_cache = {} |
@@ -61,30 +61,28 @@ class Cache: | |||
61 | return | 61 | return |
62 | 62 | ||
63 | self.has_cache = True | 63 | self.has_cache = True |
64 | self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") | 64 | self.cachefile = os.path.join(self.cachedir, "bb_cache.dat") |
65 | 65 | ||
66 | bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) | 66 | bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) |
67 | try: | 67 | bb.utils.mkdirhier(self.cachedir) |
68 | os.stat( self.cachedir ) | ||
69 | except OSError: | ||
70 | bb.mkdirhier( self.cachedir ) | ||
71 | 68 | ||
72 | # If any of configuration.data's dependencies are newer than the | 69 | # If any of configuration.data's dependencies are newer than the |
73 | # cache there isn't even any point in loading it... | 70 | # cache there isn't even any point in loading it... |
74 | newest_mtime = 0 | 71 | newest_mtime = 0 |
75 | deps = bb.data.getVar("__depends", cooker.configuration.data, True) | 72 | deps = bb.data.getVar("__depends", data) |
76 | for f,old_mtime in deps: | 73 | |
77 | if old_mtime > newest_mtime: | 74 | old_mtimes = [old_mtime for f, old_mtime in deps] |
78 | newest_mtime = old_mtime | 75 | old_mtimes.append(newest_mtime) |
76 | newest_mtime = max(old_mtimes) | ||
79 | 77 | ||
80 | if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: | 78 | if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: |
81 | try: | 79 | try: |
82 | p = pickle.Unpickler(file(self.cachefile, "rb")) | 80 | p = pickle.Unpickler(file(self.cachefile, "rb")) |
83 | self.depends_cache, version_data = p.load() | 81 | self.depends_cache, version_data = p.load() |
84 | if version_data['CACHE_VER'] != __cache_version__: | 82 | if version_data['CACHE_VER'] != __cache_version__: |
85 | raise ValueError, 'Cache Version Mismatch' | 83 | raise ValueError('Cache Version Mismatch') |
86 | if version_data['BITBAKE_VER'] != bb.__version__: | 84 | if version_data['BITBAKE_VER'] != bb.__version__: |
87 | raise ValueError, 'Bitbake Version Mismatch' | 85 | raise ValueError('Bitbake Version Mismatch') |
88 | except EOFError: | 86 | except EOFError: |
89 | bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...") | 87 | bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...") |
90 | self.depends_cache = {} | 88 | self.depends_cache = {} |
@@ -92,27 +90,23 @@ class Cache: | |||
92 | bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") | 90 | bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") |
93 | self.depends_cache = {} | 91 | self.depends_cache = {} |
94 | else: | 92 | else: |
95 | try: | 93 | if os.path.isfile(self.cachefile): |
96 | os.stat( self.cachefile ) | ||
97 | bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...") | 94 | bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...") |
98 | except OSError: | ||
99 | pass | ||
100 | 95 | ||
101 | def getVar(self, var, fn, exp = 0): | 96 | def getVar(self, var, fn, exp = 0): |
102 | """ | 97 | """ |
103 | Gets the value of a variable | 98 | Gets the value of a variable |
104 | (similar to getVar in the data class) | 99 | (similar to getVar in the data class) |
105 | 100 | ||
106 | There are two scenarios: | 101 | There are two scenarios: |
107 | 1. We have cached data - serve from depends_cache[fn] | 102 | 1. We have cached data - serve from depends_cache[fn] |
108 | 2. We're learning what data to cache - serve from data | 103 | 2. We're learning what data to cache - serve from data |
109 | backend but add a copy of the data to the cache. | 104 | backend but add a copy of the data to the cache. |
110 | """ | 105 | """ |
111 | if fn in self.clean: | 106 | if fn in self.clean: |
112 | return self.depends_cache[fn][var] | 107 | return self.depends_cache[fn][var] |
113 | 108 | ||
114 | if not fn in self.depends_cache: | 109 | self.depends_cache.setdefault(fn, {}) |
115 | self.depends_cache[fn] = {} | ||
116 | 110 | ||
117 | if fn != self.data_fn: | 111 | if fn != self.data_fn: |
118 | # We're trying to access data in the cache which doesn't exist | 112 | # We're trying to access data in the cache which doesn't exist |
@@ -134,14 +128,14 @@ class Cache: | |||
134 | self.data = data | 128 | self.data = data |
135 | 129 | ||
136 | # Make sure __depends makes the depends_cache | 130 | # Make sure __depends makes the depends_cache |
137 | # If we're a virtual class we need to make sure all our depends are appended | 131 | # If we're a virtual class we need to make sure all our depends are appended |
138 | # to the depends of fn. | 132 | # to the depends of fn. |
139 | depends = self.getVar("__depends", virtualfn, True) or [] | 133 | depends = self.getVar("__depends", virtualfn) or set() |
134 | self.depends_cache.setdefault(fn, {}) | ||
140 | if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]: | 135 | if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]: |
141 | self.depends_cache[fn]["__depends"] = depends | 136 | self.depends_cache[fn]["__depends"] = depends |
142 | for dep in depends: | 137 | else: |
143 | if dep not in self.depends_cache[fn]["__depends"]: | 138 | self.depends_cache[fn]["__depends"].update(depends) |
144 | self.depends_cache[fn]["__depends"].append(dep) | ||
145 | 139 | ||
146 | # Make sure the variants always make it into the cache too | 140 | # Make sure the variants always make it into the cache too |
147 | self.getVar('__VARIANTS', virtualfn, True) | 141 | self.getVar('__VARIANTS', virtualfn, True) |
@@ -217,7 +211,7 @@ class Cache: | |||
217 | for data in bb_data: | 211 | for data in bb_data: |
218 | virtualfn = self.realfn2virtual(fn, data) | 212 | virtualfn = self.realfn2virtual(fn, data) |
219 | self.setData(virtualfn, fn, bb_data[data]) | 213 | self.setData(virtualfn, fn, bb_data[data]) |
220 | if self.getVar("__SKIPPED", virtualfn, True): | 214 | if self.getVar("__SKIPPED", virtualfn): |
221 | skipped += 1 | 215 | skipped += 1 |
222 | bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) | 216 | bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) |
223 | else: | 217 | else: |
@@ -258,11 +252,11 @@ class Cache: | |||
258 | self.remove(fn) | 252 | self.remove(fn) |
259 | return False | 253 | return False |
260 | 254 | ||
261 | mtime = bb.parse.cached_mtime_noerror(fn) | 255 | mtime = bb.parse.cached_mtime_noerror(fn) |
262 | 256 | ||
263 | # Check file still exists | 257 | # Check file still exists |
264 | if mtime == 0: | 258 | if mtime == 0: |
265 | bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn) | 259 | bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s no longer exists" % fn) |
266 | self.remove(fn) | 260 | self.remove(fn) |
267 | return False | 261 | return False |
268 | 262 | ||
@@ -275,7 +269,7 @@ class Cache: | |||
275 | # Check dependencies are still valid | 269 | # Check dependencies are still valid |
276 | depends = self.getVar("__depends", fn, True) | 270 | depends = self.getVar("__depends", fn, True) |
277 | if depends: | 271 | if depends: |
278 | for f,old_mtime in depends: | 272 | for f, old_mtime in depends: |
279 | fmtime = bb.parse.cached_mtime_noerror(f) | 273 | fmtime = bb.parse.cached_mtime_noerror(f) |
280 | # Check if file still exists | 274 | # Check if file still exists |
281 | if old_mtime != 0 and fmtime == 0: | 275 | if old_mtime != 0 and fmtime == 0: |
@@ -345,14 +339,14 @@ class Cache: | |||
345 | 339 | ||
346 | def handle_data(self, file_name, cacheData): | 340 | def handle_data(self, file_name, cacheData): |
347 | """ | 341 | """ |
348 | Save data we need into the cache | 342 | Save data we need into the cache |
349 | """ | 343 | """ |
350 | 344 | ||
351 | pn = self.getVar('PN', file_name, True) | 345 | pn = self.getVar('PN', file_name, True) |
352 | pe = self.getVar('PE', file_name, True) or "0" | 346 | pe = self.getVar('PE', file_name, True) or "0" |
353 | pv = self.getVar('PV', file_name, True) | 347 | pv = self.getVar('PV', file_name, True) |
354 | if 'SRCREVINACTION' in pv: | 348 | if 'SRCREVINACTION' in pv: |
355 | bb.note("Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name)) | 349 | bb.msg.note(1, bb.msg.domain.Cache, "Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name)) |
356 | pr = self.getVar('PR', file_name, True) | 350 | pr = self.getVar('PR', file_name, True) |
357 | dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") | 351 | dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") |
358 | depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") | 352 | depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") |
@@ -360,7 +354,7 @@ class Cache: | |||
360 | packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() | 354 | packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() |
361 | rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split() | 355 | rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split() |
362 | 356 | ||
363 | cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True) | 357 | cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name) |
364 | 358 | ||
365 | # build PackageName to FileName lookup table | 359 | # build PackageName to FileName lookup table |
366 | if pn not in cacheData.pkg_pn: | 360 | if pn not in cacheData.pkg_pn: |
@@ -371,7 +365,7 @@ class Cache: | |||
371 | 365 | ||
372 | # build FileName to PackageName lookup table | 366 | # build FileName to PackageName lookup table |
373 | cacheData.pkg_fn[file_name] = pn | 367 | cacheData.pkg_fn[file_name] = pn |
374 | cacheData.pkg_pepvpr[file_name] = (pe,pv,pr) | 368 | cacheData.pkg_pepvpr[file_name] = (pe, pv, pr) |
375 | cacheData.pkg_dp[file_name] = dp | 369 | cacheData.pkg_dp[file_name] = dp |
376 | 370 | ||
377 | provides = [pn] | 371 | provides = [pn] |
@@ -400,13 +394,13 @@ class Cache: | |||
400 | if not dep in cacheData.all_depends: | 394 | if not dep in cacheData.all_depends: |
401 | cacheData.all_depends.append(dep) | 395 | cacheData.all_depends.append(dep) |
402 | 396 | ||
403 | # Build reverse hash for PACKAGES, so runtime dependencies | 397 | # Build reverse hash for PACKAGES, so runtime dependencies |
404 | # can be be resolved (RDEPENDS, RRECOMMENDS etc.) | 398 | # can be be resolved (RDEPENDS, RRECOMMENDS etc.) |
405 | for package in packages: | 399 | for package in packages: |
406 | if not package in cacheData.packages: | 400 | if not package in cacheData.packages: |
407 | cacheData.packages[package] = [] | 401 | cacheData.packages[package] = [] |
408 | cacheData.packages[package].append(file_name) | 402 | cacheData.packages[package].append(file_name) |
409 | rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() | 403 | rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() |
410 | 404 | ||
411 | for package in packages_dynamic: | 405 | for package in packages_dynamic: |
412 | if not package in cacheData.packages_dynamic: | 406 | if not package in cacheData.packages_dynamic: |
@@ -445,38 +439,45 @@ class Cache: | |||
445 | self.getVar('__BB_DONT_CACHE', file_name, True) | 439 | self.getVar('__BB_DONT_CACHE', file_name, True) |
446 | self.getVar('__VARIANTS', file_name, True) | 440 | self.getVar('__VARIANTS', file_name, True) |
447 | 441 | ||
448 | def load_bbfile( self, bbfile , config): | 442 | def load_bbfile( self, bbfile, config): |
449 | """ | 443 | """ |
450 | Load and parse one .bb build file | 444 | Load and parse one .bb build file |
451 | Return the data and whether parsing resulted in the file being skipped | 445 | Return the data and whether parsing resulted in the file being skipped |
452 | """ | 446 | """ |
447 | chdir_back = False | ||
453 | 448 | ||
454 | import bb | 449 | from bb import data, parse |
455 | from bb import utils, data, parse, debug, event, fatal | ||
456 | 450 | ||
457 | # expand tmpdir to include this topdir | 451 | # expand tmpdir to include this topdir |
458 | data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) | 452 | data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) |
459 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) | 453 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) |
460 | oldpath = os.path.abspath(os.getcwd()) | 454 | oldpath = os.path.abspath(os.getcwd()) |
461 | if bb.parse.cached_mtime_noerror(bbfile_loc): | 455 | parse.cached_mtime_noerror(bbfile_loc) |
462 | os.chdir(bbfile_loc) | ||
463 | bb_data = data.init_db(config) | 456 | bb_data = data.init_db(config) |
457 | # The ConfHandler first looks if there is a TOPDIR and if not | ||
458 | # then it would call getcwd(). | ||
459 | # Previously, we chdir()ed to bbfile_loc, called the handler | ||
460 | # and finally chdir()ed back, a couple of thousand times. We now | ||
461 | # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. | ||
462 | if not data.getVar('TOPDIR', bb_data): | ||
463 | chdir_back = True | ||
464 | data.setVar('TOPDIR', bbfile_loc, bb_data) | ||
464 | try: | 465 | try: |
465 | bb_data = parse.handle(bbfile, bb_data) # read .bb data | 466 | bb_data = parse.handle(bbfile, bb_data) # read .bb data |
466 | os.chdir(oldpath) | 467 | if chdir_back: os.chdir(oldpath) |
467 | return bb_data | 468 | return bb_data |
468 | except: | 469 | except: |
469 | os.chdir(oldpath) | 470 | if chdir_back: os.chdir(oldpath) |
470 | raise | 471 | raise |
471 | 472 | ||
472 | def init(cooker): | 473 | def init(cooker): |
473 | """ | 474 | """ |
474 | The Objective: Cache the minimum amount of data possible yet get to the | 475 | The Objective: Cache the minimum amount of data possible yet get to the |
475 | stage of building packages (i.e. tryBuild) without reparsing any .bb files. | 476 | stage of building packages (i.e. tryBuild) without reparsing any .bb files. |
476 | 477 | ||
477 | To do this, we intercept getVar calls and only cache the variables we see | 478 | To do this, we intercept getVar calls and only cache the variables we see |
478 | being accessed. We rely on the cache getVar calls being made for all | 479 | being accessed. We rely on the cache getVar calls being made for all |
479 | variables bitbake might need to use to reach this stage. For each cached | 480 | variables bitbake might need to use to reach this stage. For each cached |
480 | file we need to track: | 481 | file we need to track: |
481 | 482 | ||
482 | * Its mtime | 483 | * Its mtime |
@@ -486,7 +487,7 @@ def init(cooker): | |||
486 | Files causing parsing errors are evicted from the cache. | 487 | Files causing parsing errors are evicted from the cache. |
487 | 488 | ||
488 | """ | 489 | """ |
489 | return Cache(cooker) | 490 | return Cache(cooker.configuration.data) |
490 | 491 | ||
491 | 492 | ||
492 | 493 | ||