diff options
Diffstat (limited to 'bitbake/lib/bb/cache.py')
| -rw-r--r-- | bitbake/lib/bb/cache.py | 99 |
1 files changed, 50 insertions, 49 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 1f180012e0..da4546640a 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
| @@ -28,7 +28,7 @@ | |||
| 28 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | 28 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
| 29 | 29 | ||
| 30 | 30 | ||
| 31 | import os, re | 31 | import os |
| 32 | import bb.data | 32 | import bb.data |
| 33 | import bb.utils | 33 | import bb.utils |
| 34 | 34 | ||
| @@ -44,10 +44,10 @@ class Cache: | |||
| 44 | """ | 44 | """ |
| 45 | BitBake Cache implementation | 45 | BitBake Cache implementation |
| 46 | """ | 46 | """ |
| 47 | def __init__(self, cooker): | 47 | def __init__(self, data): |
| 48 | 48 | ||
| 49 | 49 | ||
| 50 | self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True) | 50 | self.cachedir = bb.data.getVar("CACHE", data, True) |
| 51 | self.clean = {} | 51 | self.clean = {} |
| 52 | self.checked = {} | 52 | self.checked = {} |
| 53 | self.depends_cache = {} | 53 | self.depends_cache = {} |
| @@ -61,30 +61,28 @@ class Cache: | |||
| 61 | return | 61 | return |
| 62 | 62 | ||
| 63 | self.has_cache = True | 63 | self.has_cache = True |
| 64 | self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") | 64 | self.cachefile = os.path.join(self.cachedir, "bb_cache.dat") |
| 65 | 65 | ||
| 66 | bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) | 66 | bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) |
| 67 | try: | 67 | bb.utils.mkdirhier(self.cachedir) |
| 68 | os.stat( self.cachedir ) | ||
| 69 | except OSError: | ||
| 70 | bb.mkdirhier( self.cachedir ) | ||
| 71 | 68 | ||
| 72 | # If any of configuration.data's dependencies are newer than the | 69 | # If any of configuration.data's dependencies are newer than the |
| 73 | # cache there isn't even any point in loading it... | 70 | # cache there isn't even any point in loading it... |
| 74 | newest_mtime = 0 | 71 | newest_mtime = 0 |
| 75 | deps = bb.data.getVar("__depends", cooker.configuration.data, True) | 72 | deps = bb.data.getVar("__depends", data) |
| 76 | for f,old_mtime in deps: | 73 | |
| 77 | if old_mtime > newest_mtime: | 74 | old_mtimes = [old_mtime for f, old_mtime in deps] |
| 78 | newest_mtime = old_mtime | 75 | old_mtimes.append(newest_mtime) |
| 76 | newest_mtime = max(old_mtimes) | ||
| 79 | 77 | ||
| 80 | if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: | 78 | if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: |
| 81 | try: | 79 | try: |
| 82 | p = pickle.Unpickler(file(self.cachefile, "rb")) | 80 | p = pickle.Unpickler(file(self.cachefile, "rb")) |
| 83 | self.depends_cache, version_data = p.load() | 81 | self.depends_cache, version_data = p.load() |
| 84 | if version_data['CACHE_VER'] != __cache_version__: | 82 | if version_data['CACHE_VER'] != __cache_version__: |
| 85 | raise ValueError, 'Cache Version Mismatch' | 83 | raise ValueError('Cache Version Mismatch') |
| 86 | if version_data['BITBAKE_VER'] != bb.__version__: | 84 | if version_data['BITBAKE_VER'] != bb.__version__: |
| 87 | raise ValueError, 'Bitbake Version Mismatch' | 85 | raise ValueError('Bitbake Version Mismatch') |
| 88 | except EOFError: | 86 | except EOFError: |
| 89 | bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...") | 87 | bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...") |
| 90 | self.depends_cache = {} | 88 | self.depends_cache = {} |
| @@ -92,27 +90,23 @@ class Cache: | |||
| 92 | bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") | 90 | bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") |
| 93 | self.depends_cache = {} | 91 | self.depends_cache = {} |
| 94 | else: | 92 | else: |
| 95 | try: | 93 | if os.path.isfile(self.cachefile): |
| 96 | os.stat( self.cachefile ) | ||
| 97 | bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...") | 94 | bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...") |
| 98 | except OSError: | ||
| 99 | pass | ||
| 100 | 95 | ||
| 101 | def getVar(self, var, fn, exp = 0): | 96 | def getVar(self, var, fn, exp = 0): |
| 102 | """ | 97 | """ |
| 103 | Gets the value of a variable | 98 | Gets the value of a variable |
| 104 | (similar to getVar in the data class) | 99 | (similar to getVar in the data class) |
| 105 | 100 | ||
| 106 | There are two scenarios: | 101 | There are two scenarios: |
| 107 | 1. We have cached data - serve from depends_cache[fn] | 102 | 1. We have cached data - serve from depends_cache[fn] |
| 108 | 2. We're learning what data to cache - serve from data | 103 | 2. We're learning what data to cache - serve from data |
| 109 | backend but add a copy of the data to the cache. | 104 | backend but add a copy of the data to the cache. |
| 110 | """ | 105 | """ |
| 111 | if fn in self.clean: | 106 | if fn in self.clean: |
| 112 | return self.depends_cache[fn][var] | 107 | return self.depends_cache[fn][var] |
| 113 | 108 | ||
| 114 | if not fn in self.depends_cache: | 109 | self.depends_cache.setdefault(fn, {}) |
| 115 | self.depends_cache[fn] = {} | ||
| 116 | 110 | ||
| 117 | if fn != self.data_fn: | 111 | if fn != self.data_fn: |
| 118 | # We're trying to access data in the cache which doesn't exist | 112 | # We're trying to access data in the cache which doesn't exist |
| @@ -134,14 +128,14 @@ class Cache: | |||
| 134 | self.data = data | 128 | self.data = data |
| 135 | 129 | ||
| 136 | # Make sure __depends makes the depends_cache | 130 | # Make sure __depends makes the depends_cache |
| 137 | # If we're a virtual class we need to make sure all our depends are appended | 131 | # If we're a virtual class we need to make sure all our depends are appended |
| 138 | # to the depends of fn. | 132 | # to the depends of fn. |
| 139 | depends = self.getVar("__depends", virtualfn, True) or [] | 133 | depends = self.getVar("__depends", virtualfn) or set() |
| 134 | self.depends_cache.setdefault(fn, {}) | ||
| 140 | if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]: | 135 | if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]: |
| 141 | self.depends_cache[fn]["__depends"] = depends | 136 | self.depends_cache[fn]["__depends"] = depends |
| 142 | for dep in depends: | 137 | else: |
| 143 | if dep not in self.depends_cache[fn]["__depends"]: | 138 | self.depends_cache[fn]["__depends"].update(depends) |
| 144 | self.depends_cache[fn]["__depends"].append(dep) | ||
| 145 | 139 | ||
| 146 | # Make sure the variants always make it into the cache too | 140 | # Make sure the variants always make it into the cache too |
| 147 | self.getVar('__VARIANTS', virtualfn, True) | 141 | self.getVar('__VARIANTS', virtualfn, True) |
| @@ -217,7 +211,7 @@ class Cache: | |||
| 217 | for data in bb_data: | 211 | for data in bb_data: |
| 218 | virtualfn = self.realfn2virtual(fn, data) | 212 | virtualfn = self.realfn2virtual(fn, data) |
| 219 | self.setData(virtualfn, fn, bb_data[data]) | 213 | self.setData(virtualfn, fn, bb_data[data]) |
| 220 | if self.getVar("__SKIPPED", virtualfn, True): | 214 | if self.getVar("__SKIPPED", virtualfn): |
| 221 | skipped += 1 | 215 | skipped += 1 |
| 222 | bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) | 216 | bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) |
| 223 | else: | 217 | else: |
| @@ -258,11 +252,11 @@ class Cache: | |||
| 258 | self.remove(fn) | 252 | self.remove(fn) |
| 259 | return False | 253 | return False |
| 260 | 254 | ||
| 261 | mtime = bb.parse.cached_mtime_noerror(fn) | 255 | mtime = bb.parse.cached_mtime_noerror(fn) |
| 262 | 256 | ||
| 263 | # Check file still exists | 257 | # Check file still exists |
| 264 | if mtime == 0: | 258 | if mtime == 0: |
| 265 | bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn) | 259 | bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s no longer exists" % fn) |
| 266 | self.remove(fn) | 260 | self.remove(fn) |
| 267 | return False | 261 | return False |
| 268 | 262 | ||
| @@ -275,7 +269,7 @@ class Cache: | |||
| 275 | # Check dependencies are still valid | 269 | # Check dependencies are still valid |
| 276 | depends = self.getVar("__depends", fn, True) | 270 | depends = self.getVar("__depends", fn, True) |
| 277 | if depends: | 271 | if depends: |
| 278 | for f,old_mtime in depends: | 272 | for f, old_mtime in depends: |
| 279 | fmtime = bb.parse.cached_mtime_noerror(f) | 273 | fmtime = bb.parse.cached_mtime_noerror(f) |
| 280 | # Check if file still exists | 274 | # Check if file still exists |
| 281 | if old_mtime != 0 and fmtime == 0: | 275 | if old_mtime != 0 and fmtime == 0: |
| @@ -345,14 +339,14 @@ class Cache: | |||
| 345 | 339 | ||
| 346 | def handle_data(self, file_name, cacheData): | 340 | def handle_data(self, file_name, cacheData): |
| 347 | """ | 341 | """ |
| 348 | Save data we need into the cache | 342 | Save data we need into the cache |
| 349 | """ | 343 | """ |
| 350 | 344 | ||
| 351 | pn = self.getVar('PN', file_name, True) | 345 | pn = self.getVar('PN', file_name, True) |
| 352 | pe = self.getVar('PE', file_name, True) or "0" | 346 | pe = self.getVar('PE', file_name, True) or "0" |
| 353 | pv = self.getVar('PV', file_name, True) | 347 | pv = self.getVar('PV', file_name, True) |
| 354 | if 'SRCREVINACTION' in pv: | 348 | if 'SRCREVINACTION' in pv: |
| 355 | bb.note("Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name)) | 349 | bb.msg.note(1, bb.msg.domain.Cache, "Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name)) |
| 356 | pr = self.getVar('PR', file_name, True) | 350 | pr = self.getVar('PR', file_name, True) |
| 357 | dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") | 351 | dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") |
| 358 | depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") | 352 | depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") |
| @@ -360,7 +354,7 @@ class Cache: | |||
| 360 | packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() | 354 | packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() |
| 361 | rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split() | 355 | rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split() |
| 362 | 356 | ||
| 363 | cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True) | 357 | cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name) |
| 364 | 358 | ||
| 365 | # build PackageName to FileName lookup table | 359 | # build PackageName to FileName lookup table |
| 366 | if pn not in cacheData.pkg_pn: | 360 | if pn not in cacheData.pkg_pn: |
| @@ -371,7 +365,7 @@ class Cache: | |||
| 371 | 365 | ||
| 372 | # build FileName to PackageName lookup table | 366 | # build FileName to PackageName lookup table |
| 373 | cacheData.pkg_fn[file_name] = pn | 367 | cacheData.pkg_fn[file_name] = pn |
| 374 | cacheData.pkg_pepvpr[file_name] = (pe,pv,pr) | 368 | cacheData.pkg_pepvpr[file_name] = (pe, pv, pr) |
| 375 | cacheData.pkg_dp[file_name] = dp | 369 | cacheData.pkg_dp[file_name] = dp |
| 376 | 370 | ||
| 377 | provides = [pn] | 371 | provides = [pn] |
| @@ -400,13 +394,13 @@ class Cache: | |||
| 400 | if not dep in cacheData.all_depends: | 394 | if not dep in cacheData.all_depends: |
| 401 | cacheData.all_depends.append(dep) | 395 | cacheData.all_depends.append(dep) |
| 402 | 396 | ||
| 403 | # Build reverse hash for PACKAGES, so runtime dependencies | 397 | # Build reverse hash for PACKAGES, so runtime dependencies |
| 404 | # can be be resolved (RDEPENDS, RRECOMMENDS etc.) | 398 | # can be be resolved (RDEPENDS, RRECOMMENDS etc.) |
| 405 | for package in packages: | 399 | for package in packages: |
| 406 | if not package in cacheData.packages: | 400 | if not package in cacheData.packages: |
| 407 | cacheData.packages[package] = [] | 401 | cacheData.packages[package] = [] |
| 408 | cacheData.packages[package].append(file_name) | 402 | cacheData.packages[package].append(file_name) |
| 409 | rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() | 403 | rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() |
| 410 | 404 | ||
| 411 | for package in packages_dynamic: | 405 | for package in packages_dynamic: |
| 412 | if not package in cacheData.packages_dynamic: | 406 | if not package in cacheData.packages_dynamic: |
| @@ -445,38 +439,45 @@ class Cache: | |||
| 445 | self.getVar('__BB_DONT_CACHE', file_name, True) | 439 | self.getVar('__BB_DONT_CACHE', file_name, True) |
| 446 | self.getVar('__VARIANTS', file_name, True) | 440 | self.getVar('__VARIANTS', file_name, True) |
| 447 | 441 | ||
| 448 | def load_bbfile( self, bbfile , config): | 442 | def load_bbfile( self, bbfile, config): |
| 449 | """ | 443 | """ |
| 450 | Load and parse one .bb build file | 444 | Load and parse one .bb build file |
| 451 | Return the data and whether parsing resulted in the file being skipped | 445 | Return the data and whether parsing resulted in the file being skipped |
| 452 | """ | 446 | """ |
| 447 | chdir_back = False | ||
| 453 | 448 | ||
| 454 | import bb | 449 | from bb import data, parse |
| 455 | from bb import utils, data, parse, debug, event, fatal | ||
| 456 | 450 | ||
| 457 | # expand tmpdir to include this topdir | 451 | # expand tmpdir to include this topdir |
| 458 | data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) | 452 | data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) |
| 459 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) | 453 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) |
| 460 | oldpath = os.path.abspath(os.getcwd()) | 454 | oldpath = os.path.abspath(os.getcwd()) |
| 461 | if bb.parse.cached_mtime_noerror(bbfile_loc): | 455 | parse.cached_mtime_noerror(bbfile_loc) |
| 462 | os.chdir(bbfile_loc) | ||
| 463 | bb_data = data.init_db(config) | 456 | bb_data = data.init_db(config) |
| 457 | # The ConfHandler first looks if there is a TOPDIR and if not | ||
| 458 | # then it would call getcwd(). | ||
| 459 | # Previously, we chdir()ed to bbfile_loc, called the handler | ||
| 460 | # and finally chdir()ed back, a couple of thousand times. We now | ||
| 461 | # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. | ||
| 462 | if not data.getVar('TOPDIR', bb_data): | ||
| 463 | chdir_back = True | ||
| 464 | data.setVar('TOPDIR', bbfile_loc, bb_data) | ||
| 464 | try: | 465 | try: |
| 465 | bb_data = parse.handle(bbfile, bb_data) # read .bb data | 466 | bb_data = parse.handle(bbfile, bb_data) # read .bb data |
| 466 | os.chdir(oldpath) | 467 | if chdir_back: os.chdir(oldpath) |
| 467 | return bb_data | 468 | return bb_data |
| 468 | except: | 469 | except: |
| 469 | os.chdir(oldpath) | 470 | if chdir_back: os.chdir(oldpath) |
| 470 | raise | 471 | raise |
| 471 | 472 | ||
| 472 | def init(cooker): | 473 | def init(cooker): |
| 473 | """ | 474 | """ |
| 474 | The Objective: Cache the minimum amount of data possible yet get to the | 475 | The Objective: Cache the minimum amount of data possible yet get to the |
| 475 | stage of building packages (i.e. tryBuild) without reparsing any .bb files. | 476 | stage of building packages (i.e. tryBuild) without reparsing any .bb files. |
| 476 | 477 | ||
| 477 | To do this, we intercept getVar calls and only cache the variables we see | 478 | To do this, we intercept getVar calls and only cache the variables we see |
| 478 | being accessed. We rely on the cache getVar calls being made for all | 479 | being accessed. We rely on the cache getVar calls being made for all |
| 479 | variables bitbake might need to use to reach this stage. For each cached | 480 | variables bitbake might need to use to reach this stage. For each cached |
| 480 | file we need to track: | 481 | file we need to track: |
| 481 | 482 | ||
| 482 | * Its mtime | 483 | * Its mtime |
| @@ -486,7 +487,7 @@ def init(cooker): | |||
| 486 | Files causing parsing errors are evicted from the cache. | 487 | Files causing parsing errors are evicted from the cache. |
| 487 | 488 | ||
| 488 | """ | 489 | """ |
| 489 | return Cache(cooker) | 490 | return Cache(cooker.configuration.data) |
| 490 | 491 | ||
| 491 | 492 | ||
| 492 | 493 | ||
