summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/cache.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/cache.py')
-rw-r--r--bitbake/lib/bb/cache.py99
1 files changed, 50 insertions, 49 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 1f180012e0..da4546640a 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -28,7 +28,7 @@
28# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 28# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
29 29
30 30
31import os, re 31import os
32import bb.data 32import bb.data
33import bb.utils 33import bb.utils
34 34
@@ -44,10 +44,10 @@ class Cache:
44 """ 44 """
45 BitBake Cache implementation 45 BitBake Cache implementation
46 """ 46 """
47 def __init__(self, cooker): 47 def __init__(self, data):
48 48
49 49
50 self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True) 50 self.cachedir = bb.data.getVar("CACHE", data, True)
51 self.clean = {} 51 self.clean = {}
52 self.checked = {} 52 self.checked = {}
53 self.depends_cache = {} 53 self.depends_cache = {}
@@ -61,30 +61,28 @@ class Cache:
61 return 61 return
62 62
63 self.has_cache = True 63 self.has_cache = True
64 self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") 64 self.cachefile = os.path.join(self.cachedir, "bb_cache.dat")
65 65
66 bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir) 66 bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s'" % self.cachedir)
67 try: 67 bb.utils.mkdirhier(self.cachedir)
68 os.stat( self.cachedir )
69 except OSError:
70 bb.mkdirhier( self.cachedir )
71 68
72 # If any of configuration.data's dependencies are newer than the 69 # If any of configuration.data's dependencies are newer than the
73 # cache there isn't even any point in loading it... 70 # cache there isn't even any point in loading it...
74 newest_mtime = 0 71 newest_mtime = 0
75 deps = bb.data.getVar("__depends", cooker.configuration.data, True) 72 deps = bb.data.getVar("__depends", data)
76 for f,old_mtime in deps: 73
77 if old_mtime > newest_mtime: 74 old_mtimes = [old_mtime for f, old_mtime in deps]
78 newest_mtime = old_mtime 75 old_mtimes.append(newest_mtime)
76 newest_mtime = max(old_mtimes)
79 77
80 if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime: 78 if bb.parse.cached_mtime_noerror(self.cachefile) >= newest_mtime:
81 try: 79 try:
82 p = pickle.Unpickler(file(self.cachefile, "rb")) 80 p = pickle.Unpickler(file(self.cachefile, "rb"))
83 self.depends_cache, version_data = p.load() 81 self.depends_cache, version_data = p.load()
84 if version_data['CACHE_VER'] != __cache_version__: 82 if version_data['CACHE_VER'] != __cache_version__:
85 raise ValueError, 'Cache Version Mismatch' 83 raise ValueError('Cache Version Mismatch')
86 if version_data['BITBAKE_VER'] != bb.__version__: 84 if version_data['BITBAKE_VER'] != bb.__version__:
87 raise ValueError, 'Bitbake Version Mismatch' 85 raise ValueError('Bitbake Version Mismatch')
88 except EOFError: 86 except EOFError:
89 bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...") 87 bb.msg.note(1, bb.msg.domain.Cache, "Truncated cache found, rebuilding...")
90 self.depends_cache = {} 88 self.depends_cache = {}
@@ -92,27 +90,23 @@ class Cache:
92 bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...") 90 bb.msg.note(1, bb.msg.domain.Cache, "Invalid cache found, rebuilding...")
93 self.depends_cache = {} 91 self.depends_cache = {}
94 else: 92 else:
95 try: 93 if os.path.isfile(self.cachefile):
96 os.stat( self.cachefile )
97 bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...") 94 bb.msg.note(1, bb.msg.domain.Cache, "Out of date cache found, rebuilding...")
98 except OSError:
99 pass
100 95
101 def getVar(self, var, fn, exp = 0): 96 def getVar(self, var, fn, exp = 0):
102 """ 97 """
103 Gets the value of a variable 98 Gets the value of a variable
104 (similar to getVar in the data class) 99 (similar to getVar in the data class)
105 100
106 There are two scenarios: 101 There are two scenarios:
107 1. We have cached data - serve from depends_cache[fn] 102 1. We have cached data - serve from depends_cache[fn]
108 2. We're learning what data to cache - serve from data 103 2. We're learning what data to cache - serve from data
109 backend but add a copy of the data to the cache. 104 backend but add a copy of the data to the cache.
110 """ 105 """
111 if fn in self.clean: 106 if fn in self.clean:
112 return self.depends_cache[fn][var] 107 return self.depends_cache[fn][var]
113 108
114 if not fn in self.depends_cache: 109 self.depends_cache.setdefault(fn, {})
115 self.depends_cache[fn] = {}
116 110
117 if fn != self.data_fn: 111 if fn != self.data_fn:
118 # We're trying to access data in the cache which doesn't exist 112 # We're trying to access data in the cache which doesn't exist
@@ -134,14 +128,14 @@ class Cache:
134 self.data = data 128 self.data = data
135 129
136 # Make sure __depends makes the depends_cache 130 # Make sure __depends makes the depends_cache
137 # If we're a virtual class we need to make sure all our depends are appended 131 # If we're a virtual class we need to make sure all our depends are appended
138 # to the depends of fn. 132 # to the depends of fn.
139 depends = self.getVar("__depends", virtualfn, True) or [] 133 depends = self.getVar("__depends", virtualfn) or set()
134 self.depends_cache.setdefault(fn, {})
140 if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]: 135 if "__depends" not in self.depends_cache[fn] or not self.depends_cache[fn]["__depends"]:
141 self.depends_cache[fn]["__depends"] = depends 136 self.depends_cache[fn]["__depends"] = depends
142 for dep in depends: 137 else:
143 if dep not in self.depends_cache[fn]["__depends"]: 138 self.depends_cache[fn]["__depends"].update(depends)
144 self.depends_cache[fn]["__depends"].append(dep)
145 139
146 # Make sure the variants always make it into the cache too 140 # Make sure the variants always make it into the cache too
147 self.getVar('__VARIANTS', virtualfn, True) 141 self.getVar('__VARIANTS', virtualfn, True)
@@ -217,7 +211,7 @@ class Cache:
217 for data in bb_data: 211 for data in bb_data:
218 virtualfn = self.realfn2virtual(fn, data) 212 virtualfn = self.realfn2virtual(fn, data)
219 self.setData(virtualfn, fn, bb_data[data]) 213 self.setData(virtualfn, fn, bb_data[data])
220 if self.getVar("__SKIPPED", virtualfn, True): 214 if self.getVar("__SKIPPED", virtualfn):
221 skipped += 1 215 skipped += 1
222 bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn) 216 bb.msg.debug(1, bb.msg.domain.Cache, "Skipping %s" % virtualfn)
223 else: 217 else:
@@ -258,11 +252,11 @@ class Cache:
258 self.remove(fn) 252 self.remove(fn)
259 return False 253 return False
260 254
261 mtime = bb.parse.cached_mtime_noerror(fn) 255 mtime = bb.parse.cached_mtime_noerror(fn)
262 256
263 # Check file still exists 257 # Check file still exists
264 if mtime == 0: 258 if mtime == 0:
265 bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s not longer exists" % fn) 259 bb.msg.debug(2, bb.msg.domain.Cache, "Cache: %s no longer exists" % fn)
266 self.remove(fn) 260 self.remove(fn)
267 return False 261 return False
268 262
@@ -275,7 +269,7 @@ class Cache:
275 # Check dependencies are still valid 269 # Check dependencies are still valid
276 depends = self.getVar("__depends", fn, True) 270 depends = self.getVar("__depends", fn, True)
277 if depends: 271 if depends:
278 for f,old_mtime in depends: 272 for f, old_mtime in depends:
279 fmtime = bb.parse.cached_mtime_noerror(f) 273 fmtime = bb.parse.cached_mtime_noerror(f)
280 # Check if file still exists 274 # Check if file still exists
281 if old_mtime != 0 and fmtime == 0: 275 if old_mtime != 0 and fmtime == 0:
@@ -345,14 +339,14 @@ class Cache:
345 339
346 def handle_data(self, file_name, cacheData): 340 def handle_data(self, file_name, cacheData):
347 """ 341 """
348 Save data we need into the cache 342 Save data we need into the cache
349 """ 343 """
350 344
351 pn = self.getVar('PN', file_name, True) 345 pn = self.getVar('PN', file_name, True)
352 pe = self.getVar('PE', file_name, True) or "0" 346 pe = self.getVar('PE', file_name, True) or "0"
353 pv = self.getVar('PV', file_name, True) 347 pv = self.getVar('PV', file_name, True)
354 if 'SRCREVINACTION' in pv: 348 if 'SRCREVINACTION' in pv:
355 bb.note("Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name)) 349 bb.msg.note(1, bb.msg.domain.Cache, "Found SRCREVINACTION in PV (%s) or %s. Please report this bug." % (pv, file_name))
356 pr = self.getVar('PR', file_name, True) 350 pr = self.getVar('PR', file_name, True)
357 dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0") 351 dp = int(self.getVar('DEFAULT_PREFERENCE', file_name, True) or "0")
358 depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "") 352 depends = bb.utils.explode_deps(self.getVar("DEPENDS", file_name, True) or "")
@@ -360,7 +354,7 @@ class Cache:
360 packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split() 354 packages_dynamic = (self.getVar('PACKAGES_DYNAMIC', file_name, True) or "").split()
361 rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split() 355 rprovides = (self.getVar("RPROVIDES", file_name, True) or "").split()
362 356
363 cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name, True) 357 cacheData.task_deps[file_name] = self.getVar("_task_deps", file_name)
364 358
365 # build PackageName to FileName lookup table 359 # build PackageName to FileName lookup table
366 if pn not in cacheData.pkg_pn: 360 if pn not in cacheData.pkg_pn:
@@ -371,7 +365,7 @@ class Cache:
371 365
372 # build FileName to PackageName lookup table 366 # build FileName to PackageName lookup table
373 cacheData.pkg_fn[file_name] = pn 367 cacheData.pkg_fn[file_name] = pn
374 cacheData.pkg_pepvpr[file_name] = (pe,pv,pr) 368 cacheData.pkg_pepvpr[file_name] = (pe, pv, pr)
375 cacheData.pkg_dp[file_name] = dp 369 cacheData.pkg_dp[file_name] = dp
376 370
377 provides = [pn] 371 provides = [pn]
@@ -400,13 +394,13 @@ class Cache:
400 if not dep in cacheData.all_depends: 394 if not dep in cacheData.all_depends:
401 cacheData.all_depends.append(dep) 395 cacheData.all_depends.append(dep)
402 396
403 # Build reverse hash for PACKAGES, so runtime dependencies 397 # Build reverse hash for PACKAGES, so runtime dependencies
404 # can be be resolved (RDEPENDS, RRECOMMENDS etc.) 398 # can be be resolved (RDEPENDS, RRECOMMENDS etc.)
405 for package in packages: 399 for package in packages:
406 if not package in cacheData.packages: 400 if not package in cacheData.packages:
407 cacheData.packages[package] = [] 401 cacheData.packages[package] = []
408 cacheData.packages[package].append(file_name) 402 cacheData.packages[package].append(file_name)
409 rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split() 403 rprovides += (self.getVar("RPROVIDES_%s" % package, file_name, 1) or "").split()
410 404
411 for package in packages_dynamic: 405 for package in packages_dynamic:
412 if not package in cacheData.packages_dynamic: 406 if not package in cacheData.packages_dynamic:
@@ -445,38 +439,45 @@ class Cache:
445 self.getVar('__BB_DONT_CACHE', file_name, True) 439 self.getVar('__BB_DONT_CACHE', file_name, True)
446 self.getVar('__VARIANTS', file_name, True) 440 self.getVar('__VARIANTS', file_name, True)
447 441
448 def load_bbfile( self, bbfile , config): 442 def load_bbfile( self, bbfile, config):
449 """ 443 """
450 Load and parse one .bb build file 444 Load and parse one .bb build file
451 Return the data and whether parsing resulted in the file being skipped 445 Return the data and whether parsing resulted in the file being skipped
452 """ 446 """
447 chdir_back = False
453 448
454 import bb 449 from bb import data, parse
455 from bb import utils, data, parse, debug, event, fatal
456 450
457 # expand tmpdir to include this topdir 451 # expand tmpdir to include this topdir
458 data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config) 452 data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
459 bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) 453 bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
460 oldpath = os.path.abspath(os.getcwd()) 454 oldpath = os.path.abspath(os.getcwd())
461 if bb.parse.cached_mtime_noerror(bbfile_loc): 455 parse.cached_mtime_noerror(bbfile_loc)
462 os.chdir(bbfile_loc)
463 bb_data = data.init_db(config) 456 bb_data = data.init_db(config)
457 # The ConfHandler first looks if there is a TOPDIR and if not
458 # then it would call getcwd().
459 # Previously, we chdir()ed to bbfile_loc, called the handler
460 # and finally chdir()ed back, a couple of thousand times. We now
461 # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
462 if not data.getVar('TOPDIR', bb_data):
463 chdir_back = True
464 data.setVar('TOPDIR', bbfile_loc, bb_data)
464 try: 465 try:
465 bb_data = parse.handle(bbfile, bb_data) # read .bb data 466 bb_data = parse.handle(bbfile, bb_data) # read .bb data
466 os.chdir(oldpath) 467 if chdir_back: os.chdir(oldpath)
467 return bb_data 468 return bb_data
468 except: 469 except:
469 os.chdir(oldpath) 470 if chdir_back: os.chdir(oldpath)
470 raise 471 raise
471 472
472def init(cooker): 473def init(cooker):
473 """ 474 """
474 The Objective: Cache the minimum amount of data possible yet get to the 475 The Objective: Cache the minimum amount of data possible yet get to the
475 stage of building packages (i.e. tryBuild) without reparsing any .bb files. 476 stage of building packages (i.e. tryBuild) without reparsing any .bb files.
476 477
477 To do this, we intercept getVar calls and only cache the variables we see 478 To do this, we intercept getVar calls and only cache the variables we see
478 being accessed. We rely on the cache getVar calls being made for all 479 being accessed. We rely on the cache getVar calls being made for all
479 variables bitbake might need to use to reach this stage. For each cached 480 variables bitbake might need to use to reach this stage. For each cached
480 file we need to track: 481 file we need to track:
481 482
482 * Its mtime 483 * Its mtime
@@ -486,7 +487,7 @@ def init(cooker):
486 Files causing parsing errors are evicted from the cache. 487 Files causing parsing errors are evicted from the cache.
487 488
488 """ 489 """
489 return Cache(cooker) 490 return Cache(cooker.configuration.data)
490 491
491 492
492 493