diff options
Diffstat (limited to 'bitbake/lib/bb/cache.py')
-rw-r--r-- | bitbake/lib/bb/cache.py | 306 |
1 files changed, 306 insertions, 0 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py new file mode 100644 index 0000000000..921a9f7589 --- /dev/null +++ b/bitbake/lib/bb/cache.py | |||
@@ -0,0 +1,306 @@ | |||
1 | #!/usr/bin/env python | ||
2 | # ex:ts=4:sw=4:sts=4:et | ||
3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
4 | """ | ||
5 | BitBake 'Event' implementation | ||
6 | |||
7 | Caching of bitbake variables before task execution | ||
8 | |||
9 | # Copyright (C) 2006 Richard Purdie | ||
10 | |||
11 | # but small sections based on code from bin/bitbake: | ||
12 | # Copyright (C) 2003, 2004 Chris Larson | ||
13 | # Copyright (C) 2003, 2004 Phil Blundell | ||
14 | # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer | ||
15 | # Copyright (C) 2005 Holger Hans Peter Freyther | ||
16 | # Copyright (C) 2005 ROAD GmbH | ||
17 | |||
18 | This program is free software; you can redistribute it and/or modify it under | ||
19 | the terms of the GNU General Public License as published by the Free Software | ||
20 | Foundation; either version 2 of the License, or (at your option) any later | ||
21 | version. | ||
22 | |||
23 | This program is distributed in the hope that it will be useful, but WITHOUT | ||
24 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
25 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. | ||
26 | |||
27 | You should have received a copy of the GNU General Public License along with | ||
28 | this program; if not, write to the Free Software Foundation, Inc., 59 Temple | ||
29 | Place, Suite 330, Boston, MA 02111-1307 USA. | ||
30 | |||
31 | """ | ||
32 | |||
33 | import os, re | ||
34 | import bb.data | ||
35 | import bb.utils | ||
36 | |||
37 | try: | ||
38 | import cPickle as pickle | ||
39 | except ImportError: | ||
40 | import pickle | ||
41 | print "NOTE: Importing cPickle failed. Falling back to a very slow implementation." | ||
42 | |||
43 | # __cache_version__ = "123" | ||
44 | __cache_version__ = "124" # changes the __depends structure | ||
45 | |||
46 | class Cache: | ||
47 | """ | ||
48 | BitBake Cache implementation | ||
49 | """ | ||
50 | def __init__(self, cooker): | ||
51 | |||
52 | |||
53 | self.cachedir = bb.data.getVar("CACHE", cooker.configuration.data, True) | ||
54 | self.clean = {} | ||
55 | self.depends_cache = {} | ||
56 | self.data = None | ||
57 | self.data_fn = None | ||
58 | |||
59 | if self.cachedir in [None, '']: | ||
60 | self.has_cache = False | ||
61 | if cooker.cb is not None: | ||
62 | print "NOTE: Not using a cache. Set CACHE = <directory> to enable." | ||
63 | else: | ||
64 | self.has_cache = True | ||
65 | self.cachefile = os.path.join(self.cachedir,"bb_cache.dat") | ||
66 | |||
67 | if cooker.cb is not None: | ||
68 | print "NOTE: Using cache in '%s'" % self.cachedir | ||
69 | try: | ||
70 | os.stat( self.cachedir ) | ||
71 | except OSError: | ||
72 | bb.mkdirhier( self.cachedir ) | ||
73 | |||
74 | if self.has_cache and (self.mtime(self.cachefile)): | ||
75 | try: | ||
76 | p = pickle.Unpickler( file(self.cachefile,"rb")) | ||
77 | self.depends_cache, version_data = p.load() | ||
78 | if version_data['CACHE_VER'] != __cache_version__: | ||
79 | raise ValueError, 'Cache Version Mismatch' | ||
80 | if version_data['BITBAKE_VER'] != bb.__version__: | ||
81 | raise ValueError, 'Bitbake Version Mismatch' | ||
82 | except (ValueError, KeyError): | ||
83 | bb.note("Invalid cache found, rebuilding...") | ||
84 | self.depends_cache = {} | ||
85 | |||
86 | if self.depends_cache: | ||
87 | for fn in self.depends_cache.keys(): | ||
88 | self.clean[fn] = "" | ||
89 | self.cacheValidUpdate(fn) | ||
90 | |||
91 | def getVar(self, var, fn, exp = 0): | ||
92 | """ | ||
93 | Gets the value of a variable | ||
94 | (similar to getVar in the data class) | ||
95 | |||
96 | There are two scenarios: | ||
97 | 1. We have cached data - serve from depends_cache[fn] | ||
98 | 2. We're learning what data to cache - serve from data | ||
99 | backend but add a copy of the data to the cache. | ||
100 | """ | ||
101 | |||
102 | if fn in self.clean: | ||
103 | return self.depends_cache[fn][var] | ||
104 | |||
105 | if not fn in self.depends_cache: | ||
106 | self.depends_cache[fn] = {} | ||
107 | |||
108 | if fn != self.data_fn: | ||
109 | # We're trying to access data in the cache which doesn't exist | ||
110 | # yet setData hasn't been called to setup the right access. Very bad. | ||
111 | bb.error("Parsing error data_fn %s and fn %s don't match" % (self.data_fn, fn)) | ||
112 | |||
113 | result = bb.data.getVar(var, self.data, exp) | ||
114 | self.depends_cache[fn][var] = result | ||
115 | return result | ||
116 | |||
117 | def setData(self, fn, data): | ||
118 | """ | ||
119 | Called to prime bb_cache ready to learn which variables to cache. | ||
120 | Will be followed by calls to self.getVar which aren't cached | ||
121 | but can be fulfilled from self.data. | ||
122 | """ | ||
123 | self.data_fn = fn | ||
124 | self.data = data | ||
125 | |||
126 | # Make sure __depends makes the depends_cache | ||
127 | self.getVar("__depends", fn, True) | ||
128 | self.depends_cache[fn]["CACHETIMESTAMP"] = bb.parse.cached_mtime(fn) | ||
129 | |||
130 | def loadDataFull(self, fn, cooker): | ||
131 | """ | ||
132 | Return a complete set of data for fn. | ||
133 | To do this, we need to parse the file. | ||
134 | """ | ||
135 | bb_data, skipped = self.load_bbfile(fn, cooker) | ||
136 | return bb_data | ||
137 | |||
138 | def loadData(self, fn, cooker): | ||
139 | """ | ||
140 | Load a subset of data for fn. | ||
141 | If the cached data is valid we do nothing, | ||
142 | To do this, we need to parse the file and set the system | ||
143 | to record the variables accessed. | ||
144 | Return the cache status and whether the file was skipped when parsed | ||
145 | """ | ||
146 | if self.cacheValid(fn): | ||
147 | if "SKIPPED" in self.depends_cache[fn]: | ||
148 | return True, True | ||
149 | return True, False | ||
150 | |||
151 | bb_data, skipped = self.load_bbfile(fn, cooker) | ||
152 | self.setData(fn, bb_data) | ||
153 | return False, skipped | ||
154 | |||
155 | def cacheValid(self, fn): | ||
156 | """ | ||
157 | Is the cache valid for fn? | ||
158 | Fast version, no timestamps checked. | ||
159 | """ | ||
160 | # Is cache enabled? | ||
161 | if not self.has_cache: | ||
162 | return False | ||
163 | if fn in self.clean: | ||
164 | return True | ||
165 | return False | ||
166 | |||
167 | def cacheValidUpdate(self, fn): | ||
168 | """ | ||
169 | Is the cache valid for fn? | ||
170 | Make thorough (slower) checks including timestamps. | ||
171 | """ | ||
172 | # Is cache enabled? | ||
173 | if not self.has_cache: | ||
174 | return False | ||
175 | |||
176 | # Check file still exists | ||
177 | if self.mtime(fn) == 0: | ||
178 | bb.debug(2, "Cache: %s not longer exists" % fn) | ||
179 | self.remove(fn) | ||
180 | return False | ||
181 | |||
182 | # File isn't in depends_cache | ||
183 | if not fn in self.depends_cache: | ||
184 | bb.debug(2, "Cache: %s is not cached" % fn) | ||
185 | self.remove(fn) | ||
186 | return False | ||
187 | |||
188 | # Check the file's timestamp | ||
189 | if bb.parse.cached_mtime(fn) > self.getVar("CACHETIMESTAMP", fn, True): | ||
190 | bb.debug(2, "Cache: %s changed" % fn) | ||
191 | self.remove(fn) | ||
192 | return False | ||
193 | |||
194 | # Check dependencies are still valid | ||
195 | depends = self.getVar("__depends", fn, True) | ||
196 | for f,old_mtime in depends: | ||
197 | new_mtime = bb.parse.cached_mtime(f) | ||
198 | if (new_mtime > old_mtime): | ||
199 | bb.debug(2, "Cache: %s's dependency %s changed" % (fn, f)) | ||
200 | self.remove(fn) | ||
201 | return False | ||
202 | |||
203 | bb.debug(2, "Depends Cache: %s is clean" % fn) | ||
204 | if not fn in self.clean: | ||
205 | self.clean[fn] = "" | ||
206 | |||
207 | return True | ||
208 | |||
209 | def skip(self, fn): | ||
210 | """ | ||
211 | Mark a fn as skipped | ||
212 | Called from the parser | ||
213 | """ | ||
214 | if not fn in self.depends_cache: | ||
215 | self.depends_cache[fn] = {} | ||
216 | self.depends_cache[fn]["SKIPPED"] = "1" | ||
217 | |||
218 | def remove(self, fn): | ||
219 | """ | ||
220 | Remove a fn from the cache | ||
221 | Called from the parser in error cases | ||
222 | """ | ||
223 | bb.debug(1, "Removing %s from cache" % fn) | ||
224 | if fn in self.depends_cache: | ||
225 | del self.depends_cache[fn] | ||
226 | if fn in self.clean: | ||
227 | del self.clean[fn] | ||
228 | |||
229 | def sync(self): | ||
230 | """ | ||
231 | Save the cache | ||
232 | Called from the parser when complete (or exitting) | ||
233 | """ | ||
234 | |||
235 | if not self.has_cache: | ||
236 | return | ||
237 | |||
238 | version_data = {} | ||
239 | version_data['CACHE_VER'] = __cache_version__ | ||
240 | version_data['BITBAKE_VER'] = bb.__version__ | ||
241 | |||
242 | p = pickle.Pickler(file(self.cachefile, "wb" ), -1 ) | ||
243 | p.dump([self.depends_cache, version_data]) | ||
244 | |||
245 | def mtime(self, cachefile): | ||
246 | try: | ||
247 | return os.stat(cachefile)[8] | ||
248 | except OSError: | ||
249 | return 0 | ||
250 | |||
251 | def load_bbfile( self, bbfile , cooker): | ||
252 | """ | ||
253 | Load and parse one .bb build file | ||
254 | Return the data and whether parsing resulted in the file being skipped | ||
255 | """ | ||
256 | |||
257 | import bb | ||
258 | from bb import utils, data, parse, debug, event, fatal | ||
259 | |||
260 | topdir = data.getVar('TOPDIR', cooker.configuration.data) | ||
261 | if not topdir: | ||
262 | topdir = os.path.abspath(os.getcwd()) | ||
263 | # set topdir to here | ||
264 | data.setVar('TOPDIR', topdir, cooker.configuration) | ||
265 | bbfile = os.path.abspath(bbfile) | ||
266 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) | ||
267 | # expand tmpdir to include this topdir | ||
268 | data.setVar('TMPDIR', data.getVar('TMPDIR', cooker.configuration.data, 1) or "", cooker.configuration.data) | ||
269 | # set topdir to location of .bb file | ||
270 | topdir = bbfile_loc | ||
271 | #data.setVar('TOPDIR', topdir, cfg) | ||
272 | # go there | ||
273 | oldpath = os.path.abspath(os.getcwd()) | ||
274 | if self.mtime(topdir): | ||
275 | os.chdir(topdir) | ||
276 | bb_data = data.init_db(cooker.configuration.data) | ||
277 | try: | ||
278 | parse.handle(bbfile, bb_data) # read .bb data | ||
279 | os.chdir(oldpath) | ||
280 | return bb_data, False | ||
281 | except bb.parse.SkipPackage: | ||
282 | os.chdir(oldpath) | ||
283 | return bb_data, True | ||
284 | except: | ||
285 | os.chdir(oldpath) | ||
286 | raise | ||
287 | |||
288 | def init(cooker): | ||
289 | """ | ||
290 | The Objective: Cache the minimum amount of data possible yet get to the | ||
291 | stage of building packages (i.e. tryBuild) without reparsing any .bb files. | ||
292 | |||
293 | To do this, we intercept getVar calls and only cache the variables we see | ||
294 | being accessed. We rely on the cache getVar calls being made for all | ||
295 | variables bitbake might need to use to reach this stage. For each cached | ||
296 | file we need to track: | ||
297 | |||
298 | * Its mtime | ||
299 | * The mtimes of all its dependencies | ||
300 | * Whether it caused a parse.SkipPackage exception | ||
301 | |||
302 | Files causing parsing errors are evicted from the cache. | ||
303 | |||
304 | """ | ||
305 | return Cache(cooker) | ||
306 | |||