diff options
| -rwxr-xr-x | bitbake/bin/bitbake-worker | 23 | ||||
| -rw-r--r-- | bitbake/lib/bb/cache.py | 46 | ||||
| -rw-r--r-- | bitbake/lib/bb/cooker.py | 23 |
3 files changed, 51 insertions, 41 deletions
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker index 963b4cdf93..1926b89882 100755 --- a/bitbake/bin/bitbake-worker +++ b/bitbake/bin/bitbake-worker | |||
| @@ -115,7 +115,7 @@ def sigterm_handler(signum, frame): | |||
| 115 | os.killpg(0, signal.SIGTERM) | 115 | os.killpg(0, signal.SIGTERM) |
| 116 | sys.exit() | 116 | sys.exit() |
| 117 | 117 | ||
| 118 | def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False): | 118 | def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, appends, taskdepdata, quieterrors=False): |
| 119 | # We need to setup the environment BEFORE the fork, since | 119 | # We need to setup the environment BEFORE the fork, since |
| 120 | # a fork() or exec*() activates PSEUDO... | 120 | # a fork() or exec*() activates PSEUDO... |
| 121 | 121 | ||
| @@ -193,15 +193,18 @@ def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, taskdepdat | |||
| 193 | if umask: | 193 | if umask: |
| 194 | os.umask(umask) | 194 | os.umask(umask) |
| 195 | 195 | ||
| 196 | data.setVar("BB_WORKERCONTEXT", "1") | ||
| 197 | data.setVar("BB_TASKDEPDATA", taskdepdata) | ||
| 198 | data.setVar("BUILDNAME", workerdata["buildname"]) | ||
| 199 | data.setVar("DATE", workerdata["date"]) | ||
| 200 | data.setVar("TIME", workerdata["time"]) | ||
| 201 | bb.parse.siggen.set_taskdata(workerdata["sigdata"]) | ||
| 202 | ret = 0 | ||
| 203 | try: | 196 | try: |
| 204 | the_data = bb.cache.Cache.loadDataFull(fn, appends, data) | 197 | bb_cache = bb.cache.NoCache(databuilder) |
| 198 | the_data = databuilder.data | ||
| 199 | the_data.setVar("BB_WORKERCONTEXT", "1") | ||
| 200 | the_data.setVar("BB_TASKDEPDATA", taskdepdata) | ||
| 201 | the_data.setVar("BUILDNAME", workerdata["buildname"]) | ||
| 202 | the_data.setVar("DATE", workerdata["date"]) | ||
| 203 | the_data.setVar("TIME", workerdata["time"]) | ||
| 204 | bb.parse.siggen.set_taskdata(workerdata["sigdata"]) | ||
| 205 | ret = 0 | ||
| 206 | |||
| 207 | the_data = bb_cache.loadDataFull(fn, appends) | ||
| 205 | the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task]) | 208 | the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task]) |
| 206 | 209 | ||
| 207 | bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN", True), taskname.replace("do_", ""))) | 210 | bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN", True), taskname.replace("do_", ""))) |
| @@ -389,7 +392,7 @@ class BitbakeWorker(object): | |||
| 389 | fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data) | 392 | fn, task, taskname, quieterrors, appends, taskdepdata = pickle.loads(data) |
| 390 | workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) | 393 | workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) |
| 391 | 394 | ||
| 392 | pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors) | 395 | pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, appends, taskdepdata, quieterrors) |
| 393 | 396 | ||
| 394 | self.build_pids[pid] = task | 397 | self.build_pids[pid] = task |
| 395 | self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout) | 398 | self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout) |
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index 8c1fe11317..5f302d68b4 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
| @@ -271,35 +271,44 @@ class NoCache(object): | |||
| 271 | self.databuilder = databuilder | 271 | self.databuilder = databuilder |
| 272 | self.data = databuilder.data | 272 | self.data = databuilder.data |
| 273 | 273 | ||
| 274 | @classmethod | 274 | def loadDataFull(self, virtualfn, appends): |
| 275 | def loadDataFull(cls, virtualfn, appends, cfgData): | ||
| 276 | """ | 275 | """ |
| 277 | Return a complete set of data for fn. | 276 | Return a complete set of data for fn. |
| 278 | To do this, we need to parse the file. | 277 | To do this, we need to parse the file. |
| 279 | """ | 278 | """ |
| 280 | 279 | logger.debug(1, "Parsing %s (full)" % virtualfn) | |
| 281 | (fn, virtual) = virtualfn2realfn(virtualfn) | 280 | (fn, virtual) = virtualfn2realfn(virtualfn) |
| 282 | 281 | bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) | |
| 283 | logger.debug(1, "Parsing %s (full)", fn) | ||
| 284 | |||
| 285 | cfgData.setVar("__ONLYFINALISE", virtual or "default") | ||
| 286 | bb_data = cls.load_bbfile(fn, appends, cfgData) | ||
| 287 | return bb_data[virtual] | 282 | return bb_data[virtual] |
| 288 | 283 | ||
| 289 | @staticmethod | 284 | def load_bbfile(self, bbfile, appends, virtonly = False): |
| 290 | def load_bbfile(bbfile, appends, config): | ||
| 291 | """ | 285 | """ |
| 292 | Load and parse one .bb build file | 286 | Load and parse one .bb build file |
| 293 | Return the data and whether parsing resulted in the file being skipped | 287 | Return the data and whether parsing resulted in the file being skipped |
| 294 | """ | 288 | """ |
| 289 | |||
| 290 | if virtonly: | ||
| 291 | (bbfile, virtual) = virtualfn2realfn(bbfile) | ||
| 292 | bb_data = self.data.createCopy() | ||
| 293 | bb_data.setVar("__BBMULTICONFIG", mc) | ||
| 294 | bb_data.setVar("__ONLYFINALISE", virtual or "default") | ||
| 295 | datastores = self._load_bbfile(bb_data, bbfile, appends) | ||
| 296 | return datastores | ||
| 297 | |||
| 298 | bb_data = self.data.createCopy() | ||
| 299 | datastores = self._load_bbfile(bb_data, bbfile, appends) | ||
| 300 | |||
| 301 | return datastores | ||
| 302 | |||
| 303 | def _load_bbfile(self, bb_data, bbfile, appends): | ||
| 295 | chdir_back = False | 304 | chdir_back = False |
| 296 | 305 | ||
| 297 | # expand tmpdir to include this topdir | 306 | # expand tmpdir to include this topdir |
| 298 | config.setVar('TMPDIR', config.getVar('TMPDIR', True) or "") | 307 | bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR', True) or "") |
| 299 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) | 308 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) |
| 300 | oldpath = os.path.abspath(os.getcwd()) | 309 | oldpath = os.path.abspath(os.getcwd()) |
| 301 | bb.parse.cached_mtime_noerror(bbfile_loc) | 310 | bb.parse.cached_mtime_noerror(bbfile_loc) |
| 302 | bb_data = config.createCopy() | 311 | |
| 303 | # The ConfHandler first looks if there is a TOPDIR and if not | 312 | # The ConfHandler first looks if there is a TOPDIR and if not |
| 304 | # then it would call getcwd(). | 313 | # then it would call getcwd(). |
| 305 | # Previously, we chdir()ed to bbfile_loc, called the handler | 314 | # Previously, we chdir()ed to bbfile_loc, called the handler |
| @@ -431,12 +440,11 @@ class Cache(NoCache): | |||
| 431 | len(self.depends_cache)), | 440 | len(self.depends_cache)), |
| 432 | self.data) | 441 | self.data) |
| 433 | 442 | ||
| 434 | @classmethod | 443 | def parse(self, filename, appends): |
| 435 | def parse(cls, filename, appends, configdata, caches_array): | ||
| 436 | """Parse the specified filename, returning the recipe information""" | 444 | """Parse the specified filename, returning the recipe information""" |
| 437 | logger.debug(1, "Parsing %s", filename) | 445 | logger.debug(1, "Parsing %s", filename) |
| 438 | infos = [] | 446 | infos = [] |
| 439 | datastores = cls.load_bbfile(filename, appends, configdata) | 447 | datastores = self.load_bbfile(filename, appends) |
| 440 | depends = [] | 448 | depends = [] |
| 441 | variants = [] | 449 | variants = [] |
| 442 | # Process the "real" fn last so we can store variants list | 450 | # Process the "real" fn last so we can store variants list |
| @@ -451,14 +459,14 @@ class Cache(NoCache): | |||
| 451 | if virtualfn == filename: | 459 | if virtualfn == filename: |
| 452 | data.setVar("__VARIANTS", " ".join(variants)) | 460 | data.setVar("__VARIANTS", " ".join(variants)) |
| 453 | info_array = [] | 461 | info_array = [] |
| 454 | for cache_class in caches_array: | 462 | for cache_class in self.caches_array: |
| 455 | info = cache_class(filename, data) | 463 | info = cache_class(filename, data) |
| 456 | info_array.append(info) | 464 | info_array.append(info) |
| 457 | infos.append((virtualfn, info_array)) | 465 | infos.append((virtualfn, info_array)) |
| 458 | 466 | ||
| 459 | return infos | 467 | return infos |
| 460 | 468 | ||
| 461 | def load(self, filename, appends, configdata): | 469 | def load(self, filename, appends): |
| 462 | """Obtain the recipe information for the specified filename, | 470 | """Obtain the recipe information for the specified filename, |
| 463 | using cached values if available, otherwise parsing. | 471 | using cached values if available, otherwise parsing. |
| 464 | 472 | ||
| @@ -479,13 +487,13 @@ class Cache(NoCache): | |||
| 479 | 487 | ||
| 480 | return cached, infos | 488 | return cached, infos |
| 481 | 489 | ||
| 482 | def loadData(self, fn, appends, cfgData, cacheData): | 490 | def loadData(self, fn, appends, cacheData): |
| 483 | """Load the recipe info for the specified filename, | 491 | """Load the recipe info for the specified filename, |
| 484 | parsing and adding to the cache if necessary, and adding | 492 | parsing and adding to the cache if necessary, and adding |
| 485 | the recipe information to the supplied CacheData instance.""" | 493 | the recipe information to the supplied CacheData instance.""" |
| 486 | skipped, virtuals = 0, 0 | 494 | skipped, virtuals = 0, 0 |
| 487 | 495 | ||
| 488 | cached, infos = self.load(fn, appends, cfgData) | 496 | cached, infos = self.load(fn, appends) |
| 489 | for virtualfn, info_array in infos: | 497 | for virtualfn, info_array in infos: |
| 490 | if info_array[0].skipped: | 498 | if info_array[0].skipped: |
| 491 | logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) | 499 | logger.debug(1, "Skipping %s: %s", virtualfn, info_array[0].skipreason) |
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 43c4f78dbc..fe95e73a12 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
| @@ -617,7 +617,8 @@ class BBCooker: | |||
| 617 | 617 | ||
| 618 | if fn: | 618 | if fn: |
| 619 | try: | 619 | try: |
| 620 | envdata = bb.cache.Cache.loadDataFull(fn, self.collection.get_file_appends(fn), self.data) | 620 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
| 621 | envdata = bb_cache.loadDataFull(fn, self.collection.get_file_appends(fn)) | ||
| 621 | except Exception as e: | 622 | except Exception as e: |
| 622 | parselog.exception("Unable to read %s", fn) | 623 | parselog.exception("Unable to read %s", fn) |
| 623 | raise | 624 | raise |
| @@ -1254,9 +1255,9 @@ class BBCooker: | |||
| 1254 | 1255 | ||
| 1255 | self.buildSetVars() | 1256 | self.buildSetVars() |
| 1256 | 1257 | ||
| 1257 | infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \ | 1258 | bb_cache = bb.cache.Cache(self.databuilder, self.data_hash, self.caches_array) |
| 1258 | self.data, | 1259 | |
| 1259 | self.caches_array) | 1260 | infos = bb_cache.parse(fn, self.collection.get_file_appends(fn)) |
| 1260 | infos = dict(infos) | 1261 | infos = dict(infos) |
| 1261 | 1262 | ||
| 1262 | fn = bb.cache.realfn2virtual(fn, cls) | 1263 | fn = bb.cache.realfn2virtual(fn, cls) |
| @@ -1943,7 +1944,7 @@ class Parser(multiprocessing.Process): | |||
| 1943 | except queue.Full: | 1944 | except queue.Full: |
| 1944 | pending.append(result) | 1945 | pending.append(result) |
| 1945 | 1946 | ||
| 1946 | def parse(self, filename, appends, caches_array): | 1947 | def parse(self, filename, appends): |
| 1947 | try: | 1948 | try: |
| 1948 | # Record the filename we're parsing into any events generated | 1949 | # Record the filename we're parsing into any events generated |
| 1949 | def parse_filter(self, record): | 1950 | def parse_filter(self, record): |
| @@ -1956,7 +1957,7 @@ class Parser(multiprocessing.Process): | |||
| 1956 | bb.event.set_class_handlers(self.handlers.copy()) | 1957 | bb.event.set_class_handlers(self.handlers.copy()) |
| 1957 | bb.event.LogHandler.filter = parse_filter | 1958 | bb.event.LogHandler.filter = parse_filter |
| 1958 | 1959 | ||
| 1959 | return True, bb.cache.Cache.parse(filename, appends, self.cfg, caches_array) | 1960 | return True, self.bb_cache.parse(filename, appends) |
| 1960 | except Exception as exc: | 1961 | except Exception as exc: |
| 1961 | tb = sys.exc_info()[2] | 1962 | tb = sys.exc_info()[2] |
| 1962 | exc.recipe = filename | 1963 | exc.recipe = filename |
| @@ -1995,7 +1996,7 @@ class CookerParser(object): | |||
| 1995 | for filename in self.filelist: | 1996 | for filename in self.filelist: |
| 1996 | appends = self.cooker.collection.get_file_appends(filename) | 1997 | appends = self.cooker.collection.get_file_appends(filename) |
| 1997 | if not self.bb_cache.cacheValid(filename, appends): | 1998 | if not self.bb_cache.cacheValid(filename, appends): |
| 1998 | self.willparse.append((filename, appends, cooker.caches_array)) | 1999 | self.willparse.append((filename, appends)) |
| 1999 | else: | 2000 | else: |
| 2000 | self.fromcache.append((filename, appends)) | 2001 | self.fromcache.append((filename, appends)) |
| 2001 | self.toparse = self.total - len(self.fromcache) | 2002 | self.toparse = self.total - len(self.fromcache) |
| @@ -2013,7 +2014,7 @@ class CookerParser(object): | |||
| 2013 | if self.toparse: | 2014 | if self.toparse: |
| 2014 | bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) | 2015 | bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) |
| 2015 | def init(): | 2016 | def init(): |
| 2016 | Parser.cfg = self.cfgdata | 2017 | Parser.bb_cache = self.bb_cache |
| 2017 | bb.utils.set_process_name(multiprocessing.current_process().name) | 2018 | bb.utils.set_process_name(multiprocessing.current_process().name) |
| 2018 | multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) | 2019 | multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) |
| 2019 | multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) | 2020 | multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) |
| @@ -2084,7 +2085,7 @@ class CookerParser(object): | |||
| 2084 | 2085 | ||
| 2085 | def load_cached(self): | 2086 | def load_cached(self): |
| 2086 | for filename, appends in self.fromcache: | 2087 | for filename, appends in self.fromcache: |
| 2087 | cached, infos = self.bb_cache.load(filename, appends, self.cfgdata) | 2088 | cached, infos = self.bb_cache.load(filename, appends) |
| 2088 | yield not cached, infos | 2089 | yield not cached, infos |
| 2089 | 2090 | ||
| 2090 | def parse_generator(self): | 2091 | def parse_generator(self): |
| @@ -2168,8 +2169,6 @@ class CookerParser(object): | |||
| 2168 | return True | 2169 | return True |
| 2169 | 2170 | ||
| 2170 | def reparse(self, filename): | 2171 | def reparse(self, filename): |
| 2171 | infos = self.bb_cache.parse(filename, | 2172 | infos = self.bb_cache.parse(filename, self.cooker.collection.get_file_appends(filename)) |
| 2172 | self.cooker.collection.get_file_appends(filename), | ||
| 2173 | self.cfgdata, self.cooker.caches_array) | ||
| 2174 | for vfn, info_array in infos: | 2173 | for vfn, info_array in infos: |
| 2175 | self.cooker.recipecache.add_from_recipeinfo(vfn, info_array) | 2174 | self.cooker.recipecache.add_from_recipeinfo(vfn, info_array) |
