summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/cooker.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r--bitbake/lib/bb/cooker.py232
1 files changed, 126 insertions, 106 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 5c52d85901..da598c9330 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -89,7 +89,6 @@ class BBCooker:
89 89
90 def __init__(self, configuration, server_registration_cb, savedenv={}): 90 def __init__(self, configuration, server_registration_cb, savedenv={}):
91 self.status = None 91 self.status = None
92 self.appendlist = {}
93 self.skiplist = {} 92 self.skiplist = {}
94 93
95 self.server_registration_cb = server_registration_cb 94 self.server_registration_cb = server_registration_cb
@@ -419,7 +418,7 @@ class BBCooker:
419 418
420 if fn: 419 if fn:
421 try: 420 try:
422 envdata = bb.cache.Cache.loadDataFull(fn, self.get_file_appends(fn), self.configuration.data) 421 envdata = bb.cache.Cache.loadDataFull(fn, self.collection.get_file_appends(fn), self.configuration.data)
423 except Exception as e: 422 except Exception as e:
424 parselog.exception("Unable to read %s", fn) 423 parselog.exception("Unable to read %s", fn)
425 raise 424 raise
@@ -698,22 +697,13 @@ class BBCooker:
698 print("}", file=tdepends_file) 697 print("}", file=tdepends_file)
699 logger.info("Task dependencies saved to 'task-depends.dot'") 698 logger.info("Task dependencies saved to 'task-depends.dot'")
700 699
701 def calc_bbfile_priority( self, filename, matched = None ):
702 for _, _, regex, pri in self.status.bbfile_config_priorities:
703 if regex.match(filename):
704 if matched != None:
705 if not regex in matched:
706 matched.add(regex)
707 return pri
708 return 0
709
710 def show_appends_with_no_recipes( self ): 700 def show_appends_with_no_recipes( self ):
711 recipes = set(os.path.basename(f) 701 recipes = set(os.path.basename(f)
712 for f in self.status.pkg_fn.iterkeys()) 702 for f in self.status.pkg_fn.iterkeys())
713 recipes |= set(os.path.basename(f) 703 recipes |= set(os.path.basename(f)
714 for f in self.skiplist.iterkeys()) 704 for f in self.skiplist.iterkeys())
715 appended_recipes = self.appendlist.iterkeys() 705 appended_recipes = self.collection.appendlist.iterkeys()
716 appends_without_recipes = [self.appendlist[recipe] 706 appends_without_recipes = [self.collection.appendlist[recipe]
717 for recipe in appended_recipes 707 for recipe in appended_recipes
718 if recipe not in recipes] 708 if recipe not in recipes]
719 if appends_without_recipes: 709 if appends_without_recipes:
@@ -747,32 +737,8 @@ class BBCooker:
747 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.status.preferred[providee]) 737 providerlog.error("conflicting preferences for %s: both %s and %s specified", providee, provider, self.status.preferred[providee])
748 self.status.preferred[providee] = provider 738 self.status.preferred[providee] = provider
749 739
750 # Calculate priorities for each file
751 matched = set()
752 for p in self.status.pkg_fn:
753 realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
754 self.status.bbfile_priority[p] = self.calc_bbfile_priority(realfn, matched)
755 740
756 # Don't show the warning if the BBFILE_PATTERN did match .bbappend files 741 self.status.bbfile_priority = self.collection.collection_priorities(self.status.pkg_fn)
757 unmatched = set()
758 for _, _, regex, pri in self.status.bbfile_config_priorities:
759 if not regex in matched:
760 unmatched.add(regex)
761
762 def findmatch(regex):
763 for bbfile in self.appendlist:
764 for append in self.appendlist[bbfile]:
765 if regex.match(append):
766 return True
767 return False
768
769 for unmatch in unmatched.copy():
770 if findmatch(unmatch):
771 unmatched.remove(unmatch)
772
773 for collection, pattern, regex, _ in self.status.bbfile_config_priorities:
774 if regex in unmatched:
775 collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
776 742
777 def findCoreBaseFiles(self, subdir, configfile): 743 def findCoreBaseFiles(self, subdir, configfile):
778 corebase = self.configuration.data.getVar('COREBASE', True) or "" 744 corebase = self.configuration.data.getVar('COREBASE', True) or ""
@@ -1111,7 +1077,9 @@ class BBCooker:
1111 """ 1077 """
1112 if bf.startswith("/") or bf.startswith("../"): 1078 if bf.startswith("/") or bf.startswith("../"):
1113 bf = os.path.abspath(bf) 1079 bf = os.path.abspath(bf)
1114 filelist, masked = self.collect_bbfiles() 1080
1081 self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
1082 filelist, masked = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
1115 try: 1083 try:
1116 os.stat(bf) 1084 os.stat(bf)
1117 bf = os.path.abspath(bf) 1085 bf = os.path.abspath(bf)
@@ -1165,7 +1133,7 @@ class BBCooker:
1165 self.buildSetVars() 1133 self.buildSetVars()
1166 1134
1167 self.status = bb.cache.CacheData(self.caches_array) 1135 self.status = bb.cache.CacheData(self.caches_array)
1168 infos = bb.cache.Cache.parse(fn, self.get_file_appends(fn), \ 1136 infos = bb.cache.Cache.parse(fn, self.collection.get_file_appends(fn), \
1169 self.configuration.data, 1137 self.configuration.data,
1170 self.caches_array) 1138 self.caches_array)
1171 infos = dict(infos) 1139 infos = dict(infos)
@@ -1332,7 +1300,9 @@ class BBCooker:
1332 for dep in self.configuration.extra_assume_provided: 1300 for dep in self.configuration.extra_assume_provided:
1333 self.status.ignored_dependencies.add(dep) 1301 self.status.ignored_dependencies.add(dep)
1334 1302
1335 (filelist, masked) = self.collect_bbfiles() 1303 self.collection = CookerCollectFiles(self.status.bbfile_config_priorities)
1304 (filelist, masked) = self.collection.collect_bbfiles(self.configuration.data, self.configuration.event_data)
1305
1336 self.configuration.data.renameVar("__depends", "__base_depends") 1306 self.configuration.data.renameVar("__depends", "__base_depends")
1337 1307
1338 self.parser = CookerParser(self, filelist, masked) 1308 self.parser = CookerParser(self, filelist, masked)
@@ -1369,7 +1339,87 @@ class BBCooker:
1369 1339
1370 return pkgs_to_build 1340 return pkgs_to_build
1371 1341
1372 def get_bbfiles( self, path = os.getcwd() ): 1342
1343
1344
1345 def pre_serve(self):
1346 # Empty the environment. The environment will be populated as
1347 # necessary from the data store.
1348 #bb.utils.empty_environment()
1349 try:
1350 prserv.serv.auto_start(self.configuration.data)
1351 except prserv.serv.PRServiceConfigError:
1352 bb.event.fire(CookerExit(), self.configuration.event_data)
1353 return
1354
1355 def post_serve(self):
1356 prserv.serv.auto_shutdown(self.configuration.data)
1357 bb.event.fire(CookerExit(), self.configuration.event_data)
1358
1359 def shutdown(self):
1360 self.state = state.shutdown
1361
1362 def stop(self):
1363 self.state = state.stop
1364
1365 def reparseFiles(self):
1366 return
1367
1368 def initialize(self):
1369 self.state = state.initial
1370 self.initConfigurationData()
1371
1372 def reset(self):
1373 self.state = state.initial
1374 self.loadConfigurationData()
1375
1376def server_main(cooker, func, *args):
1377 cooker.pre_serve()
1378
1379 if cooker.configuration.profile:
1380 try:
1381 import cProfile as profile
1382 except:
1383 import profile
1384 prof = profile.Profile()
1385
1386 ret = profile.Profile.runcall(prof, func, *args)
1387
1388 prof.dump_stats("profile.log")
1389 bb.utils.process_profilelog("profile.log")
1390 print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
1391
1392 else:
1393 ret = func(*args)
1394
1395 cooker.post_serve()
1396
1397 return ret
1398
1399class CookerExit(bb.event.Event):
1400 """
1401 Notify clients of the Cooker shutdown
1402 """
1403
1404 def __init__(self):
1405 bb.event.Event.__init__(self)
1406
1407
1408class CookerCollectFiles(object):
1409 def __init__(self, priorities):
1410 self.appendlist = {}
1411 self.bbfile_config_priorities = priorities
1412
1413 def calc_bbfile_priority( self, filename, matched = None ):
1414 for _, _, regex, pri in self.bbfile_config_priorities:
1415 if regex.match(filename):
1416 if matched != None:
1417 if not regex in matched:
1418 matched.add(regex)
1419 return pri
1420 return 0
1421
1422 def get_bbfiles(self, path = os.getcwd()):
1373 """Get list of default .bb files by reading out the current directory""" 1423 """Get list of default .bb files by reading out the current directory"""
1374 contents = os.listdir(path) 1424 contents = os.listdir(path)
1375 bbfiles = [] 1425 bbfiles = []
@@ -1379,7 +1429,7 @@ class BBCooker:
1379 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f))) 1429 bbfiles.append(os.path.abspath(os.path.join(os.getcwd(), f)))
1380 return bbfiles 1430 return bbfiles
1381 1431
1382 def find_bbfiles( self, path ): 1432 def find_bbfiles(self, path):
1383 """Find all the .bb and .bbappend files in a directory""" 1433 """Find all the .bb and .bbappend files in a directory"""
1384 from os.path import join 1434 from os.path import join
1385 1435
@@ -1392,14 +1442,14 @@ class BBCooker:
1392 1442
1393 return found 1443 return found
1394 1444
1395 def collect_bbfiles( self ): 1445 def collect_bbfiles(self, config, eventdata):
1396 """Collect all available .bb build files""" 1446 """Collect all available .bb build files"""
1397 masked = 0 1447 masked = 0
1398 1448
1399 collectlog.debug(1, "collecting .bb files") 1449 collectlog.debug(1, "collecting .bb files")
1400 1450
1401 files = (data.getVar( "BBFILES", self.configuration.data, True) or "").split() 1451 files = (config.getVar( "BBFILES", True) or "").split()
1402 data.setVar("BBFILES", " ".join(files), self.configuration.data) 1452 config.setVar("BBFILES", " ".join(files))
1403 1453
1404 # Sort files by priority 1454 # Sort files by priority
1405 files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) ) 1455 files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem) )
@@ -1409,7 +1459,7 @@ class BBCooker:
1409 1459
1410 if not len(files): 1460 if not len(files):
1411 collectlog.error("no recipe files to build, check your BBPATH and BBFILES?") 1461 collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
1412 bb.event.fire(CookerExit(), self.configuration.event_data) 1462 bb.event.fire(CookerExit(), eventdata)
1413 1463
1414 # Can't use set here as order is important 1464 # Can't use set here as order is important
1415 newfiles = [] 1465 newfiles = []
@@ -1427,7 +1477,7 @@ class BBCooker:
1427 if g not in newfiles: 1477 if g not in newfiles:
1428 newfiles.append(g) 1478 newfiles.append(g)
1429 1479
1430 bbmask = self.configuration.data.getVar('BBMASK', True) 1480 bbmask = config.getVar('BBMASK', True)
1431 1481
1432 if bbmask: 1482 if bbmask:
1433 try: 1483 try:
@@ -1475,74 +1525,44 @@ class BBCooker:
1475 def get_file_appends(self, fn): 1525 def get_file_appends(self, fn):
1476 """ 1526 """
1477 Returns a list of .bbappend files to apply to fn 1527 Returns a list of .bbappend files to apply to fn
1478 NB: collect_bbfiles() must have been called prior to this
1479 """ 1528 """
1480 f = os.path.basename(fn) 1529 f = os.path.basename(fn)
1481 if f in self.appendlist: 1530 if f in self.appendlist:
1482 return self.appendlist[f] 1531 return self.appendlist[f]
1483 return [] 1532 return []
1484 1533
1485 def pre_serve(self): 1534 def collection_priorities(self, pkgfns):
1486 # Empty the environment. The environment will be populated as
1487 # necessary from the data store.
1488 #bb.utils.empty_environment()
1489 try:
1490 prserv.serv.auto_start(self.configuration.data)
1491 except prserv.serv.PRServiceConfigError:
1492 bb.event.fire(CookerExit(), self.configuration.event_data)
1493 return
1494
1495 def post_serve(self):
1496 prserv.serv.auto_shutdown(self.configuration.data)
1497 bb.event.fire(CookerExit(), self.configuration.event_data)
1498
1499 def shutdown(self):
1500 self.state = state.shutdown
1501 1535
1502 def stop(self): 1536 priorities = {}
1503 self.state = state.stop
1504
1505 def reparseFiles(self):
1506 return
1507
1508 def initialize(self):
1509 self.state = state.initial
1510 self.initConfigurationData()
1511
1512 def reset(self):
1513 self.state = state.initial
1514 self.loadConfigurationData()
1515 1537
1516def server_main(cooker, func, *args): 1538 # Calculate priorities for each file
1517 cooker.pre_serve() 1539 matched = set()
1518 1540 for p in pkgfns:
1519 if cooker.configuration.profile: 1541 realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
1520 try: 1542 priorities[p] = self.calc_bbfile_priority(realfn, matched)
1521 import cProfile as profile 1543
1522 except: 1544 # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
1523 import profile 1545 unmatched = set()
1524 prof = profile.Profile() 1546 for _, _, regex, pri in self.bbfile_config_priorities:
1525 1547 if not regex in matched:
1526 ret = profile.Profile.runcall(prof, func, *args) 1548 unmatched.add(regex)
1527
1528 prof.dump_stats("profile.log")
1529 bb.utils.process_profilelog("profile.log")
1530 print("Raw profiling information saved to profile.log and processed statistics to profile.log.processed")
1531
1532 else:
1533 ret = func(*args)
1534 1549
1535 cooker.post_serve() 1550 def findmatch(regex):
1551 for bbfile in self.appendlist:
1552 for append in self.appendlist[bbfile]:
1553 if regex.match(append):
1554 return True
1555 return False
1536 1556
1537 return ret 1557 for unmatch in unmatched.copy():
1558 if findmatch(unmatch):
1559 unmatched.remove(unmatch)
1538 1560
1539class CookerExit(bb.event.Event): 1561 for collection, pattern, regex, _ in self.bbfile_config_priorities:
1540 """ 1562 if regex in unmatched:
1541 Notify clients of the Cooker shutdown 1563 collectlog.warn("No bb files matched BBFILE_PATTERN_%s '%s'" % (collection, pattern))
1542 """
1543 1564
1544 def __init__(self): 1565 return priorities
1545 bb.event.Event.__init__(self)
1546 1566
1547def catch_parse_error(func): 1567def catch_parse_error(func):
1548 """Exception handling bits for our parsing""" 1568 """Exception handling bits for our parsing"""
@@ -1677,7 +1697,7 @@ class CookerParser(object):
1677 self.fromcache = [] 1697 self.fromcache = []
1678 self.willparse = [] 1698 self.willparse = []
1679 for filename in self.filelist: 1699 for filename in self.filelist:
1680 appends = self.cooker.get_file_appends(filename) 1700 appends = self.cooker.collection.get_file_appends(filename)
1681 if not self.bb_cache.cacheValid(filename, appends): 1701 if not self.bb_cache.cacheValid(filename, appends):
1682 self.willparse.append((filename, appends, cooker.caches_array)) 1702 self.willparse.append((filename, appends, cooker.caches_array))
1683 else: 1703 else:
@@ -1840,7 +1860,7 @@ class CookerParser(object):
1840 1860
1841 def reparse(self, filename): 1861 def reparse(self, filename):
1842 infos = self.bb_cache.parse(filename, 1862 infos = self.bb_cache.parse(filename,
1843 self.cooker.get_file_appends(filename), 1863 self.cooker.collection.get_file_appends(filename),
1844 self.cfgdata, self.cooker.caches_array) 1864 self.cfgdata, self.cooker.caches_array)
1845 for vfn, info_array in infos: 1865 for vfn, info_array in infos:
1846 self.cooker.status.add_from_recipeinfo(vfn, info_array) 1866 self.cooker.status.add_from_recipeinfo(vfn, info_array)