summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-13 14:13:53 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-14 11:32:02 +0000
commit6c3c3e11f6f06f6ad9bf8fd2a026e03f76204ba4 (patch)
tree1d282d88490eebf4ba0e31f7d159538161b32b54 /bitbake/lib/bb
parent83f8a4002ab2026696973422ee0f5733652d9527 (diff)
downloadpoky-6c3c3e11f6f06f6ad9bf8fd2a026e03f76204ba4.tar.gz
bitbake: cooker/cache/parse: Implement pyinofity based reconfigure
Memory resident bitbake has one current flaw, changes in the base configuration are not noticed by bitbake. The parsing cache is also refreshed on each invocation of bitbake (although the mtime cache is not cleared so its pointless). This change adds in pyinotify support and adds two different watchers, one for the base configuration and one for the parsed recipes. Changes in the latter will trigger a reparse (and an update of the mtime cache). The former will trigger a complete reload of the configuration. Note that this code will also correctly handle creation of new configuration files since the __depends and __base_depends variables already track these for cache correctness purposes. We could be a little more clever about parsing cache invalidation, right now we just invalidate the whole thing and recheck. For now, its better than what we have and doesn't seem to perform that badly though. For education and QA purposes I can document a workflow that illustrates this: $ source oe-init-build-env-memres $ time bitbake bash [base configuration is loaded, recipes are parsed, bash builds] $ time bitbake bash [command returns quickly since all caches are valid] $ touch ../meta/classes/gettext.bbclass $ time bitbake bash [reparse is triggered, time is longer than above] $ echo 'FOO = "1"' >> conf/local.conf $ time bitbake bash [reparse is triggered, but with a base configuration reload too] As far as changes go, I like this one a lot, it makes memory resident bitbake truly usable and may be the tweak we need to make it the default. The new pyinotify dependency is covered in the previous commit. (Bitbake rev: 0557d03c170fba8d7efe82be1b9641d0eb229213) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/cache.py5
-rw-r--r--bitbake/lib/bb/cooker.py44
-rw-r--r--bitbake/lib/bb/parse/__init__.py5
3 files changed, 51 insertions, 3 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 715da07e8d..a1dde96425 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -623,10 +623,13 @@ class Cache(object):
623 def mtime(cachefile): 623 def mtime(cachefile):
624 return bb.parse.cached_mtime_noerror(cachefile) 624 return bb.parse.cached_mtime_noerror(cachefile)
625 625
626 def add_info(self, filename, info_array, cacheData, parsed=None): 626 def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
627 if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped): 627 if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
628 cacheData.add_from_recipeinfo(filename, info_array) 628 cacheData.add_from_recipeinfo(filename, info_array)
629 629
630 if watcher:
631 watcher(info_array[0].file_depends)
632
630 if not self.has_cache: 633 if not self.has_cache:
631 return 634 return
632 635
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 23e7abda39..f14eb64c83 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -39,6 +39,7 @@ from bb import utils, data, parse, event, cache, providers, taskdata, runqueue
39import Queue 39import Queue
40import signal 40import signal
41import prserv.serv 41import prserv.serv
42import pyinotify
42 43
43logger = logging.getLogger("BitBake") 44logger = logging.getLogger("BitBake")
44collectlog = logging.getLogger("BitBake.Collection") 45collectlog = logging.getLogger("BitBake.Collection")
@@ -120,7 +121,18 @@ class BBCooker:
120 121
121 self.configuration = configuration 122 self.configuration = configuration
122 123
124 self.configwatcher = pyinotify.WatchManager()
125 self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
126 self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
127 pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
128 pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
129 self.watcher = pyinotify.WatchManager()
130 self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
131
132
123 self.initConfigurationData() 133 self.initConfigurationData()
134 self.baseconfig_valid = True
135 self.parsecache_valid = False
124 136
125 # Take a lock so only one copy of bitbake can run against a given build 137 # Take a lock so only one copy of bitbake can run against a given build
126 # directory at a time 138 # directory at a time
@@ -156,6 +168,18 @@ class BBCooker:
156 # Let SIGHUP exit as SIGTERM 168 # Let SIGHUP exit as SIGTERM
157 signal.signal(signal.SIGHUP, self.sigterm_exception) 169 signal.signal(signal.SIGHUP, self.sigterm_exception)
158 170
171 def config_notifications(self, event):
172 bb.parse.update_cache(event.path)
173 self.baseconfig_valid = False
174
175 def notifications(self, event):
176 bb.parse.update_cache(event.path)
177 self.parsecache_valid = False
178
179 def add_filewatch(self, deps):
180 for i in deps:
181 self.watcher.add_watch(i[0], self.watchmask, rec=True)
182
159 def sigterm_exception(self, signum, stackframe): 183 def sigterm_exception(self, signum, stackframe):
160 if signum == signal.SIGTERM: 184 if signum == signal.SIGTERM:
161 bb.warn("Cooker recieved SIGTERM, shutting down...") 185 bb.warn("Cooker recieved SIGTERM, shutting down...")
@@ -1361,6 +1385,18 @@ class BBCooker:
1361 raise bb.BBHandledException() 1385 raise bb.BBHandledException()
1362 1386
1363 if self.state != state.parsing: 1387 if self.state != state.parsing:
1388 for n in [self.confignotifier, self.notifier]:
1389 if n.check_events(timeout=0):
1390 # read notified events and enqeue them
1391 n.read_events()
1392 n.process_events()
1393 if not self.baseconfig_valid:
1394 logger.debug(1, "Reloading base configuration data")
1395 self.initConfigurationData()
1396 self.baseconfig_valid = True
1397 self.parsecache_valid = False
1398
1399 if self.state != state.parsing and not self.parsecache_valid:
1364 self.parseConfiguration () 1400 self.parseConfiguration ()
1365 if CookerFeatures.SEND_SANITYEVENTS in self.featureset: 1401 if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
1366 bb.event.fire(bb.event.SanityCheck(False), self.data) 1402 bb.event.fire(bb.event.SanityCheck(False), self.data)
@@ -1375,9 +1411,13 @@ class BBCooker:
1375 (filelist, masked) = self.collection.collect_bbfiles(self.data, self.event_data) 1411 (filelist, masked) = self.collection.collect_bbfiles(self.data, self.event_data)
1376 1412
1377 self.data.renameVar("__depends", "__base_depends") 1413 self.data.renameVar("__depends", "__base_depends")
1414 for i in self.data.getVar("__base_depends"):
1415 self.wdd = self.configwatcher.add_watch(i[0], self.watchmask, rec=True)
1378 1416
1379 self.parser = CookerParser(self, filelist, masked) 1417 self.parser = CookerParser(self, filelist, masked)
1380 self.state = state.parsing 1418 self.parsecache_valid = True
1419
1420 self.state = state.parsing
1381 1421
1382 if not self.parser.parse_next(): 1422 if not self.parser.parse_next():
1383 collectlog.debug(1, "parsing complete") 1423 collectlog.debug(1, "parsing complete")
@@ -1940,7 +1980,7 @@ class CookerParser(object):
1940 self.skipped += 1 1980 self.skipped += 1
1941 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0]) 1981 self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
1942 self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecache, 1982 self.bb_cache.add_info(virtualfn, info_array, self.cooker.recipecache,
1943 parsed=parsed) 1983 parsed=parsed, watcher = self.cooker.add_filewatch)
1944 return True 1984 return True
1945 1985
1946 def reparse(self, filename): 1986 def reparse(self, filename):
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index 2303f15b9e..25effc2200 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -73,6 +73,11 @@ def update_mtime(f):
73 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME] 73 __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
74 return __mtime_cache[f] 74 return __mtime_cache[f]
75 75
76def update_cache(f):
77 if f in __mtime_cache:
78 logger.debug(1, "Updating mtime cache for %s" % f)
79 update_mtime(f)
80
76def mark_dependency(d, f): 81def mark_dependency(d, f):
77 if f.startswith('./'): 82 if f.startswith('./'):
78 f = "%s/%s" % (os.getcwd(), f[2:]) 83 f = "%s/%s" % (os.getcwd(), f[2:])