summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-11-29 17:47:42 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-12-07 10:42:22 +0000
commit083365143e844827599d9b0a78204b3a05df460c (patch)
tree5862275136012a1f932cee8644f67fbdcd2971c6 /bitbake/lib/bb
parent1b32c6ed025745cb06b7c28ca0fe9e416ce7abfa (diff)
downloadpoky-083365143e844827599d9b0a78204b3a05df460c.tar.gz
bitbake: cooker process: fire heartbeat event at regular time intervals
The intended usage is for recording current system statistics from /proc in buildstats.bbclass during a build and for improving the BB_DISKMON_DIRS implementation. All other existing hooks are less suitable because they trigger at unpredictable rates: too often can be handled by doing rate-limiting in the event handler, but not often enough (for example, when there is only one long-running task) cannot because the handler does not get called at all. The implementation of the new heartbeat event hooks into the cooker process event queue. The process already wakes up every 0.1s, which is often enough for the intentionally coarse 1s delay between heartbeats. That value was chosen to keep the overhead low while still being frequent enough for the intended usage. If necessary, BB_HEARTBEAT_EVENT can be set to a float specifying the delay in seconds between these heartbeat events. (Bitbake rev: 7cf22ea057d28c54bd98dc1ab7a43402a29ff1f5) Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/event.py10
-rw-r--r--bitbake/lib/bb/server/process.py25
-rw-r--r--bitbake/lib/bb/ui/knotty.py1
-rw-r--r--bitbake/lib/bb/ui/toasterui.py3
4 files changed, 39 insertions, 0 deletions
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 6f1cb101fc..cacbac8f56 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -48,6 +48,16 @@ class Event(object):
48 def __init__(self): 48 def __init__(self):
49 self.pid = worker_pid 49 self.pid = worker_pid
50 50
51
52class HeartbeatEvent(Event):
53 """Triggered at regular time intervals of 10 seconds. Other events can fire much more often
54 (runQueueTaskStarted when there are many short tasks) or not at all for long periods
55 of time (again runQueueTaskStarted, when there is just one long-running task), so this
56 event is more suitable for doing some task-independent work occassionally."""
57 def __init__(self, time):
58 Event.__init__(self)
59 self.time = time
60
51Registered = 10 61Registered = 10
52AlreadyRegistered = 14 62AlreadyRegistered = 14
53 63
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index 982fcf71c3..1654faf92b 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -92,6 +92,8 @@ class ProcessServer(Process, BaseImplServer):
92 self.event = EventAdapter(event_queue) 92 self.event = EventAdapter(event_queue)
93 self.featurelist = featurelist 93 self.featurelist = featurelist
94 self.quit = False 94 self.quit = False
95 self.heartbeat_seconds = 1 # default, BB_HEARTBEAT_EVENT will be checked once we have a datastore.
96 self.next_heartbeat = time.time()
95 97
96 self.quitin, self.quitout = Pipe() 98 self.quitin, self.quitout = Pipe()
97 self.event_handle = multiprocessing.Value("i") 99 self.event_handle = multiprocessing.Value("i")
@@ -101,6 +103,14 @@ class ProcessServer(Process, BaseImplServer):
101 self.event_queue.put(event) 103 self.event_queue.put(event)
102 self.event_handle.value = bb.event.register_UIHhandler(self, True) 104 self.event_handle.value = bb.event.register_UIHhandler(self, True)
103 105
106 heartbeat_event = self.cooker.data.getVar('BB_HEARTBEAT_EVENT', True)
107 if heartbeat_event:
108 try:
109 self.heartbeat_seconds = float(heartbeat_event)
110 except:
111 # Throwing an exception here causes bitbake to hang.
112 # Just warn about the invalid setting and continue
113 bb.warn('Ignoring invalid BB_HEARTBEAT_EVENT=%s, must be a float specifying seconds.' % heartbeat_event)
104 bb.cooker.server_main(self.cooker, self.main) 114 bb.cooker.server_main(self.cooker, self.main)
105 115
106 def main(self): 116 def main(self):
@@ -160,6 +170,21 @@ class ProcessServer(Process, BaseImplServer):
160 del self._idlefuns[function] 170 del self._idlefuns[function]
161 self.quit = True 171 self.quit = True
162 172
173 # Create new heartbeat event?
174 now = time.time()
175 if now >= self.next_heartbeat:
176 # We might have missed heartbeats. Just trigger once in
177 # that case and continue after the usual delay.
178 self.next_heartbeat += self.heartbeat_seconds
179 if self.next_heartbeat <= now:
180 self.next_heartbeat = now + self.heartbeat_seconds
181 heartbeat = bb.event.HeartbeatEvent(now)
182 bb.event.fire(heartbeat, self.cooker.data)
183 if nextsleep and now + nextsleep > self.next_heartbeat:
184 # Shorten timeout so that we we wake up in time for
185 # the heartbeat.
186 nextsleep = self.next_heartbeat - now
187
163 if nextsleep is not None: 188 if nextsleep is not None:
164 select.select(fds,[],[],nextsleep) 189 select.select(fds,[],[],nextsleep)
165 190
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 948f52769d..48e1223c6b 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -647,6 +647,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
647 bb.event.OperationCompleted, 647 bb.event.OperationCompleted,
648 bb.event.OperationProgress, 648 bb.event.OperationProgress,
649 bb.event.DiskFull, 649 bb.event.DiskFull,
650 bb.event.HeartbeatEvent,
650 bb.build.TaskProgress)): 651 bb.build.TaskProgress)):
651 continue 652 continue
652 653
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index b1b3684a82..17299026ab 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -236,6 +236,9 @@ def main(server, eventHandler, params):
236 # pylint: disable=protected-access 236 # pylint: disable=protected-access
237 # the code will look into the protected variables of the event; no easy way around this 237 # the code will look into the protected variables of the event; no easy way around this
238 238
239 if isinstance(event, bb.event.HeartbeatEvent):
240 continue
241
239 if isinstance(event, bb.event.ParseStarted): 242 if isinstance(event, bb.event.ParseStarted):
240 if not (build_log and build_log_file_path): 243 if not (build_log and build_log_file_path):
241 build_log, build_log_file_path = _open_build_log(log_dir) 244 build_log, build_log_file_path = _open_build_log(log_dir)