summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2013-09-18 13:15:47 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-09-22 12:19:43 +0100
commitba83eb315d1462e096e2d6682d246cbbe260b4c0 (patch)
tree7b0bf6ac84989275df250d76e1d1097fdd9bd0aa /bitbake/lib/bb
parent87e86d4fd3a22627bf5a7eff20d8551cb1e67452 (diff)
downloadpoky-ba83eb315d1462e096e2d6682d246cbbe260b4c0.tar.gz
bitbake: bitbake: cooker,xmlrpc,servers: implement CookerFeatures
Implementing feature set selection that allows a client to enable specific features in the server at connection time. Only enabling of features is supported, as there is no way to safely remove data loaded into the cooker. Once enabled, a feature will remain enabled for the life of the cooker. Client-server connection now supports specifying the feature set required by the client. This is implemented in the Process server using a managed proxy list, so the server cooker will now load dynamically needed features based on what client connects to it. In the XMLRPC server the feature set is requested by using a parameter for registerUIHandler function. This allows observer-only clients to also specify features for the server. The server code configuration now is completly separated from the client code. All hardcoding of client knowledge is removed from the server. The extra_caches is removed as the client can now specify the caches it needs using the feature. The UI modules now need to specify the desired featureSet. HOB is modified to conform to the featureSet specification. The only feature available is CookerFeatures.HOB_EXTRA_CACHES which forces loading the bb.cache_extra:HobRecipeInfo class. (Bitbake rev: 98e594837aab89ea042cfa9f3740d20a661b14e2) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/cooker.py34
-rw-r--r--bitbake/lib/bb/cookerdata.py1
-rw-r--r--bitbake/lib/bb/server/__init__.py2
-rw-r--r--bitbake/lib/bb/server/process.py23
-rw-r--r--bitbake/lib/bb/server/xmlrpc.py29
-rwxr-xr-xbitbake/lib/bb/ui/hob.py2
-rw-r--r--bitbake/lib/bb/ui/knotty.py3
-rw-r--r--bitbake/lib/bb/ui/uievent.py7
8 files changed, 82 insertions, 19 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 1d5e5f66c2..8776e187ea 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -79,6 +79,29 @@ class SkippedPackage:
79 elif reason: 79 elif reason:
80 self.skipreason = reason 80 self.skipreason = reason
81 81
82
83class CookerFeatures(object):
84 _feature_list = [HOB_EXTRA_CACHES] = range(1)
85
86 def __init__(self):
87 self._features=set()
88
89 def setFeature(self, f):
90 # validate we got a request for a feature we support
91 if f not in CookerFeatures._feature_list:
92 return
93 self._features.add(f)
94
95 def __contains__(self, f):
96 return f in self._features
97
98 def __iter__(self):
99 return self._features.__iter__()
100
101 def next(self):
102 return self._features.next()
103
104
82#============================================================================# 105#============================================================================#
83# BBCooker 106# BBCooker
84#============================================================================# 107#============================================================================#
@@ -90,6 +113,7 @@ class BBCooker:
90 def __init__(self, configuration): 113 def __init__(self, configuration):
91 self.recipecache = None 114 self.recipecache = None
92 self.skiplist = {} 115 self.skiplist = {}
116 self.featureset = CookerFeatures()
93 117
94 self.configuration = configuration 118 self.configuration = configuration
95 119
@@ -122,7 +146,13 @@ class BBCooker:
122 self.state = state.initial 146 self.state = state.initial
123 147
124 self.caches_array = [] 148 self.caches_array = []
125 caches_name_array = ['bb.cache:CoreRecipeInfo'] + self.configuration.extra_caches 149
150 all_extra_cache_names = []
151 # We hardcode all known cache types in a single place, here.
152 if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
153 all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
154
155 caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
126 156
127 # At least CoreRecipeInfo will be loaded, so caches_array will never be empty! 157 # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
128 # This is the entry point, no further check needed! 158 # This is the entry point, no further check needed!
@@ -130,7 +160,7 @@ class BBCooker:
130 try: 160 try:
131 module_name, cache_name = var.split(':') 161 module_name, cache_name = var.split(':')
132 module = __import__(module_name, fromlist=(cache_name,)) 162 module = __import__(module_name, fromlist=(cache_name,))
133 self.caches_array.append(getattr(module, cache_name)) 163 self.caches_array.append(getattr(module, cache_name))
134 except ImportError as exc: 164 except ImportError as exc:
135 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc)) 165 logger.critical("Unable to import extra RecipeInfo '%s' from '%s': %s" % (cache_name, module_name, exc))
136 sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name) 166 sys.exit("FATAL: Failed to import extra cache class '%s'." % cache_name)
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 0b278b178e..e640ed0f35 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -127,7 +127,6 @@ class CookerConfiguration(object):
127 self.dump_signatures = False 127 self.dump_signatures = False
128 self.dry_run = False 128 self.dry_run = False
129 self.tracking = False 129 self.tracking = False
130 self.extra_caches = []
131 130
132 self.env = {} 131 self.env = {}
133 132
diff --git a/bitbake/lib/bb/server/__init__.py b/bitbake/lib/bb/server/__init__.py
index 2e1c619b54..da5e480740 100644
--- a/bitbake/lib/bb/server/__init__.py
+++ b/bitbake/lib/bb/server/__init__.py
@@ -89,7 +89,7 @@ class BitBakeBaseServer(object):
89 def detach(self): 89 def detach(self):
90 return 90 return
91 91
92 def establishConnection(self): 92 def establishConnection(self, featureset):
93 raise "Must redefine the %s.establishConnection()" % self.__class__.__name__ 93 raise "Must redefine the %s.establishConnection()" % self.__class__.__name__
94 94
95 def endSession(self): 95 def endSession(self):
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index e45e0c2f6d..aa072020af 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -31,7 +31,7 @@ import sys
31import time 31import time
32import select 32import select
33from Queue import Empty 33from Queue import Empty
34from multiprocessing import Event, Process, util, Queue, Pipe, queues 34from multiprocessing import Event, Process, util, Queue, Pipe, queues, Manager
35 35
36from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer 36from . import BitBakeBaseServer, BitBakeBaseServerConnection, BaseImplServer
37 37
@@ -78,12 +78,13 @@ class ProcessServer(Process, BaseImplServer):
78 profile_filename = "profile.log" 78 profile_filename = "profile.log"
79 profile_processed_filename = "profile.log.processed" 79 profile_processed_filename = "profile.log.processed"
80 80
81 def __init__(self, command_channel, event_queue): 81 def __init__(self, command_channel, event_queue, featurelist):
82 BaseImplServer.__init__(self) 82 BaseImplServer.__init__(self)
83 Process.__init__(self) 83 Process.__init__(self, args=(featurelist))
84 self.command_channel = command_channel 84 self.command_channel = command_channel
85 self.event_queue = event_queue 85 self.event_queue = event_queue
86 self.event = EventAdapter(event_queue) 86 self.event = EventAdapter(event_queue)
87 self.featurelist = featurelist
87 self.quit = False 88 self.quit = False
88 89
89 self.keep_running = Event() 90 self.keep_running = Event()
@@ -94,6 +95,14 @@ class ProcessServer(Process, BaseImplServer):
94 for event in bb.event.ui_queue: 95 for event in bb.event.ui_queue:
95 self.event_queue.put(event) 96 self.event_queue.put(event)
96 self.event_handle.value = bb.event.register_UIHhandler(self) 97 self.event_handle.value = bb.event.register_UIHhandler(self)
98
99 # process any feature changes based on what UI requested
100 original_featureset = list(self.cooker.featureset)
101 while len(self.featurelist)> 0:
102 self.cooker.featureset.setFeature(self.featurelist.pop())
103 if (original_featureset != list(self.cooker.featureset)):
104 self.cooker.reset()
105
97 bb.cooker.server_main(self.cooker, self.main) 106 bb.cooker.server_main(self.cooker, self.main)
98 107
99 def main(self): 108 def main(self):
@@ -198,13 +207,17 @@ class BitBakeServer(BitBakeBaseServer):
198 # 207 #
199 self.ui_channel, self.server_channel = Pipe() 208 self.ui_channel, self.server_channel = Pipe()
200 self.event_queue = ProcessEventQueue(0) 209 self.event_queue = ProcessEventQueue(0)
201 self.serverImpl = ProcessServer(self.server_channel, self.event_queue) 210 manager = Manager()
211 self.featurelist = manager.list()
212 self.serverImpl = ProcessServer(self.server_channel, self.event_queue, self.featurelist)
202 213
203 def detach(self): 214 def detach(self):
204 self.serverImpl.start() 215 self.serverImpl.start()
205 return 216 return
206 217
207 def establishConnection(self): 218 def establishConnection(self, featureset):
219 for f in featureset:
220 self.featurelist.append(f)
208 self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue) 221 self.connection = BitBakeProcessServerConnection(self.serverImpl, self.ui_channel, self.event_queue)
209 signal.signal(signal.SIGTERM, lambda i, s: self.connection.terminate()) 222 signal.signal(signal.SIGTERM, lambda i, s: self.connection.terminate())
210 return self.connection 223 return self.connection
diff --git a/bitbake/lib/bb/server/xmlrpc.py b/bitbake/lib/bb/server/xmlrpc.py
index 8326623520..389327a60f 100644
--- a/bitbake/lib/bb/server/xmlrpc.py
+++ b/bitbake/lib/bb/server/xmlrpc.py
@@ -89,12 +89,23 @@ class BitBakeServerCommands():
89 self.server = server 89 self.server = server
90 self.has_client = False 90 self.has_client = False
91 91
92 def registerEventHandler(self, host, port): 92 def registerEventHandler(self, host, port, featureset = []):
93 """ 93 """
94 Register a remote UI Event Handler 94 Register a remote UI Event Handler
95 """ 95 """
96 s, t = _create_server(host, port) 96 s, t = _create_server(host, port)
97 97
98 # we don't allow connections if the cooker is running
99 if (self.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
100 return None
101
102 original_featureset = list(self.cooker.featureset)
103 for f in featureset:
104 self.cooker.featureset.setFeature(f)
105
106 if (original_featureset != list(self.cooker.featureset)):
107 self.cooker.reset()
108
98 self.event_handle = bb.event.register_UIHhandler(s) 109 self.event_handle = bb.event.register_UIHhandler(s)
99 return self.event_handle 110 return self.event_handle
100 111
@@ -263,11 +274,12 @@ class XMLRPCServer(SimpleXMLRPCServer, BaseImplServer):
263 self.connection_token = token 274 self.connection_token = token
264 275
265class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection): 276class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
266 def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False): 277 def __init__(self, serverImpl, clientinfo=("localhost", 0), observer_only = False, featureset = []):
267 self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port) 278 self.connection, self.transport = _create_server(serverImpl.host, serverImpl.port)
268 self.clientinfo = clientinfo 279 self.clientinfo = clientinfo
269 self.serverImpl = serverImpl 280 self.serverImpl = serverImpl
270 self.observer_only = observer_only 281 self.observer_only = observer_only
282 self.featureset = featureset
271 283
272 def connect(self): 284 def connect(self):
273 if not self.observer_only: 285 if not self.observer_only:
@@ -277,7 +289,8 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
277 if token is None: 289 if token is None:
278 return None 290 return None
279 self.transport.set_connection_token(token) 291 self.transport.set_connection_token(token)
280 self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo) 292
293 self.events = uievent.BBUIEventQueue(self.connection, self.clientinfo, self.featureset)
281 for event in bb.event.ui_queue: 294 for event in bb.event.ui_queue:
282 self.events.queue_event(event) 295 self.events.queue_event(event)
283 return self 296 return self
@@ -301,14 +314,15 @@ class BitBakeXMLRPCServerConnection(BitBakeBaseServerConnection):
301 314
302class BitBakeServer(BitBakeBaseServer): 315class BitBakeServer(BitBakeBaseServer):
303 def initServer(self, interface = ("localhost", 0)): 316 def initServer(self, interface = ("localhost", 0)):
317 self.interface = interface
304 self.serverImpl = XMLRPCServer(interface) 318 self.serverImpl = XMLRPCServer(interface)
305 319
306 def detach(self): 320 def detach(self):
307 daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log") 321 daemonize.createDaemon(self.serverImpl.serve_forever, "bitbake-cookerdaemon.log")
308 del self.cooker 322 del self.cooker
309 323
310 def establishConnection(self): 324 def establishConnection(self, featureset):
311 self.connection = BitBakeXMLRPCServerConnection(self.serverImpl) 325 self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, self.interface, False, featureset)
312 return self.connection.connect() 326 return self.connection.connect()
313 327
314 def set_connection_token(self, token): 328 def set_connection_token(self, token):
@@ -318,12 +332,13 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
318 332
319 def __init__(self, observer_only = False): 333 def __init__(self, observer_only = False):
320 self.observer_only = observer_only 334 self.observer_only = observer_only
335 # if we need extra caches, just tell the server to load them all
321 pass 336 pass
322 337
323 def saveConnectionDetails(self, remote): 338 def saveConnectionDetails(self, remote):
324 self.remote = remote 339 self.remote = remote
325 340
326 def establishConnection(self): 341 def establishConnection(self, featureset):
327 # The format of "remote" must be "server:port" 342 # The format of "remote" must be "server:port"
328 try: 343 try:
329 [host, port] = self.remote.split(":") 344 [host, port] = self.remote.split(":")
@@ -340,7 +355,7 @@ class BitBakeXMLRPCClient(BitBakeBaseServer):
340 except: 355 except:
341 return None 356 return None
342 self.serverImpl = XMLRPCProxyServer(host, port) 357 self.serverImpl = XMLRPCProxyServer(host, port)
343 self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only) 358 self.connection = BitBakeXMLRPCServerConnection(self.serverImpl, (ip, 0), self.observer_only, featureset)
344 return self.connection.connect() 359 return self.connection.connect()
345 360
346 def endSession(self): 361 def endSession(self):
diff --git a/bitbake/lib/bb/ui/hob.py b/bitbake/lib/bb/ui/hob.py
index bc5ee1434a..154a3b3b4a 100755
--- a/bitbake/lib/bb/ui/hob.py
+++ b/bitbake/lib/bb/ui/hob.py
@@ -46,7 +46,7 @@ from bb.ui.crumbs.hoblistmodel import RecipeListModel, PackageListModel
46from bb.ui.crumbs.hobeventhandler import HobHandler 46from bb.ui.crumbs.hobeventhandler import HobHandler
47from bb.ui.crumbs.builder import Builder 47from bb.ui.crumbs.builder import Builder
48 48
49extraCaches = ['bb.cache_extra:HobRecipeInfo'] 49featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES]
50 50
51def event_handle_idle_func(eventHandler, hobHandler): 51def event_handle_idle_func(eventHandler, hobHandler):
52 # Consume as many messages as we can in the time available to us 52 # Consume as many messages as we can in the time available to us
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 35590a2a79..0211b50aba 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -472,6 +472,9 @@ def main(server, eventHandler, params, tf = TerminalFilter):
472 event.taskid, event.taskstring, event.exitcode) 472 event.taskid, event.taskstring, event.exitcode)
473 continue 473 continue
474 474
475 if isinstance(event, bb.event.DepTreeGenerated):
476 continue
477
475 # ignore 478 # ignore
476 if isinstance(event, (bb.event.BuildBase, 479 if isinstance(event, (bb.event.BuildBase,
477 bb.event.StampUpdate, 480 bb.event.StampUpdate,
diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py
index 038029fcfa..2133b44477 100644
--- a/bitbake/lib/bb/ui/uievent.py
+++ b/bitbake/lib/bb/ui/uievent.py
@@ -28,7 +28,7 @@ import socket, threading, pickle
28from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler 28from SimpleXMLRPCServer import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
29 29
30class BBUIEventQueue: 30class BBUIEventQueue:
31 def __init__(self, BBServer, clientinfo=("localhost, 0")): 31 def __init__(self, BBServer, clientinfo=("localhost, 0"), featureset=[]):
32 32
33 self.eventQueue = [] 33 self.eventQueue = []
34 self.eventQueueLock = threading.Lock() 34 self.eventQueueLock = threading.Lock()
@@ -44,7 +44,10 @@ class BBUIEventQueue:
44 server.register_function( self.send_event, "event.sendpickle" ) 44 server.register_function( self.send_event, "event.sendpickle" )
45 server.socket.settimeout(1) 45 server.socket.settimeout(1)
46 46
47 self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port) 47 self.EventHandle = self.BBServer.registerEventHandler(self.host, self.port, featureset)
48
49 if (self.EventHandle == None):
50 bb.fatal("Could not register UI event handler")
48 51
49 self.server = server 52 self.server = server
50 53