summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2013-05-20 23:00:31 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-05-22 12:10:14 +0100
commit26d19996a3d45e2f4b8f10516e45e10778929f6d (patch)
tree1c81b0e5b2d565c1a242eac9a175d1f141fce60d
parentf0930c8d63cc9fd8ead759b3230ab708fffa6ed4 (diff)
downloadpoky-26d19996a3d45e2f4b8f10516e45e10778929f6d.tar.gz
bitbake: cooker: Split configuration parsing code into cookerdata
In order to have a memory resident bitbake and to allow task execution, we need to be able to rebuild the base configuration without a cooker. This moves the code into its own class so it can be built independently. The interface is less than ideal here but I didn't want to add parsing methods a subclassed DataSmart, at least until we've experimented further with this code and are certain that makes sense. At the very least, the methods are ugly and need cleaning up. Spliting the code out seems to be the right thing to do though and should unblock various activities on BitBake so I believe this code is a step in the right direction. Based on a patch from Alexandru Damian <alexandru.damian@intel.com> (Bitbake rev: 22a0b3cf73d2689db0c118b37aa7492632f8b0a7) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py127
-rw-r--r--bitbake/lib/bb/cookerdata.py130
2 files changed, 139 insertions, 118 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index ca544558f0..daf37ebd45 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -93,15 +93,6 @@ class BBCooker:
93 93
94 self.configuration = configuration 94 self.configuration = configuration
95 95
96 # Keep a datastore of the initial environment variables and their
97 # values from when BitBake was launched to enable child processes
98 # to use environment variables which have been cleaned from the
99 # BitBake processes env
100 self.savedenv = bb.data.init()
101 savedenv = configuration.params.environment
102 for k in savedenv:
103 self.savedenv.setVar(k, savedenv[k])
104
105 self.caches_array = [] 96 self.caches_array = []
106 # Currently, only Image Creator hob ui needs extra cache. 97 # Currently, only Image Creator hob ui needs extra cache.
107 # So, we save Extra Cache class name and container file 98 # So, we save Extra Cache class name and container file
@@ -170,16 +161,12 @@ class BBCooker:
170 self.parser = None 161 self.parser = None
171 162
172 def initConfigurationData(self): 163 def initConfigurationData(self):
173 self.configuration.data = bb.data.init() 164 worker = False
174 if self.configuration.tracking:
175 self.configuration.data.enableTracking()
176
177 if not self.configuration.server_register_idlecallback: 165 if not self.configuration.server_register_idlecallback:
178 self.configuration.data.setVar("BB_WORKERCONTEXT", "1") 166 worker = True
179 167
180 filtered_keys = bb.utils.approved_variables() 168 self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration.params, worker)
181 bb.data.inheritFromOS(self.configuration.data, self.savedenv, filtered_keys) 169 self.configuration.data = self.databuilder.data
182 self.configuration.data.setVar("BB_ORIGENV", self.savedenv)
183 170
184 def enableDataTracking(self): 171 def enableDataTracking(self):
185 self.configuration.data.enableTracking() 172 self.configuration.data.enableTracking()
@@ -189,15 +176,9 @@ class BBCooker:
189 176
190 def loadConfigurationData(self): 177 def loadConfigurationData(self):
191 self.initConfigurationData() 178 self.initConfigurationData()
192 179 self.databuilder.parseBaseConfiguration()
193 try: 180 self.configuration.data = self.databuilder.data
194 self.parseConfigurationFiles(self.configuration.prefile, 181 self.configuration.data_hash = self.databuilder.data_hash
195 self.configuration.postfile)
196 except SyntaxError:
197 sys.exit(1)
198 except Exception:
199 logger.exception("Error parsing configuration files")
200 sys.exit(1)
201 182
202 def saveConfigurationVar(self, var, val, default_file): 183 def saveConfigurationVar(self, var, val, default_file):
203 184
@@ -260,7 +241,7 @@ class BBCooker:
260 self.configuration.data.varhistory.del_var_history(var) 241 self.configuration.data.varhistory.del_var_history(var)
261 242
262 #add var to the end of default_file 243 #add var to the end of default_file
263 default_file = self._findConfigFile(default_file) 244 default_file = bb.cookerdata.findConfigFile(default_file)
264 245
265 with open(default_file, 'r') as f: 246 with open(default_file, 'r') as f:
266 contents = f.readlines() 247 contents = f.readlines()
@@ -700,7 +681,7 @@ class BBCooker:
700 Find the location on disk of configfile and if it exists and was parsed by BitBake 681 Find the location on disk of configfile and if it exists and was parsed by BitBake
701 emit the ConfigFilePathFound event with the path to the file. 682 emit the ConfigFilePathFound event with the path to the file.
702 """ 683 """
703 path = self._findConfigFile(configfile) 684 path = bb.cookerdata.findConfigFile(configfile)
704 if not path: 685 if not path:
705 return 686 return
706 687
@@ -835,76 +816,6 @@ class BBCooker:
835 else: 816 else:
836 shell.start( self ) 817 shell.start( self )
837 818
838 def _findConfigFile(self, configfile):
839 path = os.getcwd()
840 while path != "/":
841 confpath = os.path.join(path, "conf", configfile)
842 if os.path.exists(confpath):
843 return confpath
844
845 path, _ = os.path.split(path)
846 return None
847
848 def _findLayerConf(self):
849 return self._findConfigFile("bblayers.conf")
850
851 def parseConfigurationFiles(self, prefiles, postfiles):
852 data = self.configuration.data
853 bb.parse.init_parser(data)
854
855 # Parse files for loading *before* bitbake.conf and any includes
856 for f in prefiles:
857 data = _parse(f, data)
858
859 layerconf = self._findLayerConf()
860 if layerconf:
861 parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
862 data = _parse(layerconf, data)
863
864 layers = (data.getVar('BBLAYERS', True) or "").split()
865
866 data = bb.data.createCopy(data)
867 for layer in layers:
868 parselog.debug(2, "Adding layer %s", layer)
869 data.setVar('LAYERDIR', layer)
870 data = _parse(os.path.join(layer, "conf", "layer.conf"), data)
871 data.expandVarref('LAYERDIR')
872
873 data.delVar('LAYERDIR')
874
875 if not data.getVar("BBPATH", True):
876 raise SystemExit("The BBPATH variable is not set")
877
878 data = _parse(os.path.join("conf", "bitbake.conf"), data)
879
880 # Parse files for loading *after* bitbake.conf and any includes
881 for p in postfiles:
882 data = _parse(p, data)
883
884 # Handle any INHERITs and inherit the base class
885 bbclasses = ["base"] + (data.getVar('INHERIT', True) or "").split()
886 for bbclass in bbclasses:
887 data = _inherit(bbclass, data)
888
889 # Nomally we only register event handlers at the end of parsing .bb files
890 # We register any handlers we've found so far here...
891 for var in data.getVar('__BBHANDLERS') or []:
892 bb.event.register(var, data.getVar(var))
893
894 if data.getVar("BB_WORKERCONTEXT", False) is None:
895 bb.fetch.fetcher_init(data)
896 bb.codeparser.parser_cache_init(data)
897 bb.event.fire(bb.event.ConfigParsed(), data)
898
899 if data.getVar("BB_INVALIDCONF") is True:
900 data.setVar("BB_INVALIDCONF", False)
901 self.parseConfigurationFiles(self.configuration.prefile,
902 self.configuration.postfile)
903 else:
904 bb.parse.init_parser(data)
905 data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
906 self.configuration.data = data
907 self.configuration.data_hash = data.get_hash()
908 819
909 def handleCollections( self, collections ): 820 def handleCollections( self, collections ):
910 """Handle collections""" 821 """Handle collections"""
@@ -1501,26 +1412,6 @@ class CookerCollectFiles(object):
1501 1412
1502 return priorities 1413 return priorities
1503 1414
1504def catch_parse_error(func):
1505 """Exception handling bits for our parsing"""
1506 @wraps(func)
1507 def wrapped(fn, *args):
1508 try:
1509 return func(fn, *args)
1510 except (IOError, bb.parse.ParseError, bb.data_smart.ExpansionError) as exc:
1511 parselog.critical("Unable to parse %s: %s" % (fn, exc))
1512 sys.exit(1)
1513 return wrapped
1514
1515@catch_parse_error
1516def _parse(fn, data, include=True):
1517 return bb.parse.handle(fn, data, include)
1518
1519@catch_parse_error
1520def _inherit(bbclass, data):
1521 bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
1522 return data
1523
1524class ParsingFailure(Exception): 1415class ParsingFailure(Exception):
1525 def __init__(self, realexception, recipe): 1416 def __init__(self, realexception, recipe):
1526 self.realexception = realexception 1417 self.realexception = realexception
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 3c2469ef87..0b436b37e6 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -126,3 +126,133 @@ class CookerConfiguration(object):
126 def setServerRegIdleCallback(self, srcb): 126 def setServerRegIdleCallback(self, srcb):
127 self.server_register_idlecallback = srcb 127 self.server_register_idlecallback = srcb
128 128
129def catch_parse_error(func):
130 """Exception handling bits for our parsing"""
131 @wraps(func)
132 def wrapped(fn, *args):
133 try:
134 return func(fn, *args)
135 except (IOError, bb.parse.ParseError, bb.data_smart.ExpansionError) as exc:
136 parselog.critical("Unable to parse %s: %s" % (fn, exc))
137 sys.exit(1)
138 return wrapped
139
140@catch_parse_error
141def _parse(fn, data, include=True):
142 return bb.parse.handle(fn, data, include)
143
144@catch_parse_error
145def _inherit(bbclass, data):
146 bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
147 return data
148
149def findConfigFile(configfile):
150 path = os.getcwd()
151 while path != "/":
152 confpath = os.path.join(path, "conf", configfile)
153 if os.path.exists(confpath):
154 return confpath
155
156 path, _ = os.path.split(path)
157 return None
158
159class CookerDataBuilder(object):
160
161 def __init__(self, params, worker = False):
162
163 self.prefiles = params.prefile
164 self.postfiles = params.postfile
165 self.tracking = params.tracking
166
167 self.data = bb.data.init()
168 if self.tracking:
169 self.data.enableTracking()
170
171 # Keep a datastore of the initial environment variables and their
172 # values from when BitBake was launched to enable child processes
173 # to use environment variables which have been cleaned from the
174 # BitBake processes env
175 self.savedenv = bb.data.init()
176 savedenv = params.environment
177 for k in savedenv:
178 self.savedenv.setVar(k, savedenv[k])
179
180 filtered_keys = bb.utils.approved_variables()
181 bb.data.inheritFromOS(self.data, self.savedenv, filtered_keys)
182 self.data.setVar("BB_ORIGENV", self.savedenv)
183
184 if worker:
185 self.data.setVar("BB_WORKERCONTEXT", "1")
186
187 def parseBaseConfiguration(self):
188 try:
189 self.parseConfigurationFiles(self.prefiles, self.postfiles)
190 except SyntaxError:
191 sys.exit(1)
192 except Exception:
193 logger.exception("Error parsing configuration files")
194 sys.exit(1)
195
196 def _findLayerConf(self):
197 return findConfigFile("bblayers.conf")
198
199 def parseConfigurationFiles(self, prefiles, postfiles):
200 data = self.data
201 bb.parse.init_parser(data)
202
203 # Parse files for loading *before* bitbake.conf and any includes
204 for f in prefiles:
205 data = _parse(f, data)
206
207 layerconf = self._findLayerConf()
208 if layerconf:
209 parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
210 data = _parse(layerconf, data)
211
212 layers = (data.getVar('BBLAYERS', True) or "").split()
213
214 data = bb.data.createCopy(data)
215 for layer in layers:
216 parselog.debug(2, "Adding layer %s", layer)
217 data.setVar('LAYERDIR', layer)
218 data = _parse(os.path.join(layer, "conf", "layer.conf"), data)
219 data.expandVarref('LAYERDIR')
220
221 data.delVar('LAYERDIR')
222
223 if not data.getVar("BBPATH", True):
224 raise SystemExit("The BBPATH variable is not set")
225
226 data = _parse(os.path.join("conf", "bitbake.conf"), data)
227
228 # Parse files for loading *after* bitbake.conf and any includes
229 for p in postfiles:
230 data = _parse(p, data)
231
232 # Handle any INHERITs and inherit the base class
233 bbclasses = ["base"] + (data.getVar('INHERIT', True) or "").split()
234 for bbclass in bbclasses:
235 data = _inherit(bbclass, data)
236
237 # Nomally we only register event handlers at the end of parsing .bb files
238 # We register any handlers we've found so far here...
239 for var in data.getVar('__BBHANDLERS') or []:
240 bb.event.register(var, data.getVar(var))
241
242 if data.getVar("BB_WORKERCONTEXT", False) is None:
243 bb.fetch.fetcher_init(data)
244 bb.codeparser.parser_cache_init(data)
245 bb.event.fire(bb.event.ConfigParsed(), data)
246
247 if data.getVar("BB_INVALIDCONF") is True:
248 data.setVar("BB_INVALIDCONF", False)
249 self.parseConfigurationFiles(self.prefiles, self.postfiles)
250 return
251
252 bb.parse.init_parser(data)
253 data.setVar('BBINCLUDED',bb.parse.get_file_depends(data))
254 self.data = data
255 self.data_hash = data.get_hash()
256
257
258