summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2015-12-15 17:41:12 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-12-18 12:18:19 +0000
commit0019edc8180e859544da4820fa390b23d975fe08 (patch)
tree16ea6385b9faae8a718dbe5dcedaacea232e938e /bitbake
parentb14ccb2367983b2263608b6d618647ecd22bf00d (diff)
downloadpoky-0019edc8180e859544da4820fa390b23d975fe08.tar.gz
bitbake: ast/event/utils: Improve tracebacks to include file and line numbers more correctly
Currently bitbake tracebacks can have places where the line numbers are inaccurate and filenames may be missing. These changes start to try and correct this. The only way I could find to correct line numbers was to compile as a python ast, tweak the line numbers then compile to bytecode. I'm open to better ways of doing this if anyone knows of any. This does mean passing a few more parameters into functions, and putting more data into the data store about functions (i.e. their filenames and line numbers) but the improvement in debugging is more than worthwhile). Before: ---------------- ERROR: Execution of event handler 'run_buildstats' failed Traceback (most recent call last): File "run_buildstats(e)", line 43, in run_buildstats(e=<bb.build.TaskStarted object at 0x7f7b7c57a590>) NameError: global name 'notexist' is not defined ERROR: Build of do_patch failed ERROR: Traceback (most recent call last): File "/media/build1/poky/bitbake/lib/bb/build.py", line 560, in exec_task return _exec_task(fn, task, d, quieterr) File "/media/build1/poky/bitbake/lib/bb/build.py", line 497, in _exec_task event.fire(TaskStarted(task, logfn, flags, localdata), localdata) File "/media/build1/poky/bitbake/lib/bb/event.py", line 170, in fire fire_class_handlers(event, d) File "/media/build1/poky/bitbake/lib/bb/event.py", line 109, in fire_class_handlers execute_handler(name, handler, event, d) File "/media/build1/poky/bitbake/lib/bb/event.py", line 81, in execute_handler ret = handler(event) File "run_buildstats(e)", line 43, in run_buildstats NameError: global name 'notexist' is not defined ---------------- After: ---------------- ERROR: Execution of event handler 'run_buildstats' failed Traceback (most recent call last): File "/media/build1/poky/meta/classes/buildstats.bbclass", line 143, in run_buildstats(e=<bb.build.TaskStarted object at 0x7efe89284e10>): if isinstance(e, bb.build.TaskStarted): > trigger = notexist pn = d.getVar("PN", True) NameError: global name 'notexist' is not defined ERROR: Build of do_package failed ERROR: Traceback (most recent call last): File "/media/build1/poky/bitbake/lib/bb/build.py", line 560, in exec_task return _exec_task(fn, task, d, quieterr) File "/media/build1/poky/bitbake/lib/bb/build.py", line 497, in _exec_task event.fire(TaskStarted(task, logfn, flags, localdata), localdata) File "/media/build1/poky/bitbake/lib/bb/event.py", line 170, in fire fire_class_handlers(event, d) File "/media/build1/poky/bitbake/lib/bb/event.py", line 109, in fire_class_handlers execute_handler(name, handler, event, d) File "/media/build1/poky/bitbake/lib/bb/event.py", line 81, in execute_handler ret = handler(event) File "/media/build1/poky/meta/classes/buildstats.bbclass", line 143, in run_buildstats trigger = notexist NameError: global name 'notexist' is not defined ---------------- (Bitbake rev: 1ff860960919ff6f8097138bc68de85bcb5f88b0) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/cookerdata.py4
-rw-r--r--bitbake/lib/bb/event.py10
-rw-r--r--bitbake/lib/bb/methodpool.py4
-rw-r--r--bitbake/lib/bb/parse/ast.py16
-rw-r--r--bitbake/lib/bb/utils.py12
5 files changed, 32 insertions, 14 deletions
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 671c0cb0e2..b47e7f3230 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -317,7 +317,9 @@ class CookerDataBuilder(object):
317 # Nomally we only register event handlers at the end of parsing .bb files 317 # Nomally we only register event handlers at the end of parsing .bb files
318 # We register any handlers we've found so far here... 318 # We register any handlers we've found so far here...
319 for var in data.getVar('__BBHANDLERS', False) or []: 319 for var in data.getVar('__BBHANDLERS', False) or []:
320 bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask", True) or "").split()) 320 handlerfn = data.getVarFlag(var, "filename", False)
321 handlerln = int(data.getVarFlag(var, "lineno", False))
322 bb.event.register(var, data.getVar(var, False), (data.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
321 323
322 if data.getVar("BB_WORKERCONTEXT", False) is None: 324 if data.getVar("BB_WORKERCONTEXT", False) is None:
323 bb.fetch.fetcher_init(data) 325 bb.fetch.fetcher_init(data)
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index ec25ce77fb..bd2b0a4b05 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -177,7 +177,7 @@ def fire_from_worker(event, d):
177 fire_ui_handlers(event, d) 177 fire_ui_handlers(event, d)
178 178
179noop = lambda _: None 179noop = lambda _: None
180def register(name, handler, mask=None): 180def register(name, handler, mask=None, filename=None, lineno=None):
181 """Register an Event handler""" 181 """Register an Event handler"""
182 182
183 # already registered 183 # already registered
@@ -189,7 +189,13 @@ def register(name, handler, mask=None):
189 if isinstance(handler, basestring): 189 if isinstance(handler, basestring):
190 tmp = "def %s(e):\n%s" % (name, handler) 190 tmp = "def %s(e):\n%s" % (name, handler)
191 try: 191 try:
192 code = compile(tmp, "%s(e)" % name, "exec") 192 import ast
193 if filename is None:
194 filename = "%s(e)" % name
195 code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST)
196 if lineno is not None:
197 ast.increment_lineno(code, lineno-1)
198 code = compile(code, filename, "exec")
193 except SyntaxError: 199 except SyntaxError:
194 logger.error("Unable to register event handler '%s':\n%s", name, 200 logger.error("Unable to register event handler '%s':\n%s", name,
195 ''.join(traceback.format_exc(limit=0))) 201 ''.join(traceback.format_exc(limit=0)))
diff --git a/bitbake/lib/bb/methodpool.py b/bitbake/lib/bb/methodpool.py
index bf2e9f5542..b2ea1a1887 100644
--- a/bitbake/lib/bb/methodpool.py
+++ b/bitbake/lib/bb/methodpool.py
@@ -19,11 +19,11 @@
19 19
20from bb.utils import better_compile, better_exec 20from bb.utils import better_compile, better_exec
21 21
22def insert_method(modulename, code, fn): 22def insert_method(modulename, code, fn, lineno):
23 """ 23 """
24 Add code of a module should be added. The methods 24 Add code of a module should be added. The methods
25 will be simply added, no checking will be done 25 will be simply added, no checking will be done
26 """ 26 """
27 comp = better_compile(code, modulename, fn ) 27 comp = better_compile(code, modulename, fn, lineno=lineno)
28 better_exec(comp, None, code, fn) 28 better_exec(comp, None, code, fn)
29 29
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py
index 11db1801b3..e1bf82fe90 100644
--- a/bitbake/lib/bb/parse/ast.py
+++ b/bitbake/lib/bb/parse/ast.py
@@ -148,17 +148,19 @@ class MethodNode(AstNode):
148 148
149 def eval(self, data): 149 def eval(self, data):
150 text = '\n'.join(self.body) 150 text = '\n'.join(self.body)
151 funcname = self.func_name
151 if self.func_name == "__anonymous": 152 if self.func_name == "__anonymous":
152 funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl))) 153 funcname = ("__anon_%s_%s" % (self.lineno, self.filename.translate(MethodNode.tr_tbl)))
153 text = "def %s(d):\n" % (funcname) + text 154 text = "def %s(d):\n" % (funcname) + text
154 bb.methodpool.insert_method(funcname, text, self.filename) 155 bb.methodpool.insert_method(funcname, text, self.filename, self.lineno - len(self.body))
155 anonfuncs = data.getVar('__BBANONFUNCS', False) or [] 156 anonfuncs = data.getVar('__BBANONFUNCS', False) or []
156 anonfuncs.append(funcname) 157 anonfuncs.append(funcname)
157 data.setVar('__BBANONFUNCS', anonfuncs) 158 data.setVar('__BBANONFUNCS', anonfuncs)
158 data.setVar(funcname, text, parsing=True)
159 else: 159 else:
160 data.setVarFlag(self.func_name, "func", 1) 160 data.setVarFlag(self.func_name, "func", 1)
161 data.setVar(self.func_name, text, parsing=True) 161 data.setVar(funcname, text, parsing=True)
162 data.setVarFlag(funcname, 'filename', self.filename)
163 data.setVarFlag(funcname, 'lineno', str(self.lineno - len(self.body)))
162 164
163class PythonMethodNode(AstNode): 165class PythonMethodNode(AstNode):
164 def __init__(self, filename, lineno, function, modulename, body): 166 def __init__(self, filename, lineno, function, modulename, body):
@@ -172,10 +174,12 @@ class PythonMethodNode(AstNode):
172 # 'this' file. This means we will not parse methods from 174 # 'this' file. This means we will not parse methods from
173 # bb classes twice 175 # bb classes twice
174 text = '\n'.join(self.body) 176 text = '\n'.join(self.body)
175 bb.methodpool.insert_method(self.modulename, text, self.filename) 177 bb.methodpool.insert_method(self.modulename, text, self.filename, self.lineno - len(self.body) - 1)
176 data.setVarFlag(self.function, "func", 1) 178 data.setVarFlag(self.function, "func", 1)
177 data.setVarFlag(self.function, "python", 1) 179 data.setVarFlag(self.function, "python", 1)
178 data.setVar(self.function, text, parsing=True) 180 data.setVar(self.function, text, parsing=True)
181 data.setVarFlag(self.function, 'filename', self.filename)
182 data.setVarFlag(self.function, 'lineno', str(self.lineno - len(self.body) - 1))
179 183
180class MethodFlagsNode(AstNode): 184class MethodFlagsNode(AstNode):
181 def __init__(self, filename, lineno, key, m): 185 def __init__(self, filename, lineno, key, m):
@@ -317,7 +321,9 @@ def finalize(fn, d, variant = None):
317 all_handlers = {} 321 all_handlers = {}
318 for var in d.getVar('__BBHANDLERS', False) or []: 322 for var in d.getVar('__BBHANDLERS', False) or []:
319 # try to add the handler 323 # try to add the handler
320 bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split()) 324 handlerfn = d.getVarFlag(var, "filename", False)
325 handlerln = int(d.getVarFlag(var, "lineno", False))
326 bb.event.register(var, d.getVar(var, False), (d.getVarFlag(var, "eventmask", True) or "").split(), handlerfn, handlerln)
321 327
322 bb.event.fire(bb.event.RecipePreFinalise(fn), d) 328 bb.event.fire(bb.event.RecipePreFinalise(fn), d)
323 329
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 31ec2b7c9a..c5ff903cef 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -33,6 +33,7 @@ import fnmatch
33import traceback 33import traceback
34import errno 34import errno
35import signal 35import signal
36import ast
36from commands import getstatusoutput 37from commands import getstatusoutput
37from contextlib import contextmanager 38from contextlib import contextmanager
38from ctypes import cdll 39from ctypes import cdll
@@ -291,19 +292,22 @@ def _print_trace(body, line):
291 error.append(' %.4d:%s' % (i, body[i-1].rstrip())) 292 error.append(' %.4d:%s' % (i, body[i-1].rstrip()))
292 return error 293 return error
293 294
294def better_compile(text, file, realfile, mode = "exec"): 295def better_compile(text, file, realfile, mode = "exec", lineno = None):
295 """ 296 """
296 A better compile method. This method 297 A better compile method. This method
297 will print the offending lines. 298 will print the offending lines.
298 """ 299 """
299 try: 300 try:
300 return compile(text, file, mode) 301 code = compile(text, realfile, mode, ast.PyCF_ONLY_AST)
302 if lineno is not None:
303 ast.increment_lineno(code, lineno)
304 return compile(code, realfile, mode)
301 except Exception as e: 305 except Exception as e:
302 error = [] 306 error = []
303 # split the text into lines again 307 # split the text into lines again
304 body = text.split('\n') 308 body = text.split('\n')
305 error.append("Error in compiling python function in %s:\n" % realfile) 309 error.append("Error in compiling python function in %s, line %s:\n" % (realfile, lineno))
306 if e.lineno: 310 if hasattr(e, "lineno"):
307 error.append("The code lines resulting in this error were:") 311 error.append("The code lines resulting in this error were:")
308 error.extend(_print_trace(body, e.lineno)) 312 error.extend(_print_trace(body, e.lineno))
309 else: 313 else: