summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChris Larson <chris_larson@mentor.com>2010-12-09 20:14:48 -0500
committerRichard Purdie <rpurdie@linux.intel.com>2011-01-04 14:46:48 +0000
commitb4eff9fcefe2fefab1caaf22e497317e9338063e (patch)
treeabf0e3a263afb75aebf21e6d90bb80fec031acbc
parent87b6cdf5475c6fcc2368e1d80dfaac8fdee5a453 (diff)
downloadpoky-b4eff9fcefe2fefab1caaf22e497317e9338063e.tar.gz
build: use bb.process instead of os.system
(Bitbake rev: 53740977521bc81ffa37adfa7bbeb8f2a80ea165) build: write logfiles per task, not per function Based on d14f9bf6 from poky, reworked for master and other cleanup. (Bitbake rev: beadff2eca1eb95f0411115dd72ddb4c3c44c604) Signed-off-by: Chris Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
-rw-r--r--bitbake/lib/bb/build.py352
-rw-r--r--bitbake/lib/bb/data_smart.py4
-rw-r--r--bitbake/lib/bb/utils.py11
3 files changed, 185 insertions, 182 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 3f6bc875c0..79fb1def9b 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -31,9 +31,13 @@ import sys
31import logging 31import logging
32import bb 32import bb
33import bb.utils 33import bb.utils
34import bb.process
34 35
35logger = logging.getLogger("BitBake.Build") 36logger = logging.getLogger("BitBake.Build")
36 37
38NULL = open('/dev/null', 'r')
39
40
37# When we execute a python function we'd like certain things 41# When we execute a python function we'd like certain things
38# in all namespaces, hence we add them to __builtins__ 42# in all namespaces, hence we add them to __builtins__
39# If we do not do this and use the exec globals, they will 43# If we do not do this and use the exec globals, they will
@@ -53,8 +57,8 @@ class FuncFailed(Exception):
53 57
54 def __str__(self): 58 def __str__(self):
55 if self.logfile and os.path.exists(self.logfile): 59 if self.logfile and os.path.exists(self.logfile):
56 msg = "%s (see %s for further information)" % \ 60 msg = ("%s (see %s for further information)" %
57 (self.message, self.logfile) 61 (self.message, self.logfile))
58 else: 62 else:
59 msg = self.message 63 msg = self.message
60 return msg 64 return msg
@@ -95,30 +99,33 @@ class TaskInvalid(TaskBase):
95 super(TaskInvalid, self).__init__(task, metadata) 99 super(TaskInvalid, self).__init__(task, metadata)
96 self._message = "No such task '%s'" % task 100 self._message = "No such task '%s'" % task
97 101
98# functions
99 102
100def exec_func(func, d, dirs = None): 103class tee(file):
104 def write(self, string):
105 logger.plain(string)
106 file.write(self, string)
107
108 def __repr__(self):
109 return "<open[tee] file '{0}'>".format(self.name)
110
111
112def exec_func(func, d, dirs = None, logfile = NULL):
101 """Execute an BB 'function'""" 113 """Execute an BB 'function'"""
102 114
103 body = data.getVar(func, d) 115 body = data.getVar(func, d)
104 if not body: 116 if not body:
105 bb.warn("Function %s doesn't exist" % func) 117 if body is None:
118 logger.warn("Function %s doesn't exist", func)
106 return 119 return
107 120
108 flags = data.getVarFlags(func, d) 121 flags = data.getVarFlags(func, d)
109 for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot', 'task']: 122 cleandirs = flags.get('cleandirs')
110 if not item in flags:
111 flags[item] = None
112
113 ispython = flags['python']
114
115 cleandirs = flags['cleandirs']
116 if cleandirs: 123 if cleandirs:
117 for cdir in data.expand(cleandirs, d).split(): 124 for cdir in data.expand(cleandirs, d).split():
118 os.system("rm -rf %s" % cdir) 125 bb.utils.remove(cdir, True)
119 126
120 if dirs is None: 127 if dirs is None:
121 dirs = flags['dirs'] 128 dirs = flags.get('dirs')
122 if dirs: 129 if dirs:
123 dirs = data.expand(dirs, d).split() 130 dirs = data.expand(dirs, d).split()
124 131
@@ -128,214 +135,185 @@ def exec_func(func, d, dirs = None):
128 adir = dirs[-1] 135 adir = dirs[-1]
129 else: 136 else:
130 adir = data.getVar('B', d, 1) 137 adir = data.getVar('B', d, 1)
138 if not os.path.exists(adir):
139 adir = None
131 140
132 # Save current directory 141 ispython = flags.get('python')
133 try: 142 if flags.get('fakeroot') and not flags.get('task'):
134 prevdir = os.getcwd() 143 bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func)
135 except OSError:
136 prevdir = data.getVar('TOPDIR', d, True)
137
138 # Setup scriptfile
139 t = data.getVar('T', d, 1)
140 if not t:
141 raise SystemExit("T variable not set, unable to build")
142 bb.utils.mkdirhier(t)
143 runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
144 logfile = d.getVar("BB_LOGFILE", True)
145 144
146 # Change to correct directory (if specified) 145 tempdir = data.getVar('T', d, 1)
147 if adir and os.access(adir, os.F_OK): 146 runfile = os.path.join(tempdir, 'run.{0}.{1}'.format(func, os.getpid()))
148 os.chdir(adir)
149 147
150 locks = [] 148 locks = []
151 lockfiles = flags['lockfiles'] 149 lockfiles = flags.get('lockfiles')
152 if lockfiles: 150 if lockfiles:
153 for lock in data.expand(lockfiles, d).split(): 151 for lock in data.expand(lockfiles, d).split():
154 locks.append(bb.utils.lockfile(lock)) 152 locks.append(bb.utils.lockfile(lock))
155 153
156 try: 154 try:
157 # Run the function
158 if ispython: 155 if ispython:
159 exec_func_python(func, d, runfile, logfile) 156 exec_func_python(func, d, runfile, logfile, cwd=adir)
160 else: 157 else:
161 exec_func_shell(func, d, runfile, logfile, flags) 158 exec_func_shell(func, d, runfile, logfile, cwd=adir)
162
163 # Restore original directory
164 try:
165 os.chdir(prevdir)
166 except:
167 pass
168 159
169 finally: 160 finally:
170
171 # Unlock any lockfiles 161 # Unlock any lockfiles
172 for lock in locks: 162 for lock in locks:
173 bb.utils.unlockfile(lock) 163 bb.utils.unlockfile(lock)
174 164
175def exec_func_python(func, d, runfile, logfile): 165_functionfmt = """
166def {function}(d):
167{body}
168
169{function}(d)
170"""
171#logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
172def exec_func_python(func, d, runfile, logfile, cwd=None):
176 """Execute a python BB 'function'""" 173 """Execute a python BB 'function'"""
177 174
178 bbfile = bb.data.getVar('FILE', d, 1) 175 bbfile = d.getVar('FILE', True)
179 tmp = "def " + func + "(d):\n%s" % data.getVar(func, d) 176 try:
180 tmp += '\n' + func + '(d)' 177 olddir = os.getcwd()
178 except OSError:
179 olddir = None
180 code = _functionfmt.format(function=func, body=d.getVar(func, True))
181 bb.utils.mkdirhier(os.path.dirname(runfile))
182 with open(runfile, 'w') as script:
183 script.write(code)
184
185 if cwd:
186 os.chdir(cwd)
187
188 #handler = logging.StreamHandler(logfile)
189 #handler.setFormatter(logformatter)
190 #bblogger.addHandler(handler)
181 191
182 f = open(runfile, "w")
183 f.write(tmp)
184 comp = utils.better_compile(tmp, func, bbfile)
185 try: 192 try:
186 utils.better_exec(comp, {"d": d}, tmp, bbfile) 193 comp = utils.better_compile(code, func, bbfile)
194 utils.better_exec(comp, {"d": d}, code, bbfile)
187 except: 195 except:
188 if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed): 196 if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed):
189 raise 197 raise
190 198
191 raise FuncFailed(func, logfile) 199 raise FuncFailed(func, None)
192 200 finally:
193 201 #bblogger.removeHandler(handler)
194def exec_func_shell(func, d, runfile, logfile, flags): 202 if olddir:
195 """Execute a shell BB 'function' Returns true if execution was successful. 203 os.chdir(olddir)
196 204
197 For this, it creates a bash shell script in the tmp dectory, writes the local 205def exec_func_shell(function, d, runfile, logfile, cwd=None):
198 data into it and finally executes. The output of the shell will end in a log file and stdout. 206 """Execute a shell function from the metadata
199 207
200 Note on directory behavior. The 'dirs' varflag should contain a list 208 Note on directory behavior. The 'dirs' varflag should contain a list
201 of the directories you need created prior to execution. The last 209 of the directories you need created prior to execution. The last
202 item in the list is where we will chdir/cd to. 210 item in the list is where we will chdir/cd to.
203 """ 211 """
204 212
205 deps = flags['deps'] 213 # Don't let the emitted shell script override PWD
206 check = flags['check'] 214 d.delVarFlag('PWD', 'export')
207 if check in globals():
208 if globals()[check](func, deps):
209 return
210
211 f = open(runfile, "w")
212 f.write("#!/bin/sh -e\n")
213 if logger.getEffectiveLevel() <= logging.DEBUG:
214 f.write("set -x\n")
215 data.emit_func(func, f, d)
216
217 f.write("cd %s\n" % os.getcwd())
218 if func: f.write("%s\n" % func)
219 f.close()
220 os.chmod(runfile, 0775)
221 if not func:
222 raise TypeError("Function argument must be a string")
223
224 # execute function
225 if flags['fakeroot'] and not flags['task']:
226 bb.fatal("Function %s specifies fakeroot but isn't a task?!" % func)
227
228 lang_environment = "LC_ALL=C "
229 ret = os.system('%ssh -e %s' % (lang_environment, runfile))
230 215
231 if ret == 0: 216 with open(runfile, 'w') as script:
232 return 217 script.write('#!/bin/sh -e\n')
218 if logger.getEffectiveLevel() <= logging.DEBUG:
219 script.write("set -x\n")
220 data.emit_func(function, script, d)
233 221
234 raise FuncFailed(func, logfile) 222 script.write("%s\n" % function)
223 os.fchmod(script.fileno(), 0775)
235 224
225 env = {
226 'PATH': d.getVar('PATH', True),
227 'LC_ALL': 'C',
228 }
236 229
237def exec_task(fn, task, d): 230 cmd = runfile
238 """Execute an BB 'task'
239 231
240 The primary difference between executing a task versus executing 232 if logger.getEffectiveLevel() <= logging.DEBUG:
241 a function is that a task exists in the task digraph, and therefore 233 logfile = LogTee(logger, logfile)
242 has dependencies amongst other tasks."""
243 234
244 # Check whther this is a valid task 235 try:
236 bb.process.run(cmd, env=env, cwd=cwd, shell=False, stdin=NULL,
237 log=logfile)
238 except bb.process.CmdError:
239 raise FuncFailed(function, logfile.name)
240
241def _task_data(fn, task, d):
242 localdata = data.createCopy(d)
243 localdata.setVar('BB_FILENAME', fn)
244 localdata.setVar('BB_CURRENTTASK', task[3:])
245 localdata.setVar('OVERRIDES', 'task-%s:%s' %
246 (task[3:], d.getVar('OVERRIDES', False)))
247 localdata.finalize()
248 data.expandKeys(localdata)
249 return localdata
250
251def _exec_task(fn, task, d, quieterr):
252 """Execute a BB 'task'
253
254 Execution of a task involves a bit more setup than executing a function,
255 running it with its own local metadata, and with some useful variables set.
256 """
245 if not data.getVarFlag(task, 'task', d): 257 if not data.getVarFlag(task, 'task', d):
246 event.fire(TaskInvalid(task, d), d) 258 event.fire(TaskInvalid(task, d), d)
247 logger.error("No such task: %s" % task) 259 logger.error("No such task: %s" % task)
248 return 1 260 return 1
249 261
250 quieterr = False 262 logger.debug(1, "Executing task %s", task)
251 if d.getVarFlag(task, "quieterrors") is not None:
252 quieterr = True
253 263
254 try: 264 localdata = _task_data(fn, task, d)
255 logger.debug(1, "Executing task %s", task) 265 tempdir = localdata.getVar('T', True)
256 old_overrides = data.getVar('OVERRIDES', d, 0) 266 if not tempdir:
257 localdata = data.createCopy(d) 267 bb.fatal("T variable not set, unable to build")
258 data.setVar('OVERRIDES', 'task-%s:%s' % (task[3:], old_overrides), localdata)
259 data.update_data(localdata)
260 data.expandKeys(localdata)
261 data.setVar('BB_FILENAME', fn, d)
262 data.setVar('BB_CURRENTTASK', task[3:], d)
263 event.fire(TaskStarted(task, localdata), localdata)
264
265 # Setup logfiles
266 t = data.getVar('T', d, 1)
267 if not t:
268 raise SystemExit("T variable not set, unable to build")
269 bb.utils.mkdirhier(t)
270 loglink = "%s/log.%s" % (t, task)
271 logfile = "%s/log.%s.%s" % (t, task, str(os.getpid()))
272 d.setVar("BB_LOGFILE", logfile)
273
274 # Even though the log file has not yet been opened, lets create the link
275 if loglink:
276 try:
277 os.remove(loglink)
278 except OSError as e:
279 pass
280
281 try:
282 os.symlink(logfile, loglink)
283 except OSError as e:
284 pass
285
286 # Handle logfiles
287 si = file('/dev/null', 'r')
288 try:
289 so = file(logfile, 'w')
290 except OSError:
291 logger.exception("Opening log file '%s'", logfile)
292 pass
293 se = so
294 268
295 # Dup the existing fds so we dont lose them 269 bb.utils.mkdirhier(tempdir)
296 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()] 270 loglink = os.path.join(tempdir, 'log.{0}'.format(task))
297 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()] 271 logfn = os.path.join(tempdir, 'log.{0}.{1}'.format(task, os.getpid()))
298 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()] 272 if loglink:
273 bb.utils.remove(loglink)
299 274
300 # Replace those fds with our own 275 try:
301 os.dup2(si.fileno(), osi[1]) 276 os.symlink(logfn, loglink)
302 os.dup2(so.fileno(), oso[1]) 277 except OSError:
303 os.dup2(se.fileno(), ose[1]) 278 pass
304 279
305 # Since we've remapped stdout and stderr, its safe for log messages to be printed there now 280 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
306 # exec_func can nest so we have to save state 281 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
307 origstdout = bb.event.useStdout
308 bb.event.useStdout = True
309 282
283 # Handle logfiles
284 si = file('/dev/null', 'r')
285 try:
286 logfile = file(logfn, 'w')
287 except OSError:
288 logger.exception("Opening log file '%s'", logfn)
289 pass
310 290
311 prefuncs = (data.getVarFlag(task, 'prefuncs', localdata) or "").split() 291 # Dup the existing fds so we dont lose them
312 for func in prefuncs: 292 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
313 exec_func(func, localdata) 293 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
314 exec_func(task, localdata) 294 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
315 postfuncs = (data.getVarFlag(task, 'postfuncs', localdata) or "").split()
316 for func in postfuncs:
317 exec_func(func, localdata)
318 295
319 event.fire(TaskSucceeded(task, localdata), localdata) 296 # Replace those fds with our own
297 os.dup2(si.fileno(), osi[1])
298 os.dup2(logfile.fileno(), oso[1])
299 os.dup2(logfile.fileno(), ose[1])
320 300
321 # make stamp, or cause event and raise exception 301 # Since we've remapped stdout and stderr, its safe for log messages to be printed there now
322 if not data.getVarFlag(task, 'nostamp', d) and not data.getVarFlag(task, 'selfstamp', d): 302 # exec_func can nest so we have to save state
323 make_stamp(task, d) 303 origstdout = bb.event.useStdout
304 bb.event.useStdout = True
324 305
306 event.fire(TaskStarted(task, localdata), localdata)
307 try:
308 for func in (prefuncs or '').split():
309 exec_func(func, localdata, logfile=logfile)
310 exec_func(task, localdata, logfile=logfile)
311 for func in (postfuncs or '').split():
312 exec_func(func, localdata, logfile=logfile)
325 except FuncFailed as exc: 313 except FuncFailed as exc:
326 if not quieterr: 314 if not quieterr:
327 logger.error(str(exc)) 315 logger.error(str(exc))
328 failedevent = TaskFailed(exc.name, exc.logfile, task, d) 316 event.fire(TaskFailed(exc.name, exc.logfile, localdata), localdata)
329 event.fire(failedevent, d)
330 return 1
331
332 except Exception:
333 from traceback import format_exc
334 if not quieterr:
335 logger.error("Build of %s failed" % (task))
336 logger.error(format_exc())
337 failedevent = TaskFailed("Task Failed", None, task, d)
338 event.fire(failedevent, d)
339 return 1 317 return 1
340 finally: 318 finally:
341 sys.stdout.flush() 319 sys.stdout.flush()
@@ -348,26 +326,40 @@ def exec_task(fn, task, d):
348 os.dup2(oso[0], oso[1]) 326 os.dup2(oso[0], oso[1])
349 os.dup2(ose[0], ose[1]) 327 os.dup2(ose[0], ose[1])
350 328
351 # Close our logs
352 si.close()
353 so.close()
354 se.close()
355
356 if logfile and os.path.exists(logfile) and os.path.getsize(logfile) == 0:
357 logger.debug(2, "Zero size logfile %s, removing", logfile)
358 os.remove(logfile)
359 try:
360 os.remove(loglink)
361 except OSError as e:
362 pass
363
364 # Close the backup fds 329 # Close the backup fds
365 os.close(osi[0]) 330 os.close(osi[0])
366 os.close(oso[0]) 331 os.close(oso[0])
367 os.close(ose[0]) 332 os.close(ose[0])
333 si.close()
334
335 logfile.close()
336 if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
337 logger.debug(2, "Zero size logfn %s, removing", logfn)
338 bb.utils.remove(logfn)
339 bb.utils.remove(loglink)
340 event.fire(TaskSucceeded(task, localdata), localdata)
341
342 if not d.getVarFlag(task, 'nostamp') and not d.getVarFlag(task, 'selfstamp'):
343 make_stamp(task, d)
368 344
369 return 0 345 return 0
370 346
347def exec_task(fn, task, d):
348 try:
349 quieterr = False
350 if d.getVarFlag(task, "quieterrors") is not None:
351 quieterr = True
352
353 return _exec_task(fn, task, d, quieterr)
354 except Exception:
355 from traceback import format_exc
356 if not quieterr:
357 logger.error("Build of %s failed" % (task))
358 logger.error(format_exc())
359 failedevent = TaskFailed("Task Failed", None, task, d)
360 event.fire(failedevent, d)
361 return 1
362
371def extract_stamp(d, fn): 363def extract_stamp(d, fn):
372 """ 364 """
373 Extracts stamp format which is either a data dictionary (fn unset) 365 Extracts stamp format which is either a data dictionary (fn unset)
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 3f2d42c1bb..16270461a4 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -291,13 +291,13 @@ class DataSmart(MutableMapping):
291 self._makeShadowCopy(var) 291 self._makeShadowCopy(var)
292 self.dict[var][flag] = flagvalue 292 self.dict[var][flag] = flagvalue
293 293
294 def getVarFlag(self, var, flag, exp = False): 294 def getVarFlag(self, var, flag, expand = False):
295 local_var = self._findVar(var) 295 local_var = self._findVar(var)
296 value = None 296 value = None
297 if local_var: 297 if local_var:
298 if flag in local_var: 298 if flag in local_var:
299 value = copy.copy(local_var[flag]) 299 value = copy.copy(local_var[flag])
300 if exp and value: 300 if expand and value:
301 value = self.expand(value, None) 301 value = self.expand(value, None)
302 return value 302 return value
303 303
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index e4c12fd1ab..d9f543bc60 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -579,6 +579,17 @@ def build_environment(d):
579 if export: 579 if export:
580 os.environ[var] = bb.data.getVar(var, d, True) or "" 580 os.environ[var] = bb.data.getVar(var, d, True) or ""
581 581
582def remove(path, recurse=False):
583 """Equivalent to rm -f or rm -rf"""
584 import os, errno, shutil
585 try:
586 os.unlink(path)
587 except OSError, exc:
588 if recurse and exc.errno == errno.EISDIR:
589 shutil.rmtree(path)
590 elif exc.errno != errno.ENOENT:
591 raise
592
582def prunedir(topdir): 593def prunedir(topdir):
583 # Delete everything reachable from the directory named in 'topdir'. 594 # Delete everything reachable from the directory named in 'topdir'.
584 # CAUTION: This is dangerous! 595 # CAUTION: This is dangerous!