summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2012-06-22 12:55:55 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-06-25 14:57:16 +0100
commita87aa7b7af4f43d86b4d89f90a9c3b134d12f006 (patch)
tree75e4c166c0818c955a25a7f7d9b361c50065552f /bitbake
parent15a31a93ba382037786aaf9d8105d8cf10c278c0 (diff)
downloadpoky-a87aa7b7af4f43d86b4d89f90a9c3b134d12f006.tar.gz
bitbake: process: Improve _logged_communicate buffering
There are two problems with the _logged_communicate that are both caused as a result of buffering I/O issues: 1) log truncation when python fails 2) While a bitbake task is running it is impossible to see what is going on if it is only writing a small incremental log that is smaller than the buffer, or you get only a partial log, up until the task exists. It is worse in the case that stderr and stdout are separate file handles, because previous code blocks on the read of stdout and then stderr, serially. The right approach is simply to use select() to determine if there is data available and also flush the log before exiting. This is based on a patch from Jason Wessel <jason.wessel@windriver.com> with some changes to flush upon exit, abstract the non blocking file descriptor setup and drop the buffer size parameter. (Bitbake rev: 361fb71e907aa84c28ecec79fefc6ca39c39172f) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/build.py3
-rw-r--r--bitbake/lib/bb/process.py46
2 files changed, 35 insertions, 14 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 2cb7a9301b..bea2926085 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -135,7 +135,8 @@ class LogTee(object):
135 135
136 def __repr__(self): 136 def __repr__(self):
137 return '<LogTee {0}>'.format(self.name) 137 return '<LogTee {0}>'.format(self.name)
138 138 def flush(self):
139 self.outfile.flush()
139 140
140def exec_func(func, d, dirs = None): 141def exec_func(func, d, dirs = None):
141 """Execute an BB 'function'""" 142 """Execute an BB 'function'"""
diff --git a/bitbake/lib/bb/process.py b/bitbake/lib/bb/process.py
index b74cb18066..05b51725f1 100644
--- a/bitbake/lib/bb/process.py
+++ b/bitbake/lib/bb/process.py
@@ -1,6 +1,8 @@
1import logging 1import logging
2import signal 2import signal
3import subprocess 3import subprocess
4import errno
5import select
4 6
5logger = logging.getLogger('BitBake.Process') 7logger = logging.getLogger('BitBake.Process')
6 8
@@ -68,20 +70,38 @@ def _logged_communicate(pipe, log, input):
68 pipe.stdin.write(input) 70 pipe.stdin.write(input)
69 pipe.stdin.close() 71 pipe.stdin.close()
70 72
71 bufsize = 512
72 outdata, errdata = [], [] 73 outdata, errdata = [], []
73 while pipe.poll() is None: 74 rin = []
74 if pipe.stdout is not None: 75
75 data = pipe.stdout.read(bufsize) 76 if pipe.stdout is not None:
76 if data is not None: 77 bb.utils.nonblockingfd(pipe.stdout.fileno())
77 outdata.append(data) 78 rin.append(pipe.stdout)
78 log.write(data) 79 if pipe.stderr is not None:
79 80 bb.utils.nonblockingfd(pipe.stderr.fileno())
80 if pipe.stderr is not None: 81 rin.append(pipe.stderr)
81 data = pipe.stderr.read(bufsize) 82
82 if data is not None: 83 try:
83 errdata.append(data) 84 while pipe.poll() is None:
84 log.write(data) 85 rlist = rin
86 try:
87 r,w,e = select.select (rlist, [], [])
88 except OSError, e:
89 if e.errno != errno.EINTR:
90 raise
91
92 if pipe.stdout in r:
93 data = pipe.stdout.read()
94 if data is not None:
95 outdata.append(data)
96 log.write(data)
97
98 if pipe.stderr in r:
99 data = pipe.stderr.read()
100 if data is not None:
101 errdata.append(data)
102 log.write(data)
103 finally:
104 log.flush()
85 return ''.join(outdata), ''.join(errdata) 105 return ''.join(outdata), ''.join(errdata)
86 106
87def run(cmd, input=None, log=None, **options): 107def run(cmd, input=None, log=None, **options):