diff options
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 134 |
1 files changed, 58 insertions, 76 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 9127f248d5..b2d5fc01ab 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -23,6 +23,7 @@ Handles preparation and execution of a queue of tasks | |||
23 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | 23 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
24 | 24 | ||
25 | import bb, os, sys | 25 | import bb, os, sys |
26 | import subprocess | ||
26 | from bb import msg, data, event | 27 | from bb import msg, data, event |
27 | import signal | 28 | import signal |
28 | import stat | 29 | import stat |
@@ -937,6 +938,7 @@ class RunQueueExecute: | |||
937 | self.runq_complete = [] | 938 | self.runq_complete = [] |
938 | self.build_pids = {} | 939 | self.build_pids = {} |
939 | self.build_pipes = {} | 940 | self.build_pipes = {} |
941 | self.build_procs = {} | ||
940 | self.failed_fnids = [] | 942 | self.failed_fnids = [] |
941 | 943 | ||
942 | def runqueue_process_waitpid(self): | 944 | def runqueue_process_waitpid(self): |
@@ -944,19 +946,22 @@ class RunQueueExecute: | |||
944 | Return none is there are no processes awaiting result collection, otherwise | 946 | Return none is there are no processes awaiting result collection, otherwise |
945 | collect the process exit codes and close the information pipe. | 947 | collect the process exit codes and close the information pipe. |
946 | """ | 948 | """ |
947 | result = os.waitpid(-1, os.WNOHANG) | 949 | for pid in self.build_procs.keys(): |
948 | if result[0] is 0 and result[1] is 0: | 950 | proc = self.build_procs[pid] |
949 | return None | 951 | proc.poll() |
950 | task = self.build_pids[result[0]] | 952 | if proc.returncode is not None: |
951 | del self.build_pids[result[0]] | 953 | task = self.build_pids[pid] |
952 | self.build_pipes[result[0]].close() | 954 | del self.build_pids[pid] |
953 | del self.build_pipes[result[0]] | 955 | self.build_pipes[pid].close() |
954 | if result[1] != 0: | 956 | del self.build_pipes[pid] |
955 | self.task_fail(task, result[1]) | 957 | del self.build_procs[pid] |
956 | else: | 958 | if proc.returncode != 0: |
957 | self.task_complete(task) | 959 | self.task_fail(task, proc.returncode) |
958 | self.stats.taskCompleted() | 960 | else: |
959 | bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) | 961 | self.task_complete(task) |
962 | self.stats.taskCompleted() | ||
963 | bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) | ||
964 | |||
960 | 965 | ||
961 | def finish_now(self): | 966 | def finish_now(self): |
962 | if self.stats.active: | 967 | if self.stats.active: |
@@ -990,31 +995,8 @@ class RunQueueExecute: | |||
990 | def fork_off_task(self, fn, task, taskname): | 995 | def fork_off_task(self, fn, task, taskname): |
991 | sys.stdout.flush() | 996 | sys.stdout.flush() |
992 | sys.stderr.flush() | 997 | sys.stderr.flush() |
993 | try: | ||
994 | pipein, pipeout = os.pipe() | ||
995 | pid = os.fork() | ||
996 | except OSError as e: | ||
997 | bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror)) | ||
998 | if pid == 0: | ||
999 | os.close(pipein) | ||
1000 | # Save out the PID so that the event can include it the | ||
1001 | # events | ||
1002 | bb.event.worker_pid = os.getpid() | ||
1003 | bb.event.worker_pipe = pipeout | ||
1004 | |||
1005 | self.rq.state = runQueueChildProcess | ||
1006 | # Make the child the process group leader | ||
1007 | os.setpgid(0, 0) | ||
1008 | # No stdin | ||
1009 | newsi = os.open('/dev/null', os.O_RDWR) | ||
1010 | os.dup2(newsi, sys.stdin.fileno()) | ||
1011 | # Stdout to a logfile | ||
1012 | #logout = data.expand("${TMPDIR}/log/stdout.%s" % os.getpid(), self.cfgData, True) | ||
1013 | #mkdirhier(os.path.dirname(logout)) | ||
1014 | #newso = open(logout, 'w') | ||
1015 | #os.dup2(newso.fileno(), sys.stdout.fileno()) | ||
1016 | #os.dup2(newso.fileno(), sys.stderr.fileno()) | ||
1017 | 998 | ||
999 | try: | ||
1018 | bb.event.fire(runQueueTaskStarted(task, self.stats, self.rq), self.cfgData) | 1000 | bb.event.fire(runQueueTaskStarted(task, self.stats, self.rq), self.cfgData) |
1019 | bb.msg.note(1, bb.msg.domain.RunQueue, | 1001 | bb.msg.note(1, bb.msg.domain.RunQueue, |
1020 | "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.stats.active + self.stats.failed + 1, | 1002 | "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.stats.active + self.stats.failed + 1, |
@@ -1022,26 +1004,25 @@ class RunQueueExecute: | |||
1022 | task, | 1004 | task, |
1023 | self.rqdata.get_user_idstring(task))) | 1005 | self.rqdata.get_user_idstring(task))) |
1024 | 1006 | ||
1025 | bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", self, self.cooker.configuration.data) | 1007 | the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data) |
1026 | bb.data.setVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", fn, self.cooker.configuration.data) | 1008 | |
1027 | try: | 1009 | env = bb.data.export_vars(the_data) |
1028 | the_data = self.cooker.bb_cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data) | 1010 | |
1029 | 1011 | taskdep = self.rqdata.dataCache.task_deps[fn] | |
1030 | if not self.cooker.configuration.dry_run: | 1012 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']: |
1031 | bb.build.exec_task(taskname, the_data) | 1013 | envvars = the_data.getVar("FAKEROOTENV", True).split() |
1032 | os._exit(0) | 1014 | for var in envvars: |
1033 | 1015 | comps = var.split("=") | |
1034 | except bb.build.EventException as e: | 1016 | env[comps[0]] = comps[1] |
1035 | event = e.args[1] | 1017 | |
1036 | bb.msg.error(bb.msg.domain.Build, "%s event exception, aborting" % bb.event.getName(event)) | 1018 | proc = subprocess.Popen(["bitbake-runtask", fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE) |
1037 | os._exit(1) | 1019 | pipein = proc.stdout |
1038 | except Exception: | 1020 | pipeout = proc.stdin |
1039 | from traceback import format_exc | 1021 | pid = proc.pid |
1040 | bb.msg.error(bb.msg.domain.Build, "Build of %s %s failed" % (fn, taskname)) | 1022 | except OSError as e: |
1041 | bb.msg.error(bb.msg.domain.Build, format_exc()) | 1023 | bb.msg.fatal(bb.msg.domain.RunQueue, "fork failed: %d (%s)" % (e.errno, e.strerror)) |
1042 | os._exit(1) | 1024 | |
1043 | os._exit(0) | 1025 | return proc |
1044 | return pid, pipein, pipeout | ||
1045 | 1026 | ||
1046 | class RunQueueExecuteTasks(RunQueueExecute): | 1027 | class RunQueueExecuteTasks(RunQueueExecute): |
1047 | def __init__(self, rq): | 1028 | def __init__(self, rq): |
@@ -1153,10 +1134,11 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1153 | self.task_skip(task) | 1134 | self.task_skip(task) |
1154 | continue | 1135 | continue |
1155 | 1136 | ||
1156 | pid, pipein, pipeout = self.fork_off_task(fn, task, taskname) | 1137 | proc = self.fork_off_task(fn, task, taskname) |
1157 | 1138 | ||
1158 | self.build_pids[pid] = task | 1139 | self.build_pids[proc.pid] = task |
1159 | self.build_pipes[pid] = runQueuePipe(pipein, pipeout, self.cfgData) | 1140 | self.build_procs[proc.pid] = proc |
1141 | self.build_pipes[proc.pid] = runQueuePipe(proc.stdout, proc.stdin, self.cfgData) | ||
1160 | self.runq_running[task] = 1 | 1142 | self.runq_running[task] = 1 |
1161 | self.stats.taskActive() | 1143 | self.stats.taskActive() |
1162 | if self.stats.active < self.number_tasks: | 1144 | if self.stats.active < self.number_tasks: |
@@ -1356,10 +1338,11 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1356 | self.task_skip(task) | 1338 | self.task_skip(task) |
1357 | return True | 1339 | return True |
1358 | 1340 | ||
1359 | pid, pipein, pipeout = self.fork_off_task(fn, realtask, taskname) | 1341 | proc = self.fork_off_task(fn, realtask, taskname) |
1360 | 1342 | ||
1361 | self.build_pids[pid] = task | 1343 | self.build_pids[proc.pid] = task |
1362 | self.build_pipes[pid] = runQueuePipe(pipein, pipeout, self.cfgData) | 1344 | self.build_procs[proc.pid] = proc |
1345 | self.build_pipes[proc.pid] = runQueuePipe(proc.stdout, proc.stdin, self.cfgData) | ||
1363 | self.runq_running[task] = 1 | 1346 | self.runq_running[task] = 1 |
1364 | self.stats.taskActive() | 1347 | self.stats.taskActive() |
1365 | if self.stats.active < self.number_tasks: | 1348 | if self.stats.active < self.number_tasks: |
@@ -1384,7 +1367,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1384 | self.rq.state = runQueueRunInit | 1367 | self.rq.state = runQueueRunInit |
1385 | return True | 1368 | return True |
1386 | 1369 | ||
1387 | |||
1388 | class TaskFailure(Exception): | 1370 | class TaskFailure(Exception): |
1389 | """ | 1371 | """ |
1390 | Exception raised when a task in a runqueue fails | 1372 | Exception raised when a task in a runqueue fails |
@@ -1437,14 +1419,14 @@ class runQueueTaskCompleted(runQueueEvent): | |||
1437 | runQueueEvent.__init__(self, task, stats, rq) | 1419 | runQueueEvent.__init__(self, task, stats, rq) |
1438 | self.message = "Task %s completed (%s)" % (task, self.taskstring) | 1420 | self.message = "Task %s completed (%s)" % (task, self.taskstring) |
1439 | 1421 | ||
1440 | def check_stamp_fn(fn, taskname, d): | 1422 | #def check_stamp_fn(fn, taskname, d): |
1441 | rq = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", d) | 1423 | # rq = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY", d) |
1442 | fn = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", d) | 1424 | # fn = bb.data.getVar("__RUNQUEUE_DO_NOT_USE_EXTERNALLY2", d) |
1443 | fnid = rq.rqdata.taskData.getfn_id(fn) | 1425 | # fnid = rq.rqdata.taskData.getfn_id(fn) |
1444 | taskid = rq.get_task_id(fnid, taskname) | 1426 | # taskid = rq.get_task_id(fnid, taskname) |
1445 | if taskid is not None: | 1427 | # if taskid is not None: |
1446 | return rq.check_stamp_task(taskid) | 1428 | # return rq.check_stamp_task(taskid) |
1447 | return None | 1429 | # return None |
1448 | 1430 | ||
1449 | class runQueuePipe(): | 1431 | class runQueuePipe(): |
1450 | """ | 1432 | """ |
@@ -1452,7 +1434,7 @@ class runQueuePipe(): | |||
1452 | """ | 1434 | """ |
1453 | def __init__(self, pipein, pipeout, d): | 1435 | def __init__(self, pipein, pipeout, d): |
1454 | self.fd = pipein | 1436 | self.fd = pipein |
1455 | os.close(pipeout) | 1437 | pipeout.close() |
1456 | fcntl.fcntl(self.fd, fcntl.F_SETFL, fcntl.fcntl(self.fd, fcntl.F_GETFL) | os.O_NONBLOCK) | 1438 | fcntl.fcntl(self.fd, fcntl.F_SETFL, fcntl.fcntl(self.fd, fcntl.F_GETFL) | os.O_NONBLOCK) |
1457 | self.queue = "" | 1439 | self.queue = "" |
1458 | self.d = d | 1440 | self.d = d |
@@ -1460,8 +1442,8 @@ class runQueuePipe(): | |||
1460 | def read(self): | 1442 | def read(self): |
1461 | start = len(self.queue) | 1443 | start = len(self.queue) |
1462 | try: | 1444 | try: |
1463 | self.queue = self.queue + os.read(self.fd, 1024) | 1445 | self.queue = self.queue + self.fd.read(1024) |
1464 | except OSError: | 1446 | except IOError: |
1465 | pass | 1447 | pass |
1466 | end = len(self.queue) | 1448 | end = len(self.queue) |
1467 | index = self.queue.find("</event>") | 1449 | index = self.queue.find("</event>") |
@@ -1476,4 +1458,4 @@ class runQueuePipe(): | |||
1476 | continue | 1458 | continue |
1477 | if len(self.queue) > 0: | 1459 | if len(self.queue) > 0: |
1478 | print("Warning, worker left partial message") | 1460 | print("Warning, worker left partial message") |
1479 | os.close(self.fd) | 1461 | self.fd.close() |