diff options
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 75 |
1 files changed, 58 insertions, 17 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 3e694ba189..577f04a6c5 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -801,12 +801,22 @@ class RunQueue: | |||
801 | self.rqexe = None | 801 | self.rqexe = None |
802 | self.worker = None | 802 | self.worker = None |
803 | self.workerpipe = None | 803 | self.workerpipe = None |
804 | self.fakeworker = None | ||
805 | self.fakeworkerpipe = None | ||
804 | 806 | ||
805 | def _start_worker(self): | 807 | def _start_worker(self, fakeroot = False, rqexec = None): |
806 | logger.debug(1, "Starting bitbake-worker") | 808 | logger.debug(1, "Starting bitbake-worker") |
807 | worker = subprocess.Popen(["bitbake-worker", "decafbad"], stdout=subprocess.PIPE, stdin=subprocess.PIPE) | 809 | if fakeroot: |
810 | fakerootcmd = self.cfgData.getVar("FAKEROOTCMD", True) | ||
811 | fakerootenv = (self.cfgData.getVar("FAKEROOTBASEENV", True) or "").split() | ||
812 | env = os.environ.copy() | ||
813 | for key, value in (var.split('=') for var in fakerootenv): | ||
814 | env[key] = value | ||
815 | worker = subprocess.Popen([fakerootcmd, "bitbake-worker", "decafbad"], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env) | ||
816 | else: | ||
817 | worker = subprocess.Popen(["bitbake-worker", "decafbad"], stdout=subprocess.PIPE, stdin=subprocess.PIPE) | ||
808 | bb.utils.nonblockingfd(worker.stdout) | 818 | bb.utils.nonblockingfd(worker.stdout) |
809 | workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, None) | 819 | workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, rqexec) |
810 | 820 | ||
811 | workerdata = { | 821 | workerdata = { |
812 | "taskdeps" : self.rqdata.dataCache.task_deps, | 822 | "taskdeps" : self.rqdata.dataCache.task_deps, |
@@ -844,14 +854,25 @@ class RunQueue: | |||
844 | 854 | ||
845 | def start_worker(self): | 855 | def start_worker(self): |
846 | if self.worker: | 856 | if self.worker: |
847 | self.teardown_worker() | 857 | self.teardown_workers() |
848 | |||
849 | self.worker, self.workerpipe = self._start_worker() | 858 | self.worker, self.workerpipe = self._start_worker() |
850 | 859 | ||
851 | def teardown_worker(self): | 860 | def start_fakeworker(self, rqexec): |
861 | if not self.fakeworker: | ||
862 | self.fakeworker, self.fakeworkerpipe = self._start_worker(True, rqexec) | ||
863 | |||
864 | def teardown_workers(self): | ||
852 | self._teardown_worker(self.worker, self.workerpipe) | 865 | self._teardown_worker(self.worker, self.workerpipe) |
853 | self.worker = None | 866 | self.worker = None |
854 | self.workerpipe = None | 867 | self.workerpipe = None |
868 | self._teardown_worker(self.fakeworker, self.fakeworkerpipe) | ||
869 | self.fakeworker = None | ||
870 | self.fakeworkerpipe = None | ||
871 | |||
872 | def read_workers(self): | ||
873 | self.workerpipe.read() | ||
874 | if self.fakeworkerpipe: | ||
875 | self.fakeworkerpipe.read() | ||
855 | 876 | ||
856 | def check_stamp_task(self, task, taskname = None, recurse = False, cache = None): | 877 | def check_stamp_task(self, task, taskname = None, recurse = False, cache = None): |
857 | def get_timestamp(f): | 878 | def get_timestamp(f): |
@@ -964,7 +985,7 @@ class RunQueue: | |||
964 | self.rqexe.finish() | 985 | self.rqexe.finish() |
965 | 986 | ||
966 | if self.state is runQueueComplete or self.state is runQueueFailed: | 987 | if self.state is runQueueComplete or self.state is runQueueFailed: |
967 | self.teardown_worker() | 988 | self.teardown_workers() |
968 | if self.rqexe.stats.failed: | 989 | if self.rqexe.stats.failed: |
969 | logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed) | 990 | logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and %d failed.", self.rqexe.stats.completed + self.rqexe.stats.failed, self.rqexe.stats.skipped, self.rqexe.stats.failed) |
970 | else: | 991 | else: |
@@ -996,7 +1017,7 @@ class RunQueue: | |||
996 | except: | 1017 | except: |
997 | logger.error("An uncaught exception occured in runqueue, please see the failure below:") | 1018 | logger.error("An uncaught exception occured in runqueue, please see the failure below:") |
998 | try: | 1019 | try: |
999 | self.teardown_worker() | 1020 | self.teardown_workers() |
1000 | except: | 1021 | except: |
1001 | pass | 1022 | pass |
1002 | self.state = runQueueComplete | 1023 | self.state = runQueueComplete |
@@ -1047,6 +1068,8 @@ class RunQueueExecute: | |||
1047 | self.stampcache = {} | 1068 | self.stampcache = {} |
1048 | 1069 | ||
1049 | rq.workerpipe.setrunqueueexec(self) | 1070 | rq.workerpipe.setrunqueueexec(self) |
1071 | if rq.fakeworkerpipe: | ||
1072 | rq.fakeworkerpipe.setrunqueueexec(self) | ||
1050 | 1073 | ||
1051 | def runqueue_process_waitpid(self, task, status): | 1074 | def runqueue_process_waitpid(self, task, status): |
1052 | 1075 | ||
@@ -1064,6 +1087,9 @@ class RunQueueExecute: | |||
1064 | 1087 | ||
1065 | self.rq.worker.stdin.write("<finishnow></finishnow>") | 1088 | self.rq.worker.stdin.write("<finishnow></finishnow>") |
1066 | self.rq.worker.stdin.flush() | 1089 | self.rq.worker.stdin.flush() |
1090 | if self.rq.fakeworker: | ||
1091 | self.rq.fakeworker.stdin.write("<finishnow></finishnow>") | ||
1092 | self.rq.fakeworker.stdin.flush() | ||
1067 | 1093 | ||
1068 | if len(self.failed_fnids) != 0: | 1094 | if len(self.failed_fnids) != 0: |
1069 | self.rq.state = runQueueFailed | 1095 | self.rq.state = runQueueFailed |
@@ -1077,7 +1103,8 @@ class RunQueueExecute: | |||
1077 | 1103 | ||
1078 | if self.stats.active > 0: | 1104 | if self.stats.active > 0: |
1079 | bb.event.fire(runQueueExitWait(self.stats.active), self.cfgData) | 1105 | bb.event.fire(runQueueExitWait(self.stats.active), self.cfgData) |
1080 | self.rq.workerpipe.read() | 1106 | self.rq.read_workers() |
1107 | |||
1081 | return | 1108 | return |
1082 | 1109 | ||
1083 | if len(self.failed_fnids) != 0: | 1110 | if len(self.failed_fnids) != 0: |
@@ -1271,7 +1298,7 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1271 | Run the tasks in a queue prepared by rqdata.prepare() | 1298 | Run the tasks in a queue prepared by rqdata.prepare() |
1272 | """ | 1299 | """ |
1273 | 1300 | ||
1274 | self.rq.workerpipe.read() | 1301 | self.rq.read_workers() |
1275 | 1302 | ||
1276 | 1303 | ||
1277 | if self.stats.total == 0: | 1304 | if self.stats.total == 0: |
@@ -1309,8 +1336,15 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1309 | startevent = runQueueTaskStarted(task, self.stats, self.rq) | 1336 | startevent = runQueueTaskStarted(task, self.stats, self.rq) |
1310 | bb.event.fire(startevent, self.cfgData) | 1337 | bb.event.fire(startevent, self.cfgData) |
1311 | 1338 | ||
1312 | self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn))) + "</runtask>") | 1339 | taskdep = self.rqdata.dataCache.task_deps[fn] |
1313 | self.rq.worker.stdin.flush() | 1340 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']: |
1341 | if not self.rq.fakeworker: | ||
1342 | self.rq.start_fakeworker(self) | ||
1343 | self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn))) + "</runtask>") | ||
1344 | self.rq.fakeworker.stdin.flush() | ||
1345 | else: | ||
1346 | self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn))) + "</runtask>") | ||
1347 | self.rq.worker.stdin.flush() | ||
1314 | 1348 | ||
1315 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) | 1349 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) |
1316 | self.runq_running[task] = 1 | 1350 | self.runq_running[task] = 1 |
@@ -1319,7 +1353,7 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1319 | return True | 1353 | return True |
1320 | 1354 | ||
1321 | if self.stats.active > 0: | 1355 | if self.stats.active > 0: |
1322 | self.rq.workerpipe.read() | 1356 | self.rq.read_workers() |
1323 | return 0.5 | 1357 | return 0.5 |
1324 | 1358 | ||
1325 | if len(self.failed_fnids) != 0: | 1359 | if len(self.failed_fnids) != 0: |
@@ -1597,7 +1631,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1597 | Run the tasks in a queue prepared by prepare_runqueue | 1631 | Run the tasks in a queue prepared by prepare_runqueue |
1598 | """ | 1632 | """ |
1599 | 1633 | ||
1600 | self.rq.workerpipe.read() | 1634 | self.rq.read_workers() |
1601 | 1635 | ||
1602 | task = None | 1636 | task = None |
1603 | if self.stats.active < self.number_tasks: | 1637 | if self.stats.active < self.number_tasks: |
@@ -1639,8 +1673,15 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1639 | startevent = sceneQueueTaskStarted(task, self.stats, self.rq) | 1673 | startevent = sceneQueueTaskStarted(task, self.stats, self.rq) |
1640 | bb.event.fire(startevent, self.cfgData) | 1674 | bb.event.fire(startevent, self.cfgData) |
1641 | 1675 | ||
1642 | self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn))) + "</runtask>") | 1676 | taskdep = self.rqdata.dataCache.task_deps[fn] |
1643 | self.rq.worker.stdin.flush() | 1677 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']: |
1678 | if not self.rq.fakeworker: | ||
1679 | self.rq.start_fakeworker(self) | ||
1680 | self.rq.fakeworker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn))) + "</runtask>") | ||
1681 | self.rq.fakeworker.stdin.flush() | ||
1682 | else: | ||
1683 | self.rq.worker.stdin.write("<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn))) + "</runtask>") | ||
1684 | self.rq.worker.stdin.flush() | ||
1644 | 1685 | ||
1645 | self.runq_running[task] = 1 | 1686 | self.runq_running[task] = 1 |
1646 | self.stats.taskActive() | 1687 | self.stats.taskActive() |
@@ -1648,7 +1689,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1648 | return True | 1689 | return True |
1649 | 1690 | ||
1650 | if self.stats.active > 0: | 1691 | if self.stats.active > 0: |
1651 | self.rq.workerpipe.read() | 1692 | self.rq.read_workers() |
1652 | return 0.5 | 1693 | return 0.5 |
1653 | 1694 | ||
1654 | # Convert scenequeue_covered task numbers into full taskgraph ids | 1695 | # Convert scenequeue_covered task numbers into full taskgraph ids |