summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@linux.intel.com>2010-09-27 15:57:13 +0100
committerRichard Purdie <rpurdie@linux.intel.com>2010-09-28 15:34:27 +0100
commitec4d6b989aad5f4564e367c99c62c59f962b57ea (patch)
treebd1a4783352a90629a4c07950391cafb986f5373
parent15ceaaaaf777175df8fa49f08e37b23052ca2290 (diff)
downloadpoky-ec4d6b989aad5f4564e367c99c62c59f962b57ea.tar.gz
bitbake: Pass task hash information to subprocesses
Pass task has informaiton to work processes, allowing full manipulation of the hash data in the task context allowing checksums to be usable. Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
-rwxr-xr-xbitbake/bin/bitbake-runtask23
-rw-r--r--bitbake/lib/bb/runqueue.py23
-rw-r--r--bitbake/lib/bb/siggen.py6
3 files changed, 47 insertions, 5 deletions
diff --git a/bitbake/bin/bitbake-runtask b/bitbake/bin/bitbake-runtask
index 2f5ebea792..9e59b8a6f2 100755
--- a/bitbake/bin/bitbake-runtask
+++ b/bitbake/bin/bitbake-runtask
@@ -5,6 +5,12 @@ import sys
5import warnings 5import warnings
6sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) 6sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
7 7
8try:
9 import cPickle as pickle
10except ImportError:
11 import pickle
12 bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
13
8class BBConfiguration(object): 14class BBConfiguration(object):
9 """ 15 """
10 Manages build options and configurations for one run 16 Manages build options and configurations for one run
@@ -62,8 +68,9 @@ bb.event.useStdout = False
62import bb.cooker 68import bb.cooker
63 69
64cooker = bb.cooker.BBCooker(BBConfiguration(), None) 70cooker = bb.cooker.BBCooker(BBConfiguration(), None)
65buildfile = sys.argv[1] 71hashfile = sys.argv[1]
66taskname = sys.argv[2] 72buildfile = sys.argv[2]
73taskname = sys.argv[3]
67 74
68cooker.parseConfiguration() 75cooker.parseConfiguration()
69 76
@@ -84,8 +91,18 @@ cooker.bb_cache.handle_data(fn, cooker.status)
84if taskname.endswith("_setscene"): 91if taskname.endswith("_setscene"):
85 the_data.setVarFlag(taskname, "quieterrors", "1") 92 the_data.setVarFlag(taskname, "quieterrors", "1")
86 93
94p = pickle.Unpickler(file(hashfile, "rb"))
95hashdata = p.load()
96
97bb.parse.siggen.set_taskdata(hashdata["hashes"], hashdata["deps"])
98
99for h in hashdata["hashes"]:
100 bb.data.setVar("BBHASH_%s" % h, hashdata["hashes"][h], the_data)
101for h in hashdata["deps"]:
102 bb.data.setVar("BBHASHDEPS_%s" % h, hashdata["deps"][h], the_data)
103
87ret = 0 104ret = 0
88if sys.argv[3] != "True": 105if sys.argv[4] != "True":
89 ret = bb.build.exec_task(fn, taskname, the_data) 106 ret = bb.build.exec_task(fn, taskname, the_data)
90sys.exit(ret) 107sys.exit(ret)
91 108
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 201f427f45..dff4ff7f3e 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -30,6 +30,12 @@ import stat
30import fcntl 30import fcntl
31import copy 31import copy
32 32
33try:
34 import cPickle as pickle
35except ImportError:
36 import pickle
37 bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
38
33class RunQueueStats: 39class RunQueueStats:
34 """ 40 """
35 Holds statistics on the tasks handled by the associated runQueue 41 Holds statistics on the tasks handled by the associated runQueue
@@ -703,6 +709,21 @@ class RunQueueData:
703 procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep]) 709 procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
704 self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache) 710 self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache)
705 711
712 hashdata = {}
713 hashdata["hashes"] = {}
714 hashdata["deps"] = {}
715 for task in range(len(self.runq_fnid)):
716 hashdata["hashes"][self.taskData.fn_index[self.runq_fnid[task]] + "." + self.runq_task[task]] = self.runq_hash[task]
717 deps = []
718 for dep in self.runq_depends[task]:
719 deps.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep])
720 hashdata["deps"][self.taskData.fn_index[self.runq_fnid[task]] + "." + self.runq_task[task]] = deps
721
722 # Write out the hashes into a file for use by the individual tasks
723 self.hashfile = bb.data.expand("${TMPDIR}/cache/hashdata.dat", self.cooker.configuration.data)
724 p = pickle.Pickler(file(self.hashfile, "wb"), -1)
725 p.dump(hashdata)
726
706 return len(self.runq_fnid) 727 return len(self.runq_fnid)
707 728
708 def dump_data(self, taskQueue): 729 def dump_data(self, taskQueue):
@@ -1047,7 +1068,7 @@ class RunQueueExecute:
1047 sys.stdout.flush() 1068 sys.stdout.flush()
1048 sys.stderr.flush() 1069 sys.stderr.flush()
1049 1070
1050 proc = subprocess.Popen(["bitbake-runtask", fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE) 1071 proc = subprocess.Popen(["bitbake-runtask", self.rqdata.hashfile, fn, taskname, str(self.cooker.configuration.dry_run)], env=env, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
1051 pipein = proc.stdout 1072 pipein = proc.stdout
1052 pipeout = proc.stdin 1073 pipeout = proc.stdin
1053 pid = proc.pid 1074 pid = proc.pid
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index ef14471b66..eb624311a0 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -107,6 +107,10 @@ class SignatureGeneratorBasic(SignatureGenerator):
107 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) 107 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
108 return h 108 return h
109 109
110 def set_taskdata(self, hashes, deps):
111 self.runtaskdeps = deps
112 self.taskhash = hashes
113
110 def dump_sigtask(self, fn, task, stampbase, runtime): 114 def dump_sigtask(self, fn, task, stampbase, runtime):
111 k = fn + "." + task 115 k = fn + "." + task
112 if runtime == "customfile": 116 if runtime == "customfile":
@@ -128,7 +132,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
128 data['gendeps'][dep] = self.gendeps[fn][dep] 132 data['gendeps'][dep] = self.gendeps[fn][dep]
129 data['varvals'][dep] = self.lookupcache[fn][dep] 133 data['varvals'][dep] = self.lookupcache[fn][dep]
130 134
131 if runtime and runtime != "customfile": 135 if runtime:
132 data['runtaskdeps'] = self.runtaskdeps[k] 136 data['runtaskdeps'] = self.runtaskdeps[k]
133 data['runtaskhashes'] = {} 137 data['runtaskhashes'] = {}
134 for dep in data['runtaskdeps']: 138 for dep in data['runtaskdeps']: