summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2011-02-28 15:31:20 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-02-28 20:48:08 +0000
commit1b08a7eb8b708f4d0fc119cf89deb450fa62fea1 (patch)
treecdd4c9656d94264e6c55d8632788cb39a04d1d65
parentd5e12a1bfbb4d00b49209c602a68628b1b626898 (diff)
downloadpoky-1b08a7eb8b708f4d0fc119cf89deb450fa62fea1.tar.gz
bitbake/cache/runqueue.py: Move workload for recipe parsing to the child process
Parsing the recipe in the parent before forking off the child worker can mean the parent doesn't hit the idle loop and becomes a bottleneck when lauching many short lived processes. The reason we need this in the parent is to figure out the fakeroot environmental options. To address this, add the fakeroot variables to the cache and move recipe loadData into the child task. For a poky-image-sato build this results in about a 2 minute speedup (1.8%). Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cache.py10
-rw-r--r--bitbake/lib/bb/runqueue.py35
2 files changed, 32 insertions, 13 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 421bd79183..c56b4b4248 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -43,7 +43,7 @@ except ImportError:
43 logger.info("Importing cPickle failed. " 43 logger.info("Importing cPickle failed. "
44 "Falling back to a very slow implementation.") 44 "Falling back to a very slow implementation.")
45 45
46__cache_version__ = "137" 46__cache_version__ = "138"
47 47
48recipe_fields = ( 48recipe_fields = (
49 'pn', 49 'pn',
@@ -78,6 +78,8 @@ recipe_fields = (
78 'summary', 78 'summary',
79 'license', 79 'license',
80 'section', 80 'section',
81 'fakerootenv',
82 'fakerootdirs'
81) 83)
82 84
83 85
@@ -172,6 +174,8 @@ class RecipeInfo(namedtuple('RecipeInfo', recipe_fields)):
172 summary = cls.getvar('SUMMARY', metadata), 174 summary = cls.getvar('SUMMARY', metadata),
173 license = cls.getvar('LICENSE', metadata), 175 license = cls.getvar('LICENSE', metadata),
174 section = cls.getvar('SECTION', metadata), 176 section = cls.getvar('SECTION', metadata),
177 fakerootenv = cls.getvar('FAKEROOTENV', metadata),
178 fakerootdirs = cls.getvar('FAKEROOTDIRS', metadata),
175 ) 179 )
176 180
177 181
@@ -584,6 +588,8 @@ class CacheData(object):
584 self.summary = {} 588 self.summary = {}
585 self.license = {} 589 self.license = {}
586 self.section = {} 590 self.section = {}
591 self.fakerootenv = {}
592 self.fakerootdirs = {}
587 593
588 # Indirect Cache variables (set elsewhere) 594 # Indirect Cache variables (set elsewhere)
589 self.ignored_dependencies = [] 595 self.ignored_dependencies = []
@@ -647,3 +653,5 @@ class CacheData(object):
647 self.summary[fn] = info.summary 653 self.summary[fn] = info.summary
648 self.license[fn] = info.license 654 self.license[fn] = info.license
649 self.section[fn] = info.section 655 self.section[fn] = info.section
656 self.fakerootenv[fn] = info.fakerootenv
657 self.fakerootdirs[fn] = info.fakerootdirs
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 172e591522..d7d67fd508 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1060,27 +1060,23 @@ class RunQueueExecute:
1060 return 1060 return
1061 1061
1062 def fork_off_task(self, fn, task, taskname, quieterrors=False): 1062 def fork_off_task(self, fn, task, taskname, quieterrors=False):
1063 the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
1064 1063
1065 env = bb.data.export_vars(the_data) 1064 envbackup = os.environ.copy()
1066 env = bb.data.export_envvars(env, the_data) 1065 env = {}
1067 1066
1068 taskdep = self.rqdata.dataCache.task_deps[fn] 1067 taskdep = self.rqdata.dataCache.task_deps[fn]
1069 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']: 1068 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']:
1070 envvars = the_data.getVar("FAKEROOTENV", True).split() 1069 envvars = (self.rqdata.dataCache.fakerootenv[fn] or "").split()
1071 for var in envvars: 1070 for var in envvars:
1072 comps = var.split("=") 1071 comps = var.split("=")
1073 env[comps[0]] = comps[1] 1072 env[comps[0]] = comps[1]
1074 fakedirs = (the_data.getVar("FAKEROOTDIRS", True) or "").split() 1073
1074 fakedirs = (self.rqdata.dataCache.fakerootdirs[fn] or "").split()
1075 for p in fakedirs: 1075 for p in fakedirs:
1076 bb.mkdirhier(p) 1076 bb.mkdirhier(p)
1077 logger.debug(2, "Running %s:%s under fakeroot, state dir is %s" % (fn, taskname, fakedirs)) 1077 logger.debug(2, "Running %s:%s under fakeroot, state dir is %s" % (fn, taskname, fakedirs))
1078 1078 for e in env:
1079 envbackup = os.environ.copy() 1079 os.putenv(e, env[e])
1080 for e in envbackup:
1081 os.unsetenv(e)
1082 for e in env:
1083 os.putenv(e, env[e])
1084 1080
1085 sys.stdout.flush() 1081 sys.stdout.flush()
1086 sys.stderr.flush() 1082 sys.stderr.flush()
@@ -1111,6 +1107,20 @@ class RunQueueExecute:
1111 # No stdin 1107 # No stdin
1112 newsi = os.open(os.devnull, os.O_RDWR) 1108 newsi = os.open(os.devnull, os.O_RDWR)
1113 os.dup2(newsi, sys.stdin.fileno()) 1109 os.dup2(newsi, sys.stdin.fileno())
1110
1111
1112 the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.get_file_appends(fn), self.cooker.configuration.data)
1113
1114 env2 = bb.data.export_vars(the_data)
1115 env2 = bb.data.export_envvars(env2, the_data)
1116
1117 for e in os.environ:
1118 os.unsetenv(e)
1119 for e in env2:
1120 os.putenv(e, env2[e])
1121 for e in env:
1122 os.putenv(e, env[e])
1123
1114 if quieterrors: 1124 if quieterrors:
1115 the_data.setVarFlag(taskname, "quieterrors", "1") 1125 the_data.setVarFlag(taskname, "quieterrors", "1")
1116 1126
@@ -1137,7 +1147,8 @@ class RunQueueExecute:
1137 for e in env: 1147 for e in env:
1138 os.unsetenv(e) 1148 os.unsetenv(e)
1139 for e in envbackup: 1149 for e in envbackup:
1140 os.putenv(e, envbackup[e]) 1150 if e in env:
1151 os.putenv(e, envbackup[e])
1141 1152
1142 return pid, pipein, pipeout 1153 return pid, pipein, pipeout
1143 1154