summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJoshua Watt <jpewhacker@gmail.com>2018-02-12 13:39:58 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-02-16 18:05:40 +0000
commitfeded5fe2755fc6523c235fd6e82a0972e197c08 (patch)
tree91d80f0b7121167e9e7822ee6a450d56d6a06e60
parentc723f7c92a15daba5c8ec1d801ce8debb47d59bf (diff)
downloadpoky-feded5fe2755fc6523c235fd6e82a0972e197c08.tar.gz
utils.py: add parallel make helpers
The code to extract the integer number of parallel build threads and construct a new argument from them has started to be copied in multiple locations, so create two new helper utilities to aid recipes. The first helper (parallel_make()) extracts the integer number of parallel build threads from PARALLEL_MAKE. The second (parallel_make_argument()) does the same and then puts the result back into a format string, optionally clamping it to some maximum value. Additionally, rework the oe-core recipes that were manually doing this to use the new helper utilities. (From OE-Core rev: ccd1142d22b31ed85d8823b1bc9e11ccfd72b61f) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/go.bbclass19
-rw-r--r--meta/classes/waf.bbclass24
-rw-r--r--meta/lib/oe/utils.py43
-rw-r--r--meta/recipes-core/ovmf/ovmf_git.bb2
-rw-r--r--meta/recipes-support/boost/boost.inc24
5 files changed, 47 insertions, 65 deletions
diff --git a/meta/classes/go.bbclass b/meta/classes/go.bbclass
index 09b01a84c3..7ecd8c9254 100644
--- a/meta/classes/go.bbclass
+++ b/meta/classes/go.bbclass
@@ -1,23 +1,6 @@
1inherit goarch ptest 1inherit goarch ptest
2 2
3def get_go_parallel_make(d): 3GO_PARALLEL_BUILD ?= "${@oe.utils.parallel_make_argument(d, '-p %d')}"
4 pm = (d.getVar('PARALLEL_MAKE') or '').split()
5 # look for '-j' and throw other options (e.g. '-l') away
6 # because they might have a different meaning in golang
7 while pm:
8 opt = pm.pop(0)
9 if opt == '-j':
10 v = pm.pop(0)
11 elif opt.startswith('-j'):
12 v = opt[2:].strip()
13 else:
14 continue
15
16 return '-p %d' % int(v)
17
18 return ""
19
20GO_PARALLEL_BUILD ?= "${@get_go_parallel_make(d)}"
21 4
22GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go" 5GOROOT_class-native = "${STAGING_LIBDIR_NATIVE}/go"
23GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go" 6GOROOT_class-nativesdk = "${STAGING_DIR_TARGET}${libdir}/go"
diff --git a/meta/classes/waf.bbclass b/meta/classes/waf.bbclass
index bdbdc56767..f9a851d075 100644
--- a/meta/classes/waf.bbclass
+++ b/meta/classes/waf.bbclass
@@ -3,28 +3,6 @@ DISABLE_STATIC = ""
3 3
4EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}" 4EXTRA_OECONF_append = " ${PACKAGECONFIG_CONFARGS}"
5 5
6def get_waf_parallel_make(d):
7 pm = d.getVar('PARALLEL_MAKE')
8 if pm:
9 # look for '-j' and throw other options (e.g. '-l') away
10 # because they might have different meaning in bjam
11 pm = pm.split()
12 while pm:
13 v = None
14 opt = pm.pop(0)
15 if opt == '-j':
16 v = pm.pop(0)
17 elif opt.startswith('-j'):
18 v = opt[2:].strip()
19 else:
20 v = None
21
22 if v:
23 v = min(64, int(v))
24 return '-j' + str(v)
25
26 return ""
27
28python waf_preconfigure() { 6python waf_preconfigure() {
29 import subprocess 7 import subprocess
30 from distutils.version import StrictVersion 8 from distutils.version import StrictVersion
@@ -47,7 +25,7 @@ waf_do_configure() {
47 25
48do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+" 26do_compile[progress] = "outof:^\[\s*(\d+)/\s*(\d+)\]\s+"
49waf_do_compile() { 27waf_do_compile() {
50 ${S}/waf build ${@get_waf_parallel_make(d)} 28 ${S}/waf build ${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}
51} 29}
52 30
53waf_do_install() { 31waf_do_install() {
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index 7a79d752b6..ec91927233 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -156,6 +156,49 @@ def any_distro_features(d, features, truevalue="1", falsevalue=""):
156 """ 156 """
157 return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d) 157 return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
158 158
159def parallel_make(d):
160 """
161 Return the integer value for the number of parallel threads to use when
162 building, scraped out of PARALLEL_MAKE. If no parallelization option is
163 found, returns None
164
165 e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
166 """
167 pm = (d.getVar('PARALLEL_MAKE') or '').split()
168 # look for '-j' and throw other options (e.g. '-l') away
169 while pm:
170 opt = pm.pop(0)
171 if opt == '-j':
172 v = pm.pop(0)
173 elif opt.startswith('-j'):
174 v = opt[2:].strip()
175 else:
176 continue
177
178 return int(v)
179
180 return None
181
182def parallel_make_argument(d, fmt, limit=None):
183 """
184 Helper utility to construct a parallel make argument from the number of
185 parallel threads specified in PARALLEL_MAKE.
186
187 Returns the input format string `fmt` where a single '%d' will be expanded
188 with the number of parallel threads to use. If `limit` is specified, the
189 number of parallel threads will be no larger than it. If no parallelization
190 option is found in PARALLEL_MAKE, returns an empty string
191
192 e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
193 "-n 10"
194 """
195 v = parallel_make(d)
196 if v:
197 if limit:
198 v = max(limit, v)
199 return fmt % v
200 return ''
201
159def packages_filter_out_system(d): 202def packages_filter_out_system(d):
160 """ 203 """
161 Return a list of packages from PACKAGES with the "system" packages such as 204 Return a list of packages from PACKAGES with the "system" packages such as
diff --git a/meta/recipes-core/ovmf/ovmf_git.bb b/meta/recipes-core/ovmf/ovmf_git.bb
index fa0d66291d..8750b3c528 100644
--- a/meta/recipes-core/ovmf/ovmf_git.bb
+++ b/meta/recipes-core/ovmf/ovmf_git.bb
@@ -151,7 +151,7 @@ do_compile_class-native() {
151 151
152do_compile_class-target() { 152do_compile_class-target() {
153 export LFLAGS="${LDFLAGS}" 153 export LFLAGS="${LDFLAGS}"
154 PARALLEL_JOBS="${@ '${PARALLEL_MAKE}'.replace('-j', '-n ')}" 154 PARALLEL_JOBS="${@oe.utils.parallel_make_argument(d, '-n %d')}"
155 OVMF_ARCH="X64" 155 OVMF_ARCH="X64"
156 if [ "${TARGET_ARCH}" != "x86_64" ] ; then 156 if [ "${TARGET_ARCH}" != "x86_64" ] ; then
157 OVMF_ARCH="IA32" 157 OVMF_ARCH="IA32"
diff --git a/meta/recipes-support/boost/boost.inc b/meta/recipes-support/boost/boost.inc
index 41fc90fb21..0461ec6fcf 100644
--- a/meta/recipes-support/boost/boost.inc
+++ b/meta/recipes-support/boost/boost.inc
@@ -135,29 +135,7 @@ BJAM_TOOLS = "--ignore-site-config \
135 135
136# use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j 136# use PARALLEL_MAKE to speed up the build, but limit it by -j 64, greater parallelism causes bjam to segfault or to ignore -j
137# https://svn.boost.org/trac/boost/ticket/7634 137# https://svn.boost.org/trac/boost/ticket/7634
138def get_boost_parallel_make(d): 138BOOST_PARALLEL_MAKE = "${@oe.utils.parallel_make_argument(d, '-j%d', limit=64)}"
139 pm = d.getVar('PARALLEL_MAKE')
140 if pm:
141 # look for '-j' and throw other options (e.g. '-l') away
142 # because they might have different meaning in bjam
143 pm = pm.split()
144 while pm:
145 v = None
146 opt = pm.pop(0)
147 if opt == '-j':
148 v = pm.pop(0)
149 elif opt.startswith('-j'):
150 v = opt[2:].strip()
151 else:
152 v = None
153
154 if v:
155 v = min(64, int(v))
156 return '-j' + str(v)
157
158 return ""
159
160BOOST_PARALLEL_MAKE = "${@get_boost_parallel_make(d)}"
161BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \ 139BJAM_OPTS = '${BOOST_PARALLEL_MAKE} -d+2 -q \
162 ${BJAM_TOOLS} \ 140 ${BJAM_TOOLS} \
163 -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \ 141 -sBOOST_BUILD_USER_CONFIG=${WORKDIR}/user-config.jam \