summaryrefslogtreecommitdiffstats
path: root/meta/classes/icecc.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes/icecc.bbclass')
-rw-r--r--meta/classes/icecc.bbclass459
1 files changed, 0 insertions, 459 deletions
diff --git a/meta/classes/icecc.bbclass b/meta/classes/icecc.bbclass
deleted file mode 100644
index d095305ed8..0000000000
--- a/meta/classes/icecc.bbclass
+++ /dev/null
@@ -1,459 +0,0 @@
1# IceCream distributed compiling support
2#
3# Stages directories with symlinks from gcc/g++ to icecc, for both
4# native and cross compilers. Depending on each configure or compile,
5# the directories are added at the head of the PATH list and ICECC_CXX
6# and ICEC_CC are set.
7#
8# For the cross compiler, creates a tar.gz of our toolchain and sets
9# ICECC_VERSION accordingly.
10#
11# The class now handles all 3 different compile 'stages' (i.e native ,cross-kernel and target) creating the
12# necessary environment tar.gz file to be used by the remote machines.
13# It also supports meta-toolchain generation
14#
15# If ICECC_PATH is not set in local.conf then the class will try to locate it using 'bb.utils.which'
16# but nothing is sure ;)
17#
18# If ICECC_ENV_EXEC is set in local.conf, then it should point to the icecc-create-env script provided by the user
19# or the default one provided by icecc-create-env.bb will be used
20# (NOTE that this is a modified version of the script need it and *not the one that comes with icecc*
21#
22# User can specify if specific packages or packages belonging to class should not use icecc to distribute
23# compile jobs to remote machines, but handled locally, by defining ICECC_USER_CLASS_BL and ICECC_USER_PACKAGE_BL
24# with the appropriate values in local.conf. In addition the user can force to enable icecc for packages
25# which set an empty PARALLEL_MAKE variable by defining ICECC_USER_PACKAGE_WL.
26#
27#########################################################################################
28#Error checking is kept to minimum so double check any parameters you pass to the class
29###########################################################################################
30
31BB_HASHBASE_WHITELIST += "ICECC_PARALLEL_MAKE ICECC_DISABLED ICECC_USER_PACKAGE_BL \
32 ICECC_USER_CLASS_BL ICECC_USER_PACKAGE_WL ICECC_PATH ICECC_ENV_EXEC \
33 ICECC_CARET_WORKAROUND ICECC_CFLAGS ICECC_ENV_VERSION \
34 ICECC_DEBUG ICECC_LOGFILE ICECC_REPEAT_RATE ICECC_PREFERRED_HOST \
35 ICECC_CLANG_REMOTE_CPP ICECC_IGNORE_UNVERIFIED ICECC_TEST_SOCKET \
36 ICECC_ENV_DEBUG ICECC_SYSTEM_PACKAGE_BL ICECC_SYSTEM_CLASS_BL \
37 ICECC_REMOTE_CPP \
38 "
39
40ICECC_ENV_EXEC ?= "${STAGING_BINDIR_NATIVE}/icecc-create-env"
41
42HOSTTOOLS_NONFATAL += "icecc patchelf"
43
44# This version can be incremented when changes are made to the environment that
45# invalidate the version on the compile nodes. Changing it will cause a new
46# environment to be created.
47#
48# A useful thing to do for testing Icecream changes locally is to add a
49# subversion in local.conf:
50# ICECC_ENV_VERSION_append = "-my-ver-1"
51ICECC_ENV_VERSION = "2"
52
53# Default to disabling the caret workaround, If set to "1" in local.conf, icecc
54# will locally recompile any files that have warnings, which can adversely
55# affect performance.
56#
57# See: https://github.com/icecc/icecream/issues/190
58export ICECC_CARET_WORKAROUND ??= "0"
59
60export ICECC_REMOTE_CPP ??= "0"
61
62ICECC_CFLAGS = ""
63CFLAGS += "${ICECC_CFLAGS}"
64CXXFLAGS += "${ICECC_CFLAGS}"
65
66# Debug flags when generating environments
67ICECC_ENV_DEBUG ??= ""
68
69# "system" recipe blacklist contains a list of packages that can not distribute
70# compile tasks for one reason or the other. When adding new entry, please
71# document why (how it failed) so that we can re-evaluate it later e.g. when
72# there is new version
73#
74# libgcc-initial - fails with CPP sanity check error if host sysroot contains
75# cross gcc built for another target tune/variant
76# pixman - prng_state: TLS reference mismatches non-TLS reference, possibly due to
77# pragma omp threadprivate(prng_state)
78# systemtap - _HelperSDT.c undefs macros and uses the identifiers in macros emitting
79# inline assembly
80# target-sdk-provides-dummy - ${HOST_PREFIX} is empty which triggers the "NULL
81# prefix" error.
82ICECC_SYSTEM_PACKAGE_BL += "\
83 libgcc-initial \
84 pixman \
85 systemtap \
86 target-sdk-provides-dummy \
87 "
88
89# "system" classes that should be blacklisted. When adding new entry, please
90# document why (how it failed) so that we can re-evaluate it later
91#
92# image - Image aren't compiling, but the testing framework for images captures
93# PARALLEL_MAKE as part of the test environment. Many tests won't use
94# icecream, but leaving the high level of parallelism can cause them to
95# consume an unnecessary amount of resources.
96ICECC_SYSTEM_CLASS_BL += "\
97 image \
98 "
99
100def icecc_dep_prepend(d):
101 # INHIBIT_DEFAULT_DEPS doesn't apply to the patch command. Whether or not
102 # we need that built is the responsibility of the patch function / class, not
103 # the application.
104 if not d.getVar('INHIBIT_DEFAULT_DEPS'):
105 return "icecc-create-env-native"
106 return ""
107
108DEPENDS_prepend = "${@icecc_dep_prepend(d)} "
109
110get_cross_kernel_cc[vardepsexclude] += "KERNEL_CC"
111def get_cross_kernel_cc(bb,d):
112 if not icecc_is_kernel(bb, d):
113 return None
114
115 # evaluate the expression by the shell if necessary
116 kernel_cc = d.getVar('KERNEL_CC')
117 if '`' in kernel_cc or '$(' in kernel_cc:
118 import subprocess
119 kernel_cc = subprocess.check_output("echo %s" % kernel_cc, shell=True).decode("utf-8")[:-1]
120
121 kernel_cc = kernel_cc.replace('ccache', '').strip()
122 kernel_cc = kernel_cc.split(' ')[0]
123 kernel_cc = kernel_cc.strip()
124 return kernel_cc
125
126def get_icecc(d):
127 return d.getVar('ICECC_PATH') or bb.utils.which(os.getenv("PATH"), "icecc")
128
129def use_icecc(bb,d):
130 if d.getVar('ICECC_DISABLED') == "1":
131 # don't even try it, when explicitly disabled
132 return "no"
133
134 # allarch recipes don't use compiler
135 if icecc_is_allarch(bb, d):
136 return "no"
137
138 if icecc_is_cross_canadian(bb, d):
139 return "no"
140
141 if d.getVar('INHIBIT_DEFAULT_DEPS', False):
142 # We don't have a compiler, so no icecc
143 return "no"
144
145 pn = d.getVar('PN')
146 bpn = d.getVar('BPN')
147
148 # Blacklist/whitelist checks are made against BPN, because there is a good
149 # chance that if icecc should be skipped for a recipe, it should be skipped
150 # for all the variants of that recipe. PN is still checked in case a user
151 # specified a more specific recipe.
152 check_pn = set([pn, bpn])
153
154 system_class_blacklist = (d.getVar('ICECC_SYSTEM_CLASS_BL') or "").split()
155 user_class_blacklist = (d.getVar('ICECC_USER_CLASS_BL') or "none").split()
156 package_class_blacklist = system_class_blacklist + user_class_blacklist
157
158 for black in package_class_blacklist:
159 if bb.data.inherits_class(black, d):
160 bb.debug(1, "%s: class %s found in blacklist, disable icecc" % (pn, black))
161 return "no"
162
163 system_package_blacklist = (d.getVar('ICECC_SYSTEM_PACKAGE_BL') or "").split()
164 user_package_blacklist = (d.getVar('ICECC_USER_PACKAGE_BL') or "").split()
165 user_package_whitelist = (d.getVar('ICECC_USER_PACKAGE_WL') or "").split()
166 package_blacklist = system_package_blacklist + user_package_blacklist
167
168 if check_pn & set(package_blacklist):
169 bb.debug(1, "%s: found in blacklist, disable icecc" % pn)
170 return "no"
171
172 if check_pn & set(user_package_whitelist):
173 bb.debug(1, "%s: found in whitelist, enable icecc" % pn)
174 return "yes"
175
176 if d.getVar('PARALLEL_MAKE') == "":
177 bb.debug(1, "%s: has empty PARALLEL_MAKE, disable icecc" % pn)
178 return "no"
179
180 return "yes"
181
182def icecc_is_allarch(bb, d):
183 return d.getVar("PACKAGE_ARCH") == "all"
184
185def icecc_is_kernel(bb, d):
186 return \
187 bb.data.inherits_class("kernel", d);
188
189def icecc_is_native(bb, d):
190 return \
191 bb.data.inherits_class("cross", d) or \
192 bb.data.inherits_class("native", d);
193
194def icecc_is_cross_canadian(bb, d):
195 return bb.data.inherits_class("cross-canadian", d)
196
197def icecc_dir(bb, d):
198 return d.expand('${TMPDIR}/work-shared/ice')
199
200# Don't pollute allarch signatures with TARGET_FPU
201icecc_version[vardepsexclude] += "TARGET_FPU"
202def icecc_version(bb, d):
203 if use_icecc(bb, d) == "no":
204 return ""
205
206 parallel = d.getVar('ICECC_PARALLEL_MAKE') or ""
207 if not d.getVar('PARALLEL_MAKE') == "" and parallel:
208 d.setVar("PARALLEL_MAKE", parallel)
209
210 # Disable showing the caret in the GCC compiler output if the workaround is
211 # disabled
212 if d.getVar('ICECC_CARET_WORKAROUND') == '0':
213 d.setVar('ICECC_CFLAGS', '-fno-diagnostics-show-caret')
214
215 if icecc_is_native(bb, d):
216 archive_name = "local-host-env"
217 elif d.expand('${HOST_PREFIX}') == "":
218 bb.fatal(d.expand("${PN}"), " NULL prefix")
219 else:
220 prefix = d.expand('${HOST_PREFIX}' )
221 distro = d.expand('${DISTRO}')
222 target_sys = d.expand('${TARGET_SYS}')
223 float = d.getVar('TARGET_FPU') or "hard"
224 archive_name = prefix + distro + "-" + target_sys + "-" + float
225 if icecc_is_kernel(bb, d):
226 archive_name += "-kernel"
227
228 import socket
229 ice_dir = icecc_dir(bb, d)
230 tar_file = os.path.join(ice_dir, "{archive}-{version}-@VERSION@-{hostname}.tar.gz".format(
231 archive=archive_name,
232 version=d.getVar('ICECC_ENV_VERSION'),
233 hostname=socket.gethostname()
234 ))
235
236 return tar_file
237
238def icecc_path(bb,d):
239 if use_icecc(bb, d) == "no":
240 # don't create unnecessary directories when icecc is disabled
241 return
242
243 staging = os.path.join(d.expand('${STAGING_BINDIR}'), "ice")
244 if icecc_is_kernel(bb, d):
245 staging += "-kernel"
246
247 return staging
248
249def icecc_get_external_tool(bb, d, tool):
250 external_toolchain_bindir = d.expand('${EXTERNAL_TOOLCHAIN}${bindir_cross}')
251 target_prefix = d.expand('${TARGET_PREFIX}')
252 return os.path.join(external_toolchain_bindir, '%s%s' % (target_prefix, tool))
253
254def icecc_get_tool_link(tool, d):
255 import subprocess
256 try:
257 return subprocess.check_output("readlink -f %s" % tool, shell=True).decode("utf-8")[:-1]
258 except subprocess.CalledProcessError as e:
259 bb.note("icecc: one of the tools probably disappeared during recipe parsing, cmd readlink -f %s returned %d:\n%s" % (tool, e.returncode, e.output.decode("utf-8")))
260 return tool
261
262def icecc_get_path_tool(tool, d):
263 # This is a little ugly, but we want to make sure we add an actual
264 # compiler to the toolchain, not ccache. Some distros (e.g. Fedora)
265 # have ccache enabled by default using symlinks PATH, meaning ccache
266 # would be found first when looking for the compiler.
267 paths = os.getenv("PATH").split(':')
268 while True:
269 p, hist = bb.utils.which(':'.join(paths), tool, history=True)
270 if not p or os.path.basename(icecc_get_tool_link(p, d)) != 'ccache':
271 return p
272 paths = paths[len(hist):]
273
274 return ""
275
276# Don't pollute native signatures with target TUNE_PKGARCH through STAGING_BINDIR_TOOLCHAIN
277icecc_get_tool[vardepsexclude] += "STAGING_BINDIR_TOOLCHAIN"
278def icecc_get_tool(bb, d, tool):
279 if icecc_is_native(bb, d):
280 return icecc_get_path_tool(tool, d)
281 elif icecc_is_kernel(bb, d):
282 return icecc_get_path_tool(get_cross_kernel_cc(bb, d), d)
283 else:
284 ice_dir = d.expand('${STAGING_BINDIR_TOOLCHAIN}')
285 target_sys = d.expand('${TARGET_SYS}')
286 for p in ice_dir.split(':'):
287 tool_bin = os.path.join(p, "%s-%s" % (target_sys, tool))
288 if os.path.isfile(tool_bin):
289 return tool_bin
290 external_tool_bin = icecc_get_external_tool(bb, d, tool)
291 if os.path.isfile(external_tool_bin):
292 return external_tool_bin
293 return ""
294
295def icecc_get_and_check_tool(bb, d, tool):
296 # Check that g++ or gcc is not a symbolic link to icecc binary in
297 # PATH or icecc-create-env script will silently create an invalid
298 # compiler environment package.
299 t = icecc_get_tool(bb, d, tool)
300 if t:
301 link_path = icecc_get_tool_link(t, d)
302 if link_path == get_icecc(d):
303 bb.error("%s is a symlink to %s in PATH and this prevents icecc from working" % (t, link_path))
304 return ""
305 else:
306 return t
307 else:
308 return t
309
310wait_for_file() {
311 local TIME_ELAPSED=0
312 local FILE_TO_TEST=$1
313 local TIMEOUT=$2
314 until [ -f "$FILE_TO_TEST" ]
315 do
316 TIME_ELAPSED=`expr $TIME_ELAPSED + 1`
317 if [ $TIME_ELAPSED -gt $TIMEOUT ]
318 then
319 return 1
320 fi
321 sleep 1
322 done
323}
324
325def set_icecc_env():
326 # dummy python version of set_icecc_env
327 return
328
329set_icecc_env[vardepsexclude] += "KERNEL_CC"
330set_icecc_env() {
331 if [ "${@use_icecc(bb, d)}" = "no" ]
332 then
333 return
334 fi
335 ICECC_VERSION="${@icecc_version(bb, d)}"
336 if [ "x${ICECC_VERSION}" = "x" ]
337 then
338 bbwarn "Cannot use icecc: could not get ICECC_VERSION"
339 return
340 fi
341
342 ICE_PATH="${@icecc_path(bb, d)}"
343 if [ "x${ICE_PATH}" = "x" ]
344 then
345 bbwarn "Cannot use icecc: could not get ICE_PATH"
346 return
347 fi
348
349 ICECC_BIN="${@get_icecc(d)}"
350 if [ -z "${ICECC_BIN}" ]; then
351 bbwarn "Cannot use icecc: icecc binary not found"
352 return
353 fi
354 if [ -z "$(which patchelf patchelf-uninative)" ]; then
355 bbwarn "Cannot use icecc: patchelf not found"
356 return
357 fi
358
359 ICECC_CC="${@icecc_get_and_check_tool(bb, d, "gcc")}"
360 ICECC_CXX="${@icecc_get_and_check_tool(bb, d, "g++")}"
361 # cannot use icecc_get_and_check_tool here because it assumes as without target_sys prefix
362 ICECC_WHICH_AS="${@bb.utils.which(os.getenv('PATH'), 'as')}"
363 if [ ! -x "${ICECC_CC}" -o ! -x "${ICECC_CXX}" ]
364 then
365 bbwarn "Cannot use icecc: could not get ICECC_CC or ICECC_CXX"
366 return
367 fi
368
369 ICE_VERSION=`$ICECC_CC -dumpversion`
370 ICECC_VERSION=`echo ${ICECC_VERSION} | sed -e "s/@VERSION@/$ICE_VERSION/g"`
371 if [ ! -x "${ICECC_ENV_EXEC}" ]
372 then
373 bbwarn "Cannot use icecc: invalid ICECC_ENV_EXEC"
374 return
375 fi
376
377 # Create symlinks to icecc and wrapper-scripts in the recipe-sysroot directory
378 mkdir -p $ICE_PATH/symlinks
379 if [ -n "${KERNEL_CC}" ]; then
380 compilers="${@get_cross_kernel_cc(bb,d)}"
381 else
382 compilers="${HOST_PREFIX}gcc ${HOST_PREFIX}g++"
383 fi
384 for compiler in $compilers; do
385 ln -sf $ICECC_BIN $ICE_PATH/symlinks/$compiler
386 rm -f $ICE_PATH/$compiler
387 cat <<-__EOF__ > $ICE_PATH/$compiler
388 #!/bin/sh -e
389 export ICECC_VERSION=$ICECC_VERSION
390 export ICECC_CC=$ICECC_CC
391 export ICECC_CXX=$ICECC_CXX
392 $ICE_PATH/symlinks/$compiler "\$@"
393 __EOF__
394 chmod 775 $ICE_PATH/$compiler
395 done
396
397 ICECC_AS="`${ICECC_CC} -print-prog-name=as`"
398 # for target recipes should return something like:
399 # /OE/tmp-eglibc/sysroots/x86_64-linux/usr/libexec/arm920tt-oe-linux-gnueabi/gcc/arm-oe-linux-gnueabi/4.8.2/as
400 # and just "as" for native, if it returns "as" in current directory (for whatever reason) use "as" from PATH
401 if [ "`dirname "${ICECC_AS}"`" = "." ]
402 then
403 ICECC_AS="${ICECC_WHICH_AS}"
404 fi
405
406 if [ ! -f "${ICECC_VERSION}.done" ]
407 then
408 mkdir -p "`dirname "${ICECC_VERSION}"`"
409
410 # the ICECC_VERSION generation step must be locked by a mutex
411 # in order to prevent race conditions
412 if flock -n "${ICECC_VERSION}.lock" \
413 ${ICECC_ENV_EXEC} ${ICECC_ENV_DEBUG} "${ICECC_CC}" "${ICECC_CXX}" "${ICECC_AS}" "${ICECC_VERSION}"
414 then
415 touch "${ICECC_VERSION}.done"
416 elif ! wait_for_file "${ICECC_VERSION}.done" 30
417 then
418 # locking failed so wait for ${ICECC_VERSION}.done to appear
419 bbwarn "Timeout waiting for ${ICECC_VERSION}.done"
420 return
421 fi
422 fi
423
424 # Don't let ccache find the icecream compiler links that have been created, otherwise
425 # it can end up invoking icecream recursively.
426 export CCACHE_PATH="$PATH"
427 export CCACHE_DISABLE="1"
428
429 export PATH="$ICE_PATH:$PATH"
430
431 bbnote "Using icecc path: $ICE_PATH"
432 bbnote "Using icecc tarball: $ICECC_VERSION"
433}
434
435do_configure_prepend() {
436 set_icecc_env
437}
438
439do_compile_prepend() {
440 set_icecc_env
441}
442
443do_compile_kernelmodules_prepend() {
444 set_icecc_env
445}
446
447do_install_prepend() {
448 set_icecc_env
449}
450
451# IceCream is not (currently) supported in the extensible SDK
452ICECC_SDK_HOST_TASK = "nativesdk-icecc-toolchain"
453ICECC_SDK_HOST_TASK_task-populate-sdk-ext = ""
454
455# Don't include IceCream in uninative tarball
456ICECC_SDK_HOST_TASK_pn-uninative-tarball = ""
457
458# Add the toolchain scripts to the SDK
459TOOLCHAIN_HOST_TASK_append = " ${ICECC_SDK_HOST_TASK}"