From 1daa2c0e9ee2f27c9c14e9e4553f7e97e3b6b74a Mon Sep 17 00:00:00 2001 From: Scott Rifenbark Date: Wed, 11 Jun 2014 21:43:50 +0300 Subject: dev-manual: First draft of new section on debugging race conditions. Fixes [YOCTO #6390] This is a section on parallel make race situations. The draft is the first cut at the section. (From yocto-docs rev: c225d7fe121270a6f82b9fbffa78c7e3914b113d) Signed-off-by: Scott Rifenbark Signed-off-by: Richard Purdie --- .../dev-manual/dev-manual-common-tasks.xml | 241 +++++++++++++++++++++ 1 file changed, 241 insertions(+) (limited to 'documentation/dev-manual/dev-manual-common-tasks.xml') diff --git a/documentation/dev-manual/dev-manual-common-tasks.xml b/documentation/dev-manual/dev-manual-common-tasks.xml index 3922c8f0f4..70cf6adf33 100644 --- a/documentation/dev-manual/dev-manual-common-tasks.xml +++ b/documentation/dev-manual/dev-manual-common-tasks.xml @@ -7302,6 +7302,247 @@ Gateways via their Web Interfaces" +
+ Debugging Parallel Make Races + + + A parallel make race occurs when the build + consists of several parts that are run simultaneously and + a situation occurs when the output or result of one + part is not ready for use with a different part of the build that + depends on that output. + Parallel make races are annoying and can sometimes be difficult + to reproduce and fix. + However, some simple tips and tricks exist that can help + you debug and fix them. + This section presents a real-world example of an error encountered + on the Yocto Project autobuilder and the process used to fix it. + + +
+ The Failure + + + For this example, assume that you are building an image that + depends on the "neard" package. + And, during the build, BitBake runs into problems and + creates the following output. + + This example log file has longer lines artifically + broken to make the listing easier to read. + + If you examine the output or the log file, you see the + failure during the + do_compile + task for "neard": + + | DEBUG: SITE files ['endian-little', 'bit-32', 'ix86-common', 'common-linux', 'common-glibc', 'i586-linux', 'common'] + | DEBUG: Executing shell function do_compile + | NOTE: make -j 16 + | make --no-print-directory all-am + | /bin/mkdir -p include/near + | /bin/mkdir -p include/near + | /bin/mkdir -p include/near + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/types.h include/near/types.h + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/log.h include/near/log.h + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/plugin.h include/near/plugin.h + | /bin/mkdir -p include/near + | /bin/mkdir -p include/near + | /bin/mkdir -p include/near + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/tag.h include/near/tag.h + | /bin/mkdir -p include/near + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/adapter.h include/near/adapter.h + | /bin/mkdir -p include/near + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/ndef.h include/near/ndef.h + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/tlv.h include/near/tlv.h + | /bin/mkdir -p include/near + | /bin/mkdir -p include/near + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/setting.h include/near/setting.h + | /bin/mkdir -p include/near + | /bin/mkdir -p include/near + | /bin/mkdir -p include/near + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/device.h include/near/device.h + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/nfc_copy.h include/near/nfc_copy.h + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/snep.h include/near/snep.h + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/version.h include/near/version.h + | ln -s /home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/work/i586-poky-linux/neard/ + 0.14-r0/neard-0.14/include/dbus.h include/near/dbus.h + | ./src/genbuiltin nfctype1 nfctype2 nfctype3 nfctype4 p2p > src/builtin.h + | i586-poky-linux-gcc -m32 -march=i586 --sysroot=/home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/ + build/build/tmp/sysroots/qemux86 -DHAVE_CONFIG_H -I. -I./include -I./src -I./gdbus -I/home/pokybuild/ + yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/sysroots/qemux86/usr/include/glib-2.0 + -I/home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/tmp/sysroots/qemux86/usr/ + lib/glib-2.0/include -I/home/pokybuild/yocto-autobuilder/yocto-slave/nightly-x86/build/build/ + tmp/sysroots/qemux86/usr/include/dbus-1.0 -I/home/pokybuild/yocto-autobuilder/yocto-slave/ + nightly-x86/build/build/tmp/sysroots/qemux86/usr/lib/dbus-1.0/include -I/home/pokybuild/yocto-autobuilder/ + yocto-slave/nightly-x86/build/build/tmp/sysroots/qemux86/usr/include/libnl3 + -DNEAR_PLUGIN_BUILTIN -DPLUGINDIR=\""/usr/lib/near/plugins"\" + -DCONFIGDIR=\""/etc/neard\"" -O2 -pipe -g -feliminate-unused-debug-types -c + -o tools/snep-send.o tools/snep-send.c + | In file included from tools/snep-send.c:16:0: + | tools/../src/near.h:41:23: fatal error: near/dbus.h: No such file or directory + | #include <near/dbus.h> + | ^ + | compilation terminated. + | make[1]: *** [tools/snep-send.o] Error 1 + | make[1]: *** Waiting for unfinished jobs.... + | make: *** [all] Error 2 + | ERROR: oe_runmake failed + + +
+ +
+ Reproducing the Error + + + Because race conditions are intermittent, they do not + manifest themselves every time you do the build. + In fact, most times the build will complete without problems + even though the potential race condition exists. + Thus, once the error surfaces, you need a way to reproduce it. + + + + In this example, compiling the "neard" package is causing the + problem. + So the first thing to do is build "neard" locally. + Before you start the build, set the + PARALLEL_MAKE + variable in your local.conf file to + a high number (e.g. 20). + Using a high value for PARALLEL_MAKE + increases the chances of the race condition showing up: + + $ bitbake neard + + + + + Once the local build for "neard" completes, start a + devshell build: + + $ bitbake neard -c devshell + + For information on how to use a + devshell, see the + "Using a Development Shell" + section. + + + + In the devshell, do the following: + + $ make clean + $ make tools/snep-send.o + + The devshell commands cause the failure + to clearly be visible. + In this case, a missing dependency exists for the "neard" + Makefile target. + +
+ +
+ Creating a Patch for the Fix + + + Because there is a missing dependency for the Makefile + target, you need to patch the + Makefile.am file, which is generated + from Makefile.in. + You can use Quilt to create the patch: + + $ quilt new parallelmake.patch + $ quilt add Makefile.am + + For more information on using Quilt, see the + "Using a Quilt Workflow" + section. + + + + At this point you need to make the edits to + Makefile.am to add the missing + dependency. + For our example, you have to add the a + "tools/snep-send.$(OBJEXT): include/near/dbus.h" + line. + + + + Once you have edited the file, use the + refresh command to create the patch: + + $ quilt refresh + + Once the patch file exists, you need to add it back to the + originating recipe folder: + + $ cp patches/parallemake.patch <location-of-neard-0.14-recipe-folder> + + The final thing you need to do to implement the fix in the + build is to add the patch to the + SRC_URI + statement in the "neard" recipe + (neard-0.14.bb). + + + + With the patch complete and moved to the correct folder and + the SRC_URI statement updated, you can + exit the devshell: + + $ exit + + +
+ +
+ Testing the Build + + + With everything in place, you can get back to trying the + build again locally: + + $ bitbake neard + + This build should succeed. + + + + Now you can open up a devshell again + and repeat the clean and make operations as follows: + + $ make clean + $ make tools/snep-send.o + + The build should work without issue. + + + + As with all solved problems, if they originated upstream, you + need to submit the fix for the recipe in OE-Core and upstream + so that the problem is taken care of at its source. + See the + "How to Submit a Change" + section for more information. + +
+
+
Examining Builds Using the Toaster API -- cgit v1.2.3-54-g00ecf