summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorAnuj Mittal <anuj.mittal@intel.com>2018-08-16 14:26:33 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-16 22:40:28 +0100
commit371283df2e315de452891c7ea5f7563ed19cdc88 (patch)
treecfb4c09a1a91c7c80ebef1a32e8e46337fbf331e /meta
parente00b4617797ce89865a6f8318b6abedcddae3046 (diff)
downloadpoky-371283df2e315de452891c7ea5f7563ed19cdc88.tar.gz
python3: enable profile optimized builds
Enable profile guided optimization (pgo) for python3. Enabling pgo in python is generally as simple as invoking the target profile-opt which: - builds python binaries with profile instrumentation enabled, - runs a specific profile task using that python to get the profile data and, - feeds the compiler with this profile data and rebuilds python. This change invokes qemu-user for the second step of running a profile task using target python. Depending on how long profile task takes to run, this might add a significant time to compilation (which would be true for native builds too). The default profile task can be changed by the users depending on what makes sense for their use case (or can be left empty). In case qemu-user isn't supported, profile task won't be run. (From OE-Core rev: 05a2a53f9cc7e75b4a3838ab9368cadf0f15ba1b) Signed-off-by: Anuj Mittal <anuj.mittal@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch40
-rw-r--r--meta/recipes-devtools/python/python3/Use-correct-CFLAGS-for-extensions-when-cross-compili.patch56
-rw-r--r--meta/recipes-devtools/python/python3_3.5.5.bb55
3 files changed, 130 insertions, 21 deletions
diff --git a/meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch b/meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch
new file mode 100644
index 0000000000..bb01c5bcb4
--- /dev/null
+++ b/meta/recipes-devtools/python/python3/0002-Makefile-add-target-to-split-profile-generation.patch
@@ -0,0 +1,40 @@
1From 98586d6dc598e40b8b821b0dde57599e188a7ca4 Mon Sep 17 00:00:00 2001
2From: Anuj Mittal <anuj.mittal@intel.com>
3Date: Tue, 7 Aug 2018 16:43:17 +0800
4Subject: [PATCH 2/2] Makefile: add target to split profile generation
5
6We don't want to have profile task invoked from here and want to use
7qemu-user instead. Split the profile-opt task so qemu can be invoked
8once binaries have been built with instrumentation and then we can go
9ahead and build again using the profile data generated.
10
11Upstream-Status: Inappropriate [OE-specific]
12
13Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>
14---
15 Makefile.pre.in | 6 ++----
16 1 file changed, 2 insertions(+), 4 deletions(-)
17
18diff --git a/Makefile.pre.in b/Makefile.pre.in
19index 84bc3ff..017a2c4 100644
20--- a/Makefile.pre.in
21+++ b/Makefile.pre.in
22@@ -469,13 +469,11 @@ profile-opt:
23 $(MAKE) profile-removal
24 $(MAKE) build_all_generate_profile
25 $(MAKE) profile-removal
26- @echo "Running code to generate profile data (this can take a while):"
27- $(MAKE) run_profile_task
28- $(MAKE) build_all_merge_profile
29+
30+clean_and_use_profile:
31 @echo "Rebuilding with profile guided optimizations:"
32 $(MAKE) clean
33 $(MAKE) build_all_use_profile
34- $(MAKE) profile-removal
35
36 build_all_generate_profile:
37 $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(EXTRA_CFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LIBS="$(LIBS)"
38--
392.17.1
40
diff --git a/meta/recipes-devtools/python/python3/Use-correct-CFLAGS-for-extensions-when-cross-compili.patch b/meta/recipes-devtools/python/python3/Use-correct-CFLAGS-for-extensions-when-cross-compili.patch
new file mode 100644
index 0000000000..ae74ef531d
--- /dev/null
+++ b/meta/recipes-devtools/python/python3/Use-correct-CFLAGS-for-extensions-when-cross-compili.patch
@@ -0,0 +1,56 @@
1From 7fd121bb7d6c25c2e0a1c31cf76fb9bd4a9794de Mon Sep 17 00:00:00 2001
2From: Markus Lehtonen <markus.lehtonen@linux.intel.com>
3Date: Tue, 14 Aug 2018 14:11:35 +0800
4Subject: [PATCH 1/2] Use correct CFLAGS for extensions when cross-compiling
5
6Take PY_CFLAGS_NODIST into account, like in native build. This is needed
7in order to to profile-optimized build. Also, pass EXTRA_CFLAGS to
8profile-optimized build.
9
10Upstream-Status: Pending
11
12Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
13---
14 Makefile.pre.in | 4 ++--
15 setup.py | 3 ++-
16 2 files changed, 4 insertions(+), 3 deletions(-)
17
18diff --git a/Makefile.pre.in b/Makefile.pre.in
19index e2d5d3d..84bc3ff 100644
20--- a/Makefile.pre.in
21+++ b/Makefile.pre.in
22@@ -478,7 +478,7 @@ profile-opt:
23 $(MAKE) profile-removal
24
25 build_all_generate_profile:
26- $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LIBS="$(LIBS)"
27+ $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(EXTRA_CFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG) @LTOFLAGS@" LIBS="$(LIBS)"
28
29 run_profile_task:
30 : # FIXME: can't run for a cross build
31@@ -488,7 +488,7 @@ build_all_merge_profile:
32 $(LLVM_PROF_MERGER)
33
34 build_all_use_profile:
35- $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG) @LTOFLAGS@" LDFLAGS="$(LDFLAGS) @LTOFLAGS@"
36+ $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(EXTRA_CFLAGS) $(PGO_PROF_USE_FLAG) @LTOFLAGS@" LDFLAGS="$(LDFLAGS) @LTOFLAGS@"
37
38 # Compile and run with gcov
39 .PHONY=coverage coverage-lcov coverage-report
40diff --git a/setup.py b/setup.py
41index add3346..65e83b1 100644
42--- a/setup.py
43+++ b/setup.py
44@@ -263,7 +263,8 @@ class PyBuildExt(build_ext):
45 # compilers
46 if compiler is not None:
47 if cross_compiling:
48- (ccshared,cflags) = (os.environ.get('CCSHARED') or '', os.environ.get('CFLAGS') or '')
49+ (ccshared,cflags) = (os.environ.get('CCSHARED') or '',
50+ (os.environ.get('CFLAGS') or '') + ' ' + sysconfig.get_config_var('PY_CFLAGS_NODIST'))
51 else:
52 (ccshared,cflags) = sysconfig.get_config_vars('CCSHARED','CFLAGS')
53 args['compiler_so'] = compiler + ' ' + ccshared + ' ' + cflags
54--
552.17.1
56
diff --git a/meta/recipes-devtools/python/python3_3.5.5.bb b/meta/recipes-devtools/python/python3_3.5.5.bb
index 50f63e39c1..beaf3dc0ca 100644
--- a/meta/recipes-devtools/python/python3_3.5.5.bb
+++ b/meta/recipes-devtools/python/python3_3.5.5.bb
@@ -1,6 +1,6 @@
1require recipes-devtools/python/python.inc 1require recipes-devtools/python/python.inc
2 2
3DEPENDS = "python3-native libffi bzip2 gdbm openssl sqlite3 zlib virtual/libintl xz" 3DEPENDS = "python3-native libffi bzip2 gdbm openssl sqlite3 zlib virtual/libintl xz qemu-native qemu-helper-native"
4 4
5PR = "${INC_PR}.0" 5PR = "${INC_PR}.0"
6PYTHON_MAJMIN = "3.5" 6PYTHON_MAJMIN = "3.5"
@@ -37,6 +37,8 @@ SRC_URI += "\
37 file://configure.ac-fix-LIBPL.patch \ 37 file://configure.ac-fix-LIBPL.patch \
38 file://0001-Issue-21272-Use-_sysconfigdata.py-to-initialize-dist.patch \ 38 file://0001-Issue-21272-Use-_sysconfigdata.py-to-initialize-dist.patch \
39 file://pass-missing-libraries-to-Extension-for-mul.patch \ 39 file://pass-missing-libraries-to-Extension-for-mul.patch \
40 file://Use-correct-CFLAGS-for-extensions-when-cross-compili.patch \
41 file://0002-Makefile-add-target-to-split-profile-generation.patch \
40 " 42 "
41SRC_URI[md5sum] = "f3763edf9824d5d3a15f5f646083b6e0" 43SRC_URI[md5sum] = "f3763edf9824d5d3a15f5f646083b6e0"
42SRC_URI[sha256sum] = "063d2c3b0402d6191b90731e0f735c64830e7522348aeb7ed382a83165d45009" 44SRC_URI[sha256sum] = "063d2c3b0402d6191b90731e0f735c64830e7522348aeb7ed382a83165d45009"
@@ -48,7 +50,7 @@ UPSTREAM_CHECK_REGEX = "[Pp]ython-(?P<pver>\d+(\.\d+)+).tar"
48 50
49S = "${WORKDIR}/Python-${PV}" 51S = "${WORKDIR}/Python-${PV}"
50 52
51inherit autotools multilib_header python3native pkgconfig update-alternatives 53inherit autotools multilib_header python3native pkgconfig update-alternatives qemu
52 54
53MULTILIB_SUFFIX = "${@d.getVar('base_libdir',1).split('/')[-1]}" 55MULTILIB_SUFFIX = "${@d.getVar('base_libdir',1).split('/')[-1]}"
54 56
@@ -69,7 +71,8 @@ CACHED_CONFIGUREVARS = "ac_cv_have_chflags=no \
69TARGET_CC_ARCH += "-DNDEBUG -fno-inline" 71TARGET_CC_ARCH += "-DNDEBUG -fno-inline"
70SDK_CC_ARCH += "-DNDEBUG -fno-inline" 72SDK_CC_ARCH += "-DNDEBUG -fno-inline"
71EXTRA_OEMAKE += "CROSS_COMPILE=yes" 73EXTRA_OEMAKE += "CROSS_COMPILE=yes"
72EXTRA_OECONF += "CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ --without-ensurepip" 74EXTRA_OECONF += "CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ --without-ensurepip --enable-optimizations"
75PYTHON3_PROFILE_TASK ?= "${S}/Tools/pybench/pybench.py -n 10"
73 76
74export CROSS_COMPILE = "${TARGET_PREFIX}" 77export CROSS_COMPILE = "${TARGET_PREFIX}"
75export _PYTHON_PROJECT_BASE = "${B}" 78export _PYTHON_PROJECT_BASE = "${B}"
@@ -102,7 +105,6 @@ do_compile() {
102 sed -e 's,${STAGING_DIR_HOST},,g' -i *.py 105 sed -e 's,${STAGING_DIR_HOST},,g' -i *.py
103 cd - 106 cd -
104 107
105
106 # remove any bogus LD_LIBRARY_PATH 108 # remove any bogus LD_LIBRARY_PATH
107 sed -i -e s,RUNSHARED=.*,RUNSHARED=, Makefile 109 sed -i -e s,RUNSHARED=.*,RUNSHARED=, Makefile
108 110
@@ -121,23 +123,34 @@ do_compile() {
121 # then call do_install twice we get Makefile.orig == Makefile.sysroot 123 # then call do_install twice we get Makefile.orig == Makefile.sysroot
122 install -m 0644 Makefile Makefile.sysroot 124 install -m 0644 Makefile Makefile.sysroot
123 125
124 oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \ 126 oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
125 HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \ 127 HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \
126 STAGING_LIBDIR=${STAGING_LIBDIR} \ 128 STAGING_LIBDIR=${STAGING_LIBDIR} \
127 STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ 129 STAGING_INCDIR=${STAGING_INCDIR} \
128 STAGING_INCDIR=${STAGING_INCDIR} \ 130 STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
129 LIB=${baselib} \ 131 LIB=${baselib} \
130 ARCH=${TARGET_ARCH} \ 132 ARCH=${TARGET_ARCH} \
131 OPT="${CFLAGS}" libpython3.so 133 OPT="${CFLAGS}" profile-opt
134
135 if ${@bb.utils.contains('MACHINE_FEATURES', 'qemu-usermode', 'true', 'false', d)}; then
136 qemu_binary="${@qemu_wrapper_cmdline(d, '${STAGING_DIR_TARGET}', ['${B}', '${STAGING_DIR_TARGET}/${base_libdir}'])}"
137 cat > pgo-image-qemuwrapper << EOF
138#!/bin/sh
139set -x
140$qemu_binary "\$@"
141EOF
142 chmod +x pgo-image-qemuwrapper
143 ./pgo-image-qemuwrapper ${B}/python ${PYTHON3_PROFILE_TASK} || true
144 fi
132 145
133 oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \ 146 oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
134 HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \ 147 HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \
135 STAGING_LIBDIR=${STAGING_LIBDIR} \ 148 STAGING_LIBDIR=${STAGING_LIBDIR} \
136 STAGING_INCDIR=${STAGING_INCDIR} \ 149 STAGING_INCDIR=${STAGING_INCDIR} \
137 STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ 150 STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
138 LIB=${baselib} \ 151 LIB=${baselib} \
139 ARCH=${TARGET_ARCH} \ 152 ARCH=${TARGET_ARCH} \
140 OPT="${CFLAGS}" 153 OPT="${CFLAGS}" clean_and_use_profile
141} 154}
142 155
143do_install() { 156do_install() {
@@ -157,7 +170,7 @@ do_install() {
157 STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ 170 STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \
158 LIB=${baselib} \ 171 LIB=${baselib} \
159 ARCH=${TARGET_ARCH} \ 172 ARCH=${TARGET_ARCH} \
160 DESTDIR=${D} LIBDIR=${libdir} 173 DESTDIR=${D} LIBDIR=${libdir} build_all_use_profile
161 174
162 oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \ 175 oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python3-native/pgen \
163 HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \ 176 HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python3-native/python3 \