summaryrefslogtreecommitdiffstats
path: root/meta/recipes-devtools/python
diff options
context:
space:
mode:
authorDerek Straka <derek@asterius.io>2018-01-30 12:22:49 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-02-06 11:06:27 +0000
commit39e48c8b066518909ea647bb3168243061fc1414 (patch)
tree4208257e1662982bd6fc9ea67b49c19de3245ba8 /meta/recipes-devtools/python
parent863df3c81f3f9786924c98aad01d1219c540fb26 (diff)
downloadpoky-39e48c8b066518909ea647bb3168243061fc1414.tar.gz
python3: update target and native recipes to 3.5.4
Use the latest 3.5 version until the 3.6 migration is complete Removed the following upstreamed patches: - python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch - python3/upstream-random-fixes.patch Rebased the following pathes: - python3/0001-cross-compile-support.patch Regenerated the manifest based on the latest release version Updated the license checksum for the latest version that updated the copyright dates (From OE-Core rev: eb80d0391d7d4e83a61ed8850d936b102be3fa02) Signed-off-by: Derek Straka <derek@asterius.io> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/recipes-devtools/python')
-rw-r--r--meta/recipes-devtools/python/python3-native_3.5.4.bb (renamed from meta/recipes-devtools/python/python3-native_3.5.3.bb)12
-rw-r--r--meta/recipes-devtools/python/python3/0001-cross-compile-support.patch69
-rw-r--r--meta/recipes-devtools/python/python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch56
-rw-r--r--meta/recipes-devtools/python/python3/python3-manifest.json8
-rw-r--r--meta/recipes-devtools/python/python3/upstream-random-fixes.patch703
-rw-r--r--meta/recipes-devtools/python/python3_3.5.4.bb (renamed from meta/recipes-devtools/python/python3_3.5.3.bb)8
6 files changed, 52 insertions, 804 deletions
diff --git a/meta/recipes-devtools/python/python3-native_3.5.3.bb b/meta/recipes-devtools/python/python3-native_3.5.4.bb
index 12f9f2404f..c8252539a9 100644
--- a/meta/recipes-devtools/python/python3-native_3.5.3.bb
+++ b/meta/recipes-devtools/python/python3-native_3.5.4.bb
@@ -27,10 +27,10 @@ file://0001-Do-not-use-the-shell-version-of-python-config-that-w.patch \
27file://support_SOURCE_DATE_EPOCH_in_py_compile.patch \ 27file://support_SOURCE_DATE_EPOCH_in_py_compile.patch \
28" 28"
29 29
30SRC_URI[md5sum] = "57d1f8bfbabf4f2500273fb0706e6f21" 30SRC_URI[md5sum] = "fb2780baa260b4e51cbea814f111f303"
31SRC_URI[sha256sum] = "eefe2ad6575855423ab630f5b51a8ef6e5556f774584c06beab4926f930ddbb0" 31SRC_URI[sha256sum] = "94d93bfabb3b109f8a10365a325f920f9ec98c6e2380bf228f9700a14054c84c"
32 32
33LIC_FILES_CHKSUM = "file://LICENSE;md5=b680ed99aa60d350c65a65914494207e" 33LIC_FILES_CHKSUM = "file://LICENSE;md5=f741e51de91d4eeea5930b9c3c7fa69d"
34 34
35# exclude pre-releases for both python 2.x and 3.x 35# exclude pre-releases for both python 2.x and 3.x
36UPSTREAM_CHECK_REGEX = "[Pp]ython-(?P<pver>\d+(\.\d+)+).tar" 36UPSTREAM_CHECK_REGEX = "[Pp]ython-(?P<pver>\d+(\.\d+)+).tar"
@@ -61,6 +61,12 @@ do_configure_append() {
61 sed -i -e 's,#define HAVE_GETRANDOM 1,/\* #undef HAVE_GETRANDOM \*/,' ${B}/pyconfig.h 61 sed -i -e 's,#define HAVE_GETRANDOM 1,/\* #undef HAVE_GETRANDOM \*/,' ${B}/pyconfig.h
62} 62}
63 63
64# Regenerate all of the generated files
65# This ensures that pgen and friends get created during the compile phase
66do_compile_prepend() {
67 oe_runmake regen-all
68}
69
64do_install() { 70do_install() {
65 install -d ${D}${libdir}/pkgconfig 71 install -d ${D}${libdir}/pkgconfig
66 oe_runmake 'DESTDIR=${D}' install 72 oe_runmake 'DESTDIR=${D}' install
diff --git a/meta/recipes-devtools/python/python3/0001-cross-compile-support.patch b/meta/recipes-devtools/python/python3/0001-cross-compile-support.patch
index 118d75ddc5..7cd7e3b490 100644
--- a/meta/recipes-devtools/python/python3/0001-cross-compile-support.patch
+++ b/meta/recipes-devtools/python/python3/0001-cross-compile-support.patch
@@ -1,4 +1,4 @@
1From 624c029abcc73c724020ccea9a2b4b5b5c00f2a6 Mon Sep 17 00:00:00 2001 1From ecde3ea170999a9ef734e8af4d7c25be5ba81697 Mon Sep 17 00:00:00 2001
2From: Alexander Kanavin <alex.kanavin@gmail.com> 2From: Alexander Kanavin <alex.kanavin@gmail.com>
3Date: Fri, 31 Mar 2017 15:42:46 +0300 3Date: Fri, 31 Mar 2017 15:42:46 +0300
4Subject: [PATCH] cross-compile support 4Subject: [PATCH] cross-compile support
@@ -8,60 +8,63 @@ python instead of in-tree tools
8 8
9-Khem 9-Khem
10 10
11Rebased on 3.5.4
12
11Upstream-Status: Inappropriate[Configuration Specific] 13Upstream-Status: Inappropriate[Configuration Specific]
12Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> 14Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
15Signed-off-by: Derek Straka <derek@asterius.io>
13--- 16---
14 Makefile.pre.in | 14 ++++++++------ 17 Makefile.pre.in | 14 ++++++++------
15 1 file changed, 8 insertions(+), 6 deletions(-) 18 1 file changed, 8 insertions(+), 6 deletions(-)
16 19
17diff --git a/Makefile.pre.in b/Makefile.pre.in 20diff --git a/Makefile.pre.in b/Makefile.pre.in
18index a88b7d5..7cb8bb3 100644 21index 144c1f8629..f252ac2417 100644
19--- a/Makefile.pre.in 22--- a/Makefile.pre.in
20+++ b/Makefile.pre.in 23+++ b/Makefile.pre.in
21@@ -221,6 +221,7 @@ LIBOBJS= @LIBOBJS@ 24@@ -223,6 +223,7 @@ LIBOBJS= @LIBOBJS@
22 25
23 PYTHON= python$(EXE) 26 PYTHON= python$(EXE)
24 BUILDPYTHON= python$(BUILDEXE) 27 BUILDPYTHON= python$(BUILDEXE)
25+HOSTPYTHON= $(BUILDPYTHON) 28+HOSTPYTHON= $(BUILDPYTHON)
26 29
27 PYTHON_FOR_GEN=@PYTHON_FOR_GEN@ 30 PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@
28 PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@ 31 PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
29@@ -280,6 +281,7 @@ LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ 32@@ -277,6 +278,7 @@ LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@
30 ########################################################################## 33 ##########################################################################
31 # Parser 34 # Parser
32 PGEN= Parser/pgen$(EXE) 35 PGEN= Parser/pgen$(EXE)
33+HOSTPGEN= $(PGEN)$(EXE) 36+HOSTPGEN= $(PGEN)$(EXE)
34 37
35 PSRCS= \ 38 PSRCS= \
36 Parser/acceler.c \ 39 Parser/acceler.c \
37@@ -510,7 +512,7 @@ build_all_generate_profile: 40@@ -478,7 +480,7 @@ build_all_generate_profile:
38 41
39 run_profile_task: 42 run_profile_task:
40 : # FIXME: can't run for a cross build 43 : # FIXME: can't run for a cross build
41- $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true 44- $(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
42+ $(LLVM_PROF_FILE) $(RUNSHARED) $(HOSTPYTHON) $(PROFILE_TASK) || true 45+ $(LLVM_PROF_FILE) $(RUNSHARED) $(HOSTPYTHON) $(PROFILE_TASK) || true
43 46
44 build_all_merge_profile: 47 build_all_merge_profile:
45 $(LLVM_PROF_MERGER) 48 $(LLVM_PROF_MERGER)
46@@ -787,7 +789,7 @@ $(IO_OBJS): $(IO_H) 49@@ -772,7 +774,7 @@ regen-grammar: $(PGEN)
47 50 # Regenerate Include/graminit.h and Python/graminit.c
48 $(GRAMMAR_H): @GENERATED_COMMENT@ $(GRAMMAR_INPUT) $(PGEN) 51 # from Grammar/Grammar using pgen
49 @$(MKDIR_P) Include 52 @$(MKDIR_P) Include
50- $(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) 53- $(PGEN) $(srcdir)/Grammar/Grammar \
51+ $(HOSTPGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C) 54+ $(HOSTPGEN) $(srcdir)/Grammar/Grammar \
52 $(GRAMMAR_C): @GENERATED_COMMENT@ $(GRAMMAR_H) 55 $(srcdir)/Include/graminit.h \
53 touch $(GRAMMAR_C) 56 $(srcdir)/Python/graminit.c
54 57
55@@ -976,7 +978,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS) 58@@ -978,7 +980,7 @@ $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS)
56 ###################################################################### 59 ######################################################################
57 60
58 TESTOPTS= $(EXTRATESTOPTS) 61 TESTOPTS= $(EXTRATESTOPTS)
59-TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS) 62-TESTPYTHON= $(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS)
60+TESTPYTHON= $(RUNSHARED) $(HOSTPYTHON) $(TESTPYTHONOPTS) 63+TESTPYTHON= $(RUNSHARED) $(HOSTPYTHON) $(TESTPYTHONOPTS)
61 TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py 64 TESTRUNNER= $(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
62 TESTTIMEOUT= 3600 65 TESTTIMEOUT= 3600
63 66
64@@ -1468,7 +1470,7 @@ frameworkinstallstructure: $(LDLIBRARY) 67@@ -1470,7 +1472,7 @@ frameworkinstallstructure: $(LDLIBRARY)
65 fi; \ 68 fi; \
66 done 69 done
67 $(LN) -fsn include/python$(LDVERSION) $(DESTDIR)$(prefix)/Headers 70 $(LN) -fsn include/python$(LDVERSION) $(DESTDIR)$(prefix)/Headers
@@ -70,24 +73,24 @@ index a88b7d5..7cb8bb3 100644
70 $(LN) -fsn $(VERSION) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/Current 73 $(LN) -fsn $(VERSION) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/Current
71 $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/$(PYTHONFRAMEWORK) 74 $(LN) -fsn Versions/Current/$(PYTHONFRAMEWORK) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/$(PYTHONFRAMEWORK)
72 $(LN) -fsn Versions/Current/Headers $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers 75 $(LN) -fsn Versions/Current/Headers $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Headers
73@@ -1534,7 +1536,7 @@ config.status: $(srcdir)/configure 76@@ -1543,7 +1545,7 @@ Python/dtoa.o: Python/dtoa.c
74 77
75 # Run reindent on the library 78 # Run reindent on the library
76 reindent: 79 reindent:
77- ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib 80- ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib
78+ $(HOSTPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib 81+ $(HOSTPYTHON) $(srcdir)/Tools/scripts/reindent.py -r $(srcdir)/Lib
79 82
80 # Rerun configure with the same options as it was run last time, 83 # Rerun configure with the same options as it was run last time,
81 # provided the config.status script exists 84 # provided the config.status script exists
82@@ -1674,7 +1676,7 @@ funny: 85@@ -1678,7 +1680,7 @@ funny:
83 86
84 # Perform some verification checks on any modified files. 87 # Perform some verification checks on any modified files.
85 patchcheck: all 88 patchcheck: @DEF_MAKE_RULE@
86- $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py 89- $(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/patchcheck.py
87+ $(RUNSHARED) ./$(HOSTPYTHON) $(srcdir)/Tools/scripts/patchcheck.py 90+ $(RUNSHARED) $(HOSTPYTHON) $(srcdir)/Tools/scripts/patchcheck.py
88 91
89 # Dependencies 92 # Dependencies
90 93
91-- 94--
922.11.0 952.11.0
93 96
diff --git a/meta/recipes-devtools/python/python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch b/meta/recipes-devtools/python/python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch
deleted file mode 100644
index 7217c6edea..0000000000
--- a/meta/recipes-devtools/python/python3/Fix-29519-weakref-spewing-exceptions-during-interp-f.patch
+++ /dev/null
@@ -1,56 +0,0 @@
1From 62dcf34987b680e95873eb947b3f4d802199c667 Mon Sep 17 00:00:00 2001
2From: =?UTF-8?q?=C5=81ukasz=20Langa?= <lukasz@langa.pl>
3Date: Fri, 10 Feb 2017 00:14:55 -0800
4Subject: [PATCH] Fix #29519: weakref spewing exceptions during interp
5 finalization
6
7commit 9cd7e17640a49635d1c1f8c2989578a8fc2c1de6
8from https://github.com/python/cpython
9
10Upstream-Status: Backport
11
12Signed-off-by: Lukasz Langa <lukasz@langa.pl>
13---
14 Lib/weakref.py | 4 ++--
15 Misc/NEWS | 3 +++
16 2 files changed, 5 insertions(+), 2 deletions(-)
17
18diff --git a/Lib/weakref.py b/Lib/weakref.py
19index aaebd0c..787e33a 100644
20--- a/Lib/weakref.py
21+++ b/Lib/weakref.py
22@@ -106,7 +106,7 @@ class WeakValueDictionary(collections.MutableMapping):
23 self, *args = args
24 if len(args) > 1:
25 raise TypeError('expected at most 1 arguments, got %d' % len(args))
26- def remove(wr, selfref=ref(self)):
27+ def remove(wr, selfref=ref(self), _atomic_removal=_remove_dead_weakref):
28 self = selfref()
29 if self is not None:
30 if self._iterating:
31@@ -114,7 +114,7 @@ class WeakValueDictionary(collections.MutableMapping):
32 else:
33 # Atomic removal is necessary since this function
34 # can be called asynchronously by the GC
35- _remove_dead_weakref(d, wr.key)
36+ _atomic_removal(d, wr.key)
37 self._remove = remove
38 # A list of keys to be removed
39 self._pending_removals = []
40diff --git a/Misc/NEWS b/Misc/NEWS
41index 41cfdba..6d89f52 100644
42--- a/Misc/NEWS
43+++ b/Misc/NEWS
44@@ -5719,6 +5719,9 @@ Core and Builtins
45 Library
46 -------
47
48+- Issue #29519: Fix weakref spewing exceptions during interpreter shutdown
49+ when used with a rare combination of multiprocessing and custom codecs.
50+
51 - Issue #20154: Deadlock in asyncio.StreamReader.readexactly().
52
53 - Issue #16113: Remove sha3 module again.
54--
552.7.4
56
diff --git a/meta/recipes-devtools/python/python3/python3-manifest.json b/meta/recipes-devtools/python/python3/python3-manifest.json
index 5b7b70b346..b11756926b 100644
--- a/meta/recipes-devtools/python/python3/python3-manifest.json
+++ b/meta/recipes-devtools/python/python3/python3-manifest.json
@@ -797,17 +797,17 @@
797 }, 797 },
798 "numbers": { 798 "numbers": {
799 "cached": [ 799 "cached": [
800 "${libdir}/python3.5/__pycache__/_pydecimal.*.pyc",
800 "${libdir}/python3.5/__pycache__/decimal.*.pyc", 801 "${libdir}/python3.5/__pycache__/decimal.*.pyc",
801 "${libdir}/python3.5/__pycache__/fractions.*.pyc", 802 "${libdir}/python3.5/__pycache__/fractions.*.pyc",
802 "${libdir}/python3.5/__pycache__/numbers.*.pyc", 803 "${libdir}/python3.5/__pycache__/numbers.*.pyc"
803 "${libdir}/python3.5/__pycache__/_pydecimal.*.pyc"
804 ], 804 ],
805 "files": [ 805 "files": [
806 "${libdir}/python3.5/_pydecimal.py",
806 "${libdir}/python3.5/decimal.py", 807 "${libdir}/python3.5/decimal.py",
807 "${libdir}/python3.5/fractions.py", 808 "${libdir}/python3.5/fractions.py",
808 "${libdir}/python3.5/lib-dynload/_decimal.*.so", 809 "${libdir}/python3.5/lib-dynload/_decimal.*.so",
809 "${libdir}/python3.5/numbers.py", 810 "${libdir}/python3.5/numbers.py"
810 "${libdir}/python3.5/_pydecimal.py"
811 ], 811 ],
812 "rdepends": [ 812 "rdepends": [
813 "core" 813 "core"
diff --git a/meta/recipes-devtools/python/python3/upstream-random-fixes.patch b/meta/recipes-devtools/python/python3/upstream-random-fixes.patch
deleted file mode 100644
index 9b40e8ac9f..0000000000
--- a/meta/recipes-devtools/python/python3/upstream-random-fixes.patch
+++ /dev/null
@@ -1,703 +0,0 @@
1From 035ba5da3e53e45c712b39fe1f6fb743e697c032 Mon Sep 17 00:00:00 2001
2From: Victor Stinner <victor.stinner@gmail.com>
3Date: Mon, 9 Jan 2017 11:18:53 +0100
4Subject: [PATCH] Issue #29157: Prefer getrandom() over getentropy()
5
6Copy and then adapt Python/random.c from default branch. Difference between 3.5
7and default branches:
8
9* Python 3.5 only uses getrandom() in non-blocking mode: flags=GRND_NONBLOCK
10* If getrandom() fails with EAGAIN: py_getrandom() immediately fails and
11 remembers that getrandom() doesn't work.
12* Python 3.5 has no _PyOS_URandomNonblock() function: _PyOS_URandom()
13 works in non-blocking mode on Python 3.5
14
15Upstream-Status: Backport [https://github.com/python/cpython/commit/035ba5da3e53e45c712b39fe1f6fb743e697c032]
16Signed-off-by: Alexander Kanavin <alexander.kanavin@intel.com>
17
18---
19 Python/random.c | 494 +++++++++++++++++++++++++++++++++-----------------------
20 1 file changed, 294 insertions(+), 200 deletions(-)
21
22diff --git a/Python/random.c b/Python/random.c
23index d203939..31f61d0 100644
24--- a/Python/random.c
25+++ b/Python/random.c
26@@ -1,6 +1,9 @@
27 #include "Python.h"
28 #ifdef MS_WINDOWS
29 # include <windows.h>
30+/* All sample MSDN wincrypt programs include the header below. It is at least
31+ * required with Min GW. */
32+# include <wincrypt.h>
33 #else
34 # include <fcntl.h>
35 # ifdef HAVE_SYS_STAT_H
36@@ -37,10 +40,9 @@ win32_urandom_init(int raise)
37 return 0;
38
39 error:
40- if (raise)
41+ if (raise) {
42 PyErr_SetFromWindowsErr(0);
43- else
44- Py_FatalError("Failed to initialize Windows random API (CryptoGen)");
45+ }
46 return -1;
47 }
48
49@@ -53,8 +55,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise)
50
51 if (hCryptProv == 0)
52 {
53- if (win32_urandom_init(raise) == -1)
54+ if (win32_urandom_init(raise) == -1) {
55 return -1;
56+ }
57 }
58
59 while (size > 0)
60@@ -63,11 +66,9 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise)
61 if (!CryptGenRandom(hCryptProv, (DWORD)chunk, buffer))
62 {
63 /* CryptGenRandom() failed */
64- if (raise)
65+ if (raise) {
66 PyErr_SetFromWindowsErr(0);
67- else
68- Py_FatalError("Failed to initialized the randomized hash "
69- "secret using CryptoGen)");
70+ }
71 return -1;
72 }
73 buffer += chunk;
74@@ -76,58 +77,23 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise)
75 return 0;
76 }
77
78-/* Issue #25003: Don't use getentropy() on Solaris (available since
79- * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
80-#elif defined(HAVE_GETENTROPY) && !defined(sun)
81-#define PY_GETENTROPY 1
82-
83-/* Fill buffer with size pseudo-random bytes generated by getentropy().
84- Return 0 on success, or raise an exception and return -1 on error.
85-
86- If fatal is nonzero, call Py_FatalError() instead of raising an exception
87- on error. */
88-static int
89-py_getentropy(unsigned char *buffer, Py_ssize_t size, int fatal)
90-{
91- while (size > 0) {
92- Py_ssize_t len = Py_MIN(size, 256);
93- int res;
94-
95- if (!fatal) {
96- Py_BEGIN_ALLOW_THREADS
97- res = getentropy(buffer, len);
98- Py_END_ALLOW_THREADS
99-
100- if (res < 0) {
101- PyErr_SetFromErrno(PyExc_OSError);
102- return -1;
103- }
104- }
105- else {
106- res = getentropy(buffer, len);
107- if (res < 0)
108- Py_FatalError("getentropy() failed");
109- }
110-
111- buffer += len;
112- size -= len;
113- }
114- return 0;
115-}
116-
117-#else
118+#else /* !MS_WINDOWS */
119
120 #if defined(HAVE_GETRANDOM) || defined(HAVE_GETRANDOM_SYSCALL)
121 #define PY_GETRANDOM 1
122
123-/* Call getrandom()
124+/* Call getrandom() to get random bytes:
125+
126 - Return 1 on success
127- - Return 0 if getrandom() syscall is not available (failed with ENOSYS or
128- EPERM) or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom
129- not initialized yet) and raise=0.
130+ - Return 0 if getrandom() is not available (failed with ENOSYS or EPERM),
131+ or if getrandom(GRND_NONBLOCK) failed with EAGAIN (system urandom not
132+ initialized yet).
133 - Raise an exception (if raise is non-zero) and return -1 on error:
134- getrandom() failed with EINTR and the Python signal handler raised an
135- exception, or getrandom() failed with a different error. */
136+ if getrandom() failed with EINTR, raise is non-zero and the Python signal
137+ handler raised an exception, or if getrandom() failed with a different
138+ error.
139+
140+ getrandom() is retried if it failed with EINTR: interrupted by a signal. */
141 static int
142 py_getrandom(void *buffer, Py_ssize_t size, int raise)
143 {
144@@ -142,16 +108,19 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
145 * see https://bugs.python.org/issue26839. To avoid this, use the
146 * GRND_NONBLOCK flag. */
147 const int flags = GRND_NONBLOCK;
148+ char *dest;
149 long n;
150
151 if (!getrandom_works) {
152 return 0;
153 }
154
155+ dest = buffer;
156 while (0 < size) {
157 #ifdef sun
158 /* Issue #26735: On Solaris, getrandom() is limited to returning up
159- to 1024 bytes */
160+ to 1024 bytes. Call it multiple times if more bytes are
161+ requested. */
162 n = Py_MIN(size, 1024);
163 #else
164 n = Py_MIN(size, LONG_MAX);
165@@ -161,34 +130,35 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
166 #ifdef HAVE_GETRANDOM
167 if (raise) {
168 Py_BEGIN_ALLOW_THREADS
169- n = getrandom(buffer, n, flags);
170+ n = getrandom(dest, n, flags);
171 Py_END_ALLOW_THREADS
172 }
173 else {
174- n = getrandom(buffer, n, flags);
175+ n = getrandom(dest, n, flags);
176 }
177 #else
178 /* On Linux, use the syscall() function because the GNU libc doesn't
179- * expose the Linux getrandom() syscall yet. See:
180- * https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */
181+ expose the Linux getrandom() syscall yet. See:
182+ https://sourceware.org/bugzilla/show_bug.cgi?id=17252 */
183 if (raise) {
184 Py_BEGIN_ALLOW_THREADS
185- n = syscall(SYS_getrandom, buffer, n, flags);
186+ n = syscall(SYS_getrandom, dest, n, flags);
187 Py_END_ALLOW_THREADS
188 }
189 else {
190- n = syscall(SYS_getrandom, buffer, n, flags);
191+ n = syscall(SYS_getrandom, dest, n, flags);
192 }
193 #endif
194
195 if (n < 0) {
196- /* ENOSYS: getrandom() syscall not supported by the kernel (but
197- * maybe supported by the host which built Python). EPERM:
198- * getrandom() syscall blocked by SECCOMP or something else. */
199+ /* ENOSYS: the syscall is not supported by the kernel.
200+ EPERM: the syscall is blocked by a security policy (ex: SECCOMP)
201+ or something else. */
202 if (errno == ENOSYS || errno == EPERM) {
203 getrandom_works = 0;
204 return 0;
205 }
206+
207 if (errno == EAGAIN) {
208 /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system
209 urandom is not initialiazed yet. In this case, fall back on
210@@ -202,169 +172,225 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
211 }
212
213 if (errno == EINTR) {
214- if (PyErr_CheckSignals()) {
215- if (!raise) {
216- Py_FatalError("getrandom() interrupted by a signal");
217+ if (raise) {
218+ if (PyErr_CheckSignals()) {
219+ return -1;
220 }
221- return -1;
222 }
223
224- /* retry getrandom() */
225+ /* retry getrandom() if it was interrupted by a signal */
226 continue;
227 }
228
229 if (raise) {
230 PyErr_SetFromErrno(PyExc_OSError);
231 }
232- else {
233- Py_FatalError("getrandom() failed");
234- }
235 return -1;
236 }
237
238- buffer += n;
239+ dest += n;
240 size -= n;
241 }
242 return 1;
243 }
244-#endif
245
246-static struct {
247- int fd;
248- dev_t st_dev;
249- ino_t st_ino;
250-} urandom_cache = { -1 };
251+#elif defined(HAVE_GETENTROPY)
252+#define PY_GETENTROPY 1
253
254+/* Fill buffer with size pseudo-random bytes generated by getentropy():
255
256-/* Read 'size' random bytes from py_getrandom(). Fall back on reading from
257- /dev/urandom if getrandom() is not available.
258+ - Return 1 on success
259+ - Return 0 if getentropy() syscall is not available (failed with ENOSYS or
260+ EPERM).
261+ - Raise an exception (if raise is non-zero) and return -1 on error:
262+ if getentropy() failed with EINTR, raise is non-zero and the Python signal
263+ handler raised an exception, or if getentropy() failed with a different
264+ error.
265
266- Call Py_FatalError() on error. */
267-static void
268-dev_urandom_noraise(unsigned char *buffer, Py_ssize_t size)
269+ getentropy() is retried if it failed with EINTR: interrupted by a signal. */
270+static int
271+py_getentropy(char *buffer, Py_ssize_t size, int raise)
272 {
273- int fd;
274- Py_ssize_t n;
275+ /* Is getentropy() supported by the running kernel? Set to 0 if
276+ getentropy() failed with ENOSYS or EPERM. */
277+ static int getentropy_works = 1;
278
279- assert (0 < size);
280-
281-#ifdef PY_GETRANDOM
282- if (py_getrandom(buffer, size, 0) == 1) {
283- return;
284+ if (!getentropy_works) {
285+ return 0;
286 }
287- /* getrandom() failed with ENOSYS or EPERM,
288- fall back on reading /dev/urandom */
289-#endif
290
291- fd = _Py_open_noraise("/dev/urandom", O_RDONLY);
292- if (fd < 0) {
293- Py_FatalError("Failed to open /dev/urandom");
294- }
295+ while (size > 0) {
296+ /* getentropy() is limited to returning up to 256 bytes. Call it
297+ multiple times if more bytes are requested. */
298+ Py_ssize_t len = Py_MIN(size, 256);
299+ int res;
300
301- while (0 < size)
302- {
303- do {
304- n = read(fd, buffer, (size_t)size);
305- } while (n < 0 && errno == EINTR);
306+ if (raise) {
307+ Py_BEGIN_ALLOW_THREADS
308+ res = getentropy(buffer, len);
309+ Py_END_ALLOW_THREADS
310+ }
311+ else {
312+ res = getentropy(buffer, len);
313+ }
314
315- if (n <= 0) {
316- /* read() failed or returned 0 bytes */
317- Py_FatalError("Failed to read bytes from /dev/urandom");
318- break;
319+ if (res < 0) {
320+ /* ENOSYS: the syscall is not supported by the running kernel.
321+ EPERM: the syscall is blocked by a security policy (ex: SECCOMP)
322+ or something else. */
323+ if (errno == ENOSYS || errno == EPERM) {
324+ getentropy_works = 0;
325+ return 0;
326+ }
327+
328+ if (errno == EINTR) {
329+ if (raise) {
330+ if (PyErr_CheckSignals()) {
331+ return -1;
332+ }
333+ }
334+
335+ /* retry getentropy() if it was interrupted by a signal */
336+ continue;
337+ }
338+
339+ if (raise) {
340+ PyErr_SetFromErrno(PyExc_OSError);
341+ }
342+ return -1;
343 }
344- buffer += n;
345- size -= n;
346+
347+ buffer += len;
348+ size -= len;
349 }
350- close(fd);
351+ return 1;
352 }
353+#endif /* defined(HAVE_GETENTROPY) && !defined(sun) */
354
355-/* Read 'size' random bytes from py_getrandom(). Fall back on reading from
356- /dev/urandom if getrandom() is not available.
357
358- Return 0 on success. Raise an exception and return -1 on error. */
359+static struct {
360+ int fd;
361+ dev_t st_dev;
362+ ino_t st_ino;
363+} urandom_cache = { -1 };
364+
365+/* Read random bytes from the /dev/urandom device:
366+
367+ - Return 0 on success
368+ - Raise an exception (if raise is non-zero) and return -1 on error
369+
370+ Possible causes of errors:
371+
372+ - open() failed with ENOENT, ENXIO, ENODEV, EACCES: the /dev/urandom device
373+ was not found. For example, it was removed manually or not exposed in a
374+ chroot or container.
375+ - open() failed with a different error
376+ - fstat() failed
377+ - read() failed or returned 0
378+
379+ read() is retried if it failed with EINTR: interrupted by a signal.
380+
381+ The file descriptor of the device is kept open between calls to avoid using
382+ many file descriptors when run in parallel from multiple threads:
383+ see the issue #18756.
384+
385+ st_dev and st_ino fields of the file descriptor (from fstat()) are cached to
386+ check if the file descriptor was replaced by a different file (which is
387+ likely a bug in the application): see the issue #21207.
388+
389+ If the file descriptor was closed or replaced, open a new file descriptor
390+ but don't close the old file descriptor: it probably points to something
391+ important for some third-party code. */
392 static int
393-dev_urandom_python(char *buffer, Py_ssize_t size)
394+dev_urandom(char *buffer, Py_ssize_t size, int raise)
395 {
396 int fd;
397 Py_ssize_t n;
398- struct _Py_stat_struct st;
399-#ifdef PY_GETRANDOM
400- int res;
401-#endif
402-
403- if (size <= 0)
404- return 0;
405
406-#ifdef PY_GETRANDOM
407- res = py_getrandom(buffer, size, 1);
408- if (res < 0) {
409- return -1;
410- }
411- if (res == 1) {
412- return 0;
413- }
414- /* getrandom() failed with ENOSYS or EPERM,
415- fall back on reading /dev/urandom */
416-#endif
417+ if (raise) {
418+ struct _Py_stat_struct st;
419
420- if (urandom_cache.fd >= 0) {
421- /* Does the fd point to the same thing as before? (issue #21207) */
422- if (_Py_fstat_noraise(urandom_cache.fd, &st)
423- || st.st_dev != urandom_cache.st_dev
424- || st.st_ino != urandom_cache.st_ino) {
425- /* Something changed: forget the cached fd (but don't close it,
426- since it probably points to something important for some
427- third-party code). */
428- urandom_cache.fd = -1;
429- }
430- }
431- if (urandom_cache.fd >= 0)
432- fd = urandom_cache.fd;
433- else {
434- fd = _Py_open("/dev/urandom", O_RDONLY);
435- if (fd < 0) {
436- if (errno == ENOENT || errno == ENXIO ||
437- errno == ENODEV || errno == EACCES)
438- PyErr_SetString(PyExc_NotImplementedError,
439- "/dev/urandom (or equivalent) not found");
440- /* otherwise, keep the OSError exception raised by _Py_open() */
441- return -1;
442- }
443 if (urandom_cache.fd >= 0) {
444- /* urandom_fd was initialized by another thread while we were
445- not holding the GIL, keep it. */
446- close(fd);
447- fd = urandom_cache.fd;
448+ /* Does the fd point to the same thing as before? (issue #21207) */
449+ if (_Py_fstat_noraise(urandom_cache.fd, &st)
450+ || st.st_dev != urandom_cache.st_dev
451+ || st.st_ino != urandom_cache.st_ino) {
452+ /* Something changed: forget the cached fd (but don't close it,
453+ since it probably points to something important for some
454+ third-party code). */
455+ urandom_cache.fd = -1;
456+ }
457 }
458+ if (urandom_cache.fd >= 0)
459+ fd = urandom_cache.fd;
460 else {
461- if (_Py_fstat(fd, &st)) {
462- close(fd);
463+ fd = _Py_open("/dev/urandom", O_RDONLY);
464+ if (fd < 0) {
465+ if (errno == ENOENT || errno == ENXIO ||
466+ errno == ENODEV || errno == EACCES) {
467+ PyErr_SetString(PyExc_NotImplementedError,
468+ "/dev/urandom (or equivalent) not found");
469+ }
470+ /* otherwise, keep the OSError exception raised by _Py_open() */
471 return -1;
472 }
473+ if (urandom_cache.fd >= 0) {
474+ /* urandom_fd was initialized by another thread while we were
475+ not holding the GIL, keep it. */
476+ close(fd);
477+ fd = urandom_cache.fd;
478+ }
479 else {
480- urandom_cache.fd = fd;
481- urandom_cache.st_dev = st.st_dev;
482- urandom_cache.st_ino = st.st_ino;
483+ if (_Py_fstat(fd, &st)) {
484+ close(fd);
485+ return -1;
486+ }
487+ else {
488+ urandom_cache.fd = fd;
489+ urandom_cache.st_dev = st.st_dev;
490+ urandom_cache.st_ino = st.st_ino;
491+ }
492 }
493 }
494- }
495
496- do {
497- n = _Py_read(fd, buffer, (size_t)size);
498- if (n == -1) {
499- return -1;
500- }
501- if (n == 0) {
502- PyErr_Format(PyExc_RuntimeError,
503- "Failed to read %zi bytes from /dev/urandom",
504- size);
505+ do {
506+ n = _Py_read(fd, buffer, (size_t)size);
507+ if (n == -1)
508+ return -1;
509+ if (n == 0) {
510+ PyErr_Format(PyExc_RuntimeError,
511+ "Failed to read %zi bytes from /dev/urandom",
512+ size);
513+ return -1;
514+ }
515+
516+ buffer += n;
517+ size -= n;
518+ } while (0 < size);
519+ }
520+ else {
521+ fd = _Py_open_noraise("/dev/urandom", O_RDONLY);
522+ if (fd < 0) {
523 return -1;
524 }
525
526- buffer += n;
527- size -= n;
528- } while (0 < size);
529+ while (0 < size)
530+ {
531+ do {
532+ n = read(fd, buffer, (size_t)size);
533+ } while (n < 0 && errno == EINTR);
534
535+ if (n <= 0) {
536+ /* stop on error or if read(size) returned 0 */
537+ close(fd);
538+ return -1;
539+ }
540+
541+ buffer += n;
542+ size -= n;
543+ }
544+ close(fd);
545+ }
546 return 0;
547 }
548
549@@ -376,8 +402,8 @@ dev_urandom_close(void)
550 urandom_cache.fd = -1;
551 }
552 }
553+#endif /* !MS_WINDOWS */
554
555-#endif
556
557 /* Fill buffer with pseudo-random bytes generated by a linear congruent
558 generator (LCG):
559@@ -400,31 +426,100 @@ lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size)
560 }
561 }
562
563-/* Fill buffer with size pseudo-random bytes from the operating system random
564- number generator (RNG). It is suitable for most cryptographic purposes
565- except long living private keys for asymmetric encryption.
566+/* Read random bytes:
567
568- Return 0 on success, raise an exception and return -1 on error. */
569-int
570-_PyOS_URandom(void *buffer, Py_ssize_t size)
571+ - Return 0 on success
572+ - Raise an exception (if raise is non-zero) and return -1 on error
573+
574+ Used sources of entropy ordered by preference, preferred source first:
575+
576+ - CryptGenRandom() on Windows
577+ - getrandom() function (ex: Linux and Solaris): call py_getrandom()
578+ - getentropy() function (ex: OpenBSD): call py_getentropy()
579+ - /dev/urandom device
580+
581+ Read from the /dev/urandom device if getrandom() or getentropy() function
582+ is not available or does not work.
583+
584+ Prefer getrandom() over getentropy() because getrandom() supports blocking
585+ and non-blocking mode and Python requires non-blocking RNG at startup to
586+ initialize its hash secret: see the PEP 524.
587+
588+ Prefer getrandom() and getentropy() over reading directly /dev/urandom
589+ because these functions don't need file descriptors and so avoid ENFILE or
590+ EMFILE errors (too many open files): see the issue #18756.
591+
592+ Only use RNG running in the kernel. They are more secure because it is
593+ harder to get the internal state of a RNG running in the kernel land than a
594+ RNG running in the user land. The kernel has a direct access to the hardware
595+ and has access to hardware RNG, they are used as entropy sources.
596+
597+ Note: the OpenSSL RAND_pseudo_bytes() function does not automatically reseed
598+ its RNG on fork(), two child processes (with the same pid) generate the same
599+ random numbers: see issue #18747. Kernel RNGs don't have this issue,
600+ they have access to good quality entropy sources.
601+
602+ If raise is zero:
603+
604+ - Don't raise an exception on error
605+ - Don't call the Python signal handler (don't call PyErr_CheckSignals()) if
606+ a function fails with EINTR: retry directly the interrupted function
607+ - Don't release the GIL to call functions.
608+*/
609+static int
610+pyurandom(void *buffer, Py_ssize_t size, int raise)
611 {
612+#if defined(PY_GETRANDOM) || defined(PY_GETENTROPY)
613+ int res;
614+#endif
615+
616 if (size < 0) {
617- PyErr_Format(PyExc_ValueError,
618- "negative argument not allowed");
619+ if (raise) {
620+ PyErr_Format(PyExc_ValueError,
621+ "negative argument not allowed");
622+ }
623 return -1;
624 }
625- if (size == 0)
626+
627+ if (size == 0) {
628 return 0;
629+ }
630
631 #ifdef MS_WINDOWS
632- return win32_urandom((unsigned char *)buffer, size, 1);
633-#elif defined(PY_GETENTROPY)
634- return py_getentropy(buffer, size, 0);
635+ return win32_urandom((unsigned char *)buffer, size, raise);
636+#else
637+
638+#if defined(PY_GETRANDOM) || defined(PY_GETENTROPY)
639+#ifdef PY_GETRANDOM
640+ res = py_getrandom(buffer, size, raise);
641 #else
642- return dev_urandom_python((char*)buffer, size);
643+ res = py_getentropy(buffer, size, raise);
644+#endif
645+ if (res < 0) {
646+ return -1;
647+ }
648+ if (res == 1) {
649+ return 0;
650+ }
651+ /* getrandom() or getentropy() function is not available: failed with
652+ ENOSYS, EPERM or EAGAIN. Fall back on reading from /dev/urandom. */
653+#endif
654+
655+ return dev_urandom(buffer, size, raise);
656 #endif
657 }
658
659+/* Fill buffer with size pseudo-random bytes from the operating system random
660+ number generator (RNG). It is suitable for most cryptographic purposes
661+ except long living private keys for asymmetric encryption.
662+
663+ Return 0 on success. Raise an exception and return -1 on error. */
664+int
665+_PyOS_URandom(void *buffer, Py_ssize_t size)
666+{
667+ return pyurandom(buffer, size, 1);
668+}
669+
670 void
671 _PyRandom_Init(void)
672 {
673@@ -463,13 +558,14 @@ _PyRandom_Init(void)
674 }
675 }
676 else {
677-#ifdef MS_WINDOWS
678- (void)win32_urandom(secret, secret_size, 0);
679-#elif defined(PY_GETENTROPY)
680- (void)py_getentropy(secret, secret_size, 1);
681-#else
682- dev_urandom_noraise(secret, secret_size);
683-#endif
684+ int res;
685+
686+ /* _PyRandom_Init() is called very early in the Python initialization
687+ and so exceptions cannot be used (use raise=0). */
688+ res = pyurandom(secret, secret_size, 0);
689+ if (res < 0) {
690+ Py_FatalError("failed to get random numbers to initialize Python");
691+ }
692 }
693 }
694
695@@ -481,8 +577,6 @@ _PyRandom_Fini(void)
696 CryptReleaseContext(hCryptProv, 0);
697 hCryptProv = 0;
698 }
699-#elif defined(PY_GETENTROPY)
700- /* nothing to clean */
701 #else
702 dev_urandom_close();
703 #endif
diff --git a/meta/recipes-devtools/python/python3_3.5.3.bb b/meta/recipes-devtools/python/python3_3.5.4.bb
index 3ae0db2b8b..4c7a22527c 100644
--- a/meta/recipes-devtools/python/python3_3.5.3.bb
+++ b/meta/recipes-devtools/python/python3_3.5.4.bb
@@ -35,15 +35,13 @@ SRC_URI += "\
35 file://sysconfig.py-add-_PYTHON_PROJECT_SRC.patch \ 35 file://sysconfig.py-add-_PYTHON_PROJECT_SRC.patch \
36 file://setup.py-check-cross_compiling-when-get-FLAGS.patch \ 36 file://setup.py-check-cross_compiling-when-get-FLAGS.patch \
37 file://configure.ac-fix-LIBPL.patch \ 37 file://configure.ac-fix-LIBPL.patch \
38 file://upstream-random-fixes.patch \
39 file://0001-Issue-21272-Use-_sysconfigdata.py-to-initialize-dist.patch \ 38 file://0001-Issue-21272-Use-_sysconfigdata.py-to-initialize-dist.patch \
40 file://Fix-29519-weakref-spewing-exceptions-during-interp-f.patch \
41 file://pass-missing-libraries-to-Extension-for-mul.patch \ 39 file://pass-missing-libraries-to-Extension-for-mul.patch \
42 " 40 "
43SRC_URI[md5sum] = "57d1f8bfbabf4f2500273fb0706e6f21" 41SRC_URI[md5sum] = "fb2780baa260b4e51cbea814f111f303"
44SRC_URI[sha256sum] = "eefe2ad6575855423ab630f5b51a8ef6e5556f774584c06beab4926f930ddbb0" 42SRC_URI[sha256sum] = "94d93bfabb3b109f8a10365a325f920f9ec98c6e2380bf228f9700a14054c84c"
45 43
46LIC_FILES_CHKSUM = "file://LICENSE;md5=b680ed99aa60d350c65a65914494207e" 44LIC_FILES_CHKSUM = "file://LICENSE;md5=f741e51de91d4eeea5930b9c3c7fa69d"
47 45
48# exclude pre-releases for both python 2.x and 3.x 46# exclude pre-releases for both python 2.x and 3.x
49UPSTREAM_CHECK_REGEX = "[Pp]ython-(?P<pver>\d+(\.\d+)+).tar" 47UPSTREAM_CHECK_REGEX = "[Pp]ython-(?P<pver>\d+(\.\d+)+).tar"