From 62f52fdda08e2a8fad08b94ce326cae12e4b8757 Mon Sep 17 00:00:00 2001 From: Derek Straka Date: Sat, 30 Jun 2018 07:37:35 -0400 Subject: python: update to version 2.7.15 Update to the latest stable version License-Update: Copyright year updated to include 2018 Remove the alignment patch that is included upstream (From OE-Core rev: 855020053906478cea164ed254c08bedce48479d) (From OE-Core rev: ab2dd15f72a94cce528276e6e3e38c56677e7ba4) Signed-off-by: Derek Straka Signed-off-by: Ross Burton Signed-off-by: Richard Purdie [Bug fix update only, drop patches included in update] Signed-off-by: Armin Kuster Signed-off-by: Richard Purdie --- .../python/python-native_2.7.14.bb | 86 ------ .../python/python-native_2.7.15.bb | 85 ++++++ meta/recipes-devtools/python/python.inc | 10 +- .../python/python/CVE-2018-1000030-1.patch | 138 ---------- .../python/python/CVE-2018-1000030-2.patch | 306 --------------------- .../python/python/fix-gc-alignment.patch | 43 --- meta/recipes-devtools/python/python_2.7.14.bb | 282 ------------------- meta/recipes-devtools/python/python_2.7.15.bb | 281 +++++++++++++++++++ 8 files changed, 370 insertions(+), 861 deletions(-) delete mode 100644 meta/recipes-devtools/python/python-native_2.7.14.bb create mode 100644 meta/recipes-devtools/python/python-native_2.7.15.bb delete mode 100644 meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch delete mode 100644 meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch delete mode 100644 meta/recipes-devtools/python/python/fix-gc-alignment.patch delete mode 100644 meta/recipes-devtools/python/python_2.7.14.bb create mode 100644 meta/recipes-devtools/python/python_2.7.15.bb diff --git a/meta/recipes-devtools/python/python-native_2.7.14.bb b/meta/recipes-devtools/python/python-native_2.7.14.bb deleted file mode 100644 index b822583c53..0000000000 --- a/meta/recipes-devtools/python/python-native_2.7.14.bb +++ /dev/null @@ -1,86 +0,0 @@ -require python.inc -EXTRANATIVEPATH += "bzip2-native" -DEPENDS = "openssl-native bzip2-replacement-native zlib-native readline-native sqlite3-native expat-native gdbm-native db-native" -PR = "${INC_PR}.1" - -SRC_URI += "\ - file://05-enable-ctypes-cross-build.patch \ - file://10-distutils-fix-swig-parameter.patch \ - file://11-distutils-never-modify-shebang-line.patch \ - file://0001-distutils-set-the-prefix-to-be-inside-staging-direct.patch \ - file://debug.patch \ - file://unixccompiler.patch \ - file://nohostlibs.patch \ - file://multilib.patch \ - file://add-md5module-support.patch \ - file://builddir.patch \ - file://parallel-makeinst-create-bindir.patch \ - file://revert_use_of_sysconfigdata.patch \ - file://fix-gc-alignment.patch \ - " - -S = "${WORKDIR}/Python-${PV}" - -FILESEXTRAPATHS =. "${FILE_DIRNAME}/${PN}:" - -inherit native - -EXTRA_OECONF_append = " --bindir=${bindir}/${PN} --with-system-expat=${STAGING_DIR_HOST}" - -EXTRA_OEMAKE = '\ - LIBC="" \ - STAGING_LIBDIR=${STAGING_LIBDIR_NATIVE} \ - STAGING_INCDIR=${STAGING_INCDIR_NATIVE} \ -' - -do_configure_append() { - autoreconf --verbose --install --force --exclude=autopoint ../Python-${PV}/Modules/_ctypes/libffi -} - -# Regenerate all of the generated files -# This ensures that pgen and friends get created during the compile phase -do_compile_prepend() { - oe_runmake regen-all -} - -do_install() { - oe_runmake 'DESTDIR=${D}' install - install -d ${D}${bindir}/${PN} - install -m 0755 Parser/pgen ${D}${bindir}/${PN} - - # Make sure we use /usr/bin/env python - for PYTHSCRIPT in `grep -rIl ${bindir}/${PN}/python ${D}${bindir}/${PN}`; do - sed -i -e '1s|^#!.*|#!/usr/bin/env python|' $PYTHSCRIPT - done - - # Add a symlink to the native Python so that scripts can just invoke - # "nativepython" and get the right one without needing absolute paths - # (these often end up too long for the #! parser in the kernel as the - # buffer is 128 bytes long). - ln -s python-native/python ${D}${bindir}/nativepython - - # We don't want modules in ~/.local being used in preference to those - # installed in the native sysroot, so disable user site support. - sed -i -e 's,^\(ENABLE_USER_SITE = \).*,\1False,' ${D}${libdir}/python${PYTHON_MAJMIN}/site.py -} - -python(){ - - # Read JSON manifest - import json - pythondir = d.getVar('THISDIR',True) - with open(pythondir+'/python/python2-manifest.json') as manifest_file: - python_manifest=json.load(manifest_file) - - rprovides = d.getVar('RPROVIDES').split() - - # Hardcoded since it cant be python-native-foo, should be python-foo-native - pn = 'python' - - for key in python_manifest: - pypackage = pn + '-' + key + '-native' - if pypackage not in rprovides: - rprovides.append(pypackage) - - d.setVar('RPROVIDES', ' '.join(rprovides)) -} diff --git a/meta/recipes-devtools/python/python-native_2.7.15.bb b/meta/recipes-devtools/python/python-native_2.7.15.bb new file mode 100644 index 0000000000..7c491fa3e0 --- /dev/null +++ b/meta/recipes-devtools/python/python-native_2.7.15.bb @@ -0,0 +1,85 @@ +require python.inc +EXTRANATIVEPATH += "bzip2-native" +DEPENDS = "openssl-native bzip2-replacement-native zlib-native readline-native sqlite3-native expat-native gdbm-native db-native" +PR = "${INC_PR}.1" + +SRC_URI += "\ + file://05-enable-ctypes-cross-build.patch \ + file://10-distutils-fix-swig-parameter.patch \ + file://11-distutils-never-modify-shebang-line.patch \ + file://0001-distutils-set-the-prefix-to-be-inside-staging-direct.patch \ + file://debug.patch \ + file://unixccompiler.patch \ + file://nohostlibs.patch \ + file://multilib.patch \ + file://add-md5module-support.patch \ + file://builddir.patch \ + file://parallel-makeinst-create-bindir.patch \ + file://revert_use_of_sysconfigdata.patch \ + " + +S = "${WORKDIR}/Python-${PV}" + +FILESEXTRAPATHS =. "${FILE_DIRNAME}/${PN}:" + +inherit native + +EXTRA_OECONF_append = " --bindir=${bindir}/${PN} --with-system-expat=${STAGING_DIR_HOST}" + +EXTRA_OEMAKE = '\ + LIBC="" \ + STAGING_LIBDIR=${STAGING_LIBDIR_NATIVE} \ + STAGING_INCDIR=${STAGING_INCDIR_NATIVE} \ +' + +do_configure_append() { + autoreconf --verbose --install --force --exclude=autopoint ../Python-${PV}/Modules/_ctypes/libffi +} + +# Regenerate all of the generated files +# This ensures that pgen and friends get created during the compile phase +do_compile_prepend() { + oe_runmake regen-all +} + +do_install() { + oe_runmake 'DESTDIR=${D}' install + install -d ${D}${bindir}/${PN} + install -m 0755 Parser/pgen ${D}${bindir}/${PN} + + # Make sure we use /usr/bin/env python + for PYTHSCRIPT in `grep -rIl ${bindir}/${PN}/python ${D}${bindir}/${PN}`; do + sed -i -e '1s|^#!.*|#!/usr/bin/env python|' $PYTHSCRIPT + done + + # Add a symlink to the native Python so that scripts can just invoke + # "nativepython" and get the right one without needing absolute paths + # (these often end up too long for the #! parser in the kernel as the + # buffer is 128 bytes long). + ln -s python-native/python ${D}${bindir}/nativepython + + # We don't want modules in ~/.local being used in preference to those + # installed in the native sysroot, so disable user site support. + sed -i -e 's,^\(ENABLE_USER_SITE = \).*,\1False,' ${D}${libdir}/python${PYTHON_MAJMIN}/site.py +} + +python(){ + + # Read JSON manifest + import json + pythondir = d.getVar('THISDIR',True) + with open(pythondir+'/python/python2-manifest.json') as manifest_file: + python_manifest=json.load(manifest_file) + + rprovides = d.getVar('RPROVIDES').split() + + # Hardcoded since it cant be python-native-foo, should be python-foo-native + pn = 'python' + + for key in python_manifest: + pypackage = pn + '-' + key + '-native' + if pypackage not in rprovides: + rprovides.append(pypackage) + + d.setVar('RPROVIDES', ' '.join(rprovides)) +} diff --git a/meta/recipes-devtools/python/python.inc b/meta/recipes-devtools/python/python.inc index 69542c96c8..062a20f68e 100644 --- a/meta/recipes-devtools/python/python.inc +++ b/meta/recipes-devtools/python/python.inc @@ -5,14 +5,12 @@ SECTION = "devel/python" # bump this on every change in contrib/python/generate-manifest-2.7.py INC_PR = "r1" -LIC_FILES_CHKSUM = "file://LICENSE;md5=f741e51de91d4eeea5930b9c3c7fa69d" +LIC_FILES_CHKSUM = "file://LICENSE;md5=f257cc14f81685691652a3d3e1b5d754" -SRC_URI = "http://www.python.org/ftp/python/${PV}/Python-${PV}.tar.xz \ - file://CVE-2018-1000030-1.patch \ - file://CVE-2018-1000030-2.patch" +SRC_URI = "http://www.python.org/ftp/python/${PV}/Python-${PV}.tar.xz" -SRC_URI[md5sum] = "1f6db41ad91d9eb0a6f0c769b8613c5b" -SRC_URI[sha256sum] = "71ffb26e09e78650e424929b2b457b9c912ac216576e6bd9e7d204ed03296a66" +SRC_URI[md5sum] = "a80ae3cc478460b922242f43a1b4094d" +SRC_URI[sha256sum] = "22d9b1ac5b26135ad2b8c2901a9413537e08749a753356ee913c84dbd2df5574" # python recipe is actually python 2.x # also, exclude pre-releases for both python 2.x and 3.x diff --git a/meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch b/meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch deleted file mode 100644 index 06ad4c695d..0000000000 --- a/meta/recipes-devtools/python/python/CVE-2018-1000030-1.patch +++ /dev/null @@ -1,138 +0,0 @@ -From 6401e5671781eb217ee1afb4603cc0d1b0367ae6 Mon Sep 17 00:00:00 2001 -From: Serhiy Storchaka -Date: Fri, 10 Nov 2017 12:58:55 +0200 -Subject: [PATCH] [2.7] bpo-31530: Stop crashes when iterating over a file on - multiple threads. (#3672) - -CVE: CVE-2018-1000030 -Upstream-Status: Backport [https://github.com/python/cpython/commit/6401e5671781eb217ee1afb4603cc0d1b0367ae6] - -Signed-off-by: Jagadeesh Krishnanjanappa ---- - Lib/test/test_file2k.py | 32 ++++++++++++++++++++++ - .../2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | 4 +++ - Objects/fileobject.c | 19 +++++++++++-- - 3 files changed, 52 insertions(+), 3 deletions(-) - create mode 100644 Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst - -diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py -index e39ef7042e..d8966e034e 100644 ---- a/Lib/test/test_file2k.py -+++ b/Lib/test/test_file2k.py -@@ -652,6 +652,38 @@ class FileThreadingTests(unittest.TestCase): - self.f.writelines('') - self._test_close_open_io(io_func) - -+ def test_iteration_torture(self): -+ # bpo-31530: Crash when concurrently iterate over a file. -+ with open(self.filename, "wb") as fp: -+ for i in xrange(2**20): -+ fp.write(b"0"*50 + b"\n") -+ with open(self.filename, "rb") as f: -+ def iterate(): -+ try: -+ for l in f: -+ pass -+ except IOError: -+ pass -+ self._run_workers(iterate, 10) -+ -+ def test_iteration_seek(self): -+ # bpo-31530: Crash when concurrently seek and iterate over a file. -+ with open(self.filename, "wb") as fp: -+ for i in xrange(10000): -+ fp.write(b"0"*50 + b"\n") -+ with open(self.filename, "rb") as f: -+ it = iter([1] + [0]*10) # one thread reads, others seek -+ def iterate(): -+ try: -+ if next(it): -+ for l in f: -+ pass -+ else: -+ for i in range(100): -+ f.seek(i*100, 0) -+ except IOError: -+ pass -+ self._run_workers(iterate, 10) - - @unittest.skipUnless(os.name == 'posix', 'test requires a posix system.') - class TestFileSignalEINTR(unittest.TestCase): -diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst -new file mode 100644 -index 0000000000..a6cb6c9e9b ---- /dev/null -+++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst -@@ -0,0 +1,4 @@ -+Fixed crashes when iterating over a file on multiple threads. -+seek() and next() methods of file objects now raise an exception during -+concurrent operation on the same file object. -+A lock can be used to prevent the error. -diff --git a/Objects/fileobject.c b/Objects/fileobject.c -index 7e07a5376f..2f63c374d1 100644 ---- a/Objects/fileobject.c -+++ b/Objects/fileobject.c -@@ -430,7 +430,7 @@ close_the_file(PyFileObject *f) - if (f->ob_refcnt > 0) { - PyErr_SetString(PyExc_IOError, - "close() called during concurrent " -- "operation on the same file object."); -+ "operation on the same file object"); - } else { - /* This should not happen unless someone is - * carelessly playing with the PyFileObject -@@ -438,7 +438,7 @@ close_the_file(PyFileObject *f) - * pointer. */ - PyErr_SetString(PyExc_SystemError, - "PyFileObject locking error in " -- "destructor (refcnt <= 0 at close)."); -+ "destructor (refcnt <= 0 at close)"); - } - return NULL; - } -@@ -762,6 +762,12 @@ file_seek(PyFileObject *f, PyObject *args) - - if (f->f_fp == NULL) - return err_closed(); -+ if (f->unlocked_count > 0) { -+ PyErr_SetString(PyExc_IOError, -+ "seek() called during concurrent " -+ "operation on the same file object"); -+ return NULL; -+ } - drop_readahead(f); - whence = 0; - if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence)) -@@ -2238,6 +2244,7 @@ readahead(PyFileObject *f, Py_ssize_t bufsize) - { - Py_ssize_t chunksize; - -+ assert(f->unlocked_count == 0); - if (f->f_buf != NULL) { - if( (f->f_bufend - f->f_bufptr) >= 1) - return 0; -@@ -2279,6 +2286,12 @@ readahead_get_line_skip(PyFileObject *f, Py_ssize_t skip, Py_ssize_t bufsize) - char *buf; - Py_ssize_t len; - -+ if (f->unlocked_count > 0) { -+ PyErr_SetString(PyExc_IOError, -+ "next() called during concurrent " -+ "operation on the same file object"); -+ return NULL; -+ } - if (f->f_buf == NULL) - if (readahead(f, bufsize) < 0) - return NULL; -@@ -2692,7 +2705,7 @@ int PyObject_AsFileDescriptor(PyObject *o) - } - else { - PyErr_SetString(PyExc_TypeError, -- "argument must be an int, or have a fileno() method."); -+ "argument must be an int, or have a fileno() method"); - return -1; - } - --- -2.13.3 - diff --git a/meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch b/meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch deleted file mode 100644 index 9b7713be88..0000000000 --- a/meta/recipes-devtools/python/python/CVE-2018-1000030-2.patch +++ /dev/null @@ -1,306 +0,0 @@ -From dbf52e02f18dac6f5f0a64f78932f3dc6efc056b Mon Sep 17 00:00:00 2001 -From: Benjamin Peterson -Date: Tue, 2 Jan 2018 09:25:41 -0800 -Subject: [PATCH] bpo-31530: fix crash when multiple threads iterate over a - file, round 2 (#5060) - -Multiple threads iterating over a file can corrupt the file's internal readahead -buffer resulting in crashes. To fix this, cache buffer state thread-locally for -the duration of a file_iternext call and only update the file's internal state -after reading completes. - -No attempt is made to define or provide "reasonable" semantics for iterating -over a file on multiple threads. (Non-crashing) races are still -present. Duplicated, corrupt, and missing data will happen. - -This was originally fixed by 6401e5671781eb217ee1afb4603cc0d1b0367ae6, which -raised an exception from seek() and next() when concurrent operations were -detected. Alas, this simpler solution breaks legitimate use cases such as -capturing the standard streams when multiple threads are logging. - -CVE: CVE-2018-1000030 -Upstream-Status: Backport [https://github.com/python/cpython/commit/dbf52e02f18dac6f5f0a64f78932f3dc6efc056b] - -Signed-off-by: Jagadeesh Krishnanjanappa - ---- - Lib/test/test_file2k.py | 27 ++--- - .../2017-09-20-18-28-09.bpo-31530.CdLOM7.rst | 3 - - Objects/fileobject.c | 118 ++++++++++++--------- - 3 files changed, 78 insertions(+), 70 deletions(-) - -diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py -index d8966e034e..c73e8d8dc4 100644 ---- a/Lib/test/test_file2k.py -+++ b/Lib/test/test_file2k.py -@@ -653,18 +653,15 @@ class FileThreadingTests(unittest.TestCase): - self._test_close_open_io(io_func) - - def test_iteration_torture(self): -- # bpo-31530: Crash when concurrently iterate over a file. -+ # bpo-31530 - with open(self.filename, "wb") as fp: - for i in xrange(2**20): - fp.write(b"0"*50 + b"\n") - with open(self.filename, "rb") as f: -- def iterate(): -- try: -- for l in f: -- pass -- except IOError: -+ def it(): -+ for l in f: - pass -- self._run_workers(iterate, 10) -+ self._run_workers(it, 10) - - def test_iteration_seek(self): - # bpo-31530: Crash when concurrently seek and iterate over a file. -@@ -674,17 +671,15 @@ class FileThreadingTests(unittest.TestCase): - with open(self.filename, "rb") as f: - it = iter([1] + [0]*10) # one thread reads, others seek - def iterate(): -- try: -- if next(it): -- for l in f: -- pass -- else: -- for i in range(100): -- f.seek(i*100, 0) -- except IOError: -- pass -+ if next(it): -+ for l in f: -+ pass -+ else: -+ for i in xrange(100): -+ f.seek(i*100, 0) - self._run_workers(iterate, 10) - -+ - @unittest.skipUnless(os.name == 'posix', 'test requires a posix system.') - class TestFileSignalEINTR(unittest.TestCase): - def _test_reading(self, data_to_write, read_and_verify_code, method_name, -diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst -index a6cb6c9e9b..beb09b5ae6 100644 ---- a/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst -+++ b/Misc/NEWS.d/next/Core and Builtins/2017-09-20-18-28-09.bpo-31530.CdLOM7.rst -@@ -1,4 +1 @@ - Fixed crashes when iterating over a file on multiple threads. --seek() and next() methods of file objects now raise an exception during --concurrent operation on the same file object. --A lock can be used to prevent the error. -diff --git a/Objects/fileobject.c b/Objects/fileobject.c -index 8d1c5812f0..270b28264a 100644 ---- a/Objects/fileobject.c -+++ b/Objects/fileobject.c -@@ -609,7 +609,12 @@ err_iterbuffered(void) - return NULL; - } - --static void drop_readahead(PyFileObject *); -+static void -+drop_file_readahead(PyFileObject *f) -+{ -+ PyMem_FREE(f->f_buf); -+ f->f_buf = NULL; -+} - - /* Methods */ - -@@ -632,7 +637,7 @@ file_dealloc(PyFileObject *f) - Py_XDECREF(f->f_mode); - Py_XDECREF(f->f_encoding); - Py_XDECREF(f->f_errors); -- drop_readahead(f); -+ drop_file_readahead(f); - Py_TYPE(f)->tp_free((PyObject *)f); - } - -@@ -767,13 +772,7 @@ file_seek(PyFileObject *f, PyObject *args) - - if (f->f_fp == NULL) - return err_closed(); -- if (f->unlocked_count > 0) { -- PyErr_SetString(PyExc_IOError, -- "seek() called during concurrent " -- "operation on the same file object"); -- return NULL; -- } -- drop_readahead(f); -+ drop_file_readahead(f); - whence = 0; - if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &whence)) - return NULL; -@@ -2242,12 +2241,16 @@ static PyGetSetDef file_getsetlist[] = { - {0}, - }; - -+typedef struct { -+ char *buf, *bufptr, *bufend; -+} readaheadbuffer; -+ - static void --drop_readahead(PyFileObject *f) -+drop_readaheadbuffer(readaheadbuffer *rab) - { -- if (f->f_buf != NULL) { -- PyMem_Free(f->f_buf); -- f->f_buf = NULL; -+ if (rab->buf != NULL) { -+ PyMem_FREE(rab->buf); -+ rab->buf = NULL; - } - } - -@@ -2255,36 +2258,34 @@ drop_readahead(PyFileObject *f) - (unless at EOF) and no more than bufsize. Returns negative value on - error, will set MemoryError if bufsize bytes cannot be allocated. */ - static int --readahead(PyFileObject *f, Py_ssize_t bufsize) -+readahead(PyFileObject *f, readaheadbuffer *rab, Py_ssize_t bufsize) - { - Py_ssize_t chunksize; - -- assert(f->unlocked_count == 0); -- if (f->f_buf != NULL) { -- if( (f->f_bufend - f->f_bufptr) >= 1) -+ if (rab->buf != NULL) { -+ if ((rab->bufend - rab->bufptr) >= 1) - return 0; - else -- drop_readahead(f); -+ drop_readaheadbuffer(rab); - } -- if ((f->f_buf = (char *)PyMem_Malloc(bufsize)) == NULL) { -+ if ((rab->buf = PyMem_MALLOC(bufsize)) == NULL) { - PyErr_NoMemory(); - return -1; - } - FILE_BEGIN_ALLOW_THREADS(f) - errno = 0; -- chunksize = Py_UniversalNewlineFread( -- f->f_buf, bufsize, f->f_fp, (PyObject *)f); -+ chunksize = Py_UniversalNewlineFread(rab->buf, bufsize, f->f_fp, (PyObject *)f); - FILE_END_ALLOW_THREADS(f) - if (chunksize == 0) { - if (ferror(f->f_fp)) { - PyErr_SetFromErrno(PyExc_IOError); - clearerr(f->f_fp); -- drop_readahead(f); -+ drop_readaheadbuffer(rab); - return -1; - } - } -- f->f_bufptr = f->f_buf; -- f->f_bufend = f->f_buf + chunksize; -+ rab->bufptr = rab->buf; -+ rab->bufend = rab->buf + chunksize; - return 0; - } - -@@ -2294,51 +2295,43 @@ readahead(PyFileObject *f, Py_ssize_t bufsize) - logarithmic buffer growth to about 50 even when reading a 1gb line. */ - - static PyStringObject * --readahead_get_line_skip(PyFileObject *f, Py_ssize_t skip, Py_ssize_t bufsize) -+readahead_get_line_skip(PyFileObject *f, readaheadbuffer *rab, Py_ssize_t skip, Py_ssize_t bufsize) - { - PyStringObject* s; - char *bufptr; - char *buf; - Py_ssize_t len; - -- if (f->unlocked_count > 0) { -- PyErr_SetString(PyExc_IOError, -- "next() called during concurrent " -- "operation on the same file object"); -- return NULL; -- } -- if (f->f_buf == NULL) -- if (readahead(f, bufsize) < 0) -+ if (rab->buf == NULL) -+ if (readahead(f, rab, bufsize) < 0) - return NULL; - -- len = f->f_bufend - f->f_bufptr; -+ len = rab->bufend - rab->bufptr; - if (len == 0) -- return (PyStringObject *) -- PyString_FromStringAndSize(NULL, skip); -- bufptr = (char *)memchr(f->f_bufptr, '\n', len); -+ return (PyStringObject *)PyString_FromStringAndSize(NULL, skip); -+ bufptr = (char *)memchr(rab->bufptr, '\n', len); - if (bufptr != NULL) { - bufptr++; /* Count the '\n' */ -- len = bufptr - f->f_bufptr; -- s = (PyStringObject *) -- PyString_FromStringAndSize(NULL, skip + len); -+ len = bufptr - rab->bufptr; -+ s = (PyStringObject *)PyString_FromStringAndSize(NULL, skip + len); - if (s == NULL) - return NULL; -- memcpy(PyString_AS_STRING(s) + skip, f->f_bufptr, len); -- f->f_bufptr = bufptr; -- if (bufptr == f->f_bufend) -- drop_readahead(f); -+ memcpy(PyString_AS_STRING(s) + skip, rab->bufptr, len); -+ rab->bufptr = bufptr; -+ if (bufptr == rab->bufend) -+ drop_readaheadbuffer(rab); - } else { -- bufptr = f->f_bufptr; -- buf = f->f_buf; -- f->f_buf = NULL; /* Force new readahead buffer */ -+ bufptr = rab->bufptr; -+ buf = rab->buf; -+ rab->buf = NULL; /* Force new readahead buffer */ - assert(len <= PY_SSIZE_T_MAX - skip); -- s = readahead_get_line_skip(f, skip + len, bufsize + (bufsize>>2)); -+ s = readahead_get_line_skip(f, rab, skip + len, bufsize + (bufsize>>2)); - if (s == NULL) { -- PyMem_Free(buf); -+ PyMem_FREE(buf); - return NULL; - } - memcpy(PyString_AS_STRING(s) + skip, bufptr, len); -- PyMem_Free(buf); -+ PyMem_FREE(buf); - } - return s; - } -@@ -2356,7 +2349,30 @@ file_iternext(PyFileObject *f) - if (!f->readable) - return err_mode("reading"); - -- l = readahead_get_line_skip(f, 0, READAHEAD_BUFSIZE); -+ { -+ /* -+ Multiple threads can enter this method while the GIL is released -+ during file read and wreak havoc on the file object's readahead -+ buffer. To avoid dealing with cross-thread coordination issues, we -+ cache the file buffer state locally and only set it back on the file -+ object when we're done. -+ */ -+ readaheadbuffer rab = {f->f_buf, f->f_bufptr, f->f_bufend}; -+ f->f_buf = NULL; -+ l = readahead_get_line_skip(f, &rab, 0, READAHEAD_BUFSIZE); -+ /* -+ Make sure the file's internal read buffer is cleared out. This will -+ only do anything if some other thread interleaved with us during -+ readahead. We want to drop any changeling buffer, so we don't leak -+ memory. We may lose data, but that's what you get for reading the same -+ file object in multiple threads. -+ */ -+ drop_file_readahead(f); -+ f->f_buf = rab.buf; -+ f->f_bufptr = rab.bufptr; -+ f->f_bufend = rab.bufend; -+ } -+ - if (l == NULL || PyString_GET_SIZE(l) == 0) { - Py_XDECREF(l); - return NULL; --- -2.13.3 - diff --git a/meta/recipes-devtools/python/python/fix-gc-alignment.patch b/meta/recipes-devtools/python/python/fix-gc-alignment.patch deleted file mode 100644 index b63cd08747..0000000000 --- a/meta/recipes-devtools/python/python/fix-gc-alignment.patch +++ /dev/null @@ -1,43 +0,0 @@ -Upstream-Status: Submitted -Signed-off-by: Ross Burton - -Fix for over-aligned GC info -Patch by Florian Weimer - -See: https://bugzilla.redhat.com/show_bug.cgi?id=1540316 -Upstream discussion: https://mail.python.org/pipermail/python-dev/2018-January/152000.html - -diff --git a/Include/objimpl.h b/Include/objimpl.h -index 55e83eced6..aa906144dc 100644 ---- a/Include/objimpl.h -+++ b/Include/objimpl.h -@@ -248,6 +248,18 @@ PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); - /* for source compatibility with 2.2 */ - #define _PyObject_GC_Del PyObject_GC_Del - -+/* Former over-aligned definition of PyGC_Head, used to compute the -+ size of the padding for the new version below. */ -+union _gc_head; -+union _gc_head_old { -+ struct { -+ union _gc_head *gc_next; -+ union _gc_head *gc_prev; -+ Py_ssize_t gc_refs; -+ } gc; -+ long double dummy; -+}; -+ - /* GC information is stored BEFORE the object structure. */ - typedef union _gc_head { - struct { -@@ -255,7 +267,8 @@ typedef union _gc_head { - union _gc_head *gc_prev; - Py_ssize_t gc_refs; - } gc; -- long double dummy; /* force worst-case alignment */ -+ double dummy; /* force worst-case alignment */ -+ char dummy_padding[sizeof(union _gc_head_old)]; - } PyGC_Head; - - extern PyGC_Head *_PyGC_generation0; - \ No newline at end of file diff --git a/meta/recipes-devtools/python/python_2.7.14.bb b/meta/recipes-devtools/python/python_2.7.14.bb deleted file mode 100644 index b923b9237b..0000000000 --- a/meta/recipes-devtools/python/python_2.7.14.bb +++ /dev/null @@ -1,282 +0,0 @@ -require python.inc - -DEPENDS = "python-native libffi bzip2 gdbm openssl readline sqlite3 zlib" - -PR = "${INC_PR}" - -DISTRO_SRC_URI ?= "file://sitecustomize.py" -DISTRO_SRC_URI_linuxstdbase = "" -SRC_URI += "\ - file://01-use-proper-tools-for-cross-build.patch \ - file://03-fix-tkinter-detection.patch \ - file://06-avoid_usr_lib_termcap_path_in_linking.patch \ - ${DISTRO_SRC_URI} \ - file://multilib.patch \ - file://cgi_py.patch \ - file://setup_py_skip_cross_import_check.patch \ - file://add-md5module-support.patch \ - file://host_include_contamination.patch \ - file://fix_for_using_different_libdir.patch \ - file://setuptweaks.patch \ - file://check-if-target-is-64b-not-host.patch \ - file://search_db_h_in_inc_dirs_and_avoid_warning.patch \ - file://avoid_warning_about_tkinter.patch \ - file://avoid_warning_for_sunos_specific_module.patch \ - file://python-2.7.3-remove-bsdb-rpath.patch \ - file://fix-makefile-for-ptest.patch \ - file://run-ptest \ - file://parallel-makeinst-create-bindir.patch \ - file://use_sysroot_ncurses_instead_of_host.patch \ - file://add-CROSSPYTHONPATH-for-PYTHON_FOR_BUILD.patch \ - file://pass-missing-libraries-to-Extension-for-mul.patch \ - file://support_SOURCE_DATE_EPOCH_in_py_compile_2.7.patch \ - file://fix-gc-alignment.patch \ -" - -S = "${WORKDIR}/Python-${PV}" - -inherit autotools multilib_header python-dir pythonnative - -CONFIGUREOPTS += " --with-system-ffi " - -EXTRA_OECONF += "ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no" - -PACKAGECONFIG ??= "bdb" -PACKAGECONFIG[bdb] = ",,db" - -do_configure_append() { - rm -f ${S}/Makefile.orig - autoreconf -Wcross --verbose --install --force --exclude=autopoint ../Python-${PV}/Modules/_ctypes/libffi -} - -do_compile() { - # regenerate platform specific files, because they depend on system headers - cd ${S}/Lib/plat-linux2 - include=${STAGING_INCDIR} ${STAGING_BINDIR_NATIVE}/python-native/python \ - ${S}/Tools/scripts/h2py.py -i '(u_long)' \ - ${STAGING_INCDIR}/dlfcn.h \ - ${STAGING_INCDIR}/linux/cdrom.h \ - ${STAGING_INCDIR}/netinet/in.h \ - ${STAGING_INCDIR}/sys/types.h - sed -e 's,${STAGING_DIR_HOST},,g' -i *.py - cd - - - # remove any bogus LD_LIBRARY_PATH - sed -i -e s,RUNSHARED=.*,RUNSHARED=, Makefile - - if [ ! -f Makefile.orig ]; then - install -m 0644 Makefile Makefile.orig - fi - sed -i -e 's#^LDFLAGS=.*#LDFLAGS=${LDFLAGS} -L. -L${STAGING_LIBDIR}#g' \ - -e 's,libdir=${libdir},libdir=${STAGING_LIBDIR},g' \ - -e 's,libexecdir=${libexecdir},libexecdir=${STAGING_DIR_HOST}${libexecdir},g' \ - -e 's,^LIBDIR=.*,LIBDIR=${STAGING_LIBDIR},g' \ - -e 's,includedir=${includedir},includedir=${STAGING_INCDIR},g' \ - -e 's,^INCLUDEDIR=.*,INCLUDE=${STAGING_INCDIR},g' \ - -e 's,^CONFINCLUDEDIR=.*,CONFINCLUDE=${STAGING_INCDIR},g' \ - Makefile - # save copy of it now, because if we do it in do_install and - # then call do_install twice we get Makefile.orig == Makefile.sysroot - install -m 0644 Makefile Makefile.sysroot - - export CROSS_COMPILE="${TARGET_PREFIX}" - export PYTHONBUILDDIR="${B}" - - oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python-native/pgen \ - HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python-native/python \ - STAGING_LIBDIR=${STAGING_LIBDIR} \ - STAGING_INCDIR=${STAGING_INCDIR} \ - STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ - OPT="${CFLAGS}" -} - -do_install() { - # make install needs the original Makefile, or otherwise the inclues would - # go to ${D}${STAGING...}/... - install -m 0644 Makefile.orig Makefile - - export CROSS_COMPILE="${TARGET_PREFIX}" - export PYTHONBUILDDIR="${B}" - - # After swizzling the makefile, we need to run the build again. - # install can race with the build so we have to run this first, then install - oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python-native/pgen \ - HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python-native/python \ - CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ \ - STAGING_LIBDIR=${STAGING_LIBDIR} \ - STAGING_INCDIR=${STAGING_INCDIR} \ - STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ - DESTDIR=${D} LIBDIR=${libdir} - - oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python-native/pgen \ - HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python-native/python \ - CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ \ - STAGING_LIBDIR=${STAGING_LIBDIR} \ - STAGING_INCDIR=${STAGING_INCDIR} \ - STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ - DESTDIR=${D} LIBDIR=${libdir} install - - install -m 0644 Makefile.sysroot ${D}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile - - if [ -e ${WORKDIR}/sitecustomize.py ]; then - install -m 0644 ${WORKDIR}/sitecustomize.py ${D}/${libdir}/python${PYTHON_MAJMIN} - fi - - oe_multilib_header python${PYTHON_MAJMIN}/pyconfig.h - - if [ -z "${@bb.utils.filter('PACKAGECONFIG', 'bdb', d)}" ]; then - rm -rf ${D}/${libdir}/python${PYTHON_MAJMIN}/bsddb - fi -} - -do_install_append_class-nativesdk () { - create_wrapper ${D}${bindir}/python2.7 PYTHONHOME='${prefix}' TERMINFO_DIRS='${sysconfdir}/terminfo:/etc/terminfo:/usr/share/terminfo:/usr/share/misc/terminfo:/lib/terminfo' PYTHONNOUSERSITE='1' -} - -SSTATE_SCAN_FILES += "Makefile" -PACKAGE_PREPROCESS_FUNCS += "py_package_preprocess" - -py_package_preprocess () { - # copy back the old Makefile to fix target package - install -m 0644 ${B}/Makefile.orig ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile - - # Remove references to buildmachine paths in target Makefile and _sysconfigdata - sed -i -e 's:--sysroot=${STAGING_DIR_TARGET}::g' -e s:'--with-libtool-sysroot=${STAGING_DIR_TARGET}'::g \ - -e 's|${DEBUG_PREFIX_MAP}||g' \ - -e 's:${HOSTTOOLS_DIR}/::g' \ - -e 's:${RECIPE_SYSROOT_NATIVE}::g' \ - -e 's:${RECIPE_SYSROOT}::g' \ - -e 's:${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}::g' \ - ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile \ - ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py - (cd ${PKGD}; python -m py_compile ./${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py) -} - - -# manual dependency additions -RPROVIDES_${PN}-core = "${PN}" -RRECOMMENDS_${PN}-core_append_class-nativesdk = " nativesdk-python-modules" -RRECOMMENDS_${PN}-crypt = "openssl" - -# package libpython2 -PACKAGES =+ "lib${BPN}2" -FILES_lib${BPN}2 = "${libdir}/libpython*.so.*" - -# catch all the rest (unsorted) -PACKAGES += "${PN}-misc" -FILES_${PN}-misc = "${libdir}/python${PYTHON_MAJMIN}" -RDEPENDS_${PN}-modules += "${PN}-misc" - -# ptest -RDEPENDS_${PN}-ptest = "${PN}-modules ${PN}-tests" -#inherit ptest after "require python-${PYTHON_MAJMIN}-manifest.inc" so PACKAGES doesn't get overwritten -inherit ptest - -# This must come after inherit ptest for the override to take effect -do_install_ptest() { - cp ${B}/Makefile ${D}${PTEST_PATH} - sed -e s:LIBDIR/python/ptest:${PTEST_PATH}:g \ - -e s:LIBDIR:${libdir}:g \ - -i ${D}${PTEST_PATH}/run-ptest - - #Remove build host references - sed -i \ - -e 's:--with-libtool-sysroot=${STAGING_DIR_TARGET}'::g \ - -e 's:--sysroot=${STAGING_DIR_TARGET}::g' \ - -e 's|${DEBUG_PREFIX_MAP}||g' \ - -e 's:${HOSTTOOLS_DIR}/::g' \ - -e 's:${RECIPE_SYSROOT}::g' \ - -e 's:${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}::g' \ - ${D}/${PTEST_PATH}/Makefile -} - -# catch manpage -PACKAGES += "${PN}-man" -FILES_${PN}-man = "${datadir}/man" - -# Nasty but if bdb isn't enabled the package won't be generated -RDEPENDS_${PN}-modules_remove = "${@bb.utils.contains('PACKAGECONFIG', 'bdb', '', '${PN}-bsddb', d)}" - -BBCLASSEXTEND = "nativesdk" - -RPROVIDES_${PN} += "${PN}-modules" - -# We want bytecode precompiled .py files (.pyc's) by default -# but the user may set it on their own conf - -INCLUDE_PYCS ?= "1" - -python(){ - import json - - filename = os.path.join(d.getVar('THISDIR'), 'python', 'python2-manifest.json') - # This python changes the datastore based on the contents of a file, so mark - # that dependency. - bb.parse.mark_dependency(d, filename) - - with open(filename) as manifest_file: - python_manifest=json.load(manifest_file) - - include_pycs = d.getVar('INCLUDE_PYCS') - - packages = d.getVar('PACKAGES').split() - pn = d.getVar('PN') - - newpackages=[] - - for key in python_manifest: - pypackage= pn + '-' + key - - if pypackage not in packages: - # We need to prepend, otherwise python-misc gets everything - # so we use a new variable - newpackages.append(pypackage) - - # "Build" python's manifest FILES, RDEPENDS and SUMMARY - d.setVar('FILES_' + pypackage, '') - for value in python_manifest[key]['files']: - d.appendVar('FILES_' + pypackage, ' ' + value) - if include_pycs == '1': - if value.endswith('.py'): - d.appendVar('FILES_' + pypackage, ' ' + value + 'c') - - d.setVar('RDEPENDS_' + pypackage, '') - for value in python_manifest[key]['rdepends']: - # Make it work with or without $PN - if '${PN}' in value: - value=value.split('-')[1] - d.appendVar('RDEPENDS_' + pypackage, ' ' + pn + '-' + value) - d.setVar('SUMMARY_' + pypackage, python_manifest[key]['summary']) - - # We need to ensure staticdev packages match for files first so we sort in reverse - newpackages.sort(reverse=True) - # Prepending so to avoid python-misc getting everything - packages = newpackages + packages - d.setVar('PACKAGES', ' '.join(packages)) - d.setVar('ALLOW_EMPTY_${PN}-modules', '1') -} - -# Files needed to create a new manifest -SRC_URI += "file://create_manifest2.py file://get_module_deps2.py file://python2-manifest.json" - -do_create_manifest() { - # This task should be run with every new release of Python. - # We must ensure that PACKAGECONFIG enables everything when creating - # a new manifest, this is to base our new manifest on a complete - # native python build, containing all dependencies, otherwise the task - # wont be able to find the required files. - # e.g. BerkeleyDB is an optional build dependency so it may or may not - # be present, we must ensure it is. - - cd ${WORKDIR} - # This needs to be executed by python-native and NOT by HOST's python - nativepython create_manifest2.py - cp python2-manifest.json.new ${THISDIR}/python/python2-manifest.json -} - -# bitbake python -c create_manifest -addtask do_create_manifest - -# Make sure we have native python ready when we create a new manifest -do_create_manifest[depends] += "python:do_prepare_recipe_sysroot" -do_create_manifest[depends] += "python:do_patch" diff --git a/meta/recipes-devtools/python/python_2.7.15.bb b/meta/recipes-devtools/python/python_2.7.15.bb new file mode 100644 index 0000000000..3044c3d363 --- /dev/null +++ b/meta/recipes-devtools/python/python_2.7.15.bb @@ -0,0 +1,281 @@ +require python.inc + +DEPENDS = "python-native libffi bzip2 gdbm openssl readline sqlite3 zlib" + +PR = "${INC_PR}" + +DISTRO_SRC_URI ?= "file://sitecustomize.py" +DISTRO_SRC_URI_linuxstdbase = "" +SRC_URI += "\ + file://01-use-proper-tools-for-cross-build.patch \ + file://03-fix-tkinter-detection.patch \ + file://06-avoid_usr_lib_termcap_path_in_linking.patch \ + ${DISTRO_SRC_URI} \ + file://multilib.patch \ + file://cgi_py.patch \ + file://setup_py_skip_cross_import_check.patch \ + file://add-md5module-support.patch \ + file://host_include_contamination.patch \ + file://fix_for_using_different_libdir.patch \ + file://setuptweaks.patch \ + file://check-if-target-is-64b-not-host.patch \ + file://search_db_h_in_inc_dirs_and_avoid_warning.patch \ + file://avoid_warning_about_tkinter.patch \ + file://avoid_warning_for_sunos_specific_module.patch \ + file://python-2.7.3-remove-bsdb-rpath.patch \ + file://fix-makefile-for-ptest.patch \ + file://run-ptest \ + file://parallel-makeinst-create-bindir.patch \ + file://use_sysroot_ncurses_instead_of_host.patch \ + file://add-CROSSPYTHONPATH-for-PYTHON_FOR_BUILD.patch \ + file://pass-missing-libraries-to-Extension-for-mul.patch \ + file://support_SOURCE_DATE_EPOCH_in_py_compile_2.7.patch \ +" + +S = "${WORKDIR}/Python-${PV}" + +inherit autotools multilib_header python-dir pythonnative + +CONFIGUREOPTS += " --with-system-ffi " + +EXTRA_OECONF += "ac_cv_file__dev_ptmx=yes ac_cv_file__dev_ptc=no" + +PACKAGECONFIG ??= "bdb" +PACKAGECONFIG[bdb] = ",,db" + +do_configure_append() { + rm -f ${S}/Makefile.orig + autoreconf -Wcross --verbose --install --force --exclude=autopoint ../Python-${PV}/Modules/_ctypes/libffi +} + +do_compile() { + # regenerate platform specific files, because they depend on system headers + cd ${S}/Lib/plat-linux2 + include=${STAGING_INCDIR} ${STAGING_BINDIR_NATIVE}/python-native/python \ + ${S}/Tools/scripts/h2py.py -i '(u_long)' \ + ${STAGING_INCDIR}/dlfcn.h \ + ${STAGING_INCDIR}/linux/cdrom.h \ + ${STAGING_INCDIR}/netinet/in.h \ + ${STAGING_INCDIR}/sys/types.h + sed -e 's,${STAGING_DIR_HOST},,g' -i *.py + cd - + + # remove any bogus LD_LIBRARY_PATH + sed -i -e s,RUNSHARED=.*,RUNSHARED=, Makefile + + if [ ! -f Makefile.orig ]; then + install -m 0644 Makefile Makefile.orig + fi + sed -i -e 's#^LDFLAGS=.*#LDFLAGS=${LDFLAGS} -L. -L${STAGING_LIBDIR}#g' \ + -e 's,libdir=${libdir},libdir=${STAGING_LIBDIR},g' \ + -e 's,libexecdir=${libexecdir},libexecdir=${STAGING_DIR_HOST}${libexecdir},g' \ + -e 's,^LIBDIR=.*,LIBDIR=${STAGING_LIBDIR},g' \ + -e 's,includedir=${includedir},includedir=${STAGING_INCDIR},g' \ + -e 's,^INCLUDEDIR=.*,INCLUDE=${STAGING_INCDIR},g' \ + -e 's,^CONFINCLUDEDIR=.*,CONFINCLUDE=${STAGING_INCDIR},g' \ + Makefile + # save copy of it now, because if we do it in do_install and + # then call do_install twice we get Makefile.orig == Makefile.sysroot + install -m 0644 Makefile Makefile.sysroot + + export CROSS_COMPILE="${TARGET_PREFIX}" + export PYTHONBUILDDIR="${B}" + + oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python-native/pgen \ + HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python-native/python \ + STAGING_LIBDIR=${STAGING_LIBDIR} \ + STAGING_INCDIR=${STAGING_INCDIR} \ + STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ + OPT="${CFLAGS}" +} + +do_install() { + # make install needs the original Makefile, or otherwise the inclues would + # go to ${D}${STAGING...}/... + install -m 0644 Makefile.orig Makefile + + export CROSS_COMPILE="${TARGET_PREFIX}" + export PYTHONBUILDDIR="${B}" + + # After swizzling the makefile, we need to run the build again. + # install can race with the build so we have to run this first, then install + oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python-native/pgen \ + HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python-native/python \ + CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ \ + STAGING_LIBDIR=${STAGING_LIBDIR} \ + STAGING_INCDIR=${STAGING_INCDIR} \ + STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ + DESTDIR=${D} LIBDIR=${libdir} + + oe_runmake HOSTPGEN=${STAGING_BINDIR_NATIVE}/python-native/pgen \ + HOSTPYTHON=${STAGING_BINDIR_NATIVE}/python-native/python \ + CROSSPYTHONPATH=${STAGING_LIBDIR_NATIVE}/python${PYTHON_MAJMIN}/lib-dynload/ \ + STAGING_LIBDIR=${STAGING_LIBDIR} \ + STAGING_INCDIR=${STAGING_INCDIR} \ + STAGING_BASELIBDIR=${STAGING_BASELIBDIR} \ + DESTDIR=${D} LIBDIR=${libdir} install + + install -m 0644 Makefile.sysroot ${D}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile + + if [ -e ${WORKDIR}/sitecustomize.py ]; then + install -m 0644 ${WORKDIR}/sitecustomize.py ${D}/${libdir}/python${PYTHON_MAJMIN} + fi + + oe_multilib_header python${PYTHON_MAJMIN}/pyconfig.h + + if [ -z "${@bb.utils.filter('PACKAGECONFIG', 'bdb', d)}" ]; then + rm -rf ${D}/${libdir}/python${PYTHON_MAJMIN}/bsddb + fi +} + +do_install_append_class-nativesdk () { + create_wrapper ${D}${bindir}/python2.7 PYTHONHOME='${prefix}' TERMINFO_DIRS='${sysconfdir}/terminfo:/etc/terminfo:/usr/share/terminfo:/usr/share/misc/terminfo:/lib/terminfo' PYTHONNOUSERSITE='1' +} + +SSTATE_SCAN_FILES += "Makefile" +PACKAGE_PREPROCESS_FUNCS += "py_package_preprocess" + +py_package_preprocess () { + # copy back the old Makefile to fix target package + install -m 0644 ${B}/Makefile.orig ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile + + # Remove references to buildmachine paths in target Makefile and _sysconfigdata + sed -i -e 's:--sysroot=${STAGING_DIR_TARGET}::g' -e s:'--with-libtool-sysroot=${STAGING_DIR_TARGET}'::g \ + -e 's|${DEBUG_PREFIX_MAP}||g' \ + -e 's:${HOSTTOOLS_DIR}/::g' \ + -e 's:${RECIPE_SYSROOT_NATIVE}::g' \ + -e 's:${RECIPE_SYSROOT}::g' \ + -e 's:${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}::g' \ + ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/config/Makefile \ + ${PKGD}/${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py + (cd ${PKGD}; python -m py_compile ./${libdir}/python${PYTHON_MAJMIN}/_sysconfigdata.py) +} + + +# manual dependency additions +RPROVIDES_${PN}-core = "${PN}" +RRECOMMENDS_${PN}-core_append_class-nativesdk = " nativesdk-python-modules" +RRECOMMENDS_${PN}-crypt = "openssl" + +# package libpython2 +PACKAGES =+ "lib${BPN}2" +FILES_lib${BPN}2 = "${libdir}/libpython*.so.*" + +# catch all the rest (unsorted) +PACKAGES += "${PN}-misc" +FILES_${PN}-misc = "${libdir}/python${PYTHON_MAJMIN}" +RDEPENDS_${PN}-modules += "${PN}-misc" + +# ptest +RDEPENDS_${PN}-ptest = "${PN}-modules ${PN}-tests" +#inherit ptest after "require python-${PYTHON_MAJMIN}-manifest.inc" so PACKAGES doesn't get overwritten +inherit ptest + +# This must come after inherit ptest for the override to take effect +do_install_ptest() { + cp ${B}/Makefile ${D}${PTEST_PATH} + sed -e s:LIBDIR/python/ptest:${PTEST_PATH}:g \ + -e s:LIBDIR:${libdir}:g \ + -i ${D}${PTEST_PATH}/run-ptest + + #Remove build host references + sed -i \ + -e 's:--with-libtool-sysroot=${STAGING_DIR_TARGET}'::g \ + -e 's:--sysroot=${STAGING_DIR_TARGET}::g' \ + -e 's|${DEBUG_PREFIX_MAP}||g' \ + -e 's:${HOSTTOOLS_DIR}/::g' \ + -e 's:${RECIPE_SYSROOT}::g' \ + -e 's:${BASE_WORKDIR}/${MULTIMACH_TARGET_SYS}::g' \ + ${D}/${PTEST_PATH}/Makefile +} + +# catch manpage +PACKAGES += "${PN}-man" +FILES_${PN}-man = "${datadir}/man" + +# Nasty but if bdb isn't enabled the package won't be generated +RDEPENDS_${PN}-modules_remove = "${@bb.utils.contains('PACKAGECONFIG', 'bdb', '', '${PN}-bsddb', d)}" + +BBCLASSEXTEND = "nativesdk" + +RPROVIDES_${PN} += "${PN}-modules" + +# We want bytecode precompiled .py files (.pyc's) by default +# but the user may set it on their own conf + +INCLUDE_PYCS ?= "1" + +python(){ + import json + + filename = os.path.join(d.getVar('THISDIR'), 'python', 'python2-manifest.json') + # This python changes the datastore based on the contents of a file, so mark + # that dependency. + bb.parse.mark_dependency(d, filename) + + with open(filename) as manifest_file: + python_manifest=json.load(manifest_file) + + include_pycs = d.getVar('INCLUDE_PYCS') + + packages = d.getVar('PACKAGES').split() + pn = d.getVar('PN') + + newpackages=[] + + for key in python_manifest: + pypackage= pn + '-' + key + + if pypackage not in packages: + # We need to prepend, otherwise python-misc gets everything + # so we use a new variable + newpackages.append(pypackage) + + # "Build" python's manifest FILES, RDEPENDS and SUMMARY + d.setVar('FILES_' + pypackage, '') + for value in python_manifest[key]['files']: + d.appendVar('FILES_' + pypackage, ' ' + value) + if include_pycs == '1': + if value.endswith('.py'): + d.appendVar('FILES_' + pypackage, ' ' + value + 'c') + + d.setVar('RDEPENDS_' + pypackage, '') + for value in python_manifest[key]['rdepends']: + # Make it work with or without $PN + if '${PN}' in value: + value=value.split('-')[1] + d.appendVar('RDEPENDS_' + pypackage, ' ' + pn + '-' + value) + d.setVar('SUMMARY_' + pypackage, python_manifest[key]['summary']) + + # We need to ensure staticdev packages match for files first so we sort in reverse + newpackages.sort(reverse=True) + # Prepending so to avoid python-misc getting everything + packages = newpackages + packages + d.setVar('PACKAGES', ' '.join(packages)) + d.setVar('ALLOW_EMPTY_${PN}-modules', '1') +} + +# Files needed to create a new manifest +SRC_URI += "file://create_manifest2.py file://get_module_deps2.py file://python2-manifest.json" + +do_create_manifest() { + # This task should be run with every new release of Python. + # We must ensure that PACKAGECONFIG enables everything when creating + # a new manifest, this is to base our new manifest on a complete + # native python build, containing all dependencies, otherwise the task + # wont be able to find the required files. + # e.g. BerkeleyDB is an optional build dependency so it may or may not + # be present, we must ensure it is. + + cd ${WORKDIR} + # This needs to be executed by python-native and NOT by HOST's python + nativepython create_manifest2.py + cp python2-manifest.json.new ${THISDIR}/python/python2-manifest.json +} + +# bitbake python -c create_manifest +addtask do_create_manifest + +# Make sure we have native python ready when we create a new manifest +do_create_manifest[depends] += "python:do_prepare_recipe_sysroot" +do_create_manifest[depends] += "python:do_patch" -- cgit v1.2.3-54-g00ecf