summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--meta/recipes-devtools/python/python-native_2.7.15.bb6
-rw-r--r--meta/recipes-devtools/python/python/create_manifest2.py21
-rw-r--r--meta/recipes-devtools/python/python/python2-manifest.json90
-rw-r--r--meta/recipes-devtools/python/python_2.7.15.bb6
4 files changed, 117 insertions, 6 deletions
diff --git a/meta/recipes-devtools/python/python-native_2.7.15.bb b/meta/recipes-devtools/python/python-native_2.7.15.bb
index 7c491fa3e0..de35104c60 100644
--- a/meta/recipes-devtools/python/python-native_2.7.15.bb
+++ b/meta/recipes-devtools/python/python-native_2.7.15.bb
@@ -69,7 +69,11 @@ python(){
69 import json 69 import json
70 pythondir = d.getVar('THISDIR',True) 70 pythondir = d.getVar('THISDIR',True)
71 with open(pythondir+'/python/python2-manifest.json') as manifest_file: 71 with open(pythondir+'/python/python2-manifest.json') as manifest_file:
72 python_manifest=json.load(manifest_file) 72 manifest_str = manifest_file.read()
73 json_start = manifest_str.find('# EOC') + 6
74 manifest_file.seek(json_start)
75 manifest_str = manifest_file.read()
76 python_manifest = json.loads(manifest_str)
73 77
74 rprovides = d.getVar('RPROVIDES').split() 78 rprovides = d.getVar('RPROVIDES').split()
75 79
diff --git a/meta/recipes-devtools/python/python/create_manifest2.py b/meta/recipes-devtools/python/python/create_manifest2.py
index 87999991d7..b6748653b6 100644
--- a/meta/recipes-devtools/python/python/create_manifest2.py
+++ b/meta/recipes-devtools/python/python/create_manifest2.py
@@ -22,7 +22,7 @@
22# 22#
23# 23#
24# This way we will create a new manifest from the data structure that was built during 24# This way we will create a new manifest from the data structure that was built during
25# this process, ont this new manifest each package will contain specifically only 25# this process, on this new manifest each package will contain specifically only
26# what it needs to run. 26# what it needs to run.
27# 27#
28# There are some caveats which we try to deal with, such as repeated files on different 28# There are some caveats which we try to deal with, such as repeated files on different
@@ -30,7 +30,7 @@
30# Its also important to note that this method only works for python files, and shared 30# Its also important to note that this method only works for python files, and shared
31# libraries. Static libraries, header files and binaries need to be dealt with manually. 31# libraries. Static libraries, header files and binaries need to be dealt with manually.
32# 32#
33# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> 33# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com>
34 34
35 35
36import sys 36import sys
@@ -62,10 +62,21 @@ def isFolder(value):
62 else: 62 else:
63 return False 63 return False
64 64
65def prepend_comments(comments, json_manifest):
66 with open(json_manifest, 'r+') as manifest:
67 json_contents = manifest.read()
68 manifest.seek(0, 0)
69 manifest.write(comments + json_contents)
70
65# Read existing JSON manifest 71# Read existing JSON manifest
66with open('python2-manifest.json') as manifest: 72with open('python2-manifest.json') as manifest:
67 old_manifest = json.load(manifest, object_pairs_hook=collections.OrderedDict) 73 # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker
68 74 manifest_str = manifest.read()
75 json_start = manifest_str.find('# EOC') + 6 # EOC + \n
76 manifest.seek(0)
77 comments = manifest.read(json_start)
78 manifest_str = manifest.read()
79 old_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
69 80
70# First pass to get core-package functionality, because we base everything on the fact that core is actually working 81# First pass to get core-package functionality, because we base everything on the fact that core is actually working
71# Not exactly the same so it should not be a function 82# Not exactly the same so it should not be a function
@@ -277,3 +288,5 @@ for key in new_manifest:
277# Create the manifest from the data structure that was built 288# Create the manifest from the data structure that was built
278with open('python2-manifest.json.new','w') as outfile: 289with open('python2-manifest.json.new','w') as outfile:
279 json.dump(new_manifest,outfile, indent=4) 290 json.dump(new_manifest,outfile, indent=4)
291
292prepend_comments(comments,'python2-manifest.json.new')
diff --git a/meta/recipes-devtools/python/python/python2-manifest.json b/meta/recipes-devtools/python/python/python2-manifest.json
index a9f392475c..4fff54a95b 100644
--- a/meta/recipes-devtools/python/python/python2-manifest.json
+++ b/meta/recipes-devtools/python/python/python2-manifest.json
@@ -1,3 +1,93 @@
1# DO NOT (entirely) modify this file manually, please read.
2#
3# IMPORTANT NOTE:
4# Please keep in mind that the create_manifest task relies on the fact the the
5# target and native Python packages are the same, and it also needs to be executed
6# with a fully working native package (with all the PACKAGECONFIGs enabled and all
7# and all the modules should be working, check log.do_compile), otherwise the script
8# will fail to find dependencies correctly, this note is valid either if you are
9# upgrading to a new Python version or adding a new package.
10#
11#
12# If you are adding a new package please follow the next steps:
13# How to add a new package:
14# - If a user wants to add a new package all that has to be done is:
15# Modify the python2-manifest.json file, and add the required file(s) to the FILES list,
16# fill up the SUMMARY section as well, the script should handle all the rest.
17#
18# Real example:
19# We want to add a web browser package, including the file webbrowser.py
20# which at the moment is on python-misc.
21# "webbrowser": {
22# "files": ["${libdir}/python2.7/lib-dynload/webbrowser.py"],
23# "rdepends": [],
24# "summary": "Python Web Browser support"}
25#
26# * Note that the rdepends field was left empty
27#
28# We run $ bitbake python -c create_manifest and the resulting manifest
29# should be completed after a few seconds, showing something like:
30# "webbrowser": {
31# "files": ["${libdir}/python2.7/webbrowser.py"],
32# "rdepends": ["core","fcntl","io","pickle","shell","subprocess"],
33# "summary": "Python Web Browser support"}
34#
35#
36# If you are upgrading Python to a new version please follow the next steps:
37# After each Python upgrade, the create_manifest task should be executed, because we
38# don't control what changes on upstream Python, so, some module dependency
39# might have changed without us realizing it, a certain module can either have
40# more or less dependencies, or could be depending on a new file that was just
41# created on the new release and for obvious reasons we wouldn't have it on our
42# old manifest, all of these issues would cause runtime errors on our system.
43#
44# - Upgrade both the native and target Python packages to a new version
45# - Run the create_manifest task for the target Python package as its shown below:
46#
47# $ bitbake python -c create_manifest
48#
49# This will automatically replace your manifest file located under the Python directory
50# with an new one, which contains the new dependencies (if any).
51#
52# Several things could have gone wrong here, I will try to explain a few:
53#
54# a) A new file was introduced on this release, e.g. sha3*.so:
55# The task will check what its needed to import every module, more than one module would
56# would probably depend on sha3*.so, although only one module should contain it.
57#
58# After running the task, the new manifest will have the sha3*.so file on more than one
59# module, you need to manually decide which one of them should get it and delete it from
60# the others, for example sha3*.so should likely be on ${PN}-crypt.
61# Once you have deleted from the others you need to run the create_manifest task again,
62# this will populate the other module's rdepends fields, with ${PN}-crypt and you should be
63# good to go.
64#
65# b) The native package wasn't built correctly and its missing a certain module:
66# As mentioned before, you need to make sure the native package was built with all the modules
67# because it is used as base to build the manifest file, you need to manually check log.do_compile
68# since it won't error out the compile function if its only missing a couple of modules.
69#
70# e.g. missing the _uuid module, log.do_compile would show the following:
71# Python build finished successfully!
72# The necessary bits to build these optional modules were not found:
73# _uuid
74#
75# What will happen here is that the new manifest would not be aware that the _uuid module exists, so
76# not only we won't know of any dependencies to it, but also, the _uuid* files will be packaged on
77# the misc package (which is where any file that doesn't belong anywhere else ends up).
78#
79# This will eventually cause runtime errors on our system if we don't include the misc package on
80# on our image, because the _uuid files will be missing.
81# If we build the _uuid module correctly and run the create_manifest task the _uuid files will be
82# detected correctly along with its dependencies, and we will get a working manifest.
83#
84# This is the reason why it is important to make sure we have a fully working native build,
85# so we can avoid these errors.
86#
87#
88#
89# DO NOT MODIFY THE NEXT LINE!, IT IS USED AS A MARKER FOR THE ACTUAL JSON MANIFEST
90# EOC
1{ 91{
2 "tests": { 92 "tests": {
3 "summary": "Python test suite", 93 "summary": "Python test suite",
diff --git a/meta/recipes-devtools/python/python_2.7.15.bb b/meta/recipes-devtools/python/python_2.7.15.bb
index 6a60aa7f5b..7a6da884e5 100644
--- a/meta/recipes-devtools/python/python_2.7.15.bb
+++ b/meta/recipes-devtools/python/python_2.7.15.bb
@@ -199,7 +199,11 @@ python(){
199 bb.parse.mark_dependency(d, filename) 199 bb.parse.mark_dependency(d, filename)
200 200
201 with open(filename) as manifest_file: 201 with open(filename) as manifest_file:
202 python_manifest=json.load(manifest_file, object_pairs_hook=collections.OrderedDict) 202 manifest_str = manifest_file.read()
203 json_start = manifest_str.find('# EOC') + 6
204 manifest_file.seek(json_start)
205 manifest_str = manifest_file.read()
206 python_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
203 207
204 include_pycs = d.getVar('INCLUDE_PYCS') 208 include_pycs = d.getVar('INCLUDE_PYCS')
205 209