summaryrefslogtreecommitdiffstats
path: root/meta/recipes-devtools
diff options
context:
space:
mode:
authorAlejandro Enedino Hernandez Samaniego <alejandro.enedino.hernandez-samaniego@xilinx.com>2018-11-16 11:31:46 -0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-12-01 11:38:37 +0000
commit8627773268fdb58711ef23cc5d54274cc82052b8 (patch)
treedb88c04ec89307707328779d48c60c4b99951000 /meta/recipes-devtools
parent6c524c09067a57c6e78e41a3860387dd4b31dc2b (diff)
downloadpoky-8627773268fdb58711ef23cc5d54274cc82052b8.tar.gz
python: Adds instructions to the manifest file
While there is a bit of documentation regarding building a new manifest file for python, it seems that users usually only read the manifest file. The manifest file is in JSON format which doesn't allow comments, hence why instructions were initially put elsewhere. This patch hacks the call to open the JSON manifest file by using a marker to trick it into reading only part of the file as the manifest itself, and keep the other part as comments, which contain instructions for the user to run the create_manifest task after an upgrade or when adding a new package. (From OE-Core rev: 5641a24a70b54544012c04c6a082514d9a5aa49a) Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandr@xilinx.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/recipes-devtools')
-rw-r--r--meta/recipes-devtools/python/python-native_2.7.15.bb6
-rw-r--r--meta/recipes-devtools/python/python/create_manifest2.py21
-rw-r--r--meta/recipes-devtools/python/python/python2-manifest.json90
-rw-r--r--meta/recipes-devtools/python/python_2.7.15.bb6
4 files changed, 117 insertions, 6 deletions
diff --git a/meta/recipes-devtools/python/python-native_2.7.15.bb b/meta/recipes-devtools/python/python-native_2.7.15.bb
index 7c491fa3e0..de35104c60 100644
--- a/meta/recipes-devtools/python/python-native_2.7.15.bb
+++ b/meta/recipes-devtools/python/python-native_2.7.15.bb
@@ -69,7 +69,11 @@ python(){
69 import json 69 import json
70 pythondir = d.getVar('THISDIR',True) 70 pythondir = d.getVar('THISDIR',True)
71 with open(pythondir+'/python/python2-manifest.json') as manifest_file: 71 with open(pythondir+'/python/python2-manifest.json') as manifest_file:
72 python_manifest=json.load(manifest_file) 72 manifest_str = manifest_file.read()
73 json_start = manifest_str.find('# EOC') + 6
74 manifest_file.seek(json_start)
75 manifest_str = manifest_file.read()
76 python_manifest = json.loads(manifest_str)
73 77
74 rprovides = d.getVar('RPROVIDES').split() 78 rprovides = d.getVar('RPROVIDES').split()
75 79
diff --git a/meta/recipes-devtools/python/python/create_manifest2.py b/meta/recipes-devtools/python/python/create_manifest2.py
index 87999991d7..b6748653b6 100644
--- a/meta/recipes-devtools/python/python/create_manifest2.py
+++ b/meta/recipes-devtools/python/python/create_manifest2.py
@@ -22,7 +22,7 @@
22# 22#
23# 23#
24# This way we will create a new manifest from the data structure that was built during 24# This way we will create a new manifest from the data structure that was built during
25# this process, ont this new manifest each package will contain specifically only 25# this process, on this new manifest each package will contain specifically only
26# what it needs to run. 26# what it needs to run.
27# 27#
28# There are some caveats which we try to deal with, such as repeated files on different 28# There are some caveats which we try to deal with, such as repeated files on different
@@ -30,7 +30,7 @@
30# Its also important to note that this method only works for python files, and shared 30# Its also important to note that this method only works for python files, and shared
31# libraries. Static libraries, header files and binaries need to be dealt with manually. 31# libraries. Static libraries, header files and binaries need to be dealt with manually.
32# 32#
33# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> 33# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com>
34 34
35 35
36import sys 36import sys
@@ -62,10 +62,21 @@ def isFolder(value):
62 else: 62 else:
63 return False 63 return False
64 64
65def prepend_comments(comments, json_manifest):
66 with open(json_manifest, 'r+') as manifest:
67 json_contents = manifest.read()
68 manifest.seek(0, 0)
69 manifest.write(comments + json_contents)
70
65# Read existing JSON manifest 71# Read existing JSON manifest
66with open('python2-manifest.json') as manifest: 72with open('python2-manifest.json') as manifest:
67 old_manifest = json.load(manifest, object_pairs_hook=collections.OrderedDict) 73 # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker
68 74 manifest_str = manifest.read()
75 json_start = manifest_str.find('# EOC') + 6 # EOC + \n
76 manifest.seek(0)
77 comments = manifest.read(json_start)
78 manifest_str = manifest.read()
79 old_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
69 80
70# First pass to get core-package functionality, because we base everything on the fact that core is actually working 81# First pass to get core-package functionality, because we base everything on the fact that core is actually working
71# Not exactly the same so it should not be a function 82# Not exactly the same so it should not be a function
@@ -277,3 +288,5 @@ for key in new_manifest:
277# Create the manifest from the data structure that was built 288# Create the manifest from the data structure that was built
278with open('python2-manifest.json.new','w') as outfile: 289with open('python2-manifest.json.new','w') as outfile:
279 json.dump(new_manifest,outfile, indent=4) 290 json.dump(new_manifest,outfile, indent=4)
291
292prepend_comments(comments,'python2-manifest.json.new')
diff --git a/meta/recipes-devtools/python/python/python2-manifest.json b/meta/recipes-devtools/python/python/python2-manifest.json
index a9f392475c..4fff54a95b 100644
--- a/meta/recipes-devtools/python/python/python2-manifest.json
+++ b/meta/recipes-devtools/python/python/python2-manifest.json
@@ -1,3 +1,93 @@
1# DO NOT (entirely) modify this file manually, please read.
2#
3# IMPORTANT NOTE:
4# Please keep in mind that the create_manifest task relies on the fact the the
5# target and native Python packages are the same, and it also needs to be executed
6# with a fully working native package (with all the PACKAGECONFIGs enabled and all
7# and all the modules should be working, check log.do_compile), otherwise the script
8# will fail to find dependencies correctly, this note is valid either if you are
9# upgrading to a new Python version or adding a new package.
10#
11#
12# If you are adding a new package please follow the next steps:
13# How to add a new package:
14# - If a user wants to add a new package all that has to be done is:
15# Modify the python2-manifest.json file, and add the required file(s) to the FILES list,
16# fill up the SUMMARY section as well, the script should handle all the rest.
17#
18# Real example:
19# We want to add a web browser package, including the file webbrowser.py
20# which at the moment is on python-misc.
21# "webbrowser": {
22# "files": ["${libdir}/python2.7/lib-dynload/webbrowser.py"],
23# "rdepends": [],
24# "summary": "Python Web Browser support"}
25#
26# * Note that the rdepends field was left empty
27#
28# We run $ bitbake python -c create_manifest and the resulting manifest
29# should be completed after a few seconds, showing something like:
30# "webbrowser": {
31# "files": ["${libdir}/python2.7/webbrowser.py"],
32# "rdepends": ["core","fcntl","io","pickle","shell","subprocess"],
33# "summary": "Python Web Browser support"}
34#
35#
36# If you are upgrading Python to a new version please follow the next steps:
37# After each Python upgrade, the create_manifest task should be executed, because we
38# don't control what changes on upstream Python, so, some module dependency
39# might have changed without us realizing it, a certain module can either have
40# more or less dependencies, or could be depending on a new file that was just
41# created on the new release and for obvious reasons we wouldn't have it on our
42# old manifest, all of these issues would cause runtime errors on our system.
43#
44# - Upgrade both the native and target Python packages to a new version
45# - Run the create_manifest task for the target Python package as its shown below:
46#
47# $ bitbake python -c create_manifest
48#
49# This will automatically replace your manifest file located under the Python directory
50# with an new one, which contains the new dependencies (if any).
51#
52# Several things could have gone wrong here, I will try to explain a few:
53#
54# a) A new file was introduced on this release, e.g. sha3*.so:
55# The task will check what its needed to import every module, more than one module would
56# would probably depend on sha3*.so, although only one module should contain it.
57#
58# After running the task, the new manifest will have the sha3*.so file on more than one
59# module, you need to manually decide which one of them should get it and delete it from
60# the others, for example sha3*.so should likely be on ${PN}-crypt.
61# Once you have deleted from the others you need to run the create_manifest task again,
62# this will populate the other module's rdepends fields, with ${PN}-crypt and you should be
63# good to go.
64#
65# b) The native package wasn't built correctly and its missing a certain module:
66# As mentioned before, you need to make sure the native package was built with all the modules
67# because it is used as base to build the manifest file, you need to manually check log.do_compile
68# since it won't error out the compile function if its only missing a couple of modules.
69#
70# e.g. missing the _uuid module, log.do_compile would show the following:
71# Python build finished successfully!
72# The necessary bits to build these optional modules were not found:
73# _uuid
74#
75# What will happen here is that the new manifest would not be aware that the _uuid module exists, so
76# not only we won't know of any dependencies to it, but also, the _uuid* files will be packaged on
77# the misc package (which is where any file that doesn't belong anywhere else ends up).
78#
79# This will eventually cause runtime errors on our system if we don't include the misc package on
80# on our image, because the _uuid files will be missing.
81# If we build the _uuid module correctly and run the create_manifest task the _uuid files will be
82# detected correctly along with its dependencies, and we will get a working manifest.
83#
84# This is the reason why it is important to make sure we have a fully working native build,
85# so we can avoid these errors.
86#
87#
88#
89# DO NOT MODIFY THE NEXT LINE!, IT IS USED AS A MARKER FOR THE ACTUAL JSON MANIFEST
90# EOC
1{ 91{
2 "tests": { 92 "tests": {
3 "summary": "Python test suite", 93 "summary": "Python test suite",
diff --git a/meta/recipes-devtools/python/python_2.7.15.bb b/meta/recipes-devtools/python/python_2.7.15.bb
index 6a60aa7f5b..7a6da884e5 100644
--- a/meta/recipes-devtools/python/python_2.7.15.bb
+++ b/meta/recipes-devtools/python/python_2.7.15.bb
@@ -199,7 +199,11 @@ python(){
199 bb.parse.mark_dependency(d, filename) 199 bb.parse.mark_dependency(d, filename)
200 200
201 with open(filename) as manifest_file: 201 with open(filename) as manifest_file:
202 python_manifest=json.load(manifest_file, object_pairs_hook=collections.OrderedDict) 202 manifest_str = manifest_file.read()
203 json_start = manifest_str.find('# EOC') + 6
204 manifest_file.seek(json_start)
205 manifest_str = manifest_file.read()
206 python_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
203 207
204 include_pycs = d.getVar('INCLUDE_PYCS') 208 include_pycs = d.getVar('INCLUDE_PYCS')
205 209