summaryrefslogtreecommitdiffstats
path: root/meta/recipes-devtools/python
diff options
context:
space:
mode:
authorAlejandro Enedino Hernandez Samaniego <alejandro.enedino.hernandez-samaniego@xilinx.com>2018-11-16 11:31:45 -0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-12-01 11:38:37 +0000
commit6c524c09067a57c6e78e41a3860387dd4b31dc2b (patch)
treebf24aaa5b511179ea6edbd8d8840c00c946e624b /meta/recipes-devtools/python
parenta1e2c4e9bde9af41d797a2d0b7e4a3448c47bd64 (diff)
downloadpoky-6c524c09067a57c6e78e41a3860387dd4b31dc2b.tar.gz
python3: Adds instructions to the manifest file
While there is a bit of documentation regarding building a new manifest file for python, it seems that users usually only read the manifest file. The manifest file is in JSON format which doesn't allow comments, hence why instructions were initially put elsewhere. This patch hacks the call to open the JSON manifest file by using a marker to trick it into reading only part of the file as the manifest itself, and keep the other part as comments, which contain instructions for the user to run the create_manifest task after an upgrade or when adding a new package. (From OE-Core rev: 3eab24c6dc095fd2305b9be8467aab1191141e35) Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandr@xilinx.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/recipes-devtools/python')
-rw-r--r--meta/recipes-devtools/python/python3-native_3.5.6.bb6
-rw-r--r--meta/recipes-devtools/python/python3/create_manifest3.py20
-rw-r--r--meta/recipes-devtools/python/python3/python3-manifest.json94
-rw-r--r--meta/recipes-devtools/python/python3_3.5.6.bb6
4 files changed, 119 insertions, 7 deletions
diff --git a/meta/recipes-devtools/python/python3-native_3.5.6.bb b/meta/recipes-devtools/python/python3-native_3.5.6.bb
index 1e98c036c3..4c8faf88c0 100644
--- a/meta/recipes-devtools/python/python3-native_3.5.6.bb
+++ b/meta/recipes-devtools/python/python3-native_3.5.6.bb
@@ -85,7 +85,11 @@ python(){
85 import json 85 import json
86 pythondir = d.getVar('THISDIR',True) 86 pythondir = d.getVar('THISDIR',True)
87 with open(pythondir+'/python3/python3-manifest.json') as manifest_file: 87 with open(pythondir+'/python3/python3-manifest.json') as manifest_file:
88 python_manifest=json.load(manifest_file) 88 manifest_str = manifest_file.read()
89 json_start = manifest_str.find('# EOC') + 6
90 manifest_file.seek(json_start)
91 manifest_str = manifest_file.read()
92 python_manifest = json.loads(manifest_str)
89 93
90 rprovides = d.getVar('RPROVIDES').split() 94 rprovides = d.getVar('RPROVIDES').split()
91 95
diff --git a/meta/recipes-devtools/python/python3/create_manifest3.py b/meta/recipes-devtools/python/python3/create_manifest3.py
index fddb23cdc4..f7d4587030 100644
--- a/meta/recipes-devtools/python/python3/create_manifest3.py
+++ b/meta/recipes-devtools/python/python3/create_manifest3.py
@@ -22,7 +22,7 @@
22# 22#
23# 23#
24# This way we will create a new manifest from the data structure that was built during 24# This way we will create a new manifest from the data structure that was built during
25# this process, ont this new manifest each package will contain specifically only 25# this process, on this new manifest each package will contain specifically only
26# what it needs to run. 26# what it needs to run.
27# 27#
28# There are some caveats which we try to deal with, such as repeated files on different 28# There are some caveats which we try to deal with, such as repeated files on different
@@ -36,7 +36,7 @@
36# Tha method to handle cached files does not work when a module includes a folder which 36# Tha method to handle cached files does not work when a module includes a folder which
37# itself contains the pycache folder, gladly this is almost never the case. 37# itself contains the pycache folder, gladly this is almost never the case.
38# 38#
39# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> 39# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com>
40 40
41 41
42import sys 42import sys
@@ -78,9 +78,21 @@ def isCached(item):
78 else: 78 else:
79 return False 79 return False
80 80
81def prepend_comments(comments, json_manifest):
82 with open(json_manifest, 'r+') as manifest:
83 json_contents = manifest.read()
84 manifest.seek(0, 0)
85 manifest.write(comments + json_contents)
86
81# Read existing JSON manifest 87# Read existing JSON manifest
82with open('python3-manifest.json') as manifest: 88with open('python3-manifest.json') as manifest:
83 old_manifest = json.load(manifest, object_pairs_hook=collections.OrderedDict) 89 # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker
90 manifest_str = manifest.read()
91 json_start = manifest_str.find('# EOC') + 6 # EOC + \n
92 manifest.seek(0)
93 comments = manifest.read(json_start)
94 manifest_str = manifest.read()
95 old_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
84 96
85# 97#
86# First pass to get core-package functionality, because we base everything on the fact that core is actually working 98# First pass to get core-package functionality, because we base everything on the fact that core is actually working
@@ -402,6 +414,8 @@ with open('python3-manifest.json.new','w') as outfile:
402 json.dump(new_manifest,outfile, indent=4) 414 json.dump(new_manifest,outfile, indent=4)
403 outfile.write('\n') 415 outfile.write('\n')
404 416
417prepend_comments(comments,'python3-manifest.json.new')
418
405if (repeated): 419if (repeated):
406 error_msg = '\n\nERROR:\n' 420 error_msg = '\n\nERROR:\n'
407 error_msg += 'The following files are repeated (contained in more than one package),\n' 421 error_msg += 'The following files are repeated (contained in more than one package),\n'
diff --git a/meta/recipes-devtools/python/python3/python3-manifest.json b/meta/recipes-devtools/python/python3/python3-manifest.json
index 617256a3e0..3641d5bd7b 100644
--- a/meta/recipes-devtools/python/python3/python3-manifest.json
+++ b/meta/recipes-devtools/python/python3/python3-manifest.json
@@ -1,3 +1,93 @@
1# DO NOT (entirely) modify this file manually, please read.
2#
3# IMPORTANT NOTE:
4# Please keep in mind that the create_manifest task relies on the fact the the
5# target and native Python packages are the same, and it also needs to be executed
6# with a fully working native package (with all the PACKAGECONFIGs enabled and all
7# and all the modules should be working, check log.do_compile), otherwise the script
8# will fail to find dependencies correctly, this note is valid either if you are
9# upgrading to a new Python version or adding a new package.
10#
11#
12# If you are adding a new package please follow the next steps:
13# How to add a new package:
14# - If a user wants to add a new package all that has to be done is:
15# Modify the python3-manifest.json file, and add the required file(s) to the FILES list,
16# fill up the SUMMARY section as well, the script should handle all the rest.
17#
18# Real example:
19# We want to add a web browser package, including the file webbrowser.py
20# which at the moment is on python3-misc.
21# "webbrowser": {
22# "files": ["${libdir}/python${PYTHON_MAJMIN}/lib-dynload/webbrowser.py"],
23# "rdepends": [],
24# "summary": "Python Web Browser support"}
25#
26# * Note that the rdepends field was left empty
27#
28# We run $ bitbake python3 -c create_manifest and the resulting manifest
29# should be completed after a few seconds, showing something like:
30# "webbrowser": {
31# "files": ["${libdir}/python${PYTHON_MAJMIN}/webbrowser.py"],
32# "rdepends": ["core","fcntl","io","pickle","shell","subprocess"],
33# "summary": "Python Web Browser support"}
34#
35#
36# If you are upgrading Python to a new version please follow the next steps:
37# After each Python upgrade, the create_manifest task should be executed, because we
38# don't control what changes on upstream Python, so, some module dependency
39# might have changed without us realizing it, a certain module can either have
40# more or less dependencies, or could be depending on a new file that was just
41# created on the new release and for obvious reasons we wouldn't have it on our
42# old manifest, all of these issues would cause runtime errors on our system.
43#
44# - Upgrade both the native and target Python packages to a new version
45# - Run the create_manifest task for the target Python package as its shown below:
46#
47# $ bitbake python3 -c create_manifest
48#
49# This will automatically replace your manifest file located under the Python directory
50# with an new one, which contains the new dependencies (if any).
51#
52# Several things could have gone wrong here, I will try to explain a few:
53#
54# a) A new file was introduced on this release, e.g. sha3*.so:
55# The task will check what its needed to import every module, more than one module would
56# would probably depend on sha3*.so, although only one module should contain it.
57#
58# After running the task, the new manifest will have the sha3*.so file on more than one
59# module, you need to manually decide which one of them should get it and delete it from
60# the others, for example sha3*.so should likely be on ${PN}-crypt.
61# Once you have deleted from the others you need to run the create_manifest task again,
62# this will populate the other module's rdepends fields, with ${PN}-crypt and you should be
63# good to go.
64#
65# b) The native package wasn't built correctly and its missing a certain module:
66# As mentioned before, you need to make sure the native package was built with all the modules
67# because it is used as base to build the manifest file, you need to manually check log.do_compile
68# since it won't error out the compile function if its only missing a couple of modules.
69#
70# e.g. missing the _uuid module, log.do_compile would show the following:
71# Python build finished successfully!
72# The necessary bits to build these optional modules were not found:
73# _uuid
74#
75# What will happen here is that the new manifest would not be aware that the _uuid module exists, so
76# not only we won't know of any dependencies to it, but also, the _uuid* files will be packaged on
77# the misc package (which is where any file that doesn't belong anywhere else ends up).
78#
79# This will eventually cause runtime errors on our system if we don't include the misc package on
80# on our image, because the _uuid files will be missing.
81# If we build the _uuid module correctly and run the create_manifest task the _uuid files will be
82# detected correctly along with its dependencies, and we will get a working manifest.
83#
84# This is the reason why it is important to make sure we have a fully working native build,
85# so we can avoid these errors.
86#
87#
88#
89# DO NOT MODIFY THE NEXT LINE!, IT IS USED AS A MARKER FOR THE ACTUAL JSON MANIFEST
90# EOC
1{ 91{
2 "tests": { 92 "tests": {
3 "summary": "Python test suite", 93 "summary": "Python test suite",
@@ -1120,8 +1210,8 @@
1120 "unixadmin" 1210 "unixadmin"
1121 ], 1211 ],
1122 "files": [ 1212 "files": [
1123 "${libdir}/python${PYTHON_MAJMIN}/venv", 1213 "${bindir}/pyvenv*",
1124 "${bindir}/pyvenv*" 1214 "${libdir}/python${PYTHON_MAJMIN}/venv"
1125 ], 1215 ],
1126 "cached": [] 1216 "cached": []
1127 }, 1217 },
diff --git a/meta/recipes-devtools/python/python3_3.5.6.bb b/meta/recipes-devtools/python/python3_3.5.6.bb
index 25c47b6192..430c577785 100644
--- a/meta/recipes-devtools/python/python3_3.5.6.bb
+++ b/meta/recipes-devtools/python/python3_3.5.6.bb
@@ -265,7 +265,11 @@ python(){
265 bb.parse.mark_dependency(d, filename) 265 bb.parse.mark_dependency(d, filename)
266 266
267 with open(filename) as manifest_file: 267 with open(filename) as manifest_file:
268 python_manifest=json.load(manifest_file, object_pairs_hook=collections.OrderedDict) 268 manifest_str = manifest_file.read()
269 json_start = manifest_str.find('# EOC') + 6
270 manifest_file.seek(json_start)
271 manifest_str = manifest_file.read()
272 python_manifest = json.loads(manifest_str, object_pairs_hook=collections.OrderedDict)
269 273
270 include_pycs = d.getVar('INCLUDE_PYCS') 274 include_pycs = d.getVar('INCLUDE_PYCS')
271 275