diff options
author | Alejandro Enedino Hernandez Samaniego <alejandro@enedino.org> | 2021-04-16 18:48:34 -0600 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-04-18 11:37:26 +0100 |
commit | 51b7ecfef378edbceda48ba55f6b87c923240545 (patch) | |
tree | 0f5658b99c62f8b9896a26532f4d74810409dabb /meta/recipes-devtools/python/python3/create_manifest3.py | |
parent | 18007c25bd1820abba336cd40706fa0e3e6f472f (diff) | |
download | poky-51b7ecfef378edbceda48ba55f6b87c923240545.tar.gz |
python3: Improve logging, syntax and update deprecated modules to create_manifest
The imp module has een deprecated by upstream python, drop its usage
(imp.get_tag) in favor of sys.implementation.cache_tag.
Avoid incorrectly getting dependencies for running script and
multiprocessing module.
Improve logging behavior of the create_manifest task:
- Use indentation.
- Logs on temp directory.
- Use a proper debug flag.
- Standarize syntax.
(From OE-Core rev: a3ac339f5b8549a050308ba94c4ef9093f10e303)
Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandro@enedino.org>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/recipes-devtools/python/python3/create_manifest3.py')
-rw-r--r-- | meta/recipes-devtools/python/python3/create_manifest3.py | 47 |
1 files changed, 29 insertions, 18 deletions
diff --git a/meta/recipes-devtools/python/python3/create_manifest3.py b/meta/recipes-devtools/python/python3/create_manifest3.py index 4da02a2991..045240ea0b 100644 --- a/meta/recipes-devtools/python/python3/create_manifest3.py +++ b/meta/recipes-devtools/python/python3/create_manifest3.py | |||
@@ -36,7 +36,7 @@ | |||
36 | # Tha method to handle cached files does not work when a module includes a folder which | 36 | # Tha method to handle cached files does not work when a module includes a folder which |
37 | # itself contains the pycache folder, gladly this is almost never the case. | 37 | # itself contains the pycache folder, gladly this is almost never the case. |
38 | # | 38 | # |
39 | # Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com> | 39 | # Author: Alejandro Enedino Hernandez Samaniego <alejandro at enedino dot org> |
40 | 40 | ||
41 | 41 | ||
42 | import sys | 42 | import sys |
@@ -45,6 +45,11 @@ import json | |||
45 | import os | 45 | import os |
46 | import collections | 46 | import collections |
47 | 47 | ||
48 | if '-d' in sys.argv: | ||
49 | debugFlag = '-d' | ||
50 | else: | ||
51 | debugFlag = '' | ||
52 | |||
48 | # Get python version from ${PYTHON_MAJMIN} | 53 | # Get python version from ${PYTHON_MAJMIN} |
49 | pyversion = str(sys.argv[1]) | 54 | pyversion = str(sys.argv[1]) |
50 | 55 | ||
@@ -84,6 +89,12 @@ def prepend_comments(comments, json_manifest): | |||
84 | manifest.seek(0, 0) | 89 | manifest.seek(0, 0) |
85 | manifest.write(comments + json_contents) | 90 | manifest.write(comments + json_contents) |
86 | 91 | ||
92 | def print_indent(msg, offset): | ||
93 | for l in msg.splitlines(): | ||
94 | msg = ' ' * offset + l | ||
95 | print(msg) | ||
96 | |||
97 | |||
87 | # Read existing JSON manifest | 98 | # Read existing JSON manifest |
88 | with open('python3-manifest.json') as manifest: | 99 | with open('python3-manifest.json') as manifest: |
89 | # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker | 100 | # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker |
@@ -99,7 +110,7 @@ with open('python3-manifest.json') as manifest: | |||
99 | # Not exactly the same so it should not be a function | 110 | # Not exactly the same so it should not be a function |
100 | # | 111 | # |
101 | 112 | ||
102 | print ('Getting dependencies for package: core') | 113 | print_indent('Getting dependencies for package: core', 0) |
103 | 114 | ||
104 | 115 | ||
105 | # This special call gets the core dependencies and | 116 | # This special call gets the core dependencies and |
@@ -109,7 +120,7 @@ print ('Getting dependencies for package: core') | |||
109 | # on the new core package, they will still find them | 120 | # on the new core package, they will still find them |
110 | # even when checking the old_manifest | 121 | # even when checking the old_manifest |
111 | 122 | ||
112 | output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8') | 123 | output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package', '%s' % debugFlag]).decode('utf8') |
113 | for coredep in output.split(): | 124 | for coredep in output.split(): |
114 | coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}') | 125 | coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}') |
115 | if isCached(coredep): | 126 | if isCached(coredep): |
@@ -149,17 +160,16 @@ for filedep in old_manifest['core']['files']: | |||
149 | # Get actual module name , shouldnt be affected by libdir/bindir, etc. | 160 | # Get actual module name , shouldnt be affected by libdir/bindir, etc. |
150 | pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0] | 161 | pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0] |
151 | 162 | ||
152 | |||
153 | # We now know that were dealing with a python module, so we can import it | 163 | # We now know that were dealing with a python module, so we can import it |
154 | # and check what its dependencies are. | 164 | # and check what its dependencies are. |
155 | # We launch a separate task for each module for deterministic behavior. | 165 | # We launch a separate task for each module for deterministic behavior. |
156 | # Each module will only import what is necessary for it to work in specific. | 166 | # Each module will only import what is necessary for it to work in specific. |
157 | # The output of each task will contain each module's dependencies | 167 | # The output of each task will contain each module's dependencies |
158 | 168 | ||
159 | print ('Getting dependencies for module: %s' % pymodule) | 169 | print_indent('Getting dependencies for module: %s' % pymodule, 2) |
160 | output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') | 170 | output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') |
161 | print ('The following dependencies were found for module %s:\n' % pymodule) | 171 | print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) |
162 | print (output) | 172 | print_indent(output, 6) |
163 | 173 | ||
164 | 174 | ||
165 | for pymodule_dep in output.split(): | 175 | for pymodule_dep in output.split(): |
@@ -178,12 +188,13 @@ for filedep in old_manifest['core']['files']: | |||
178 | # all others will use this a base. | 188 | # all others will use this a base. |
179 | 189 | ||
180 | 190 | ||
191 | print('\n\nChecking for directories...\n') | ||
181 | # To improve the script speed, we check which packages contain directories | 192 | # To improve the script speed, we check which packages contain directories |
182 | # since we will be looping through (only) those later. | 193 | # since we will be looping through (only) those later. |
183 | for pypkg in old_manifest: | 194 | for pypkg in old_manifest: |
184 | for filedep in old_manifest[pypkg]['files']: | 195 | for filedep in old_manifest[pypkg]['files']: |
185 | if isFolder(filedep): | 196 | if isFolder(filedep): |
186 | print ('%s is a folder' % filedep) | 197 | print_indent('%s is a directory' % filedep, 2) |
187 | if pypkg not in hasfolders: | 198 | if pypkg not in hasfolders: |
188 | hasfolders.append(pypkg) | 199 | hasfolders.append(pypkg) |
189 | if filedep not in allfolders: | 200 | if filedep not in allfolders: |
@@ -221,14 +232,14 @@ for pypkg in old_manifest: | |||
221 | 232 | ||
222 | print('\n') | 233 | print('\n') |
223 | print('--------------------------') | 234 | print('--------------------------') |
224 | print ('Handling package %s' % pypkg) | 235 | print('Handling package %s' % pypkg) |
225 | print('--------------------------') | 236 | print('--------------------------') |
226 | 237 | ||
227 | # Handle special cases, we assume that when they were manually added | 238 | # Handle special cases, we assume that when they were manually added |
228 | # to the manifest we knew what we were doing. | 239 | # to the manifest we knew what we were doing. |
229 | special_packages = ['misc', 'modules', 'dev', 'tests'] | 240 | special_packages = ['misc', 'modules', 'dev', 'tests'] |
230 | if pypkg in special_packages or 'staticdev' in pypkg: | 241 | if pypkg in special_packages or 'staticdev' in pypkg: |
231 | print('Passing %s package directly' % pypkg) | 242 | print_indent('Passing %s package directly' % pypkg, 2) |
232 | new_manifest[pypkg] = old_manifest[pypkg] | 243 | new_manifest[pypkg] = old_manifest[pypkg] |
233 | continue | 244 | continue |
234 | 245 | ||
@@ -259,7 +270,7 @@ for pypkg in old_manifest: | |||
259 | 270 | ||
260 | # Get actual module name , shouldnt be affected by libdir/bindir, etc. | 271 | # Get actual module name , shouldnt be affected by libdir/bindir, etc. |
261 | # We need to check if the imported module comes from another (e.g. sqlite3.dump) | 272 | # We need to check if the imported module comes from another (e.g. sqlite3.dump) |
262 | path,pymodule = os.path.split(filedep) | 273 | path, pymodule = os.path.split(filedep) |
263 | path = os.path.basename(path) | 274 | path = os.path.basename(path) |
264 | pymodule = os.path.splitext(os.path.basename(pymodule))[0] | 275 | pymodule = os.path.splitext(os.path.basename(pymodule))[0] |
265 | 276 | ||
@@ -279,10 +290,10 @@ for pypkg in old_manifest: | |||
279 | # Each module will only import what is necessary for it to work in specific. | 290 | # Each module will only import what is necessary for it to work in specific. |
280 | # The output of each task will contain each module's dependencies | 291 | # The output of each task will contain each module's dependencies |
281 | 292 | ||
282 | print ('\nGetting dependencies for module: %s' % pymodule) | 293 | print_indent('\nGetting dependencies for module: %s' % pymodule, 2) |
283 | output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') | 294 | output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8') |
284 | print ('The following dependencies were found for module %s:\n' % pymodule) | 295 | print_indent('The following dependencies were found for module %s:\n' % pymodule, 4) |
285 | print (output) | 296 | print_indent(output, 6) |
286 | 297 | ||
287 | reportFILES = [] | 298 | reportFILES = [] |
288 | reportRDEPS = [] | 299 | reportRDEPS = [] |
@@ -325,7 +336,7 @@ for pypkg in old_manifest: | |||
325 | # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder)) | 336 | # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder)) |
326 | for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']: | 337 | for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']: |
327 | if folder_dep == folder: | 338 | if folder_dep == folder: |
328 | print ('%s folder found in %s' % (folder, pypkg_with_folder)) | 339 | print ('%s directory found in %s' % (folder, pypkg_with_folder)) |
329 | folderFound = True | 340 | folderFound = True |
330 | if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg: | 341 | if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg: |
331 | new_manifest[pypkg]['rdepends'].append(pypkg_with_folder) | 342 | new_manifest[pypkg]['rdepends'].append(pypkg_with_folder) |
@@ -424,7 +435,7 @@ prepend_comments(comments,'python3-manifest.json.new') | |||
424 | 435 | ||
425 | if (repeated): | 436 | if (repeated): |
426 | error_msg = '\n\nERROR:\n' | 437 | error_msg = '\n\nERROR:\n' |
427 | error_msg += 'The following files are repeated (contained in more than one package),\n' | 438 | error_msg += 'The following files were found in more than one package),\n' |
428 | error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n' | 439 | error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n' |
429 | error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n' | 440 | error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n' |
430 | error_msg += '\n'.join(repeated) | 441 | error_msg += '\n'.join(repeated) |