summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlejandro Enedino Hernandez Samaniego <alejandro@enedino.org>2021-04-16 18:48:34 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-04-30 14:37:25 +0100
commitc135ee79f6c9c5b571a2b0f5133712e8de134fd2 (patch)
tree79adc4586eb8d5a0a0075dd6ee1c4d0860453251
parentd2f044b1829ed9f617138437268751828e56ce92 (diff)
downloadpoky-c135ee79f6c9c5b571a2b0f5133712e8de134fd2.tar.gz
python3: Improve logging, syntax and update deprecated modules to create_manifest
The imp module has een deprecated by upstream python, drop its usage (imp.get_tag) in favor of sys.implementation.cache_tag. Avoid incorrectly getting dependencies for running script and multiprocessing module. Improve logging behavior of the create_manifest task: - Use indentation. - Logs on temp directory. - Use a proper debug flag. - Standarize syntax. (From OE-Core rev: 003d73d74791e5d7dcdeb4f29fc7b05e35b345ea) Signed-off-by: Alejandro Enedino Hernandez Samaniego <alejandro@enedino.org> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> (cherry picked from commit a3ac339f5b8549a050308ba94c4ef9093f10e303) Signed-off-by: Anuj Mittal <anuj.mittal@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/recipes-devtools/python/python3/create_manifest3.py47
-rw-r--r--meta/recipes-devtools/python/python3/get_module_deps3.py72
2 files changed, 71 insertions, 48 deletions
diff --git a/meta/recipes-devtools/python/python3/create_manifest3.py b/meta/recipes-devtools/python/python3/create_manifest3.py
index 4da02a2991..045240ea0b 100644
--- a/meta/recipes-devtools/python/python3/create_manifest3.py
+++ b/meta/recipes-devtools/python/python3/create_manifest3.py
@@ -36,7 +36,7 @@
36# Tha method to handle cached files does not work when a module includes a folder which 36# Tha method to handle cached files does not work when a module includes a folder which
37# itself contains the pycache folder, gladly this is almost never the case. 37# itself contains the pycache folder, gladly this is almost never the case.
38# 38#
39# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29 at gmail dot com> 39# Author: Alejandro Enedino Hernandez Samaniego <alejandro at enedino dot org>
40 40
41 41
42import sys 42import sys
@@ -45,6 +45,11 @@ import json
45import os 45import os
46import collections 46import collections
47 47
48if '-d' in sys.argv:
49 debugFlag = '-d'
50else:
51 debugFlag = ''
52
48# Get python version from ${PYTHON_MAJMIN} 53# Get python version from ${PYTHON_MAJMIN}
49pyversion = str(sys.argv[1]) 54pyversion = str(sys.argv[1])
50 55
@@ -84,6 +89,12 @@ def prepend_comments(comments, json_manifest):
84 manifest.seek(0, 0) 89 manifest.seek(0, 0)
85 manifest.write(comments + json_contents) 90 manifest.write(comments + json_contents)
86 91
92def print_indent(msg, offset):
93 for l in msg.splitlines():
94 msg = ' ' * offset + l
95 print(msg)
96
97
87# Read existing JSON manifest 98# Read existing JSON manifest
88with open('python3-manifest.json') as manifest: 99with open('python3-manifest.json') as manifest:
89 # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker 100 # The JSON format doesn't allow comments so we hack the call to keep the comments using a marker
@@ -99,7 +110,7 @@ with open('python3-manifest.json') as manifest:
99# Not exactly the same so it should not be a function 110# Not exactly the same so it should not be a function
100# 111#
101 112
102print ('Getting dependencies for package: core') 113print_indent('Getting dependencies for package: core', 0)
103 114
104 115
105# This special call gets the core dependencies and 116# This special call gets the core dependencies and
@@ -109,7 +120,7 @@ print ('Getting dependencies for package: core')
109# on the new core package, they will still find them 120# on the new core package, they will still find them
110# even when checking the old_manifest 121# even when checking the old_manifest
111 122
112output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package']).decode('utf8') 123output = subprocess.check_output([sys.executable, 'get_module_deps3.py', 'python-core-package', '%s' % debugFlag]).decode('utf8')
113for coredep in output.split(): 124for coredep in output.split():
114 coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}') 125 coredep = coredep.replace(pyversion,'${PYTHON_MAJMIN}')
115 if isCached(coredep): 126 if isCached(coredep):
@@ -149,17 +160,16 @@ for filedep in old_manifest['core']['files']:
149 # Get actual module name , shouldnt be affected by libdir/bindir, etc. 160 # Get actual module name , shouldnt be affected by libdir/bindir, etc.
150 pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0] 161 pymodule = os.path.splitext(os.path.basename(os.path.normpath(filedep)))[0]
151 162
152
153 # We now know that were dealing with a python module, so we can import it 163 # We now know that were dealing with a python module, so we can import it
154 # and check what its dependencies are. 164 # and check what its dependencies are.
155 # We launch a separate task for each module for deterministic behavior. 165 # We launch a separate task for each module for deterministic behavior.
156 # Each module will only import what is necessary for it to work in specific. 166 # Each module will only import what is necessary for it to work in specific.
157 # The output of each task will contain each module's dependencies 167 # The output of each task will contain each module's dependencies
158 168
159 print ('Getting dependencies for module: %s' % pymodule) 169 print_indent('Getting dependencies for module: %s' % pymodule, 2)
160 output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') 170 output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8')
161 print ('The following dependencies were found for module %s:\n' % pymodule) 171 print_indent('The following dependencies were found for module %s:\n' % pymodule, 4)
162 print (output) 172 print_indent(output, 6)
163 173
164 174
165 for pymodule_dep in output.split(): 175 for pymodule_dep in output.split():
@@ -178,12 +188,13 @@ for filedep in old_manifest['core']['files']:
178# all others will use this a base. 188# all others will use this a base.
179 189
180 190
191print('\n\nChecking for directories...\n')
181# To improve the script speed, we check which packages contain directories 192# To improve the script speed, we check which packages contain directories
182# since we will be looping through (only) those later. 193# since we will be looping through (only) those later.
183for pypkg in old_manifest: 194for pypkg in old_manifest:
184 for filedep in old_manifest[pypkg]['files']: 195 for filedep in old_manifest[pypkg]['files']:
185 if isFolder(filedep): 196 if isFolder(filedep):
186 print ('%s is a folder' % filedep) 197 print_indent('%s is a directory' % filedep, 2)
187 if pypkg not in hasfolders: 198 if pypkg not in hasfolders:
188 hasfolders.append(pypkg) 199 hasfolders.append(pypkg)
189 if filedep not in allfolders: 200 if filedep not in allfolders:
@@ -221,14 +232,14 @@ for pypkg in old_manifest:
221 232
222 print('\n') 233 print('\n')
223 print('--------------------------') 234 print('--------------------------')
224 print ('Handling package %s' % pypkg) 235 print('Handling package %s' % pypkg)
225 print('--------------------------') 236 print('--------------------------')
226 237
227 # Handle special cases, we assume that when they were manually added 238 # Handle special cases, we assume that when they were manually added
228 # to the manifest we knew what we were doing. 239 # to the manifest we knew what we were doing.
229 special_packages = ['misc', 'modules', 'dev', 'tests'] 240 special_packages = ['misc', 'modules', 'dev', 'tests']
230 if pypkg in special_packages or 'staticdev' in pypkg: 241 if pypkg in special_packages or 'staticdev' in pypkg:
231 print('Passing %s package directly' % pypkg) 242 print_indent('Passing %s package directly' % pypkg, 2)
232 new_manifest[pypkg] = old_manifest[pypkg] 243 new_manifest[pypkg] = old_manifest[pypkg]
233 continue 244 continue
234 245
@@ -259,7 +270,7 @@ for pypkg in old_manifest:
259 270
260 # Get actual module name , shouldnt be affected by libdir/bindir, etc. 271 # Get actual module name , shouldnt be affected by libdir/bindir, etc.
261 # We need to check if the imported module comes from another (e.g. sqlite3.dump) 272 # We need to check if the imported module comes from another (e.g. sqlite3.dump)
262 path,pymodule = os.path.split(filedep) 273 path, pymodule = os.path.split(filedep)
263 path = os.path.basename(path) 274 path = os.path.basename(path)
264 pymodule = os.path.splitext(os.path.basename(pymodule))[0] 275 pymodule = os.path.splitext(os.path.basename(pymodule))[0]
265 276
@@ -279,10 +290,10 @@ for pypkg in old_manifest:
279 # Each module will only import what is necessary for it to work in specific. 290 # Each module will only import what is necessary for it to work in specific.
280 # The output of each task will contain each module's dependencies 291 # The output of each task will contain each module's dependencies
281 292
282 print ('\nGetting dependencies for module: %s' % pymodule) 293 print_indent('\nGetting dependencies for module: %s' % pymodule, 2)
283 output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule]).decode('utf8') 294 output = subprocess.check_output([sys.executable, 'get_module_deps3.py', '%s' % pymodule, '%s' % debugFlag]).decode('utf8')
284 print ('The following dependencies were found for module %s:\n' % pymodule) 295 print_indent('The following dependencies were found for module %s:\n' % pymodule, 4)
285 print (output) 296 print_indent(output, 6)
286 297
287 reportFILES = [] 298 reportFILES = []
288 reportRDEPS = [] 299 reportRDEPS = []
@@ -325,7 +336,7 @@ for pypkg in old_manifest:
325 # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder)) 336 # print('Checking folder %s on package %s' % (pymodule_dep,pypkg_with_folder))
326 for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']: 337 for folder_dep in old_manifest[pypkg_with_folder]['files'] or folder_dep in old_manifest[pypkg_with_folder]['cached']:
327 if folder_dep == folder: 338 if folder_dep == folder:
328 print ('%s folder found in %s' % (folder, pypkg_with_folder)) 339 print ('%s directory found in %s' % (folder, pypkg_with_folder))
329 folderFound = True 340 folderFound = True
330 if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg: 341 if pypkg_with_folder not in new_manifest[pypkg]['rdepends'] and pypkg_with_folder != pypkg:
331 new_manifest[pypkg]['rdepends'].append(pypkg_with_folder) 342 new_manifest[pypkg]['rdepends'].append(pypkg_with_folder)
@@ -424,7 +435,7 @@ prepend_comments(comments,'python3-manifest.json.new')
424 435
425if (repeated): 436if (repeated):
426 error_msg = '\n\nERROR:\n' 437 error_msg = '\n\nERROR:\n'
427 error_msg += 'The following files are repeated (contained in more than one package),\n' 438 error_msg += 'The following files were found in more than one package),\n'
428 error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n' 439 error_msg += 'this is likely to happen when new files are introduced after an upgrade,\n'
429 error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n' 440 error_msg += 'please check which package should get it,\n modify the manifest accordingly and re-run the create_manifest task:\n'
430 error_msg += '\n'.join(repeated) 441 error_msg += '\n'.join(repeated)
diff --git a/meta/recipes-devtools/python/python3/get_module_deps3.py b/meta/recipes-devtools/python/python3/get_module_deps3.py
index 6806f23172..1f4c982aed 100644
--- a/meta/recipes-devtools/python/python3/get_module_deps3.py
+++ b/meta/recipes-devtools/python/python3/get_module_deps3.py
@@ -3,14 +3,18 @@
3# them out, the output of this execution will have all dependencies 3# them out, the output of this execution will have all dependencies
4# for a specific module, which will be parsed an dealt on create_manifest.py 4# for a specific module, which will be parsed an dealt on create_manifest.py
5# 5#
6# Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> 6# Author: Alejandro Enedino Hernandez Samaniego <alejandro at enedino dot org>
7 7
8# We can get a log per module, for all the dependencies that were found, but its messy.
9debug=False
10 8
11import sys 9import sys
12import os 10import os
13 11
12# We can get a log per module, for all the dependencies that were found, but its messy.
13if '-d' in sys.argv:
14 debug = True
15else:
16 debug = False
17
14# We can get a list of the modules which are currently required to run python 18# We can get a list of the modules which are currently required to run python
15# so we run python-core and get its modules, we then import what we need 19# so we run python-core and get its modules, we then import what we need
16# and check what modules are currently running, if we substract them from the 20# and check what modules are currently running, if we substract them from the
@@ -19,13 +23,13 @@ import os
19# We use importlib to achieve this, so we also need to know what modules importlib needs 23# We use importlib to achieve this, so we also need to know what modules importlib needs
20import importlib 24import importlib
21 25
22core_deps=set(sys.modules) 26core_deps = set(sys.modules)
23 27
24def fix_path(dep_path): 28def fix_path(dep_path):
25 import os 29 import os
26 # We DONT want the path on our HOST system 30 # We DONT want the path on our HOST system
27 pivot='recipe-sysroot-native' 31 pivot = 'recipe-sysroot-native'
28 dep_path=dep_path[dep_path.find(pivot)+len(pivot):] 32 dep_path = dep_path[dep_path.find(pivot)+len(pivot):]
29 33
30 if '/usr/bin' in dep_path: 34 if '/usr/bin' in dep_path:
31 dep_path = dep_path.replace('/usr/bin''${bindir}') 35 dep_path = dep_path.replace('/usr/bin''${bindir}')
@@ -46,8 +50,8 @@ def fix_path(dep_path):
46 50
47# Module to import was passed as an argument 51# Module to import was passed as an argument
48current_module = str(sys.argv[1]).rstrip() 52current_module = str(sys.argv[1]).rstrip()
49if(debug==True): 53if debug == True:
50 log = open('log_%s' % current_module,'w') 54 log = open('temp/log_%s' % current_module.strip('.*'),'w')
51 log.write('Module %s generated the following dependencies:\n' % current_module) 55 log.write('Module %s generated the following dependencies:\n' % current_module)
52try: 56try:
53 m = importlib.import_module(current_module) 57 m = importlib.import_module(current_module)
@@ -63,13 +67,13 @@ try:
63 except: 67 except:
64 pass # ignore all import or other exceptions raised during import 68 pass # ignore all import or other exceptions raised during import
65except ImportError as e: 69except ImportError as e:
66 if (debug==True): 70 if debug == True:
67 log.write('Module was not found') 71 log.write('Module was not found\n')
68 pass 72 pass
69 73
70 74
71# Get current module dependencies, dif will contain a list of specific deps for this module 75# Get current module dependencies, dif will contain a list of specific deps for this module
72module_deps=set(sys.modules) 76module_deps = set(sys.modules)
73 77
74# We handle the core package (1st pass on create_manifest.py) as a special case 78# We handle the core package (1st pass on create_manifest.py) as a special case
75if current_module == 'python-core-package': 79if current_module == 'python-core-package':
@@ -81,14 +85,18 @@ else:
81 85
82# Check where each dependency came from 86# Check where each dependency came from
83for item in dif: 87for item in dif:
84 dep_path='' 88 # Main module returns script filename, __main matches mp_main__ as well
89 if 'main__' in item:
90 continue
91
92 dep_path = ''
85 try: 93 try:
86 if (debug==True): 94 if debug == True:
87 log.write('Calling: sys.modules[' + '%s' % item + '].__file__\n') 95 log.write('\nCalling: sys.modules[' + '%s' % item + '].__file__\n')
88 dep_path = sys.modules['%s' % item].__file__ 96 dep_path = sys.modules['%s' % item].__file__
89 except AttributeError as e: 97 except AttributeError as e:
90 # Deals with thread (builtin module) not having __file__ attribute 98 # Deals with thread (builtin module) not having __file__ attribute
91 if debug==True: 99 if debug == True:
92 log.write(item + ' ') 100 log.write(item + ' ')
93 log.write(str(e)) 101 log.write(str(e))
94 log.write('\n') 102 log.write('\n')
@@ -96,11 +104,16 @@ for item in dif:
96 except NameError as e: 104 except NameError as e:
97 # Deals with NameError: name 'dep_path' is not defined 105 # Deals with NameError: name 'dep_path' is not defined
98 # because module is not found (wasn't compiled?), e.g. bddsm 106 # because module is not found (wasn't compiled?), e.g. bddsm
99 if (debug==True): 107 if debug == True:
100 log.write(item+' ') 108 log.write(item+' ')
101 log.write(str(e)) 109 log.write(str(e))
102 pass 110 pass
103 111
112 if dep_path == '':
113 continue
114 if debug == True:
115 log.write('Dependency path found:\n%s\n' % dep_path)
116
104 # Site-customize is a special case since we (OpenEmbedded) put it there manually 117 # Site-customize is a special case since we (OpenEmbedded) put it there manually
105 if 'sitecustomize' in dep_path: 118 if 'sitecustomize' in dep_path:
106 dep_path = '${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py' 119 dep_path = '${libdir}/python${PYTHON_MAJMIN}/sitecustomize.py'
@@ -111,52 +124,51 @@ for item in dif:
111 dep_path = fix_path(dep_path) 124 dep_path = fix_path(dep_path)
112 125
113 import sysconfig 126 import sysconfig
114 soabi=sysconfig.get_config_var('SOABI') 127 soabi = sysconfig.get_config_var('SOABI')
115 # Check if its a shared library and deconstruct it 128 # Check if its a shared library and deconstruct it
116 if soabi in dep_path: 129 if soabi in dep_path:
117 if (debug==True): 130 if debug == True:
118 log.write('Shared library found in %s' % dep_path) 131 log.write('Shared library found in %s\n' % dep_path)
119 dep_path = dep_path.replace(soabi,'*') 132 dep_path = dep_path.replace(soabi,'*')
120 print (dep_path) 133 print (dep_path)
121 continue 134 continue
122 if "_sysconfigdata" in dep_path: 135 if "_sysconfigdata" in dep_path:
123 dep_path = dep_path.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*") 136 dep_path = dep_path.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*")
124 137
125 if (debug==True): 138 if debug == True:
126 log.write(dep_path+'\n') 139 log.write(dep_path+'\n')
127 # Prints out result, which is what will be used by create_manifest 140 # Prints out result, which is what will be used by create_manifest
128 print (dep_path) 141 print (dep_path)
129 142
130 143
131 import imp 144 cpython_tag = sys.implementation.cache_tag
132 cpython_tag = imp.get_tag() 145 cached = ''
133 cached=''
134 # Theres no naive way to find *.pyc files on python3 146 # Theres no naive way to find *.pyc files on python3
135 try: 147 try:
136 if (debug==True): 148 if debug == True:
137 log.write('Calling: sys.modules[' + '%s' % item + '].__cached__\n') 149 log.write('\nCalling: sys.modules[' + '%s' % item + '].__cached__\n')
138 cached = sys.modules['%s' % item].__cached__ 150 cached = sys.modules['%s' % item].__cached__
139 except AttributeError as e: 151 except AttributeError as e:
140 # Deals with thread (builtin module) not having __cached__ attribute 152 # Deals with thread (builtin module) not having __cached__ attribute
141 if debug==True: 153 if debug == True:
142 log.write(item + ' ') 154 log.write(item + ' ')
143 log.write(str(e)) 155 log.write(str(e))
144 log.write('\n') 156 log.write('\n')
145 pass 157 pass
146 except NameError as e: 158 except NameError as e:
147 # Deals with NameError: name 'cached' is not defined 159 # Deals with NameError: name 'cached' is not defined
148 if (debug==True): 160 if debug == True:
149 log.write(item+' ') 161 log.write(item+' ')
150 log.write(str(e)) 162 log.write(str(e))
151 pass 163 pass
152 if cached is not None: 164 if cached is not None:
153 if (debug==True): 165 if debug == True:
154 log.write(cached) 166 log.write(cached + '\n')
155 cached = fix_path(cached) 167 cached = fix_path(cached)
156 cached = cached.replace(cpython_tag,'*') 168 cached = cached.replace(cpython_tag,'*')
157 if "_sysconfigdata" in cached: 169 if "_sysconfigdata" in cached:
158 cached = cached.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*") 170 cached = cached.replace(sysconfig._get_sysconfigdata_name(), "_sysconfigdata*")
159 print (cached) 171 print (cached)
160 172
161if debug==True: 173if debug == True:
162 log.close() 174 log.close()