diff options
Diffstat (limited to 'meta/recipes-devtools/python/python/create_manifest2.py')
-rw-r--r-- | meta/recipes-devtools/python/python/create_manifest2.py | 277 |
1 files changed, 277 insertions, 0 deletions
diff --git a/meta/recipes-devtools/python/python/create_manifest2.py b/meta/recipes-devtools/python/python/create_manifest2.py new file mode 100644 index 0000000000..bd64ec3ee7 --- /dev/null +++ b/meta/recipes-devtools/python/python/create_manifest2.py | |||
@@ -0,0 +1,277 @@ | |||
1 | # This script is used as a bitbake task to create a new python manifest | ||
2 | # $ bitbake python -c create_manifest | ||
3 | # | ||
4 | # Our goal is to keep python-core as small as posible and add other python | ||
5 | # packages only when the user needs them, hence why we split upstream python | ||
6 | # into several packages. | ||
7 | # | ||
8 | # In a very simplistic way what this does is: | ||
9 | # Launch python and see specifically what is required for it to run at a minimum | ||
10 | # | ||
11 | # Go through the python-manifest file and launch a separate task for every single | ||
12 | # one of the files on each package, this task will check what was required for that | ||
13 | # specific module to run, these modules will be called dependencies. | ||
14 | # The output of such task will be a list of the modules or dependencies that were | ||
15 | # found for that file. | ||
16 | # | ||
17 | # Such output will be parsed by this script, we will look for each dependency on the | ||
18 | # manifest and if we find that another package already includes it, then we will add | ||
19 | # that package as an RDEPENDS to the package we are currently checking; in case we dont | ||
20 | # find the current dependency on any other package we will add it to the current package | ||
21 | # as part of FILES. | ||
22 | # | ||
23 | # | ||
24 | # This way we will create a new manifest from the data structure that was built during | ||
25 | # this process, ont this new manifest each package will contain specifically only | ||
26 | # what it needs to run. | ||
27 | # | ||
28 | # There are some caveats which we try to deal with, such as repeated files on different | ||
29 | # packages, packages that include folders, wildcards, and special packages. | ||
30 | # Its also important to note that this method only works for python files, and shared | ||
31 | # libraries. Static libraries, header files and binaries need to be dealt with manually. | ||
32 | # | ||
33 | # Author: Alejandro Enedino Hernandez Samaniego "aehs29" <aehs29@gmail.com> | ||
34 | |||
35 | |||
36 | import sys | ||
37 | import subprocess | ||
38 | import json | ||
39 | import os | ||
40 | |||
41 | # Hack to get native python search path (for folders), not fond of it but it works for now | ||
42 | pivot='recipe-sysroot-native' | ||
43 | for p in sys.path: | ||
44 | if pivot in p: | ||
45 | nativelibfolder=p[:p.find(pivot)+len(pivot)] | ||
46 | |||
47 | # Empty dict to hold the whole manifest | ||
48 | new_manifest = {} | ||
49 | |||
50 | # Check for repeated files, folders and wildcards | ||
51 | allfiles=[] | ||
52 | repeated=[] | ||
53 | wildcards=[] | ||
54 | |||
55 | hasfolders=[] | ||
56 | allfolders=[] | ||
57 | |||
58 | def isFolder(value): | ||
59 | if os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib64')) or os.path.isdir(value.replace('${libdir}',nativelibfolder+'/usr/lib32')): | ||
60 | return True | ||
61 | else: | ||
62 | return False | ||
63 | |||
64 | # Read existing JSON manifest | ||
65 | with open('python2-manifest.json') as manifest: | ||
66 | old_manifest=json.load(manifest) | ||
67 | |||
68 | |||
69 | # First pass to get core-package functionality, because we base everything on the fact that core is actually working | ||
70 | # Not exactly the same so it should not be a function | ||
71 | print ("Getting dependencies for core package:") | ||
72 | |||
73 | # Special call to check for core package | ||
74 | output = subprocess.check_output([sys.executable, 'get_module_deps2.py', 'python-core-package']) | ||
75 | for item in output.split(): | ||
76 | # We append it so it doesnt hurt what we currently have: | ||
77 | if item not in old_manifest['core']['files']: | ||
78 | # We use the same data structure since its the one which will be used to check | ||
79 | # dependencies for other packages | ||
80 | old_manifest['core']['files'].append(item) | ||
81 | |||
82 | for value in old_manifest['core']['files']: | ||
83 | # Ignore folders, since we don't import those, difficult to handle multilib | ||
84 | if isFolder(value): | ||
85 | # Pass it directly | ||
86 | if value not in old_manifest['core']['files']: | ||
87 | old_manifest['core']['files'].append(value) | ||
88 | # Ignore binaries, since we don't import those, assume it was added correctly (manually) | ||
89 | if '${bindir}' in value: | ||
90 | # Pass it directly | ||
91 | if value not in old_manifest['core']['files']: | ||
92 | old_manifest['core']['files'].append(value) | ||
93 | continue | ||
94 | # Ignore empty values | ||
95 | if value == '': | ||
96 | continue | ||
97 | if '${includedir}' in value: | ||
98 | if value not in old_manifest['core']['files']: | ||
99 | old_manifest['core']['files'].append(value) | ||
100 | continue | ||
101 | # Get module name , shouldnt be affected by libdir/bindir | ||
102 | value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0] | ||
103 | |||
104 | |||
105 | # Launch separate task for each module for deterministic behavior | ||
106 | # Each module will only import what is necessary for it to work in specific | ||
107 | print ('Getting dependencies for module: %s' % value) | ||
108 | output = subprocess.check_output([sys.executable, 'get_module_deps2.py', '%s' % value]) | ||
109 | for item in output.split(): | ||
110 | # We append it so it doesnt hurt what we currently have: | ||
111 | if item not in old_manifest['core']['files']: | ||
112 | old_manifest['core']['files'].append(item) | ||
113 | |||
114 | # We check which packages include folders | ||
115 | for key in old_manifest: | ||
116 | for value in old_manifest[key]['files']: | ||
117 | # Ignore folders, since we don't import those, difficult to handle multilib | ||
118 | if isFolder(value): | ||
119 | print ('%s is a folder' % value) | ||
120 | if key not in hasfolders: | ||
121 | hasfolders.append(key) | ||
122 | if value not in allfolders: | ||
123 | allfolders.append(value) | ||
124 | |||
125 | for key in old_manifest: | ||
126 | # Use an empty dict as data structure to hold data for each package and fill it up | ||
127 | new_manifest[key]={} | ||
128 | new_manifest[key]['files']=[] | ||
129 | new_manifest[key]['rdepends']=[] | ||
130 | # All packages should depend on core | ||
131 | if key != 'core': | ||
132 | new_manifest[key]['rdepends'].append('core') | ||
133 | new_manifest[key]['summary']=old_manifest[key]['summary'] | ||
134 | |||
135 | # Handle special cases, we assume that when they were manually added | ||
136 | # to the manifest we knew what we were doing. | ||
137 | print ('Handling package %s' % key) | ||
138 | special_packages=['misc', 'modules', 'dev'] | ||
139 | if key in special_packages or 'staticdev' in key: | ||
140 | print('Passing %s package directly' % key) | ||
141 | new_manifest[key]=old_manifest[key] | ||
142 | continue | ||
143 | |||
144 | for value in old_manifest[key]['files']: | ||
145 | # We already handled core on the first pass | ||
146 | if key == 'core': | ||
147 | new_manifest[key]['files'].append(value) | ||
148 | continue | ||
149 | # Ignore folders, since we don't import those, difficult to handle multilib | ||
150 | if isFolder(value): | ||
151 | # Pass folders directly | ||
152 | new_manifest[key]['files'].append(value) | ||
153 | # Ignore binaries, since we don't import those | ||
154 | if '${bindir}' in value: | ||
155 | # Pass it directly to the new manifest data structure | ||
156 | if value not in new_manifest[key]['files']: | ||
157 | new_manifest[key]['files'].append(value) | ||
158 | continue | ||
159 | # Ignore empty values | ||
160 | if value == '': | ||
161 | continue | ||
162 | if '${includedir}' in value: | ||
163 | if value not in new_manifest[key]['files']: | ||
164 | new_manifest[key]['files'].append(value) | ||
165 | continue | ||
166 | # Get module name , shouldnt be affected by libdir/bindir | ||
167 | value = os.path.splitext(os.path.basename(os.path.normpath(value)))[0] | ||
168 | |||
169 | # Launch separate task for each module for deterministic behavior | ||
170 | # Each module will only import what is necessary for it to work in specific | ||
171 | print ('Getting dependencies for module: %s' % value) | ||
172 | output = subprocess.check_output([sys.executable, 'get_module_deps2.py', '%s' % value]) | ||
173 | |||
174 | # We can print dependencies for debugging purposes | ||
175 | #print (output) | ||
176 | # Output will have all dependencies | ||
177 | for item in output.split(): | ||
178 | |||
179 | # Warning: This first part is ugly | ||
180 | # One of the dependencies that was found, could be inside of one of the folders included by another package | ||
181 | # We need to check if this happens so we can add the package containing the folder as an RDEPENDS | ||
182 | # e.g. Folder encodings contained in codecs | ||
183 | # This would be solved if no packages included any folders | ||
184 | |||
185 | # This can be done in two ways: | ||
186 | # 1 - We assume that if we take out the filename from the path we would get | ||
187 | # the folder string, then we would check if folder string is in the list of folders | ||
188 | # This would not work if a package contains a folder which contains another folder | ||
189 | # e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2 | ||
190 | # folder_string would not match any value contained in the list of folders | ||
191 | # | ||
192 | # 2 - We do it the other way around, checking if the folder is contained in the path | ||
193 | # e.g. path/folder1/folder2/filename folder_string= path/folder1/folder2 | ||
194 | # is folder_string inside path/folder1/folder2/filename?, | ||
195 | # Yes, it works, but we waste a couple of milliseconds. | ||
196 | |||
197 | inFolders=False | ||
198 | for folder in allfolders: | ||
199 | if folder in item: | ||
200 | inFolders = True # Did we find a folder? | ||
201 | folderFound = False # Second flag to break inner for | ||
202 | # Loop only through packages which contain folders | ||
203 | for keyfolder in hasfolders: | ||
204 | if (folderFound == False): | ||
205 | #print("Checking folder %s on package %s" % (item,keyfolder)) | ||
206 | for file_folder in old_manifest[keyfolder]['files']: | ||
207 | if file_folder==folder: | ||
208 | print ('%s found in %s' % (folder, keyfolder)) | ||
209 | folderFound = True | ||
210 | if keyfolder not in new_manifest[key]['rdepends'] and keyfolder != key: | ||
211 | new_manifest[key]['rdepends'].append(keyfolder) | ||
212 | else: | ||
213 | break | ||
214 | |||
215 | # A folder was found so we're done with this item, we can go on | ||
216 | if inFolders: | ||
217 | continue | ||
218 | |||
219 | # We might already have it on the dictionary since it could depend on a (previously checked) module | ||
220 | if item not in new_manifest[key]['files']: | ||
221 | # Handle core as a special package, we already did it so we pass it to NEW data structure directly | ||
222 | if key=='core': | ||
223 | print('Adding %s to %s FILES' % (item, key)) | ||
224 | if item.endswith('*'): | ||
225 | wildcards.append(item) | ||
226 | new_manifest[key]['files'].append(item) | ||
227 | |||
228 | # Check for repeated files | ||
229 | if item not in allfiles: | ||
230 | allfiles.append(item) | ||
231 | else: | ||
232 | repeated.append(item) | ||
233 | |||
234 | else: | ||
235 | |||
236 | # Check if this dependency is already contained on another package, so we add it | ||
237 | # as an RDEPENDS, or if its not, it means it should be contained on the current | ||
238 | # package, so we should add it to FILES | ||
239 | for newkey in old_manifest: | ||
240 | # Debug | ||
241 | #print("Checking %s " % item + " in %s" % newkey) | ||
242 | if item in old_manifest[newkey]['files']: | ||
243 | # Since were nesting, we need to check its not the same key | ||
244 | if(newkey!=key): | ||
245 | if newkey not in new_manifest[key]['rdepends']: | ||
246 | # Add it to the new manifest data struct | ||
247 | # Debug | ||
248 | print('Adding %s to %s RDEPENDS, because it contains %s' % (newkey, key, item)) | ||
249 | new_manifest[key]['rdepends'].append(newkey) | ||
250 | break | ||
251 | else: | ||
252 | # Debug | ||
253 | print('Adding %s to %s FILES' % (item, key)) | ||
254 | # Since it wasnt found on another package, its not an RDEP, so add it to FILES for this package | ||
255 | new_manifest[key]['files'].append(item) | ||
256 | if item.endswith('*'): | ||
257 | wildcards.append(item) | ||
258 | if item not in allfiles: | ||
259 | allfiles.append(item) | ||
260 | else: | ||
261 | repeated.append(item) | ||
262 | |||
263 | print ('The following files are repeated (contained in more than one package), please check which package should get it:') | ||
264 | print (repeated) | ||
265 | print('The following files contain wildcards, please check they are necessary') | ||
266 | print(wildcards) | ||
267 | print('The following files contain folders, please check they are necessary') | ||
268 | print(hasfolders) | ||
269 | |||
270 | # Sort it just so it looks nice | ||
271 | for key in new_manifest: | ||
272 | new_manifest[key]['files'].sort() | ||
273 | new_manifest[key]['rdepends'].sort() | ||
274 | |||
275 | # Create the manifest from the data structure that was built | ||
276 | with open('python2-manifest.json.new','w') as outfile: | ||
277 | json.dump(new_manifest,outfile,sort_keys=True, indent=4) | ||