diff options
Diffstat (limited to 'meta/lib/oe/image.py')
-rw-r--r-- | meta/lib/oe/image.py | 337 |
1 files changed, 337 insertions, 0 deletions
diff --git a/meta/lib/oe/image.py b/meta/lib/oe/image.py new file mode 100644 index 0000000000..c9b9033132 --- /dev/null +++ b/meta/lib/oe/image.py | |||
@@ -0,0 +1,337 @@ | |||
1 | from oe.utils import execute_pre_post_process | ||
2 | import os | ||
3 | import subprocess | ||
4 | import multiprocessing | ||
5 | |||
6 | |||
7 | def generate_image(arg): | ||
8 | (type, subimages, create_img_cmd) = arg | ||
9 | |||
10 | bb.note("Running image creation script for %s: %s ..." % | ||
11 | (type, create_img_cmd)) | ||
12 | |||
13 | try: | ||
14 | subprocess.check_output(create_img_cmd, stderr=subprocess.STDOUT) | ||
15 | except subprocess.CalledProcessError as e: | ||
16 | return("Error: The image creation script '%s' returned %d:\n%s" % | ||
17 | (e.cmd, e.returncode, e.output)) | ||
18 | |||
19 | return None | ||
20 | |||
21 | |||
22 | """ | ||
23 | This class will help compute IMAGE_FSTYPE dependencies and group them in batches | ||
24 | that can be executed in parallel. | ||
25 | |||
26 | The next example is for illustration purposes, highly unlikely to happen in real life. | ||
27 | It's just one of the test cases I used to test the algorithm: | ||
28 | |||
29 | For: | ||
30 | IMAGE_FSTYPES = "i1 i2 i3 i4 i5" | ||
31 | IMAGE_TYPEDEP_i4 = "i2" | ||
32 | IMAGE_TYPEDEP_i5 = "i6 i4" | ||
33 | IMAGE_TYPEDEP_i6 = "i7" | ||
34 | IMAGE_TYPEDEP_i7 = "i2" | ||
35 | |||
36 | We get the following list of batches that can be executed in parallel, having the | ||
37 | dependencies satisfied: | ||
38 | |||
39 | [['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']] | ||
40 | """ | ||
41 | class ImageDepGraph(object): | ||
42 | def __init__(self, d): | ||
43 | self.d = d | ||
44 | self.graph = dict() | ||
45 | self.deps_array = dict() | ||
46 | |||
47 | def _construct_dep_graph(self, image_fstypes): | ||
48 | graph = dict() | ||
49 | |||
50 | def add_node(node): | ||
51 | deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "") | ||
52 | if deps != "": | ||
53 | graph[node] = deps | ||
54 | |||
55 | for dep in deps.split(): | ||
56 | if not dep in graph: | ||
57 | add_node(dep) | ||
58 | else: | ||
59 | graph[node] = "" | ||
60 | |||
61 | for fstype in image_fstypes: | ||
62 | add_node(fstype) | ||
63 | |||
64 | return graph | ||
65 | |||
66 | def _clean_graph(self): | ||
67 | # Live and VMDK images will be processed via inheriting | ||
68 | # bbclass and does not get processed here. Remove them from the fstypes | ||
69 | # graph. Their dependencies are already added, so no worries here. | ||
70 | remove_list = (self.d.getVar('IMAGE_TYPES_MASKED', True) or "").split() | ||
71 | |||
72 | for item in remove_list: | ||
73 | self.graph.pop(item, None) | ||
74 | |||
75 | def _compute_dependencies(self): | ||
76 | """ | ||
77 | returns dict object of nodes with [no_of_depends_on, no_of_depended_by] | ||
78 | for each node | ||
79 | """ | ||
80 | deps_array = dict() | ||
81 | for node in self.graph: | ||
82 | deps_array[node] = [0, 0] | ||
83 | |||
84 | for node in self.graph: | ||
85 | deps = self.graph[node].split() | ||
86 | deps_array[node][0] += len(deps) | ||
87 | for dep in deps: | ||
88 | deps_array[dep][1] += 1 | ||
89 | |||
90 | return deps_array | ||
91 | |||
92 | def _sort_graph(self): | ||
93 | sorted_list = [] | ||
94 | group = [] | ||
95 | for node in self.graph: | ||
96 | if node not in self.deps_array: | ||
97 | continue | ||
98 | |||
99 | depends_on = self.deps_array[node][0] | ||
100 | |||
101 | if depends_on == 0: | ||
102 | group.append(node) | ||
103 | |||
104 | if len(group) == 0 and len(self.deps_array) != 0: | ||
105 | bb.fatal("possible fstype circular dependency...") | ||
106 | |||
107 | sorted_list.append(group) | ||
108 | |||
109 | # remove added nodes from deps_array | ||
110 | for item in group: | ||
111 | for node in self.graph: | ||
112 | if item in self.graph[node]: | ||
113 | self.deps_array[node][0] -= 1 | ||
114 | |||
115 | self.deps_array.pop(item, None) | ||
116 | |||
117 | if len(self.deps_array): | ||
118 | # recursive call, to find the next group | ||
119 | sorted_list += self._sort_graph() | ||
120 | |||
121 | return sorted_list | ||
122 | |||
123 | def group_fstypes(self, image_fstypes): | ||
124 | self.graph = self._construct_dep_graph(image_fstypes) | ||
125 | |||
126 | self._clean_graph() | ||
127 | |||
128 | self.deps_array = self._compute_dependencies() | ||
129 | |||
130 | alltypes = [node for node in self.graph] | ||
131 | |||
132 | return (alltypes, self._sort_graph()) | ||
133 | |||
134 | |||
135 | class Image(ImageDepGraph): | ||
136 | def __init__(self, d): | ||
137 | self.d = d | ||
138 | |||
139 | super(Image, self).__init__(d) | ||
140 | |||
141 | def _get_rootfs_size(self): | ||
142 | """compute the rootfs size""" | ||
143 | rootfs_alignment = int(self.d.getVar('IMAGE_ROOTFS_ALIGNMENT', True)) | ||
144 | overhead_factor = float(self.d.getVar('IMAGE_OVERHEAD_FACTOR', True)) | ||
145 | rootfs_req_size = int(self.d.getVar('IMAGE_ROOTFS_SIZE', True)) | ||
146 | rootfs_extra_space = eval(self.d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True)) | ||
147 | |||
148 | output = subprocess.check_output(['du', '-ks', | ||
149 | self.d.getVar('IMAGE_ROOTFS', True)]) | ||
150 | size_kb = int(output.split()[0]) | ||
151 | base_size = size_kb * overhead_factor | ||
152 | base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \ | ||
153 | rootfs_extra_space | ||
154 | |||
155 | if base_size != int(base_size): | ||
156 | base_size = int(base_size + 1) | ||
157 | |||
158 | base_size += rootfs_alignment - 1 | ||
159 | base_size -= base_size % rootfs_alignment | ||
160 | |||
161 | return base_size | ||
162 | |||
163 | def _create_symlinks(self, subimages): | ||
164 | """create symlinks to the newly created image""" | ||
165 | deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) | ||
166 | img_name = self.d.getVar('IMAGE_NAME', True) | ||
167 | link_name = self.d.getVar('IMAGE_LINK_NAME', True) | ||
168 | manifest_name = self.d.getVar('IMAGE_MANIFEST', True) | ||
169 | |||
170 | os.chdir(deploy_dir) | ||
171 | |||
172 | if link_name is not None: | ||
173 | for type in subimages: | ||
174 | if os.path.exists(img_name + ".rootfs." + type): | ||
175 | dst = link_name + "." + type | ||
176 | src = img_name + ".rootfs." + type | ||
177 | bb.note("Creating symlink: %s -> %s" % (dst, src)) | ||
178 | os.symlink(src, dst) | ||
179 | |||
180 | if manifest_name is not None and \ | ||
181 | os.path.exists(manifest_name) and \ | ||
182 | not os.path.exists(link_name + ".manifest"): | ||
183 | os.symlink(os.path.basename(manifest_name), | ||
184 | link_name + ".manifest") | ||
185 | |||
186 | def _remove_old_symlinks(self): | ||
187 | """remove the symlinks to old binaries""" | ||
188 | |||
189 | if self.d.getVar('IMAGE_LINK_NAME', True): | ||
190 | deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True) | ||
191 | for img in os.listdir(deploy_dir): | ||
192 | if img.find(self.d.getVar('IMAGE_LINK_NAME', True)) == 0: | ||
193 | img = os.path.join(deploy_dir, img) | ||
194 | if os.path.islink(img): | ||
195 | if self.d.getVar('RM_OLD_IMAGE', True) == "1" and \ | ||
196 | os.path.exists(os.path.realpath(img)): | ||
197 | os.remove(os.path.realpath(img)) | ||
198 | |||
199 | os.remove(img) | ||
200 | |||
201 | """ | ||
202 | This function will just filter out the compressed image types from the | ||
203 | fstype groups returning a (filtered_fstype_groups, cimages) tuple. | ||
204 | """ | ||
205 | def _filter_out_commpressed(self, fstype_groups): | ||
206 | ctypes = self.d.getVar('COMPRESSIONTYPES', True).split() | ||
207 | cimages = {} | ||
208 | |||
209 | filtered_groups = [] | ||
210 | for group in fstype_groups: | ||
211 | filtered_group = [] | ||
212 | for type in group: | ||
213 | basetype = None | ||
214 | for ctype in ctypes: | ||
215 | if type.endswith("." + ctype): | ||
216 | basetype = type[:-len("." + ctype)] | ||
217 | if basetype not in filtered_group: | ||
218 | filtered_group.append(basetype) | ||
219 | if basetype not in cimages: | ||
220 | cimages[basetype] = [] | ||
221 | if ctype not in cimages[basetype]: | ||
222 | cimages[basetype].append(ctype) | ||
223 | break | ||
224 | if not basetype and type not in filtered_group: | ||
225 | filtered_group.append(type) | ||
226 | |||
227 | filtered_groups.append(filtered_group) | ||
228 | |||
229 | return (filtered_groups, cimages) | ||
230 | |||
231 | def _get_image_types(self): | ||
232 | """returns a (types, cimages) tuple""" | ||
233 | |||
234 | alltypes, fstype_groups = self.group_fstypes(self.d.getVar('IMAGE_FSTYPES', True).split()) | ||
235 | |||
236 | filtered_groups, cimages = self._filter_out_commpressed(fstype_groups) | ||
237 | |||
238 | return (alltypes, filtered_groups, cimages) | ||
239 | |||
240 | def _write_script(self, type, cmds): | ||
241 | tempdir = self.d.getVar('T', True) | ||
242 | script_name = os.path.join(tempdir, "create_image." + type) | ||
243 | |||
244 | self.d.setVar('img_creation_func', '\n'.join(cmds)) | ||
245 | self.d.setVarFlag('img_creation_func', 'func', 1) | ||
246 | self.d.setVarFlag('img_creation_func', 'fakeroot', 1) | ||
247 | |||
248 | with open(script_name, "w+") as script: | ||
249 | script.write("%s" % bb.build.shell_trap_code()) | ||
250 | script.write("export ROOTFS_SIZE=%d\n" % self._get_rootfs_size()) | ||
251 | bb.data.emit_func('img_creation_func', script, self.d) | ||
252 | script.write("img_creation_func\n") | ||
253 | |||
254 | os.chmod(script_name, 0775) | ||
255 | |||
256 | return script_name | ||
257 | |||
258 | def _get_imagecmds(self): | ||
259 | old_overrides = self.d.getVar('OVERRIDES', 0) | ||
260 | |||
261 | alltypes, fstype_groups, cimages = self._get_image_types() | ||
262 | |||
263 | image_cmd_groups = [] | ||
264 | |||
265 | bb.note("The image creation groups are: %s" % str(fstype_groups)) | ||
266 | for fstype_group in fstype_groups: | ||
267 | image_cmds = [] | ||
268 | for type in fstype_group: | ||
269 | cmds = [] | ||
270 | subimages = [] | ||
271 | |||
272 | localdata = bb.data.createCopy(self.d) | ||
273 | localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides)) | ||
274 | bb.data.update_data(localdata) | ||
275 | localdata.setVar('type', type) | ||
276 | |||
277 | cmds.append("\t" + localdata.getVar("IMAGE_CMD", True)) | ||
278 | cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}")) | ||
279 | |||
280 | if type in cimages: | ||
281 | for ctype in cimages[type]: | ||
282 | cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True)) | ||
283 | subimages.append(type + "." + ctype) | ||
284 | |||
285 | if type not in alltypes: | ||
286 | cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}")) | ||
287 | else: | ||
288 | subimages.append(type) | ||
289 | |||
290 | script_name = self._write_script(type, cmds) | ||
291 | |||
292 | image_cmds.append((type, subimages, script_name)) | ||
293 | |||
294 | image_cmd_groups.append(image_cmds) | ||
295 | |||
296 | return image_cmd_groups | ||
297 | |||
298 | def create(self): | ||
299 | bb.note("###### Generate images #######") | ||
300 | pre_process_cmds = self.d.getVar("IMAGE_PREPROCESS_COMMAND", True) | ||
301 | post_process_cmds = self.d.getVar("IMAGE_POSTPROCESS_COMMAND", True) | ||
302 | |||
303 | execute_pre_post_process(self.d, pre_process_cmds) | ||
304 | |||
305 | self._remove_old_symlinks() | ||
306 | |||
307 | image_cmd_groups = self._get_imagecmds() | ||
308 | |||
309 | for image_cmds in image_cmd_groups: | ||
310 | # create the images in parallel | ||
311 | nproc = multiprocessing.cpu_count() | ||
312 | pool = bb.utils.multiprocessingpool(nproc) | ||
313 | results = list(pool.imap(generate_image, image_cmds)) | ||
314 | pool.close() | ||
315 | pool.join() | ||
316 | |||
317 | for result in results: | ||
318 | if result is not None: | ||
319 | bb.fatal(result) | ||
320 | |||
321 | for image_type, subimages, script in image_cmds: | ||
322 | bb.note("Creating symlinks for %s image ..." % image_type) | ||
323 | self._create_symlinks(subimages) | ||
324 | |||
325 | execute_pre_post_process(self.d, post_process_cmds) | ||
326 | |||
327 | |||
328 | def create_image(d): | ||
329 | Image(d).create() | ||
330 | |||
331 | if __name__ == "__main__": | ||
332 | """ | ||
333 | Image creation can be called independent from bitbake environment. | ||
334 | """ | ||
335 | """ | ||
336 | TBD | ||
337 | """ | ||