summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe/image.py
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oe/image.py')
-rw-r--r--meta/lib/oe/image.py337
1 files changed, 337 insertions, 0 deletions
diff --git a/meta/lib/oe/image.py b/meta/lib/oe/image.py
new file mode 100644
index 0000000000..c9b9033132
--- /dev/null
+++ b/meta/lib/oe/image.py
@@ -0,0 +1,337 @@
1from oe.utils import execute_pre_post_process
2import os
3import subprocess
4import multiprocessing
5
6
7def generate_image(arg):
8 (type, subimages, create_img_cmd) = arg
9
10 bb.note("Running image creation script for %s: %s ..." %
11 (type, create_img_cmd))
12
13 try:
14 subprocess.check_output(create_img_cmd, stderr=subprocess.STDOUT)
15 except subprocess.CalledProcessError as e:
16 return("Error: The image creation script '%s' returned %d:\n%s" %
17 (e.cmd, e.returncode, e.output))
18
19 return None
20
21
22"""
23This class will help compute IMAGE_FSTYPE dependencies and group them in batches
24that can be executed in parallel.
25
26The next example is for illustration purposes, highly unlikely to happen in real life.
27It's just one of the test cases I used to test the algorithm:
28
29For:
30IMAGE_FSTYPES = "i1 i2 i3 i4 i5"
31IMAGE_TYPEDEP_i4 = "i2"
32IMAGE_TYPEDEP_i5 = "i6 i4"
33IMAGE_TYPEDEP_i6 = "i7"
34IMAGE_TYPEDEP_i7 = "i2"
35
36We get the following list of batches that can be executed in parallel, having the
37dependencies satisfied:
38
39[['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']]
40"""
41class ImageDepGraph(object):
42 def __init__(self, d):
43 self.d = d
44 self.graph = dict()
45 self.deps_array = dict()
46
47 def _construct_dep_graph(self, image_fstypes):
48 graph = dict()
49
50 def add_node(node):
51 deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "")
52 if deps != "":
53 graph[node] = deps
54
55 for dep in deps.split():
56 if not dep in graph:
57 add_node(dep)
58 else:
59 graph[node] = ""
60
61 for fstype in image_fstypes:
62 add_node(fstype)
63
64 return graph
65
66 def _clean_graph(self):
67 # Live and VMDK images will be processed via inheriting
68 # bbclass and does not get processed here. Remove them from the fstypes
69 # graph. Their dependencies are already added, so no worries here.
70 remove_list = (self.d.getVar('IMAGE_TYPES_MASKED', True) or "").split()
71
72 for item in remove_list:
73 self.graph.pop(item, None)
74
75 def _compute_dependencies(self):
76 """
77 returns dict object of nodes with [no_of_depends_on, no_of_depended_by]
78 for each node
79 """
80 deps_array = dict()
81 for node in self.graph:
82 deps_array[node] = [0, 0]
83
84 for node in self.graph:
85 deps = self.graph[node].split()
86 deps_array[node][0] += len(deps)
87 for dep in deps:
88 deps_array[dep][1] += 1
89
90 return deps_array
91
92 def _sort_graph(self):
93 sorted_list = []
94 group = []
95 for node in self.graph:
96 if node not in self.deps_array:
97 continue
98
99 depends_on = self.deps_array[node][0]
100
101 if depends_on == 0:
102 group.append(node)
103
104 if len(group) == 0 and len(self.deps_array) != 0:
105 bb.fatal("possible fstype circular dependency...")
106
107 sorted_list.append(group)
108
109 # remove added nodes from deps_array
110 for item in group:
111 for node in self.graph:
112 if item in self.graph[node]:
113 self.deps_array[node][0] -= 1
114
115 self.deps_array.pop(item, None)
116
117 if len(self.deps_array):
118 # recursive call, to find the next group
119 sorted_list += self._sort_graph()
120
121 return sorted_list
122
123 def group_fstypes(self, image_fstypes):
124 self.graph = self._construct_dep_graph(image_fstypes)
125
126 self._clean_graph()
127
128 self.deps_array = self._compute_dependencies()
129
130 alltypes = [node for node in self.graph]
131
132 return (alltypes, self._sort_graph())
133
134
135class Image(ImageDepGraph):
136 def __init__(self, d):
137 self.d = d
138
139 super(Image, self).__init__(d)
140
141 def _get_rootfs_size(self):
142 """compute the rootfs size"""
143 rootfs_alignment = int(self.d.getVar('IMAGE_ROOTFS_ALIGNMENT', True))
144 overhead_factor = float(self.d.getVar('IMAGE_OVERHEAD_FACTOR', True))
145 rootfs_req_size = int(self.d.getVar('IMAGE_ROOTFS_SIZE', True))
146 rootfs_extra_space = eval(self.d.getVar('IMAGE_ROOTFS_EXTRA_SPACE', True))
147
148 output = subprocess.check_output(['du', '-ks',
149 self.d.getVar('IMAGE_ROOTFS', True)])
150 size_kb = int(output.split()[0])
151 base_size = size_kb * overhead_factor
152 base_size = (base_size, rootfs_req_size)[base_size < rootfs_req_size] + \
153 rootfs_extra_space
154
155 if base_size != int(base_size):
156 base_size = int(base_size + 1)
157
158 base_size += rootfs_alignment - 1
159 base_size -= base_size % rootfs_alignment
160
161 return base_size
162
163 def _create_symlinks(self, subimages):
164 """create symlinks to the newly created image"""
165 deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True)
166 img_name = self.d.getVar('IMAGE_NAME', True)
167 link_name = self.d.getVar('IMAGE_LINK_NAME', True)
168 manifest_name = self.d.getVar('IMAGE_MANIFEST', True)
169
170 os.chdir(deploy_dir)
171
172 if link_name is not None:
173 for type in subimages:
174 if os.path.exists(img_name + ".rootfs." + type):
175 dst = link_name + "." + type
176 src = img_name + ".rootfs." + type
177 bb.note("Creating symlink: %s -> %s" % (dst, src))
178 os.symlink(src, dst)
179
180 if manifest_name is not None and \
181 os.path.exists(manifest_name) and \
182 not os.path.exists(link_name + ".manifest"):
183 os.symlink(os.path.basename(manifest_name),
184 link_name + ".manifest")
185
186 def _remove_old_symlinks(self):
187 """remove the symlinks to old binaries"""
188
189 if self.d.getVar('IMAGE_LINK_NAME', True):
190 deploy_dir = self.d.getVar('DEPLOY_DIR_IMAGE', True)
191 for img in os.listdir(deploy_dir):
192 if img.find(self.d.getVar('IMAGE_LINK_NAME', True)) == 0:
193 img = os.path.join(deploy_dir, img)
194 if os.path.islink(img):
195 if self.d.getVar('RM_OLD_IMAGE', True) == "1" and \
196 os.path.exists(os.path.realpath(img)):
197 os.remove(os.path.realpath(img))
198
199 os.remove(img)
200
201 """
202 This function will just filter out the compressed image types from the
203 fstype groups returning a (filtered_fstype_groups, cimages) tuple.
204 """
205 def _filter_out_commpressed(self, fstype_groups):
206 ctypes = self.d.getVar('COMPRESSIONTYPES', True).split()
207 cimages = {}
208
209 filtered_groups = []
210 for group in fstype_groups:
211 filtered_group = []
212 for type in group:
213 basetype = None
214 for ctype in ctypes:
215 if type.endswith("." + ctype):
216 basetype = type[:-len("." + ctype)]
217 if basetype not in filtered_group:
218 filtered_group.append(basetype)
219 if basetype not in cimages:
220 cimages[basetype] = []
221 if ctype not in cimages[basetype]:
222 cimages[basetype].append(ctype)
223 break
224 if not basetype and type not in filtered_group:
225 filtered_group.append(type)
226
227 filtered_groups.append(filtered_group)
228
229 return (filtered_groups, cimages)
230
231 def _get_image_types(self):
232 """returns a (types, cimages) tuple"""
233
234 alltypes, fstype_groups = self.group_fstypes(self.d.getVar('IMAGE_FSTYPES', True).split())
235
236 filtered_groups, cimages = self._filter_out_commpressed(fstype_groups)
237
238 return (alltypes, filtered_groups, cimages)
239
240 def _write_script(self, type, cmds):
241 tempdir = self.d.getVar('T', True)
242 script_name = os.path.join(tempdir, "create_image." + type)
243
244 self.d.setVar('img_creation_func', '\n'.join(cmds))
245 self.d.setVarFlag('img_creation_func', 'func', 1)
246 self.d.setVarFlag('img_creation_func', 'fakeroot', 1)
247
248 with open(script_name, "w+") as script:
249 script.write("%s" % bb.build.shell_trap_code())
250 script.write("export ROOTFS_SIZE=%d\n" % self._get_rootfs_size())
251 bb.data.emit_func('img_creation_func', script, self.d)
252 script.write("img_creation_func\n")
253
254 os.chmod(script_name, 0775)
255
256 return script_name
257
258 def _get_imagecmds(self):
259 old_overrides = self.d.getVar('OVERRIDES', 0)
260
261 alltypes, fstype_groups, cimages = self._get_image_types()
262
263 image_cmd_groups = []
264
265 bb.note("The image creation groups are: %s" % str(fstype_groups))
266 for fstype_group in fstype_groups:
267 image_cmds = []
268 for type in fstype_group:
269 cmds = []
270 subimages = []
271
272 localdata = bb.data.createCopy(self.d)
273 localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides))
274 bb.data.update_data(localdata)
275 localdata.setVar('type', type)
276
277 cmds.append("\t" + localdata.getVar("IMAGE_CMD", True))
278 cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}"))
279
280 if type in cimages:
281 for ctype in cimages[type]:
282 cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True))
283 subimages.append(type + "." + ctype)
284
285 if type not in alltypes:
286 cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}"))
287 else:
288 subimages.append(type)
289
290 script_name = self._write_script(type, cmds)
291
292 image_cmds.append((type, subimages, script_name))
293
294 image_cmd_groups.append(image_cmds)
295
296 return image_cmd_groups
297
298 def create(self):
299 bb.note("###### Generate images #######")
300 pre_process_cmds = self.d.getVar("IMAGE_PREPROCESS_COMMAND", True)
301 post_process_cmds = self.d.getVar("IMAGE_POSTPROCESS_COMMAND", True)
302
303 execute_pre_post_process(self.d, pre_process_cmds)
304
305 self._remove_old_symlinks()
306
307 image_cmd_groups = self._get_imagecmds()
308
309 for image_cmds in image_cmd_groups:
310 # create the images in parallel
311 nproc = multiprocessing.cpu_count()
312 pool = bb.utils.multiprocessingpool(nproc)
313 results = list(pool.imap(generate_image, image_cmds))
314 pool.close()
315 pool.join()
316
317 for result in results:
318 if result is not None:
319 bb.fatal(result)
320
321 for image_type, subimages, script in image_cmds:
322 bb.note("Creating symlinks for %s image ..." % image_type)
323 self._create_symlinks(subimages)
324
325 execute_pre_post_process(self.d, post_process_cmds)
326
327
328def create_image(d):
329 Image(d).create()
330
331if __name__ == "__main__":
332 """
333 Image creation can be called independent from bitbake environment.
334 """
335 """
336 TBD
337 """