summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe
diff options
context:
space:
mode:
authorLaurentiu Palcu <laurentiu.palcu@intel.com>2014-02-18 15:49:44 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-02-20 14:28:10 +0000
commit84210fd82824e5136b75c71bdc5eed0f33b0cdb9 (patch)
treef3de6dd79095e6264b1c3bef45b82871c29cb2f7 /meta/lib/oe
parentf866b2e4b78c91f6c10f1da723505c0c9d99e856 (diff)
downloadpoky-84210fd82824e5136b75c71bdc5eed0f33b0cdb9.tar.gz
lib/oe/image.py: add image dependency mechanism
This commit adds a dependency mechanism to image creation, so that we can split the images creation execution in groups, that can be executed in parallel, having the dependencies satisfied in the same time. The old code didn't need this since everything was serialized. Technically, it adds a dependency graph topological sort class that the main Image class can use to sort out the dependencies. Images that have dependencies have to declare them using the NEW IMAGE_TYPEDEP variable, like in the example below: For: IMAGE_FSTYPES = "i1 i2 i3 i4 i5" IMAGE_TYPEDEP_i4 = "i2" IMAGE_TYPEDEP_i5 = "i6 i4" IMAGE_TYPEDEP_i6 = "i7" IMAGE_TYPEDEP_i7 = "i2" We'll get the following image groups, sorted out by their dependencies: [['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']] The algorithm can probably be optimized but, given the small size of the graphs, it'll do. [YOCTO #5830] (From OE-Core rev: db9dd4b4ef9120baccbccae77d9c31f54a6eb9a1) Signed-off-by: Laurentiu Palcu <laurentiu.palcu@intel.com> Signed-off-by: Saul Wold <sgw@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oe')
-rw-r--r--meta/lib/oe/image.py282
1 files changed, 191 insertions, 91 deletions
diff --git a/meta/lib/oe/image.py b/meta/lib/oe/image.py
index c15296f5c0..488683e42a 100644
--- a/meta/lib/oe/image.py
+++ b/meta/lib/oe/image.py
@@ -19,9 +19,124 @@ def generate_image(arg):
19 return None 19 return None
20 20
21 21
22class Image(object): 22"""
23This class will help compute IMAGE_FSTYPE dependencies and group them in batches
24that can be executed in parallel.
25
26The next example is for illustration purposes, highly unlikely to happen in real life.
27It's just one of the test cases I used to test the algorithm:
28
29For:
30IMAGE_FSTYPES = "i1 i2 i3 i4 i5"
31IMAGE_TYPEDEP_i4 = "i2"
32IMAGE_TYPEDEP_i5 = "i6 i4"
33IMAGE_TYPEDEP_i6 = "i7"
34IMAGE_TYPEDEP_i7 = "i2"
35
36We get the following list of batches that can be executed in parallel, having the
37dependencies satisfied:
38
39[['i1', 'i3', 'i2'], ['i4', 'i7'], ['i6'], ['i5']]
40"""
41class ImageDepGraph(object):
23 def __init__(self, d): 42 def __init__(self, d):
24 self.d = d 43 self.d = d
44 self.graph = dict()
45 self.deps_array = dict()
46
47 def _construct_dep_graph(self, image_fstypes):
48 graph = dict()
49
50 def add_node(node):
51 deps = (self.d.getVar('IMAGE_TYPEDEP_' + node, True) or "")
52 if deps != "":
53 graph[node] = deps
54
55 for dep in deps.split():
56 if not dep in graph:
57 add_node(dep)
58 else:
59 graph[node] = ""
60
61 for fstype in image_fstypes:
62 add_node(fstype)
63
64 return graph
65
66 def _clean_graph(self):
67 # Live and VMDK images will be processed via inheriting
68 # bbclass and does not get processed here. Remove them from the fstypes
69 # graph. Their dependencies are already added, so no worries here.
70 remove_list = (self.d.getVar('IMAGE_TYPES_MASKED', True) or "").split()
71
72 for item in remove_list:
73 self.graph.pop(item, None)
74
75 def _compute_dependencies(self):
76 """
77 returns dict object of nodes with [no_of_depends_on, no_of_depended_by]
78 for each node
79 """
80 deps_array = dict()
81 for node in self.graph:
82 deps_array[node] = [0, 0]
83
84 for node in self.graph:
85 deps = self.graph[node].split()
86 deps_array[node][0] += len(deps)
87 for dep in deps:
88 deps_array[dep][1] += 1
89
90 return deps_array
91
92 def _sort_graph(self):
93 sorted_list = []
94 group = []
95 for node in self.graph:
96 if node not in self.deps_array:
97 continue
98
99 depends_on = self.deps_array[node][0]
100
101 if depends_on == 0:
102 group.append(node)
103
104 if len(group) == 0 and len(self.deps_array) != 0:
105 bb.fatal("possible fstype circular dependency...")
106
107 sorted_list.append(group)
108
109 # remove added nodes from deps_array
110 for item in group:
111 for node in self.graph:
112 if item in self.graph[node]:
113 self.deps_array[node][0] -= 1
114
115 self.deps_array.pop(item, None)
116
117 if len(self.deps_array):
118 # recursive call, to find the next group
119 sorted_list += self._sort_graph()
120
121 return sorted_list
122
123 def group_fstypes(self, image_fstypes):
124 self.graph = self._construct_dep_graph(image_fstypes)
125
126 self._clean_graph()
127
128 self.deps_array = self._compute_dependencies()
129
130 alltypes = [node for node in self.graph]
131
132 return (alltypes, self._sort_graph())
133
134
135class Image(ImageDepGraph):
136 def __init__(self, d):
137 self.d = d
138
139 super(Image, self).__init__(d)
25 140
26 def _get_rootfs_size(self): 141 def _get_rootfs_size(self):
27 """compute the rootfs size""" 142 """compute the rootfs size"""
@@ -82,66 +197,44 @@ class Image(object):
82 197
83 os.remove(img) 198 os.remove(img)
84 199
200 """
201 This function will just filter out the compressed image types from the
202 fstype groups returning a (filtered_fstype_groups, cimages) tuple.
203 """
204 def _filter_out_commpressed(self, fstype_groups):
205 ctypes = self.d.getVar('COMPRESSIONTYPES', True).split()
206 cimages = {}
207
208 filtered_groups = []
209 for group in fstype_groups:
210 filtered_group = []
211 for type in group:
212 basetype = None
213 for ctype in ctypes:
214 if type.endswith("." + ctype):
215 basetype = type[:-len("." + ctype)]
216 if basetype not in filtered_group:
217 filtered_group.append(basetype)
218 if basetype not in cimages:
219 cimages[basetype] = []
220 if ctype not in cimages[basetype]:
221 cimages[basetype].append(ctype)
222 break
223 if not basetype and type not in filtered_group:
224 filtered_group.append(type)
225
226 filtered_groups.append(filtered_group)
227
228 return (filtered_groups, cimages)
229
85 def _get_image_types(self): 230 def _get_image_types(self):
86 """returns a (types, cimages) tuple""" 231 """returns a (types, cimages) tuple"""
87 232
88 alltypes = self.d.getVar('IMAGE_FSTYPES', True).split() 233 alltypes, fstype_groups = self.group_fstypes(self.d.getVar('IMAGE_FSTYPES', True).split())
89 types = []
90 ctypes = self.d.getVar('COMPRESSIONTYPES', True).split()
91 cimages = {}
92 234
93 # Image type b depends on a having been generated first 235 filtered_groups, cimages = self._filter_out_commpressed(fstype_groups)
94 def addtypedepends(a, b):
95 if a in alltypes:
96 alltypes.remove(a)
97 if b not in alltypes:
98 alltypes.append(b)
99 alltypes.append(a)
100
101 # The elf image depends on the cpio.gz image already having
102 # been created, so we add that explicit ordering here.
103 addtypedepends("elf", "cpio.gz")
104
105 # Filter out all the compressed images from alltypes
106 for type in alltypes:
107 basetype = None
108 for ctype in ctypes:
109 if type.endswith("." + ctype):
110 basetype = type[:-len("." + ctype)]
111 if basetype not in types:
112 types.append(basetype)
113 if basetype not in cimages:
114 cimages[basetype] = []
115 if ctype not in cimages[basetype]:
116 cimages[basetype].append(ctype)
117 break
118 if not basetype and type not in types:
119 types.append(type)
120 236
121 # Live and VMDK images will be processed via inheriting 237 return (alltypes, filtered_groups, cimages)
122 # bbclass and does not get processed here.
123 # vmdk depend on live images also depend on ext3 so ensure its present
124 # Note: we need to ensure ext3 is in alltypes, otherwise, subimages may
125 # not contain ext3 and the .rootfs.ext3 file won't be created.
126 if "vmdk" in types:
127 if "ext3" not in types:
128 types.append("ext3")
129 if "ext3" not in alltypes:
130 alltypes.append("ext3")
131 types.remove("vmdk")
132 if "live" in types or "iso" in types or "hddimg" in types:
133 if "ext3" not in types:
134 types.append("ext3")
135 if "ext3" not in alltypes:
136 alltypes.append("ext3")
137 if "live" in types:
138 types.remove("live")
139 if "iso" in types:
140 types.remove("iso")
141 if "hddimg" in types:
142 types.remove("hddimg")
143
144 return (alltypes, types, cimages)
145 238
146 def _write_script(self, type, cmds): 239 def _write_script(self, type, cmds):
147 tempdir = self.d.getVar('T', True) 240 tempdir = self.d.getVar('T', True)
@@ -164,36 +257,42 @@ class Image(object):
164 def _get_imagecmds(self): 257 def _get_imagecmds(self):
165 old_overrides = self.d.getVar('OVERRIDES', 0) 258 old_overrides = self.d.getVar('OVERRIDES', 0)
166 259
167 alltypes, types, cimages = self._get_image_types() 260 alltypes, fstype_groups, cimages = self._get_image_types()
168 261
169 image_cmds = [] 262 image_cmd_groups = []
170 for type in types:
171 cmds = []
172 subimages = []
173 263
174 localdata = bb.data.createCopy(self.d) 264 bb.note("The image creation groups are: %s" % str(fstype_groups))
175 localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides)) 265 for fstype_group in fstype_groups:
176 bb.data.update_data(localdata) 266 image_cmds = []
177 localdata.setVar('type', type) 267 for type in fstype_group:
268 cmds = []
269 subimages = []
178 270
179 cmds.append("\t" + localdata.getVar("IMAGE_CMD", True)) 271 localdata = bb.data.createCopy(self.d)
180 cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}")) 272 localdata.setVar('OVERRIDES', '%s:%s' % (type, old_overrides))
273 bb.data.update_data(localdata)
274 localdata.setVar('type', type)
181 275
182 if type in cimages: 276 cmds.append("\t" + localdata.getVar("IMAGE_CMD", True))
183 for ctype in cimages[type]: 277 cmds.append(localdata.expand("\tcd ${DEPLOY_DIR_IMAGE}"))
184 cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True))
185 subimages.append(type + "." + ctype)
186 278
187 if type not in alltypes: 279 if type in cimages:
188 cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}")) 280 for ctype in cimages[type]:
189 else: 281 cmds.append("\t" + localdata.getVar("COMPRESS_CMD_" + ctype, True))
190 subimages.append(type) 282 subimages.append(type + "." + ctype)
283
284 if type not in alltypes:
285 cmds.append(localdata.expand("\trm ${IMAGE_NAME}.rootfs.${type}"))
286 else:
287 subimages.append(type)
288
289 script_name = self._write_script(type, cmds)
191 290
192 script_name = self._write_script(type, cmds) 291 image_cmds.append((type, subimages, script_name))
193 292
194 image_cmds.append((type, subimages, script_name)) 293 image_cmd_groups.append(image_cmds)
195 294
196 return image_cmds 295 return image_cmd_groups
197 296
198 def create(self): 297 def create(self):
199 bb.note("###### Generate images #######") 298 bb.note("###### Generate images #######")
@@ -204,22 +303,23 @@ class Image(object):
204 303
205 self._remove_old_symlinks() 304 self._remove_old_symlinks()
206 305
207 image_cmds = self._get_imagecmds() 306 image_cmd_groups = self._get_imagecmds()
208 307
209 # create the images in parallel 308 for image_cmds in image_cmd_groups:
210 nproc = multiprocessing.cpu_count() 309 # create the images in parallel
211 pool = bb.utils.multiprocessingpool(nproc) 310 nproc = multiprocessing.cpu_count()
212 results = list(pool.imap(generate_image, image_cmds)) 311 pool = bb.utils.multiprocessingpool(nproc)
213 pool.close() 312 results = list(pool.imap(generate_image, image_cmds))
214 pool.join() 313 pool.close()
314 pool.join()
215 315
216 for result in results: 316 for result in results:
217 if result is not None: 317 if result is not None:
218 bb.fatal(result) 318 bb.fatal(result)
219 319
220 for image_type, subimages, script in image_cmds: 320 for image_type, subimages, script in image_cmds:
221 bb.note("Creating symlinks for %s image ..." % image_type) 321 bb.note("Creating symlinks for %s image ..." % image_type)
222 self._create_symlinks(subimages) 322 self._create_symlinks(subimages)
223 323
224 execute_pre_post_process(self.d, post_process_cmds) 324 execute_pre_post_process(self.d, post_process_cmds)
225 325