diff options
author | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
---|---|---|
committer | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
commit | 972dcfcdbfe75dcfeb777150c136576cf1a71e99 (patch) | |
tree | 97a61cd7e293d7ae9d56ef7ed0f81253365bb026 /meta/classes/toaster.bbclass | |
download | poky-972dcfcdbfe75dcfeb777150c136576cf1a71e99.tar.gz |
initial commit for Enea Linux 5.0 arm
Signed-off-by: Tudor Florea <tudor.florea@enea.com>
Diffstat (limited to 'meta/classes/toaster.bbclass')
-rw-r--r-- | meta/classes/toaster.bbclass | 343 |
1 files changed, 343 insertions, 0 deletions
diff --git a/meta/classes/toaster.bbclass b/meta/classes/toaster.bbclass new file mode 100644 index 0000000000..a7dd0aa854 --- /dev/null +++ b/meta/classes/toaster.bbclass | |||
@@ -0,0 +1,343 @@ | |||
1 | # | ||
2 | # Toaster helper class | ||
3 | # | ||
4 | # Copyright (C) 2013 Intel Corporation | ||
5 | # | ||
6 | # Released under the MIT license (see COPYING.MIT) | ||
7 | # | ||
8 | # This bbclass is designed to extract data used by OE-Core during the build process, | ||
9 | # for recording in the Toaster system. | ||
10 | # The data access is synchronous, preserving the build data integrity across | ||
11 | # different builds. | ||
12 | # | ||
13 | # The data is transferred through the event system, using the MetadataEvent objects. | ||
14 | # | ||
15 | # The model is to enable the datadump functions as postfuncs, and have the dump | ||
16 | # executed after the real taskfunc has been executed. This prevents task signature changing | ||
17 | # is toaster is enabled or not. Build performance is not affected if Toaster is not enabled. | ||
18 | # | ||
19 | # To enable, use INHERIT in local.conf: | ||
20 | # | ||
21 | # INHERIT += "toaster" | ||
22 | # | ||
23 | # | ||
24 | # | ||
25 | # | ||
26 | |||
27 | # Find and dump layer info when we got the layers parsed | ||
28 | |||
29 | |||
30 | |||
31 | python toaster_layerinfo_dumpdata() { | ||
32 | import subprocess | ||
33 | |||
34 | def _get_git_branch(layer_path): | ||
35 | branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0] | ||
36 | branch = branch.replace('refs/heads/', '').rstrip() | ||
37 | return branch | ||
38 | |||
39 | def _get_git_revision(layer_path): | ||
40 | revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip() | ||
41 | return revision | ||
42 | |||
43 | def _get_url_map_name(layer_name): | ||
44 | """ Some layers have a different name on openembedded.org site, | ||
45 | this method returns the correct name to use in the URL | ||
46 | """ | ||
47 | |||
48 | url_name = layer_name | ||
49 | url_mapping = {'meta': 'openembedded-core'} | ||
50 | |||
51 | for key in url_mapping.keys(): | ||
52 | if key == layer_name: | ||
53 | url_name = url_mapping[key] | ||
54 | |||
55 | return url_name | ||
56 | |||
57 | def _get_layer_version_information(layer_path): | ||
58 | |||
59 | layer_version_info = {} | ||
60 | layer_version_info['branch'] = _get_git_branch(layer_path) | ||
61 | layer_version_info['commit'] = _get_git_revision(layer_path) | ||
62 | layer_version_info['priority'] = 0 | ||
63 | |||
64 | return layer_version_info | ||
65 | |||
66 | |||
67 | def _get_layer_dict(layer_path): | ||
68 | |||
69 | layer_info = {} | ||
70 | layer_name = layer_path.split('/')[-1] | ||
71 | layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/' | ||
72 | layer_url_name = _get_url_map_name(layer_name) | ||
73 | |||
74 | layer_info['name'] = layer_url_name | ||
75 | layer_info['local_path'] = layer_path | ||
76 | layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name) | ||
77 | layer_info['version'] = _get_layer_version_information(layer_path) | ||
78 | |||
79 | return layer_info | ||
80 | |||
81 | |||
82 | bblayers = e.data.getVar("BBLAYERS", True) | ||
83 | |||
84 | llayerinfo = {} | ||
85 | |||
86 | for layer in { l for l in bblayers.strip().split(" ") if len(l) }: | ||
87 | llayerinfo[layer] = _get_layer_dict(layer) | ||
88 | |||
89 | |||
90 | bb.event.fire(bb.event.MetadataEvent("LayerInfo", llayerinfo), e.data) | ||
91 | } | ||
92 | |||
93 | # Dump package file info data | ||
94 | |||
95 | def _toaster_load_pkgdatafile(dirpath, filepath): | ||
96 | import json | ||
97 | import re | ||
98 | pkgdata = {} | ||
99 | with open(os.path.join(dirpath, filepath), "r") as fin: | ||
100 | for line in fin: | ||
101 | try: | ||
102 | kn, kv = line.strip().split(": ", 1) | ||
103 | m = re.match(r"^PKG_([^A-Z:]*)", kn) | ||
104 | if m: | ||
105 | pkgdata['OPKGN'] = m.group(1) | ||
106 | kn = "_".join([x for x in kn.split("_") if x.isupper()]) | ||
107 | pkgdata[kn] = kv.strip() | ||
108 | if kn == 'FILES_INFO': | ||
109 | pkgdata[kn] = json.loads(kv) | ||
110 | |||
111 | except ValueError: | ||
112 | pass # ignore lines without valid key: value pairs | ||
113 | return pkgdata | ||
114 | |||
115 | |||
116 | python toaster_package_dumpdata() { | ||
117 | """ | ||
118 | Dumps the data created by emit_pkgdata | ||
119 | """ | ||
120 | # replicate variables from the package.bbclass | ||
121 | |||
122 | packages = d.getVar('PACKAGES', True) | ||
123 | pkgdest = d.getVar('PKGDEST', True) | ||
124 | |||
125 | pkgdatadir = d.getVar('PKGDESTWORK', True) | ||
126 | |||
127 | # scan and send data for each package | ||
128 | |||
129 | lpkgdata = {} | ||
130 | for pkg in packages.split(): | ||
131 | |||
132 | lpkgdata = _toaster_load_pkgdatafile(pkgdatadir + "/runtime/", pkg) | ||
133 | |||
134 | # Fire an event containing the pkg data | ||
135 | bb.event.fire(bb.event.MetadataEvent("SinglePackageInfo", lpkgdata), d) | ||
136 | } | ||
137 | |||
138 | # 2. Dump output image files information | ||
139 | |||
140 | python toaster_image_dumpdata() { | ||
141 | """ | ||
142 | Image filename for output images is not standardized. | ||
143 | image_types.bbclass will spell out IMAGE_CMD_xxx variables that actually | ||
144 | have hardcoded ways to create image file names in them. | ||
145 | So we look for files starting with the set name. | ||
146 | """ | ||
147 | |||
148 | deploy_dir_image = d.getVar('DEPLOY_DIR_IMAGE', True); | ||
149 | image_name = d.getVar('IMAGE_NAME', True); | ||
150 | |||
151 | image_info_data = {} | ||
152 | |||
153 | for dirpath, dirnames, filenames in os.walk(deploy_dir_image): | ||
154 | for fn in filenames: | ||
155 | if fn.startswith(image_name): | ||
156 | image_output = os.path.join(dirpath, fn) | ||
157 | image_info_data[image_output] = os.stat(image_output).st_size | ||
158 | |||
159 | bb.event.fire(bb.event.MetadataEvent("ImageFileSize",image_info_data), d) | ||
160 | } | ||
161 | |||
162 | |||
163 | |||
164 | # collect list of buildstats files based on fired events; when the build completes, collect all stats and fire an event with collected data | ||
165 | |||
166 | python toaster_collect_task_stats() { | ||
167 | import bb.build | ||
168 | import bb.event | ||
169 | import bb.data | ||
170 | import bb.utils | ||
171 | import os | ||
172 | |||
173 | if not e.data.getVar('BUILDSTATS_BASE', True): | ||
174 | return # if we don't have buildstats, we cannot collect stats | ||
175 | |||
176 | def _append_read_list(v): | ||
177 | lock = bb.utils.lockfile(e.data.expand("${TOPDIR}/toaster.lock"), False, True) | ||
178 | |||
179 | with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "a") as fout: | ||
180 | bn = get_bn(e) | ||
181 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
182 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
183 | fout.write("%s:%s:%s:%s\n" % (e.taskfile, e.taskname, os.path.join(taskdir, e.task), e.data.expand("${PN}"))) | ||
184 | |||
185 | bb.utils.unlockfile(lock) | ||
186 | |||
187 | def _read_stats(filename): | ||
188 | cpu_usage = 0 | ||
189 | disk_io = 0 | ||
190 | startio = '0' | ||
191 | endio = '0' | ||
192 | started = '0' | ||
193 | ended = '0' | ||
194 | pn = '' | ||
195 | taskname = '' | ||
196 | statinfo = {} | ||
197 | |||
198 | with open(filename, 'r') as task_bs: | ||
199 | for line in task_bs.readlines(): | ||
200 | k,v = line.strip().split(": ", 1) | ||
201 | statinfo[k] = v | ||
202 | |||
203 | if "CPU usage" in statinfo: | ||
204 | cpu_usage = str(statinfo["CPU usage"]).strip('% \n\r') | ||
205 | |||
206 | if "EndTimeIO" in statinfo: | ||
207 | endio = str(statinfo["EndTimeIO"]).strip('% \n\r') | ||
208 | |||
209 | if "StartTimeIO" in statinfo: | ||
210 | startio = str(statinfo["StartTimeIO"]).strip('% \n\r') | ||
211 | |||
212 | if "Started" in statinfo: | ||
213 | started = str(statinfo["Started"]).strip('% \n\r') | ||
214 | |||
215 | if "Ended" in statinfo: | ||
216 | ended = str(statinfo["Ended"]).strip('% \n\r') | ||
217 | |||
218 | disk_io = int(endio) - int(startio) | ||
219 | |||
220 | elapsed_time = float(ended) - float(started) | ||
221 | |||
222 | cpu_usage = float(cpu_usage) | ||
223 | |||
224 | return {'cpu_usage': cpu_usage, 'disk_io': disk_io, 'elapsed_time': elapsed_time} | ||
225 | |||
226 | |||
227 | if isinstance(e, (bb.build.TaskSucceeded, bb.build.TaskFailed)): | ||
228 | _append_read_list(e) | ||
229 | pass | ||
230 | |||
231 | |||
232 | if isinstance(e, bb.event.BuildCompleted) and os.path.exists(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")): | ||
233 | events = [] | ||
234 | with open(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist"), "r") as fin: | ||
235 | for line in fin: | ||
236 | (taskfile, taskname, filename, recipename) = line.strip().split(":") | ||
237 | events.append((taskfile, taskname, _read_stats(filename), recipename)) | ||
238 | bb.event.fire(bb.event.MetadataEvent("BuildStatsList", events), e.data) | ||
239 | os.unlink(os.path.join(e.data.getVar('BUILDSTATS_BASE', True), "toasterstatlist")) | ||
240 | } | ||
241 | |||
242 | # dump relevant build history data as an event when the build is completed | ||
243 | |||
244 | python toaster_buildhistory_dump() { | ||
245 | import re | ||
246 | BUILDHISTORY_DIR = e.data.expand("${TOPDIR}/buildhistory") | ||
247 | BUILDHISTORY_DIR_IMAGE_BASE = e.data.expand("%s/images/${MACHINE_ARCH}/${TCLIBC}/"% BUILDHISTORY_DIR) | ||
248 | pkgdata_dir = e.data.getVar("PKGDATA_DIR", True) | ||
249 | |||
250 | |||
251 | # scan the build targets for this build | ||
252 | images = {} | ||
253 | allpkgs = {} | ||
254 | files = {} | ||
255 | for target in e._pkgs: | ||
256 | installed_img_path = e.data.expand(os.path.join(BUILDHISTORY_DIR_IMAGE_BASE, target)) | ||
257 | if os.path.exists(installed_img_path): | ||
258 | images[target] = {} | ||
259 | files[target] = {} | ||
260 | files[target]['dirs'] = [] | ||
261 | files[target]['syms'] = [] | ||
262 | files[target]['files'] = [] | ||
263 | with open("%s/installed-package-sizes.txt" % installed_img_path, "r") as fin: | ||
264 | for line in fin: | ||
265 | line = line.rstrip(";") | ||
266 | psize, px = line.split("\t") | ||
267 | punit, pname = px.split(" ") | ||
268 | # this size is "installed-size" as it measures how much space it takes on disk | ||
269 | images[target][pname.strip()] = {'size':int(psize)*1024, 'depends' : []} | ||
270 | |||
271 | with open("%s/depends.dot" % installed_img_path, "r") as fin: | ||
272 | p = re.compile(r' -> ') | ||
273 | dot = re.compile(r'.*style=dotted') | ||
274 | for line in fin: | ||
275 | line = line.rstrip(';') | ||
276 | linesplit = p.split(line) | ||
277 | if len(linesplit) == 2: | ||
278 | pname = linesplit[0].rstrip('"').strip('"') | ||
279 | dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"') | ||
280 | deptype = "depends" | ||
281 | if dot.match(line): | ||
282 | deptype = "recommends" | ||
283 | if not pname in images[target]: | ||
284 | images[target][pname] = {'size': 0, 'depends' : []} | ||
285 | if not dependsname in images[target]: | ||
286 | images[target][dependsname] = {'size': 0, 'depends' : []} | ||
287 | images[target][pname]['depends'].append((dependsname, deptype)) | ||
288 | |||
289 | with open("%s/files-in-image.txt" % installed_img_path, "r") as fin: | ||
290 | for line in fin: | ||
291 | lc = [ x for x in line.strip().split(" ") if len(x) > 0 ] | ||
292 | if lc[0].startswith("l"): | ||
293 | files[target]['syms'].append(lc) | ||
294 | elif lc[0].startswith("d"): | ||
295 | files[target]['dirs'].append(lc) | ||
296 | else: | ||
297 | files[target]['files'].append(lc) | ||
298 | |||
299 | for pname in images[target]: | ||
300 | if not pname in allpkgs: | ||
301 | try: | ||
302 | pkgdata = _toaster_load_pkgdatafile("%s/runtime-reverse/" % pkgdata_dir, pname) | ||
303 | except IOError as err: | ||
304 | if err.errno == 2: | ||
305 | # We expect this e.g. for RRECOMMENDS that are unsatisfied at runtime | ||
306 | continue | ||
307 | else: | ||
308 | raise | ||
309 | allpkgs[pname] = pkgdata | ||
310 | |||
311 | |||
312 | data = { 'pkgdata' : allpkgs, 'imgdata' : images, 'filedata' : files } | ||
313 | |||
314 | bb.event.fire(bb.event.MetadataEvent("ImagePkgList", data), e.data) | ||
315 | |||
316 | } | ||
317 | |||
318 | # dump information related to license manifest path | ||
319 | |||
320 | python toaster_licensemanifest_dump() { | ||
321 | deploy_dir = d.getVar('DEPLOY_DIR', True); | ||
322 | image_name = d.getVar('IMAGE_NAME', True); | ||
323 | |||
324 | data = { 'deploy_dir' : deploy_dir, 'image_name' : image_name } | ||
325 | |||
326 | bb.event.fire(bb.event.MetadataEvent("LicenseManifestPath", data), d) | ||
327 | } | ||
328 | |||
329 | # set event handlers | ||
330 | addhandler toaster_layerinfo_dumpdata | ||
331 | toaster_layerinfo_dumpdata[eventmask] = "bb.event.TreeDataPreparationCompleted" | ||
332 | |||
333 | addhandler toaster_collect_task_stats | ||
334 | toaster_collect_task_stats[eventmask] = "bb.event.BuildCompleted bb.build.TaskSucceeded bb.build.TaskFailed" | ||
335 | |||
336 | addhandler toaster_buildhistory_dump | ||
337 | toaster_buildhistory_dump[eventmask] = "bb.event.BuildCompleted" | ||
338 | do_package[postfuncs] += "toaster_package_dumpdata " | ||
339 | do_package[vardepsexclude] += "toaster_package_dumpdata " | ||
340 | |||
341 | do_rootfs[postfuncs] += "toaster_image_dumpdata " | ||
342 | do_rootfs[postfuncs] += "toaster_licensemanifest_dump " | ||
343 | do_rootfs[vardepsexclude] += "toaster_image_dumpdata toaster_licensemanifest_dump" | ||