summaryrefslogtreecommitdiffstats
path: root/scripts/contrib
diff options
context:
space:
mode:
authorMarkus Lehtonen <markus.lehtonen@linux.intel.com>2017-03-31 17:07:30 +0300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-04-01 23:28:20 +0100
commit034702f52070c495415db83e18e77fbe6632e54e (patch)
treec1e242182134e0cb5ce37b6b51696c694d8682cd /scripts/contrib
parent9f299876f716f253b0a3d70eb4473a023c593fc5 (diff)
downloadpoky-034702f52070c495415db83e18e77fbe6632e54e.tar.gz
scripts/contrib: add oe-build-perf-report-email
Script for sending build perf test reports as an email. Mangles an html report, generated by oe-build-perf-report, into a format suitable for html emails. Supports multipart emails where a plaintext alternative can be included in the same email. Dependencies required to be installed on the host: - phantomjs - optipng [YOCTO #10931] (From OE-Core rev: 9e97ff174458f7245fc27a4c407f21a9d2e317ab) Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/contrib')
-rwxr-xr-xscripts/contrib/oe-build-perf-report-email.py266
1 files changed, 266 insertions, 0 deletions
diff --git a/scripts/contrib/oe-build-perf-report-email.py b/scripts/contrib/oe-build-perf-report-email.py
new file mode 100755
index 0000000000..7f4274efed
--- /dev/null
+++ b/scripts/contrib/oe-build-perf-report-email.py
@@ -0,0 +1,266 @@
1#!/usr/bin/python3
2#
3# Send build performance test report emails
4#
5# Copyright (c) 2017, Intel Corporation.
6#
7# This program is free software; you can redistribute it and/or modify it
8# under the terms and conditions of the GNU General Public License,
9# version 2, as published by the Free Software Foundation.
10#
11# This program is distributed in the hope it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14# more details.
15#
16import argparse
17import base64
18import logging
19import os
20import pwd
21import re
22import shutil
23import smtplib
24import subprocess
25import sys
26import tempfile
27from email.mime.multipart import MIMEMultipart
28from email.mime.text import MIMEText
29
30
31# Setup logging
32logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
33log = logging.getLogger('oe-build-perf-report')
34
35
36# Find js scaper script
37SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf',
38 'scrape-html-report.js')
39if not os.path.isfile(SCRAPE_JS):
40 log.error("Unableto find oe-build-perf-report-scrape.js")
41 sys.exit(1)
42
43
44class ReportError(Exception):
45 """Local errors"""
46 pass
47
48
49def check_utils():
50 """Check that all needed utils are installed in the system"""
51 missing = []
52 for cmd in ('phantomjs', 'optipng'):
53 if not shutil.which(cmd):
54 missing.append(cmd)
55 if missing:
56 log.error("The following tools are missing: %s", ' '.join(missing))
57 sys.exit(1)
58
59
60def parse_args(argv):
61 """Parse command line arguments"""
62 description = """Email build perf test report"""
63 parser = argparse.ArgumentParser(
64 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
65 description=description)
66
67 parser.add_argument('--debug', '-d', action='store_true',
68 help="Verbose logging")
69 parser.add_argument('--quiet', '-q', action='store_true',
70 help="Only print errors")
71 parser.add_argument('--to', action='append',
72 help="Recipients of the email")
73 parser.add_argument('--subject', default="Yocto build perf test report",
74 help="Email subject")
75 parser.add_argument('--outdir', '-o',
76 help="Store files in OUTDIR. Can be used to preserve "
77 "the email parts")
78 parser.add_argument('--text',
79 help="Plain text message")
80 parser.add_argument('--html',
81 help="HTML peport generated by oe-build-perf-report")
82 parser.add_argument('--phantomjs-args', action='append',
83 help="Extra command line arguments passed to PhantomJS")
84
85 args = parser.parse_args(argv)
86
87 if not args.html and not args.text:
88 parser.error("Please specify --html and/or --text")
89
90 return args
91
92
93def decode_png(infile, outfile):
94 """Parse/decode/optimize png data from a html element"""
95 with open(infile) as f:
96 raw_data = f.read()
97
98 # Grab raw base64 data
99 b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1)
100 b64_data = re.sub('">.+$', '', b64_data, 1)
101
102 # Replace file with proper decoded png
103 with open(outfile, 'wb') as f:
104 f.write(base64.b64decode(b64_data))
105
106 subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT)
107
108
109def encode_png(pngfile):
110 """Encode png into a <img> html element"""
111 with open(pngfile, 'rb') as f:
112 data = f.read()
113
114 b64_data = base64.b64encode(data)
115 return '<img src="data:image/png;base64,' + b64_data.decode('utf-8') + '">\n'
116
117
118def mangle_html_report(infile, outfile, pngs):
119 """Mangle html file into a email compatible format"""
120 paste = True
121 png_dir = os.path.dirname(outfile)
122 with open(infile) as f_in:
123 with open(outfile, 'w') as f_out:
124 for line in f_in.readlines():
125 stripped = line.strip()
126 # Strip out scripts
127 if stripped == '<!--START-OF-SCRIPTS-->':
128 paste = False
129 elif stripped == '<!--END-OF-SCRIPTS-->':
130 paste = True
131 elif paste:
132 if re.match('^.+href="data:image/png;base64', stripped):
133 # Strip out encoded pngs (as they're huge in size)
134 continue
135 elif 'www.gstatic.com' in stripped:
136 # HACK: drop references to external static pages
137 continue
138
139 # Replace charts with <img> elements
140 match = re.match('<div id="(?P<id>\w+)"', stripped)
141 if match and match.group('id') in pngs:
142 #f_out.write('<img src="{}">\n'.format(match.group('id') + '.png'))
143 png_file = os.path.join(png_dir, match.group('id') + '.png')
144 f_out.write(encode_png(png_file))
145 else:
146 f_out.write(line)
147
148
149def scrape_html_report(report, outdir, phantomjs_extra_args=None):
150 """Scrape html report into a format sendable by email"""
151 tmpdir = tempfile.mkdtemp(dir='.')
152 log.debug("Using tmpdir %s for phantomjs output", tmpdir)
153
154 if not os.path.isdir(outdir):
155 os.mkdir(outdir)
156 if os.path.splitext(report)[1] not in ('.html', '.htm'):
157 raise ReportError("Invalid file extension for report, needs to be "
158 "'.html' or '.htm'")
159
160 try:
161 log.info("Scraping HTML report with PhangomJS")
162 extra_args = phantomjs_extra_args if phantomjs_extra_args else []
163 subprocess.check_output(['phantomjs', '--debug=true'] + extra_args +
164 [SCRAPE_JS, report, tmpdir],
165 stderr=subprocess.STDOUT)
166
167 pngs = []
168 attachments = []
169 for fname in os.listdir(tmpdir):
170 base, ext = os.path.splitext(fname)
171 if ext == '.png':
172 log.debug("Decoding %s", fname)
173 decode_png(os.path.join(tmpdir, fname),
174 os.path.join(outdir, fname))
175 pngs.append(base)
176 attachments.append(fname)
177 elif ext in ('.html', '.htm'):
178 report_file = fname
179 else:
180 log.warning("Unknown file extension: '%s'", ext)
181 #shutil.move(os.path.join(tmpdir, fname), outdir)
182
183 log.debug("Mangling html report file %s", report_file)
184 mangle_html_report(os.path.join(tmpdir, report_file),
185 os.path.join(outdir, report_file), pngs)
186 return report_file, attachments
187 finally:
188 shutil.rmtree(tmpdir)
189
190def send_email(text_fn, html_fn, subject, recipients):
191 """Send email"""
192 # Generate email message
193 text_msg = html_msg = None
194 if text_fn:
195 with open(text_fn) as f:
196 text_msg = MIMEText("Yocto build performance test report.\n" +
197 f.read(), 'plain')
198 if html_fn:
199 with open(html_fn) as f:
200 html_msg = MIMEText(f.read(), 'html')
201
202 if text_msg and html_msg:
203 msg = MIMEMultipart('alternative')
204 msg.attach(text_msg)
205 msg.attach(html_msg)
206 elif text_msg:
207 msg = text_msg
208 elif html_msg:
209 msg = html_msg
210 else:
211 raise ReportError("Neither plain text nor html body specified")
212
213 full_name = pwd.getpwuid(os.getuid()).pw_gecos.split(',')[0]
214 email = os.environ.get('EMAIL', os.getlogin())
215 msg['From'] = "{} <{}>".format(full_name, email)
216 msg['To'] = ', '.join(recipients)
217 msg['Subject'] = subject
218
219 # Send email
220 with smtplib.SMTP('localhost') as smtp:
221 smtp.send_message(msg)
222
223
224def main(argv=None):
225 """Script entry point"""
226 args = parse_args(argv)
227 if args.quiet:
228 log.setLevel(logging.ERROR)
229 if args.debug:
230 log.setLevel(logging.DEBUG)
231
232 check_utils()
233
234 if args.outdir:
235 outdir = args.outdir
236 if not os.path.exists(outdir):
237 os.mkdir(outdir)
238 else:
239 outdir = tempfile.mkdtemp(dir='.')
240
241 try:
242 log.debug("Storing email parts in %s", outdir)
243 html_report = None
244 if args.html:
245 scrape_html_report(args.html, outdir, args.phantomjs_args)
246 html_report = os.path.join(outdir, args.html)
247
248 if args.to:
249 log.info("Sending email to %s", ', '.join(args.to))
250 send_email(args.text, html_report, args.subject, args.to)
251 except subprocess.CalledProcessError as err:
252 log.error("%s, with output:\n%s", str(err), err.output.decode())
253 return 1
254 except ReportError as err:
255 log.error(err)
256 return 1
257 finally:
258 if not args.outdir:
259 log.debug("Wiping %s", outdir)
260 shutil.rmtree(outdir)
261
262 return 0
263
264
265if __name__ == "__main__":
266 sys.exit(main())