summaryrefslogtreecommitdiffstats
path: root/scripts/contrib
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2022-03-15 17:29:13 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-03-16 10:31:41 +0000
commit10d9a8ba7a6ab2ce0e434e83f44727c160d26982 (patch)
treee67ff58c16db536a6e0b88f9fe6ec999d56a950d /scripts/contrib
parent65d4385d46163a3a1dd7c3ea7575b419eb743675 (diff)
downloadpoky-10d9a8ba7a6ab2ce0e434e83f44727c160d26982.tar.gz
perf-build-test/report: Drop phantomjs and html email reports support
phantomjs isn't reliable and we've moved to sharing the reports via a webserver. Update the scripts to more match those being used in the autobuilder helper where the html email support was removed. (From OE-Core rev: b5c131006e3fad0a15e6cdf81f71dc1e96647028) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/contrib')
-rwxr-xr-xscripts/contrib/build-perf-test-wrapper.sh15
-rwxr-xr-xscripts/contrib/oe-build-perf-report-email.py154
2 files changed, 8 insertions, 161 deletions
diff --git a/scripts/contrib/build-perf-test-wrapper.sh b/scripts/contrib/build-perf-test-wrapper.sh
index fa71d4a2e9..0a85e6e708 100755
--- a/scripts/contrib/build-perf-test-wrapper.sh
+++ b/scripts/contrib/build-perf-test-wrapper.sh
@@ -87,21 +87,10 @@ if [ $# -ne 0 ]; then
87 exit 1 87 exit 1
88fi 88fi
89 89
90if [ -n "$email_to" ]; then
91 if ! [ -x "$(command -v phantomjs)" ]; then
92 echo "ERROR: Sending email needs phantomjs."
93 exit 1
94 fi
95 if ! [ -x "$(command -v optipng)" ]; then
96 echo "ERROR: Sending email needs optipng."
97 exit 1
98 fi
99fi
100
101# Open a file descriptor for flock and acquire lock 90# Open a file descriptor for flock and acquire lock
102LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock" 91LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock"
103if ! exec 3> "$LOCK_FILE"; then 92if ! exec 3> "$LOCK_FILE"; then
104 echo "ERROR: Unable to open lock file" 93 echo "ERROR: Unable to open loemack file"
105 exit 1 94 exit 1
106fi 95fi
107if ! flock -n 3; then 96if ! flock -n 3; then
@@ -226,7 +215,7 @@ if [ -n "$results_repo" ]; then
226 if [ -n "$email_to" ]; then 215 if [ -n "$email_to" ]; then
227 echo "Emailing test report" 216 echo "Emailing test report"
228 os_name=`get_os_release_var PRETTY_NAME` 217 os_name=`get_os_release_var PRETTY_NAME`
229 "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt --html $report_html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" 218 "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}"
230 fi 219 fi
231 220
232 # Upload report files, unless we're on detached head 221 # Upload report files, unless we're on detached head
diff --git a/scripts/contrib/oe-build-perf-report-email.py b/scripts/contrib/oe-build-perf-report-email.py
index de3862c897..c900720f6e 100755
--- a/scripts/contrib/oe-build-perf-report-email.py
+++ b/scripts/contrib/oe-build-perf-report-email.py
@@ -19,8 +19,6 @@ import socket
19import subprocess 19import subprocess
20import sys 20import sys
21import tempfile 21import tempfile
22from email.mime.image import MIMEImage
23from email.mime.multipart import MIMEMultipart
24from email.mime.text import MIMEText 22from email.mime.text import MIMEText
25 23
26 24
@@ -29,19 +27,6 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
29log = logging.getLogger('oe-build-perf-report') 27log = logging.getLogger('oe-build-perf-report')
30 28
31 29
32# Find js scaper script
33SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf',
34 'scrape-html-report.js')
35if not os.path.isfile(SCRAPE_JS):
36 log.error("Unableto find oe-build-perf-report-scrape.js")
37 sys.exit(1)
38
39
40class ReportError(Exception):
41 """Local errors"""
42 pass
43
44
45def check_utils(): 30def check_utils():
46 """Check that all needed utils are installed in the system""" 31 """Check that all needed utils are installed in the system"""
47 missing = [] 32 missing = []
@@ -77,137 +62,19 @@ def parse_args(argv):
77 "the email parts") 62 "the email parts")
78 parser.add_argument('--text', 63 parser.add_argument('--text',
79 help="Plain text message") 64 help="Plain text message")
80 parser.add_argument('--html',
81 help="HTML peport generated by oe-build-perf-report")
82 parser.add_argument('--phantomjs-args', action='append',
83 help="Extra command line arguments passed to PhantomJS")
84 65
85 args = parser.parse_args(argv) 66 args = parser.parse_args(argv)
86 67
87 if not args.html and not args.text: 68 if not args.text:
88 parser.error("Please specify --html and/or --text") 69 parser.error("Please specify --text")
89 70
90 return args 71 return args
91 72
92 73
93def decode_png(infile, outfile): 74def send_email(text_fn, subject, recipients, copy=[], blind_copy=[]):
94 """Parse/decode/optimize png data from a html element"""
95 with open(infile) as f:
96 raw_data = f.read()
97
98 # Grab raw base64 data
99 b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1)
100 b64_data = re.sub('">.+$', '', b64_data, 1)
101
102 # Replace file with proper decoded png
103 with open(outfile, 'wb') as f:
104 f.write(base64.b64decode(b64_data))
105
106 subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT)
107
108
109def mangle_html_report(infile, outfile, pngs):
110 """Mangle html file into a email compatible format"""
111 paste = True
112 png_dir = os.path.dirname(outfile)
113 with open(infile) as f_in:
114 with open(outfile, 'w') as f_out:
115 for line in f_in.readlines():
116 stripped = line.strip()
117 # Strip out scripts
118 if stripped == '<!--START-OF-SCRIPTS-->':
119 paste = False
120 elif stripped == '<!--END-OF-SCRIPTS-->':
121 paste = True
122 elif paste:
123 if re.match('^.+href="data:image/png;base64', stripped):
124 # Strip out encoded pngs (as they're huge in size)
125 continue
126 elif 'www.gstatic.com' in stripped:
127 # HACK: drop references to external static pages
128 continue
129
130 # Replace charts with <img> elements
131 match = re.match('<div id="(?P<id>\w+)"', stripped)
132 if match and match.group('id') in pngs:
133 f_out.write('<img src="cid:{}"\n'.format(match.group('id')))
134 else:
135 f_out.write(line)
136
137
138def scrape_html_report(report, outdir, phantomjs_extra_args=None):
139 """Scrape html report into a format sendable by email"""
140 tmpdir = tempfile.mkdtemp(dir='.')
141 log.debug("Using tmpdir %s for phantomjs output", tmpdir)
142
143 if not os.path.isdir(outdir):
144 os.mkdir(outdir)
145 if os.path.splitext(report)[1] not in ('.html', '.htm'):
146 raise ReportError("Invalid file extension for report, needs to be "
147 "'.html' or '.htm'")
148
149 try:
150 log.info("Scraping HTML report with PhangomJS")
151 extra_args = phantomjs_extra_args if phantomjs_extra_args else []
152 subprocess.check_output(['phantomjs', '--debug=true'] + extra_args +
153 [SCRAPE_JS, report, tmpdir],
154 stderr=subprocess.STDOUT)
155
156 pngs = []
157 images = []
158 for fname in os.listdir(tmpdir):
159 base, ext = os.path.splitext(fname)
160 if ext == '.png':
161 log.debug("Decoding %s", fname)
162 decode_png(os.path.join(tmpdir, fname),
163 os.path.join(outdir, fname))
164 pngs.append(base)
165 images.append(fname)
166 elif ext in ('.html', '.htm'):
167 report_file = fname
168 else:
169 log.warning("Unknown file extension: '%s'", ext)
170 #shutil.move(os.path.join(tmpdir, fname), outdir)
171
172 log.debug("Mangling html report file %s", report_file)
173 mangle_html_report(os.path.join(tmpdir, report_file),
174 os.path.join(outdir, report_file), pngs)
175 return (os.path.join(outdir, report_file),
176 [os.path.join(outdir, i) for i in images])
177 finally:
178 shutil.rmtree(tmpdir)
179
180def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[],
181 blind_copy=[]):
182 """Send email"""
183 # Generate email message 75 # Generate email message
184 text_msg = html_msg = None 76 with open(text_fn) as f:
185 if text_fn: 77 msg = MIMEText("Yocto build performance test report.\n" + f.read(), 'plain')
186 with open(text_fn) as f:
187 text_msg = MIMEText("Yocto build performance test report.\n" +
188 f.read(), 'plain')
189 if html_fn:
190 html_msg = msg = MIMEMultipart('related')
191 with open(html_fn) as f:
192 html_msg.attach(MIMEText(f.read(), 'html'))
193 for img_fn in image_fns:
194 # Expect that content id is same as the filename
195 cid = os.path.splitext(os.path.basename(img_fn))[0]
196 with open(img_fn, 'rb') as f:
197 image_msg = MIMEImage(f.read())
198 image_msg['Content-ID'] = '<{}>'.format(cid)
199 html_msg.attach(image_msg)
200
201 if text_msg and html_msg:
202 msg = MIMEMultipart('alternative')
203 msg.attach(text_msg)
204 msg.attach(html_msg)
205 elif text_msg:
206 msg = text_msg
207 elif html_msg:
208 msg = html_msg
209 else:
210 raise ReportError("Neither plain text nor html body specified")
211 78
212 pw_data = pwd.getpwuid(os.getuid()) 79 pw_data = pwd.getpwuid(os.getuid())
213 full_name = pw_data.pw_gecos.split(',')[0] 80 full_name = pw_data.pw_gecos.split(',')[0]
@@ -245,25 +112,16 @@ def main(argv=None):
245 112
246 try: 113 try:
247 log.debug("Storing email parts in %s", outdir) 114 log.debug("Storing email parts in %s", outdir)
248 html_report = images = None
249 if args.html:
250 html_report, images = scrape_html_report(args.html, outdir,
251 args.phantomjs_args)
252
253 if args.to: 115 if args.to:
254 log.info("Sending email to %s", ', '.join(args.to)) 116 log.info("Sending email to %s", ', '.join(args.to))
255 if args.cc: 117 if args.cc:
256 log.info("Copying to %s", ', '.join(args.cc)) 118 log.info("Copying to %s", ', '.join(args.cc))
257 if args.bcc: 119 if args.bcc:
258 log.info("Blind copying to %s", ', '.join(args.bcc)) 120 log.info("Blind copying to %s", ', '.join(args.bcc))
259 send_email(args.text, html_report, images, args.subject, 121 send_email(args.text, args.subject, args.to, args.cc, args.bcc)
260 args.to, args.cc, args.bcc)
261 except subprocess.CalledProcessError as err: 122 except subprocess.CalledProcessError as err:
262 log.error("%s, with output:\n%s", str(err), err.output.decode()) 123 log.error("%s, with output:\n%s", str(err), err.output.decode())
263 return 1 124 return 1
264 except ReportError as err:
265 log.error(err)
266 return 1
267 finally: 125 finally:
268 if not args.outdir: 126 if not args.outdir:
269 log.debug("Wiping %s", outdir) 127 log.debug("Wiping %s", outdir)