diff options
Diffstat (limited to 'scripts/contrib')
-rwxr-xr-x | scripts/contrib/build-perf-test-wrapper.sh | 15 | ||||
-rwxr-xr-x | scripts/contrib/convert-srcuri.py | 77 | ||||
-rwxr-xr-x | scripts/contrib/documentation-audit.sh | 2 | ||||
-rwxr-xr-x | scripts/contrib/oe-build-perf-report-email.py | 167 |
4 files changed, 86 insertions, 175 deletions
diff --git a/scripts/contrib/build-perf-test-wrapper.sh b/scripts/contrib/build-perf-test-wrapper.sh index fa71d4a2e9..0a85e6e708 100755 --- a/scripts/contrib/build-perf-test-wrapper.sh +++ b/scripts/contrib/build-perf-test-wrapper.sh | |||
@@ -87,21 +87,10 @@ if [ $# -ne 0 ]; then | |||
87 | exit 1 | 87 | exit 1 |
88 | fi | 88 | fi |
89 | 89 | ||
90 | if [ -n "$email_to" ]; then | ||
91 | if ! [ -x "$(command -v phantomjs)" ]; then | ||
92 | echo "ERROR: Sending email needs phantomjs." | ||
93 | exit 1 | ||
94 | fi | ||
95 | if ! [ -x "$(command -v optipng)" ]; then | ||
96 | echo "ERROR: Sending email needs optipng." | ||
97 | exit 1 | ||
98 | fi | ||
99 | fi | ||
100 | |||
101 | # Open a file descriptor for flock and acquire lock | 90 | # Open a file descriptor for flock and acquire lock |
102 | LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock" | 91 | LOCK_FILE="/tmp/oe-build-perf-test-wrapper.lock" |
103 | if ! exec 3> "$LOCK_FILE"; then | 92 | if ! exec 3> "$LOCK_FILE"; then |
104 | echo "ERROR: Unable to open lock file" | 93 | echo "ERROR: Unable to open loemack file" |
105 | exit 1 | 94 | exit 1 |
106 | fi | 95 | fi |
107 | if ! flock -n 3; then | 96 | if ! flock -n 3; then |
@@ -226,7 +215,7 @@ if [ -n "$results_repo" ]; then | |||
226 | if [ -n "$email_to" ]; then | 215 | if [ -n "$email_to" ]; then |
227 | echo "Emailing test report" | 216 | echo "Emailing test report" |
228 | os_name=`get_os_release_var PRETTY_NAME` | 217 | os_name=`get_os_release_var PRETTY_NAME` |
229 | "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt --html $report_html "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" | 218 | "$script_dir"/oe-build-perf-report-email.py --to "$email_to" --subject "Build Perf Test Report for $os_name" --text $report_txt "${OE_BUILD_PERF_REPORT_EMAIL_EXTRA_ARGS[@]}" |
230 | fi | 219 | fi |
231 | 220 | ||
232 | # Upload report files, unless we're on detached head | 221 | # Upload report files, unless we're on detached head |
diff --git a/scripts/contrib/convert-srcuri.py b/scripts/contrib/convert-srcuri.py new file mode 100755 index 0000000000..5b362ea2e8 --- /dev/null +++ b/scripts/contrib/convert-srcuri.py | |||
@@ -0,0 +1,77 @@ | |||
1 | #!/usr/bin/env python3 | ||
2 | # | ||
3 | # Conversion script to update SRC_URI to add branch to git urls | ||
4 | # | ||
5 | # Copyright (C) 2021 Richard Purdie | ||
6 | # | ||
7 | # SPDX-License-Identifier: GPL-2.0-only | ||
8 | # | ||
9 | |||
10 | import re | ||
11 | import os | ||
12 | import sys | ||
13 | import tempfile | ||
14 | import shutil | ||
15 | import mimetypes | ||
16 | |||
17 | if len(sys.argv) < 2: | ||
18 | print("Please specify a directory to run the conversion script against.") | ||
19 | sys.exit(1) | ||
20 | |||
21 | def processfile(fn): | ||
22 | def matchline(line): | ||
23 | if "MIRROR" in line or ".*" in line or "GNOME_GIT" in line: | ||
24 | return False | ||
25 | return True | ||
26 | print("processing file '%s'" % fn) | ||
27 | try: | ||
28 | if "distro_alias.inc" in fn or "linux-yocto-custom.bb" in fn: | ||
29 | return | ||
30 | fh, abs_path = tempfile.mkstemp() | ||
31 | modified = False | ||
32 | with os.fdopen(fh, 'w') as new_file: | ||
33 | with open(fn, "r") as old_file: | ||
34 | for line in old_file: | ||
35 | if ("git://" in line or "gitsm://" in line) and "branch=" not in line and matchline(line): | ||
36 | if line.endswith('"\n'): | ||
37 | line = line.replace('"\n', ';branch=master"\n') | ||
38 | elif line.endswith(" \\\n"): | ||
39 | line = line.replace(' \\\n', ';branch=master \\\n') | ||
40 | modified = True | ||
41 | if ("git://" in line or "gitsm://" in line) and "github.com" in line and "protocol=https" not in line and matchline(line): | ||
42 | if "protocol=git" in line: | ||
43 | line = line.replace('protocol=git', 'protocol=https') | ||
44 | elif line.endswith('"\n'): | ||
45 | line = line.replace('"\n', ';protocol=https"\n') | ||
46 | elif line.endswith(" \\\n"): | ||
47 | line = line.replace(' \\\n', ';protocol=https \\\n') | ||
48 | modified = True | ||
49 | new_file.write(line) | ||
50 | if modified: | ||
51 | shutil.copymode(fn, abs_path) | ||
52 | os.remove(fn) | ||
53 | shutil.move(abs_path, fn) | ||
54 | except UnicodeDecodeError: | ||
55 | pass | ||
56 | |||
57 | ourname = os.path.basename(sys.argv[0]) | ||
58 | ourversion = "0.1" | ||
59 | |||
60 | if os.path.isfile(sys.argv[1]): | ||
61 | processfile(sys.argv[1]) | ||
62 | sys.exit(0) | ||
63 | |||
64 | for targetdir in sys.argv[1:]: | ||
65 | print("processing directory '%s'" % targetdir) | ||
66 | for root, dirs, files in os.walk(targetdir): | ||
67 | for name in files: | ||
68 | if name == ourname: | ||
69 | continue | ||
70 | fn = os.path.join(root, name) | ||
71 | if os.path.islink(fn): | ||
72 | continue | ||
73 | if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"): | ||
74 | continue | ||
75 | processfile(fn) | ||
76 | |||
77 | print("All files processed with version %s" % ourversion) | ||
diff --git a/scripts/contrib/documentation-audit.sh b/scripts/contrib/documentation-audit.sh index 1191f57a8e..f436f9bae0 100755 --- a/scripts/contrib/documentation-audit.sh +++ b/scripts/contrib/documentation-audit.sh | |||
@@ -27,7 +27,7 @@ fi | |||
27 | 27 | ||
28 | echo "REMINDER: you need to build for MACHINE=qemux86 or you won't get useful results" | 28 | echo "REMINDER: you need to build for MACHINE=qemux86 or you won't get useful results" |
29 | echo "REMINDER: you need to set LICENSE_FLAGS_WHITELIST appropriately in local.conf or " | 29 | echo "REMINDER: you need to set LICENSE_FLAGS_WHITELIST appropriately in local.conf or " |
30 | echo " you'll get false positives. For example, LICENSE_FLAGS_WHITELIST = \"Commercial\"" | 30 | echo " you'll get false positives. For example, LICENSE_FLAGS_WHITELIST = \"commercial\"" |
31 | 31 | ||
32 | for pkg in `bitbake -s | awk '{ print \$1 }'`; do | 32 | for pkg in `bitbake -s | awk '{ print \$1 }'`; do |
33 | if [[ "$pkg" == "Loading" || "$pkg" == "Loaded" || | 33 | if [[ "$pkg" == "Loading" || "$pkg" == "Loaded" || |
diff --git a/scripts/contrib/oe-build-perf-report-email.py b/scripts/contrib/oe-build-perf-report-email.py index de3862c897..7192113c28 100755 --- a/scripts/contrib/oe-build-perf-report-email.py +++ b/scripts/contrib/oe-build-perf-report-email.py | |||
@@ -19,8 +19,6 @@ import socket | |||
19 | import subprocess | 19 | import subprocess |
20 | import sys | 20 | import sys |
21 | import tempfile | 21 | import tempfile |
22 | from email.mime.image import MIMEImage | ||
23 | from email.mime.multipart import MIMEMultipart | ||
24 | from email.mime.text import MIMEText | 22 | from email.mime.text import MIMEText |
25 | 23 | ||
26 | 24 | ||
@@ -29,30 +27,6 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") | |||
29 | log = logging.getLogger('oe-build-perf-report') | 27 | log = logging.getLogger('oe-build-perf-report') |
30 | 28 | ||
31 | 29 | ||
32 | # Find js scaper script | ||
33 | SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf', | ||
34 | 'scrape-html-report.js') | ||
35 | if not os.path.isfile(SCRAPE_JS): | ||
36 | log.error("Unableto find oe-build-perf-report-scrape.js") | ||
37 | sys.exit(1) | ||
38 | |||
39 | |||
40 | class ReportError(Exception): | ||
41 | """Local errors""" | ||
42 | pass | ||
43 | |||
44 | |||
45 | def check_utils(): | ||
46 | """Check that all needed utils are installed in the system""" | ||
47 | missing = [] | ||
48 | for cmd in ('phantomjs', 'optipng'): | ||
49 | if not shutil.which(cmd): | ||
50 | missing.append(cmd) | ||
51 | if missing: | ||
52 | log.error("The following tools are missing: %s", ' '.join(missing)) | ||
53 | sys.exit(1) | ||
54 | |||
55 | |||
56 | def parse_args(argv): | 30 | def parse_args(argv): |
57 | """Parse command line arguments""" | 31 | """Parse command line arguments""" |
58 | description = """Email build perf test report""" | 32 | description = """Email build perf test report""" |
@@ -77,137 +51,19 @@ def parse_args(argv): | |||
77 | "the email parts") | 51 | "the email parts") |
78 | parser.add_argument('--text', | 52 | parser.add_argument('--text', |
79 | help="Plain text message") | 53 | help="Plain text message") |
80 | parser.add_argument('--html', | ||
81 | help="HTML peport generated by oe-build-perf-report") | ||
82 | parser.add_argument('--phantomjs-args', action='append', | ||
83 | help="Extra command line arguments passed to PhantomJS") | ||
84 | 54 | ||
85 | args = parser.parse_args(argv) | 55 | args = parser.parse_args(argv) |
86 | 56 | ||
87 | if not args.html and not args.text: | 57 | if not args.text: |
88 | parser.error("Please specify --html and/or --text") | 58 | parser.error("Please specify --text") |
89 | 59 | ||
90 | return args | 60 | return args |
91 | 61 | ||
92 | 62 | ||
93 | def decode_png(infile, outfile): | 63 | def send_email(text_fn, subject, recipients, copy=[], blind_copy=[]): |
94 | """Parse/decode/optimize png data from a html element""" | ||
95 | with open(infile) as f: | ||
96 | raw_data = f.read() | ||
97 | |||
98 | # Grab raw base64 data | ||
99 | b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1) | ||
100 | b64_data = re.sub('">.+$', '', b64_data, 1) | ||
101 | |||
102 | # Replace file with proper decoded png | ||
103 | with open(outfile, 'wb') as f: | ||
104 | f.write(base64.b64decode(b64_data)) | ||
105 | |||
106 | subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT) | ||
107 | |||
108 | |||
109 | def mangle_html_report(infile, outfile, pngs): | ||
110 | """Mangle html file into a email compatible format""" | ||
111 | paste = True | ||
112 | png_dir = os.path.dirname(outfile) | ||
113 | with open(infile) as f_in: | ||
114 | with open(outfile, 'w') as f_out: | ||
115 | for line in f_in.readlines(): | ||
116 | stripped = line.strip() | ||
117 | # Strip out scripts | ||
118 | if stripped == '<!--START-OF-SCRIPTS-->': | ||
119 | paste = False | ||
120 | elif stripped == '<!--END-OF-SCRIPTS-->': | ||
121 | paste = True | ||
122 | elif paste: | ||
123 | if re.match('^.+href="data:image/png;base64', stripped): | ||
124 | # Strip out encoded pngs (as they're huge in size) | ||
125 | continue | ||
126 | elif 'www.gstatic.com' in stripped: | ||
127 | # HACK: drop references to external static pages | ||
128 | continue | ||
129 | |||
130 | # Replace charts with <img> elements | ||
131 | match = re.match('<div id="(?P<id>\w+)"', stripped) | ||
132 | if match and match.group('id') in pngs: | ||
133 | f_out.write('<img src="cid:{}"\n'.format(match.group('id'))) | ||
134 | else: | ||
135 | f_out.write(line) | ||
136 | |||
137 | |||
138 | def scrape_html_report(report, outdir, phantomjs_extra_args=None): | ||
139 | """Scrape html report into a format sendable by email""" | ||
140 | tmpdir = tempfile.mkdtemp(dir='.') | ||
141 | log.debug("Using tmpdir %s for phantomjs output", tmpdir) | ||
142 | |||
143 | if not os.path.isdir(outdir): | ||
144 | os.mkdir(outdir) | ||
145 | if os.path.splitext(report)[1] not in ('.html', '.htm'): | ||
146 | raise ReportError("Invalid file extension for report, needs to be " | ||
147 | "'.html' or '.htm'") | ||
148 | |||
149 | try: | ||
150 | log.info("Scraping HTML report with PhangomJS") | ||
151 | extra_args = phantomjs_extra_args if phantomjs_extra_args else [] | ||
152 | subprocess.check_output(['phantomjs', '--debug=true'] + extra_args + | ||
153 | [SCRAPE_JS, report, tmpdir], | ||
154 | stderr=subprocess.STDOUT) | ||
155 | |||
156 | pngs = [] | ||
157 | images = [] | ||
158 | for fname in os.listdir(tmpdir): | ||
159 | base, ext = os.path.splitext(fname) | ||
160 | if ext == '.png': | ||
161 | log.debug("Decoding %s", fname) | ||
162 | decode_png(os.path.join(tmpdir, fname), | ||
163 | os.path.join(outdir, fname)) | ||
164 | pngs.append(base) | ||
165 | images.append(fname) | ||
166 | elif ext in ('.html', '.htm'): | ||
167 | report_file = fname | ||
168 | else: | ||
169 | log.warning("Unknown file extension: '%s'", ext) | ||
170 | #shutil.move(os.path.join(tmpdir, fname), outdir) | ||
171 | |||
172 | log.debug("Mangling html report file %s", report_file) | ||
173 | mangle_html_report(os.path.join(tmpdir, report_file), | ||
174 | os.path.join(outdir, report_file), pngs) | ||
175 | return (os.path.join(outdir, report_file), | ||
176 | [os.path.join(outdir, i) for i in images]) | ||
177 | finally: | ||
178 | shutil.rmtree(tmpdir) | ||
179 | |||
180 | def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[], | ||
181 | blind_copy=[]): | ||
182 | """Send email""" | ||
183 | # Generate email message | 64 | # Generate email message |
184 | text_msg = html_msg = None | 65 | with open(text_fn) as f: |
185 | if text_fn: | 66 | msg = MIMEText("Yocto build performance test report.\n" + f.read(), 'plain') |
186 | with open(text_fn) as f: | ||
187 | text_msg = MIMEText("Yocto build performance test report.\n" + | ||
188 | f.read(), 'plain') | ||
189 | if html_fn: | ||
190 | html_msg = msg = MIMEMultipart('related') | ||
191 | with open(html_fn) as f: | ||
192 | html_msg.attach(MIMEText(f.read(), 'html')) | ||
193 | for img_fn in image_fns: | ||
194 | # Expect that content id is same as the filename | ||
195 | cid = os.path.splitext(os.path.basename(img_fn))[0] | ||
196 | with open(img_fn, 'rb') as f: | ||
197 | image_msg = MIMEImage(f.read()) | ||
198 | image_msg['Content-ID'] = '<{}>'.format(cid) | ||
199 | html_msg.attach(image_msg) | ||
200 | |||
201 | if text_msg and html_msg: | ||
202 | msg = MIMEMultipart('alternative') | ||
203 | msg.attach(text_msg) | ||
204 | msg.attach(html_msg) | ||
205 | elif text_msg: | ||
206 | msg = text_msg | ||
207 | elif html_msg: | ||
208 | msg = html_msg | ||
209 | else: | ||
210 | raise ReportError("Neither plain text nor html body specified") | ||
211 | 67 | ||
212 | pw_data = pwd.getpwuid(os.getuid()) | 68 | pw_data = pwd.getpwuid(os.getuid()) |
213 | full_name = pw_data.pw_gecos.split(',')[0] | 69 | full_name = pw_data.pw_gecos.split(',')[0] |
@@ -234,8 +90,6 @@ def main(argv=None): | |||
234 | if args.debug: | 90 | if args.debug: |
235 | log.setLevel(logging.DEBUG) | 91 | log.setLevel(logging.DEBUG) |
236 | 92 | ||
237 | check_utils() | ||
238 | |||
239 | if args.outdir: | 93 | if args.outdir: |
240 | outdir = args.outdir | 94 | outdir = args.outdir |
241 | if not os.path.exists(outdir): | 95 | if not os.path.exists(outdir): |
@@ -245,25 +99,16 @@ def main(argv=None): | |||
245 | 99 | ||
246 | try: | 100 | try: |
247 | log.debug("Storing email parts in %s", outdir) | 101 | log.debug("Storing email parts in %s", outdir) |
248 | html_report = images = None | ||
249 | if args.html: | ||
250 | html_report, images = scrape_html_report(args.html, outdir, | ||
251 | args.phantomjs_args) | ||
252 | |||
253 | if args.to: | 102 | if args.to: |
254 | log.info("Sending email to %s", ', '.join(args.to)) | 103 | log.info("Sending email to %s", ', '.join(args.to)) |
255 | if args.cc: | 104 | if args.cc: |
256 | log.info("Copying to %s", ', '.join(args.cc)) | 105 | log.info("Copying to %s", ', '.join(args.cc)) |
257 | if args.bcc: | 106 | if args.bcc: |
258 | log.info("Blind copying to %s", ', '.join(args.bcc)) | 107 | log.info("Blind copying to %s", ', '.join(args.bcc)) |
259 | send_email(args.text, html_report, images, args.subject, | 108 | send_email(args.text, args.subject, args.to, args.cc, args.bcc) |
260 | args.to, args.cc, args.bcc) | ||
261 | except subprocess.CalledProcessError as err: | 109 | except subprocess.CalledProcessError as err: |
262 | log.error("%s, with output:\n%s", str(err), err.output.decode()) | 110 | log.error("%s, with output:\n%s", str(err), err.output.decode()) |
263 | return 1 | 111 | return 1 |
264 | except ReportError as err: | ||
265 | log.error(err) | ||
266 | return 1 | ||
267 | finally: | 112 | finally: |
268 | if not args.outdir: | 113 | if not args.outdir: |
269 | log.debug("Wiping %s", outdir) | 114 | log.debug("Wiping %s", outdir) |