summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster/contrib
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2015-07-30 19:25:09 +0300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-08-01 11:26:13 +0100
commit1640a65091c56eeed0f6bb1020fd156cdc279a6e (patch)
treeee36a0a10c1fea4d5e10e645b9077f80d6d67567 /bitbake/lib/toaster/contrib
parent160d610604fff0b119de2b70a9da8a5f73d93715 (diff)
downloadpoky-1640a65091c56eeed0f6bb1020fd156cdc279a6e.tar.gz
bitbake: toaster: tts: fix pylint warnings
This patch brings TTS to the pylint coding standards. Pylint was run with some disables: disable=logging-too-many-args,line-too-long,missing-docstring and achieved Your code has been rated at 10.00/10 There are no functional changes. (Bitbake rev: 2b40b412ff6a7e3fd4cc32707bd3cd713bc09ddb) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster/contrib')
-rw-r--r--bitbake/lib/toaster/contrib/tts/config.py37
-rwxr-xr-xbitbake/lib/toaster/contrib/tts/launcher.py35
-rwxr-xr-xbitbake/lib/toaster/contrib/tts/recv.py13
-rwxr-xr-xbitbake/lib/toaster/contrib/tts/runner.py87
-rw-r--r--bitbake/lib/toaster/contrib/tts/scratchpad.py20
-rw-r--r--bitbake/lib/toaster/contrib/tts/shellutils.py64
-rw-r--r--bitbake/lib/toaster/contrib/tts/tests.py43
-rw-r--r--bitbake/lib/toaster/contrib/tts/urlcheck.py29
-rw-r--r--bitbake/lib/toaster/contrib/tts/urllist.py90
9 files changed, 208 insertions, 210 deletions
diff --git a/bitbake/lib/toaster/contrib/tts/config.py b/bitbake/lib/toaster/contrib/tts/config.py
index c0e05369e6..40d45f3b7a 100644
--- a/bitbake/lib/toaster/contrib/tts/config.py
+++ b/bitbake/lib/toaster/contrib/tts/config.py
@@ -28,42 +28,48 @@ LOGDIR = "log"
28SETTINGS_FILE = os.path.join(os.path.dirname(__file__), "settings.json") 28SETTINGS_FILE = os.path.join(os.path.dirname(__file__), "settings.json")
29TEST_DIR_NAME = "tts_testdir" 29TEST_DIR_NAME = "tts_testdir"
30 30
31DEBUG = True
32
31OWN_PID = os.getpid() 33OWN_PID = os.getpid()
32 34
33W3C_VALIDATOR = "http://icarus.local/w3c-validator/check?doctype=HTML5&uri=" 35W3C_VALIDATOR = "http://icarus.local/w3c-validator/check?doctype=HTML5&uri="
34 36
35#TODO assign port dynamically 37TOASTER_PORT = 56789
36TOASTER_PORT=56789 38
39TESTDIR = None
37 40
38#we parse the w3c URL to know where to connect 41#we parse the w3c URL to know where to connect
39 42
40import urlparse 43import urlparse
41 44
42def get_public_ip(): 45def get_public_ip():
43 s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 46 temp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
44 p = urlparse.urlparse("http://icarus.local/w3c-validator/check?doctype=HTML5&uri=") 47 parsed_url = urlparse.urlparse("http://icarus.local/w3c-validator/check?doctype=HTML5&uri=")
45 s.connect(( p.netloc, 80 if p.port is None else p.port)) 48 temp_socket.connect((parsed_url.netloc, 80 if parsed_url.port is None else parsed_url.port))
46 hn = s.getsockname()[0] 49 public_ip = temp_socket.getsockname()[0]
47 s.close() 50 temp_socket.close()
48 return hn 51 return public_ip
49 52
50TOASTER_BASEURL="http://%s:%d/" % (get_public_ip(), TOASTER_PORT) 53TOASTER_BASEURL = "http://%s:%d/" % (get_public_ip(), TOASTER_PORT)
51 54
52 55
53OWN_EMAIL_ADDRESS = "Toaster Testing Framework <alexandru.damian@intel.com>" 56OWN_EMAIL_ADDRESS = "Toaster Testing Framework <alexandru.damian@intel.com>"
54REPORT_EMAIL_ADDRESS = "alexandru.damian@intel.com" 57REPORT_EMAIL_ADDRESS = "alexandru.damian@intel.com"
55 58
56# make sure we have the basic logging infrastructure 59# make sure we have the basic logging infrastructure
60
61#pylint: disable=invalid-name
62# we disable the invalid name because the module-level "logger" is used througout bitbake
57logger = logging.getLogger("toastertest") 63logger = logging.getLogger("toastertest")
58__console = logging.StreamHandler(sys.stdout) 64__console__ = logging.StreamHandler(sys.stdout)
59__console.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s")) 65__console__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
60logger.addHandler(__console) 66logger.addHandler(__console__)
61logger.setLevel(logging.DEBUG) 67logger.setLevel(logging.DEBUG)
62 68
63 69
64# singleton file names 70# singleton file names
65LOCKFILE="/tmp/ttf.lock" 71LOCKFILE = "/tmp/ttf.lock"
66BACKLOGFILE=os.path.join(os.path.dirname(__file__), "backlog.txt") 72BACKLOGFILE = os.path.join(os.path.dirname(__file__), "backlog.txt")
67 73
68# task states 74# task states
69def enum(*sequential, **named): 75def enum(*sequential, **named):
@@ -73,7 +79,8 @@ def enum(*sequential, **named):
73 return type('Enum', (), enums) 79 return type('Enum', (), enums)
74 80
75 81
76class TASKS: 82class TASKS(object):
83 #pylint: disable=too-few-public-methods
77 PENDING = "PENDING" 84 PENDING = "PENDING"
78 INPROGRESS = "INPROGRESS" 85 INPROGRESS = "INPROGRESS"
79 DONE = "DONE" 86 DONE = "DONE"
diff --git a/bitbake/lib/toaster/contrib/tts/launcher.py b/bitbake/lib/toaster/contrib/tts/launcher.py
index 90c6d95c58..e5794c1c56 100755
--- a/bitbake/lib/toaster/contrib/tts/launcher.py
+++ b/bitbake/lib/toaster/contrib/tts/launcher.py
@@ -31,13 +31,11 @@ import smtplib
31# Import the email modules we'll need 31# Import the email modules we'll need
32from email.mime.text import MIMEText 32from email.mime.text import MIMEText
33 33
34DEBUG=True
35
36def _take_lockfile(): 34def _take_lockfile():
37 return shellutils.lockfile(shellutils.mk_lock_filename()) 35 return shellutils.lockfile(shellutils.mk_lock_filename())
38 36
39 37
40def read_next_task_by_state(task_state, task_name = None): 38def read_next_task_by_state(task_state, task_name=None):
41 if not os.path.exists(os.path.join(os.path.dirname(__file__), config.BACKLOGFILE)): 39 if not os.path.exists(os.path.join(os.path.dirname(__file__), config.BACKLOGFILE)):
42 return None 40 return None
43 os.rename(config.BACKLOGFILE, config.BACKLOGFILE + ".tmp") 41 os.rename(config.BACKLOGFILE, config.BACKLOGFILE + ".tmp")
@@ -56,28 +54,28 @@ def read_next_task_by_state(task_state, task_name = None):
56 os.remove(config.BACKLOGFILE + ".tmp") 54 os.remove(config.BACKLOGFILE + ".tmp")
57 return task 55 return task
58 56
59def send_report(task_name, plaintext, errtext = None): 57def send_report(task_name, plaintext, errtext=None):
60 if errtext is None: 58 if errtext is None:
61 msg = MIMEText(plaintext) 59 msg = MIMEText(plaintext)
62 else: 60 else:
63 if plaintext is None: 61 if plaintext is None:
64 plaintext="" 62 plaintext = ""
65 msg = MIMEText("--STDOUT dump--\n\n%s\n\n--STDERR dump--\n\n%s" % (plaintext, errtext)) 63 msg = MIMEText("--STDOUT dump--\n\n%s\n\n--STDERR dump--\n\n%s" % (plaintext, errtext))
66 64
67 msg['Subject'] = "[review-request] %s - smoke test results" % task_name 65 msg['Subject'] = "[review-request] %s - smoke test results" % task_name
68 msg['From'] = config.OWN_EMAIL_ADDRESS 66 msg['From'] = config.OWN_EMAIL_ADDRESS
69 msg['To'] = config.REPORT_EMAIL_ADDRESS 67 msg['To'] = config.REPORT_EMAIL_ADDRESS
70 68
71 s = smtplib.SMTP("localhost") 69 smtp_connection = smtplib.SMTP("localhost")
72 s.sendmail(config.OWN_EMAIL_ADDRESS, [config.REPORT_EMAIL_ADDRESS], msg.as_string()) 70 smtp_connection.sendmail(config.OWN_EMAIL_ADDRESS, [config.REPORT_EMAIL_ADDRESS], msg.as_string())
73 s.quit() 71 smtp_connection.quit()
74 72
75if __name__ == "__main__": 73def main():
76 # we don't do anything if we have another instance of us running 74 # we don't do anything if we have another instance of us running
77 lf = _take_lockfile() 75 lock_file = _take_lockfile()
78 76
79 if lf is None: 77 if lock_file is None:
80 if DEBUG: 78 if config.DEBUG:
81 print("Concurrent script in progress, exiting") 79 print("Concurrent script in progress, exiting")
82 sys.exit(1) 80 sys.exit(1)
83 81
@@ -88,13 +86,16 @@ if __name__ == "__main__":
88 out = None 86 out = None
89 try: 87 try:
90 out = shellutils.run_shell_cmd("%s %s" % (os.path.join(os.path.dirname(__file__), "runner.py"), next_task)) 88 out = shellutils.run_shell_cmd("%s %s" % (os.path.join(os.path.dirname(__file__), "runner.py"), next_task))
91 pass 89 except ShellCmdException as exc:
92 except ShellCmdException as e: 90 print("Failed while running the test runner: %s", exc)
93 print("Failed while running the test runner: %s", e) 91 errtext = exc.__str__()
94 errtext = e.__str__()
95 send_report(next_task, out, errtext) 92 send_report(next_task, out, errtext)
96 read_next_task_by_state(config.TASKS.INPROGRESS, next_task) 93 read_next_task_by_state(config.TASKS.INPROGRESS, next_task)
97 else: 94 else:
98 print("No task") 95 print("No task")
99 96
100 shellutils.unlockfile(lf) 97 shellutils.unlockfile(lock_file)
98
99
100if __name__ == "__main__":
101 main()
diff --git a/bitbake/lib/toaster/contrib/tts/recv.py b/bitbake/lib/toaster/contrib/tts/recv.py
index 168294acab..2faf8d08c7 100755
--- a/bitbake/lib/toaster/contrib/tts/recv.py
+++ b/bitbake/lib/toaster/contrib/tts/recv.py
@@ -26,7 +26,6 @@
26 26
27from __future__ import print_function 27from __future__ import print_function
28import sys, os, config, shellutils 28import sys, os, config, shellutils
29from shellutils import ShellCmdException
30 29
31from email.parser import Parser 30from email.parser import Parser
32 31
@@ -34,9 +33,13 @@ def recv_mail(datastring):
34 headers = Parser().parsestr(datastring) 33 headers = Parser().parsestr(datastring)
35 return headers['subject'] 34 return headers['subject']
36 35
36def main():
37 lock_file = shellutils.lockfile(shellutils.mk_lock_filename(), retry=True)
37 38
38if __name__ == "__main__": 39 if lock_file is None:
39 lf = shellutils.lockfile(shellutils.mk_lock_filename(), retry = True) 40 if config.DEBUG:
41 print("Concurrent script in progress, exiting")
42 sys.exit(1)
40 43
41 subject = recv_mail(sys.stdin.read()) 44 subject = recv_mail(sys.stdin.read())
42 45
@@ -47,5 +50,7 @@ if __name__ == "__main__":
47 line = "%s|%s\n" % (task_name, config.TASKS.PENDING) 50 line = "%s|%s\n" % (task_name, config.TASKS.PENDING)
48 fout.write(line) 51 fout.write(line)
49 52
50 shellutils.unlockfile(lf) 53 shellutils.unlockfile(lock_file)
51 54
55if __name__ == "__main__":
56 main()
diff --git a/bitbake/lib/toaster/contrib/tts/runner.py b/bitbake/lib/toaster/contrib/tts/runner.py
index e7e4dd2d2e..809fb35c5c 100755
--- a/bitbake/lib/toaster/contrib/tts/runner.py
+++ b/bitbake/lib/toaster/contrib/tts/runner.py
@@ -28,24 +28,22 @@
28 28
29 29
30from __future__ import print_function 30from __future__ import print_function
31import optparse
32import sys, os 31import sys, os
33import unittest, inspect, importlib 32import unittest, importlib
34import logging, pprint, json 33import logging, pprint, json
35 34
36from shellutils import * 35from shellutils import ShellCmdException, mkdirhier, run_shell_cmd
37 36
38import config 37import config
39 38
40# we also log to a file, in addition to console, because our output is important 39# we also log to a file, in addition to console, because our output is important
41__log_file_name =os.path.join(os.path.dirname(__file__),"log/tts_%d.log" % config.OWN_PID) 40__log_file_name__ = os.path.join(os.path.dirname(__file__), "log/tts_%d.log" % config.OWN_PID)
42mkdirhier(os.path.dirname(__log_file_name)) 41mkdirhier(os.path.dirname(__log_file_name__))
43__log_file = open(__log_file_name, "w") 42__log_file__ = open(__log_file_name__, "w")
44__file_handler = logging.StreamHandler(__log_file) 43__file_handler__ = logging.StreamHandler(__log_file__)
45__file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s")) 44__file_handler__.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
46
47config.logger.addHandler(__file_handler)
48 45
46config.logger.addHandler(__file_handler__)
49 47
50# set up log directory 48# set up log directory
51try: 49try:
@@ -54,12 +52,12 @@ try:
54 else: 52 else:
55 if not os.path.isdir(config.LOGDIR): 53 if not os.path.isdir(config.LOGDIR):
56 raise Exception("Expected log dir '%s' is not actually a directory." % config.LOGDIR) 54 raise Exception("Expected log dir '%s' is not actually a directory." % config.LOGDIR)
57except OSError as e: 55except OSError as exc:
58 raise e 56 raise exc
59 57
60# creates the under-test-branch as a separate directory 58# creates the under-test-branch as a separate directory
61def set_up_test_branch(settings, branch_name): 59def set_up_test_branch(settings, branch_name):
62 testdir = "%s/%s.%d" % (settings['workdir'], config.TEST_DIR_NAME, config.OWN_PID) 60 testdir = "%s/%s.%d" % (settings['workdir'], config.TEST_DIR_NAME, config.OWN_PID)
63 61
64 # creates the host dir 62 # creates the host dir
65 if os.path.exists(testdir): 63 if os.path.exists(testdir):
@@ -70,11 +68,11 @@ def set_up_test_branch(settings, branch_name):
70 run_shell_cmd("cp -a '%s'/.git '%s'" % (settings['localclone'], testdir)) 68 run_shell_cmd("cp -a '%s'/.git '%s'" % (settings['localclone'], testdir))
71 69
72 # add the remote if it doesn't exist 70 # add the remote if it doesn't exist
73 crt_remotes = run_shell_cmd("git remote -v", cwd = testdir) 71 crt_remotes = run_shell_cmd("git remote -v", cwd=testdir)
74 remotes = [word for line in crt_remotes.split("\n") for word in line.split()] 72 remotes = [word for line in crt_remotes.split("\n") for word in line.split()]
75 if not config.CONTRIB_REPO in remotes: 73 if not config.CONTRIB_REPO in remotes:
76 remote_name = "tts_contrib" 74 remote_name = "tts_contrib"
77 run_shell_cmd("git remote add %s %s" % (remote_name, config.CONTRIB_REPO), cwd = testdir) 75 run_shell_cmd("git remote add %s %s" % (remote_name, config.CONTRIB_REPO), cwd=testdir)
78 else: 76 else:
79 remote_name = remotes[remotes.index(config.CONTRIB_REPO) - 1] 77 remote_name = remotes[remotes.index(config.CONTRIB_REPO) - 1]
80 78
@@ -82,7 +80,7 @@ def set_up_test_branch(settings, branch_name):
82 run_shell_cmd("git fetch %s -p" % remote_name, cwd=testdir) 80 run_shell_cmd("git fetch %s -p" % remote_name, cwd=testdir)
83 81
84 # do the checkout 82 # do the checkout
85 run_shell_cmd("git checkout origin/master && git branch -D %s; git checkout %s/%s -b %s && git reset --hard" % (branch_name,remote_name,branch_name,branch_name), cwd=testdir) 83 run_shell_cmd("git checkout origin/master && git branch -D %s; git checkout %s/%s -b %s && git reset --hard" % (branch_name, remote_name, branch_name, branch_name), cwd=testdir)
86 84
87 return testdir 85 return testdir
88 86
@@ -90,15 +88,15 @@ def set_up_test_branch(settings, branch_name):
90def __search_for_tests(): 88def __search_for_tests():
91 # we find all classes that can run, and run them 89 # we find all classes that can run, and run them
92 tests = [] 90 tests = []
93 for dir_name, dirs_list, files_list in os.walk(os.path.dirname(os.path.abspath(__file__))): 91 for _, _, files_list in os.walk(os.path.dirname(os.path.abspath(__file__))):
94 for f in [f[:-3] for f in files_list if f.endswith(".py") and not f.startswith("__init__")]: 92 for module_file in [f[:-3] for f in files_list if f.endswith(".py") and not f.startswith("__init__")]:
95 config.logger.debug("Inspecting module %s", f) 93 config.logger.debug("Inspecting module %s", module_file)
96 current_module = importlib.import_module(f) 94 current_module = importlib.import_module(module_file)
97 crtclass_names = vars(current_module) 95 crtclass_names = vars(current_module)
98 for v in crtclass_names: 96 for name in crtclass_names:
99 t = crtclass_names[v] 97 tested_value = crtclass_names[name]
100 if isinstance(t, type(unittest.TestCase)) and issubclass(t, unittest.TestCase): 98 if isinstance(tested_value, type(unittest.TestCase)) and issubclass(tested_value, unittest.TestCase):
101 tests.append((f,v)) 99 tests.append((module_file, name))
102 break 100 break
103 return tests 101 return tests
104 102
@@ -117,27 +115,31 @@ def execute_tests(dir_under_test, testname):
117 os.chdir(dir_under_test) 115 os.chdir(dir_under_test)
118 116
119 # execute each module 117 # execute each module
118 # pylint: disable=broad-except
119 # we disable the broad-except because we want to actually catch all possible exceptions
120 try: 120 try:
121 config.logger.debug("Discovered test clases: %s" % pprint.pformat(tests)) 121 config.logger.debug("Discovered test clases: %s", pprint.pformat(tests))
122 suite = unittest.TestSuite() 122 suite = unittest.TestSuite()
123 loader = unittest.TestLoader() 123 loader = unittest.TestLoader()
124 result = unittest.TestResult() 124 result = unittest.TestResult()
125 for m,t in tests: 125 for module_file, name in tests:
126 suite.addTest(loader.loadTestsFromName("%s.%s" % (m,t))) 126 suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, name)))
127 config.logger.info("Running %d test(s)", suite.countTestCases()) 127 config.logger.info("Running %d test(s)", suite.countTestCases())
128 suite.run(result) 128 suite.run(result)
129 129
130 if len(result.errors) > 0: 130 for error in result.errors:
131 map(lambda x: config.logger.error("Exception on test: %s" % pprint.pformat(x)), result.errors) 131 config.logger.error("Exception on test: %s\n%s", error[0],
132 "\n".join(["-- %s" % x for x in error[1].split("\n")]))
132 133
133 if len(result.failures) > 0: 134 for failure in result.failures:
134 map(lambda x: config.logger.error("Failed test: %s:\n%s\n" % (pprint.pformat(x[0]), "\n".join(["-- %s" % x for x in eval(pprint.pformat(x[1])).split("\n")]))), result.failures) 135 config.logger.error("Failed test: %s:\n%s\n", failure[0],
136 "\n".join(["-- %s" % x for x in failure[1].split("\n")]))
135 137
136 config.logger.info("Test results: %d ran, %d errors, %d failures" % (result.testsRun, len(result.errors), len(result.failures))) 138 config.logger.info("Test results: %d ran, %d errors, %d failures", result.testsRun, len(result.errors), len(result.failures))
137 139
138 except Exception as e: 140 except Exception as exc:
139 import traceback 141 import traceback
140 config.logger.error("Exception while running test. Tracedump: \n%s", traceback.format_exc(e)) 142 config.logger.error("Exception while running test. Tracedump: \n%s", traceback.format_exc(exc))
141 finally: 143 finally:
142 os.chdir(crt_dir) 144 os.chdir(crt_dir)
143 return len(result.failures) 145 return len(result.failures)
@@ -161,17 +163,15 @@ def validate_args():
161# load the configuration options 163# load the configuration options
162def read_settings(): 164def read_settings():
163 if not os.path.exists(config.SETTINGS_FILE) or not os.path.isfile(config.SETTINGS_FILE): 165 if not os.path.exists(config.SETTINGS_FILE) or not os.path.isfile(config.SETTINGS_FILE):
164 raise Exception("Config file '%s' cannot be openend" % config.SETTINGS_FILE); 166 raise Exception("Config file '%s' cannot be openend" % config.SETTINGS_FILE)
165 return json.loads(open(config.SETTINGS_FILE, "r").read()) 167 return json.loads(open(config.SETTINGS_FILE, "r").read())
166 168
167 169
168# cleanup ! 170# cleanup !
169def clean_up(testdir): 171def clean_up(testdir):
170 # TODO: delete the test dir
171 run_shell_cmd("rm -rf -- '%s'" % testdir) 172 run_shell_cmd("rm -rf -- '%s'" % testdir)
172 pass
173 173
174if __name__ == "__main__": 174def main():
175 (options, args) = validate_args() 175 (options, args) = validate_args()
176 176
177 settings = read_settings() 177 settings = read_settings()
@@ -182,19 +182,22 @@ if __name__ == "__main__":
182 try: 182 try:
183 if options.testdir is not None and os.path.exists(options.testdir): 183 if options.testdir is not None and os.path.exists(options.testdir):
184 testdir = os.path.abspath(options.testdir) 184 testdir = os.path.abspath(options.testdir)
185 config.logger.info("No checkout, using %s" % testdir) 185 config.logger.info("No checkout, using %s", testdir)
186 else: 186 else:
187 need_cleanup = True 187 need_cleanup = True
188 testdir = set_up_test_branch(settings, args[0]) # we expect a branch name as first argument 188 testdir = set_up_test_branch(settings, args[0]) # we expect a branch name as first argument
189 189
190 config.testdir = testdir # we let tests know where to run 190 config.TESTDIR = testdir # we let tests know where to run
191 no_failures = execute_tests(testdir, options.singletest) 191 no_failures = execute_tests(testdir, options.singletest)
192 192
193 except ShellCmdException as e : 193 except ShellCmdException as exc:
194 import traceback 194 import traceback
195 config.logger.error("Error while setting up testing. Traceback: \n%s" % traceback.format_exc(e)) 195 config.logger.error("Error while setting up testing. Traceback: \n%s", traceback.format_exc(exc))
196 finally: 196 finally:
197 if need_cleanup and testdir is not None: 197 if need_cleanup and testdir is not None:
198 clean_up(testdir) 198 clean_up(testdir)
199 199
200 sys.exit(no_failures) 200 sys.exit(no_failures)
201
202if __name__ == "__main__":
203 main()
diff --git a/bitbake/lib/toaster/contrib/tts/scratchpad.py b/bitbake/lib/toaster/contrib/tts/scratchpad.py
deleted file mode 100644
index b276fb598b..0000000000
--- a/bitbake/lib/toaster/contrib/tts/scratchpad.py
+++ /dev/null
@@ -1,20 +0,0 @@
1import config
2
3# Code testing section
4def _code_test():
5 def callback_writeeventlog(opt, opt_str, value, parser):
6 if len(parser.rargs) < 1 or parser.rargs[0].startswith("-"):
7 value = ""
8 else:
9 value = parser.rargs[0]
10 del parser.rargs[0]
11
12 setattr(parser.values, opt.dest, value)
13
14 parser = optparse.OptionParser()
15 parser.add_option("-w", "--write-log", help = "Writes the event log of the build to a bitbake event json file.",
16 action = "callback", callback=callback_writeeventlog, dest = "writeeventlog")
17
18 options, targets = parser.parse_args(sys.argv)
19
20 print (options, targets)
diff --git a/bitbake/lib/toaster/contrib/tts/shellutils.py b/bitbake/lib/toaster/contrib/tts/shellutils.py
index 2b7f0f1d2e..c2012edf83 100644
--- a/bitbake/lib/toaster/contrib/tts/shellutils.py
+++ b/bitbake/lib/toaster/contrib/tts/shellutils.py
@@ -37,16 +37,16 @@ def mkdirhier(directory):
37 37
38 try: 38 try:
39 os.makedirs(directory) 39 os.makedirs(directory)
40 except OSError as e: 40 except OSError as exc:
41 if e.errno != errno.EEXIST: 41 if exc.errno != errno.EEXIST:
42 raise e 42 raise exc
43 43
44def lockfile(name, shared=False, retry=True): 44def lockfile(name, shared=False, retry=True):
45 """ 45 """
46 Use the file fn as a lock file, return when the lock has been acquired. 46 Use the file fn as a lock file, return when the lock has been acquired.
47 Returns a variable to pass to unlockfile(). 47 Returns a variable to pass to unlockfile().
48 """ 48 """
49 config.logger.debug("take lockfile %s" % name) 49 config.logger.debug("take lockfile %s", name)
50 dirname = os.path.dirname(name) 50 dirname = os.path.dirname(name)
51 mkdirhier(dirname) 51 mkdirhier(dirname)
52 52
@@ -55,11 +55,11 @@ def lockfile(name, shared=False, retry=True):
55 name) 55 name)
56 sys.exit(1) 56 sys.exit(1)
57 57
58 op = fcntl.LOCK_EX 58 operation = fcntl.LOCK_EX
59 if shared: 59 if shared:
60 op = fcntl.LOCK_SH 60 operation = fcntl.LOCK_SH
61 if not retry: 61 if not retry:
62 op = op | fcntl.LOCK_NB 62 operation = operation | fcntl.LOCK_NB
63 63
64 while True: 64 while True:
65 # If we leave the lockfiles lying around there is no problem 65 # If we leave the lockfiles lying around there is no problem
@@ -72,38 +72,40 @@ def lockfile(name, shared=False, retry=True):
72 # This implementation is unfair since the last person to request the 72 # This implementation is unfair since the last person to request the
73 # lock is the most likely to win it. 73 # lock is the most likely to win it.
74 74
75 # pylint: disable=broad-except
76 # we disable the broad-except because we want to actually catch all possible exceptions
75 try: 77 try:
76 lf = open(name, 'a+') 78 lock_file = open(name, 'a+')
77 fileno = lf.fileno() 79 fileno = lock_file.fileno()
78 fcntl.flock(fileno, op) 80 fcntl.flock(fileno, operation)
79 statinfo = os.fstat(fileno) 81 statinfo = os.fstat(fileno)
80 if os.path.exists(lf.name): 82 if os.path.exists(lock_file.name):
81 statinfo2 = os.stat(lf.name) 83 statinfo2 = os.stat(lock_file.name)
82 if statinfo.st_ino == statinfo2.st_ino: 84 if statinfo.st_ino == statinfo2.st_ino:
83 return lf 85 return lock_file
84 lf.close() 86 lock_file.close()
85 except Exception: 87 except Exception as exc:
86 try: 88 try:
87 lf.close() 89 lock_file.close()
88 except Exception: 90 except Exception as exc2:
89 pass 91 config.logger.error("Failed to close the lockfile: %s", exc2)
90 pass 92 config.logger.error("Failed to acquire the lockfile: %s", exc)
91 if not retry: 93 if not retry:
92 return None 94 return None
93 95
94def unlockfile(lf): 96def unlockfile(lock_file):
95 """ 97 """
96 Unlock a file locked using lockfile() 98 Unlock a file locked using lockfile()
97 """ 99 """
98 try: 100 try:
99 # If we had a shared lock, we need to promote to exclusive before 101 # If we had a shared lock, we need to promote to exclusive before
100 # removing the lockfile. Attempt this, ignore failures. 102 # removing the lockfile. Attempt this, ignore failures.
101 fcntl.flock(lf.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB) 103 fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
102 os.unlink(lf.name) 104 os.unlink(lock_file.name)
103 except (IOError, OSError): 105 except (IOError, OSError):
104 pass 106 pass
105 fcntl.flock(lf.fileno(), fcntl.LOCK_UN) 107 fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
106 lf.close() 108 lock_file.close()
107 109
108#ENDOFCOPY 110#ENDOFCOPY
109 111
@@ -118,20 +120,20 @@ def mk_lock_filename():
118class ShellCmdException(Exception): 120class ShellCmdException(Exception):
119 pass 121 pass
120 122
121def run_shell_cmd(command, cwd = None): 123def run_shell_cmd(command, cwd=None):
122 if cwd is None: 124 if cwd is None:
123 cwd = os.getcwd() 125 cwd = os.getcwd()
124 126
125 config.logger.debug("_shellcmd: (%s) %s" % (cwd, command)) 127 config.logger.debug("_shellcmd: (%s) %s", cwd, command)
126 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 128 process = subprocess.Popen(command, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
127 (out,err) = p.communicate() 129 (out, err) = process.communicate()
128 p.wait() 130 process.wait()
129 if p.returncode: 131 if process.returncode:
130 if len(err) == 0: 132 if len(err) == 0:
131 err = "command: %s \n%s" % (command, out) 133 err = "command: %s \n%s" % (command, out)
132 else: 134 else:
133 err = "command: %s \n%s" % (command, err) 135 err = "command: %s \n%s" % (command, err)
134 config.logger.warn("_shellcmd: error \n%s\n%s" % (out, err)) 136 config.logger.warn("_shellcmd: error \n%s\n%s", out, err)
135 raise ShellCmdException(err) 137 raise ShellCmdException(err)
136 else: 138 else:
137 #config.logger.debug("localhostbecontroller: shellcmd success\n%s" % out) 139 #config.logger.debug("localhostbecontroller: shellcmd success\n%s" % out)
diff --git a/bitbake/lib/toaster/contrib/tts/tests.py b/bitbake/lib/toaster/contrib/tts/tests.py
index 15a9a874ff..eb91947d83 100644
--- a/bitbake/lib/toaster/contrib/tts/tests.py
+++ b/bitbake/lib/toaster/contrib/tts/tests.py
@@ -24,7 +24,8 @@
24# as this file. 24# as this file.
25 25
26import unittest 26import unittest
27from shellutils import * 27from shellutils import run_shell_cmd, ShellCmdException
28import config
28 29
29import pexpect 30import pexpect
30import sys, os, signal, time 31import sys, os, signal, time
@@ -33,12 +34,9 @@ class TestPyCompilable(unittest.TestCase):
33 ''' Verifies that all Python files are syntactically correct ''' 34 ''' Verifies that all Python files are syntactically correct '''
34 def test_compile_file(self): 35 def test_compile_file(self):
35 try: 36 try:
36 out = run_shell_cmd("find . -name *py -type f -print0 | xargs -0 -n1 -P20 python -m py_compile", config.testdir) 37 run_shell_cmd("find . -name *py -type f -print0 | xargs -0 -n1 -P20 python -m py_compile", config.TESTDIR)
37 except ShellCmdException as e: 38 except ShellCmdException as exc:
38 self.fail("Error compiling python files: %s" % (e)) 39 self.fail("Error compiling python files: %s" % (exc))
39 except Exception as e:
40 self.fail("Unknown error: %s" % e)
41
42 40
43class TestPySystemStart(unittest.TestCase): 41class TestPySystemStart(unittest.TestCase):
44 ''' Attempts to start Toaster, verify that it is succesfull, and stop it ''' 42 ''' Attempts to start Toaster, verify that it is succesfull, and stop it '''
@@ -47,29 +45,28 @@ class TestPySystemStart(unittest.TestCase):
47 45
48 def test_start_interactive_mode(self): 46 def test_start_interactive_mode(self):
49 try: 47 try:
50 run_shell_cmd("bash -c 'source %s/oe-init-build-env && source toaster start webport=%d && source toaster stop'" % (config.testdir, config.TOASTER_PORT), config.testdir) 48 run_shell_cmd("bash -c 'source %s/oe-init-build-env && source toaster start webport=%d && source toaster stop'" % (config.TESTDIR, config.TOASTER_PORT), config.TESTDIR)
51 except ShellCmdException as e: 49 except ShellCmdException as exc:
52 self.fail("Failed starting interactive mode: %s" % (e)) 50 self.fail("Failed starting interactive mode: %s" % (exc))
53 51
54 def test_start_managed_mode(self): 52 def test_start_managed_mode(self):
55 try: 53 try:
56 run_shell_cmd("%s/bitbake/bin/toaster webport=%d nobrowser & sleep 10 && curl http://localhost:%d/ && kill -2 %1" % (config.testdir, config.TOASTER_PORT, config.TOASTER_PORT), config.testdir) 54 run_shell_cmd("%s/bitbake/bin/toaster webport=%d nobrowser & sleep 10 && curl http://localhost:%d/ && kill -2 %%1" % (config.TESTDIR, config.TOASTER_PORT, config.TOASTER_PORT), config.TESTDIR)
57 pass 55 except ShellCmdException as exc:
58 except ShellCmdException as e: 56 self.fail("Failed starting managed mode: %s" % (exc))
59 self.fail("Failed starting managed mode: %s" % (e))
60 57
61class TestHTML5Compliance(unittest.TestCase): 58class TestHTML5Compliance(unittest.TestCase):
62 def setUp(self): 59 def setUp(self):
63 self.origdir = os.getcwd() 60 self.origdir = os.getcwd()
64 self.crtdir = os.path.dirname(config.testdir) 61 self.crtdir = os.path.dirname(config.TESTDIR)
65 os.chdir(self.crtdir) 62 os.chdir(self.crtdir)
66 if not os.path.exists(os.path.join(self.crtdir, "toaster.sqlite")): 63 if not os.path.exists(os.path.join(self.crtdir, "toaster.sqlite")):
67 run_shell_cmd("%s/bitbake/lib/toaster/manage.py syncdb --noinput" % config.testdir) 64 run_shell_cmd("%s/bitbake/lib/toaster/manage.py syncdb --noinput" % config.TESTDIR)
68 run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate orm" % config.testdir) 65 run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate orm" % config.TESTDIR)
69 run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate bldcontrol" % config.testdir) 66 run_shell_cmd("%s/bitbake/lib/toaster/manage.py migrate bldcontrol" % config.TESTDIR)
70 run_shell_cmd("%s/bitbake/lib/toaster/manage.py loadconf %s/meta-yocto/conf/toasterconf.json" % (config.testdir, config.testdir)) 67 run_shell_cmd("%s/bitbake/lib/toaster/manage.py loadconf %s/meta-yocto/conf/toasterconf.json" % (config.TESTDIR, config.TESTDIR))
71 68
72 setup = pexpect.spawn("%s/bitbake/lib/toaster/manage.py checksettings" % config.testdir) 69 setup = pexpect.spawn("%s/bitbake/lib/toaster/manage.py checksettings" % config.TESTDIR)
73 setup.logfile = sys.stdout 70 setup.logfile = sys.stdout
74 setup.expect(r".*or type the full path to a different directory: ") 71 setup.expect(r".*or type the full path to a different directory: ")
75 setup.sendline('') 72 setup.sendline('')
@@ -79,8 +76,8 @@ class TestHTML5Compliance(unittest.TestCase):
79 setup.expect(r"Enter your option: ") 76 setup.expect(r"Enter your option: ")
80 setup.sendline('0') 77 setup.sendline('0')
81 78
82 self.child = pexpect.spawn("%s/bitbake/bin/toaster webport=%d nobrowser" % (config.testdir, config.TOASTER_PORT)) 79 self.child = pexpect.spawn("%s/bitbake/bin/toaster webport=%d nobrowser" % (config.TESTDIR, config.TOASTER_PORT))
83 self.child.logfile=sys.stdout 80 self.child.logfile = sys.stdout
84 self.child.expect("Toaster is now running. You can stop it with Ctrl-C") 81 self.child.expect("Toaster is now running. You can stop it with Ctrl-C")
85 82
86 def test_html5_compliance(self): 83 def test_html5_compliance(self):
@@ -95,7 +92,7 @@ class TestHTML5Compliance(unittest.TestCase):
95 failed.append((url, results[url])) 92 failed.append((url, results[url]))
96 93
97 94
98 self.assertTrue(len(failed)== 0, "Not all URLs validate: \n%s " % "\n".join(map(lambda x: "".join(str(x)),failed))) 95 self.assertTrue(len(failed) == 0, "Not all URLs validate: \n%s " % "\n".join(["".join(str(x)) for x in failed]))
99 96
100 #(config.TOASTER_BASEURL + url, status, errors, warnings)) 97 #(config.TOASTER_BASEURL + url, status, errors, warnings))
101 98
diff --git a/bitbake/lib/toaster/contrib/tts/urlcheck.py b/bitbake/lib/toaster/contrib/tts/urlcheck.py
index 86d7caac2a..8514de8956 100644
--- a/bitbake/lib/toaster/contrib/tts/urlcheck.py
+++ b/bitbake/lib/toaster/contrib/tts/urlcheck.py
@@ -2,40 +2,45 @@ from __future__ import print_function
2import sys 2import sys
3 3
4import httplib2 4import httplib2
5import time
6 5
7import config 6import config
8import urllist 7import urllist
9 8
10 9
11# TODO: turn to a test
12def validate_html5(url): 10def validate_html5(url):
13 h = httplib2.Http(".cache") 11 http_client = httplib2.Http(None)
14 status = "Failed" 12 status = "Failed"
15 errors = -1 13 errors = -1
16 warnings = -1 14 warnings = -1
17 15
18 # TODO: the w3c-validator must be a configurable setting
19 urlrequest = config.W3C_VALIDATOR+url 16 urlrequest = config.W3C_VALIDATOR+url
17
18 # pylint: disable=broad-except
19 # we disable the broad-except because we want to actually catch all possible exceptions
20 try: 20 try:
21 resp, content = h.request(urlrequest, "HEAD") 21 resp, _ = http_client.request(urlrequest, "HEAD")
22 if resp['x-w3c-validator-status'] != "Abort": 22 if resp['x-w3c-validator-status'] != "Abort":
23 status = resp['x-w3c-validator-status'] 23 status = resp['x-w3c-validator-status']
24 errors = int(resp['x-w3c-validator-errors']) 24 errors = int(resp['x-w3c-validator-errors'])
25 warnings = int(resp['x-w3c-validator-warnings']) 25 warnings = int(resp['x-w3c-validator-warnings'])
26 except Exception as e: 26 except Exception as exc:
27 config.logger.warn("Failed validation call: %s" % e.__str__()) 27 config.logger.warn("Failed validation call: %s", exc)
28 return (status, errors, warnings) 28 return (status, errors, warnings)
29 29
30if __name__ == "__main__":
31 print("Testing %s with %s" % (config.TOASTER_BASEURL, config.W3C_VALIDATOR))
32 30
33 def print_validation(url): 31def print_validation(url):
34 status, errors, warnings = validate_html5(url) 32 status, errors, warnings = validate_html5(url)
35 config.logger.error("url %s is %s\terrors %s warnings %s (check at %s)" % (url, status, errors, warnings, config.W3C_VALIDATOR+url)) 33 config.logger.error("url %s is %s\terrors %s warnings %s (check at %s)", url, status, errors, warnings, config.W3C_VALIDATOR+url)
34
35
36def main():
37 print("Testing %s with %s" % (config.TOASTER_BASEURL, config.W3C_VALIDATOR))
36 38
37 if len(sys.argv) > 1: 39 if len(sys.argv) > 1:
38 print_validation(sys.argv[1]) 40 print_validation(sys.argv[1])
39 else: 41 else:
40 for url in urllist.URLS: 42 for url in urllist.URLS:
41 print_validation(config.TOASTER_BASEURL+url) 43 print_validation(config.TOASTER_BASEURL+url)
44
45if __name__ == "__main__":
46 main()
diff --git a/bitbake/lib/toaster/contrib/tts/urllist.py b/bitbake/lib/toaster/contrib/tts/urllist.py
index 0226334f0b..3f3a7c2d6e 100644
--- a/bitbake/lib/toaster/contrib/tts/urllist.py
+++ b/bitbake/lib/toaster/contrib/tts/urllist.py
@@ -1,48 +1,46 @@
1import config
2
3URLS = [ 1URLS = [
4'toastergui/landing/', 2 'toastergui/landing/',
5'toastergui/builds/', 3 'toastergui/builds/',
6'toastergui/build/1', 4 'toastergui/build/1',
7'toastergui/build/1/tasks/', 5 'toastergui/build/1/tasks/',
8'toastergui/build/1/tasks/1/', 6 'toastergui/build/1/tasks/1/',
9'toastergui/build/1/task/1', 7 'toastergui/build/1/task/1',
10'toastergui/build/1/recipes/', 8 'toastergui/build/1/recipes/',
11'toastergui/build/1/recipe/1/active_tab/1', 9 'toastergui/build/1/recipe/1/active_tab/1',
12'toastergui/build/1/recipe/1', 10 'toastergui/build/1/recipe/1',
13'toastergui/build/1/recipe_packages/1', 11 'toastergui/build/1/recipe_packages/1',
14'toastergui/build/1/packages/', 12 'toastergui/build/1/packages/',
15'toastergui/build/1/package/1', 13 'toastergui/build/1/package/1',
16'toastergui/build/1/package_built_dependencies/1', 14 'toastergui/build/1/package_built_dependencies/1',
17'toastergui/build/1/package_included_detail/1/1', 15 'toastergui/build/1/package_included_detail/1/1',
18'toastergui/build/1/package_included_dependencies/1/1', 16 'toastergui/build/1/package_included_dependencies/1/1',
19'toastergui/build/1/package_included_reverse_dependencies/1/1', 17 'toastergui/build/1/package_included_reverse_dependencies/1/1',
20'toastergui/build/1/target/1', 18 'toastergui/build/1/target/1',
21'toastergui/build/1/target/1/targetpkg', 19 'toastergui/build/1/target/1/targetpkg',
22'toastergui/dentries/build/1/target/1', 20 'toastergui/dentries/build/1/target/1',
23'toastergui/build/1/target/1/dirinfo', 21 'toastergui/build/1/target/1/dirinfo',
24'toastergui/build/1/target/1/dirinfo_filepath/_/bin/bash', 22 'toastergui/build/1/target/1/dirinfo_filepath/_/bin/bash',
25'toastergui/build/1/configuration', 23 'toastergui/build/1/configuration',
26'toastergui/build/1/configvars', 24 'toastergui/build/1/configvars',
27'toastergui/build/1/buildtime', 25 'toastergui/build/1/buildtime',
28'toastergui/build/1/cpuusage', 26 'toastergui/build/1/cpuusage',
29'toastergui/build/1/diskio', 27 'toastergui/build/1/diskio',
30'toastergui/build/1/target/1/packagefile/1', 28 'toastergui/build/1/target/1/packagefile/1',
31'toastergui/newproject/', 29 'toastergui/newproject/',
32'toastergui/projects/', 30 'toastergui/projects/',
33'toastergui/project/', 31 'toastergui/project/',
34'toastergui/project/1', 32 'toastergui/project/1',
35'toastergui/project/1/configuration', 33 'toastergui/project/1/configuration',
36'toastergui/project/1/builds/', 34 'toastergui/project/1/builds/',
37'toastergui/project/1/layers/', 35 'toastergui/project/1/layers/',
38'toastergui/project/1/layer/1', 36 'toastergui/project/1/layer/1',
39'toastergui/project/1/layer/', 37 'toastergui/project/1/layer/',
40'toastergui/project/1/importlayer', 38 'toastergui/project/1/importlayer',
41'toastergui/project/1/targets/', 39 'toastergui/project/1/targets/',
42'toastergui/project/1/machines/', 40 'toastergui/project/1/machines/',
43'toastergui/xhr_configvaredit/1', 41 'toastergui/xhr_configvaredit/1',
44'toastergui/xhr_importlayer/', 42 'toastergui/xhr_importlayer/',
45'toastergui/xhr_updatelayer/', 43 'toastergui/xhr_updatelayer/',
46'toastergui/project/1/buildrequest/1', 44 'toastergui/project/1/buildrequest/1',
47'toastergui/', 45 'toastergui/',
48] 46]