From dfc54a477bca7773d075041156897867cfc8ad67 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Mon, 19 Oct 2020 13:50:19 +0100 Subject: oeqa: Add sync call to command execution We previously put a sync call into devtool to try and combat the bitbake timeout issues on the autobuilder. It isn't enough as the timeouts occur mid test. They are also occurring on non-devtool tests. Add in sync calls around command execution instead. (From OE-Core rev: ceca5ed121e2b54415a7ab3a217882e4ea86923a) Signed-off-by: Richard Purdie --- meta/lib/oeqa/selftest/cases/devtool.py | 7 ------- meta/lib/oeqa/selftest/cases/runcmd.py | 16 ++++++++-------- meta/lib/oeqa/utils/commands.py | 8 +++++++- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/meta/lib/oeqa/selftest/cases/devtool.py b/meta/lib/oeqa/selftest/cases/devtool.py index 0185e670ad..d3d2e04c20 100644 --- a/meta/lib/oeqa/selftest/cases/devtool.py +++ b/meta/lib/oeqa/selftest/cases/devtool.py @@ -107,13 +107,6 @@ class DevtoolBase(OESelftestTestCase): 'under the build directory') self.append_config(self.sstate_conf) - def tearDown(self): - # devtools tests are heavy on IO and if bitbake can't write out its caches, we see timeouts. - # call sync around the tests to ensure the IO queue doesn't get too large, taking any IO - # hit here rather than in bitbake shutdown. - super().tearDown() - os.system("sync") - def _check_src_repo(self, repo_dir): """Check srctree git repository""" self.assertTrue(os.path.isdir(os.path.join(repo_dir, '.git')), diff --git a/meta/lib/oeqa/selftest/cases/runcmd.py b/meta/lib/oeqa/selftest/cases/runcmd.py index a5ef1ea95f..fa6113d7fa 100644 --- a/meta/lib/oeqa/selftest/cases/runcmd.py +++ b/meta/lib/oeqa/selftest/cases/runcmd.py @@ -64,12 +64,12 @@ class RunCmdTests(OESelftestTestCase): runCmd, "echo foobar >&2; false", shell=True, assert_error=False) def test_output(self): - result = runCmd("echo stdout; echo stderr >&2", shell=True) + result = runCmd("echo stdout; echo stderr >&2", shell=True, sync=False) self.assertEqual("stdout\nstderr", result.output) self.assertEqual("", result.error) def test_output_split(self): - result = runCmd("echo stdout; echo stderr >&2", shell=True, stderr=subprocess.PIPE) + result = runCmd("echo stdout; echo stderr >&2", shell=True, stderr=subprocess.PIPE, sync=False) self.assertEqual("stdout", result.output) self.assertEqual("stderr", result.error) @@ -77,7 +77,7 @@ class RunCmdTests(OESelftestTestCase): numthreads = threading.active_count() start = time.time() # Killing a hanging process only works when not using a shell?! - result = runCmd(['sleep', '60'], timeout=self.TIMEOUT, ignore_status=True) + result = runCmd(['sleep', '60'], timeout=self.TIMEOUT, ignore_status=True, sync=False) self.assertEqual(result.status, -signal.SIGTERM) end = time.time() self.assertLess(end - start, self.TIMEOUT + self.DELTA) @@ -87,7 +87,7 @@ class RunCmdTests(OESelftestTestCase): numthreads = threading.active_count() start = time.time() # Killing a hanging process only works when not using a shell?! - result = runCmd(['sleep', '60'], timeout=self.TIMEOUT, ignore_status=True, stderr=subprocess.PIPE) + result = runCmd(['sleep', '60'], timeout=self.TIMEOUT, ignore_status=True, stderr=subprocess.PIPE, sync=False) self.assertEqual(result.status, -signal.SIGTERM) end = time.time() self.assertLess(end - start, self.TIMEOUT + self.DELTA) @@ -95,7 +95,7 @@ class RunCmdTests(OESelftestTestCase): def test_stdin(self): numthreads = threading.active_count() - result = runCmd("cat", data=b"hello world", timeout=self.TIMEOUT) + result = runCmd("cat", data=b"hello world", timeout=self.TIMEOUT, sync=False) self.assertEqual("hello world", result.output) self.assertEqual(numthreads, threading.active_count(), msg="Thread counts were not equal before (%s) and after (%s), active threads: %s" % (numthreads, threading.active_count(), threading.enumerate())) self.assertEqual(numthreads, 1) @@ -103,7 +103,7 @@ class RunCmdTests(OESelftestTestCase): def test_stdin_timeout(self): numthreads = threading.active_count() start = time.time() - result = runCmd(['sleep', '60'], data=b"hello world", timeout=self.TIMEOUT, ignore_status=True) + result = runCmd(['sleep', '60'], data=b"hello world", timeout=self.TIMEOUT, ignore_status=True, sync=False) self.assertEqual(result.status, -signal.SIGTERM) end = time.time() self.assertLess(end - start, self.TIMEOUT + self.DELTA) @@ -111,12 +111,12 @@ class RunCmdTests(OESelftestTestCase): def test_log(self): log = MemLogger() - result = runCmd("echo stdout; echo stderr >&2", shell=True, output_log=log) + result = runCmd("echo stdout; echo stderr >&2", shell=True, output_log=log, sync=False) self.assertEqual(["Running: echo stdout; echo stderr >&2", "stdout", "stderr"], log.info_msgs) self.assertEqual([], log.error_msgs) def test_log_split(self): log = MemLogger() - result = runCmd("echo stdout; echo stderr >&2", shell=True, output_log=log, stderr=subprocess.PIPE) + result = runCmd("echo stdout; echo stderr >&2", shell=True, output_log=log, stderr=subprocess.PIPE, sync=False) self.assertEqual(["Running: echo stdout; echo stderr >&2", "stdout"], log.info_msgs) self.assertEqual(["stderr"], log.error_msgs) diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index f7f8c16bf0..8059cbce3e 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -167,7 +167,7 @@ class Result(object): pass -def runCmd(command, ignore_status=False, timeout=None, assert_error=True, +def runCmd(command, ignore_status=False, timeout=None, assert_error=True, sync=True, native_sysroot=None, limit_exc_output=0, output_log=None, **options): result = Result() @@ -184,6 +184,12 @@ def runCmd(command, ignore_status=False, timeout=None, assert_error=True, cmd = Command(command, timeout=timeout, output_log=output_log, **options) cmd.run() + # tests can be heavy on IO and if bitbake can't write out its caches, we see timeouts. + # call sync around the tests to ensure the IO queue doesn't get too large, taking any IO + # hit here rather than in bitbake shutdown. + if sync: + os.system("sync") + result.command = command result.status = cmd.status result.output = cmd.output -- cgit v1.2.3-54-g00ecf