From 09ad3c88175e9009856c8303ba77b002a33cf68c Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 1 Jul 2019 17:00:23 +0200 Subject: Add ability to supply METADIR to envsetup.sh So that we can bypass the "relative path from the script" trick that doesn't work with symlinks. Signed-off-by: Laurent Bonnans --- scripts/envsetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/envsetup.sh b/scripts/envsetup.sh index ff78681..e912824 100755 --- a/scripts/envsetup.sh +++ b/scripts/envsetup.sh @@ -21,7 +21,7 @@ if [[ $SOURCED -ne 1 ]]; then exit 1 fi -METADIR="${SOURCEDIR}/../.." +METADIR=${METADIR:-${SOURCEDIR}/../..} if [[ ! -f "${BUILDDIR}/conf/local.conf" ]]; then source "$METADIR/poky/oe-init-build-env" "$BUILDDIR" -- cgit v1.2.3-54-g00ecf From 144d9b25815300fc98157ea411081e898b7457f9 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Fri, 5 Jul 2019 17:41:29 +0200 Subject: Fix aktualizr-native run in oe-selftest It did not work when using an empty build dir with sstate-cache, the proper working way is to bitbake build_sysroots. Also change the way we fetches some of the akautalizr package files which was broken too. Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/testutils.py | 25 +++++++++++-------------- lib/oeqa/selftest/cases/updater_native.py | 1 + lib/oeqa/selftest/cases/updater_qemux86_64.py | 20 ++++++++++---------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/lib/oeqa/selftest/cases/testutils.py b/lib/oeqa/selftest/cases/testutils.py index f8b1904..900d15c 100644 --- a/lib/oeqa/selftest/cases/testutils.py +++ b/lib/oeqa/selftest/cases/testutils.py @@ -1,4 +1,5 @@ import os +import oe.path import logging import re import subprocess @@ -72,20 +73,16 @@ def akt_native_run(testInst, cmd, **kwargs): # run a command supplied by aktualizr-native and checks that: # - the executable exists # - the command runs without error - # NOTE: the base test class must have built aktualizr-native (in - # setUpClass, for example) - bb_vars = get_bb_vars(['SYSROOT_DESTDIR', 'base_prefix', 'libdir', 'bindir'], - 'aktualizr-native') - sysroot = bb_vars['SYSROOT_DESTDIR'] + bb_vars['base_prefix'] - sysrootbin = bb_vars['SYSROOT_DESTDIR'] + bb_vars['bindir'] - libdir = bb_vars['libdir'] - - program, *_ = cmd.split(' ') - p = '{}/{}'.format(sysrootbin, program) - testInst.assertTrue(os.path.isfile(p), msg="No {} found ({})".format(program, p)) - env = dict(os.environ) - env['LD_LIBRARY_PATH'] = libdir - result = runCmd(cmd, env=env, native_sysroot=sysroot, ignore_status=True, **kwargs) + # + # Requirements in base test class (setUpClass for example): + # bitbake aktualizr-native + # bitbake build-sysroots -c build_native_sysroot + # + # (technique found in poky/meta/lib/oeqa/selftest/cases/package.py) + bb_vars = get_bb_vars(['STAGING_DIR', 'BUILD_ARCH']) + sysroot = oe.path.join(bb_vars['STAGING_DIR'], bb_vars['BUILD_ARCH']) + + result = runCmd(cmd, native_sysroot=sysroot, ignore_status=True, **kwargs) testInst.assertEqual(result.status, 0, "Status not equal to 0. output: %s" % result.output) diff --git a/lib/oeqa/selftest/cases/updater_native.py b/lib/oeqa/selftest/cases/updater_native.py index 1fc9cb8..d2bf341 100644 --- a/lib/oeqa/selftest/cases/updater_native.py +++ b/lib/oeqa/selftest/cases/updater_native.py @@ -14,6 +14,7 @@ class SotaToolsTests(OESelftestTestCase): logger = logging.getLogger("selftest") logger.info('Running bitbake to build aktualizr-native tools') bitbake('aktualizr-native') + bitbake('build-sysroots -c build_native_sysroot') def test_push_help(self): akt_native_run(self, 'garage-push --help') diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py index e26a022..c61c9b2 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py @@ -39,7 +39,8 @@ class AktualizrToolsTests(OESelftestTestCase): super(AktualizrToolsTests, cls).setUpClass() logger = logging.getLogger("selftest") logger.info('Running bitbake to build aktualizr-native tools') - bitbake('aktualizr-native') + bitbake('aktualizr-native aktualizr-device-prov') + bitbake('build-sysroots -c build_native_sysroot') def test_cert_provider_help(self): akt_native_run(self, 'aktualizr-cert-provider --help') @@ -47,12 +48,11 @@ class AktualizrToolsTests(OESelftestTestCase): def test_cert_provider_local_output(self): logger = logging.getLogger("selftest") logger.info('Running bitbake to build aktualizr-device-prov') - bitbake('aktualizr-device-prov') bb_vars = get_bb_vars(['SOTA_PACKED_CREDENTIALS', 'T'], 'aktualizr-native') creds = bb_vars['SOTA_PACKED_CREDENTIALS'] temp_dir = bb_vars['T'] - bb_vars_prov = get_bb_vars(['STAGING_DIR_HOST', 'libdir'], 'aktualizr-device-prov') - config = bb_vars_prov['STAGING_DIR_HOST'] + bb_vars_prov['libdir'] + '/sota/sota-device-cred.toml' + bb_vars_prov = get_bb_vars(['WORKDIR', 'libdir'], 'aktualizr-device-prov') + config = bb_vars_prov['WORKDIR'] + '/sysroot-destdir' + bb_vars_prov['libdir'] + '/sota/conf.d/20-sota-device-cred.toml' akt_native_run(self, 'aktualizr-cert-provider -c {creds} -r -l {temp} -g {config}' .format(creds=creds, temp=temp_dir, config=config)) @@ -173,8 +173,8 @@ class DeviceCredProvTests(OESelftestTestCase): self.append_config('MACHINE = "qemux86-64"') self.append_config('SOTA_CLIENT_PROV = " aktualizr-device-prov "') self.append_config('SOTA_DEPLOY_CREDENTIALS = "0"') - runCmd('bitbake -c cleanall aktualizr aktualizr-device-prov') self.qemu, self.s = qemu_launch(machine='qemux86-64') + bitbake('build-sysroots -c build_native_sysroot') def tearDownLocal(self): qemu_terminate(self.s) @@ -201,8 +201,8 @@ class DeviceCredProvTests(OESelftestTestCase): # Run aktualizr-cert-provider. bb_vars = get_bb_vars(['SOTA_PACKED_CREDENTIALS'], 'aktualizr-native') creds = bb_vars['SOTA_PACKED_CREDENTIALS'] - bb_vars_prov = get_bb_vars(['STAGING_DIR_HOST', 'libdir'], 'aktualizr-device-prov') - config = bb_vars_prov['STAGING_DIR_HOST'] + bb_vars_prov['libdir'] + '/sota/sota-device-cred.toml' + bb_vars_prov = get_bb_vars(['WORKDIR', 'libdir'], 'aktualizr-device-prov') + config = bb_vars_prov['WORKDIR'] + '/sysroot-destdir' + bb_vars_prov['libdir'] + '/sota/conf.d/20-sota-device-cred.toml' print('Provisining at root@localhost:%d' % self.qemu.ssh_port) akt_native_run(self, 'aktualizr-cert-provider -c {creds} -t root@localhost -p {port} -s -u -r -g {config}' @@ -231,8 +231,8 @@ class DeviceCredProvHsmTests(OESelftestTestCase): self.append_config('SOTA_DEPLOY_CREDENTIALS = "0"') self.append_config('SOTA_CLIENT_FEATURES = "hsm"') self.append_config('IMAGE_INSTALL_append = " softhsm-testtoken"') - runCmd('bitbake -c cleanall aktualizr aktualizr-device-prov-hsm') self.qemu, self.s = qemu_launch(machine='qemux86-64') + bitbake('build-sysroots -c build_native_sysroot') def tearDownLocal(self): qemu_terminate(self.s) @@ -269,8 +269,8 @@ class DeviceCredProvHsmTests(OESelftestTestCase): # Run aktualizr-cert-provider. bb_vars = get_bb_vars(['SOTA_PACKED_CREDENTIALS'], 'aktualizr-native') creds = bb_vars['SOTA_PACKED_CREDENTIALS'] - bb_vars_prov = get_bb_vars(['STAGING_DIR_HOST', 'libdir'], 'aktualizr-device-prov-hsm') - config = bb_vars_prov['STAGING_DIR_HOST'] + bb_vars_prov['libdir'] + '/sota/sota-device-cred-hsm.toml' + bb_vars_prov = get_bb_vars(['WORKDIR', 'libdir'], 'aktualizr-device-prov-hsm') + config = bb_vars_prov['WORKDIR'] + '/sysroot-destdir' + bb_vars_prov['libdir'] + '/sota/conf.d/20-sota-device-cred-hsm.toml' akt_native_run(self, 'aktualizr-cert-provider -c {creds} -t root@localhost -p {port} -r -s -u -g {config}' .format(creds=creds, port=self.qemu.ssh_port, config=config)) -- cgit v1.2.3-54-g00ecf From 4ab468bc0cb76f5d13b24af35e0d7746a0c035ad Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 8 Jul 2019 11:28:34 +0200 Subject: Factor out ugly function in oe-selftests Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/testutils.py | 10 +++++ lib/oeqa/selftest/cases/updater_minnowboard.py | 12 ++---- lib/oeqa/selftest/cases/updater_qemux86_64.py | 47 ++++------------------ .../selftest/cases/updater_qemux86_64_ptest.py | 9 +---- lib/oeqa/selftest/cases/updater_raspberrypi.py | 13 +++--- 5 files changed, 28 insertions(+), 63 deletions(-) diff --git a/lib/oeqa/selftest/cases/testutils.py b/lib/oeqa/selftest/cases/testutils.py index 900d15c..e67f30c 100644 --- a/lib/oeqa/selftest/cases/testutils.py +++ b/lib/oeqa/selftest/cases/testutils.py @@ -69,6 +69,16 @@ def qemu_send_command(port, command, timeout=60): return stdout, stderr, s2.returncode +def metadir(): + # Assume the directory layout for finding other layers. We could also + # make assumptions by using 'show-layers', but either way, if the + # layers we need aren't where we expect them, we are out of luck. + path = os.path.abspath(os.path.dirname(__file__)) + metadir = path + "/../../../../../" + + return metadir + + def akt_native_run(testInst, cmd, **kwargs): # run a command supplied by aktualizr-native and checks that: # - the executable exists diff --git a/lib/oeqa/selftest/cases/updater_minnowboard.py b/lib/oeqa/selftest/cases/updater_minnowboard.py index 267445b..855bd20 100644 --- a/lib/oeqa/selftest/cases/updater_minnowboard.py +++ b/lib/oeqa/selftest/cases/updater_minnowboard.py @@ -1,9 +1,8 @@ -import os import re from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, get_bb_var -from testutils import qemu_launch, qemu_send_command, qemu_terminate, verifyProvisioned +from testutils import metadir, qemu_launch, qemu_send_command, qemu_terminate, verifyProvisioned class MinnowTests(OESelftestTestCase): @@ -12,18 +11,13 @@ class MinnowTests(OESelftestTestCase): layer_intel = "meta-intel" layer_minnow = "meta-updater-minnowboard" result = runCmd('bitbake-layers show-layers') - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of luck. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" if re.search(layer_intel, result.output) is None: - self.meta_intel = metadir + layer_intel + self.meta_intel = metadir() + layer_intel runCmd('bitbake-layers add-layer "%s"' % self.meta_intel) else: self.meta_intel = None if re.search(layer_minnow, result.output) is None: - self.meta_minnow = metadir + layer_minnow + self.meta_minnow = metadir() + layer_minnow runCmd('bitbake-layers add-layer "%s"' % self.meta_minnow) else: self.meta_minnow = None diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py index c61c9b2..7228529 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py @@ -9,7 +9,8 @@ from uuid import uuid4 from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars from testutils import qemu_launch, qemu_send_command, qemu_terminate, \ - akt_native_run, verifyNotProvisioned, verifyProvisioned, qemu_bake_image, qemu_boot_image + metadir, akt_native_run, verifyNotProvisioned, verifyProvisioned, \ + qemu_bake_image, qemu_boot_image class GeneralTests(OESelftestTestCase): @@ -46,8 +47,6 @@ class AktualizrToolsTests(OESelftestTestCase): akt_native_run(self, 'aktualizr-cert-provider --help') def test_cert_provider_local_output(self): - logger = logging.getLogger("selftest") - logger.info('Running bitbake to build aktualizr-device-prov') bb_vars = get_bb_vars(['SOTA_PACKED_CREDENTIALS', 'T'], 'aktualizr-native') creds = bb_vars['SOTA_PACKED_CREDENTIALS'] temp_dir = bb_vars['T'] @@ -75,12 +74,7 @@ class SharedCredProvTests(OESelftestTestCase): layer = "meta-updater-qemux86-64" result = runCmd('bitbake-layers show-layers') if re.search(layer, result.output) is None: - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of luck. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" - self.meta_qemu = metadir + layer + self.meta_qemu = metadir() + layer runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu) else: self.meta_qemu = None @@ -117,12 +111,7 @@ class ManualControlTests(OESelftestTestCase): layer = "meta-updater-qemux86-64" result = runCmd('bitbake-layers show-layers') if re.search(layer, result.output) is None: - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of like. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" - self.meta_qemu = metadir + layer + self.meta_qemu = metadir() + layer runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu) else: self.meta_qemu = None @@ -161,12 +150,7 @@ class DeviceCredProvTests(OESelftestTestCase): layer = "meta-updater-qemux86-64" result = runCmd('bitbake-layers show-layers') if re.search(layer, result.output) is None: - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of luck. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" - self.meta_qemu = metadir + layer + self.meta_qemu = metadir() + layer runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu) else: self.meta_qemu = None @@ -217,12 +201,7 @@ class DeviceCredProvHsmTests(OESelftestTestCase): layer = "meta-updater-qemux86-64" result = runCmd('bitbake-layers show-layers') if re.search(layer, result.output) is None: - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of luck. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" - self.meta_qemu = metadir + layer + self.meta_qemu = metadir() + layer runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu) else: self.meta_qemu = None @@ -399,12 +378,7 @@ class IpSecondaryTests(OESelftestTestCase): layer = "meta-updater-qemux86-64" result = runCmd('bitbake-layers show-layers') if re.search(layer, result.output) is None: - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of luck. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" - self.meta_qemu = metadir + layer + self.meta_qemu = metadir() + layer runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu) else: self.meta_qemu = None @@ -446,12 +420,7 @@ class ResourceControlTests(OESelftestTestCase): layer = "meta-updater-qemux86-64" result = runCmd('bitbake-layers show-layers') if re.search(layer, result.output) is None: - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of luck. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" - self.meta_qemu = metadir + layer + self.meta_qemu = metadir() + layer runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu) else: self.meta_qemu = None diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py index 0f0f491..88e773b 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py @@ -4,7 +4,7 @@ import re from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd -from testutils import qemu_launch, qemu_send_command, qemu_terminate +from testutils import metadir, qemu_launch, qemu_send_command, qemu_terminate class PtestTests(OESelftestTestCase): @@ -13,12 +13,7 @@ class PtestTests(OESelftestTestCase): layer = "meta-updater-qemux86-64" result = runCmd('bitbake-layers show-layers') if re.search(layer, result.output) is None: - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of like. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" - self.meta_qemu = metadir + layer + self.meta_qemu = metadir() + layer runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu) else: self.meta_qemu = None diff --git a/lib/oeqa/selftest/cases/updater_raspberrypi.py b/lib/oeqa/selftest/cases/updater_raspberrypi.py index f6ae903..26d5c4c 100644 --- a/lib/oeqa/selftest/cases/updater_raspberrypi.py +++ b/lib/oeqa/selftest/cases/updater_raspberrypi.py @@ -7,6 +7,8 @@ import unittest from oeqa.selftest.case import OESelftestTestCase from oeqa.utils.commands import runCmd, bitbake, get_bb_var +from testutils import metadir + class RpiTests(OESelftestTestCase): @@ -17,23 +19,18 @@ class RpiTests(OESelftestTestCase): layer_rpi = "meta-raspberrypi" layer_upd_rpi = "meta-updater-raspberrypi" result = runCmd('bitbake-layers show-layers') - # Assume the directory layout for finding other layers. We could also - # make assumptions by using 'show-layers', but either way, if the - # layers we need aren't where we expect them, we are out of luck. - path = os.path.abspath(os.path.dirname(__file__)) - metadir = path + "/../../../../../" if re.search(layer_python, result.output) is None: - self.meta_python = metadir + layer_python + self.meta_python = metadir() + layer_python runCmd('bitbake-layers add-layer "%s"' % self.meta_python) else: self.meta_python = None if re.search(layer_rpi, result.output) is None: - self.meta_rpi = metadir + layer_rpi + self.meta_rpi = metadir() + layer_rpi runCmd('bitbake-layers add-layer "%s"' % self.meta_rpi) else: self.meta_rpi = None if re.search(layer_upd_rpi, result.output) is None: - self.meta_upd_rpi = metadir + layer_upd_rpi + self.meta_upd_rpi = metadir() + layer_upd_rpi runCmd('bitbake-layers add-layer "%s"' % self.meta_upd_rpi) else: self.meta_upd_rpi = None -- cgit v1.2.3-54-g00ecf From 8837fe8267d8eb108aecab9ebc7ea8a1270a9f24 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Tue, 9 Jul 2019 12:06:22 +0200 Subject: Fix some oe-selftest timeouts issues Was causing problems on CI Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/testutils.py | 2 +- lib/oeqa/selftest/cases/updater_qemux86_64.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/oeqa/selftest/cases/testutils.py b/lib/oeqa/selftest/cases/testutils.py index e67f30c..208f822 100644 --- a/lib/oeqa/selftest/cases/testutils.py +++ b/lib/oeqa/selftest/cases/testutils.py @@ -61,7 +61,7 @@ def qemu_bake_image(imagename): bitbake(imagename) -def qemu_send_command(port, command, timeout=60): +def qemu_send_command(port, command, timeout=120): command = ['ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@localhost -p ' + str(port) + ' "' + command + '"'] s2 = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py index 7228529..033b224 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py @@ -2,6 +2,7 @@ import os import logging import re +import subprocess import unittest from time import sleep from uuid import uuid4 @@ -307,8 +308,8 @@ class IpSecondaryTests(OESelftestTestCase): self.configure() qemu_bake_image(self.imagename) - def send_command(self, cmd): - stdout, stderr, retcode = qemu_send_command(self.qemu.ssh_port, cmd, timeout=60) + def send_command(self, cmd, timeout=60): + stdout, stderr, retcode = qemu_send_command(self.qemu.ssh_port, cmd, timeout=timeout) return str(stdout), str(stderr), retcode def __enter__(self): @@ -323,7 +324,7 @@ class IpSecondaryTests(OESelftestTestCase): def wait_till_sshable(self): # qemu_send_command tries to ssh into the qemu VM and blocks until it gets there or timeout happens # so it helps us to block q control flow until the VM is booted and a target binary/daemon is running there - self.stdout, self.stderr, self.retcode = self.send_command(self.binaryname + ' --help') + self.stdout, self.stderr, self.retcode = self.send_command(self.binaryname + ' --help', timeout=300) def was_successfully_booted(self): return self.retcode == 0 @@ -445,10 +446,13 @@ class ResourceControlTests(OESelftestTestCase): ran_ok = False for delay in [5, 5, 5, 5]: sleep(delay) - stdout, stderr, retcode = self.qemu_command('systemctl --no-pager show aktualizr') - if retcode == 0 and b'ExecMainStatus=9' in stdout: - ran_ok = True - break + try: + stdout, stderr, retcode = self.qemu_command('systemctl --no-pager show aktualizr') + if retcode == 0 and b'ExecMainStatus=9' in stdout: + ran_ok = True + break + except subprocess.TimeoutExpired: + pass self.assertTrue(ran_ok, 'Aktualizr was not killed') self.assertIn(b'CPUWeight=1000', stdout, 'CPUWeight was not set correctly') -- cgit v1.2.3-54-g00ecf From 57fc9c149b8659305ec507226d9a1e0238dc2ae3 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Tue, 9 Jul 2019 14:33:13 +0200 Subject: Do not push ostree images during most of oe-selftest One failure point less Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/updater_qemux86_64.py | 9 +++++++++ lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py | 1 + 2 files changed, 10 insertions(+) diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py index 033b224..81c2e5e 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py @@ -19,6 +19,9 @@ class GeneralTests(OESelftestTestCase): logger = logging.getLogger("selftest") logger.info('Running bitbake to build core-image-minimal') self.append_config('SOTA_CLIENT_PROV = "aktualizr-shared-prov"') + + # note: this also tests ostreepush/garagesign/garagecheck which are + # omitted from other test cases bitbake('core-image-minimal') credentials = get_bb_var('SOTA_PACKED_CREDENTIALS') # skip the test if the variable SOTA_PACKED_CREDENTIALS is not set @@ -81,6 +84,7 @@ class SharedCredProvTests(OESelftestTestCase): self.meta_qemu = None self.append_config('MACHINE = "qemux86-64"') self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "') + self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') self.qemu, self.s = qemu_launch(machine='qemux86-64') def tearDownLocal(self): @@ -119,6 +123,7 @@ class ManualControlTests(OESelftestTestCase): self.append_config('MACHINE = "qemux86-64"') self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "') self.append_config('SYSTEMD_AUTO_ENABLE_aktualizr = "disable"') + self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') self.qemu, self.s = qemu_launch(machine='qemux86-64') def tearDownLocal(self): @@ -158,6 +163,7 @@ class DeviceCredProvTests(OESelftestTestCase): self.append_config('MACHINE = "qemux86-64"') self.append_config('SOTA_CLIENT_PROV = " aktualizr-device-prov "') self.append_config('SOTA_DEPLOY_CREDENTIALS = "0"') + self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') self.qemu, self.s = qemu_launch(machine='qemux86-64') bitbake('build-sysroots -c build_native_sysroot') @@ -211,6 +217,7 @@ class DeviceCredProvHsmTests(OESelftestTestCase): self.append_config('SOTA_DEPLOY_CREDENTIALS = "0"') self.append_config('SOTA_CLIENT_FEATURES = "hsm"') self.append_config('IMAGE_INSTALL_append = " softhsm-testtoken"') + self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') self.qemu, self.s = qemu_launch(machine='qemux86-64') bitbake('build-sysroots -c build_native_sysroot') @@ -384,6 +391,7 @@ class IpSecondaryTests(OESelftestTestCase): else: self.meta_qemu = None + self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') self.primary = IpSecondaryTests.Primary(self) self.secondary = IpSecondaryTests.Secondary(self) @@ -427,6 +435,7 @@ class ResourceControlTests(OESelftestTestCase): self.meta_qemu = None self.append_config('MACHINE = "qemux86-64"') self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "') + self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') self.append_config('IMAGE_INSTALL_append += " aktualizr-resource-control "') self.append_config('RESOURCE_CPU_WEIGHT_pn-aktualizr = "1000"') self.append_config('RESOURCE_MEMORY_HIGH_pn-aktualizr = "50M"') diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py index 88e773b..83f5841 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py @@ -21,6 +21,7 @@ class PtestTests(OESelftestTestCase): self.append_config('SYSTEMD_AUTO_ENABLE_aktualizr = "disable"') self.append_config('PTEST_ENABLED_pn-aktualizr = "1"') self.append_config('IMAGE_INSTALL_append += "aktualizr-ptest ptest-runner "') + self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') self.qemu, self.s = qemu_launch(machine='qemux86-64') def tearDownLocal(self): -- cgit v1.2.3-54-g00ecf From 2cfbf9bd383994580ac2bf6cf44b7b9119689486 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Thu, 11 Jul 2019 11:28:15 +0200 Subject: Fix race condition in oe-selftest HSM test Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/updater_qemux86_64.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py index 81c2e5e..80d135f 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py @@ -263,18 +263,17 @@ class DeviceCredProvHsmTests(OESelftestTestCase): .format(creds=creds, port=self.qemu.ssh_port, config=config)) # Verify that HSM is able to initialize. - ran_ok = False for delay in [5, 5, 5, 5, 10]: sleep(delay) p11_out, p11_err, p11_ret = self.qemu_command(pkcs11_command) hsm_out, hsm_err, hsm_ret = self.qemu_command(softhsm2_command) - if p11_ret == 0 and hsm_ret == 0 and hsm_err == b'': - ran_ok = True + if (p11_ret == 0 and hsm_ret == 0 and hsm_err == b'' and + b'X.509 cert' in p11_out and b'present token' in p11_err): break - self.assertTrue(ran_ok, 'pkcs11-tool or softhsm2-tool failed: ' + p11_err.decode() + - p11_out.decode() + hsm_err.decode() + hsm_out.decode()) - self.assertIn(b'present token', p11_err, 'pkcs11-tool failed: ' + p11_err.decode() + p11_out.decode()) - self.assertIn(b'X.509 cert', p11_out, 'pkcs11-tool failed: ' + p11_err.decode() + p11_out.decode()) + else: + self.fail('pkcs11-tool or softhsm2-tool failed: ' + p11_err.decode() + + p11_out.decode() + hsm_err.decode() + hsm_out.decode()) + self.assertIn(b'Initialized: yes', hsm_out, 'softhsm2-tool failed: ' + hsm_err.decode() + hsm_out.decode()) self.assertIn(b'User PIN init.: yes', hsm_out, 'softhsm2-tool failed: ' + -- cgit v1.2.3-54-g00ecf From aaee0a8f502e588a0a161d62f7393bf616e01526 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 15 Jul 2019 15:37:43 +0200 Subject: Use 128M by default for qemu in oe-selftests Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/testutils.py | 1 + lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py | 3 +-- scripts/qemucommand.py | 9 ++++++--- scripts/run-qemu-ota | 1 + 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/oeqa/selftest/cases/testutils.py b/lib/oeqa/selftest/cases/testutils.py index 208f822..8d618a6 100644 --- a/lib/oeqa/selftest/cases/testutils.py +++ b/lib/oeqa/selftest/cases/testutils.py @@ -34,6 +34,7 @@ def qemu_boot_image(imagename, **kwargs): args.dir = 'tmp/deploy/images' args.efi = kwargs.get('efi', False) args.machine = kwargs.get('machine', None) + args.mem = kwargs.get('mem', '128M') qemu_use_kvm = get_bb_var("QEMU_USE_KVM") if qemu_use_kvm and \ (qemu_use_kvm == 'True' and 'x86' in args.machine or diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py index 83f5841..fe09371 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py @@ -1,5 +1,4 @@ # pylint: disable=C0111,C0325 -import os import re from oeqa.selftest.case import OESelftestTestCase @@ -22,7 +21,7 @@ class PtestTests(OESelftestTestCase): self.append_config('PTEST_ENABLED_pn-aktualizr = "1"') self.append_config('IMAGE_INSTALL_append += "aktualizr-ptest ptest-runner "') self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"') - self.qemu, self.s = qemu_launch(machine='qemux86-64') + self.qemu, self.s = qemu_launch(machine='qemux86-64', mem="768M") def tearDownLocal(self): qemu_terminate(self.s) diff --git a/scripts/qemucommand.py b/scripts/qemucommand.py index 1049d7c..3045b45 100644 --- a/scripts/qemucommand.py +++ b/scripts/qemucommand.py @@ -2,7 +2,7 @@ from os.path import exists, join, realpath, abspath from os import listdir import random import socket -from subprocess import check_output, CalledProcessError +from subprocess import check_output EXTENSIONS = { 'intel-corei7-64': 'wic', @@ -68,6 +68,10 @@ class QemuCommand(object): self.mac_address = random_mac() self.serial_port = find_local_port(8990) self.ssh_port = find_local_port(2222) + if args.mem: + self.mem = args.mem + else: + self.mem = "1G" if args.kvm is None: # Autodetect KVM using 'kvm-ok' try: @@ -95,7 +99,7 @@ class QemuCommand(object): cmdline += ["-drive", "file=%s,if=ide,format=raw,snapshot=on" % self.image] cmdline += [ "-serial", "tcp:127.0.0.1:%d,server,nowait" % self.serial_port, - "-m", "1G", + "-m", self.mem, "-usb", "-object", "rng-random,id=rng0,filename=/dev/urandom", "-device", "virtio-rng-pci,rng=rng0", @@ -131,4 +135,3 @@ class QemuCommand(object): "-f", "qcow2", self.overlay] return cmdline - diff --git a/scripts/run-qemu-ota b/scripts/run-qemu-ota index b2f55e9..de63297 100755 --- a/scripts/run-qemu-ota +++ b/scripts/run-qemu-ota @@ -26,6 +26,7 @@ def main(): dest='kvm', action='store_true', default=None) kvm_group.add_argument('--no-kvm', help='Disable KVM in QEMU', dest='kvm', action='store_false') + parser.add_argument('--mem', default=None, help="Amount of memory the machine boots with") parser.add_argument('--no-gui', help='Disable GUI', action='store_true') parser.add_argument('--gdb', help='Export gdbserver port 2159 from the image', action='store_true') parser.add_argument('--pcap', default=None, help='Dump all network traffic') -- cgit v1.2.3-54-g00ecf From da4fbf300ee8c166aa77298d8ce2f3d794909478 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Wed, 17 Jul 2019 09:39:52 +0200 Subject: Avoid unicode decode error in ptest The full suite logs often contains invalid utf8 sequences. Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py index fe09371..d20a9f0 100644 --- a/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py +++ b/lib/oeqa/selftest/cases/updater_qemux86_64_ptest.py @@ -41,7 +41,7 @@ class PtestTests(OESelftestTestCase): if has_failure: print("Full test suite log:") stdout, _, _ = self.qemu_command('cat /tmp/aktualizr-ptest.log || cat /tmp/aktualizr-ptest.log.tmp', timeout=None) - print(stdout.decode()) + print(stdout.decode(errors='replace')) self.assertEqual(retcode, 0) self.assertFalse(has_failure) -- cgit v1.2.3-54-g00ecf From c42c0dbd59cb21b77d01e0c0f7d7d2e9434ae408 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Fri, 19 Jul 2019 15:41:21 +0200 Subject: Give more memory for minnowboard selftest Signed-off-by: Laurent Bonnans --- lib/oeqa/selftest/cases/updater_minnowboard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/oeqa/selftest/cases/updater_minnowboard.py b/lib/oeqa/selftest/cases/updater_minnowboard.py index 855bd20..f4da360 100644 --- a/lib/oeqa/selftest/cases/updater_minnowboard.py +++ b/lib/oeqa/selftest/cases/updater_minnowboard.py @@ -24,7 +24,7 @@ class MinnowTests(OESelftestTestCase): self.append_config('MACHINE = "intel-corei7-64"') self.append_config('OSTREE_BOOTLOADER = "grub"') self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "') - self.qemu, self.s = qemu_launch(efi=True, machine='intel-corei7-64') + self.qemu, self.s = qemu_launch(efi=True, machine='intel-corei7-64', mem='512M') def tearDownLocal(self): qemu_terminate(self.s) -- cgit v1.2.3-54-g00ecf From ba1c931dde251637f2718e33532f7e469954470f Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Wed, 10 Jul 2019 17:08:10 +0200 Subject: Get CI jobs from master rpi and minnowboard are enabled here by default Signed-off-by: Laurent Bonnans --- .gitlab-ci.yml | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..6ad00ea --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,91 @@ +stages: + - docker + - checkout + - test + +variables: + BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-bitbake + BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-checkout + +include: + - project: 'olp/edge/ota/connect/client/meta-updater' + ref: 'master' + file: 'scripts/ci/gitlab/docker.yml' + - project: 'olp/edge/ota/connect/client/meta-updater' + ref: 'master' + file: 'scripts/ci/gitlab/checkout.yml' + - project: 'olp/edge/ota/connect/client/meta-updater' + ref: 'master' + file: 'scripts/ci/gitlab/tests.yml' + +Docker setup: + extends: .bb_docker_remote + + stage: docker + except: + - pushes + +Checkout: + extends: .bb_checkout + + stage: checkout + variables: + MANIFEST: thud + CURRENT_PROJECT: meta-updater + except: + - pushes + +Build core-image-minimal: + extends: .bitbake + + stage: test + variables: + TEST_BUILD_DIR: 'build-core-image-minimal' + BITBAKE_TARGETS: 'core-image-minimal' + except: + - pushes + +Oe-selftest qemux86_64: + extends: .oe-selftest + + stage: test + variables: + TEST_BUILD_DIR: 'build-oe-qemux86_64' + OE_SELFTESTS: 'updater_native updater_qemux86_64' + except: + - pushes + +Oe-selftest minnowboard: + extends: .oe-selftest + + stage: test + variables: + TEST_BUILD_DIR: 'build-oe-minnowboard' + OE_SELFTESTS: 'updater_minnowboard' + except: + - pushes + +Oe-selftest rpi: + extends: .oe-selftest + + stage: test + variables: + TEST_BUILD_DIR: 'build-oe-rpi' + OE_SELFTESTS: 'updater_raspberrypi' + except: + - pushes + +# Not run by default + +Ptest qemux86_64: + extends: .oe-selftest + + stage: test + variables: + TEST_BUILD_DIR: 'build-oe-qemux86_64-ptest' + OE_SELFTESTS: 'updater_qemux86_64_ptest' + except: + - pushes + only: + variables: + - $OE_PTEST -- cgit v1.2.3-54-g00ecf