Merge pull request #634 from advancedtelematic/feat/thud/2019.10

Feat/thud/2019.10

14 files changed, 136 insertions, 51 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6ad00ea..6a7ae47 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -45,6 +45,17 @@ Build core-image-minimal:
   except:
     - pushes
 
+Build core-image-minimal (rpi):
+  extends: .bitbake
+
+  stage: test
+  variables:
+    TEST_BUILD_DIR: 'build-core-image-minimal-rpi'
+    BITBAKE_TARGETS: 'core-image-minimal'
+    TEST_MACHINE: 'raspberrypi3'
+  except:
+    - pushes
+
 Oe-selftest qemux86_64:
   extends: .oe-selftest
 
diff --git a/classes/image_types_ostree.bbclass b/classes/image_types_ostree.bbclass
index 7ffe99d..0f0fd28 100644
--- a/classes/image_types_ostree.bbclass
+++ b/classes/image_types_ostree.bbclass
@@ -13,11 +13,11 @@ BUILD_OSTREE_TARBALL ??= "1"
 SYSTEMD_USED = "${@oe.utils.ifelse(d.getVar('VIRTUAL-RUNTIME_init_manager') == 'systemd', 'true', '')}"
 
 IMAGE_CMD_TAR = "tar --xattrs --xattrs-include=*"
-CONVERSION_CMD_tar = "touch ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}; ${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.tar -C ${OTA_IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+CONVERSION_CMD_tar = "touch ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}; ${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.tar -C ${TAR_IMAGE_ROOTFS} . || [ $? -eq 1 ]"
 CONVERSIONTYPES_append = " tar"
 
 REQUIRED_DISTRO_FEATURES = "usrmerge"
-OTA_IMAGE_ROOTFS_task-image-ostree = "${OSTREE_ROOTFS}"
+TAR_IMAGE_ROOTFS_task-image-ostree = "${OSTREE_ROOTFS}"
 do_image_ostree[dirs] = "${OSTREE_ROOTFS}"
 do_image_ostree[cleandirs] = "${OSTREE_ROOTFS}"
 do_image_ostree[depends] = "coreutils-native:do_populate_sysroot virtual/kernel:do_deploy ${INITRAMFS_IMAGE}:do_image_complete"
@@ -186,13 +186,20 @@ IMAGE_CMD_ostreecommit () {
 IMAGE_TYPEDEP_ostreepush = "ostreecommit"
 do_image_ostreepush[depends] += "aktualizr-native:do_populate_sysroot ca-certificates-native:do_populate_sysroot"
 IMAGE_CMD_ostreepush () {
-    # Print warnings if credetials are not set or if the file has not been found.
+    # send a copy of the repo manifest to backend if available
+    local SEND_MANIFEST=""
+    # check if garage-push supports the --repo-manifest option before trying
+    if $(garage-push --help | grep -q '^\s*--repo-manifest') && [ -f ${IMAGE_ROOTFS}${sysconfdir}/manifest.xml ]; then
+        SEND_MANIFEST="--repo-manifest ${IMAGE_ROOTFS}${sysconfdir}/manifest.xml"
+    fi
+
     if [ -n "${SOTA_PACKED_CREDENTIALS}" ]; then
         if [ -e ${SOTA_PACKED_CREDENTIALS} ]; then
             garage-push -vv --repo=${OSTREE_REPO} \
                         --ref=${OSTREE_BRANCHNAME} \
                         --credentials=${SOTA_PACKED_CREDENTIALS} \
-                        --cacert=${STAGING_ETCDIR_NATIVE}/ssl/certs/ca-certificates.crt
+                        --cacert=${STAGING_ETCDIR_NATIVE}/ssl/certs/ca-certificates.crt \
+                        $SEND_MANIFEST
         else
             bbwarn "SOTA_PACKED_CREDENTIALS file does not exist."
         fi
diff --git a/classes/image_types_ota.bbclass b/classes/image_types_ota.bbclass
index 894919d..857161a 100644
--- a/classes/image_types_ota.bbclass
+++ b/classes/image_types_ota.bbclass
@@ -38,7 +38,7 @@ calculate_size () {
 }
 
 OTA_SYSROOT = "${WORKDIR}/ota-sysroot"
-OTA_IMAGE_ROOTFS_task-image-ota = "${OTA_SYSROOT}"
+TAR_IMAGE_ROOTFS_task-image-ota = "${OTA_SYSROOT}"
 IMAGE_TYPEDEP_ota = "ostreecommit"
 do_image_ota[dirs] = "${OTA_SYSROOT}"
 do_image_ota[cleandirs] = "${OTA_SYSROOT}"
diff --git a/conf/distro/poky-sota-systemd.conf b/conf/distro/poky-sota-systemd.conf
index b30b322..0dc50b0 100644
--- a/conf/distro/poky-sota-systemd.conf
+++ b/conf/distro/poky-sota-systemd.conf
@@ -2,7 +2,7 @@ require conf/distro/poky.conf
 
 require conf/distro/sota.conf.inc
 
-DISTRO = "poky-sota"
+DISTRO = "poky-sota-systemd"
 DISTRO_NAME = "OTA-enabled Linux"
 DISTRO_VERSION = "1.0"
 DISTRO_CODENAME = "sota"
diff --git a/conf/distro/poky-sota.conf b/conf/distro/poky-sota.conf
index 3fb1d20..bfac90f 100644
--- a/conf/distro/poky-sota.conf
+++ b/conf/distro/poky-sota.conf
@@ -5,3 +5,5 @@ DISTRO = "poky-sota"
 DISTRO_NAME = "OTA-enabled Linux"
 DISTRO_VERSION = "1.0"
 DISTRO_CODENAME = "sota"
+
+IMAGE_INSTALL_append_sota = " ostree-booted"
diff --git a/conf/local.conf.base.append b/conf/local.conf.base.append
new file mode 100644
index 0000000..36b2f59
--- /dev/null
+++ b/conf/local.conf.base.append
@@ -0,0 +1,18 @@
+#
+# meta-updater configuration, see README.adoc and aktualizr's
+# documentation for more options and detailed documentation
+#
+
+MACHINE = "##MACHINE##"
+DISTRO = "##DISTRO##"
+
+# General SOTA setup
+#SOTA_CLIENT_PROV = "aktualizr-shared-prov"
+#SOTA_PACKED_CREDENTIALS = "/path/to/credentials.zip"
+
+# Uncomment this line to start an ssh server at boot automatically
+#IMAGE_FEATURES += "ssh-server-dropbear"
+
+# Uncomment this line to set the log level of aktualizr to 'debug' (from 'info'
+# by default)
+#IMAGE_INSTALL_append += " aktualizr-log-debug"
diff --git a/conf/local.conf.sample.append b/conf/local.conf.systemd.append
index fc565f8..12e0182 100644
--- a/conf/local.conf.sample.append
+++ b/conf/local.conf.systemd.append
@@ -1,23 +1,3 @@
-
-#
-# meta-updater configuration, see README.adoc and aktualizr's
-# documentation for more options and detailed documentation
-#
-
-MACHINE = "##MACHINE##"
-DISTRO = "poky-sota-systemd"
-
-# General SOTA setup
-#SOTA_CLIENT_PROV = "aktualizr-shared-prov"
-#SOTA_PACKED_CREDENTIALS = "/path/to/credentials.zip"
-
-# Uncomment this line to start an ssh server at boot automatically
-#IMAGE_FEATURES += "ssh-server-dropbear"
-
-# Uncomment this line to set the log level of aktualizr to 'debug' (from 'info'
-# by default)
-#IMAGE_INSTALL_append += " aktualizr-log-debug"
-
 # Store systemd logs in persistent storage
 #
 # It greatly helps diagnosing issues on testing devices but should be
diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py
index 4506300..5f152ad 100644
--- a/lib/oeqa/selftest/cases/updater_qemux86_64.py
+++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py
@@ -465,4 +465,40 @@ class ResourceControlTests(OESelftestTestCase):
         stdout, stderr, retcode = self.qemu_command('systemctl --no-pager show --property=ExecMainStatus aktualizr')
         self.assertIn(b'ExecMainStatus=0', stdout, 'Aktualizr did not restart')
 
+
+class NonSystemdTests(OESelftestTestCase):
+    def setUpLocal(self):
+        layer = "meta-updater-qemux86-64"
+        result = runCmd('bitbake-layers show-layers')
+        if re.search(layer, result.output) is None:
+            self.meta_qemu = metadir() + layer
+            runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu)
+        else:
+            self.meta_qemu = None
+        self.append_config('MACHINE = "qemux86-64"')
+        self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "')
+        self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"')
+        self.append_config('DISTRO = "poky-sota"')
+        self.append_config('IMAGE_INSTALL_remove += " aktualizr-resource-control"')
+        self.qemu, self.s = qemu_launch(machine='qemux86-64')
+
+    def tearDownLocal(self):
+        qemu_terminate(self.s)
+        if self.meta_qemu:
+            runCmd('bitbake-layers remove-layer "%s"' % self.meta_qemu, ignore_status=True)
+
+    def qemu_command(self, command):
+        return qemu_send_command(self.qemu.ssh_port, command)
+
+    def test_provisioning(self):
+        print('Checking if systemd is not installed...')
+        stdout, stderr, retcode = self.qemu_command('systemctl')
+        self.assertTrue(retcode != 0, 'systemd is installed while it is not supposed to: ' + str(stdout))
+
+        stdout, stderr, retcode = self.qemu_command('aktualizr --run-mode once')
+        self.assertEqual(retcode, 0, 'Failed to run aktualizr: ' + str(stdout) + str(stderr))
+
+        machine = get_bb_var('MACHINE', 'core-image-minimal')
+        verifyProvisioned(self, machine)
+
 # vim:set ts=4 sw=4 sts=4 expandtab:
diff --git a/lib/oeqa/selftest/cases/updater_raspberrypi.py b/lib/oeqa/selftest/cases/updater_raspberrypi.py
index 26d5c4c..25c5f12 100644
--- a/lib/oeqa/selftest/cases/updater_raspberrypi.py
+++ b/lib/oeqa/selftest/cases/updater_raspberrypi.py
@@ -35,26 +35,10 @@ class RpiTests(OESelftestTestCase):
         else:
             self.meta_upd_rpi = None
 
-        # This is trickier that I would've thought. The fundamental problem is
-        # that the qemu layer changes the u-boot file extension to .rom, but
-        # raspberrypi still expects .bin. To prevent this, the qemu layer must
-        # be temporarily removed if it is present. It has to be removed by name
-        # without the complete path, but to add it back when we are done, we
-        # need the full path.
-        p = re.compile(r'meta-updater-qemux86-64\s*(\S*meta-updater-qemux86-64)\s')
-        m = p.search(result.output)
-        if m and m.lastindex > 0:
-            self.meta_qemu = m.group(1)
-            runCmd('bitbake-layers remove-layer meta-updater-qemux86-64')
-        else:
-            self.meta_qemu = None
-
         self.append_config('MACHINE = "raspberrypi3"')
         self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "')
 
     def tearDownLocal(self):
-        if self.meta_qemu:
-            runCmd('bitbake-layers add-layer "%s"' % self.meta_qemu, ignore_status=True)
         if self.meta_upd_rpi:
             runCmd('bitbake-layers remove-layer "%s"' % self.meta_upd_rpi, ignore_status=True)
         if self.meta_rpi:
diff --git a/recipes-connectivity/networkd-dhcp-conf/networkd-dhcp-conf.bb b/recipes-connectivity/networkd-dhcp-conf/networkd-dhcp-conf.bb
index 394531e..40a3958 100644
--- a/recipes-connectivity/networkd-dhcp-conf/networkd-dhcp-conf.bb
+++ b/recipes-connectivity/networkd-dhcp-conf/networkd-dhcp-conf.bb
@@ -15,7 +15,7 @@ SRC_URI = " \
   "
 PR = "r1"
 
-RDEPENDS_${PN} = "systemd"
+REQUIRED_DISTRO_FEATURES_${PN} = "systemd"
 RCONFLICTS_${PN} = "connman"
 
 S = "${WORKDIR}"
diff --git a/recipes-sota/aktualizr/aktualizr_git.bb b/recipes-sota/aktualizr/aktualizr_git.bb
index 0427c3c..6cec05a 100644
--- a/recipes-sota/aktualizr/aktualizr_git.bb
+++ b/recipes-sota/aktualizr/aktualizr_git.bb
@@ -15,7 +15,7 @@ RDEPENDS_${PN}-ptest += "bash cmake curl python3-misc python3-modules openssl-bi
 PV = "1.0+git${SRCPV}"
 PR = "7"
 
-GARAGE_SIGN_PV = "0.7.0-49-g5ffd420"
+GARAGE_SIGN_PV = "0.7.0-59-gf6013d6"
 
 SRC_URI = " \
   gitsm://github.com/advancedtelematic/aktualizr;branch=${BRANCH};name=aktualizr \
@@ -27,10 +27,10 @@ SRC_URI = " \
   ${@ d.expand("https://ats-tuf-cli-releases.s3-eu-central-1.amazonaws.com/cli-${GARAGE_SIGN_PV}.tgz;unpack=0;name=garagesign") if d.getVar('GARAGE_SIGN_AUTOVERSION') != '1' else ''} \
   "
 
-SRC_URI[garagesign.md5sum] = "de0877ecb693fd48ec11052e51b0ff1a"
-SRC_URI[garagesign.sha256sum] = "cf25759574c9c1206835daeaf6fc345f6db7b5ccdb95fb828c86d7451f78f0aa"
+SRC_URI[garagesign.md5sum] = "3d38908f9b536a02cc73778b11bbc32e"
+SRC_URI[garagesign.sha256sum] = "eceeb16a781e0e8d1f554defbcd5bbcea86d448ebd350fc6a2529372bf867dba"
 
-SRCREV = "fa59e33208d3b1dc690a30ce8339b3b4162f8022"
+SRCREV = "662aa23f0b9c97a2c93a428438dacae72333c751"
 BRANCH ?= "master"
 
 S = "${WORKDIR}/git"
@@ -55,7 +55,6 @@ PACKAGECONFIG[warning-as-error] = "-DWARNING_AS_ERROR=ON,-DWARNING_AS_ERROR=OFF,
 PACKAGECONFIG[ostree] = "-DBUILD_OSTREE=ON,-DBUILD_OSTREE=OFF,ostree,"
 PACKAGECONFIG[hsm] = "-DBUILD_P11=ON,-DBUILD_P11=OFF,libp11,"
 PACKAGECONFIG[sota-tools] = "-DBUILD_SOTA_TOOLS=ON ${GARAGE_SIGN_OPS},-DBUILD_SOTA_TOOLS=OFF,glib-2.0,"
-PACKAGECONFIG[systemd] = "-DBUILD_SYSTEMD=ON,-DBUILD_SYSTEMD=OFF,systemd,"
 PACKAGECONFIG[load-tests] = "-DBUILD_LOAD_TESTS=ON,-DBUILD_LOAD_TESTS=OFF,"
 PACKAGECONFIG[serialcan] = ",,,slcand-start"
 PACKAGECONFIG[ubootenv] = ",,,u-boot-fw-utils aktualizr-uboot-env-rollback"
diff --git a/recipes-sota/ostree/files/touch-ostree b/recipes-sota/ostree/files/touch-ostree
new file mode 100755
index 0000000..28cb672
--- /dev/null
+++ b/recipes-sota/ostree/files/touch-ostree
@@ -0,0 +1,21 @@
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides:          touch-ostree
+# Required-Start:    $network $remote_fs
+# Required-Stop:     $network $remote_fs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Indicate OSTree boot
+### END INIT INFO
+
+case "$1" in
+  start)
+        touch /run/ostree-booted
+        ;;
+  stop)
+        ;;
+  *)
+        echo "Usage: /etc/init.d/touch-ostree {start|stop}"
+        exit 1
+        ;;
+esac
diff --git a/recipes-sota/ostree/ostree-booted_1.0.bb b/recipes-sota/ostree/ostree-booted_1.0.bb
new file mode 100644
index 0000000..d74cf24
--- /dev/null
+++ b/recipes-sota/ostree/ostree-booted_1.0.bb
@@ -0,0 +1,15 @@
+SUMMARY = "Indicate an OSTree boot"
+DESCRIPTION =  "Indicate an OSTree boot"
+LICENSE = "MPL-2.0"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MPL-2.0;md5=815ca599c9df247a0c7f619bab123dad"
+SRC_URI = "file://touch-ostree"
+
+inherit allarch update-rc.d
+
+INITSCRIPT_NAME = "touch-ostree"
+INITSCRIPT_PARAMS = "start 8 2 3 4 5 . stop 20 0 1 6 ."
+
+do_install() {
+        install -d ${D}${sysconfdir}/init.d
+        install -m 0755 ${WORKDIR}/touch-ostree ${D}${sysconfdir}/init.d/touch-ostree
+}
diff --git a/scripts/envsetup.sh b/scripts/envsetup.sh
index 5827bc2..19a5c94 100755
--- a/scripts/envsetup.sh
+++ b/scripts/envsetup.sh
@@ -3,9 +3,13 @@
 SCRIPT="envsetup.sh"
 MACHINE="$1"
 BUILDDIR="build"
+DISTRO="poky-sota-systemd"
+BASE_CONF="local.conf.base.append"
+declare -A supported_distros=( ["poky-sota-systemd"]="local.conf.systemd.append" ["poky-sota"]="local.conf.base.append" )
 
-[[ "$#" -lt 1 ]] && { echo "Usage: ${SCRIPT} <machine> [builddir]"; return 1; }
-[[ "$#" -eq 2 ]] && { BUILDDIR="$2"; }
+[[ "$#" -lt 1 ]] && { echo "Usage: ${SCRIPT} <machine> [builddir] [distro=< poky-sota-systemd | poky-sota >]"; return 1; }
+[[ "$#" -ge 2 ]] && { BUILDDIR="$2"; }
+[[ "$#" -eq 3 ]] && { DISTRO="$3"; }
 
 # detect if this script is sourced: see http://stackoverflow.com/a/38128348/6255594
 SOURCED=0
@@ -22,6 +26,8 @@ if [[ $SOURCED -ne 1 ]]; then
 fi
 
 METADIR=${METADIR:-${SOURCEDIR}/../..}
+DISTRO_CONF=${supported_distros[$DISTRO]}
+[[ -n $DISTRO_CONF ]] && { echo "Using $DISTRO_CONF for the specified distro $DISTRO"; } || { echo "The specified distro $DISTRO is not supported"; return 1; }
 
 if [[ ! -f "${BUILDDIR}/conf/local.conf" ]]; then
   source "$METADIR/poky/oe-init-build-env" "$BUILDDIR"
@@ -30,7 +36,13 @@ if [[ ! -f "${BUILDDIR}/conf/local.conf" ]]; then
   cat "${METADIR}/meta-updater/conf/include/bblayers/sota.inc" >> conf/bblayers.conf
   cat "${METADIR}/meta-updater/conf/include/bblayers/sota_${MACHINE}.inc" >> conf/bblayers.conf
 
-  sed "s/##MACHINE##/$MACHINE/g" "${METADIR}/meta-updater/conf/local.conf.sample.append" >> conf/local.conf
+  sed -e "s/##MACHINE##/$MACHINE/g" \
+      -e "s/##DISTRO##/$DISTRO/g" \
+         "${METADIR}/meta-updater/conf/$BASE_CONF" >> conf/local.conf
+
+  if [ "$BASE_CONF" != "$DISTRO_CONF" ]; then
+    cat "${METADIR}/meta-updater/conf/$DISTRO_CONF" >> conf/local.conf
+  fi
 else
   source "$METADIR/poky/oe-init-build-env" "$BUILDDIR"
 fi