Merge pull request #731 from advancedtelematic/feat/dunfell/2020.7

2020.7 on dunfell

9 files changed, 110 insertions, 54 deletions

diff --git a/classes/image_types_ostree.bbclass b/classes/image_types_ostree.bbclass
index ce9f984..fd38d28 100644
--- a/classes/image_types_ostree.bbclass
+++ b/classes/image_types_ostree.bbclass
@@ -20,14 +20,24 @@ CONVERSION_CMD_tar = "touch ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${
 CONVERSIONTYPES_append = " tar"
 
 TAR_IMAGE_ROOTFS_task-image-ostree = "${OSTREE_ROOTFS}"
+
+python prepare_ostree_rootfs() {
+    import oe.path
+    import shutil
+
+    ostree_rootfs = d.getVar("OSTREE_ROOTFS")
+    if os.path.lexists(ostree_rootfs):
+        bb.utils.remove(ostree_rootfs, True)
+
+    # Copy required as we change permissions on some files.
+    image_rootfs = d.getVar("IMAGE_ROOTFS")
+    oe.path.copyhardlinktree(image_rootfs, ostree_rootfs)
+}
+
 do_image_ostree[dirs] = "${OSTREE_ROOTFS}"
-do_image_ostree[cleandirs] = "${OSTREE_ROOTFS}"
+do_image_ostree[prefuncs] += "prepare_ostree_rootfs"
 do_image_ostree[depends] = "coreutils-native:do_populate_sysroot virtual/kernel:do_deploy ${INITRAMFS_IMAGE}:do_image_complete"
 IMAGE_CMD_ostree () {
-    cp -a ${IMAGE_ROOTFS}/* ${OSTREE_ROOTFS}
-    chmod a+rx ${OSTREE_ROOTFS}
-    sync
-
     for d in var/*; do
       if [ "${d}" != "var/local" ]; then
         rm -rf ${d}
@@ -38,9 +48,6 @@ IMAGE_CMD_ostree () {
     mkdir sysroot
     ln -sf sysroot/ostree ostree
 
-    rm -rf tmp/*
-    ln -sf sysroot/tmp tmp
-
     mkdir -p usr/rootdirs
 
     mv etc usr/
@@ -49,13 +56,11 @@ IMAGE_CMD_ostree () {
         mkdir -p usr/etc/tmpfiles.d
         tmpfiles_conf=usr/etc/tmpfiles.d/00ostree-tmpfiles.conf
         echo "d /var/rootdirs 0755 root root -" >>${tmpfiles_conf}
-        echo "L /var/rootdirs/home - - - - /sysroot/home" >>${tmpfiles_conf}
     else
         mkdir -p usr/etc/init.d
         tmpfiles_conf=usr/etc/init.d/tmpfiles.sh
         echo '#!/bin/sh' > ${tmpfiles_conf}
         echo "mkdir -p /var/rootdirs; chmod 755 /var/rootdirs" >> ${tmpfiles_conf}
-        echo "ln -sf /sysroot/home /var/rootdirs/home" >> ${tmpfiles_conf}
 
         ln -s ../init.d/tmpfiles.sh usr/etc/rcS.d/S20tmpfiles.sh
     fi
@@ -64,13 +69,11 @@ IMAGE_CMD_ostree () {
     mkdir -p usr/share/sota/
     echo -n "${OSTREE_BRANCHNAME}" > usr/share/sota/branchname
 
-    # Preserve data in /home to be later copied to /sysroot/home by sysroot
-    # generating procedure
-    mkdir -p usr/homedirs
-    if [ -d "home" ] && [ ! -L "home" ]; then
-        mv home usr/homedirs/home
-        ln -sf var/rootdirs/home home
-    fi
+    # home directories get copied from the OE root later to the final sysroot
+    # Create a symlink to var/rootdirs/home to make sure the OSTree deployment
+    # redirects /home to /var/rootdirs/home.
+    rm -rf home/
+    ln -sf var/rootdirs/home home
 
     # Move persistent directories to /var
     dirs="opt mnt media srv"
@@ -138,7 +141,7 @@ IMAGE_CMD_ostree () {
         checksum=$(sha256sum ${DEPLOY_DIR_IMAGE}/${OSTREE_KERNEL} | cut -f 1 -d " ")
         touch boot/initramfs-${checksum}
     else
-        if [ "${OSTREE_DEPLOY_DEVICETREE}" = "1"  ] && [ -n "${KERNEL_DEVICETREE}" ]; then
+        if [ ${@ oe.types.boolean('${OSTREE_DEPLOY_DEVICETREE}')} = True ] && [ -n "${KERNEL_DEVICETREE}" ]; then
             checksum=$(cat ${DEPLOY_DIR_IMAGE}/${OSTREE_KERNEL} ${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE}-${MACHINE}.${INITRAMFS_FSTYPES} ${KERNEL_DEVICETREE} | sha256sum | cut -f 1 -d " ")
             for DTS_FILE in ${KERNEL_DEVICETREE}; do
                 DTS_FILE_BASENAME=$(basename ${DTS_FILE})
@@ -165,25 +168,20 @@ IMAGE_CMD_ostreecommit () {
     fi
 
     # Commit the result
-    ostree --repo=${OSTREE_REPO} commit \
+    ostree_target_hash=$(ostree --repo=${OSTREE_REPO} commit \
            --tree=dir=${OSTREE_ROOTFS} \
            --skip-if-unchanged \
            --branch=${OSTREE_BRANCHNAME} \
            --subject="${OSTREE_COMMIT_SUBJECT}" \
            --body="${OSTREE_COMMIT_BODY}" \
            --add-metadata-string=version="${OSTREE_COMMIT_VERSION}" \
-           --bind-ref="${OSTREE_BRANCHNAME}-${IMAGE_BASENAME}"
+           ${EXTRA_OSTREE_COMMIT})
 
-    if [ "${OSTREE_UPDATE_SUMMARY}" = "1" ]; then
+    echo $ostree_target_hash > ${WORKDIR}/ostree_manifest
+
+    if [ ${@ oe.types.boolean('${OSTREE_UPDATE_SUMMARY}')} = True ]; then
         ostree --repo=${OSTREE_REPO} summary -u
     fi
-
-    # To enable simultaneous bitbaking of two images with the same branch name,
-    # create a new ref in the repo using the basename of the image. (This first
-    # requires deleting it if it already exists.) Fixes OTA-2211.
-    ostree --repo=${OSTREE_REPO} refs --delete ${OSTREE_BRANCHNAME}-${IMAGE_BASENAME}
-    ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME})
-    ostree --repo=${OSTREE_REPO} refs --create=${OSTREE_BRANCHNAME}-${IMAGE_BASENAME} ${ostree_target_hash}
 }
 
 IMAGE_TYPEDEP_ostreepush = "ostreecommit"
@@ -198,7 +196,7 @@ IMAGE_CMD_ostreepush () {
 
     if [ -n "${SOTA_PACKED_CREDENTIALS}" ]; then
         if [ -e ${SOTA_PACKED_CREDENTIALS} ]; then
-            garage-push -vv --repo=${OSTREE_REPO} \
+            garage-push --loglevel 0 --repo=${OSTREE_REPO} \
                         --ref=${OSTREE_BRANCHNAME} \
                         --credentials=${SOTA_PACKED_CREDENTIALS} \
                         --cacert=${STAGING_ETCDIR_NATIVE}/ssl/certs/ca-certificates.crt \
@@ -233,7 +231,7 @@ IMAGE_CMD_garagesign () {
                          --home-dir ${GARAGE_SIGN_REPO} \
                          --credentials ${SOTA_PACKED_CREDENTIALS}
 
-        ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME}-${IMAGE_BASENAME})
+        ostree_target_hash=$(cat ${WORKDIR}/ostree_manifest)
 
         # Use OSTree target hash as version if none was provided by the user
         target_version=${ostree_target_hash}
@@ -309,7 +307,7 @@ IMAGE_CMD_garagecheck () {
         # if credentials are issued by a server that doesn't support offline signing, exit silently
         unzip -p ${SOTA_PACKED_CREDENTIALS} root.json targets.pub targets.sec tufrepo.url 2>&1 >/dev/null || exit 0
 
-        ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME}-${IMAGE_BASENAME})
+        ostree_target_hash=$(cat ${WORKDIR}/ostree_manifest)
 
         garage-check --ref=${ostree_target_hash} \
                      --credentials=${SOTA_PACKED_CREDENTIALS} \
diff --git a/classes/image_types_ota.bbclass b/classes/image_types_ota.bbclass
index 2cc8913..a8a19b5 100644
--- a/classes/image_types_ota.bbclass
+++ b/classes/image_types_ota.bbclass
@@ -45,14 +45,17 @@ do_image_ota[cleandirs] = "${OTA_SYSROOT}"
 do_image_ota[depends] = "${@'grub:do_populate_sysroot' if d.getVar('OSTREE_BOOTLOADER') == 'grub' else ''} \
                          ${@'virtual/bootloader:do_deploy' if d.getVar('OSTREE_BOOTLOADER') == 'u-boot' else ''}"
 IMAGE_CMD_ota () {
-        export OSTREE_BOOT_PARTITION=${OSTREE_BOOT_PARTITION}
-
-        ostree admin --sysroot=${OTA_SYSROOT} init-fs ${OTA_SYSROOT}
+        ostree admin --sysroot=${OTA_SYSROOT} init-fs --modern ${OTA_SYSROOT}
         ostree admin --sysroot=${OTA_SYSROOT} os-init ${OSTREE_OSNAME}
+
+        # Preparation required to steer ostree bootloader detection
         mkdir -p ${OTA_SYSROOT}/boot/loader.0
         ln -s loader.0 ${OTA_SYSROOT}/boot/loader
 
         if [ "${OSTREE_BOOTLOADER}" = "grub" ]; then
+                # Used by ostree-grub-generator called by the ostree binary
+                export OSTREE_BOOT_PARTITION=${OSTREE_BOOT_PARTITION}
+
                 mkdir -p ${OTA_SYSROOT}/boot/grub2
                 ln -s ../loader/grub.cfg ${OTA_SYSROOT}/boot/grub2/grub.cfg
         elif [ "${OSTREE_BOOTLOADER}" = "u-boot" ]; then
@@ -61,15 +64,25 @@ IMAGE_CMD_ota () {
                 bbfatal "Invalid bootloader: ${OSTREE_BOOTLOADER}"
         fi
 
-        ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME}-${IMAGE_BASENAME})
+        ostree_target_hash=$(cat ${WORKDIR}/ostree_manifest)
 
+        # Use OSTree hash to avoid any potential race conditions between
+        # multiple builds accessing the same ${OSTREE_REPO}.
         ostree --repo=${OTA_SYSROOT}/ostree/repo pull-local --remote=${OSTREE_OSNAME} ${OSTREE_REPO} ${ostree_target_hash}
         kargs_list=""
         for arg in ${OSTREE_KERNEL_ARGS}; do
                 kargs_list="${kargs_list} --karg-append=$arg"
         done
 
-        ostree admin --sysroot=${OTA_SYSROOT} deploy ${kargs_list} --os=${OSTREE_OSNAME} ${ostree_target_hash}
+        # Create the same reference on the device we use in the archive OSTree
+        # repo in ${OSTREE_REPO}. This reference will show up when showing the
+        # deployment on the device:
+        # ostree admin status
+        # If a remote with the name ${OSTREE_OSNAME} is configured, this also
+        # will allow to use:
+        # ostree admin upgrade
+        ostree --repo=${OTA_SYSROOT}/ostree/repo refs --create=${OSTREE_OSNAME}:${OSTREE_BRANCHNAME} ${ostree_target_hash}
+        ostree admin --sysroot=${OTA_SYSROOT} deploy ${kargs_list} --os=${OSTREE_OSNAME} ${OSTREE_OSNAME}:${OSTREE_BRANCHNAME}
 
         cp -a ${IMAGE_ROOTFS}/var/sota ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
         # Create /var/sota if it doesn't exist yet
@@ -77,8 +90,11 @@ IMAGE_CMD_ota () {
         # Ensure the permissions are correctly set
         chmod 700 ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota
 
-        cp -a ${OSTREE_ROOTFS}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
-        cp -a ${OSTREE_ROOTFS}/usr/homedirs/home ${OTA_SYSROOT}/ || true
+        cp -a ${IMAGE_ROOTFS}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
+
+        mkdir -p ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/rootdirs
+        cp -a ${IMAGE_ROOTFS}/home ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/rootdirs/home || true
+
         # Ensure that /var/local exists (AGL symlinks /usr/local to /var/local)
         install -d ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/local
         # Set package version for the first deployment
diff --git a/classes/sota_sanity.bbclass b/classes/sota_sanity.bbclass
index 74973eb..02ca2e7 100644
--- a/classes/sota_sanity.bbclass
+++ b/classes/sota_sanity.bbclass
@@ -1,5 +1,12 @@
 # Sanity check the sota setup for common misconfigurations
 
+def sota_check_boolean_variable(var, d):
+    try:
+        oe.types.boolean(d.getVar(var))
+    except:
+        return False
+    return True
+
 def sota_check_overrides(status, d):
     for var in (d.getVar('SOTA_OVERRIDES_BLACKLIST') or "").split():
         if var in d.getVar('OVERRIDES').split(':'):
@@ -47,14 +54,14 @@ def sota_check_variables_validity(status, d):
         path = os.path.abspath(credentials)
         if not os.path.exists(path):
             status.addresult("SOTA_PACKED_CREDENTIALS is not set correctly. The zipped credentials file does not exist.\n")
-    if d.getVar("OSTREE_UPDATE_SUMMARY") and d.getVar("OSTREE_UPDATE_SUMMARY") not in ("0", "1", ""):
-        status.addresult("OSTREE_UPDATE_SUMMARY should be set to 0 or 1.\n")
-    if d.getVar("OSTREE_DEPLOY_DEVICETREE") and d.getVar("OSTREE_DEPLOY_DEVICETREE") not in ("0", "1", ""):
-        status.addresult("OSTREE_DEPLOY_DEVICETREE should be set to 0 or 1.\n")
-    if d.getVar("GARAGE_SIGN_AUTOVERSION") and d.getVar("GARAGE_SIGN_AUTOVERSION") not in ("0", "1", ""):
-        status.addresult("GARAGE_SIGN_AUTOVERSION should be set to 0 or 1.\n")
-    if d.getVar("SOTA_DEPLOY_CREDENTIALS") and d.getVar("SOTA_DEPLOY_CREDENTIALS") not in ("0", "1", ""):
-        status.addresult("SOTA_DEPLOY_CREDENTIALS should be set to 0 or 1.\n")
+    if not sota_check_boolean_variable("OSTREE_UPDATE_SUMMARY", d):
+        status.addresult("OSTREE_UPDATE_SUMMARY (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("OSTREE_UPDATE_SUMMARY"))
+    if not sota_check_boolean_variable("OSTREE_DEPLOY_DEVICETREE", d):
+        status.addresult("OSTREE_DEPLOY_DEVICETREE (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("OSTREE_DEPLOY_DEVICETREE"))
+    if not sota_check_boolean_variable("GARAGE_SIGN_AUTOVERSION", d):
+        status.addresult("GARAGE_SIGN_AUTOVERSION (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("GARAGE_SIGN_AUTOVERSION"))
+    if not sota_check_boolean_variable("SOTA_DEPLOY_CREDENTIALS", d):
+        status.addresult("SOTA_DEPLOY_CREDENTIALS (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("SOTA_DEPLOY_CREDENTIALS"))
     
 def sota_raise_sanity_error(msg, d):
     if d.getVar("SANITY_USE_EVENTS") == "1":
diff --git a/recipes-core/images/initramfs-ostree-image.bb b/recipes-core/images/initramfs-ostree-image.bb
index 936c59a..bc21ed3 100644
--- a/recipes-core/images/initramfs-ostree-image.bb
+++ b/recipes-core/images/initramfs-ostree-image.bb
@@ -13,7 +13,7 @@ IMAGE_LINGUAS = ""
 
 LICENSE = "MIT"
 
-IMAGE_CLASSES_remove = "image_repo_manifest"
+IMAGE_CLASSES_remove = "image_repo_manifest qemuboot"
 
 IMAGE_FSTYPES = "${INITRAMFS_FSTYPES}"
 
diff --git a/recipes-extended/collectd/collectd_%.bbappend b/recipes-extended/collectd/collectd_%.bbappend
new file mode 100644
index 0000000..fb3e6c4
--- /dev/null
+++ b/recipes-extended/collectd/collectd_%.bbappend
@@ -0,0 +1,5 @@
+do_install_append() {
+    printf "<Include \"${sysconfdir}/collectd.conf.d\">\nFilter \"*.conf\"\n</Include>\n" >> ${D}/${sysconfdir}/collectd.conf
+
+    install -d ${D}/${sysconfdir}/collectd.conf.d
+}
diff --git a/recipes-sota/aktualizr/aktualizr-collectd.bb b/recipes-sota/aktualizr/aktualizr-collectd.bb
new file mode 100644
index 0000000..c1fc717
--- /dev/null
+++ b/recipes-sota/aktualizr/aktualizr-collectd.bb
@@ -0,0 +1,21 @@
+SUMMARY = "Aktualizr metric collection"
+HOMEPAGE = "https://github.com/advancedtelematic/aktualizr"
+SECTION = "base"
+LICENSE = "MPL-2.0"
+LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MPL-2.0;md5=815ca599c9df247a0c7f619bab123dad"
+
+RDEPENDS_${PN} = "collectd"
+
+SRC_URI = " file://aktualizr-collectd.conf"
+
+S = "${WORKDIR}"
+
+do_install() {
+    install -d ${D}${sysconfdir}/collectd.conf.d
+    install -m 0644 ${WORKDIR}/aktualizr-collectd.conf ${D}${sysconfdir}/collectd.conf.d/aktualizr.conf
+}
+
+FILES_${PN} = " \
+                ${sysconfdir}/collectd.conf.d \
+                ${sysconfdir}/collectd.conf.d/aktualizr.conf \
+                "
diff --git a/recipes-sota/aktualizr/aktualizr-shared-prov.bb b/recipes-sota/aktualizr/aktualizr-shared-prov.bb
index 2ee47a1..0f712df 100644
--- a/recipes-sota/aktualizr/aktualizr-shared-prov.bb
+++ b/recipes-sota/aktualizr/aktualizr-shared-prov.bb
@@ -10,7 +10,7 @@ inherit allarch
 # We need to get the config files from the aktualizr-host-tools package built by
 # the aktualizr (target) recipe.
 DEPENDS = "aktualizr"
-RDEPENDS_${PN}_append = "${@' aktualizr-shared-prov-creds' if d.getVar('SOTA_DEPLOY_CREDENTIALS') == '1' else ''}"
+RDEPENDS_${PN}_append = "${@' aktualizr-shared-prov-creds' if oe.types.boolean(d.getVar('SOTA_DEPLOY_CREDENTIALS')) else ''}"
 
 # If the config file from aktualizr used here is changed, you will need to bump
 # the version here because of SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS!
diff --git a/recipes-sota/aktualizr/aktualizr_git.bb b/recipes-sota/aktualizr/aktualizr_git.bb
index a46eab0..56a6389 100644
--- a/recipes-sota/aktualizr/aktualizr_git.bb
+++ b/recipes-sota/aktualizr/aktualizr_git.bb
@@ -17,7 +17,7 @@ PRIVATE_LIBS_${PN}-ptest = "libaktualizr.so libaktualizr_secondary.so"
 PV = "1.0+git${SRCPV}"
 PR = "7"
 
-GARAGE_SIGN_PV = "0.7.1"
+GARAGE_SIGN_PV = "0.7.1-4-gf10c1da"
 
 SRC_URI = " \
   gitsm://github.com/advancedtelematic/aktualizr;branch=${BRANCH};name=aktualizr \
@@ -26,13 +26,13 @@ SRC_URI = " \
   file://aktualizr-secondary.service \
   file://aktualizr-serialcan.service \
   file://10-resource-control.conf \
-  ${@ d.expand("https://tuf-cli-releases.ota.here.com/cli-${GARAGE_SIGN_PV}.tgz;unpack=0;name=garagesign") if d.getVar('GARAGE_SIGN_AUTOVERSION') != '1' else ''} \
+  ${@ d.expand("https://tuf-cli-releases.ota.here.com/cli-${GARAGE_SIGN_PV}.tgz;unpack=0;name=garagesign") if not oe.types.boolean(d.getVar('GARAGE_SIGN_AUTOVERSION')) else ''} \
   "
 
-SRC_URI[garagesign.md5sum] = "febc186527b324b23c5be3affcf90e54"
-SRC_URI[garagesign.sha256sum] = "a87c3f39d61492d6f813754159ed7ef1e59966c15726edef4cd188a63cde60d6"
+SRC_URI[garagesign.md5sum] = "5f8eea81d1559d6fcb28d49c4298727c"
+SRC_URI[garagesign.sha256sum] = "8b2f5bb164f19b41972069d3377e39c2eb9edffd471777161691039e12a71738"
 
-SRCREV = "a6392dec3fb9dda3cb8ab8aa10a81b2c0494cb3c"
+SRCREV = "cf44da79555d1897115eb350cbc43db1e213db03"
 BRANCH ?= "master"
 
 S = "${WORKDIR}/git"
@@ -49,7 +49,7 @@ SYSTEMD_SERVICE_${PN}-secondary = "aktualizr-secondary.service"
 
 EXTRA_OECMAKE = "-DCMAKE_BUILD_TYPE=Release ${@bb.utils.contains('PTEST_ENABLED', '1', '-DTESTSUITE_VALGRIND=on', '', d)}"
 
-GARAGE_SIGN_OPS = "${@ d.expand('-DGARAGE_SIGN_ARCHIVE=${WORKDIR}/cli-${GARAGE_SIGN_PV}.tgz') if d.getVar('GARAGE_SIGN_AUTOVERSION') != '1' else ''}"
+GARAGE_SIGN_OPS = "${@ d.expand('-DGARAGE_SIGN_ARCHIVE=${WORKDIR}/cli-${GARAGE_SIGN_PV}.tgz') if not oe.types.boolean(d.getVar('GARAGE_SIGN_AUTOVERSION')) else ''}"
 PKCS11_ENGINE_PATH = "${libdir}/engines-1.1/pkcs11.so"
 
 PACKAGECONFIG ?= "ostree ${@bb.utils.filter('SOTA_CLIENT_FEATURES', 'hsm serialcan ubootenv', d)}"
diff --git a/recipes-sota/aktualizr/files/aktualizr-collectd.conf b/recipes-sota/aktualizr/files/aktualizr-collectd.conf
new file mode 100644
index 0000000..35a1f61
--- /dev/null
+++ b/recipes-sota/aktualizr/files/aktualizr-collectd.conf
@@ -0,0 +1,9 @@
+<LoadPlugin processes>
+        Interval 1
+</LoadPlugin>
+<Plugin processes>
+        CollectFileDescriptor true
+        CollectContextSwitch true
+        CollectMemoryMaps true
+        Process "aktualizr"
+</Plugin>