Merge pull request #731 from advancedtelematic/feat/dunfell/2020.7

2020.7 on dunfell

3 files changed, 67 insertions, 46 deletions

diff --git a/classes/image_types_ostree.bbclass b/classes/image_types_ostree.bbclass
index ce9f984..fd38d28 100644
--- a/classes/image_types_ostree.bbclass
+++ b/classes/image_types_ostree.bbclass
@@ -20,14 +20,24 @@ CONVERSION_CMD_tar = "touch ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${
 CONVERSIONTYPES_append = " tar"
 
 TAR_IMAGE_ROOTFS_task-image-ostree = "${OSTREE_ROOTFS}"
+
+python prepare_ostree_rootfs() {
+    import oe.path
+    import shutil
+
+    ostree_rootfs = d.getVar("OSTREE_ROOTFS")
+    if os.path.lexists(ostree_rootfs):
+        bb.utils.remove(ostree_rootfs, True)
+
+    # Copy required as we change permissions on some files.
+    image_rootfs = d.getVar("IMAGE_ROOTFS")
+    oe.path.copyhardlinktree(image_rootfs, ostree_rootfs)
+}
+
 do_image_ostree[dirs] = "${OSTREE_ROOTFS}"
-do_image_ostree[cleandirs] = "${OSTREE_ROOTFS}"
+do_image_ostree[prefuncs] += "prepare_ostree_rootfs"
 do_image_ostree[depends] = "coreutils-native:do_populate_sysroot virtual/kernel:do_deploy ${INITRAMFS_IMAGE}:do_image_complete"
 IMAGE_CMD_ostree () {
-    cp -a ${IMAGE_ROOTFS}/* ${OSTREE_ROOTFS}
-    chmod a+rx ${OSTREE_ROOTFS}
-    sync
-
     for d in var/*; do
       if [ "${d}" != "var/local" ]; then
         rm -rf ${d}
@@ -38,9 +48,6 @@ IMAGE_CMD_ostree () {
     mkdir sysroot
     ln -sf sysroot/ostree ostree
 
-    rm -rf tmp/*
-    ln -sf sysroot/tmp tmp
-
     mkdir -p usr/rootdirs
 
     mv etc usr/
@@ -49,13 +56,11 @@ IMAGE_CMD_ostree () {
         mkdir -p usr/etc/tmpfiles.d
         tmpfiles_conf=usr/etc/tmpfiles.d/00ostree-tmpfiles.conf
         echo "d /var/rootdirs 0755 root root -" >>${tmpfiles_conf}
-        echo "L /var/rootdirs/home - - - - /sysroot/home" >>${tmpfiles_conf}
     else
         mkdir -p usr/etc/init.d
         tmpfiles_conf=usr/etc/init.d/tmpfiles.sh
         echo '#!/bin/sh' > ${tmpfiles_conf}
         echo "mkdir -p /var/rootdirs; chmod 755 /var/rootdirs" >> ${tmpfiles_conf}
-        echo "ln -sf /sysroot/home /var/rootdirs/home" >> ${tmpfiles_conf}
 
         ln -s ../init.d/tmpfiles.sh usr/etc/rcS.d/S20tmpfiles.sh
     fi
@@ -64,13 +69,11 @@ IMAGE_CMD_ostree () {
     mkdir -p usr/share/sota/
     echo -n "${OSTREE_BRANCHNAME}" > usr/share/sota/branchname
 
-    # Preserve data in /home to be later copied to /sysroot/home by sysroot
-    # generating procedure
-    mkdir -p usr/homedirs
-    if [ -d "home" ] && [ ! -L "home" ]; then
-        mv home usr/homedirs/home
-        ln -sf var/rootdirs/home home
-    fi
+    # home directories get copied from the OE root later to the final sysroot
+    # Create a symlink to var/rootdirs/home to make sure the OSTree deployment
+    # redirects /home to /var/rootdirs/home.
+    rm -rf home/
+    ln -sf var/rootdirs/home home
 
     # Move persistent directories to /var
     dirs="opt mnt media srv"
@@ -138,7 +141,7 @@ IMAGE_CMD_ostree () {
         checksum=$(sha256sum ${DEPLOY_DIR_IMAGE}/${OSTREE_KERNEL} | cut -f 1 -d " ")
         touch boot/initramfs-${checksum}
     else
-        if [ "${OSTREE_DEPLOY_DEVICETREE}" = "1"  ] && [ -n "${KERNEL_DEVICETREE}" ]; then
+        if [ ${@ oe.types.boolean('${OSTREE_DEPLOY_DEVICETREE}')} = True ] && [ -n "${KERNEL_DEVICETREE}" ]; then
             checksum=$(cat ${DEPLOY_DIR_IMAGE}/${OSTREE_KERNEL} ${DEPLOY_DIR_IMAGE}/${INITRAMFS_IMAGE}-${MACHINE}.${INITRAMFS_FSTYPES} ${KERNEL_DEVICETREE} | sha256sum | cut -f 1 -d " ")
             for DTS_FILE in ${KERNEL_DEVICETREE}; do
                 DTS_FILE_BASENAME=$(basename ${DTS_FILE})
@@ -165,25 +168,20 @@ IMAGE_CMD_ostreecommit () {
     fi
 
     # Commit the result
-    ostree --repo=${OSTREE_REPO} commit \
+    ostree_target_hash=$(ostree --repo=${OSTREE_REPO} commit \
            --tree=dir=${OSTREE_ROOTFS} \
            --skip-if-unchanged \
            --branch=${OSTREE_BRANCHNAME} \
            --subject="${OSTREE_COMMIT_SUBJECT}" \
            --body="${OSTREE_COMMIT_BODY}" \
            --add-metadata-string=version="${OSTREE_COMMIT_VERSION}" \
-           --bind-ref="${OSTREE_BRANCHNAME}-${IMAGE_BASENAME}"
+           ${EXTRA_OSTREE_COMMIT})
 
-    if [ "${OSTREE_UPDATE_SUMMARY}" = "1" ]; then
+    echo $ostree_target_hash > ${WORKDIR}/ostree_manifest
+
+    if [ ${@ oe.types.boolean('${OSTREE_UPDATE_SUMMARY}')} = True ]; then
         ostree --repo=${OSTREE_REPO} summary -u
     fi
-
-    # To enable simultaneous bitbaking of two images with the same branch name,
-    # create a new ref in the repo using the basename of the image. (This first
-    # requires deleting it if it already exists.) Fixes OTA-2211.
-    ostree --repo=${OSTREE_REPO} refs --delete ${OSTREE_BRANCHNAME}-${IMAGE_BASENAME}
-    ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME})
-    ostree --repo=${OSTREE_REPO} refs --create=${OSTREE_BRANCHNAME}-${IMAGE_BASENAME} ${ostree_target_hash}
 }
 
 IMAGE_TYPEDEP_ostreepush = "ostreecommit"
@@ -198,7 +196,7 @@ IMAGE_CMD_ostreepush () {
 
     if [ -n "${SOTA_PACKED_CREDENTIALS}" ]; then
         if [ -e ${SOTA_PACKED_CREDENTIALS} ]; then
-            garage-push -vv --repo=${OSTREE_REPO} \
+            garage-push --loglevel 0 --repo=${OSTREE_REPO} \
                         --ref=${OSTREE_BRANCHNAME} \
                         --credentials=${SOTA_PACKED_CREDENTIALS} \
                         --cacert=${STAGING_ETCDIR_NATIVE}/ssl/certs/ca-certificates.crt \
@@ -233,7 +231,7 @@ IMAGE_CMD_garagesign () {
                          --home-dir ${GARAGE_SIGN_REPO} \
                          --credentials ${SOTA_PACKED_CREDENTIALS}
 
-        ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME}-${IMAGE_BASENAME})
+        ostree_target_hash=$(cat ${WORKDIR}/ostree_manifest)
 
         # Use OSTree target hash as version if none was provided by the user
         target_version=${ostree_target_hash}
@@ -309,7 +307,7 @@ IMAGE_CMD_garagecheck () {
         # if credentials are issued by a server that doesn't support offline signing, exit silently
         unzip -p ${SOTA_PACKED_CREDENTIALS} root.json targets.pub targets.sec tufrepo.url 2>&1 >/dev/null || exit 0
 
-        ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME}-${IMAGE_BASENAME})
+        ostree_target_hash=$(cat ${WORKDIR}/ostree_manifest)
 
         garage-check --ref=${ostree_target_hash} \
                      --credentials=${SOTA_PACKED_CREDENTIALS} \
diff --git a/classes/image_types_ota.bbclass b/classes/image_types_ota.bbclass
index 2cc8913..a8a19b5 100644
--- a/classes/image_types_ota.bbclass
+++ b/classes/image_types_ota.bbclass
@@ -45,14 +45,17 @@ do_image_ota[cleandirs] = "${OTA_SYSROOT}"
 do_image_ota[depends] = "${@'grub:do_populate_sysroot' if d.getVar('OSTREE_BOOTLOADER') == 'grub' else ''} \
                          ${@'virtual/bootloader:do_deploy' if d.getVar('OSTREE_BOOTLOADER') == 'u-boot' else ''}"
 IMAGE_CMD_ota () {
-        export OSTREE_BOOT_PARTITION=${OSTREE_BOOT_PARTITION}
-
-        ostree admin --sysroot=${OTA_SYSROOT} init-fs ${OTA_SYSROOT}
+        ostree admin --sysroot=${OTA_SYSROOT} init-fs --modern ${OTA_SYSROOT}
         ostree admin --sysroot=${OTA_SYSROOT} os-init ${OSTREE_OSNAME}
+
+        # Preparation required to steer ostree bootloader detection
         mkdir -p ${OTA_SYSROOT}/boot/loader.0
         ln -s loader.0 ${OTA_SYSROOT}/boot/loader
 
         if [ "${OSTREE_BOOTLOADER}" = "grub" ]; then
+                # Used by ostree-grub-generator called by the ostree binary
+                export OSTREE_BOOT_PARTITION=${OSTREE_BOOT_PARTITION}
+
                 mkdir -p ${OTA_SYSROOT}/boot/grub2
                 ln -s ../loader/grub.cfg ${OTA_SYSROOT}/boot/grub2/grub.cfg
         elif [ "${OSTREE_BOOTLOADER}" = "u-boot" ]; then
@@ -61,15 +64,25 @@ IMAGE_CMD_ota () {
                 bbfatal "Invalid bootloader: ${OSTREE_BOOTLOADER}"
         fi
 
-        ostree_target_hash=$(cat ${OSTREE_REPO}/refs/heads/${OSTREE_BRANCHNAME}-${IMAGE_BASENAME})
+        ostree_target_hash=$(cat ${WORKDIR}/ostree_manifest)
 
+        # Use OSTree hash to avoid any potential race conditions between
+        # multiple builds accessing the same ${OSTREE_REPO}.
         ostree --repo=${OTA_SYSROOT}/ostree/repo pull-local --remote=${OSTREE_OSNAME} ${OSTREE_REPO} ${ostree_target_hash}
         kargs_list=""
         for arg in ${OSTREE_KERNEL_ARGS}; do
                 kargs_list="${kargs_list} --karg-append=$arg"
         done
 
-        ostree admin --sysroot=${OTA_SYSROOT} deploy ${kargs_list} --os=${OSTREE_OSNAME} ${ostree_target_hash}
+        # Create the same reference on the device we use in the archive OSTree
+        # repo in ${OSTREE_REPO}. This reference will show up when showing the
+        # deployment on the device:
+        # ostree admin status
+        # If a remote with the name ${OSTREE_OSNAME} is configured, this also
+        # will allow to use:
+        # ostree admin upgrade
+        ostree --repo=${OTA_SYSROOT}/ostree/repo refs --create=${OSTREE_OSNAME}:${OSTREE_BRANCHNAME} ${ostree_target_hash}
+        ostree admin --sysroot=${OTA_SYSROOT} deploy ${kargs_list} --os=${OSTREE_OSNAME} ${OSTREE_OSNAME}:${OSTREE_BRANCHNAME}
 
         cp -a ${IMAGE_ROOTFS}/var/sota ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
         # Create /var/sota if it doesn't exist yet
@@ -77,8 +90,11 @@ IMAGE_CMD_ota () {
         # Ensure the permissions are correctly set
         chmod 700 ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota
 
-        cp -a ${OSTREE_ROOTFS}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
-        cp -a ${OSTREE_ROOTFS}/usr/homedirs/home ${OTA_SYSROOT}/ || true
+        cp -a ${IMAGE_ROOTFS}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
+
+        mkdir -p ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/rootdirs
+        cp -a ${IMAGE_ROOTFS}/home ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/rootdirs/home || true
+
         # Ensure that /var/local exists (AGL symlinks /usr/local to /var/local)
         install -d ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/local
         # Set package version for the first deployment
diff --git a/classes/sota_sanity.bbclass b/classes/sota_sanity.bbclass
index 74973eb..02ca2e7 100644
--- a/classes/sota_sanity.bbclass
+++ b/classes/sota_sanity.bbclass
@@ -1,5 +1,12 @@
 # Sanity check the sota setup for common misconfigurations
 
+def sota_check_boolean_variable(var, d):
+    try:
+        oe.types.boolean(d.getVar(var))
+    except:
+        return False
+    return True
+
 def sota_check_overrides(status, d):
     for var in (d.getVar('SOTA_OVERRIDES_BLACKLIST') or "").split():
         if var in d.getVar('OVERRIDES').split(':'):
@@ -47,14 +54,14 @@ def sota_check_variables_validity(status, d):
         path = os.path.abspath(credentials)
         if not os.path.exists(path):
             status.addresult("SOTA_PACKED_CREDENTIALS is not set correctly. The zipped credentials file does not exist.\n")
-    if d.getVar("OSTREE_UPDATE_SUMMARY") and d.getVar("OSTREE_UPDATE_SUMMARY") not in ("0", "1", ""):
-        status.addresult("OSTREE_UPDATE_SUMMARY should be set to 0 or 1.\n")
-    if d.getVar("OSTREE_DEPLOY_DEVICETREE") and d.getVar("OSTREE_DEPLOY_DEVICETREE") not in ("0", "1", ""):
-        status.addresult("OSTREE_DEPLOY_DEVICETREE should be set to 0 or 1.\n")
-    if d.getVar("GARAGE_SIGN_AUTOVERSION") and d.getVar("GARAGE_SIGN_AUTOVERSION") not in ("0", "1", ""):
-        status.addresult("GARAGE_SIGN_AUTOVERSION should be set to 0 or 1.\n")
-    if d.getVar("SOTA_DEPLOY_CREDENTIALS") and d.getVar("SOTA_DEPLOY_CREDENTIALS") not in ("0", "1", ""):
-        status.addresult("SOTA_DEPLOY_CREDENTIALS should be set to 0 or 1.\n")
+    if not sota_check_boolean_variable("OSTREE_UPDATE_SUMMARY", d):
+        status.addresult("OSTREE_UPDATE_SUMMARY (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("OSTREE_UPDATE_SUMMARY"))
+    if not sota_check_boolean_variable("OSTREE_DEPLOY_DEVICETREE", d):
+        status.addresult("OSTREE_DEPLOY_DEVICETREE (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("OSTREE_DEPLOY_DEVICETREE"))
+    if not sota_check_boolean_variable("GARAGE_SIGN_AUTOVERSION", d):
+        status.addresult("GARAGE_SIGN_AUTOVERSION (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("GARAGE_SIGN_AUTOVERSION"))
+    if not sota_check_boolean_variable("SOTA_DEPLOY_CREDENTIALS", d):
+        status.addresult("SOTA_DEPLOY_CREDENTIALS (=%s) should be set to yes/y/true/t/1 or no/n/false/f/0.\n" % d.getVar("SOTA_DEPLOY_CREDENTIALS"))
     
 def sota_raise_sanity_error(msg, d):
     if d.getVar("SANITY_USE_EVENTS") == "1":