image_types_ostree/ota.bbclass: refactor ostree task

There are several flaws with ostree tasks, as follows:
- ${IMAGE_NAME}.rootfs.ostree.tar.bz2 is generated, but it's not being
  used during ostree commit, so it should be removed if it's just a
  intermittent artifact. Or if we intend to deploy this tar.bz2 file,
  it should be tracked by sstate cache, that is to day, it should be
  generated in ${IMGDEPLOYDIR} rather than in ${DEPLOY_DIR_IMAGE}.
- There are quite a few redundant code like mktemp/cd/rm a directory,
  which can be replaced by setting 'dirs', 'cleandirs' varflags.
- There are some redundant variable check logic in image_types_ostree
  and image_types_ota bbclass.

To fix the above, we make the following changes:
- Introduce a new conversion image type 'tar', it could convert ostree
  and ota to ostree.tar, ota.tar, hence we can drop the code generating
  ostree.tar.bz2 in image_types_ostree.bbclass, and also drop the
  do_image_ota-tar task. To let this conversion type take effect, the
  otasetup task needs to be changed to ota.
- Introduce BUILD_OSTREE_TARBALL variable, when being set to 1, a
  ostree.tar.bz2 tarball would be built, BUILD_OSTREE_TARBALL defaults
  to be 1, to be consistent with original behavior.
- Replace 'ota-tar ota-tar.xz' with ota.tar.xz in IMAGE_FSTYPES.
- Add a sanity check bbclass sota_sanity.bbclass, to ensure ostree or
  ota is not in OVERRIDES, this is to prevent potential naming
  pollution with other meta layers, and also check the required
  variables are not empty. This sota_sanity.bbclass is a common class
  that could be extended easily in furture, and one of its most
  advantages is that all the check are done in bb.event.SanityCheck
  event handler, so the end users could get the error message at very
  beginning of the recipe parsing time.

Signed-off-by: Ming Liu <liu.ming50@gmail.com>

4 files changed, 88 insertions, 84 deletions

diff --git a/classes/image_types_ostree.bbclass b/classes/image_types_ostree.bbclass
index 44a3aa4..41b8d0d 100644
--- a/classes/image_types_ostree.bbclass
+++ b/classes/image_types_ostree.bbclass
@@ -1,35 +1,28 @@
 # OSTree deployment
 
-do_image_ostree[depends] += "ostree-native:do_populate_sysroot \
-                        coreutils-native:do_populate_sysroot \
-                        virtual/kernel:do_deploy \
-                        ${INITRAMFS_IMAGE}:do_image_complete \
-"
-do_image_ostree[lockfiles] += "${OSTREE_REPO}/ostree.lock"
-
 OSTREE_KERNEL ??= "${KERNEL_IMAGETYPE}"
+OSTREE_ROOTFS ??= "${WORKDIR}/ostree-rootfs"
 OSTREE_COMMIT_SUBJECT ??= "Commit-id: ${IMAGE_NAME}"
 OSTREE_COMMIT_BODY ??= ""
 OSTREE_UPDATE_SUMMARY ??= "0"
 
-SYSTEMD_USED = "${@oe.utils.ifelse(d.getVar('VIRTUAL-RUNTIME_init_manager', True) == 'systemd', 'true', '')}"
+BUILD_OSTREE_TARBALL ??= "1"
 
-IMAGE_CMD_ostree () {
-    if [ -z "$OSTREE_REPO" ]; then
-        bbfatal "OSTREE_REPO should be set in your local.conf"
-    fi
+SYSTEMD_USED = "${@oe.utils.ifelse(d.getVar('VIRTUAL-RUNTIME_init_manager', True) == 'systemd', 'true', '')}"
 
-    if [ -z "$OSTREE_BRANCHNAME" ]; then
-        bbfatal "OSTREE_BRANCHNAME should be set in your local.conf"
-    fi
+IMAGE_CMD_TAR = "tar --xattrs --xattrs-include=*"
+CONVERSION_CMD_tar = "touch ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}; ${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.tar -C ${OTA_IMAGE_ROOTFS} . || [ $? -eq 1 ]"
+CONVERSIONTYPES_append = " tar"
 
-    OSTREE_ROOTFS=`mktemp -du ${WORKDIR}/ostree-root-XXXXX`
-    cp -a ${IMAGE_ROOTFS} ${OSTREE_ROOTFS}
+OTA_IMAGE_ROOTFS_task-image-ostree = "${OSTREE_ROOTFS}"
+do_image_ostree[dirs] = "${OSTREE_ROOTFS}"
+do_image_ostree[cleandirs] = "${OSTREE_ROOTFS}"
+do_image_ostree[depends] = "coreutils-native:do_populate_sysroot virtual/kernel:do_deploy ${INITRAMFS_IMAGE}:do_image_complete"
+IMAGE_CMD_ostree () {
+    cp -a ${IMAGE_ROOTFS}/* ${OSTREE_ROOTFS}
     chmod a+rx ${OSTREE_ROOTFS}
     sync
 
-    cd ${OSTREE_ROOTFS}
-
     for d in var/*; do
       if [ "${d}" != "var/local" ]; then
         rm -rf ${d}
@@ -132,17 +125,12 @@ IMAGE_CMD_ostree () {
 
     # Copy image manifest
     cat ${IMAGE_MANIFEST} | cut -d " " -f1,3 > usr/package.manifest
+}
 
-    cd ${WORKDIR}
-
-    # Create a tarball that can be then commited to OSTree repo
-    OSTREE_TAR=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ostree.tar.bz2
-    tar -C ${OSTREE_ROOTFS} --xattrs --xattrs-include='*' -cjf ${OSTREE_TAR} .
-    sync
-
-    rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.rootfs.ostree.tar.bz2
-    ln -s ${IMAGE_NAME}.rootfs.ostree.tar.bz2 ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.rootfs.ostree.tar.bz2
-
+IMAGE_TYPEDEP_ostreecommit = "ostree"
+do_image_ostreecommit[depends] += "ostree-native:do_populate_sysroot"
+do_image_ostreecommit[lockfiles] += "${WORKDIR}/${OSTREE_REPO}-commit.lock"
+IMAGE_CMD_ostreecommit () {
     if ! ostree --repo=${OSTREE_REPO} refs 2>&1 > /dev/null; then
         ostree --repo=${OSTREE_REPO} init --mode=archive-z2
     fi
@@ -158,11 +146,9 @@ IMAGE_CMD_ostree () {
     if [ "${OSTREE_UPDATE_SUMMARY}" = "1" ]; then
         ostree --repo=${OSTREE_REPO} summary -u
     fi
-
-    rm -rf ${OSTREE_ROOTFS}
 }
 
-IMAGE_TYPEDEP_ostreepush = "ostree"
+IMAGE_TYPEDEP_ostreepush = "ostreecommit"
 do_image_ostreepush[depends] += "aktualizr-native:do_populate_sysroot ca-certificates-native:do_populate_sysroot"
 IMAGE_CMD_ostreepush () {
     # Print warnings if credetials are not set or if the file has not been found.
diff --git a/classes/image_types_ota.bbclass b/classes/image_types_ota.bbclass
index 79de909..09f1095 100644
--- a/classes/image_types_ota.bbclass
+++ b/classes/image_types_ota.bbclass
@@ -1,15 +1,9 @@
 # Image to use with u-boot as BIOS and OSTree deployment system
 
-#inherit image_types
-
 # Boot filesystem size in MiB
 # OSTree updates may require some space on boot file system for
 # boot scripts, kernel and initramfs images
 #
-
-
-do_image_ota_ext4[depends] += "e2fsprogs-native:do_populate_sysroot"
-
 calculate_size () {
         BASE=$1
         SCALE=$2
@@ -44,26 +38,13 @@ calculate_size () {
 }
 
 OTA_SYSROOT = "${WORKDIR}/ota-sysroot"
-
-## Common OTA image setup
-fakeroot do_otasetup () {
-        
-        if [ -z "$OSTREE_REPO" ]; then
-                bbfatal "OSTREE_REPO should be set in your local.conf"
-        fi
-
-        if [ -z "$OSTREE_OSNAME" ]; then
-                bbfatal "OSTREE_OSNAME should be set in your local.conf"
-        fi
-
-        if [ -z "$OSTREE_BRANCHNAME" ]; then
-                bbfatal "OSTREE_BRANCHNAME should be set in your local.conf"
-        fi
-
-        # HaX! Since we are using a peristent directory, we need to be sure to clean it on run.
-        mkdir -p ${OTA_SYSROOT}
-        rm -rf ${OTA_SYSROOT}/*
-
+OTA_IMAGE_ROOTFS_task-image-ota = "${OTA_SYSROOT}"
+IMAGE_TYPEDEP_ota = "ostreecommit"
+do_image_ota[dirs] = "${OTA_SYSROOT}"
+do_image_ota[cleandirs] = "${OTA_SYSROOT}"
+do_image_ota[depends] = "${@'grub:do_populate_sysroot' if d.getVar('OSTREE_BOOTLOADER', True) == 'grub' else ''} \
+                         ${@'virtual/bootloader:do_deploy' if d.getVar('OSTREE_BOOTLOADER', True) == 'u-boot' else ''}"
+IMAGE_CMD_ota () {
         ostree admin --sysroot=${OTA_SYSROOT} init-fs ${OTA_SYSROOT}
         ostree admin --sysroot=${OTA_SYSROOT} os-init ${OSTREE_OSNAME}
         mkdir -p ${OTA_SYSROOT}/boot/loader.0
@@ -88,19 +69,14 @@ fakeroot do_otasetup () {
 
         ostree admin --sysroot=${OTA_SYSROOT} deploy ${kargs_list} --os=${OSTREE_OSNAME} ${ostree_target_hash}
 
-        # Copy deployment /home and /var/sota to sysroot
-        HOME_TMP=`mktemp -d ${WORKDIR}/home-tmp-XXXXX`
-
-        tar --xattrs --xattrs-include='*' -C ${HOME_TMP} -xf ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.rootfs.ostree.tar.bz2 ./usr/homedirs ./var/local || true
-
         cp -a ${IMAGE_ROOTFS}/var/sota ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
         # Create /var/sota if it doesn't exist yet
         mkdir -p ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota
         # Ensure the permissions are correctly set
         chmod 700 ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota
 
-        mv ${HOME_TMP}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
-        mv ${HOME_TMP}/usr/homedirs/home ${OTA_SYSROOT}/ || true
+        cp -a ${OSTREE_ROOTFS}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true
+        cp -a ${OSTREE_ROOTFS}/usr/homedirs/home ${OTA_SYSROOT}/ || true
         # Ensure that /var/local exists (AGL symlinks /usr/local to /var/local)
         install -d ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/local
         # Set package version for the first deployment
@@ -112,10 +88,10 @@ fakeroot do_otasetup () {
         fi
         mkdir -p ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota/import
         echo "{\"${ostree_target_hash}\":\"${GARAGE_TARGET_NAME}-${target_version}\"}" > ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota/import/installed_versions
-        echo "All done. Cleaning up dir: ${HOME_TMP}"
-        rm -rf ${HOME_TMP}
 }
 
+IMAGE_TYPEDEP_ota-ext4 = "ota"
+do_image_ota_ext4[depends] = "e2fsprogs-native:do_populate_sysroot"
 IMAGE_CMD_ota-ext4 () {
         # Calculate image type
         OTA_ROOTFS_SIZE=$(calculate_size `du -ks $OTA_SYSROOT | cut -f 1`  "${IMAGE_OVERHEAD_FACTOR}" "${IMAGE_ROOTFS_SIZE}" "${IMAGE_ROOTFS_MAXSIZE}" `expr ${IMAGE_ROOTFS_EXTRA_SPACE}` "${IMAGE_ROOTFS_ALIGNMENT}")
@@ -133,17 +109,3 @@ IMAGE_CMD_ota-ext4 () {
         dd if=/dev/zero of=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ota-ext4 seek=${OTA_ROOTFS_SIZE} count=$COUNT bs=1024
         mkfs.ext4 -O ^64bit ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ota-ext4 -L otaroot -d ${OTA_SYSROOT}
 }
-
-IMAGE_CMD_ota-tar () {
-        tar -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ota-tar -C ${OTA_SYSROOT} .
-}
-
-do_otasetup[doc] = "Sets up the base ota rootfs used for subsequent image generation"
-do_otasetup[depends] += "virtual/fakeroot-native:do_populate_sysroot \
-                        ${@'grub:do_populate_sysroot' if d.getVar('OSTREE_BOOTLOADER', True) == 'grub' else ''} \
-                        ${@'virtual/bootloader:do_deploy' if d.getVar('OSTREE_BOOTLOADER', True) == 'u-boot' else ''}"
-
-addtask do_otasetup after do_image_ostree before do_image_ota_ext4 do_image_ota_tar
-
-IMAGE_TYPEDEP_ota-ext4 = "ostree"
-IMAGE_TYPEDEP_ota-tar = "ostree"
diff --git a/classes/sota.bbclass b/classes/sota.bbclass
index e654071..7118807 100644
--- a/classes/sota.bbclass
+++ b/classes/sota.bbclass
@@ -16,7 +16,8 @@ IMAGE_INSTALL_append_sota = " ostree os-release ${SOTA_CLIENT} ${SOTA_CLIENT_PRO
 IMAGE_CLASSES += " image_types_ostree image_types_ota"
 
 IMAGE_FSTYPES += "${@bb.utils.contains('DISTRO_FEATURES', 'sota', 'ostreepush garagesign garagecheck ota-ext4 wic', ' ', d)}"
-IMAGE_FSTYPES += "${@bb.utils.contains('BUILD_OTA_TARBALL', '1', 'ota-tar ota-tar.xz', ' ', d)}"
+IMAGE_FSTYPES += "${@bb.utils.contains('BUILD_OSTREE_TARBALL', '1', 'ostree.tar.bz2', ' ', d)}"
+IMAGE_FSTYPES += "${@bb.utils.contains('BUILD_OTA_TARBALL', '1', 'ota.tar.xz', ' ', d)}"
 
 PACKAGECONFIG_append_pn-curl = " ssl"
 PACKAGECONFIG_remove_pn-curl = "gnutls"
@@ -50,6 +51,7 @@ SOTA_MACHINE_intel-corei7-64 ?= "minnowboard"
 SOTA_MACHINE_qemux86-64 ?= "qemux86-64"
 SOTA_MACHINE_am335x-evm ?= "am335x-evm-wifi"
 
-inherit sota_${SOTA_MACHINE}
+SOTA_OVERRIDES_BLACKLIST = "ostree ota"
+SOTA_REQUIRED_VARIABLES = "OSTREE_REPO OSTREE_BRANCHNAME OSTREE_OSNAME OSTREE_BOOTLOADER OSTREE_BOOT_PARTITION GARAGE_SIGN_REPO GARAGE_TARGET_NAME"
 
-inherit image_repo_manifest
+inherit sota_sanity sota_${SOTA_MACHINE} image_repo_manifest
diff --git a/classes/sota_sanity.bbclass b/classes/sota_sanity.bbclass
new file mode 100644
index 0000000..e47de19
--- /dev/null
+++ b/classes/sota_sanity.bbclass
@@ -0,0 +1,54 @@
+# Sanity check the sota setup for common misconfigurations
+
+def sota_check_overrides(status, d):
+    for var in (d.getVar('SOTA_OVERRIDES_BLACKLIST', True) or "").split():
+        if var in d.getVar('OVERRIDES', True).split(':'):
+            status.addresult("%s should not be a overrides, because it is a image fstype in updater layer, please check your OVERRIDES setting.\n" % var)
+
+def sota_check_required_variables(status, d):
+    for var in (d.getVar('SOTA_REQUIRED_VARIABLES', True) or "").split():
+        if not d.getVar(var, True):
+            status.addresult("%s should be set in your local.conf.\n" % var)
+
+def sota_raise_sanity_error(msg, d):
+    if d.getVar("SANITY_USE_EVENTS", True) == "1":
+        bb.event.fire(bb.event.SanityCheckFailed(msg), d)
+        return
+
+    bb.fatal("Sota's config sanity checker detected a potential misconfiguration.\n"
+             "Please fix the cause of this error then you can continue to build.\n"
+             "Following is the list of potential problems / advisories:\n"
+             "\n%s" % msg)
+
+def sota_check_sanity(sanity_data):
+    class SanityStatus(object):
+        def __init__(self):
+            self.messages = ""
+            self.reparse = False
+
+        def addresult(self, message):
+            if message:
+                self.messages = self.messages + message
+
+    status = SanityStatus()
+
+    sota_check_overrides(status, sanity_data)
+    sota_check_required_variables(status, sanity_data)
+
+    if status.messages != "":
+        sota_raise_sanity_error(sanity_data.expand(status.messages), sanity_data)
+
+addhandler sota_check_sanity_eventhandler
+sota_check_sanity_eventhandler[eventmask] = "bb.event.SanityCheck"
+
+python sota_check_sanity_eventhandler() {
+    if bb.event.getName(e) == "SanityCheck":
+        sanity_data = copy_data(e)
+        if e.generateevents:
+            sanity_data.setVar("SANITY_USE_EVENTS", "1")
+        reparse = sota_check_sanity(sanity_data)
+        e.data.setVar("BB_INVALIDCONF", reparse)
+        bb.event.fire(bb.event.SanityCheckPassed(), e.data)
+
+    return
+}