From e97975d6113ca1ffa6cbe9b005bd518bd146fd9c Mon Sep 17 00:00:00 2001 From: Ming Liu Date: Tue, 20 Nov 2018 09:03:25 +0100 Subject: image_types_ostree/ota.bbclass: refactor ostree task There are several flaws with ostree tasks, as follows: - ${IMAGE_NAME}.rootfs.ostree.tar.bz2 is generated, but it's not being used during ostree commit, so it should be removed if it's just a intermittent artifact. Or if we intend to deploy this tar.bz2 file, it should be tracked by sstate cache, that is to day, it should be generated in ${IMGDEPLOYDIR} rather than in ${DEPLOY_DIR_IMAGE}. - There are quite a few redundant code like mktemp/cd/rm a directory, which can be replaced by setting 'dirs', 'cleandirs' varflags. - There are some redundant variable check logic in image_types_ostree and image_types_ota bbclass. To fix the above, we make the following changes: - Introduce a new conversion image type 'tar', it could convert ostree and ota to ostree.tar, ota.tar, hence we can drop the code generating ostree.tar.bz2 in image_types_ostree.bbclass, and also drop the do_image_ota-tar task. To let this conversion type take effect, the otasetup task needs to be changed to ota. - Introduce BUILD_OSTREE_TARBALL variable, when being set to 1, a ostree.tar.bz2 tarball would be built, BUILD_OSTREE_TARBALL defaults to be 1, to be consistent with original behavior. - Replace 'ota-tar ota-tar.xz' with ota.tar.xz in IMAGE_FSTYPES. - Add a sanity check bbclass sota_sanity.bbclass, to ensure ostree or ota is not in OVERRIDES, this is to prevent potential naming pollution with other meta layers, and also check the required variables are not empty. This sota_sanity.bbclass is a common class that could be extended easily in furture, and one of its most advantages is that all the check are done in bb.event.SanityCheck event handler, so the end users could get the error message at very beginning of the recipe parsing time. Signed-off-by: Ming Liu --- classes/image_types_ostree.bbclass | 50 ++++++++++++------------------- classes/image_types_ota.bbclass | 60 +++++++------------------------------- classes/sota.bbclass | 8 +++-- classes/sota_sanity.bbclass | 54 ++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 84 deletions(-) create mode 100644 classes/sota_sanity.bbclass diff --git a/classes/image_types_ostree.bbclass b/classes/image_types_ostree.bbclass index 44a3aa4..41b8d0d 100644 --- a/classes/image_types_ostree.bbclass +++ b/classes/image_types_ostree.bbclass @@ -1,35 +1,28 @@ # OSTree deployment -do_image_ostree[depends] += "ostree-native:do_populate_sysroot \ - coreutils-native:do_populate_sysroot \ - virtual/kernel:do_deploy \ - ${INITRAMFS_IMAGE}:do_image_complete \ -" -do_image_ostree[lockfiles] += "${OSTREE_REPO}/ostree.lock" - OSTREE_KERNEL ??= "${KERNEL_IMAGETYPE}" +OSTREE_ROOTFS ??= "${WORKDIR}/ostree-rootfs" OSTREE_COMMIT_SUBJECT ??= "Commit-id: ${IMAGE_NAME}" OSTREE_COMMIT_BODY ??= "" OSTREE_UPDATE_SUMMARY ??= "0" -SYSTEMD_USED = "${@oe.utils.ifelse(d.getVar('VIRTUAL-RUNTIME_init_manager', True) == 'systemd', 'true', '')}" +BUILD_OSTREE_TARBALL ??= "1" -IMAGE_CMD_ostree () { - if [ -z "$OSTREE_REPO" ]; then - bbfatal "OSTREE_REPO should be set in your local.conf" - fi +SYSTEMD_USED = "${@oe.utils.ifelse(d.getVar('VIRTUAL-RUNTIME_init_manager', True) == 'systemd', 'true', '')}" - if [ -z "$OSTREE_BRANCHNAME" ]; then - bbfatal "OSTREE_BRANCHNAME should be set in your local.conf" - fi +IMAGE_CMD_TAR = "tar --xattrs --xattrs-include=*" +CONVERSION_CMD_tar = "touch ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}; ${IMAGE_CMD_TAR} --numeric-owner -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.tar -C ${OTA_IMAGE_ROOTFS} . || [ $? -eq 1 ]" +CONVERSIONTYPES_append = " tar" - OSTREE_ROOTFS=`mktemp -du ${WORKDIR}/ostree-root-XXXXX` - cp -a ${IMAGE_ROOTFS} ${OSTREE_ROOTFS} +OTA_IMAGE_ROOTFS_task-image-ostree = "${OSTREE_ROOTFS}" +do_image_ostree[dirs] = "${OSTREE_ROOTFS}" +do_image_ostree[cleandirs] = "${OSTREE_ROOTFS}" +do_image_ostree[depends] = "coreutils-native:do_populate_sysroot virtual/kernel:do_deploy ${INITRAMFS_IMAGE}:do_image_complete" +IMAGE_CMD_ostree () { + cp -a ${IMAGE_ROOTFS}/* ${OSTREE_ROOTFS} chmod a+rx ${OSTREE_ROOTFS} sync - cd ${OSTREE_ROOTFS} - for d in var/*; do if [ "${d}" != "var/local" ]; then rm -rf ${d} @@ -132,17 +125,12 @@ IMAGE_CMD_ostree () { # Copy image manifest cat ${IMAGE_MANIFEST} | cut -d " " -f1,3 > usr/package.manifest +} - cd ${WORKDIR} - - # Create a tarball that can be then commited to OSTree repo - OSTREE_TAR=${DEPLOY_DIR_IMAGE}/${IMAGE_NAME}.rootfs.ostree.tar.bz2 - tar -C ${OSTREE_ROOTFS} --xattrs --xattrs-include='*' -cjf ${OSTREE_TAR} . - sync - - rm -f ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.rootfs.ostree.tar.bz2 - ln -s ${IMAGE_NAME}.rootfs.ostree.tar.bz2 ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.rootfs.ostree.tar.bz2 - +IMAGE_TYPEDEP_ostreecommit = "ostree" +do_image_ostreecommit[depends] += "ostree-native:do_populate_sysroot" +do_image_ostreecommit[lockfiles] += "${WORKDIR}/${OSTREE_REPO}-commit.lock" +IMAGE_CMD_ostreecommit () { if ! ostree --repo=${OSTREE_REPO} refs 2>&1 > /dev/null; then ostree --repo=${OSTREE_REPO} init --mode=archive-z2 fi @@ -158,11 +146,9 @@ IMAGE_CMD_ostree () { if [ "${OSTREE_UPDATE_SUMMARY}" = "1" ]; then ostree --repo=${OSTREE_REPO} summary -u fi - - rm -rf ${OSTREE_ROOTFS} } -IMAGE_TYPEDEP_ostreepush = "ostree" +IMAGE_TYPEDEP_ostreepush = "ostreecommit" do_image_ostreepush[depends] += "aktualizr-native:do_populate_sysroot ca-certificates-native:do_populate_sysroot" IMAGE_CMD_ostreepush () { # Print warnings if credetials are not set or if the file has not been found. diff --git a/classes/image_types_ota.bbclass b/classes/image_types_ota.bbclass index 79de909..09f1095 100644 --- a/classes/image_types_ota.bbclass +++ b/classes/image_types_ota.bbclass @@ -1,15 +1,9 @@ # Image to use with u-boot as BIOS and OSTree deployment system -#inherit image_types - # Boot filesystem size in MiB # OSTree updates may require some space on boot file system for # boot scripts, kernel and initramfs images # - - -do_image_ota_ext4[depends] += "e2fsprogs-native:do_populate_sysroot" - calculate_size () { BASE=$1 SCALE=$2 @@ -44,26 +38,13 @@ calculate_size () { } OTA_SYSROOT = "${WORKDIR}/ota-sysroot" - -## Common OTA image setup -fakeroot do_otasetup () { - - if [ -z "$OSTREE_REPO" ]; then - bbfatal "OSTREE_REPO should be set in your local.conf" - fi - - if [ -z "$OSTREE_OSNAME" ]; then - bbfatal "OSTREE_OSNAME should be set in your local.conf" - fi - - if [ -z "$OSTREE_BRANCHNAME" ]; then - bbfatal "OSTREE_BRANCHNAME should be set in your local.conf" - fi - - # HaX! Since we are using a peristent directory, we need to be sure to clean it on run. - mkdir -p ${OTA_SYSROOT} - rm -rf ${OTA_SYSROOT}/* - +OTA_IMAGE_ROOTFS_task-image-ota = "${OTA_SYSROOT}" +IMAGE_TYPEDEP_ota = "ostreecommit" +do_image_ota[dirs] = "${OTA_SYSROOT}" +do_image_ota[cleandirs] = "${OTA_SYSROOT}" +do_image_ota[depends] = "${@'grub:do_populate_sysroot' if d.getVar('OSTREE_BOOTLOADER', True) == 'grub' else ''} \ + ${@'virtual/bootloader:do_deploy' if d.getVar('OSTREE_BOOTLOADER', True) == 'u-boot' else ''}" +IMAGE_CMD_ota () { ostree admin --sysroot=${OTA_SYSROOT} init-fs ${OTA_SYSROOT} ostree admin --sysroot=${OTA_SYSROOT} os-init ${OSTREE_OSNAME} mkdir -p ${OTA_SYSROOT}/boot/loader.0 @@ -88,19 +69,14 @@ fakeroot do_otasetup () { ostree admin --sysroot=${OTA_SYSROOT} deploy ${kargs_list} --os=${OSTREE_OSNAME} ${ostree_target_hash} - # Copy deployment /home and /var/sota to sysroot - HOME_TMP=`mktemp -d ${WORKDIR}/home-tmp-XXXXX` - - tar --xattrs --xattrs-include='*' -C ${HOME_TMP} -xf ${DEPLOY_DIR_IMAGE}/${IMAGE_LINK_NAME}.rootfs.ostree.tar.bz2 ./usr/homedirs ./var/local || true - cp -a ${IMAGE_ROOTFS}/var/sota ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true # Create /var/sota if it doesn't exist yet mkdir -p ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota # Ensure the permissions are correctly set chmod 700 ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota - mv ${HOME_TMP}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true - mv ${HOME_TMP}/usr/homedirs/home ${OTA_SYSROOT}/ || true + cp -a ${OSTREE_ROOTFS}/var/local ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/ || true + cp -a ${OSTREE_ROOTFS}/usr/homedirs/home ${OTA_SYSROOT}/ || true # Ensure that /var/local exists (AGL symlinks /usr/local to /var/local) install -d ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/local # Set package version for the first deployment @@ -112,10 +88,10 @@ fakeroot do_otasetup () { fi mkdir -p ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota/import echo "{\"${ostree_target_hash}\":\"${GARAGE_TARGET_NAME}-${target_version}\"}" > ${OTA_SYSROOT}/ostree/deploy/${OSTREE_OSNAME}/var/sota/import/installed_versions - echo "All done. Cleaning up dir: ${HOME_TMP}" - rm -rf ${HOME_TMP} } +IMAGE_TYPEDEP_ota-ext4 = "ota" +do_image_ota_ext4[depends] = "e2fsprogs-native:do_populate_sysroot" IMAGE_CMD_ota-ext4 () { # Calculate image type OTA_ROOTFS_SIZE=$(calculate_size `du -ks $OTA_SYSROOT | cut -f 1` "${IMAGE_OVERHEAD_FACTOR}" "${IMAGE_ROOTFS_SIZE}" "${IMAGE_ROOTFS_MAXSIZE}" `expr ${IMAGE_ROOTFS_EXTRA_SPACE}` "${IMAGE_ROOTFS_ALIGNMENT}") @@ -133,17 +109,3 @@ IMAGE_CMD_ota-ext4 () { dd if=/dev/zero of=${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ota-ext4 seek=${OTA_ROOTFS_SIZE} count=$COUNT bs=1024 mkfs.ext4 -O ^64bit ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ota-ext4 -L otaroot -d ${OTA_SYSROOT} } - -IMAGE_CMD_ota-tar () { - tar -cf ${IMGDEPLOYDIR}/${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.ota-tar -C ${OTA_SYSROOT} . -} - -do_otasetup[doc] = "Sets up the base ota rootfs used for subsequent image generation" -do_otasetup[depends] += "virtual/fakeroot-native:do_populate_sysroot \ - ${@'grub:do_populate_sysroot' if d.getVar('OSTREE_BOOTLOADER', True) == 'grub' else ''} \ - ${@'virtual/bootloader:do_deploy' if d.getVar('OSTREE_BOOTLOADER', True) == 'u-boot' else ''}" - -addtask do_otasetup after do_image_ostree before do_image_ota_ext4 do_image_ota_tar - -IMAGE_TYPEDEP_ota-ext4 = "ostree" -IMAGE_TYPEDEP_ota-tar = "ostree" diff --git a/classes/sota.bbclass b/classes/sota.bbclass index e654071..7118807 100644 --- a/classes/sota.bbclass +++ b/classes/sota.bbclass @@ -16,7 +16,8 @@ IMAGE_INSTALL_append_sota = " ostree os-release ${SOTA_CLIENT} ${SOTA_CLIENT_PRO IMAGE_CLASSES += " image_types_ostree image_types_ota" IMAGE_FSTYPES += "${@bb.utils.contains('DISTRO_FEATURES', 'sota', 'ostreepush garagesign garagecheck ota-ext4 wic', ' ', d)}" -IMAGE_FSTYPES += "${@bb.utils.contains('BUILD_OTA_TARBALL', '1', 'ota-tar ota-tar.xz', ' ', d)}" +IMAGE_FSTYPES += "${@bb.utils.contains('BUILD_OSTREE_TARBALL', '1', 'ostree.tar.bz2', ' ', d)}" +IMAGE_FSTYPES += "${@bb.utils.contains('BUILD_OTA_TARBALL', '1', 'ota.tar.xz', ' ', d)}" PACKAGECONFIG_append_pn-curl = " ssl" PACKAGECONFIG_remove_pn-curl = "gnutls" @@ -50,6 +51,7 @@ SOTA_MACHINE_intel-corei7-64 ?= "minnowboard" SOTA_MACHINE_qemux86-64 ?= "qemux86-64" SOTA_MACHINE_am335x-evm ?= "am335x-evm-wifi" -inherit sota_${SOTA_MACHINE} +SOTA_OVERRIDES_BLACKLIST = "ostree ota" +SOTA_REQUIRED_VARIABLES = "OSTREE_REPO OSTREE_BRANCHNAME OSTREE_OSNAME OSTREE_BOOTLOADER OSTREE_BOOT_PARTITION GARAGE_SIGN_REPO GARAGE_TARGET_NAME" -inherit image_repo_manifest +inherit sota_sanity sota_${SOTA_MACHINE} image_repo_manifest diff --git a/classes/sota_sanity.bbclass b/classes/sota_sanity.bbclass new file mode 100644 index 0000000..e47de19 --- /dev/null +++ b/classes/sota_sanity.bbclass @@ -0,0 +1,54 @@ +# Sanity check the sota setup for common misconfigurations + +def sota_check_overrides(status, d): + for var in (d.getVar('SOTA_OVERRIDES_BLACKLIST', True) or "").split(): + if var in d.getVar('OVERRIDES', True).split(':'): + status.addresult("%s should not be a overrides, because it is a image fstype in updater layer, please check your OVERRIDES setting.\n" % var) + +def sota_check_required_variables(status, d): + for var in (d.getVar('SOTA_REQUIRED_VARIABLES', True) or "").split(): + if not d.getVar(var, True): + status.addresult("%s should be set in your local.conf.\n" % var) + +def sota_raise_sanity_error(msg, d): + if d.getVar("SANITY_USE_EVENTS", True) == "1": + bb.event.fire(bb.event.SanityCheckFailed(msg), d) + return + + bb.fatal("Sota's config sanity checker detected a potential misconfiguration.\n" + "Please fix the cause of this error then you can continue to build.\n" + "Following is the list of potential problems / advisories:\n" + "\n%s" % msg) + +def sota_check_sanity(sanity_data): + class SanityStatus(object): + def __init__(self): + self.messages = "" + self.reparse = False + + def addresult(self, message): + if message: + self.messages = self.messages + message + + status = SanityStatus() + + sota_check_overrides(status, sanity_data) + sota_check_required_variables(status, sanity_data) + + if status.messages != "": + sota_raise_sanity_error(sanity_data.expand(status.messages), sanity_data) + +addhandler sota_check_sanity_eventhandler +sota_check_sanity_eventhandler[eventmask] = "bb.event.SanityCheck" + +python sota_check_sanity_eventhandler() { + if bb.event.getName(e) == "SanityCheck": + sanity_data = copy_data(e) + if e.generateevents: + sanity_data.setVar("SANITY_USE_EVENTS", "1") + reparse = sota_check_sanity(sanity_data) + e.data.setVar("BB_INVALIDCONF", reparse) + bb.event.fire(bb.event.SanityCheckPassed(), e.data) + + return +} -- cgit v1.2.3-54-g00ecf