Merge pull request #581 from advancedtelematic/backport/thud-19-08-14

Backport/thud 19 08 14

11 files changed, 127 insertions, 47 deletions

diff --git a/README.adoc b/README.adoc
index b047f91..dd07425 100644
--- a/README.adoc
+++ b/README.adoc
@@ -61,7 +61,7 @@ and get as a result an `ostree_repo` folder in your images directory (`tmp/deplo
 
 * your OSTree repository, with the rootfs committed as an OSTree deployment,
 * an `ota-ext4` bootstrap image, which is an OSTree physical sysroot as a burnable filesystem image, and optionally
-* some machine-dependent live images (e.g. `.wic` for Raspberry Pi or `.porter-sdimg-ota` Renesas Porter board).
+* some machine-dependent live images (e.g. `.wic` for Raspberry Pi or `.porter-sdimg-ota` for Renesas Porter board).
 
 Although `aglsetup.sh` hooks provide reasonable defaults for SOTA-related variables, you may want to tune some of them.
 
@@ -71,11 +71,18 @@ Ubuntu users that encounter an error due to missing `Python.h` should install `l
 
 == Supported boards
 
-Currently supported platforms are
+Currently supported platforms are:
 
-* https://github.com/advancedtelematic/meta-updater-raspberrypi[Raspberry Pi3]
-* https://github.com/advancedtelematic/meta-updater-minnowboard[Minnowboard]
+* https://github.com/advancedtelematic/meta-updater-raspberrypi[Raspberry Pi 2 and 3]
+* https://github.com/advancedtelematic/meta-updater-minnowboard[Intel Minnowboard]
 * https://github.com/advancedtelematic/meta-updater-qemux86-64[Native QEMU emulation]
+* Renesas R-Car H3 and M3
+* https://github.com/advancedtelematic/meta-updater-ti/[TI BeagleBone Black] (rocko only, using TI SDK 05.03)
+* https://github.com/advancedtelematic/meta-updater-ti/[TI AM65x industrial development kit] (rocko only, using TI SDK 05.03)
+
+Additionally, there is community support for https://github.com/ricardosalveti/meta-updater-riscv[RISC-V] boards, in particular the Freedom U540.
+
+We also historically supported the https://github.com/advancedtelematic/meta-updater-porter[Renesas Porter] board.
 
 === Adding support for your board
 
@@ -92,6 +99,7 @@ Your images will also need network connectivity to be able to reach an actual OT
 
 == SOTA-related variables in local.conf
 
+* `OSTREE_BRANCHNAME` - OSTree branch name. Defaults to `${SOTA_HARDWARE_ID}`. Particularly useful for grouping similar images.
 * `OSTREE_REPO` - path to your OSTree repository. Defaults to `$\{DEPLOY_DIR_IMAGE}/ostree_repo`
 * `OSTREE_OSNAME` - OS deployment name on your target device. For more information about deployments and osnames see the https://ostree.readthedocs.io/en/latest/manual/deployment/[OSTree documentation]. Defaults to "poky".
 * `OSTREE_COMMIT_BODY` - Message attached to OSTree commit. Empty by default.
@@ -110,7 +118,7 @@ Your images will also need network connectivity to be able to reach an actual OT
 * `SOTA_DT_OVERLAYS` - whitespace-separated list of used device tree overlays for FIT image. This list is OSTree-updateable as well.
 * `SOTA_EXTRA_CONF_FRAGS` - extra https://lxr.missinglinkelectronics.com/uboot/doc/uImage.FIT/overlay-fdt-boot.txt[configuration fragments] for FIT image.
 * `RESOURCE_xxx_pn-aktualizr` - controls maximum resource usage of the aktualizr service, when `aktualizr-resource-control` is installed on the image. See <<aktualizr service resource control>> for details.
-* `SOTA_POLLING_SEC` - sets polling interval for aktualizr to check for updates if aktualizr-polling-sec is included in the image.
+* `SOTA_POLLING_SEC` - sets polling interval for aktualizr to check for updates if aktualizr-polling-interval is included in the image.
 
 == Usage
 
@@ -191,7 +199,23 @@ RESOURCE_CPU_WEIGHT_pn-aktualizr = "50"
 
 == Development configuration
 
-There are a few settings that can be controlled in `local.conf` to simplify the development process:
+=== Logging
+
+To troubleshoot problems that you might encounter during development, we recommend that you enable persistent `systemd` logging. This setting is enabled by default for newly configured environments (see link:conf/local.conf.sample.append[]). To enable it manually, put this to your `local.conf`:
+
+....
+IMAGE_INSTALL_append += " systemd-journald-persistent"
+....
+
+It may also be helpful to run with debug logging enabled in aktualizr. To do so, add this to your `local.conf`:
+
+....
+IMAGE_INSTALL_append += " aktualizr-log-debug"
+....
+
+=== Custom aktualizr versions
+
+You can override the version of aktualizr included in your image. This requires that the version you wish to run is pushed to the https://github.com/advancedtelematic/aktualizr[aktualizr github repo]. You can then use these settings in your `local.conf` to simplify the development process:
 
 [options="header"]
 |======================
diff --git a/classes/image_types_ostree.bbclass b/classes/image_types_ostree.bbclass
index 56d4d76..2e8e8f5 100644
--- a/classes/image_types_ostree.bbclass
+++ b/classes/image_types_ostree.bbclass
@@ -78,15 +78,15 @@ IMAGE_CMD_ostree () {
             if [ "$(ls -A $dir)" ]; then
                 bbwarn "Data in /$dir directory is not preserved by OSTree. Consider moving it under /usr"
             fi
-
-            if [ -n "${SYSTEMD_USED}" ]; then
-                echo "d /var/rootdirs/${dir} 0755 root root -" >>${tmpfiles_conf}
-            else
-                echo "mkdir -p /var/rootdirs/${dir}; chown 755 /var/rootdirs/${dir}" >>${tmpfiles_conf}
-            fi
             rm -rf ${dir}
-            ln -sf var/rootdirs/${dir} ${dir}
         fi
+
+        if [ -n "${SYSTEMD_USED}" ]; then
+            echo "d /var/rootdirs/${dir} 0755 root root -" >>${tmpfiles_conf}
+        else
+            echo "mkdir -p /var/rootdirs/${dir}; chown 755 /var/rootdirs/${dir}" >>${tmpfiles_conf}
+        fi
+        ln -sf var/rootdirs/${dir} ${dir}
     done
 
     if [ -d root ] && [ ! -L root ]; then
@@ -95,15 +95,40 @@ IMAGE_CMD_ostree () {
         fi
 
         if [ -n "${SYSTEMD_USED}" ]; then
-            echo "d /var/roothome 0755 root root -" >>${tmpfiles_conf}
+            echo "d /var/roothome 0700 root root -" >>${tmpfiles_conf}
         else
-            echo "mkdir -p /var/roothome; chown 755 /var/roothome" >>${tmpfiles_conf}
+            echo "mkdir -p /var/roothome; chown 700 /var/roothome" >>${tmpfiles_conf}
         fi
 
         rm -rf root
         ln -sf var/roothome root
     fi
 
+    if [ -d usr/local ] && [ ! -L usr/local ]; then
+        if [ "$(ls -A usr/local)" ]; then
+            bbfatal "Data in /usr/local directory is not preserved by OSTree."
+        fi
+        rm -rf usr/local
+    fi
+
+    if [ -n "${SYSTEMD_USED}" ]; then
+        echo "d /var/usrlocal 0755 root root -" >>${tmpfiles_conf}
+    else
+        echo "mkdir -p /var/usrlocal; chown 755 /var/usrlocal" >>${tmpfiles_conf}
+    fi
+
+    dirs="bin etc games include lib man sbin share src"
+
+    for dir in ${dirs}; do
+        if [ -n "${SYSTEMD_USED}" ]; then
+            echo "d /var/usrlocal/${dir} 0755 root root -" >>${tmpfiles_conf}
+        else
+            echo "mkdir -p /var/usrlocal/${dir}; chown 755 /var/usrlocal/${dir}" >>${tmpfiles_conf}
+        fi
+    done
+
+    ln -sf ../var/usrlocal usr/local
+
     if [ "${KERNEL_IMAGETYPE}" = "fitImage" ]; then
         # this is a hack for ostree not to override init= in kernel cmdline -
         # make it think that the initramfs is present (while it is in FIT image)
@@ -229,6 +254,12 @@ IMAGE_CMD_garagesign () {
                                     ${target_url} \
                                     --sha256 ${ostree_target_hash} \
                                     --hardwareids ${SOTA_HARDWARE_ID}
+            if [ -n "${GARAGE_CUSTOMIZE_TARGET}" ]; then
+                bbplain "Running command(${GARAGE_CUSTOMIZE_TARGET}) to customize target"
+                ${GARAGE_CUSTOMIZE_TARGET} \
+                    ${GARAGE_SIGN_REPO}/tufrepo/roles/unsigned/targets.json \
+                    ${GARAGE_TARGET_NAME}-${target_version}
+            fi
             garage-sign targets sign --repo tufrepo \
                                      --home-dir ${GARAGE_SIGN_REPO} \
                                      --key-name=targets
diff --git a/classes/image_types_ota.bbclass b/classes/image_types_ota.bbclass
index 12375ec..894919d 100644
--- a/classes/image_types_ota.bbclass
+++ b/classes/image_types_ota.bbclass
@@ -45,6 +45,7 @@ do_image_ota[cleandirs] = "${OTA_SYSROOT}"
 do_image_ota[depends] = "${@'grub:do_populate_sysroot' if d.getVar('OSTREE_BOOTLOADER') == 'grub' else ''} \
                          ${@'virtual/bootloader:do_deploy' if d.getVar('OSTREE_BOOTLOADER') == 'u-boot' else ''}"
 IMAGE_CMD_ota () {
+        export OSTREE_BOOT_PARTITION=${OSTREE_BOOT_PARTITION}
         ostree admin --sysroot=${OTA_SYSROOT} init-fs ${OTA_SYSROOT}
         ostree admin --sysroot=${OTA_SYSROOT} os-init ${OSTREE_OSNAME}
         mkdir -p ${OTA_SYSROOT}/boot/loader.0
diff --git a/classes/sota.bbclass b/classes/sota.bbclass
index bf27b6d..5506428 100644
--- a/classes/sota.bbclass
+++ b/classes/sota.bbclass
@@ -38,11 +38,11 @@ EXTRA_IMAGEDEPENDS_append_sota = " parted-native mtools-native dosfstools-native
 INITRAMFS_FSTYPES ?= "${@oe.utils.ifelse(d.getVar('OSTREE_BOOTLOADER') == 'u-boot', 'cpio.gz.u-boot', 'cpio.gz')}"
 
 # Please redefine OSTREE_REPO in order to have a persistent OSTree repo
-export OSTREE_REPO ?= "${DEPLOY_DIR_IMAGE}/ostree_repo"
-export OSTREE_BRANCHNAME ?= "${SOTA_HARDWARE_ID}"
-export OSTREE_OSNAME ?= "poky"
-export OSTREE_BOOTLOADER ??= 'u-boot'
-export OSTREE_BOOT_PARTITION ??= "/boot"
+OSTREE_REPO ?= "${DEPLOY_DIR_IMAGE}/ostree_repo"
+OSTREE_BRANCHNAME ?= "${SOTA_HARDWARE_ID}"
+OSTREE_OSNAME ?= "poky"
+OSTREE_BOOTLOADER ??= 'u-boot'
+OSTREE_BOOT_PARTITION ??= "/boot"
 
 INITRAMFS_IMAGE ?= "initramfs-ostree-image"
 
@@ -51,6 +51,7 @@ GARAGE_SIGN_KEYNAME ?= "garage-key"
 GARAGE_TARGET_NAME ?= "${OSTREE_BRANCHNAME}"
 GARAGE_TARGET_VERSION ?= ""
 GARAGE_TARGET_URL ?= ""
+GARAGE_CUSTOMIZE_TARGET ?= ""
 
 SOTA_MACHINE ??="none"
 SOTA_MACHINE_rpi ?= "raspberrypi"
diff --git a/conf/local.conf.sample.append b/conf/local.conf.sample.append
index 4588ec3..fc565f8 100644
--- a/conf/local.conf.sample.append
+++ b/conf/local.conf.sample.append
@@ -8,7 +8,7 @@ MACHINE = "##MACHINE##"
 DISTRO = "poky-sota-systemd"
 
 # General SOTA setup
-#SOTA_CLIENT_PROV = "aktualizr-auto-prov"
+#SOTA_CLIENT_PROV = "aktualizr-shared-prov"
 #SOTA_PACKED_CREDENTIALS = "/path/to/credentials.zip"
 
 # Uncomment this line to start an ssh server at boot automatically
diff --git a/lib/oeqa/selftest/cases/testutils.py b/lib/oeqa/selftest/cases/testutils.py
index 8d618a6..802d665 100644
--- a/lib/oeqa/selftest/cases/testutils.py
+++ b/lib/oeqa/selftest/cases/testutils.py
@@ -119,20 +119,29 @@ def verifyNotProvisioned(testInst, machine):
                       'Device already provisioned!? ' + stderr.decode() + stdout.decode())
 
 
-def verifyProvisioned(testInst, machine):
+def verifyProvisioned(testInst, machine, hwid=''):
     # Verify that device HAS provisioned.
+    # First loop while waiting for the device to boot.
     ran_ok = False
     for delay in [5, 5, 5, 5, 10, 10, 10, 10]:
         stdout, stderr, retcode = testInst.qemu_command('aktualizr-info')
-        if retcode == 0 and stderr == b'' and stdout.decode().find('Fetched metadata: yes') >= 0:
+        if retcode == 0 and stderr == b'':
             ran_ok = True
             break
         sleep(delay)
     testInst.assertTrue(ran_ok, 'aktualizr-info failed: ' + stderr.decode() + stdout.decode())
-
+    # Then wait for aktualizr to provision.
+    if stdout.decode().find('Fetched metadata: yes') < 0:
+        stdout, stderr, retcode = testInst.qemu_command('aktualizr-info --wait-until-provisioned')
+        testInst.assertFalse(retcode, 'aktualizr-info failed: ' + stderr.decode() + stdout.decode())
+        testInst.assertEqual(stderr, b'', 'aktualizr-info failed: ' + stderr.decode() + stdout.decode())
     testInst.assertIn(b'Device ID: ', stdout, 'Provisioning failed: ' + stderr.decode() + stdout.decode())
-    testInst.assertIn(b'Primary ecu hardware ID: ' + machine.encode(), stdout,
-                      'Provisioning failed: ' + stderr.decode() + stdout.decode())
+    if hwid == '':
+        testInst.assertIn(b'Primary ecu hardware ID: ' + machine.encode(), stdout,
+                  'Provisioning failed: ' + stderr.decode() + stdout.decode())
+    else:
+        testInst.assertIn(b'Primary ecu hardware ID: ' + hwid.encode(), stdout,
+                  'Provisioning failed: ' + stderr.decode() + stdout.decode())
     testInst.assertIn(b'Fetched metadata: yes', stdout, 'Provisioning failed: ' + stderr.decode() + stdout.decode())
     p = re.compile(r'Device ID: ([a-z0-9-]*)\n')
     m = p.search(stdout.decode())
diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py
index 2b4726c..24abcff 100644
--- a/lib/oeqa/selftest/cases/updater_qemux86_64.py
+++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py
@@ -85,6 +85,7 @@ class SharedCredProvTests(OESelftestTestCase):
         self.append_config('MACHINE = "qemux86-64"')
         self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "')
         self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"')
+        self.append_config('SOTA_HARDWARE_ID = "plain_reibekuchen_314"')
         self.qemu, self.s = qemu_launch(machine='qemux86-64')
 
     def tearDownLocal(self):
@@ -107,7 +108,8 @@ class SharedCredProvTests(OESelftestTestCase):
         self.assertEqual(value, machine,
                          'MACHINE does not match hostname: ' + machine + ', ' + value)
 
-        verifyProvisioned(self, machine)
+        hwid = get_bb_var('SOTA_HARDWARE_ID')
+        verifyProvisioned(self, machine, hwid)
 
 
 class ManualControlTests(OESelftestTestCase):
@@ -358,7 +360,7 @@ class IpSecondaryTests(OESelftestTestCase):
             self._test_ctx.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "')
 
         def is_ecu_registered(self, ecu_id):
-            max_number_of_tries = 40
+            max_number_of_tries = 120
             try_counter = 0
 
             # aktualizr-info is not always able to load ECU serials from DB
diff --git a/recipes-core/images/initramfs-ostree-image.bb b/recipes-core/images/initramfs-ostree-image.bb
index e77499e..b21be1c 100644
--- a/recipes-core/images/initramfs-ostree-image.bb
+++ b/recipes-core/images/initramfs-ostree-image.bb
@@ -15,6 +15,9 @@ LICENSE = "MIT"
 
 IMAGE_FSTYPES = "${INITRAMFS_FSTYPES}"
 
+# Avoid circular dependencies
+EXTRA_IMAGEDEPENDS = ""
+
 inherit core-image
 
 IMAGE_ROOTFS_SIZE = "8192"
diff --git a/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb b/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb
index dbb5fde..2701c07 100644
--- a/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb
+++ b/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb
@@ -14,14 +14,17 @@ require credentials.inc
 do_install() {
     if [ -n "${SOTA_PACKED_CREDENTIALS}" ]; then
         install -m 0700 -d ${D}${localstatedir}/sota
-        cp "${SOTA_PACKED_CREDENTIALS}" ${D}${localstatedir}/sota/sota_provisioning_credentials.zip
-        # Device should not be able to push data to treehub
-        zip -d ${D}${localstatedir}/sota/sota_provisioning_credentials.zip treehub.json
-        # Device has no use for the API Gateway. Remove if present. See:
-        # https://github.com/advancedtelematic/ota-plus-server/pull/1913/
-        if unzip -l ${D}${localstatedir}/sota/sota_provisioning_credentials.zip api_gateway.url > /dev/null; then
-            zip -d ${D}${localstatedir}/sota/sota_provisioning_credentials.zip api_gateway.url 
-        fi
+        # root.json contains the root metadata for bootstrapping the Uptane metadata verification process.
+        # autoprov.url has the URL to the device gateway on the server, which is where we send most of our requests.
+        # autoprov_credentials.p12 contains the shared provisioning credentials.
+        for var in root.json autoprov.url autoprov_credentials.p12; do
+            if unzip -l "${SOTA_PACKED_CREDENTIALS}" $var > /dev/null; then
+                unzip "${SOTA_PACKED_CREDENTIALS}" $var -d ${T}
+                zip -mj -q ${D}${localstatedir}/sota/sota_provisioning_credentials.zip ${T}/$var
+            else
+                bbwarn "$var is missing from credentials.zip"
+            fi
+        done
     fi
 }
 
diff --git a/recipes-sota/aktualizr/aktualizr_git.bb b/recipes-sota/aktualizr/aktualizr_git.bb
index 5de341e..9adfce8 100644
--- a/recipes-sota/aktualizr/aktualizr_git.bb
+++ b/recipes-sota/aktualizr/aktualizr_git.bb
@@ -15,7 +15,7 @@ RDEPENDS_${PN}-ptest += "bash cmake curl python3-misc python3-modules openssl-bi
 PV = "1.0+git${SRCPV}"
 PR = "7"
 
-GARAGE_SIGN_PV = "0.7.0-3-gf5ba640"
+GARAGE_SIGN_PV = "0.7.0-19-g89ec974"
 
 SRC_URI = " \
   gitsm://github.com/advancedtelematic/aktualizr;branch=${BRANCH} \
@@ -28,10 +28,10 @@ SRC_URI = " \
   "
 
 # for garage-sign archive
-SRC_URI[md5sum] = "e104ccd4f32e52571a5fc0e5042db050"
-SRC_URI[sha256sum] = "c590be1a57523bfe097af82279eda5c97cf40ae47fb27162cf33c469702c8a9b"
+SRC_URI[md5sum] = "3ed27c1142860cd9b4a2594067312b8b"
+SRC_URI[sha256sum] = "e54eef3863118f373c3ebd9e2877f9de5bab4950ed157a15fb4f4ec575bc2ece"
 
-SRCREV = "9c592cf9d8dfcd995d47753f2be7bd1a2b56c7da"
+SRCREV = "03778511cc937d07bf53a8092f8b268e65f5d9a6"
 BRANCH ?= "master"
 
 S = "${WORKDIR}/git"
diff --git a/scripts/qemucommand.py b/scripts/qemucommand.py
index 3045b45..a869d4d 100644
--- a/scripts/qemucommand.py
+++ b/scripts/qemucommand.py
@@ -1,4 +1,4 @@
-from os.path import exists, join, realpath, abspath
+from os.path import exists, isdir, join, realpath, abspath
 from os import listdir
 import random
 import socket
@@ -42,6 +42,8 @@ class QemuCommand(object):
         if args.machine:
             self.machine = args.machine
         else:
+            if not isdir(args.dir):
+                raise ValueError("Directory %s does not exist, please specify a --machine or a valid images directory" % args.dir)
             machines = listdir(args.dir)
             if len(machines) == 1:
                 self.machine = machines[0]
@@ -100,12 +102,8 @@ class QemuCommand(object):
         cmdline += [
             "-serial", "tcp:127.0.0.1:%d,server,nowait" % self.serial_port,
             "-m", self.mem,
-            "-usb",
             "-object", "rng-random,id=rng0,filename=/dev/urandom",
             "-device", "virtio-rng-pci,rng=rng0",
-            "-device", "usb-tablet",
-            "-show-cursor",
-            "-vga", "std",
             "-net", netuser,
             "-net", "nic,macaddr=%s" % self.mac_address
         ]
@@ -117,9 +115,17 @@ class QemuCommand(object):
                 '-device', 'e1000,netdev=vlan1,mac='+random_mac(),
             ]
         if self.gui:
-            cmdline += ["-serial", "stdio"]
+            cmdline += [
+                    "-usb",
+                    "-device", "usb-tablet",
+                    "-show-cursor",
+                    "-vga", "std"
+            ]
         else:
-            cmdline.append('-nographic')
+            cmdline += [
+                    "-nographic",
+                    "-monitor", "null",
+            ]
         if self.kvm:
             cmdline += ['-enable-kvm', '-cpu', 'host']
         else: