Backport/sumo 19 08 14 (#580)

Backport/sumo 19 08 14

11 files changed, 128 insertions, 56 deletions

diff --git a/README.adoc b/README.adoc
index 8711e86..8bded9a 100644
--- a/README.adoc
+++ b/README.adoc
@@ -61,7 +61,7 @@ and get as a result an `ostree_repo` folder in your images directory (`tmp/deplo
 
 * your OSTree repository, with the rootfs committed as an OSTree deployment,
 * an `otaimg` bootstrap image, which is an OSTree physical sysroot as a burnable filesystem image, and optionally
-* some machine-dependent live images (e.g. `.wic` for Raspberry Pi or `.porter-sdimg-ota` Renesas Porter board).
+* some machine-dependent live images (e.g. `.wic` for Raspberry Pi or `.porter-sdimg-ota` for Renesas Porter board).
 
 Although `aglsetup.sh` hooks provide reasonable defaults for SOTA-related variables, you may want to tune some of them.
 
@@ -71,11 +71,16 @@ Ubuntu users that encounter an error due to missing `Python.h` should install `l
 
 == Supported boards
 
-Currently supported platforms are
+Currently supported platforms are:
 
-* https://github.com/advancedtelematic/meta-updater-raspberrypi[Raspberry Pi3]
-* https://github.com/advancedtelematic/meta-updater-minnowboard[Minnowboard]
+* https://github.com/advancedtelematic/meta-updater-raspberrypi[Raspberry Pi 2 and 3]
+* https://github.com/advancedtelematic/meta-updater-minnowboard[Intel Minnowboard]
 * https://github.com/advancedtelematic/meta-updater-qemux86-64[Native QEMU emulation]
+* Renesas R-Car H3 and M3
+* https://github.com/advancedtelematic/meta-updater-ti/[TI BeagleBone Black] (rocko only, using TI SDK 05.03)
+* https://github.com/advancedtelematic/meta-updater-ti/[TI AM65x industrial development kit] (rocko only, using TI SDK 05.03)
+
+We also historically supported the https://github.com/advancedtelematic/meta-updater-porter[Renesas Porter] board.
 
 === Adding support for your board
 
@@ -92,6 +97,7 @@ Your images will also need network connectivity to be able to reach an actual OT
 
 == SOTA-related variables in local.conf
 
+* `OSTREE_BRANCHNAME` - OSTree branch name. Defaults to `${SOTA_HARDWARE_ID}`. Particularly useful for grouping similar images.
 * `OSTREE_REPO` - path to your OSTree repository. Defaults to `$\{DEPLOY_DIR_IMAGE}/ostree_repo`
 * `OSTREE_OSNAME` - OS deployment name on your target device. For more information about deployments and osnames see the https://ostree.readthedocs.io/en/latest/manual/deployment/[OSTree documentation]. Defaults to "poky".
 * `OSTREE_INITRAMFS_IMAGE` - initramfs/initrd image that is used as a proxy while booting into OSTree deployment. Do not change this setting unless you are sure that your initramfs can serve as such a proxy.
@@ -106,7 +112,7 @@ Your images will also need network connectivity to be able to reach an actual OT
 * `SOTA_SECONDARY_CONFIG` - a file containing JSON configuration for secondaries. It will be installed into `/etc/sota/ecus` on the device and automatically provided to aktualizr. See link:https://github.com/advancedtelematic/aktualizr/blob/master/docs/posix-secondaries-bitbaking.adoc[here] for more details.
 * `SOTA_HARDWARE_ID` - a custom hardware ID that will be written to the aktualizr config. Defaults to MACHINE if not set.
 * `RESOURCE_xxx_pn-aktualizr` - controls maximum resource usage of the aktualizr service, when `aktualizr-resource-control` is installed on the image. See <<aktualizr service resource control>> for details.
-* `SOTA_POLLING_SEC` - sets polling interval for aktualizr to check for updates if aktualizr-polling-sec is included in the image.
+* `SOTA_POLLING_SEC` - sets polling interval for aktualizr to check for updates if aktualizr-polling-interval is included in the image.
 
 == Usage
 
@@ -187,7 +193,23 @@ RESOURCE_CPU_WEIGHT_pn-aktualizr = "50"
 
 == Development configuration
 
-There are a few settings that can be controlled in `local.conf` to simplify the development process:
+=== Logging
+
+To troubleshoot problems that you might encounter during development, we recommend that you enable persistent `systemd` logging. This setting is enabled by default for newly configured environments (see link:conf/local.conf.sample.append[]). To enable it manually, put this to your `local.conf`:
+
+....
+IMAGE_INSTALL_append += " systemd-journald-persistent"
+....
+
+It may also be helpful to run with debug logging enabled in aktualizr. To do so, add this to your `local.conf`:
+
+....
+IMAGE_INSTALL_append += " aktualizr-log-debug"
+....
+
+=== Custom aktualizr versions
+
+You can override the version of aktualizr included in your image. This requires that the version you wish to run is pushed to the https://github.com/advancedtelematic/aktualizr[aktualizr github repo]. You can then use these settings in your `local.conf` to simplify the development process:
 
 [options="header"]
 |======================
diff --git a/classes/image_types_ostree.bbclass b/classes/image_types_ostree.bbclass
index 134f5f5..edddf37 100644
--- a/classes/image_types_ostree.bbclass
+++ b/classes/image_types_ostree.bbclass
@@ -16,11 +16,11 @@ OSTREE_UPDATE_SUMMARY ??= "0"
 SYSTEMD_USED = "${@oe.utils.ifelse(d.getVar('VIRTUAL-RUNTIME_init_manager', True) == 'systemd', 'true', '')}"
 
 IMAGE_CMD_ostree () {
-    if [ -z "$OSTREE_REPO" ]; then
+    if [ -z "${OSTREE_REPO}" ]; then
         bbfatal "OSTREE_REPO should be set in your local.conf"
     fi
 
-    if [ -z "$OSTREE_BRANCHNAME" ]; then
+    if [ -z "${OSTREE_BRANCHNAME}" ]; then
         bbfatal "OSTREE_BRANCHNAME should be set in your local.conf"
     fi
 
@@ -93,15 +93,15 @@ IMAGE_CMD_ostree () {
             if [ "$(ls -A $dir)" ]; then
                 bbwarn "Data in /$dir directory is not preserved by OSTree. Consider moving it under /usr"
             fi
-
-            if [ -n "${SYSTEMD_USED}" ]; then
-                echo "d /var/rootdirs/${dir} 0755 root root -" >>${tmpfiles_conf}
-            else
-                echo "mkdir -p /var/rootdirs/${dir}; chown 755 /var/rootdirs/${dir}" >>${tmpfiles_conf}
-            fi
             rm -rf ${dir}
-            ln -sf var/rootdirs/${dir} ${dir}
         fi
+
+        if [ -n "${SYSTEMD_USED}" ]; then
+            echo "d /var/rootdirs/${dir} 0755 root root -" >>${tmpfiles_conf}
+        else
+            echo "mkdir -p /var/rootdirs/${dir}; chown 755 /var/rootdirs/${dir}" >>${tmpfiles_conf}
+        fi
+        ln -sf var/rootdirs/${dir} ${dir}
     done
 
     if [ -d root ] && [ ! -L root ]; then
@@ -110,20 +110,39 @@ IMAGE_CMD_ostree () {
         fi
 
         if [ -n "${SYSTEMD_USED}" ]; then
-            echo "d /var/roothome 0755 root root -" >>${tmpfiles_conf}
+            echo "d /var/roothome 0700 root root -" >>${tmpfiles_conf}
         else
-            echo "mkdir -p /var/roothome; chown 755 /var/roothome" >>${tmpfiles_conf}
+            echo "mkdir -p /var/roothome; chown 700 /var/roothome" >>${tmpfiles_conf}
         fi
 
         rm -rf root
         ln -sf var/roothome root
     fi
 
-    # Creating boot directories is required for "ostree admin deploy"
+    if [ -d usr/local ] && [ ! -L usr/local ]; then
+        if [ "$(ls -A usr/local)" ]; then
+            bbfatal "Data in /usr/local directory is not preserved by OSTree."
+        fi
+        rm -rf usr/local
+    fi
 
-    mkdir -p boot/loader.0
-    mkdir -p boot/loader.1
-    ln -sf boot/loader.0 boot/loader
+    if [ -n "${SYSTEMD_USED}" ]; then
+        echo "d /var/usrlocal 0755 root root -" >>${tmpfiles_conf}
+    else
+        echo "mkdir -p /var/usrlocal; chown 755 /var/usrlocal" >>${tmpfiles_conf}
+    fi
+
+    dirs="bin etc games include lib man sbin share src"
+
+    for dir in ${dirs}; do
+        if [ -n "${SYSTEMD_USED}" ]; then
+            echo "d /var/usrlocal/${dir} 0755 root root -" >>${tmpfiles_conf}
+        else
+            echo "mkdir -p /var/usrlocal/${dir}; chown 755 /var/usrlocal/${dir}" >>${tmpfiles_conf}
+        fi
+    done
+
+    ln -sf ../var/usrlocal usr/local
 
     checksum=`sha256sum ${DEPLOY_DIR_IMAGE}/${OSTREE_KERNEL} | cut -f 1 -d " "`
 
@@ -241,6 +260,12 @@ IMAGE_CMD_garagesign () {
                                     ${target_url} \
                                     --sha256 ${ostree_target_hash} \
                                     --hardwareids ${SOTA_HARDWARE_ID}
+            if [ -n "${GARAGE_CUSTOMIZE_TARGET}" ]; then
+                bbplain "Running command(${GARAGE_CUSTOMIZE_TARGET}) to customize target"
+                ${GARAGE_CUSTOMIZE_TARGET} \
+                    ${GARAGE_SIGN_REPO}/tufrepo/roles/unsigned/targets.json \
+                    ${GARAGE_TARGET_NAME}-${target_version}
+            fi
             garage-sign targets sign --repo tufrepo \
                                      --home-dir ${GARAGE_SIGN_REPO} \
                                      --key-name=targets
diff --git a/classes/image_types_ota.bbclass b/classes/image_types_ota.bbclass
index b14d2bb..9f4bd40 100644
--- a/classes/image_types_ota.bbclass
+++ b/classes/image_types_ota.bbclass
@@ -48,18 +48,19 @@ IMAGE_CMD_otaimg () {
         if ${@bb.utils.contains('IMAGE_FSTYPES', 'otaimg', 'false', 'true', d)}; then
                 return
         fi
-        if [ -z "$OSTREE_REPO" ]; then
+        if [ -z "${OSTREE_REPO}" ]; then
                 bbfatal "OSTREE_REPO should be set in your local.conf"
         fi
 
-        if [ -z "$OSTREE_OSNAME" ]; then
+        if [ -z "${OSTREE_OSNAME}" ]; then
                 bbfatal "OSTREE_OSNAME should be set in your local.conf"
         fi
 
-        if [ -z "$OSTREE_BRANCHNAME" ]; then
+        if [ -z "${OSTREE_BRANCHNAME}" ]; then
                 bbfatal "OSTREE_BRANCHNAME should be set in your local.conf"
         fi
 
+        export OSTREE_BOOT_PARTITION=${OSTREE_BOOT_PARTITION}
         PHYS_SYSROOT=`mktemp -d ${WORKDIR}/ota-sysroot-XXXXX`
 
         ostree admin --sysroot=${PHYS_SYSROOT} init-fs ${PHYS_SYSROOT}
diff --git a/classes/sota.bbclass b/classes/sota.bbclass
index 4465e6c..85401d5 100644
--- a/classes/sota.bbclass
+++ b/classes/sota.bbclass
@@ -40,11 +40,11 @@ EXTRA_IMAGEDEPENDS_append_sota = " parted-native mtools-native dosfstools-native
 OSTREE_INITRAMFS_FSTYPES ??= "${@oe.utils.ifelse(d.getVar('OSTREE_BOOTLOADER', True) == 'u-boot', 'ext4.gz.u-boot', 'ext4.gz')}"
 
 # Please redefine OSTREE_REPO in order to have a persistent OSTree repo
-export OSTREE_REPO ?= "${DEPLOY_DIR_IMAGE}/ostree_repo"
-export OSTREE_BRANCHNAME ?= "${SOTA_HARDWARE_ID}"
-export OSTREE_OSNAME ?= "poky"
-export OSTREE_BOOTLOADER ??= 'u-boot'
-export OSTREE_BOOT_PARTITION ??= "/boot"
+OSTREE_REPO ?= "${DEPLOY_DIR_IMAGE}/ostree_repo"
+OSTREE_BRANCHNAME ?= "${SOTA_HARDWARE_ID}"
+OSTREE_OSNAME ?= "poky"
+OSTREE_BOOTLOADER ??= 'u-boot'
+OSTREE_BOOT_PARTITION ??= "/boot"
 
 OSTREE_INITRAMFS_IMAGE ?= "initramfs-ostree-image"
 
@@ -53,6 +53,7 @@ GARAGE_SIGN_KEYNAME ?= "garage-key"
 GARAGE_TARGET_NAME ?= "${OSTREE_BRANCHNAME}"
 GARAGE_TARGET_VERSION ?= ""
 GARAGE_TARGET_URL ?= ""
+GARAGE_CUSTOMIZE_TARGET ?= ""
 
 SOTA_MACHINE ??="none"
 SOTA_MACHINE_rpi ?= "raspberrypi"
diff --git a/conf/local.conf.sample.append b/conf/local.conf.sample.append
index 4588ec3..fc565f8 100644
--- a/conf/local.conf.sample.append
+++ b/conf/local.conf.sample.append
@@ -8,7 +8,7 @@ MACHINE = "##MACHINE##"
 DISTRO = "poky-sota-systemd"
 
 # General SOTA setup
-#SOTA_CLIENT_PROV = "aktualizr-auto-prov"
+#SOTA_CLIENT_PROV = "aktualizr-shared-prov"
 #SOTA_PACKED_CREDENTIALS = "/path/to/credentials.zip"
 
 # Uncomment this line to start an ssh server at boot automatically
diff --git a/lib/oeqa/selftest/cases/testutils.py b/lib/oeqa/selftest/cases/testutils.py
index 8d618a6..802d665 100644
--- a/lib/oeqa/selftest/cases/testutils.py
+++ b/lib/oeqa/selftest/cases/testutils.py
@@ -119,20 +119,29 @@ def verifyNotProvisioned(testInst, machine):
                       'Device already provisioned!? ' + stderr.decode() + stdout.decode())
 
 
-def verifyProvisioned(testInst, machine):
+def verifyProvisioned(testInst, machine, hwid=''):
     # Verify that device HAS provisioned.
+    # First loop while waiting for the device to boot.
     ran_ok = False
     for delay in [5, 5, 5, 5, 10, 10, 10, 10]:
         stdout, stderr, retcode = testInst.qemu_command('aktualizr-info')
-        if retcode == 0 and stderr == b'' and stdout.decode().find('Fetched metadata: yes') >= 0:
+        if retcode == 0 and stderr == b'':
             ran_ok = True
             break
         sleep(delay)
     testInst.assertTrue(ran_ok, 'aktualizr-info failed: ' + stderr.decode() + stdout.decode())
-
+    # Then wait for aktualizr to provision.
+    if stdout.decode().find('Fetched metadata: yes') < 0:
+        stdout, stderr, retcode = testInst.qemu_command('aktualizr-info --wait-until-provisioned')
+        testInst.assertFalse(retcode, 'aktualizr-info failed: ' + stderr.decode() + stdout.decode())
+        testInst.assertEqual(stderr, b'', 'aktualizr-info failed: ' + stderr.decode() + stdout.decode())
     testInst.assertIn(b'Device ID: ', stdout, 'Provisioning failed: ' + stderr.decode() + stdout.decode())
-    testInst.assertIn(b'Primary ecu hardware ID: ' + machine.encode(), stdout,
-                      'Provisioning failed: ' + stderr.decode() + stdout.decode())
+    if hwid == '':
+        testInst.assertIn(b'Primary ecu hardware ID: ' + machine.encode(), stdout,
+                  'Provisioning failed: ' + stderr.decode() + stdout.decode())
+    else:
+        testInst.assertIn(b'Primary ecu hardware ID: ' + hwid.encode(), stdout,
+                  'Provisioning failed: ' + stderr.decode() + stdout.decode())
     testInst.assertIn(b'Fetched metadata: yes', stdout, 'Provisioning failed: ' + stderr.decode() + stdout.decode())
     p = re.compile(r'Device ID: ([a-z0-9-]*)\n')
     m = p.search(stdout.decode())
diff --git a/lib/oeqa/selftest/cases/updater_qemux86_64.py b/lib/oeqa/selftest/cases/updater_qemux86_64.py
index 2b4726c..24abcff 100644
--- a/lib/oeqa/selftest/cases/updater_qemux86_64.py
+++ b/lib/oeqa/selftest/cases/updater_qemux86_64.py
@@ -85,6 +85,7 @@ class SharedCredProvTests(OESelftestTestCase):
         self.append_config('MACHINE = "qemux86-64"')
         self.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "')
         self.append_config('IMAGE_FSTYPES_remove = "ostreepush garagesign garagecheck"')
+        self.append_config('SOTA_HARDWARE_ID = "plain_reibekuchen_314"')
         self.qemu, self.s = qemu_launch(machine='qemux86-64')
 
     def tearDownLocal(self):
@@ -107,7 +108,8 @@ class SharedCredProvTests(OESelftestTestCase):
         self.assertEqual(value, machine,
                          'MACHINE does not match hostname: ' + machine + ', ' + value)
 
-        verifyProvisioned(self, machine)
+        hwid = get_bb_var('SOTA_HARDWARE_ID')
+        verifyProvisioned(self, machine, hwid)
 
 
 class ManualControlTests(OESelftestTestCase):
@@ -358,7 +360,7 @@ class IpSecondaryTests(OESelftestTestCase):
             self._test_ctx.append_config('SOTA_CLIENT_PROV = " aktualizr-shared-prov "')
 
         def is_ecu_registered(self, ecu_id):
-            max_number_of_tries = 40
+            max_number_of_tries = 120
             try_counter = 0
 
             # aktualizr-info is not always able to load ECU serials from DB
diff --git a/recipes-core/images/initramfs-ostree-image.bb b/recipes-core/images/initramfs-ostree-image.bb
index b2d9e27..26a70d3 100644
--- a/recipes-core/images/initramfs-ostree-image.bb
+++ b/recipes-core/images/initramfs-ostree-image.bb
@@ -15,6 +15,9 @@ LICENSE = "MIT"
 
 IMAGE_FSTYPES = "${OSTREE_INITRAMFS_FSTYPES}"
 
+# Avoid circular dependencies
+EXTRA_IMAGEDEPENDS = ""
+
 inherit core-image
 
 IMAGE_ROOTFS_SIZE = "8192"
diff --git a/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb b/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb
index dbb5fde..2701c07 100644
--- a/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb
+++ b/recipes-sota/aktualizr/aktualizr-shared-prov-creds.bb
@@ -14,14 +14,17 @@ require credentials.inc
 do_install() {
     if [ -n "${SOTA_PACKED_CREDENTIALS}" ]; then
         install -m 0700 -d ${D}${localstatedir}/sota
-        cp "${SOTA_PACKED_CREDENTIALS}" ${D}${localstatedir}/sota/sota_provisioning_credentials.zip
-        # Device should not be able to push data to treehub
-        zip -d ${D}${localstatedir}/sota/sota_provisioning_credentials.zip treehub.json
-        # Device has no use for the API Gateway. Remove if present. See:
-        # https://github.com/advancedtelematic/ota-plus-server/pull/1913/
-        if unzip -l ${D}${localstatedir}/sota/sota_provisioning_credentials.zip api_gateway.url > /dev/null; then
-            zip -d ${D}${localstatedir}/sota/sota_provisioning_credentials.zip api_gateway.url 
-        fi
+        # root.json contains the root metadata for bootstrapping the Uptane metadata verification process.
+        # autoprov.url has the URL to the device gateway on the server, which is where we send most of our requests.
+        # autoprov_credentials.p12 contains the shared provisioning credentials.
+        for var in root.json autoprov.url autoprov_credentials.p12; do
+            if unzip -l "${SOTA_PACKED_CREDENTIALS}" $var > /dev/null; then
+                unzip "${SOTA_PACKED_CREDENTIALS}" $var -d ${T}
+                zip -mj -q ${D}${localstatedir}/sota/sota_provisioning_credentials.zip ${T}/$var
+            else
+                bbwarn "$var is missing from credentials.zip"
+            fi
+        done
     fi
 }
 
diff --git a/recipes-sota/aktualizr/aktualizr_git.bb b/recipes-sota/aktualizr/aktualizr_git.bb
index ddc0de1..8d1decd 100644
--- a/recipes-sota/aktualizr/aktualizr_git.bb
+++ b/recipes-sota/aktualizr/aktualizr_git.bb
@@ -15,7 +15,7 @@ RDEPENDS_${PN}-ptest += "bash cmake curl python3-misc python3-modules sqlite3 va
 PV = "1.0+git${SRCPV}"
 PR = "7"
 
-GARAGE_SIGN_PV = "0.7.0-3-gf5ba640"
+GARAGE_SIGN_PV = "0.7.0-19-g89ec974"
 
 SRC_URI = " \
   gitsm://github.com/advancedtelematic/aktualizr;branch=${BRANCH} \
@@ -28,10 +28,10 @@ SRC_URI = " \
   "
 
 # for garage-sign archive
-SRC_URI[md5sum] = "e104ccd4f32e52571a5fc0e5042db050"
-SRC_URI[sha256sum] = "c590be1a57523bfe097af82279eda5c97cf40ae47fb27162cf33c469702c8a9b"
+SRC_URI[md5sum] = "3ed27c1142860cd9b4a2594067312b8b"
+SRC_URI[sha256sum] = "e54eef3863118f373c3ebd9e2877f9de5bab4950ed157a15fb4f4ec575bc2ece"
 
-SRCREV = "9c592cf9d8dfcd995d47753f2be7bd1a2b56c7da"
+SRCREV = "03778511cc937d07bf53a8092f8b268e65f5d9a6"
 BRANCH ?= "master"
 
 S = "${WORKDIR}/git"
diff --git a/scripts/qemucommand.py b/scripts/qemucommand.py
index 8b7da2b..df6907c 100644
--- a/scripts/qemucommand.py
+++ b/scripts/qemucommand.py
@@ -1,4 +1,4 @@
-from os.path import exists, join, realpath, abspath
+from os.path import exists, isdir, join, realpath, abspath
 from os import listdir
 import random
 import socket
@@ -42,6 +42,8 @@ class QemuCommand(object):
         if args.machine:
             self.machine = args.machine
         else:
+            if not isdir(args.dir):
+                raise ValueError("Directory %s does not exist, please specify a --machine or a valid images directory" % args.dir)
             machines = listdir(args.dir)
             if len(machines) == 1:
                 self.machine = machines[0]
@@ -100,12 +102,8 @@ class QemuCommand(object):
         cmdline += [
             "-serial", "tcp:127.0.0.1:%d,server,nowait" % self.serial_port,
             "-m", self.mem,
-            "-usb",
             "-object", "rng-random,id=rng0,filename=/dev/urandom",
             "-device", "virtio-rng-pci,rng=rng0",
-            "-device", "usb-tablet",
-            "-show-cursor",
-            "-vga", "std",
             "-net", netuser,
             "-net", "nic,macaddr=%s" % self.mac_address
         ]
@@ -117,9 +115,17 @@ class QemuCommand(object):
                 '-device', 'e1000,netdev=vlan1,mac='+random_mac(),
             ]
         if self.gui:
-            cmdline += ["-serial", "stdio"]
+            cmdline += [
+                    "-usb",
+                    "-device", "usb-tablet",
+                    "-show-cursor",
+                    "-vga", "std"
+            ]
         else:
-            cmdline.append('-nographic')
+            cmdline += [
+                    "-nographic",
+                    "-monitor", "null",
+            ]
         if self.kvm:
             cmdline += ['-enable-kvm', '-cpu', 'host']
         else: