Merge pull request #537 from advancedtelematic/ci/oe-selftest

Ci/oe selftest

13 files changed, 241 insertions, 20 deletions

diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake
index c91f94c..51eaa57 100644
--- a/scripts/ci/Dockerfile.bitbake
+++ b/scripts/ci/Dockerfile.bitbake
@@ -1,15 +1,17 @@
-FROM debian:stable
+FROM debian:stretch
 LABEL Description="Image for bitbaking"
 
-RUN sed -i 's#deb http://deb.debian.org/debian stable main#deb http://deb.debian.org/debian stable main contrib#g' /etc/apt/sources.list
-RUN sed -i 's#deb http://deb.debian.org/debian stable-updates main#deb http://deb.debian.org/debian stable-updates main contrib#g' /etc/apt/sources.list
-RUN apt-get update -q && apt-get install -qy \
+RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
+RUN apt-get update -q && apt-get install --no-install-suggests --no-install-recommends -qy \
+     awscli \
      build-essential \
      bzip2 \
      chrpath \
      cpio \
      default-jre \
      diffstat \
+     file \
      gawk \
      gcc-multilib \
      git-core \
@@ -17,26 +19,40 @@ RUN apt-get update -q && apt-get install -qy \
      iproute \
      libpython-dev \
      libsdl1.2-dev \
+     libvirt-clients \
+     libvirt-daemon-system \
      locales \
      ovmf \
+     openssh-client \
      procps \
      python \
      python3 \
      python3-pexpect \
-     qemu \
+     qemu-kvm \
      socat \
+     sudo \
      texinfo \
      unzip \
      wget \
      xterm \
      xz-utils
 
-ARG uid=1000
-ARG gid=1000
+ARG uid=4321
+ARG gid=4321
 RUN groupadd -g $gid bitbake
-RUN useradd -m -u $uid -g $gid bitbake
+RUN useradd -m -u $uid -g $gid -s /bin/bash bitbake
 
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
 ENV LC_ALL="en_US.UTF-8"
 ENV LANG="en_US.UTF-8"
 ENV LANGUAGE="en_US.UTF-8"
+
+# script to mirror kvm group id with host
+RUN echo "bitbake ALL=NOPASSWD: /usr/local/bin/setup_kvm.sh" >> /etc/sudoers
+COPY ./docker/setup_kvm.sh /usr/local/bin/setup_kvm.sh
+
+# other ci scripts
+RUN mkdir /scripts
+COPY configure.sh build.sh oe-selftest.sh /scripts/
+
+USER "bitbake"
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
new file mode 100644
index 0000000..5210c6b
--- /dev/null
+++ b/scripts/ci/Dockerfile.checkout
@@ -0,0 +1,13 @@
+FROM debian:stretch-slim
+LABEL Description="Image for checking out updater-repo"
+
+RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
+RUN apt-get update -q && apt-get install -qy \
+    git \
+    repo \
+    xmlstarlet
+
+# checkout script
+RUN mkdir /scripts
+COPY checkout-oe.sh /scripts/
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index 6235428..9fbae98 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -12,7 +12,10 @@ IMAGE_NAME=${1:-core-image-minimal}
 (
 set +euo pipefail
 set +x
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
 
+set -x
 bitbake "${IMAGE_NAME}"
 )
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
new file mode 100755
index 0000000..a99f235
--- /dev/null
+++ b/scripts/ci/checkout-oe.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+set -x
+
+REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
+MANIFEST=${MANIFEST:-master}
+CURRENT_PROJECT=${CURRENT_PROJECT:-}
+
+# list of projects to pin to one version in the format:
+# "project:rev;project2:rev2..."
+PIN_LIST=${PIN_LIST:-}
+
+#CURRENT_REV=$(git rev-parse HEAD)
+LOCAL_REPO=$PWD
+
+mkdir -p updater-repo
+
+cd updater-repo
+
+repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo"
+
+git -C .repo/manifests reset --hard
+
+# patch manifest:
+# - add a new "ats" remote that points to "$REMOTE_SOURCE"
+# - change projects that contain "advancedtelematic" to use the ats remote
+MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
+xmlstarlet ed --omit-decl -L \
+    -s "/manifest" -t elem -n "remote" -v "" \
+    -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
+    -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
+    -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \
+    -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \
+    "$MANIFEST_FILE"
+
+# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix
+sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
+
+# pin projects from the list
+(
+IFS=";"
+for pin in $PIN_LIST; do
+    IFS=":"
+    read -r project rev <<< "$pin"
+    xmlstarlet ed --omit-decl -L \
+        -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \
+        -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \
+        "$MANIFEST_FILE"
+    IFS=";"
+done
+)
+
+# Remove the current project from the manifest if we have it checked out
+if [ -n "$CURRENT_PROJECT" ]; then
+    xmlstarlet ed --omit-decl -L \
+        -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \
+        "$MANIFEST_FILE"
+fi
+
+repo manifest
+
+repo forall -c 'git reset --hard ; git clean -fdx'
+
+repo sync -d --force-sync
+
+if [ -n "$CURRENT_PROJECT" ]; then
+    rm -f "$CURRENT_PROJECT"
+    ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
+fi
+
+repo manifest -r
diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh
index 960a0cc..a8968da 100755
--- a/scripts/ci/configure.sh
+++ b/scripts/ci/configure.sh
@@ -8,9 +8,13 @@ TEST_BUILD_DIR=${TEST_BUILD_DIR:-build}
 TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
 TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-}
 
-TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
-TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
-TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
+TEST_AKTUALIZR_REMOTE=${TEST_AKTUALIZR_REMOTE:-}
+if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
+    TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
+    TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
+    TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
+fi
+
 TEST_AKTUALIZR_CREDENTIALS=${TEST_AKTUALIZR_CREDENTIALS:-}
 
 # move existing conf directory to backup, before generating a new one
@@ -21,6 +25,8 @@ mv "$TEST_BUILD_DIR/conf" "$TEST_BUILD_DIR/conf.old" || true
 set +euo pipefail
 set +x
 echo ">> Running envsetup.sh"
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "$TEST_REPO_DIR/meta-updater/scripts/envsetup.sh" "$TEST_MACHINE" "$TEST_BUILD_DIR"
 )
 
@@ -31,19 +37,19 @@ SITE_CONF="$TEST_BUILD_DIR/conf/site.conf"
 echo ">> Set common bitbake config options"
 cat << EOF > "$SITE_CONF"
 SANITY_TESTED_DISTROS = ""
-SSTATE_MIRRORS ?= "file://.* https://bitbake-cache.atsgarage.com/PATH;downloadfilename=PATH"
 IMAGE_FEATURES += "ssh-server-openssh"
 
 EOF
 
-echo ">> Set aktualizr branch in bitbake's config"
-cat << EOF >> "$SITE_CONF"
+if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
+    echo ">> Set aktualizr branch in bitbake's config"
+    cat << EOF >> "$SITE_CONF"
 SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV"
 SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}"
 BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH"
 BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}"
-
 EOF
+fi
 
 if [[ -n $TEST_AKTUALIZR_CREDENTIALS ]]; then
     echo ">> Set aktualizr credentials"
diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh
new file mode 100755
index 0000000..1ffbbf5
--- /dev/null
+++ b/scripts/ci/docker/setup_kvm.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# This script makes the gid of the 'kvm' group to match the group
+# owner of '/dev/kvm'
+#
+# These two are not guaranteed to match when a docker image starts
+# with access to '/dev/kvm' that comes from the host
+
+set -euo pipefail
+
+kvm_gid=$(stat -c "%g" /dev/kvm)
+groupmod -g "$kvm_gid" kvm
+usermod -a -G kvm bitbake
+ln -s /bin/true /usr/bin/kvm-ok
diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml
new file mode 100644
index 0000000..28c9177
--- /dev/null
+++ b/scripts/ci/gitlab/checkout.yml
@@ -0,0 +1,16 @@
+.bb_checkout:
+  # parameters:
+  #   - MANIFEST
+  #   - REMOTE_SOURCE
+  #   - CURRENT_PROJECT (will be symlinked instead of pulled)
+
+  image: $BITBAKE_CHECKOUT_IMAGE
+  cache:
+    paths:
+      - updater-repo
+  artifacts:
+    expire_in: "1 day"
+    paths:
+      - updater-repo
+  script:
+    - /scripts/checkout-oe.sh
diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml
new file mode 100644
index 0000000..827bb47
--- /dev/null
+++ b/scripts/ci/gitlab/docker.yml
@@ -0,0 +1,37 @@
+.bb_docker_local:
+  # intended to be run on meta-updater's master branch which contains the
+  # reference docker files
+  # parameters:
+  #   - BITBAKE_IMAGE
+  #   - BITBAKE_CHECKOUT_IMAGE
+  #   - BITBKAE_IMAGE_MASTER
+  #   - BITBAKE_CHECKOUT_IMAGE_MASTER
+  image: docker:stable
+  stage: docker
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+  script:
+    - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true
+    - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci
+    - docker push "$BITBAKE_IMAGE"
+
+    - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true
+    - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci
+    - docker push "$BITBAKE_CHECKOUT_IMAGE"
+
+.bb_docker_remote:
+  # intended to be run on other branches and repos: just pulls the last master image
+  # parameters:
+  #   - BITBAKE_IMAGE
+  #   - BITBAKE_CHECKOUT_IMAGE
+  image: docker:stable
+  stage: docker
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+  script:
+    - docker pull "$BITBAKE_IMAGE"
+    - docker pull "$BITBAKE_CHECKOUT_IMAGE"
diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml
new file mode 100644
index 0000000..4bd5844
--- /dev/null
+++ b/scripts/ci/gitlab/tests.yml
@@ -0,0 +1,36 @@
+.bitbake:
+  # parameters:
+  #   - TEST_BUILD_DIR
+  #   - TEST_MACHINE (defaults to qemux86-64)
+  #   - BITBAKE_TARGETS
+  image: $BITBAKE_IMAGE
+  dependencies:
+    - Checkout
+  tags:
+    - bitbake
+  script:
+    - /scripts/configure.sh
+    - /scripts/build.sh $BITBAKE_TARGETS
+
+.oe-selftest:
+  # parameters:
+  #   - TEST_BUILD_DIR
+  #   - TEST_MACHINE (defaults to qemux86-64)
+  #   - OE_SELFTESTS
+  image: $BITBAKE_IMAGE
+  dependencies:
+    - Checkout
+  tags:
+    - bitbake
+  variables:
+    TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip
+  before_script:
+    - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip
+    - sudo /usr/local/bin/setup_kvm.sh
+  script:
+    - |
+      # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh)
+      sg kvm << EOS
+      /scripts/configure.sh
+      /scripts/oe-selftest.sh $OE_SELFTESTS
+      EOS
diff --git a/scripts/ci/oe-selftest.sh b/scripts/ci/oe-selftest.sh
index 3124cce..66b1fc9 100755
--- a/scripts/ci/oe-selftest.sh
+++ b/scripts/ci/oe-selftest.sh
@@ -12,7 +12,10 @@ TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
 (
 set +euo pipefail
 set +x
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
 
-oe-selftest -r updater
+set -x
+oe-selftest -r "$@"
 )
diff --git a/scripts/envsetup.sh b/scripts/envsetup.sh
index 6103cf6..5827bc2 100755
--- a/scripts/envsetup.sh
+++ b/scripts/envsetup.sh
@@ -21,7 +21,7 @@ if [[ $SOURCED -ne 1 ]]; then
   exit 1
 fi
 
-METADIR="${SOURCEDIR}/../.."
+METADIR=${METADIR:-${SOURCEDIR}/../..}
 
 if [[ ! -f "${BUILDDIR}/conf/local.conf" ]]; then
   source "$METADIR/poky/oe-init-build-env" "$BUILDDIR"
diff --git a/scripts/qemucommand.py b/scripts/qemucommand.py
index 1049d7c..3045b45 100644
--- a/scripts/qemucommand.py
+++ b/scripts/qemucommand.py
@@ -2,7 +2,7 @@ from os.path import exists, join, realpath, abspath
 from os import listdir
 import random
 import socket
-from subprocess import check_output, CalledProcessError
+from subprocess import check_output
 
 EXTENSIONS = {
     'intel-corei7-64': 'wic',
@@ -68,6 +68,10 @@ class QemuCommand(object):
             self.mac_address = random_mac()
         self.serial_port = find_local_port(8990)
         self.ssh_port = find_local_port(2222)
+        if args.mem:
+            self.mem = args.mem
+        else:
+            self.mem = "1G"
         if args.kvm is None:
             # Autodetect KVM using 'kvm-ok'
             try:
@@ -95,7 +99,7 @@ class QemuCommand(object):
             cmdline += ["-drive", "file=%s,if=ide,format=raw,snapshot=on" % self.image]
         cmdline += [
             "-serial", "tcp:127.0.0.1:%d,server,nowait" % self.serial_port,
-            "-m", "1G",
+            "-m", self.mem,
             "-usb",
             "-object", "rng-random,id=rng0,filename=/dev/urandom",
             "-device", "virtio-rng-pci,rng=rng0",
@@ -131,4 +135,3 @@ class QemuCommand(object):
             "-f", "qcow2",
             self.overlay]
         return cmdline
-
diff --git a/scripts/run-qemu-ota b/scripts/run-qemu-ota
index b2f55e9..de63297 100755
--- a/scripts/run-qemu-ota
+++ b/scripts/run-qemu-ota
@@ -26,6 +26,7 @@ def main():
                            dest='kvm', action='store_true', default=None)
     kvm_group.add_argument('--no-kvm', help='Disable KVM in QEMU',
                            dest='kvm', action='store_false')
+    parser.add_argument('--mem', default=None, help="Amount of memory the machine boots with")
     parser.add_argument('--no-gui', help='Disable GUI', action='store_true')
     parser.add_argument('--gdb', help='Export gdbserver port 2159 from the image', action='store_true')
     parser.add_argument('--pcap', default=None, help='Dump all network traffic')