Reusable meta-updater pipeline suiteBackport/thud/secondaries

Squashed:

* Oe-selftest GitLab pipeline stage

* Use credentials for CI's oe-selftest

* Setup kvm trick for docker on CI

  The gid of the kvm group needs to match the one from the host

* Run ci scripts from Docker images

* Template out jobs and split .gitlab-ci.yml

  To be easily reused in other branches and projects

  Rely on this gitlab feature: https://docs.gitlab.com/ee/ci/yaml/#extends

* More flexible checkout script for CI

  - can work without $CURRENT_PROJECT (checks out everything)
  - can take a list of pinned versions

* Add optional CI jobs

  ptest and other oe-selftests

* Publish bitbaked images as artifacts

Signed-off-by: Laurent Bonnans <laurent.bonnans@here.com>

8 files changed, 230 insertions, 48 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 94a614c..3682753 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,61 +3,96 @@ stages:
   - checkout
   - test
 
-# notes:
-# - could be useful https://docs.gitlab.com/ee/ci/yaml/#include
-
 variables:
-  # Needs $PR_BASE_BRANCH to be defined as a pipeline variable to work correctly
-
-  CHECKOUT_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-checkout
-  CHECKOUT_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout
+  BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake
+  BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout
 
-  BITBAKE_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-bitbake
-  BITBAKE_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake
+include:
+  - local: "scripts/ci/gitlab/docker.yml"
+  - local: "scripts/ci/gitlab/checkout.yml"
+  - local: "scripts/ci/gitlab/tests.yml"
 
+Docker setup:
+  extends: .bb_docker_local
 
-Docker Setup:
-  image: docker:stable
   stage: docker
-  services:
-    - docker:dind
+  variables:
+    BITBKAE_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-bitbake
+    BITBAKE_CHECKOUT_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-checkout
   except:
     - pushes
-  before_script:
-    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
-  script:
-    - docker pull "$BITBAKE_PR_IMAGE" || docker pull "$BITBAKE_MASTER_IMAGE" || true
-    - docker build --pull --cache-from "$BITBAKE_MASTER_IMAGE" --cache-from "$BITBAKE_PR_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_PR_IMAGE" ./scripts/ci
-    - docker push "$BITBAKE_PR_IMAGE"
-
-    - docker pull "$CHECKOUT_PR_IMAGE" || docker pull "$CHECKOUT_MASTER_IMAGE" || true
-    - docker build --pull --cache-from "$CHECKOUT_MASTER_IMAGE" --cache-from "$CHECKOUT_PR_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$CHECKOUT_PR_IMAGE" ./scripts/ci
-    - docker push "$CHECKOUT_PR_IMAGE"
 
 Checkout:
-  image: "$CHECKOUT_PR_IMAGE"
+  extends: .bb_checkout
+
   stage: checkout
+  variables:
+    MANIFEST: master
+    CURRENT_PROJECT: meta-updater
   except:
     - pushes
-  cache:
-    paths:
-      - updater-repo
+
+Build core-image-minimal:
+  extends: .bitbake
+
+  stage: test
+  variables:
+    TEST_BUILD_DIR: 'build-core-image-minimal'
+    BITBAKE_TARGETS: 'core-image-minimal'
   artifacts:
-    expire_in: "1 day"
+    name: "core-image-minimal_$CI_COMMIT_REF_SLUG"
     paths:
-      - updater-repo
-  script:
-    - MANIFEST=$PR_BASE_BRANCH ./scripts/ci/checkout-oe.sh
+      - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal*
+  except:
+    - pushes
+
+Oe-selftest qemux86_64:
+  extends: .oe-selftest
+
+  stage: test
+  variables:
+    TEST_BUILD_DIR: 'build-oe-qemux86_64'
+    OE_SELFTESTS: 'updater_native updater_qemux86_64'
+  except:
+    - pushes
+
+# Not run by default, triggered on nightlies
+
+Oe-selftest minnowboard:
+  extends: .oe-selftest
+
+  stage: test
+  variables:
+    TEST_BUILD_DIR: 'build-oe-minnowboard'
+    OE_SELFTESTS: 'updater_minnowboard'
+  except:
+    - pushes
+  only:
+    variables:
+      - $OE_MINNOWBOARD
+
+Oe-selftest rpi:
+  extends: .oe-selftest
+
+  stage: test
+  variables:
+    TEST_BUILD_DIR: 'build-oe-rpi'
+    OE_SELFTESTS: 'updater_rpi'
+  except:
+    - pushes
+  only:
+    variables:
+      - $OE_RPI
+
+Ptest qemux86_64:
+  extends: .oe-selftest
 
-Build core-image-minimal:
-  image: "$BITBAKE_PR_IMAGE"
   stage: test
+  variables:
+    TEST_BUILD_DIR: 'build-oe-qemux86_64-ptest'
+    OE_SELFTESTS: 'updater_qemux86_64_ptest'
   except:
     - pushes
-  dependencies:
-    - Checkout
-  tags:
-    - bitbake
-  script:
-    - ./scripts/ci/configure.sh
-    - ./scripts/ci/build.sh core-image-minimal
+  only:
+    variables:
+      - $OE_PTEST
diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake
index 75bad14..51eaa57 100644
--- a/scripts/ci/Dockerfile.bitbake
+++ b/scripts/ci/Dockerfile.bitbake
@@ -3,13 +3,15 @@ LABEL Description="Image for bitbaking"
 
 RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
 RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
-RUN apt-get update -q && apt-get install -qy \
+RUN apt-get update -q && apt-get install --no-install-suggests --no-install-recommends -qy \
+     awscli \
      build-essential \
      bzip2 \
      chrpath \
      cpio \
      default-jre \
      diffstat \
+     file \
      gawk \
      gcc-multilib \
      git-core \
@@ -17,14 +19,18 @@ RUN apt-get update -q && apt-get install -qy \
      iproute \
      libpython-dev \
      libsdl1.2-dev \
+     libvirt-clients \
+     libvirt-daemon-system \
      locales \
      ovmf \
+     openssh-client \
      procps \
      python \
      python3 \
      python3-pexpect \
-     qemu \
+     qemu-kvm \
      socat \
+     sudo \
      texinfo \
      unzip \
      wget \
@@ -34,11 +40,19 @@ RUN apt-get update -q && apt-get install -qy \
 ARG uid=4321
 ARG gid=4321
 RUN groupadd -g $gid bitbake
-RUN useradd -m -u $uid -g $gid bitbake
+RUN useradd -m -u $uid -g $gid -s /bin/bash bitbake
 
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
 ENV LC_ALL="en_US.UTF-8"
 ENV LANG="en_US.UTF-8"
 ENV LANGUAGE="en_US.UTF-8"
 
+# script to mirror kvm group id with host
+RUN echo "bitbake ALL=NOPASSWD: /usr/local/bin/setup_kvm.sh" >> /etc/sudoers
+COPY ./docker/setup_kvm.sh /usr/local/bin/setup_kvm.sh
+
+# other ci scripts
+RUN mkdir /scripts
+COPY configure.sh build.sh oe-selftest.sh /scripts/
+
 USER "bitbake"
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
index efec234..5210c6b 100644
--- a/scripts/ci/Dockerfile.checkout
+++ b/scripts/ci/Dockerfile.checkout
@@ -7,3 +7,7 @@ RUN apt-get update -q && apt-get install -qy \
     git \
     repo \
     xmlstarlet
+
+# checkout script
+RUN mkdir /scripts
+COPY checkout-oe.sh /scripts/
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
index 8744b2f..a99f235 100755
--- a/scripts/ci/checkout-oe.sh
+++ b/scripts/ci/checkout-oe.sh
@@ -6,7 +6,11 @@ set -x
 
 REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
 MANIFEST=${MANIFEST:-master}
-CURRENT_PROJECT=${CURRENT_PROJECT:-meta-updater}
+CURRENT_PROJECT=${CURRENT_PROJECT:-}
+
+# list of projects to pin to one version in the format:
+# "project:rev;project2:rev2..."
+PIN_LIST=${PIN_LIST:-}
 
 #CURRENT_REV=$(git rev-parse HEAD)
 LOCAL_REPO=$PWD
@@ -22,26 +26,48 @@ git -C .repo/manifests reset --hard
 # patch manifest:
 # - add a new "ats" remote that points to "$REMOTE_SOURCE"
 # - change projects that contain "advancedtelematic" to use the ats remote
-# - remove the current project from the manifest
 MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
 xmlstarlet ed --omit-decl -L \
     -s "/manifest" -t elem -n "remote" -v "" \
     -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
     -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
+    -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \
     -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \
-    -d "/manifest/project[@path=\"$CURRENT_PROJECT\"]" \
     "$MANIFEST_FILE"
 
 # hack: sed on `advancedtelematic/` names, to remove this unwanted prefix
 sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
 
+# pin projects from the list
+(
+IFS=";"
+for pin in $PIN_LIST; do
+    IFS=":"
+    read -r project rev <<< "$pin"
+    xmlstarlet ed --omit-decl -L \
+        -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \
+        -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \
+        "$MANIFEST_FILE"
+    IFS=";"
+done
+)
+
+# Remove the current project from the manifest if we have it checked out
+if [ -n "$CURRENT_PROJECT" ]; then
+    xmlstarlet ed --omit-decl -L \
+        -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \
+        "$MANIFEST_FILE"
+fi
+
 repo manifest
 
 repo forall -c 'git reset --hard ; git clean -fdx'
 
 repo sync -d --force-sync
 
-rm -f "$CURRENT_PROJECT"
-ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
+if [ -n "$CURRENT_PROJECT" ]; then
+    rm -f "$CURRENT_PROJECT"
+    ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
+fi
 
 repo manifest -r
diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh
new file mode 100755
index 0000000..1ffbbf5
--- /dev/null
+++ b/scripts/ci/docker/setup_kvm.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# This script makes the gid of the 'kvm' group to match the group
+# owner of '/dev/kvm'
+#
+# These two are not guaranteed to match when a docker image starts
+# with access to '/dev/kvm' that comes from the host
+
+set -euo pipefail
+
+kvm_gid=$(stat -c "%g" /dev/kvm)
+groupmod -g "$kvm_gid" kvm
+usermod -a -G kvm bitbake
+ln -s /bin/true /usr/bin/kvm-ok
diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml
new file mode 100644
index 0000000..28c9177
--- /dev/null
+++ b/scripts/ci/gitlab/checkout.yml
@@ -0,0 +1,16 @@
+.bb_checkout:
+  # parameters:
+  #   - MANIFEST
+  #   - REMOTE_SOURCE
+  #   - CURRENT_PROJECT (will be symlinked instead of pulled)
+
+  image: $BITBAKE_CHECKOUT_IMAGE
+  cache:
+    paths:
+      - updater-repo
+  artifacts:
+    expire_in: "1 day"
+    paths:
+      - updater-repo
+  script:
+    - /scripts/checkout-oe.sh
diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml
new file mode 100644
index 0000000..827bb47
--- /dev/null
+++ b/scripts/ci/gitlab/docker.yml
@@ -0,0 +1,37 @@
+.bb_docker_local:
+  # intended to be run on meta-updater's master branch which contains the
+  # reference docker files
+  # parameters:
+  #   - BITBAKE_IMAGE
+  #   - BITBAKE_CHECKOUT_IMAGE
+  #   - BITBKAE_IMAGE_MASTER
+  #   - BITBAKE_CHECKOUT_IMAGE_MASTER
+  image: docker:stable
+  stage: docker
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+  script:
+    - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true
+    - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci
+    - docker push "$BITBAKE_IMAGE"
+
+    - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true
+    - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci
+    - docker push "$BITBAKE_CHECKOUT_IMAGE"
+
+.bb_docker_remote:
+  # intended to be run on other branches and repos: just pulls the last master image
+  # parameters:
+  #   - BITBAKE_IMAGE
+  #   - BITBAKE_CHECKOUT_IMAGE
+  image: docker:stable
+  stage: docker
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+  script:
+    - docker pull "$BITBAKE_IMAGE"
+    - docker pull "$BITBAKE_CHECKOUT_IMAGE"
diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml
new file mode 100644
index 0000000..4bd5844
--- /dev/null
+++ b/scripts/ci/gitlab/tests.yml
@@ -0,0 +1,36 @@
+.bitbake:
+  # parameters:
+  #   - TEST_BUILD_DIR
+  #   - TEST_MACHINE (defaults to qemux86-64)
+  #   - BITBAKE_TARGETS
+  image: $BITBAKE_IMAGE
+  dependencies:
+    - Checkout
+  tags:
+    - bitbake
+  script:
+    - /scripts/configure.sh
+    - /scripts/build.sh $BITBAKE_TARGETS
+
+.oe-selftest:
+  # parameters:
+  #   - TEST_BUILD_DIR
+  #   - TEST_MACHINE (defaults to qemux86-64)
+  #   - OE_SELFTESTS
+  image: $BITBAKE_IMAGE
+  dependencies:
+    - Checkout
+  tags:
+    - bitbake
+  variables:
+    TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip
+  before_script:
+    - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip
+    - sudo /usr/local/bin/setup_kvm.sh
+  script:
+    - |
+      # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh)
+      sg kvm << EOS
+      /scripts/configure.sh
+      /scripts/oe-selftest.sh $OE_SELFTESTS
+      EOS