Gitlab pipeline for building core-image-minimalci/gitlab-core-image-minimal

Including these points (initially as separate commits):

* Try caching updater-repo in gitlab
* Only patch aktualizr recipe in CI if needed
* Run under user 4321 in bitbake docker image
* Add ability to supply METADIR to envsetup.sh

  So that we can bypass the "relative path from the script" trick that
  doesn't work with symlinks.
* Checkout the PR base branch
* do not run on pushes
* run on runners with "gitlab" tag
* Print all revisions after repo checkout on CI
* Pull CI image from the base branch instead of master
* More robust checkout in CI
* Only store updater-repo artifacts for one day
* Use "stretch" instead of "stable" for Debian release in Dockerfile
  (in fact, stable just changed to buster)
* Move UPDATER_REPO variable out of .gitlab-ci.yml
  To be defined in the project's settings
* Make docker CI stage buildable on shared runners

Signed-off-by: Laurent Bonnans <laurent.bonnans@here.com>

8 files changed, 149 insertions, 14 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..788fad8
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,63 @@
+stages:
+  - docker
+  - checkout
+  - test
+
+# notes:
+# - could be useful https://docs.gitlab.com/ee/ci/yaml/#include
+
+variables:
+  # Needs $PR_BASE_BRANCH to be defined as a pipeline variable to work correctly
+
+  BITBAKE_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-bitbake
+  BITBAKE_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake
+
+  CHECKOUT_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-xml
+  CHECKOUT_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-xml
+
+
+Docker Setup:
+  image: docker:stable
+  stage: docker
+  services:
+    - docker:dind
+  except:
+    - pushes
+  before_script:
+    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
+  script:
+    - docker pull "$BITBAKE_PR_IMAGE" || docker pull "$BITBAKE_MASTER_IMAGE" || true
+    - docker build --pull --cache-from "$BITBAKE_MASTER_IMAGE" --cache-from "$BITBAKE_PR_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_PR_IMAGE" ./scripts/ci
+    - docker push "$BITBAKE_PR_IMAGE"
+
+    - docker pull "$CHECKOUT_PR_IMAGE" || docker pull "$CHECKOUT_MASTER_IMAGE" || true
+    - docker build --pull --cache-from "$CHECKOUT_MASTER_IMAGE" --cache-from "$CHECKOUT_PR_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$CHECKOUT_PR_IMAGE" ./scripts/ci
+    - docker push "$CHECKOUT_PR_IMAGE"
+
+Checkout:
+  image: "$CHECKOUT_PR_IMAGE"
+  stage: checkout
+  except:
+    - pushes
+  cache:
+    paths:
+      - updater-repo
+  artifacts:
+    expire_in: "1 day"
+    paths:
+      - updater-repo
+  script:
+    - MANIFEST=$PR_BASE_BRANCH ./scripts/ci/checkout-oe.sh
+
+Build core-image-minimal:
+  image: "$BITBAKE_PR_IMAGE"
+  stage: test
+  except:
+    - pushes
+  dependencies:
+    - Checkout
+  tags:
+    - bitbake
+  script:
+    - ./scripts/ci/configure.sh
+    - ./scripts/ci/build.sh core-image-minimal
diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake
index c91f94c..75bad14 100644
--- a/scripts/ci/Dockerfile.bitbake
+++ b/scripts/ci/Dockerfile.bitbake
@@ -1,8 +1,8 @@
-FROM debian:stable
+FROM debian:stretch
 LABEL Description="Image for bitbaking"
 
-RUN sed -i 's#deb http://deb.debian.org/debian stable main#deb http://deb.debian.org/debian stable main contrib#g' /etc/apt/sources.list
-RUN sed -i 's#deb http://deb.debian.org/debian stable-updates main#deb http://deb.debian.org/debian stable-updates main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
 RUN apt-get update -q && apt-get install -qy \
      build-essential \
      bzip2 \
@@ -31,8 +31,8 @@ RUN apt-get update -q && apt-get install -qy \
      xterm \
      xz-utils
 
-ARG uid=1000
-ARG gid=1000
+ARG uid=4321
+ARG gid=4321
 RUN groupadd -g $gid bitbake
 RUN useradd -m -u $uid -g $gid bitbake
 
@@ -40,3 +40,5 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
 ENV LC_ALL="en_US.UTF-8"
 ENV LANG="en_US.UTF-8"
 ENV LANGUAGE="en_US.UTF-8"
+
+USER "bitbake"
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
new file mode 100644
index 0000000..efec234
--- /dev/null
+++ b/scripts/ci/Dockerfile.checkout
@@ -0,0 +1,9 @@
+FROM debian:stretch-slim
+LABEL Description="Image for checking out updater-repo"
+
+RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
+RUN apt-get update -q && apt-get install -qy \
+    git \
+    repo \
+    xmlstarlet
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index 6235428..9fbae98 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -12,7 +12,10 @@ IMAGE_NAME=${1:-core-image-minimal}
 (
 set +euo pipefail
 set +x
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
 
+set -x
 bitbake "${IMAGE_NAME}"
 )
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
new file mode 100755
index 0000000..9a4ade2
--- /dev/null
+++ b/scripts/ci/checkout-oe.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+set -x
+
+REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
+MANIFEST=${MANIFEST:-master}
+CURRENT_PROJECT=${CURRENT_PROJECT:-meta-updater}
+
+#CURRENT_REV=$(git rev-parse HEAD)
+LOCAL_REPO=$PWD
+
+mkdir -p updater-repo
+
+cd updater-repo
+
+repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo"
+
+git -C .repo/manifests reset --hard
+
+# patch manifest
+MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
+xmlstarlet ed --omit-decl -L \
+    -s "/manifest" -t elem -n "remote" -v "" \
+    -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
+    -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
+    -u "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" -v "ats" \
+    -d "/manifest/project[@path=\"$CURRENT_PROJECT\"]" \
+    "$MANIFEST_FILE"
+
+# hack: sed on `advancedtelematic/` names
+sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
+
+repo manifest
+
+repo forall -c 'git reset --hard ; git clean -fdx'
+
+repo sync -d --force-sync
+
+rm -f "$CURRENT_PROJECT"
+ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
+
+repo manifest -r
diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh
index 960a0cc..a8968da 100755
--- a/scripts/ci/configure.sh
+++ b/scripts/ci/configure.sh
@@ -8,9 +8,13 @@ TEST_BUILD_DIR=${TEST_BUILD_DIR:-build}
 TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
 TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-}
 
-TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
-TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
-TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
+TEST_AKTUALIZR_REMOTE=${TEST_AKTUALIZR_REMOTE:-}
+if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
+    TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
+    TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
+    TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
+fi
+
 TEST_AKTUALIZR_CREDENTIALS=${TEST_AKTUALIZR_CREDENTIALS:-}
 
 # move existing conf directory to backup, before generating a new one
@@ -21,6 +25,8 @@ mv "$TEST_BUILD_DIR/conf" "$TEST_BUILD_DIR/conf.old" || true
 set +euo pipefail
 set +x
 echo ">> Running envsetup.sh"
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "$TEST_REPO_DIR/meta-updater/scripts/envsetup.sh" "$TEST_MACHINE" "$TEST_BUILD_DIR"
 )
 
@@ -31,19 +37,19 @@ SITE_CONF="$TEST_BUILD_DIR/conf/site.conf"
 echo ">> Set common bitbake config options"
 cat << EOF > "$SITE_CONF"
 SANITY_TESTED_DISTROS = ""
-SSTATE_MIRRORS ?= "file://.* https://bitbake-cache.atsgarage.com/PATH;downloadfilename=PATH"
 IMAGE_FEATURES += "ssh-server-openssh"
 
 EOF
 
-echo ">> Set aktualizr branch in bitbake's config"
-cat << EOF >> "$SITE_CONF"
+if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
+    echo ">> Set aktualizr branch in bitbake's config"
+    cat << EOF >> "$SITE_CONF"
 SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV"
 SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}"
 BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH"
 BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}"
-
 EOF
+fi
 
 if [[ -n $TEST_AKTUALIZR_CREDENTIALS ]]; then
     echo ">> Set aktualizr credentials"
diff --git a/scripts/ci/oe-selftest.sh b/scripts/ci/oe-selftest.sh
index 3124cce..5400388 100755
--- a/scripts/ci/oe-selftest.sh
+++ b/scripts/ci/oe-selftest.sh
@@ -9,10 +9,18 @@ TEST_MACHINE=${TEST_MACHINE:-qemux86-64}
 TEST_BUILD_DIR=${TEST_BUILD_DIR:-build}
 TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
 
+OEST_ARGS=()
+for v in "$@"; do
+    OEST_ARGS+=("-r" "$v")
+done
+
 (
 set +euo pipefail
 set +x
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
 
-oe-selftest -r updater
+set -x
+oe-selftest "${OEST_ARGS[@]}"
 )
diff --git a/scripts/envsetup.sh b/scripts/envsetup.sh
index 6103cf6..5827bc2 100755
--- a/scripts/envsetup.sh
+++ b/scripts/envsetup.sh
@@ -21,7 +21,7 @@ if [[ $SOURCED -ne 1 ]]; then
   exit 1
 fi
 
-METADIR="${SOURCEDIR}/../.."
+METADIR=${METADIR:-${SOURCEDIR}/../..}
 
 if [[ ! -f "${BUILDDIR}/conf/local.conf" ]]; then
   source "$METADIR/poky/oe-init-build-env" "$BUILDDIR"