Gitlab pipeline for building core-image-minimal

Including these points (initially as separate commits):

* Try caching updater-repo in gitlab
* Only patch aktualizr recipe in CI if needed
* Run under user 4321 in bitbake docker image
* Add ability to supply METADIR to envsetup.sh

  So that we can bypass the "relative path from the script" trick that
  doesn't work with symlinks.
* Checkout the PR base branch
* do not run on pushes
* run on runners with "gitlab" tag
* Print all revisions after repo checkout on CI
* Pull CI image from the base branch instead of master
* More robust checkout in CI
* Only store updater-repo artifacts for one day
* Use "stretch" instead of "stable" for Debian release in Dockerfile
  (in fact, stable just changed to buster)
* Move UPDATER_REPO variable out of .gitlab-ci.yml
  To be defined in the project's settings
* Make docker CI stage buildable on shared runners

Signed-off-by: Laurent Bonnans <laurent.bonnans@here.com>

7 files changed, 84 insertions, 14 deletions

diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake
index c91f94c..75bad14 100644
--- a/scripts/ci/Dockerfile.bitbake
+++ b/scripts/ci/Dockerfile.bitbake
@@ -1,8 +1,8 @@
-FROM debian:stable
+FROM debian:stretch
 LABEL Description="Image for bitbaking"
 
-RUN sed -i 's#deb http://deb.debian.org/debian stable main#deb http://deb.debian.org/debian stable main contrib#g' /etc/apt/sources.list
-RUN sed -i 's#deb http://deb.debian.org/debian stable-updates main#deb http://deb.debian.org/debian stable-updates main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
 RUN apt-get update -q && apt-get install -qy \
      build-essential \
      bzip2 \
@@ -31,8 +31,8 @@ RUN apt-get update -q && apt-get install -qy \
      xterm \
      xz-utils
 
-ARG uid=1000
-ARG gid=1000
+ARG uid=4321
+ARG gid=4321
 RUN groupadd -g $gid bitbake
 RUN useradd -m -u $uid -g $gid bitbake
 
@@ -40,3 +40,5 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
 ENV LC_ALL="en_US.UTF-8"
 ENV LANG="en_US.UTF-8"
 ENV LANGUAGE="en_US.UTF-8"
+
+USER "bitbake"
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
new file mode 100644
index 0000000..efec234
--- /dev/null
+++ b/scripts/ci/Dockerfile.checkout
@@ -0,0 +1,9 @@
+FROM debian:stretch-slim
+LABEL Description="Image for checking out updater-repo"
+
+RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
+RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
+RUN apt-get update -q && apt-get install -qy \
+    git \
+    repo \
+    xmlstarlet
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index 6235428..9fbae98 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -12,7 +12,10 @@ IMAGE_NAME=${1:-core-image-minimal}
 (
 set +euo pipefail
 set +x
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
 
+set -x
 bitbake "${IMAGE_NAME}"
 )
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
new file mode 100755
index 0000000..8744b2f
--- /dev/null
+++ b/scripts/ci/checkout-oe.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+set -x
+
+REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
+MANIFEST=${MANIFEST:-master}
+CURRENT_PROJECT=${CURRENT_PROJECT:-meta-updater}
+
+#CURRENT_REV=$(git rev-parse HEAD)
+LOCAL_REPO=$PWD
+
+mkdir -p updater-repo
+
+cd updater-repo
+
+repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo"
+
+git -C .repo/manifests reset --hard
+
+# patch manifest:
+# - add a new "ats" remote that points to "$REMOTE_SOURCE"
+# - change projects that contain "advancedtelematic" to use the ats remote
+# - remove the current project from the manifest
+MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
+xmlstarlet ed --omit-decl -L \
+    -s "/manifest" -t elem -n "remote" -v "" \
+    -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
+    -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
+    -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \
+    -d "/manifest/project[@path=\"$CURRENT_PROJECT\"]" \
+    "$MANIFEST_FILE"
+
+# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix
+sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
+
+repo manifest
+
+repo forall -c 'git reset --hard ; git clean -fdx'
+
+repo sync -d --force-sync
+
+rm -f "$CURRENT_PROJECT"
+ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
+
+repo manifest -r
diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh
index 960a0cc..a8968da 100755
--- a/scripts/ci/configure.sh
+++ b/scripts/ci/configure.sh
@@ -8,9 +8,13 @@ TEST_BUILD_DIR=${TEST_BUILD_DIR:-build}
 TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
 TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-}
 
-TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
-TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
-TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
+TEST_AKTUALIZR_REMOTE=${TEST_AKTUALIZR_REMOTE:-}
+if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
+    TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
+    TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
+    TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
+fi
+
 TEST_AKTUALIZR_CREDENTIALS=${TEST_AKTUALIZR_CREDENTIALS:-}
 
 # move existing conf directory to backup, before generating a new one
@@ -21,6 +25,8 @@ mv "$TEST_BUILD_DIR/conf" "$TEST_BUILD_DIR/conf.old" || true
 set +euo pipefail
 set +x
 echo ">> Running envsetup.sh"
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "$TEST_REPO_DIR/meta-updater/scripts/envsetup.sh" "$TEST_MACHINE" "$TEST_BUILD_DIR"
 )
 
@@ -31,19 +37,19 @@ SITE_CONF="$TEST_BUILD_DIR/conf/site.conf"
 echo ">> Set common bitbake config options"
 cat << EOF > "$SITE_CONF"
 SANITY_TESTED_DISTROS = ""
-SSTATE_MIRRORS ?= "file://.* https://bitbake-cache.atsgarage.com/PATH;downloadfilename=PATH"
 IMAGE_FEATURES += "ssh-server-openssh"
 
 EOF
 
-echo ">> Set aktualizr branch in bitbake's config"
-cat << EOF >> "$SITE_CONF"
+if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
+    echo ">> Set aktualizr branch in bitbake's config"
+    cat << EOF >> "$SITE_CONF"
 SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV"
 SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}"
 BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH"
 BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}"
-
 EOF
+fi
 
 if [[ -n $TEST_AKTUALIZR_CREDENTIALS ]]; then
     echo ">> Set aktualizr credentials"
diff --git a/scripts/ci/oe-selftest.sh b/scripts/ci/oe-selftest.sh
index 3124cce..66b1fc9 100755
--- a/scripts/ci/oe-selftest.sh
+++ b/scripts/ci/oe-selftest.sh
@@ -12,7 +12,10 @@ TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
 (
 set +euo pipefail
 set +x
+METADIR=$(realpath "$TEST_REPO_DIR")
+export METADIR
 . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
 
-oe-selftest -r updater
+set -x
+oe-selftest -r "$@"
 )
diff --git a/scripts/envsetup.sh b/scripts/envsetup.sh
index 6103cf6..5827bc2 100755
--- a/scripts/envsetup.sh
+++ b/scripts/envsetup.sh
@@ -21,7 +21,7 @@ if [[ $SOURCED -ne 1 ]]; then
   exit 1
 fi
 
-METADIR="${SOURCEDIR}/../.."
+METADIR=${METADIR:-${SOURCEDIR}/../..}
 
 if [[ ! -f "${BUILDDIR}/conf/local.conf" ]]; then
   source "$METADIR/poky/oe-init-build-env" "$BUILDDIR"