Refactor CI scripts for dunfell support.

Signed-off-by: Patrick Vacek <patrickvacek@gmail.com>

7 files changed, 28 insertions, 223 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2fade39..5f7f36f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,23 +2,27 @@ stages:
   - docker
   - checkout
   - test
+  - trigger
 
 variables:
-  BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake
-  BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout
+  BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-bitbake
+  BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-checkout
 
 include:
-  - local: "scripts/ci/gitlab/docker.yml"
-  - local: "scripts/ci/gitlab/checkout.yml"
-  - local: "scripts/ci/gitlab/tests.yml"
+  - project: 'olp/edge/ota/connect/client/meta-updater'
+    ref: 'master'
+    file: 'scripts/ci/gitlab/docker.yml'
+  - project: 'olp/edge/ota/connect/client/meta-updater'
+    ref: 'master'
+    file: 'scripts/ci/gitlab/checkout.yml'
+  - project: 'olp/edge/ota/connect/client/meta-updater'
+    ref: 'master'
+    file: 'scripts/ci/gitlab/tests.yml'
 
 Docker setup:
-  extends: .bb_docker_local
+  extends: .bb_docker_remote
 
   stage: docker
-  variables:
-    BITBKAE_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-bitbake
-    BITBAKE_CHECKOUT_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-checkout
   except:
     - pushes
 
@@ -27,7 +31,7 @@ Checkout:
 
   stage: checkout
   variables:
-    MANIFEST: master
+    MANIFEST: dunfell
     CURRENT_PROJECT: meta-updater
   except:
     - pushes
@@ -39,10 +43,6 @@ Build core-image-minimal:
   variables:
     TEST_BUILD_DIR: 'build-core-image-minimal'
     BITBAKE_TARGETS: 'core-image-minimal'
-  artifacts:
-    name: "core-image-minimal_$CI_COMMIT_REF_SLUG"
-    paths:
-      - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal*
   except:
     - pushes
 
@@ -54,10 +54,6 @@ Build core-image-minimal (rpi):
     TEST_BUILD_DIR: 'build-core-image-minimal-rpi'
     BITBAKE_TARGETS: 'core-image-minimal'
     TEST_MACHINE: 'raspberrypi3'
-  artifacts:
-    name: "core-image-minimal-rpi_$CI_COMMIT_REF_SLUG"
-    paths:
-      - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal*
   except:
     - pushes
 
@@ -71,8 +67,6 @@ Oe-selftest qemux86_64:
   except:
     - pushes
 
-# Not run by default, triggered on nightlies
-
 Oe-selftest minnowboard:
   extends: .oe-selftest
 
@@ -82,9 +76,6 @@ Oe-selftest minnowboard:
     OE_SELFTESTS: 'updater_minnowboard'
   except:
     - pushes
-  only:
-    variables:
-      - $OE_MINNOWBOARD
 
 Oe-selftest rpi:
   extends: .oe-selftest
@@ -95,9 +86,8 @@ Oe-selftest rpi:
     OE_SELFTESTS: 'updater_raspberrypi'
   except:
     - pushes
-  only:
-    variables:
-      - $OE_RPI
+
+# Not run by default
 
 Ptest qemux86_64:
   extends: .oe-selftest
@@ -111,3 +101,15 @@ Ptest qemux86_64:
   only:
     variables:
       - $OE_PTEST
+
+# -- otf
+
+trigger-otf-on-pr:
+  stage: trigger
+  when: always
+  except:
+    - pushes
+    - schedules
+  script:
+    - apk add --no-cache curl
+    - curl -X POST -F "token=$CI_JOB_TOKEN" -F "ref=master" -F "variables[BITBAKE_JOB_ONLY]=true" -F "variables[BITBAKE_ENV]=dunfell" -F "variables[PROJECT_NAME]=meta-updater" -F "variables[PROJECT_SHA]=$CI_COMMIT_SHA" https://main.gitlab.in.here.com/api/v4/projects/163/trigger/pipeline
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
deleted file mode 100644
index 5210c6b..0000000
--- a/scripts/ci/Dockerfile.checkout
+++ /dev/null
@@ -1,13 +0,0 @@
-FROM debian:stretch-slim
-LABEL Description="Image for checking out updater-repo"
-
-RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
-RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
-RUN apt-get update -q && apt-get install -qy \
-    git \
-    repo \
-    xmlstarlet
-
-# checkout script
-RUN mkdir /scripts
-COPY checkout-oe.sh /scripts/
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
deleted file mode 100755
index ed74595..0000000
--- a/scripts/ci/checkout-oe.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-set -x
-
-REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
-MANIFEST=${MANIFEST:-master}
-CURRENT_PROJECT=${CURRENT_PROJECT:-}
-
-# list of projects to pin to one version in the format:
-# "project:rev;project2:rev2..."
-PIN_LIST=${PIN_LIST:-}
-
-#CURRENT_REV=$(git rev-parse HEAD)
-LOCAL_REPO=$PWD
-
-mkdir -p updater-repo
-
-cd updater-repo
-
-# it seems like it's always required now, even if repo is only used to fetch
-git config --global user.email "meta-updater-ci@example.org"
-git config --global user.name "meta-updater-ci"
-
-if [ -d .repo/manifests ]; then
-    git -C .repo/manifests reset --hard
-fi
-
-repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo"
-
-# patch manifest:
-# - add a new "ats" remote that points to "$REMOTE_SOURCE"
-# - change projects that contain "advancedtelematic" to use the ats remote
-MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
-xmlstarlet ed --omit-decl -L \
-    -s "/manifest" -t elem -n "remote" -v "" \
-    -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
-    -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
-    -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \
-    -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \
-    "$MANIFEST_FILE"
-
-# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix
-sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
-
-# pin projects from the list
-(
-IFS=";"
-for pin in $PIN_LIST; do
-    IFS=":"
-    read -r project rev <<< "$pin"
-    xmlstarlet ed --omit-decl -L \
-        -d "/manifest/project[@name=\"$project\"]/@revision" \
-        -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \
-        -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \
-        "$MANIFEST_FILE"
-    IFS=";"
-done
-)
-
-# Remove the current project from the manifest if we have it checked out
-if [ -n "$CURRENT_PROJECT" ]; then
-    xmlstarlet ed --omit-decl -L \
-        -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \
-        "$MANIFEST_FILE"
-fi
-
-repo manifest
-
-# Try to clean up the repos in case they are messed up, but this returns an
-# error code if the repos don't exist, which the subsequent repo sync will fix.
-repo forall -c 'git reset --hard ; git clean -fdx' || true
-
-repo sync -d --force-sync
-
-if [ -n "$CURRENT_PROJECT" ]; then
-    rm -f "$CURRENT_PROJECT"
-    ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
-fi
-
-repo manifest -r
diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh
deleted file mode 100755
index 1ffbbf5..0000000
--- a/scripts/ci/docker/setup_kvm.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-# This script makes the gid of the 'kvm' group to match the group
-# owner of '/dev/kvm'
-#
-# These two are not guaranteed to match when a docker image starts
-# with access to '/dev/kvm' that comes from the host
-
-set -euo pipefail
-
-kvm_gid=$(stat -c "%g" /dev/kvm)
-groupmod -g "$kvm_gid" kvm
-usermod -a -G kvm bitbake
-ln -s /bin/true /usr/bin/kvm-ok
diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml
deleted file mode 100644
index 28c9177..0000000
--- a/scripts/ci/gitlab/checkout.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-.bb_checkout:
-  # parameters:
-  #   - MANIFEST
-  #   - REMOTE_SOURCE
-  #   - CURRENT_PROJECT (will be symlinked instead of pulled)
-
-  image: $BITBAKE_CHECKOUT_IMAGE
-  cache:
-    paths:
-      - updater-repo
-  artifacts:
-    expire_in: "1 day"
-    paths:
-      - updater-repo
-  script:
-    - /scripts/checkout-oe.sh
diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml
deleted file mode 100644
index ba488c6..0000000
--- a/scripts/ci/gitlab/docker.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-.bb_docker_local:
-  # intended to be run on meta-updater's master branch which contains the
-  # reference docker files
-  # parameters:
-  #   - BITBAKE_IMAGE
-  #   - BITBAKE_CHECKOUT_IMAGE
-  #   - BITBKAE_IMAGE_MASTER
-  #   - BITBAKE_CHECKOUT_IMAGE_MASTER
-  image: docker:18
-  stage: docker
-  services:
-    - docker:18-dind
-  script:
-    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
-
-    - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true
-    - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci
-    - docker push "$BITBAKE_IMAGE"
-
-    - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true
-    - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci
-    - docker push "$BITBAKE_CHECKOUT_IMAGE"
-
-.bb_docker_remote:
-  # intended to be run on other branches and repos: just pulls the last master image
-  # parameters:
-  #   - BITBAKE_IMAGE
-  #   - BITBAKE_CHECKOUT_IMAGE
-  image: docker:18
-  stage: docker
-  services:
-    - docker:18-dind
-  script:
-    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
-
-    - docker pull "$BITBAKE_IMAGE"
-    - docker pull "$BITBAKE_CHECKOUT_IMAGE"
diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml
deleted file mode 100644
index 4175412..0000000
--- a/scripts/ci/gitlab/tests.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-.bitbake:
-  # parameters:
-  #   - TEST_BUILD_DIR
-  #   - TEST_MACHINE (defaults to qemux86-64)
-  #   - BITBAKE_TARGETS
-  image: $BITBAKE_IMAGE
-  dependencies:
-    - Checkout
-  tags:
-    - bitbake
-  script:
-    - /scripts/configure.sh
-    - /scripts/build.sh $BITBAKE_TARGETS
-
-.oe-selftest:
-  # parameters:
-  #   - TEST_BUILD_DIR
-  #   - TEST_MACHINE (defaults to qemux86-64)
-  #   - OE_SELFTESTS
-  image: $BITBAKE_IMAGE
-  dependencies:
-    - Checkout
-  tags:
-    - bitbake
-  variables:
-    TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip
-  script:
-    - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip
-    - sudo /usr/local/bin/setup_kvm.sh
-    - |
-      # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh)
-      sg kvm << EOS
-      /scripts/configure.sh
-      /scripts/oe-selftest.sh $OE_SELFTESTS
-      EOS