From 62fbb2b1ba053635f154e5a041eb4c797f501514 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Mon, 6 Apr 2020 11:49:00 +0200 Subject: Refactor CI scripts for dunfell support. Signed-off-by: Patrick Vacek --- .gitlab-ci.yml | 54 ++++++++++++++-------------- scripts/ci/Dockerfile.checkout | 13 ------- scripts/ci/checkout-oe.sh | 82 ------------------------------------------ scripts/ci/docker/setup_kvm.sh | 14 -------- scripts/ci/gitlab/checkout.yml | 16 --------- scripts/ci/gitlab/docker.yml | 37 ------------------- scripts/ci/gitlab/tests.yml | 35 ------------------ 7 files changed, 28 insertions(+), 223 deletions(-) delete mode 100644 scripts/ci/Dockerfile.checkout delete mode 100755 scripts/ci/checkout-oe.sh delete mode 100755 scripts/ci/docker/setup_kvm.sh delete mode 100644 scripts/ci/gitlab/checkout.yml delete mode 100644 scripts/ci/gitlab/docker.yml delete mode 100644 scripts/ci/gitlab/tests.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fade39..5f7f36f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,23 +2,27 @@ stages: - docker - checkout - test + - trigger variables: - BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake - BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout + BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-bitbake + BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-checkout include: - - local: "scripts/ci/gitlab/docker.yml" - - local: "scripts/ci/gitlab/checkout.yml" - - local: "scripts/ci/gitlab/tests.yml" + - project: 'olp/edge/ota/connect/client/meta-updater' + ref: 'master' + file: 'scripts/ci/gitlab/docker.yml' + - project: 'olp/edge/ota/connect/client/meta-updater' + ref: 'master' + file: 'scripts/ci/gitlab/checkout.yml' + - project: 'olp/edge/ota/connect/client/meta-updater' + ref: 'master' + file: 'scripts/ci/gitlab/tests.yml' Docker setup: - extends: .bb_docker_local + extends: .bb_docker_remote stage: docker - variables: - BITBKAE_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-bitbake - BITBAKE_CHECKOUT_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-checkout except: - pushes @@ -27,7 +31,7 @@ Checkout: stage: checkout variables: - MANIFEST: master + MANIFEST: dunfell CURRENT_PROJECT: meta-updater except: - pushes @@ -39,10 +43,6 @@ Build core-image-minimal: variables: TEST_BUILD_DIR: 'build-core-image-minimal' BITBAKE_TARGETS: 'core-image-minimal' - artifacts: - name: "core-image-minimal_$CI_COMMIT_REF_SLUG" - paths: - - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal* except: - pushes @@ -54,10 +54,6 @@ Build core-image-minimal (rpi): TEST_BUILD_DIR: 'build-core-image-minimal-rpi' BITBAKE_TARGETS: 'core-image-minimal' TEST_MACHINE: 'raspberrypi3' - artifacts: - name: "core-image-minimal-rpi_$CI_COMMIT_REF_SLUG" - paths: - - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal* except: - pushes @@ -71,8 +67,6 @@ Oe-selftest qemux86_64: except: - pushes -# Not run by default, triggered on nightlies - Oe-selftest minnowboard: extends: .oe-selftest @@ -82,9 +76,6 @@ Oe-selftest minnowboard: OE_SELFTESTS: 'updater_minnowboard' except: - pushes - only: - variables: - - $OE_MINNOWBOARD Oe-selftest rpi: extends: .oe-selftest @@ -95,9 +86,8 @@ Oe-selftest rpi: OE_SELFTESTS: 'updater_raspberrypi' except: - pushes - only: - variables: - - $OE_RPI + +# Not run by default Ptest qemux86_64: extends: .oe-selftest @@ -111,3 +101,15 @@ Ptest qemux86_64: only: variables: - $OE_PTEST + +# -- otf + +trigger-otf-on-pr: + stage: trigger + when: always + except: + - pushes + - schedules + script: + - apk add --no-cache curl + - curl -X POST -F "token=$CI_JOB_TOKEN" -F "ref=master" -F "variables[BITBAKE_JOB_ONLY]=true" -F "variables[BITBAKE_ENV]=dunfell" -F "variables[PROJECT_NAME]=meta-updater" -F "variables[PROJECT_SHA]=$CI_COMMIT_SHA" https://main.gitlab.in.here.com/api/v4/projects/163/trigger/pipeline diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout deleted file mode 100644 index 5210c6b..0000000 --- a/scripts/ci/Dockerfile.checkout +++ /dev/null @@ -1,13 +0,0 @@ -FROM debian:stretch-slim -LABEL Description="Image for checking out updater-repo" - -RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list -RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list -RUN apt-get update -q && apt-get install -qy \ - git \ - repo \ - xmlstarlet - -# checkout script -RUN mkdir /scripts -COPY checkout-oe.sh /scripts/ diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh deleted file mode 100755 index ed74595..0000000 --- a/scripts/ci/checkout-oe.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -set -x - -REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic} -MANIFEST=${MANIFEST:-master} -CURRENT_PROJECT=${CURRENT_PROJECT:-} - -# list of projects to pin to one version in the format: -# "project:rev;project2:rev2..." -PIN_LIST=${PIN_LIST:-} - -#CURRENT_REV=$(git rev-parse HEAD) -LOCAL_REPO=$PWD - -mkdir -p updater-repo - -cd updater-repo - -# it seems like it's always required now, even if repo is only used to fetch -git config --global user.email "meta-updater-ci@example.org" -git config --global user.name "meta-updater-ci" - -if [ -d .repo/manifests ]; then - git -C .repo/manifests reset --hard -fi - -repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo" - -# patch manifest: -# - add a new "ats" remote that points to "$REMOTE_SOURCE" -# - change projects that contain "advancedtelematic" to use the ats remote -MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml" -xmlstarlet ed --omit-decl -L \ - -s "/manifest" -t elem -n "remote" -v "" \ - -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \ - -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \ - -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \ - -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \ - "$MANIFEST_FILE" - -# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix -sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE" - -# pin projects from the list -( -IFS=";" -for pin in $PIN_LIST; do - IFS=":" - read -r project rev <<< "$pin" - xmlstarlet ed --omit-decl -L \ - -d "/manifest/project[@name=\"$project\"]/@revision" \ - -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \ - -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \ - "$MANIFEST_FILE" - IFS=";" -done -) - -# Remove the current project from the manifest if we have it checked out -if [ -n "$CURRENT_PROJECT" ]; then - xmlstarlet ed --omit-decl -L \ - -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \ - "$MANIFEST_FILE" -fi - -repo manifest - -# Try to clean up the repos in case they are messed up, but this returns an -# error code if the repos don't exist, which the subsequent repo sync will fix. -repo forall -c 'git reset --hard ; git clean -fdx' || true - -repo sync -d --force-sync - -if [ -n "$CURRENT_PROJECT" ]; then - rm -f "$CURRENT_PROJECT" - ln -s "$LOCAL_REPO" "$CURRENT_PROJECT" -fi - -repo manifest -r diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh deleted file mode 100755 index 1ffbbf5..0000000 --- a/scripts/ci/docker/setup_kvm.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# This script makes the gid of the 'kvm' group to match the group -# owner of '/dev/kvm' -# -# These two are not guaranteed to match when a docker image starts -# with access to '/dev/kvm' that comes from the host - -set -euo pipefail - -kvm_gid=$(stat -c "%g" /dev/kvm) -groupmod -g "$kvm_gid" kvm -usermod -a -G kvm bitbake -ln -s /bin/true /usr/bin/kvm-ok diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml deleted file mode 100644 index 28c9177..0000000 --- a/scripts/ci/gitlab/checkout.yml +++ /dev/null @@ -1,16 +0,0 @@ -.bb_checkout: - # parameters: - # - MANIFEST - # - REMOTE_SOURCE - # - CURRENT_PROJECT (will be symlinked instead of pulled) - - image: $BITBAKE_CHECKOUT_IMAGE - cache: - paths: - - updater-repo - artifacts: - expire_in: "1 day" - paths: - - updater-repo - script: - - /scripts/checkout-oe.sh diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml deleted file mode 100644 index ba488c6..0000000 --- a/scripts/ci/gitlab/docker.yml +++ /dev/null @@ -1,37 +0,0 @@ -.bb_docker_local: - # intended to be run on meta-updater's master branch which contains the - # reference docker files - # parameters: - # - BITBAKE_IMAGE - # - BITBAKE_CHECKOUT_IMAGE - # - BITBKAE_IMAGE_MASTER - # - BITBAKE_CHECKOUT_IMAGE_MASTER - image: docker:18 - stage: docker - services: - - docker:18-dind - script: - - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" - - - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true - - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci - - docker push "$BITBAKE_IMAGE" - - - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true - - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci - - docker push "$BITBAKE_CHECKOUT_IMAGE" - -.bb_docker_remote: - # intended to be run on other branches and repos: just pulls the last master image - # parameters: - # - BITBAKE_IMAGE - # - BITBAKE_CHECKOUT_IMAGE - image: docker:18 - stage: docker - services: - - docker:18-dind - script: - - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" - - - docker pull "$BITBAKE_IMAGE" - - docker pull "$BITBAKE_CHECKOUT_IMAGE" diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml deleted file mode 100644 index 4175412..0000000 --- a/scripts/ci/gitlab/tests.yml +++ /dev/null @@ -1,35 +0,0 @@ -.bitbake: - # parameters: - # - TEST_BUILD_DIR - # - TEST_MACHINE (defaults to qemux86-64) - # - BITBAKE_TARGETS - image: $BITBAKE_IMAGE - dependencies: - - Checkout - tags: - - bitbake - script: - - /scripts/configure.sh - - /scripts/build.sh $BITBAKE_TARGETS - -.oe-selftest: - # parameters: - # - TEST_BUILD_DIR - # - TEST_MACHINE (defaults to qemux86-64) - # - OE_SELFTESTS - image: $BITBAKE_IMAGE - dependencies: - - Checkout - tags: - - bitbake - variables: - TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip - script: - - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip - - sudo /usr/local/bin/setup_kvm.sh - - | - # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh) - sg kvm << EOS - /scripts/configure.sh - /scripts/oe-selftest.sh $OE_SELFTESTS - EOS -- cgit v1.2.3-54-g00ecf