From 32255ab2a79ca6d99b24ee43446aeaa943b21178 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Fri, 5 Jul 2019 17:45:39 +0200 Subject: Reusable meta-updater pipeline suite Squashed: * Oe-selftest GitLab pipeline stage * Use credentials for CI's oe-selftest * Setup kvm trick for docker on CI The gid of the kvm group needs to match the one from the host * Run ci scripts from Docker images * Template out jobs and split .gitlab-ci.yml To be easily reused in other branches and projects Rely on this gitlab feature: https://docs.gitlab.com/ee/ci/yaml/#extends * More flexible checkout script for CI - can work without $CURRENT_PROJECT (checks out everything) - can take a list of pinned versions * Add optional CI jobs ptest and other oe-selftests * Publish bitbaked images as artifacts Signed-off-by: Laurent Bonnans --- .gitlab-ci.yml | 115 +++++++++++++++++++++++++++-------------- scripts/ci/Dockerfile.checkout | 4 ++ scripts/ci/checkout-oe.sh | 36 +++++++++++-- scripts/ci/docker/setup_kvm.sh | 14 +++++ scripts/ci/gitlab/checkout.yml | 16 ++++++ scripts/ci/gitlab/docker.yml | 37 +++++++++++++ scripts/ci/gitlab/tests.yml | 36 +++++++++++++ 7 files changed, 213 insertions(+), 45 deletions(-) create mode 100755 scripts/ci/docker/setup_kvm.sh create mode 100644 scripts/ci/gitlab/checkout.yml create mode 100644 scripts/ci/gitlab/docker.yml create mode 100644 scripts/ci/gitlab/tests.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 94a614c..3682753 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,61 +3,96 @@ stages: - checkout - test -# notes: -# - could be useful https://docs.gitlab.com/ee/ci/yaml/#include - variables: - # Needs $PR_BASE_BRANCH to be defined as a pipeline variable to work correctly - - CHECKOUT_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-checkout - CHECKOUT_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout + BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake + BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout - BITBAKE_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-bitbake - BITBAKE_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake +include: + - local: "scripts/ci/gitlab/docker.yml" + - local: "scripts/ci/gitlab/checkout.yml" + - local: "scripts/ci/gitlab/tests.yml" +Docker setup: + extends: .bb_docker_local -Docker Setup: - image: docker:stable stage: docker - services: - - docker:dind + variables: + BITBKAE_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-bitbake + BITBAKE_CHECKOUT_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-checkout except: - pushes - before_script: - - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" - script: - - docker pull "$BITBAKE_PR_IMAGE" || docker pull "$BITBAKE_MASTER_IMAGE" || true - - docker build --pull --cache-from "$BITBAKE_MASTER_IMAGE" --cache-from "$BITBAKE_PR_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_PR_IMAGE" ./scripts/ci - - docker push "$BITBAKE_PR_IMAGE" - - - docker pull "$CHECKOUT_PR_IMAGE" || docker pull "$CHECKOUT_MASTER_IMAGE" || true - - docker build --pull --cache-from "$CHECKOUT_MASTER_IMAGE" --cache-from "$CHECKOUT_PR_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$CHECKOUT_PR_IMAGE" ./scripts/ci - - docker push "$CHECKOUT_PR_IMAGE" Checkout: - image: "$CHECKOUT_PR_IMAGE" + extends: .bb_checkout + stage: checkout + variables: + MANIFEST: master + CURRENT_PROJECT: meta-updater except: - pushes - cache: - paths: - - updater-repo + +Build core-image-minimal: + extends: .bitbake + + stage: test + variables: + TEST_BUILD_DIR: 'build-core-image-minimal' + BITBAKE_TARGETS: 'core-image-minimal' artifacts: - expire_in: "1 day" + name: "core-image-minimal_$CI_COMMIT_REF_SLUG" paths: - - updater-repo - script: - - MANIFEST=$PR_BASE_BRANCH ./scripts/ci/checkout-oe.sh + - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal* + except: + - pushes + +Oe-selftest qemux86_64: + extends: .oe-selftest + + stage: test + variables: + TEST_BUILD_DIR: 'build-oe-qemux86_64' + OE_SELFTESTS: 'updater_native updater_qemux86_64' + except: + - pushes + +# Not run by default, triggered on nightlies + +Oe-selftest minnowboard: + extends: .oe-selftest + + stage: test + variables: + TEST_BUILD_DIR: 'build-oe-minnowboard' + OE_SELFTESTS: 'updater_minnowboard' + except: + - pushes + only: + variables: + - $OE_MINNOWBOARD + +Oe-selftest rpi: + extends: .oe-selftest + + stage: test + variables: + TEST_BUILD_DIR: 'build-oe-rpi' + OE_SELFTESTS: 'updater_rpi' + except: + - pushes + only: + variables: + - $OE_RPI + +Ptest qemux86_64: + extends: .oe-selftest -Build core-image-minimal: - image: "$BITBAKE_PR_IMAGE" stage: test + variables: + TEST_BUILD_DIR: 'build-oe-qemux86_64-ptest' + OE_SELFTESTS: 'updater_qemux86_64_ptest' except: - pushes - dependencies: - - Checkout - tags: - - bitbake - script: - - ./scripts/ci/configure.sh - - ./scripts/ci/build.sh core-image-minimal + only: + variables: + - $OE_PTEST diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout index efec234..5210c6b 100644 --- a/scripts/ci/Dockerfile.checkout +++ b/scripts/ci/Dockerfile.checkout @@ -7,3 +7,7 @@ RUN apt-get update -q && apt-get install -qy \ git \ repo \ xmlstarlet + +# checkout script +RUN mkdir /scripts +COPY checkout-oe.sh /scripts/ diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh index 8744b2f..a99f235 100755 --- a/scripts/ci/checkout-oe.sh +++ b/scripts/ci/checkout-oe.sh @@ -6,7 +6,11 @@ set -x REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic} MANIFEST=${MANIFEST:-master} -CURRENT_PROJECT=${CURRENT_PROJECT:-meta-updater} +CURRENT_PROJECT=${CURRENT_PROJECT:-} + +# list of projects to pin to one version in the format: +# "project:rev;project2:rev2..." +PIN_LIST=${PIN_LIST:-} #CURRENT_REV=$(git rev-parse HEAD) LOCAL_REPO=$PWD @@ -22,26 +26,48 @@ git -C .repo/manifests reset --hard # patch manifest: # - add a new "ats" remote that points to "$REMOTE_SOURCE" # - change projects that contain "advancedtelematic" to use the ats remote -# - remove the current project from the manifest MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml" xmlstarlet ed --omit-decl -L \ -s "/manifest" -t elem -n "remote" -v "" \ -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \ -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \ + -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \ -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \ - -d "/manifest/project[@path=\"$CURRENT_PROJECT\"]" \ "$MANIFEST_FILE" # hack: sed on `advancedtelematic/` names, to remove this unwanted prefix sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE" +# pin projects from the list +( +IFS=";" +for pin in $PIN_LIST; do + IFS=":" + read -r project rev <<< "$pin" + xmlstarlet ed --omit-decl -L \ + -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \ + -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \ + "$MANIFEST_FILE" + IFS=";" +done +) + +# Remove the current project from the manifest if we have it checked out +if [ -n "$CURRENT_PROJECT" ]; then + xmlstarlet ed --omit-decl -L \ + -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \ + "$MANIFEST_FILE" +fi + repo manifest repo forall -c 'git reset --hard ; git clean -fdx' repo sync -d --force-sync -rm -f "$CURRENT_PROJECT" -ln -s "$LOCAL_REPO" "$CURRENT_PROJECT" +if [ -n "$CURRENT_PROJECT" ]; then + rm -f "$CURRENT_PROJECT" + ln -s "$LOCAL_REPO" "$CURRENT_PROJECT" +fi repo manifest -r diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh new file mode 100755 index 0000000..1ffbbf5 --- /dev/null +++ b/scripts/ci/docker/setup_kvm.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This script makes the gid of the 'kvm' group to match the group +# owner of '/dev/kvm' +# +# These two are not guaranteed to match when a docker image starts +# with access to '/dev/kvm' that comes from the host + +set -euo pipefail + +kvm_gid=$(stat -c "%g" /dev/kvm) +groupmod -g "$kvm_gid" kvm +usermod -a -G kvm bitbake +ln -s /bin/true /usr/bin/kvm-ok diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml new file mode 100644 index 0000000..28c9177 --- /dev/null +++ b/scripts/ci/gitlab/checkout.yml @@ -0,0 +1,16 @@ +.bb_checkout: + # parameters: + # - MANIFEST + # - REMOTE_SOURCE + # - CURRENT_PROJECT (will be symlinked instead of pulled) + + image: $BITBAKE_CHECKOUT_IMAGE + cache: + paths: + - updater-repo + artifacts: + expire_in: "1 day" + paths: + - updater-repo + script: + - /scripts/checkout-oe.sh diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml new file mode 100644 index 0000000..827bb47 --- /dev/null +++ b/scripts/ci/gitlab/docker.yml @@ -0,0 +1,37 @@ +.bb_docker_local: + # intended to be run on meta-updater's master branch which contains the + # reference docker files + # parameters: + # - BITBAKE_IMAGE + # - BITBAKE_CHECKOUT_IMAGE + # - BITBKAE_IMAGE_MASTER + # - BITBAKE_CHECKOUT_IMAGE_MASTER + image: docker:stable + stage: docker + services: + - docker:dind + before_script: + - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" + script: + - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true + - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci + - docker push "$BITBAKE_IMAGE" + + - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true + - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci + - docker push "$BITBAKE_CHECKOUT_IMAGE" + +.bb_docker_remote: + # intended to be run on other branches and repos: just pulls the last master image + # parameters: + # - BITBAKE_IMAGE + # - BITBAKE_CHECKOUT_IMAGE + image: docker:stable + stage: docker + services: + - docker:dind + before_script: + - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" + script: + - docker pull "$BITBAKE_IMAGE" + - docker pull "$BITBAKE_CHECKOUT_IMAGE" diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml new file mode 100644 index 0000000..4bd5844 --- /dev/null +++ b/scripts/ci/gitlab/tests.yml @@ -0,0 +1,36 @@ +.bitbake: + # parameters: + # - TEST_BUILD_DIR + # - TEST_MACHINE (defaults to qemux86-64) + # - BITBAKE_TARGETS + image: $BITBAKE_IMAGE + dependencies: + - Checkout + tags: + - bitbake + script: + - /scripts/configure.sh + - /scripts/build.sh $BITBAKE_TARGETS + +.oe-selftest: + # parameters: + # - TEST_BUILD_DIR + # - TEST_MACHINE (defaults to qemux86-64) + # - OE_SELFTESTS + image: $BITBAKE_IMAGE + dependencies: + - Checkout + tags: + - bitbake + variables: + TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip + before_script: + - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip + - sudo /usr/local/bin/setup_kvm.sh + script: + - | + # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh) + sg kvm << EOS + /scripts/configure.sh + /scripts/oe-selftest.sh $OE_SELFTESTS + EOS -- cgit v1.2.3-54-g00ecf