From 9eebc8d15a297d68fc35dd4d77fcb945d2f395b2 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Thu, 4 Jul 2019 15:35:44 +0200 Subject: Gitlab pipeline for building core-image-minimal Including these points (initially as separate commits): * Try caching updater-repo in gitlab * Only patch aktualizr recipe in CI if needed * Run under user 4321 in bitbake docker image * Add ability to supply METADIR to envsetup.sh So that we can bypass the "relative path from the script" trick that doesn't work with symlinks. * Checkout the PR base branch * do not run on pushes * run on runners with "gitlab" tag * Print all revisions after repo checkout on CI * Pull CI image from the base branch instead of master * More robust checkout in CI * Only store updater-repo artifacts for one day * Use "stretch" instead of "stable" for Debian release in Dockerfile (in fact, stable just changed to buster) * Move UPDATER_REPO variable out of .gitlab-ci.yml To be defined in the project's settings * Make docker CI stage buildable on shared runners Signed-off-by: Laurent Bonnans --- .gitlab-ci.yml | 63 ++++++++++++++++++++++++++++++++++++++++++ scripts/ci/Dockerfile.bitbake | 12 ++++---- scripts/ci/Dockerfile.checkout | 9 ++++++ scripts/ci/build.sh | 3 ++ scripts/ci/checkout-oe.sh | 47 +++++++++++++++++++++++++++++++ scripts/ci/configure.sh | 20 +++++++++----- scripts/ci/oe-selftest.sh | 5 +++- scripts/envsetup.sh | 2 +- 8 files changed, 147 insertions(+), 14 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 scripts/ci/Dockerfile.checkout create mode 100755 scripts/ci/checkout-oe.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..94a614c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,63 @@ +stages: + - docker + - checkout + - test + +# notes: +# - could be useful https://docs.gitlab.com/ee/ci/yaml/#include + +variables: + # Needs $PR_BASE_BRANCH to be defined as a pipeline variable to work correctly + + CHECKOUT_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-checkout + CHECKOUT_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout + + BITBAKE_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-bitbake + BITBAKE_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake + + +Docker Setup: + image: docker:stable + stage: docker + services: + - docker:dind + except: + - pushes + before_script: + - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" + script: + - docker pull "$BITBAKE_PR_IMAGE" || docker pull "$BITBAKE_MASTER_IMAGE" || true + - docker build --pull --cache-from "$BITBAKE_MASTER_IMAGE" --cache-from "$BITBAKE_PR_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_PR_IMAGE" ./scripts/ci + - docker push "$BITBAKE_PR_IMAGE" + + - docker pull "$CHECKOUT_PR_IMAGE" || docker pull "$CHECKOUT_MASTER_IMAGE" || true + - docker build --pull --cache-from "$CHECKOUT_MASTER_IMAGE" --cache-from "$CHECKOUT_PR_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$CHECKOUT_PR_IMAGE" ./scripts/ci + - docker push "$CHECKOUT_PR_IMAGE" + +Checkout: + image: "$CHECKOUT_PR_IMAGE" + stage: checkout + except: + - pushes + cache: + paths: + - updater-repo + artifacts: + expire_in: "1 day" + paths: + - updater-repo + script: + - MANIFEST=$PR_BASE_BRANCH ./scripts/ci/checkout-oe.sh + +Build core-image-minimal: + image: "$BITBAKE_PR_IMAGE" + stage: test + except: + - pushes + dependencies: + - Checkout + tags: + - bitbake + script: + - ./scripts/ci/configure.sh + - ./scripts/ci/build.sh core-image-minimal diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake index c91f94c..75bad14 100644 --- a/scripts/ci/Dockerfile.bitbake +++ b/scripts/ci/Dockerfile.bitbake @@ -1,8 +1,8 @@ -FROM debian:stable +FROM debian:stretch LABEL Description="Image for bitbaking" -RUN sed -i 's#deb http://deb.debian.org/debian stable main#deb http://deb.debian.org/debian stable main contrib#g' /etc/apt/sources.list -RUN sed -i 's#deb http://deb.debian.org/debian stable-updates main#deb http://deb.debian.org/debian stable-updates main contrib#g' /etc/apt/sources.list +RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list +RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list RUN apt-get update -q && apt-get install -qy \ build-essential \ bzip2 \ @@ -31,8 +31,8 @@ RUN apt-get update -q && apt-get install -qy \ xterm \ xz-utils -ARG uid=1000 -ARG gid=1000 +ARG uid=4321 +ARG gid=4321 RUN groupadd -g $gid bitbake RUN useradd -m -u $uid -g $gid bitbake @@ -40,3 +40,5 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen ENV LC_ALL="en_US.UTF-8" ENV LANG="en_US.UTF-8" ENV LANGUAGE="en_US.UTF-8" + +USER "bitbake" diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout new file mode 100644 index 0000000..efec234 --- /dev/null +++ b/scripts/ci/Dockerfile.checkout @@ -0,0 +1,9 @@ +FROM debian:stretch-slim +LABEL Description="Image for checking out updater-repo" + +RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list +RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list +RUN apt-get update -q && apt-get install -qy \ + git \ + repo \ + xmlstarlet diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 6235428..9fbae98 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -12,7 +12,10 @@ IMAGE_NAME=${1:-core-image-minimal} ( set +euo pipefail set +x +METADIR=$(realpath "$TEST_REPO_DIR") +export METADIR . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}" +set -x bitbake "${IMAGE_NAME}" ) diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh new file mode 100755 index 0000000..8744b2f --- /dev/null +++ b/scripts/ci/checkout-oe.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -euo pipefail + +set -x + +REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic} +MANIFEST=${MANIFEST:-master} +CURRENT_PROJECT=${CURRENT_PROJECT:-meta-updater} + +#CURRENT_REV=$(git rev-parse HEAD) +LOCAL_REPO=$PWD + +mkdir -p updater-repo + +cd updater-repo + +repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo" + +git -C .repo/manifests reset --hard + +# patch manifest: +# - add a new "ats" remote that points to "$REMOTE_SOURCE" +# - change projects that contain "advancedtelematic" to use the ats remote +# - remove the current project from the manifest +MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml" +xmlstarlet ed --omit-decl -L \ + -s "/manifest" -t elem -n "remote" -v "" \ + -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \ + -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \ + -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \ + -d "/manifest/project[@path=\"$CURRENT_PROJECT\"]" \ + "$MANIFEST_FILE" + +# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix +sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE" + +repo manifest + +repo forall -c 'git reset --hard ; git clean -fdx' + +repo sync -d --force-sync + +rm -f "$CURRENT_PROJECT" +ln -s "$LOCAL_REPO" "$CURRENT_PROJECT" + +repo manifest -r diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh index 960a0cc..a8968da 100755 --- a/scripts/ci/configure.sh +++ b/scripts/ci/configure.sh @@ -8,9 +8,13 @@ TEST_BUILD_DIR=${TEST_BUILD_DIR:-build} TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo} TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-} -TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.} -TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master} -TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")} +TEST_AKTUALIZR_REMOTE=${TEST_AKTUALIZR_REMOTE:-} +if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then + TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.} + TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master} + TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")} +fi + TEST_AKTUALIZR_CREDENTIALS=${TEST_AKTUALIZR_CREDENTIALS:-} # move existing conf directory to backup, before generating a new one @@ -21,6 +25,8 @@ mv "$TEST_BUILD_DIR/conf" "$TEST_BUILD_DIR/conf.old" || true set +euo pipefail set +x echo ">> Running envsetup.sh" +METADIR=$(realpath "$TEST_REPO_DIR") +export METADIR . "$TEST_REPO_DIR/meta-updater/scripts/envsetup.sh" "$TEST_MACHINE" "$TEST_BUILD_DIR" ) @@ -31,19 +37,19 @@ SITE_CONF="$TEST_BUILD_DIR/conf/site.conf" echo ">> Set common bitbake config options" cat << EOF > "$SITE_CONF" SANITY_TESTED_DISTROS = "" -SSTATE_MIRRORS ?= "file://.* https://bitbake-cache.atsgarage.com/PATH;downloadfilename=PATH" IMAGE_FEATURES += "ssh-server-openssh" EOF -echo ">> Set aktualizr branch in bitbake's config" -cat << EOF >> "$SITE_CONF" +if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then + echo ">> Set aktualizr branch in bitbake's config" + cat << EOF >> "$SITE_CONF" SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV" SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}" BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH" BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}" - EOF +fi if [[ -n $TEST_AKTUALIZR_CREDENTIALS ]]; then echo ">> Set aktualizr credentials" diff --git a/scripts/ci/oe-selftest.sh b/scripts/ci/oe-selftest.sh index 3124cce..66b1fc9 100755 --- a/scripts/ci/oe-selftest.sh +++ b/scripts/ci/oe-selftest.sh @@ -12,7 +12,10 @@ TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo} ( set +euo pipefail set +x +METADIR=$(realpath "$TEST_REPO_DIR") +export METADIR . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}" -oe-selftest -r updater +set -x +oe-selftest -r "$@" ) diff --git a/scripts/envsetup.sh b/scripts/envsetup.sh index 6103cf6..5827bc2 100755 --- a/scripts/envsetup.sh +++ b/scripts/envsetup.sh @@ -21,7 +21,7 @@ if [[ $SOURCED -ne 1 ]]; then exit 1 fi -METADIR="${SOURCEDIR}/../.." +METADIR=${METADIR:-${SOURCEDIR}/../..} if [[ ! -f "${BUILDDIR}/conf/local.conf" ]]; then source "$METADIR/poky/oe-init-build-env" "$BUILDDIR" -- cgit v1.2.3-54-g00ecf