summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorLaurent Bonnans <laurent.bonnans@here.com>2019-07-05 17:45:39 +0200
committerLaurent Bonnans <laurent.bonnans@here.com>2019-07-16 16:47:13 +0200
commit5463396b1d47a122f5aef9a619244fd1777c0b56 (patch)
tree8f5430210f314fc3ef7aead29beecf99047b8894 /scripts
parent9eebc8d15a297d68fc35dd4d77fcb945d2f395b2 (diff)
downloadmeta-updater-5463396b1d47a122f5aef9a619244fd1777c0b56.tar.gz
Reusable meta-updater pipeline suiteci/oe-selftest
Squashed: * Oe-selftest GitLab pipeline stage * Use credentials for CI's oe-selftest * Setup kvm trick for docker on CI The gid of the kvm group needs to match the one from the host * Run ci scripts from Docker images * Template out jobs and split .gitlab-ci.yml To be easily reused in other branches and projects Rely on this gitlab feature: https://docs.gitlab.com/ee/ci/yaml/#extends * More flexible checkout script for CI - can work without $CURRENT_PROJECT (checks out everything) - can take a list of pinned versions * Add optional CI jobs ptest and other oe-selftests * Publish bitbaked images as artifacts Signed-off-by: Laurent Bonnans <laurent.bonnans@here.com>
Diffstat (limited to 'scripts')
-rw-r--r--scripts/ci/Dockerfile.bitbake20
-rw-r--r--scripts/ci/Dockerfile.checkout4
-rwxr-xr-xscripts/ci/checkout-oe.sh36
-rwxr-xr-xscripts/ci/docker/setup_kvm.sh14
-rw-r--r--scripts/ci/gitlab/checkout.yml16
-rw-r--r--scripts/ci/gitlab/docker.yml37
-rw-r--r--scripts/ci/gitlab/tests.yml36
7 files changed, 155 insertions, 8 deletions
diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake
index 75bad14..51eaa57 100644
--- a/scripts/ci/Dockerfile.bitbake
+++ b/scripts/ci/Dockerfile.bitbake
@@ -3,13 +3,15 @@ LABEL Description="Image for bitbaking"
3 3
4RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list 4RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
5RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list 5RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
6RUN apt-get update -q && apt-get install -qy \ 6RUN apt-get update -q && apt-get install --no-install-suggests --no-install-recommends -qy \
7 awscli \
7 build-essential \ 8 build-essential \
8 bzip2 \ 9 bzip2 \
9 chrpath \ 10 chrpath \
10 cpio \ 11 cpio \
11 default-jre \ 12 default-jre \
12 diffstat \ 13 diffstat \
14 file \
13 gawk \ 15 gawk \
14 gcc-multilib \ 16 gcc-multilib \
15 git-core \ 17 git-core \
@@ -17,14 +19,18 @@ RUN apt-get update -q && apt-get install -qy \
17 iproute \ 19 iproute \
18 libpython-dev \ 20 libpython-dev \
19 libsdl1.2-dev \ 21 libsdl1.2-dev \
22 libvirt-clients \
23 libvirt-daemon-system \
20 locales \ 24 locales \
21 ovmf \ 25 ovmf \
26 openssh-client \
22 procps \ 27 procps \
23 python \ 28 python \
24 python3 \ 29 python3 \
25 python3-pexpect \ 30 python3-pexpect \
26 qemu \ 31 qemu-kvm \
27 socat \ 32 socat \
33 sudo \
28 texinfo \ 34 texinfo \
29 unzip \ 35 unzip \
30 wget \ 36 wget \
@@ -34,11 +40,19 @@ RUN apt-get update -q && apt-get install -qy \
34ARG uid=4321 40ARG uid=4321
35ARG gid=4321 41ARG gid=4321
36RUN groupadd -g $gid bitbake 42RUN groupadd -g $gid bitbake
37RUN useradd -m -u $uid -g $gid bitbake 43RUN useradd -m -u $uid -g $gid -s /bin/bash bitbake
38 44
39RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen 45RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
40ENV LC_ALL="en_US.UTF-8" 46ENV LC_ALL="en_US.UTF-8"
41ENV LANG="en_US.UTF-8" 47ENV LANG="en_US.UTF-8"
42ENV LANGUAGE="en_US.UTF-8" 48ENV LANGUAGE="en_US.UTF-8"
43 49
50# script to mirror kvm group id with host
51RUN echo "bitbake ALL=NOPASSWD: /usr/local/bin/setup_kvm.sh" >> /etc/sudoers
52COPY ./docker/setup_kvm.sh /usr/local/bin/setup_kvm.sh
53
54# other ci scripts
55RUN mkdir /scripts
56COPY configure.sh build.sh oe-selftest.sh /scripts/
57
44USER "bitbake" 58USER "bitbake"
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
index efec234..5210c6b 100644
--- a/scripts/ci/Dockerfile.checkout
+++ b/scripts/ci/Dockerfile.checkout
@@ -7,3 +7,7 @@ RUN apt-get update -q && apt-get install -qy \
7 git \ 7 git \
8 repo \ 8 repo \
9 xmlstarlet 9 xmlstarlet
10
11# checkout script
12RUN mkdir /scripts
13COPY checkout-oe.sh /scripts/
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
index 8744b2f..a99f235 100755
--- a/scripts/ci/checkout-oe.sh
+++ b/scripts/ci/checkout-oe.sh
@@ -6,7 +6,11 @@ set -x
6 6
7REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic} 7REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
8MANIFEST=${MANIFEST:-master} 8MANIFEST=${MANIFEST:-master}
9CURRENT_PROJECT=${CURRENT_PROJECT:-meta-updater} 9CURRENT_PROJECT=${CURRENT_PROJECT:-}
10
11# list of projects to pin to one version in the format:
12# "project:rev;project2:rev2..."
13PIN_LIST=${PIN_LIST:-}
10 14
11#CURRENT_REV=$(git rev-parse HEAD) 15#CURRENT_REV=$(git rev-parse HEAD)
12LOCAL_REPO=$PWD 16LOCAL_REPO=$PWD
@@ -22,26 +26,48 @@ git -C .repo/manifests reset --hard
22# patch manifest: 26# patch manifest:
23# - add a new "ats" remote that points to "$REMOTE_SOURCE" 27# - add a new "ats" remote that points to "$REMOTE_SOURCE"
24# - change projects that contain "advancedtelematic" to use the ats remote 28# - change projects that contain "advancedtelematic" to use the ats remote
25# - remove the current project from the manifest
26MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml" 29MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
27xmlstarlet ed --omit-decl -L \ 30xmlstarlet ed --omit-decl -L \
28 -s "/manifest" -t elem -n "remote" -v "" \ 31 -s "/manifest" -t elem -n "remote" -v "" \
29 -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \ 32 -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
30 -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \ 33 -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
34 -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \
31 -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \ 35 -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \
32 -d "/manifest/project[@path=\"$CURRENT_PROJECT\"]" \
33 "$MANIFEST_FILE" 36 "$MANIFEST_FILE"
34 37
35# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix 38# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix
36sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE" 39sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
37 40
41# pin projects from the list
42(
43IFS=";"
44for pin in $PIN_LIST; do
45 IFS=":"
46 read -r project rev <<< "$pin"
47 xmlstarlet ed --omit-decl -L \
48 -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \
49 -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \
50 "$MANIFEST_FILE"
51 IFS=";"
52done
53)
54
55# Remove the current project from the manifest if we have it checked out
56if [ -n "$CURRENT_PROJECT" ]; then
57 xmlstarlet ed --omit-decl -L \
58 -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \
59 "$MANIFEST_FILE"
60fi
61
38repo manifest 62repo manifest
39 63
40repo forall -c 'git reset --hard ; git clean -fdx' 64repo forall -c 'git reset --hard ; git clean -fdx'
41 65
42repo sync -d --force-sync 66repo sync -d --force-sync
43 67
44rm -f "$CURRENT_PROJECT" 68if [ -n "$CURRENT_PROJECT" ]; then
45ln -s "$LOCAL_REPO" "$CURRENT_PROJECT" 69 rm -f "$CURRENT_PROJECT"
70 ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
71fi
46 72
47repo manifest -r 73repo manifest -r
diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh
new file mode 100755
index 0000000..1ffbbf5
--- /dev/null
+++ b/scripts/ci/docker/setup_kvm.sh
@@ -0,0 +1,14 @@
1#!/usr/bin/env bash
2
3# This script makes the gid of the 'kvm' group to match the group
4# owner of '/dev/kvm'
5#
6# These two are not guaranteed to match when a docker image starts
7# with access to '/dev/kvm' that comes from the host
8
9set -euo pipefail
10
11kvm_gid=$(stat -c "%g" /dev/kvm)
12groupmod -g "$kvm_gid" kvm
13usermod -a -G kvm bitbake
14ln -s /bin/true /usr/bin/kvm-ok
diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml
new file mode 100644
index 0000000..28c9177
--- /dev/null
+++ b/scripts/ci/gitlab/checkout.yml
@@ -0,0 +1,16 @@
1.bb_checkout:
2 # parameters:
3 # - MANIFEST
4 # - REMOTE_SOURCE
5 # - CURRENT_PROJECT (will be symlinked instead of pulled)
6
7 image: $BITBAKE_CHECKOUT_IMAGE
8 cache:
9 paths:
10 - updater-repo
11 artifacts:
12 expire_in: "1 day"
13 paths:
14 - updater-repo
15 script:
16 - /scripts/checkout-oe.sh
diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml
new file mode 100644
index 0000000..827bb47
--- /dev/null
+++ b/scripts/ci/gitlab/docker.yml
@@ -0,0 +1,37 @@
1.bb_docker_local:
2 # intended to be run on meta-updater's master branch which contains the
3 # reference docker files
4 # parameters:
5 # - BITBAKE_IMAGE
6 # - BITBAKE_CHECKOUT_IMAGE
7 # - BITBKAE_IMAGE_MASTER
8 # - BITBAKE_CHECKOUT_IMAGE_MASTER
9 image: docker:stable
10 stage: docker
11 services:
12 - docker:dind
13 before_script:
14 - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
15 script:
16 - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true
17 - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci
18 - docker push "$BITBAKE_IMAGE"
19
20 - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true
21 - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci
22 - docker push "$BITBAKE_CHECKOUT_IMAGE"
23
24.bb_docker_remote:
25 # intended to be run on other branches and repos: just pulls the last master image
26 # parameters:
27 # - BITBAKE_IMAGE
28 # - BITBAKE_CHECKOUT_IMAGE
29 image: docker:stable
30 stage: docker
31 services:
32 - docker:dind
33 before_script:
34 - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
35 script:
36 - docker pull "$BITBAKE_IMAGE"
37 - docker pull "$BITBAKE_CHECKOUT_IMAGE"
diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml
new file mode 100644
index 0000000..4bd5844
--- /dev/null
+++ b/scripts/ci/gitlab/tests.yml
@@ -0,0 +1,36 @@
1.bitbake:
2 # parameters:
3 # - TEST_BUILD_DIR
4 # - TEST_MACHINE (defaults to qemux86-64)
5 # - BITBAKE_TARGETS
6 image: $BITBAKE_IMAGE
7 dependencies:
8 - Checkout
9 tags:
10 - bitbake
11 script:
12 - /scripts/configure.sh
13 - /scripts/build.sh $BITBAKE_TARGETS
14
15.oe-selftest:
16 # parameters:
17 # - TEST_BUILD_DIR
18 # - TEST_MACHINE (defaults to qemux86-64)
19 # - OE_SELFTESTS
20 image: $BITBAKE_IMAGE
21 dependencies:
22 - Checkout
23 tags:
24 - bitbake
25 variables:
26 TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip
27 before_script:
28 - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip
29 - sudo /usr/local/bin/setup_kvm.sh
30 script:
31 - |
32 # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh)
33 sg kvm << EOS
34 /scripts/configure.sh
35 /scripts/oe-selftest.sh $OE_SELFTESTS
36 EOS