summaryrefslogtreecommitdiffstats
path: root/scripts/ci
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/ci')
-rw-r--r--scripts/ci/Dockerfile.bitbake32
-rw-r--r--scripts/ci/Dockerfile.checkout13
-rwxr-xr-xscripts/ci/build.sh3
-rwxr-xr-xscripts/ci/checkout-oe.sh73
-rwxr-xr-xscripts/ci/configure.sh20
-rwxr-xr-xscripts/ci/docker/setup_kvm.sh14
-rw-r--r--scripts/ci/gitlab/checkout.yml16
-rw-r--r--scripts/ci/gitlab/docker.yml37
-rw-r--r--scripts/ci/gitlab/tests.yml36
-rwxr-xr-xscripts/ci/oe-selftest.sh5
10 files changed, 233 insertions, 16 deletions
diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake
index c91f94c..51eaa57 100644
--- a/scripts/ci/Dockerfile.bitbake
+++ b/scripts/ci/Dockerfile.bitbake
@@ -1,15 +1,17 @@
1FROM debian:stable 1FROM debian:stretch
2LABEL Description="Image for bitbaking" 2LABEL Description="Image for bitbaking"
3 3
4RUN sed -i 's#deb http://deb.debian.org/debian stable main#deb http://deb.debian.org/debian stable main contrib#g' /etc/apt/sources.list 4RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
5RUN sed -i 's#deb http://deb.debian.org/debian stable-updates main#deb http://deb.debian.org/debian stable-updates main contrib#g' /etc/apt/sources.list 5RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
6RUN apt-get update -q && apt-get install -qy \ 6RUN apt-get update -q && apt-get install --no-install-suggests --no-install-recommends -qy \
7 awscli \
7 build-essential \ 8 build-essential \
8 bzip2 \ 9 bzip2 \
9 chrpath \ 10 chrpath \
10 cpio \ 11 cpio \
11 default-jre \ 12 default-jre \
12 diffstat \ 13 diffstat \
14 file \
13 gawk \ 15 gawk \
14 gcc-multilib \ 16 gcc-multilib \
15 git-core \ 17 git-core \
@@ -17,26 +19,40 @@ RUN apt-get update -q && apt-get install -qy \
17 iproute \ 19 iproute \
18 libpython-dev \ 20 libpython-dev \
19 libsdl1.2-dev \ 21 libsdl1.2-dev \
22 libvirt-clients \
23 libvirt-daemon-system \
20 locales \ 24 locales \
21 ovmf \ 25 ovmf \
26 openssh-client \
22 procps \ 27 procps \
23 python \ 28 python \
24 python3 \ 29 python3 \
25 python3-pexpect \ 30 python3-pexpect \
26 qemu \ 31 qemu-kvm \
27 socat \ 32 socat \
33 sudo \
28 texinfo \ 34 texinfo \
29 unzip \ 35 unzip \
30 wget \ 36 wget \
31 xterm \ 37 xterm \
32 xz-utils 38 xz-utils
33 39
34ARG uid=1000 40ARG uid=4321
35ARG gid=1000 41ARG gid=4321
36RUN groupadd -g $gid bitbake 42RUN groupadd -g $gid bitbake
37RUN useradd -m -u $uid -g $gid bitbake 43RUN useradd -m -u $uid -g $gid -s /bin/bash bitbake
38 44
39RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen 45RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen
40ENV LC_ALL="en_US.UTF-8" 46ENV LC_ALL="en_US.UTF-8"
41ENV LANG="en_US.UTF-8" 47ENV LANG="en_US.UTF-8"
42ENV LANGUAGE="en_US.UTF-8" 48ENV LANGUAGE="en_US.UTF-8"
49
50# script to mirror kvm group id with host
51RUN echo "bitbake ALL=NOPASSWD: /usr/local/bin/setup_kvm.sh" >> /etc/sudoers
52COPY ./docker/setup_kvm.sh /usr/local/bin/setup_kvm.sh
53
54# other ci scripts
55RUN mkdir /scripts
56COPY configure.sh build.sh oe-selftest.sh /scripts/
57
58USER "bitbake"
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
new file mode 100644
index 0000000..5210c6b
--- /dev/null
+++ b/scripts/ci/Dockerfile.checkout
@@ -0,0 +1,13 @@
1FROM debian:stretch-slim
2LABEL Description="Image for checking out updater-repo"
3
4RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list
5RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list
6RUN apt-get update -q && apt-get install -qy \
7 git \
8 repo \
9 xmlstarlet
10
11# checkout script
12RUN mkdir /scripts
13COPY checkout-oe.sh /scripts/
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
index 6235428..9fbae98 100755
--- a/scripts/ci/build.sh
+++ b/scripts/ci/build.sh
@@ -12,7 +12,10 @@ IMAGE_NAME=${1:-core-image-minimal}
12( 12(
13set +euo pipefail 13set +euo pipefail
14set +x 14set +x
15METADIR=$(realpath "$TEST_REPO_DIR")
16export METADIR
15. "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}" 17. "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
16 18
19set -x
17bitbake "${IMAGE_NAME}" 20bitbake "${IMAGE_NAME}"
18) 21)
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
new file mode 100755
index 0000000..a99f235
--- /dev/null
+++ b/scripts/ci/checkout-oe.sh
@@ -0,0 +1,73 @@
1#!/usr/bin/env bash
2
3set -euo pipefail
4
5set -x
6
7REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
8MANIFEST=${MANIFEST:-master}
9CURRENT_PROJECT=${CURRENT_PROJECT:-}
10
11# list of projects to pin to one version in the format:
12# "project:rev;project2:rev2..."
13PIN_LIST=${PIN_LIST:-}
14
15#CURRENT_REV=$(git rev-parse HEAD)
16LOCAL_REPO=$PWD
17
18mkdir -p updater-repo
19
20cd updater-repo
21
22repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo"
23
24git -C .repo/manifests reset --hard
25
26# patch manifest:
27# - add a new "ats" remote that points to "$REMOTE_SOURCE"
28# - change projects that contain "advancedtelematic" to use the ats remote
29MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
30xmlstarlet ed --omit-decl -L \
31 -s "/manifest" -t elem -n "remote" -v "" \
32 -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
33 -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
34 -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \
35 -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \
36 "$MANIFEST_FILE"
37
38# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix
39sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
40
41# pin projects from the list
42(
43IFS=";"
44for pin in $PIN_LIST; do
45 IFS=":"
46 read -r project rev <<< "$pin"
47 xmlstarlet ed --omit-decl -L \
48 -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \
49 -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \
50 "$MANIFEST_FILE"
51 IFS=";"
52done
53)
54
55# Remove the current project from the manifest if we have it checked out
56if [ -n "$CURRENT_PROJECT" ]; then
57 xmlstarlet ed --omit-decl -L \
58 -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \
59 "$MANIFEST_FILE"
60fi
61
62repo manifest
63
64repo forall -c 'git reset --hard ; git clean -fdx'
65
66repo sync -d --force-sync
67
68if [ -n "$CURRENT_PROJECT" ]; then
69 rm -f "$CURRENT_PROJECT"
70 ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
71fi
72
73repo manifest -r
diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh
index 960a0cc..a8968da 100755
--- a/scripts/ci/configure.sh
+++ b/scripts/ci/configure.sh
@@ -8,9 +8,13 @@ TEST_BUILD_DIR=${TEST_BUILD_DIR:-build}
8TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo} 8TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
9TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-} 9TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-}
10 10
11TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.} 11TEST_AKTUALIZR_REMOTE=${TEST_AKTUALIZR_REMOTE:-}
12TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master} 12if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
13TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")} 13 TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.}
14 TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master}
15 TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")}
16fi
17
14TEST_AKTUALIZR_CREDENTIALS=${TEST_AKTUALIZR_CREDENTIALS:-} 18TEST_AKTUALIZR_CREDENTIALS=${TEST_AKTUALIZR_CREDENTIALS:-}
15 19
16# move existing conf directory to backup, before generating a new one 20# move existing conf directory to backup, before generating a new one
@@ -21,6 +25,8 @@ mv "$TEST_BUILD_DIR/conf" "$TEST_BUILD_DIR/conf.old" || true
21set +euo pipefail 25set +euo pipefail
22set +x 26set +x
23echo ">> Running envsetup.sh" 27echo ">> Running envsetup.sh"
28METADIR=$(realpath "$TEST_REPO_DIR")
29export METADIR
24. "$TEST_REPO_DIR/meta-updater/scripts/envsetup.sh" "$TEST_MACHINE" "$TEST_BUILD_DIR" 30. "$TEST_REPO_DIR/meta-updater/scripts/envsetup.sh" "$TEST_MACHINE" "$TEST_BUILD_DIR"
25) 31)
26 32
@@ -31,19 +37,19 @@ SITE_CONF="$TEST_BUILD_DIR/conf/site.conf"
31echo ">> Set common bitbake config options" 37echo ">> Set common bitbake config options"
32cat << EOF > "$SITE_CONF" 38cat << EOF > "$SITE_CONF"
33SANITY_TESTED_DISTROS = "" 39SANITY_TESTED_DISTROS = ""
34SSTATE_MIRRORS ?= "file://.* https://bitbake-cache.atsgarage.com/PATH;downloadfilename=PATH"
35IMAGE_FEATURES += "ssh-server-openssh" 40IMAGE_FEATURES += "ssh-server-openssh"
36 41
37EOF 42EOF
38 43
39echo ">> Set aktualizr branch in bitbake's config" 44if [ -n "$TEST_AKTUALIZR_REMOTE" ]; then
40cat << EOF >> "$SITE_CONF" 45 echo ">> Set aktualizr branch in bitbake's config"
46 cat << EOF >> "$SITE_CONF"
41SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV" 47SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV"
42SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}" 48SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}"
43BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH" 49BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH"
44BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}" 50BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}"
45
46EOF 51EOF
52fi
47 53
48if [[ -n $TEST_AKTUALIZR_CREDENTIALS ]]; then 54if [[ -n $TEST_AKTUALIZR_CREDENTIALS ]]; then
49 echo ">> Set aktualizr credentials" 55 echo ">> Set aktualizr credentials"
diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh
new file mode 100755
index 0000000..1ffbbf5
--- /dev/null
+++ b/scripts/ci/docker/setup_kvm.sh
@@ -0,0 +1,14 @@
1#!/usr/bin/env bash
2
3# This script makes the gid of the 'kvm' group to match the group
4# owner of '/dev/kvm'
5#
6# These two are not guaranteed to match when a docker image starts
7# with access to '/dev/kvm' that comes from the host
8
9set -euo pipefail
10
11kvm_gid=$(stat -c "%g" /dev/kvm)
12groupmod -g "$kvm_gid" kvm
13usermod -a -G kvm bitbake
14ln -s /bin/true /usr/bin/kvm-ok
diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml
new file mode 100644
index 0000000..28c9177
--- /dev/null
+++ b/scripts/ci/gitlab/checkout.yml
@@ -0,0 +1,16 @@
1.bb_checkout:
2 # parameters:
3 # - MANIFEST
4 # - REMOTE_SOURCE
5 # - CURRENT_PROJECT (will be symlinked instead of pulled)
6
7 image: $BITBAKE_CHECKOUT_IMAGE
8 cache:
9 paths:
10 - updater-repo
11 artifacts:
12 expire_in: "1 day"
13 paths:
14 - updater-repo
15 script:
16 - /scripts/checkout-oe.sh
diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml
new file mode 100644
index 0000000..827bb47
--- /dev/null
+++ b/scripts/ci/gitlab/docker.yml
@@ -0,0 +1,37 @@
1.bb_docker_local:
2 # intended to be run on meta-updater's master branch which contains the
3 # reference docker files
4 # parameters:
5 # - BITBAKE_IMAGE
6 # - BITBAKE_CHECKOUT_IMAGE
7 # - BITBKAE_IMAGE_MASTER
8 # - BITBAKE_CHECKOUT_IMAGE_MASTER
9 image: docker:stable
10 stage: docker
11 services:
12 - docker:dind
13 before_script:
14 - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
15 script:
16 - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true
17 - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci
18 - docker push "$BITBAKE_IMAGE"
19
20 - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true
21 - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci
22 - docker push "$BITBAKE_CHECKOUT_IMAGE"
23
24.bb_docker_remote:
25 # intended to be run on other branches and repos: just pulls the last master image
26 # parameters:
27 # - BITBAKE_IMAGE
28 # - BITBAKE_CHECKOUT_IMAGE
29 image: docker:stable
30 stage: docker
31 services:
32 - docker:dind
33 before_script:
34 - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
35 script:
36 - docker pull "$BITBAKE_IMAGE"
37 - docker pull "$BITBAKE_CHECKOUT_IMAGE"
diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml
new file mode 100644
index 0000000..4bd5844
--- /dev/null
+++ b/scripts/ci/gitlab/tests.yml
@@ -0,0 +1,36 @@
1.bitbake:
2 # parameters:
3 # - TEST_BUILD_DIR
4 # - TEST_MACHINE (defaults to qemux86-64)
5 # - BITBAKE_TARGETS
6 image: $BITBAKE_IMAGE
7 dependencies:
8 - Checkout
9 tags:
10 - bitbake
11 script:
12 - /scripts/configure.sh
13 - /scripts/build.sh $BITBAKE_TARGETS
14
15.oe-selftest:
16 # parameters:
17 # - TEST_BUILD_DIR
18 # - TEST_MACHINE (defaults to qemux86-64)
19 # - OE_SELFTESTS
20 image: $BITBAKE_IMAGE
21 dependencies:
22 - Checkout
23 tags:
24 - bitbake
25 variables:
26 TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip
27 before_script:
28 - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip
29 - sudo /usr/local/bin/setup_kvm.sh
30 script:
31 - |
32 # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh)
33 sg kvm << EOS
34 /scripts/configure.sh
35 /scripts/oe-selftest.sh $OE_SELFTESTS
36 EOS
diff --git a/scripts/ci/oe-selftest.sh b/scripts/ci/oe-selftest.sh
index 3124cce..66b1fc9 100755
--- a/scripts/ci/oe-selftest.sh
+++ b/scripts/ci/oe-selftest.sh
@@ -12,7 +12,10 @@ TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo}
12( 12(
13set +euo pipefail 13set +euo pipefail
14set +x 14set +x
15METADIR=$(realpath "$TEST_REPO_DIR")
16export METADIR
15. "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}" 17. "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}"
16 18
17oe-selftest -r updater 19set -x
20oe-selftest -r "$@"
18) 21)