summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLaurent Bonnans <laurent.bonnans@here.com>2019-07-05 17:45:39 +0200
committerMykhaylo Sul <ext-mykhaylo.sul@here.com>2019-07-17 16:08:28 +0300
commit32255ab2a79ca6d99b24ee43446aeaa943b21178 (patch)
tree668aad47c787a12a52b8f9f4bf37fbffc95ab1dc
parent14bedef5860abc352d7cfe957f2c3781cf29d52c (diff)
downloadmeta-updater-Backport/rocko/secondaries.tar.gz
Reusable meta-updater pipeline suiteBackport/rocko/secondaries
Squashed: * Oe-selftest GitLab pipeline stage * Use credentials for CI's oe-selftest * Setup kvm trick for docker on CI The gid of the kvm group needs to match the one from the host * Run ci scripts from Docker images * Template out jobs and split .gitlab-ci.yml To be easily reused in other branches and projects Rely on this gitlab feature: https://docs.gitlab.com/ee/ci/yaml/#extends * More flexible checkout script for CI - can work without $CURRENT_PROJECT (checks out everything) - can take a list of pinned versions * Add optional CI jobs ptest and other oe-selftests * Publish bitbaked images as artifacts Signed-off-by: Laurent Bonnans <laurent.bonnans@here.com>
-rw-r--r--.gitlab-ci.yml115
-rw-r--r--scripts/ci/Dockerfile.checkout4
-rwxr-xr-xscripts/ci/checkout-oe.sh36
-rwxr-xr-xscripts/ci/docker/setup_kvm.sh14
-rw-r--r--scripts/ci/gitlab/checkout.yml16
-rw-r--r--scripts/ci/gitlab/docker.yml37
-rw-r--r--scripts/ci/gitlab/tests.yml36
7 files changed, 213 insertions, 45 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 94a614c..3682753 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,61 +3,96 @@ stages:
3 - checkout 3 - checkout
4 - test 4 - test
5 5
6# notes:
7# - could be useful https://docs.gitlab.com/ee/ci/yaml/#include
8
9variables: 6variables:
10 # Needs $PR_BASE_BRANCH to be defined as a pipeline variable to work correctly 7 BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake
11 8 BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout
12 CHECKOUT_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-checkout
13 CHECKOUT_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout
14 9
15 BITBAKE_MASTER_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${PR_BASE_BRANCH}-bitbake 10include:
16 BITBAKE_PR_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake 11 - local: "scripts/ci/gitlab/docker.yml"
12 - local: "scripts/ci/gitlab/checkout.yml"
13 - local: "scripts/ci/gitlab/tests.yml"
17 14
15Docker setup:
16 extends: .bb_docker_local
18 17
19Docker Setup:
20 image: docker:stable
21 stage: docker 18 stage: docker
22 services: 19 variables:
23 - docker:dind 20 BITBKAE_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-bitbake
21 BITBAKE_CHECKOUT_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-checkout
24 except: 22 except:
25 - pushes 23 - pushes
26 before_script:
27 - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
28 script:
29 - docker pull "$BITBAKE_PR_IMAGE" || docker pull "$BITBAKE_MASTER_IMAGE" || true
30 - docker build --pull --cache-from "$BITBAKE_MASTER_IMAGE" --cache-from "$BITBAKE_PR_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_PR_IMAGE" ./scripts/ci
31 - docker push "$BITBAKE_PR_IMAGE"
32
33 - docker pull "$CHECKOUT_PR_IMAGE" || docker pull "$CHECKOUT_MASTER_IMAGE" || true
34 - docker build --pull --cache-from "$CHECKOUT_MASTER_IMAGE" --cache-from "$CHECKOUT_PR_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$CHECKOUT_PR_IMAGE" ./scripts/ci
35 - docker push "$CHECKOUT_PR_IMAGE"
36 24
37Checkout: 25Checkout:
38 image: "$CHECKOUT_PR_IMAGE" 26 extends: .bb_checkout
27
39 stage: checkout 28 stage: checkout
29 variables:
30 MANIFEST: master
31 CURRENT_PROJECT: meta-updater
40 except: 32 except:
41 - pushes 33 - pushes
42 cache: 34
43 paths: 35Build core-image-minimal:
44 - updater-repo 36 extends: .bitbake
37
38 stage: test
39 variables:
40 TEST_BUILD_DIR: 'build-core-image-minimal'
41 BITBAKE_TARGETS: 'core-image-minimal'
45 artifacts: 42 artifacts:
46 expire_in: "1 day" 43 name: "core-image-minimal_$CI_COMMIT_REF_SLUG"
47 paths: 44 paths:
48 - updater-repo 45 - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal*
49 script: 46 except:
50 - MANIFEST=$PR_BASE_BRANCH ./scripts/ci/checkout-oe.sh 47 - pushes
48
49Oe-selftest qemux86_64:
50 extends: .oe-selftest
51
52 stage: test
53 variables:
54 TEST_BUILD_DIR: 'build-oe-qemux86_64'
55 OE_SELFTESTS: 'updater_native updater_qemux86_64'
56 except:
57 - pushes
58
59# Not run by default, triggered on nightlies
60
61Oe-selftest minnowboard:
62 extends: .oe-selftest
63
64 stage: test
65 variables:
66 TEST_BUILD_DIR: 'build-oe-minnowboard'
67 OE_SELFTESTS: 'updater_minnowboard'
68 except:
69 - pushes
70 only:
71 variables:
72 - $OE_MINNOWBOARD
73
74Oe-selftest rpi:
75 extends: .oe-selftest
76
77 stage: test
78 variables:
79 TEST_BUILD_DIR: 'build-oe-rpi'
80 OE_SELFTESTS: 'updater_rpi'
81 except:
82 - pushes
83 only:
84 variables:
85 - $OE_RPI
86
87Ptest qemux86_64:
88 extends: .oe-selftest
51 89
52Build core-image-minimal:
53 image: "$BITBAKE_PR_IMAGE"
54 stage: test 90 stage: test
91 variables:
92 TEST_BUILD_DIR: 'build-oe-qemux86_64-ptest'
93 OE_SELFTESTS: 'updater_qemux86_64_ptest'
55 except: 94 except:
56 - pushes 95 - pushes
57 dependencies: 96 only:
58 - Checkout 97 variables:
59 tags: 98 - $OE_PTEST
60 - bitbake
61 script:
62 - ./scripts/ci/configure.sh
63 - ./scripts/ci/build.sh core-image-minimal
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout
index efec234..5210c6b 100644
--- a/scripts/ci/Dockerfile.checkout
+++ b/scripts/ci/Dockerfile.checkout
@@ -7,3 +7,7 @@ RUN apt-get update -q && apt-get install -qy \
7 git \ 7 git \
8 repo \ 8 repo \
9 xmlstarlet 9 xmlstarlet
10
11# checkout script
12RUN mkdir /scripts
13COPY checkout-oe.sh /scripts/
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh
index 8744b2f..a99f235 100755
--- a/scripts/ci/checkout-oe.sh
+++ b/scripts/ci/checkout-oe.sh
@@ -6,7 +6,11 @@ set -x
6 6
7REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic} 7REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic}
8MANIFEST=${MANIFEST:-master} 8MANIFEST=${MANIFEST:-master}
9CURRENT_PROJECT=${CURRENT_PROJECT:-meta-updater} 9CURRENT_PROJECT=${CURRENT_PROJECT:-}
10
11# list of projects to pin to one version in the format:
12# "project:rev;project2:rev2..."
13PIN_LIST=${PIN_LIST:-}
10 14
11#CURRENT_REV=$(git rev-parse HEAD) 15#CURRENT_REV=$(git rev-parse HEAD)
12LOCAL_REPO=$PWD 16LOCAL_REPO=$PWD
@@ -22,26 +26,48 @@ git -C .repo/manifests reset --hard
22# patch manifest: 26# patch manifest:
23# - add a new "ats" remote that points to "$REMOTE_SOURCE" 27# - add a new "ats" remote that points to "$REMOTE_SOURCE"
24# - change projects that contain "advancedtelematic" to use the ats remote 28# - change projects that contain "advancedtelematic" to use the ats remote
25# - remove the current project from the manifest
26MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml" 29MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml"
27xmlstarlet ed --omit-decl -L \ 30xmlstarlet ed --omit-decl -L \
28 -s "/manifest" -t elem -n "remote" -v "" \ 31 -s "/manifest" -t elem -n "remote" -v "" \
29 -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \ 32 -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \
30 -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \ 33 -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \
34 -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \
31 -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \ 35 -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \
32 -d "/manifest/project[@path=\"$CURRENT_PROJECT\"]" \
33 "$MANIFEST_FILE" 36 "$MANIFEST_FILE"
34 37
35# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix 38# hack: sed on `advancedtelematic/` names, to remove this unwanted prefix
36sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE" 39sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE"
37 40
41# pin projects from the list
42(
43IFS=";"
44for pin in $PIN_LIST; do
45 IFS=":"
46 read -r project rev <<< "$pin"
47 xmlstarlet ed --omit-decl -L \
48 -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \
49 -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \
50 "$MANIFEST_FILE"
51 IFS=";"
52done
53)
54
55# Remove the current project from the manifest if we have it checked out
56if [ -n "$CURRENT_PROJECT" ]; then
57 xmlstarlet ed --omit-decl -L \
58 -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \
59 "$MANIFEST_FILE"
60fi
61
38repo manifest 62repo manifest
39 63
40repo forall -c 'git reset --hard ; git clean -fdx' 64repo forall -c 'git reset --hard ; git clean -fdx'
41 65
42repo sync -d --force-sync 66repo sync -d --force-sync
43 67
44rm -f "$CURRENT_PROJECT" 68if [ -n "$CURRENT_PROJECT" ]; then
45ln -s "$LOCAL_REPO" "$CURRENT_PROJECT" 69 rm -f "$CURRENT_PROJECT"
70 ln -s "$LOCAL_REPO" "$CURRENT_PROJECT"
71fi
46 72
47repo manifest -r 73repo manifest -r
diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh
new file mode 100755
index 0000000..1ffbbf5
--- /dev/null
+++ b/scripts/ci/docker/setup_kvm.sh
@@ -0,0 +1,14 @@
1#!/usr/bin/env bash
2
3# This script makes the gid of the 'kvm' group to match the group
4# owner of '/dev/kvm'
5#
6# These two are not guaranteed to match when a docker image starts
7# with access to '/dev/kvm' that comes from the host
8
9set -euo pipefail
10
11kvm_gid=$(stat -c "%g" /dev/kvm)
12groupmod -g "$kvm_gid" kvm
13usermod -a -G kvm bitbake
14ln -s /bin/true /usr/bin/kvm-ok
diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml
new file mode 100644
index 0000000..28c9177
--- /dev/null
+++ b/scripts/ci/gitlab/checkout.yml
@@ -0,0 +1,16 @@
1.bb_checkout:
2 # parameters:
3 # - MANIFEST
4 # - REMOTE_SOURCE
5 # - CURRENT_PROJECT (will be symlinked instead of pulled)
6
7 image: $BITBAKE_CHECKOUT_IMAGE
8 cache:
9 paths:
10 - updater-repo
11 artifacts:
12 expire_in: "1 day"
13 paths:
14 - updater-repo
15 script:
16 - /scripts/checkout-oe.sh
diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml
new file mode 100644
index 0000000..827bb47
--- /dev/null
+++ b/scripts/ci/gitlab/docker.yml
@@ -0,0 +1,37 @@
1.bb_docker_local:
2 # intended to be run on meta-updater's master branch which contains the
3 # reference docker files
4 # parameters:
5 # - BITBAKE_IMAGE
6 # - BITBAKE_CHECKOUT_IMAGE
7 # - BITBKAE_IMAGE_MASTER
8 # - BITBAKE_CHECKOUT_IMAGE_MASTER
9 image: docker:stable
10 stage: docker
11 services:
12 - docker:dind
13 before_script:
14 - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
15 script:
16 - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true
17 - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci
18 - docker push "$BITBAKE_IMAGE"
19
20 - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true
21 - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci
22 - docker push "$BITBAKE_CHECKOUT_IMAGE"
23
24.bb_docker_remote:
25 # intended to be run on other branches and repos: just pulls the last master image
26 # parameters:
27 # - BITBAKE_IMAGE
28 # - BITBAKE_CHECKOUT_IMAGE
29 image: docker:stable
30 stage: docker
31 services:
32 - docker:dind
33 before_script:
34 - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
35 script:
36 - docker pull "$BITBAKE_IMAGE"
37 - docker pull "$BITBAKE_CHECKOUT_IMAGE"
diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml
new file mode 100644
index 0000000..4bd5844
--- /dev/null
+++ b/scripts/ci/gitlab/tests.yml
@@ -0,0 +1,36 @@
1.bitbake:
2 # parameters:
3 # - TEST_BUILD_DIR
4 # - TEST_MACHINE (defaults to qemux86-64)
5 # - BITBAKE_TARGETS
6 image: $BITBAKE_IMAGE
7 dependencies:
8 - Checkout
9 tags:
10 - bitbake
11 script:
12 - /scripts/configure.sh
13 - /scripts/build.sh $BITBAKE_TARGETS
14
15.oe-selftest:
16 # parameters:
17 # - TEST_BUILD_DIR
18 # - TEST_MACHINE (defaults to qemux86-64)
19 # - OE_SELFTESTS
20 image: $BITBAKE_IMAGE
21 dependencies:
22 - Checkout
23 tags:
24 - bitbake
25 variables:
26 TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip
27 before_script:
28 - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip
29 - sudo /usr/local/bin/setup_kvm.sh
30 script:
31 - |
32 # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh)
33 sg kvm << EOS
34 /scripts/configure.sh
35 /scripts/oe-selftest.sh $OE_SELFTESTS
36 EOS