diff options
author | Patrick Vacek <patrickvacek@gmail.com> | 2020-04-06 11:49:00 +0200 |
---|---|---|
committer | Patrick Vacek <patrickvacek@gmail.com> | 2020-04-06 13:56:50 +0200 |
commit | 62fbb2b1ba053635f154e5a041eb4c797f501514 (patch) | |
tree | b677da064978dd31f0e4563b26b038ccdce76ae0 | |
parent | 177081df3a793ee0e028b33bd416cdff97eee471 (diff) | |
download | meta-updater-62fbb2b1ba053635f154e5a041eb4c797f501514.tar.gz |
Refactor CI scripts for dunfell support.
Signed-off-by: Patrick Vacek <patrickvacek@gmail.com>
-rw-r--r-- | .gitlab-ci.yml | 54 | ||||
-rw-r--r-- | scripts/ci/Dockerfile.checkout | 13 | ||||
-rwxr-xr-x | scripts/ci/checkout-oe.sh | 82 | ||||
-rwxr-xr-x | scripts/ci/docker/setup_kvm.sh | 14 | ||||
-rw-r--r-- | scripts/ci/gitlab/checkout.yml | 16 | ||||
-rw-r--r-- | scripts/ci/gitlab/docker.yml | 37 | ||||
-rw-r--r-- | scripts/ci/gitlab/tests.yml | 35 |
7 files changed, 28 insertions, 223 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2fade39..5f7f36f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml | |||
@@ -2,23 +2,27 @@ stages: | |||
2 | - docker | 2 | - docker |
3 | - checkout | 3 | - checkout |
4 | - test | 4 | - test |
5 | - trigger | ||
5 | 6 | ||
6 | variables: | 7 | variables: |
7 | BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-bitbake | 8 | BITBAKE_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-bitbake |
8 | BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-${CI_COMMIT_REF_SLUG}-checkout | 9 | BITBAKE_CHECKOUT_IMAGE: ${CI_REGISTRY_IMAGE}:ci-master-checkout |
9 | 10 | ||
10 | include: | 11 | include: |
11 | - local: "scripts/ci/gitlab/docker.yml" | 12 | - project: 'olp/edge/ota/connect/client/meta-updater' |
12 | - local: "scripts/ci/gitlab/checkout.yml" | 13 | ref: 'master' |
13 | - local: "scripts/ci/gitlab/tests.yml" | 14 | file: 'scripts/ci/gitlab/docker.yml' |
15 | - project: 'olp/edge/ota/connect/client/meta-updater' | ||
16 | ref: 'master' | ||
17 | file: 'scripts/ci/gitlab/checkout.yml' | ||
18 | - project: 'olp/edge/ota/connect/client/meta-updater' | ||
19 | ref: 'master' | ||
20 | file: 'scripts/ci/gitlab/tests.yml' | ||
14 | 21 | ||
15 | Docker setup: | 22 | Docker setup: |
16 | extends: .bb_docker_local | 23 | extends: .bb_docker_remote |
17 | 24 | ||
18 | stage: docker | 25 | stage: docker |
19 | variables: | ||
20 | BITBKAE_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-bitbake | ||
21 | BITBAKE_CHECKOUT_IMAGE_MASTER: ${CI_REGISTRY_IMAGE}:ci-master-checkout | ||
22 | except: | 26 | except: |
23 | - pushes | 27 | - pushes |
24 | 28 | ||
@@ -27,7 +31,7 @@ Checkout: | |||
27 | 31 | ||
28 | stage: checkout | 32 | stage: checkout |
29 | variables: | 33 | variables: |
30 | MANIFEST: master | 34 | MANIFEST: dunfell |
31 | CURRENT_PROJECT: meta-updater | 35 | CURRENT_PROJECT: meta-updater |
32 | except: | 36 | except: |
33 | - pushes | 37 | - pushes |
@@ -39,10 +43,6 @@ Build core-image-minimal: | |||
39 | variables: | 43 | variables: |
40 | TEST_BUILD_DIR: 'build-core-image-minimal' | 44 | TEST_BUILD_DIR: 'build-core-image-minimal' |
41 | BITBAKE_TARGETS: 'core-image-minimal' | 45 | BITBAKE_TARGETS: 'core-image-minimal' |
42 | artifacts: | ||
43 | name: "core-image-minimal_$CI_COMMIT_REF_SLUG" | ||
44 | paths: | ||
45 | - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal* | ||
46 | except: | 46 | except: |
47 | - pushes | 47 | - pushes |
48 | 48 | ||
@@ -54,10 +54,6 @@ Build core-image-minimal (rpi): | |||
54 | TEST_BUILD_DIR: 'build-core-image-minimal-rpi' | 54 | TEST_BUILD_DIR: 'build-core-image-minimal-rpi' |
55 | BITBAKE_TARGETS: 'core-image-minimal' | 55 | BITBAKE_TARGETS: 'core-image-minimal' |
56 | TEST_MACHINE: 'raspberrypi3' | 56 | TEST_MACHINE: 'raspberrypi3' |
57 | artifacts: | ||
58 | name: "core-image-minimal-rpi_$CI_COMMIT_REF_SLUG" | ||
59 | paths: | ||
60 | - $TEST_BUILD_DIR/tmp/deploy/images/*/core-image-minimal* | ||
61 | except: | 57 | except: |
62 | - pushes | 58 | - pushes |
63 | 59 | ||
@@ -71,8 +67,6 @@ Oe-selftest qemux86_64: | |||
71 | except: | 67 | except: |
72 | - pushes | 68 | - pushes |
73 | 69 | ||
74 | # Not run by default, triggered on nightlies | ||
75 | |||
76 | Oe-selftest minnowboard: | 70 | Oe-selftest minnowboard: |
77 | extends: .oe-selftest | 71 | extends: .oe-selftest |
78 | 72 | ||
@@ -82,9 +76,6 @@ Oe-selftest minnowboard: | |||
82 | OE_SELFTESTS: 'updater_minnowboard' | 76 | OE_SELFTESTS: 'updater_minnowboard' |
83 | except: | 77 | except: |
84 | - pushes | 78 | - pushes |
85 | only: | ||
86 | variables: | ||
87 | - $OE_MINNOWBOARD | ||
88 | 79 | ||
89 | Oe-selftest rpi: | 80 | Oe-selftest rpi: |
90 | extends: .oe-selftest | 81 | extends: .oe-selftest |
@@ -95,9 +86,8 @@ Oe-selftest rpi: | |||
95 | OE_SELFTESTS: 'updater_raspberrypi' | 86 | OE_SELFTESTS: 'updater_raspberrypi' |
96 | except: | 87 | except: |
97 | - pushes | 88 | - pushes |
98 | only: | 89 | |
99 | variables: | 90 | # Not run by default |
100 | - $OE_RPI | ||
101 | 91 | ||
102 | Ptest qemux86_64: | 92 | Ptest qemux86_64: |
103 | extends: .oe-selftest | 93 | extends: .oe-selftest |
@@ -111,3 +101,15 @@ Ptest qemux86_64: | |||
111 | only: | 101 | only: |
112 | variables: | 102 | variables: |
113 | - $OE_PTEST | 103 | - $OE_PTEST |
104 | |||
105 | # -- otf | ||
106 | |||
107 | trigger-otf-on-pr: | ||
108 | stage: trigger | ||
109 | when: always | ||
110 | except: | ||
111 | - pushes | ||
112 | - schedules | ||
113 | script: | ||
114 | - apk add --no-cache curl | ||
115 | - curl -X POST -F "token=$CI_JOB_TOKEN" -F "ref=master" -F "variables[BITBAKE_JOB_ONLY]=true" -F "variables[BITBAKE_ENV]=dunfell" -F "variables[PROJECT_NAME]=meta-updater" -F "variables[PROJECT_SHA]=$CI_COMMIT_SHA" https://main.gitlab.in.here.com/api/v4/projects/163/trigger/pipeline | ||
diff --git a/scripts/ci/Dockerfile.checkout b/scripts/ci/Dockerfile.checkout deleted file mode 100644 index 5210c6b..0000000 --- a/scripts/ci/Dockerfile.checkout +++ /dev/null | |||
@@ -1,13 +0,0 @@ | |||
1 | FROM debian:stretch-slim | ||
2 | LABEL Description="Image for checking out updater-repo" | ||
3 | |||
4 | RUN sed -i 's#deb http://deb.debian.org/debian stretch main#deb http://deb.debian.org/debian stretch main contrib#g' /etc/apt/sources.list | ||
5 | RUN sed -i 's#deb http://deb.debian.org/debian stretch-updates main#deb http://deb.debian.org/debian stretch-updates main contrib#g' /etc/apt/sources.list | ||
6 | RUN apt-get update -q && apt-get install -qy \ | ||
7 | git \ | ||
8 | repo \ | ||
9 | xmlstarlet | ||
10 | |||
11 | # checkout script | ||
12 | RUN mkdir /scripts | ||
13 | COPY checkout-oe.sh /scripts/ | ||
diff --git a/scripts/ci/checkout-oe.sh b/scripts/ci/checkout-oe.sh deleted file mode 100755 index ed74595..0000000 --- a/scripts/ci/checkout-oe.sh +++ /dev/null | |||
@@ -1,82 +0,0 @@ | |||
1 | #!/usr/bin/env bash | ||
2 | |||
3 | set -euo pipefail | ||
4 | |||
5 | set -x | ||
6 | |||
7 | REMOTE_SOURCE=${REMOTE_SOURCE:-https://github.com/advancedtelematic} | ||
8 | MANIFEST=${MANIFEST:-master} | ||
9 | CURRENT_PROJECT=${CURRENT_PROJECT:-} | ||
10 | |||
11 | # list of projects to pin to one version in the format: | ||
12 | # "project:rev;project2:rev2..." | ||
13 | PIN_LIST=${PIN_LIST:-} | ||
14 | |||
15 | #CURRENT_REV=$(git rev-parse HEAD) | ||
16 | LOCAL_REPO=$PWD | ||
17 | |||
18 | mkdir -p updater-repo | ||
19 | |||
20 | cd updater-repo | ||
21 | |||
22 | # it seems like it's always required now, even if repo is only used to fetch | ||
23 | git config --global user.email "meta-updater-ci@example.org" | ||
24 | git config --global user.name "meta-updater-ci" | ||
25 | |||
26 | if [ -d .repo/manifests ]; then | ||
27 | git -C .repo/manifests reset --hard | ||
28 | fi | ||
29 | |||
30 | repo init -m "${MANIFEST}.xml" -u "$REMOTE_SOURCE/updater-repo" | ||
31 | |||
32 | # patch manifest: | ||
33 | # - add a new "ats" remote that points to "$REMOTE_SOURCE" | ||
34 | # - change projects that contain "advancedtelematic" to use the ats remote | ||
35 | MANIFEST_FILE=".repo/manifests/${MANIFEST}.xml" | ||
36 | xmlstarlet ed --omit-decl -L \ | ||
37 | -s "/manifest" -t elem -n "remote" -v "" \ | ||
38 | -i "/manifest/remote[last()]" -t attr -n "name" -v "ats" \ | ||
39 | -i "/manifest/remote[last()]" -t attr -n "fetch" -v "$REMOTE_SOURCE" \ | ||
40 | -d "/manifest/project[contains(@name, 'advancedtelematic')]/@remote" \ | ||
41 | -i "/manifest/project[contains(@name, 'advancedtelematic')]" -t attr -n "remote" -v "ats" \ | ||
42 | "$MANIFEST_FILE" | ||
43 | |||
44 | # hack: sed on `advancedtelematic/` names, to remove this unwanted prefix | ||
45 | sed -i 's#name="advancedtelematic/#name="#g' "$MANIFEST_FILE" | ||
46 | |||
47 | # pin projects from the list | ||
48 | ( | ||
49 | IFS=";" | ||
50 | for pin in $PIN_LIST; do | ||
51 | IFS=":" | ||
52 | read -r project rev <<< "$pin" | ||
53 | xmlstarlet ed --omit-decl -L \ | ||
54 | -d "/manifest/project[@name=\"$project\"]/@revision" \ | ||
55 | -i "/manifest/project[@name=\"$project\"]/@revision" -t attr -n "revision" -v "$rev" \ | ||
56 | -i "/manifest/project[@name=\"$project\"]" -t attr -n "revision" -v "$rev" \ | ||
57 | "$MANIFEST_FILE" | ||
58 | IFS=";" | ||
59 | done | ||
60 | ) | ||
61 | |||
62 | # Remove the current project from the manifest if we have it checked out | ||
63 | if [ -n "$CURRENT_PROJECT" ]; then | ||
64 | xmlstarlet ed --omit-decl -L \ | ||
65 | -d "/manifest/project[@name=\"$CURRENT_PROJECT\"]" \ | ||
66 | "$MANIFEST_FILE" | ||
67 | fi | ||
68 | |||
69 | repo manifest | ||
70 | |||
71 | # Try to clean up the repos in case they are messed up, but this returns an | ||
72 | # error code if the repos don't exist, which the subsequent repo sync will fix. | ||
73 | repo forall -c 'git reset --hard ; git clean -fdx' || true | ||
74 | |||
75 | repo sync -d --force-sync | ||
76 | |||
77 | if [ -n "$CURRENT_PROJECT" ]; then | ||
78 | rm -f "$CURRENT_PROJECT" | ||
79 | ln -s "$LOCAL_REPO" "$CURRENT_PROJECT" | ||
80 | fi | ||
81 | |||
82 | repo manifest -r | ||
diff --git a/scripts/ci/docker/setup_kvm.sh b/scripts/ci/docker/setup_kvm.sh deleted file mode 100755 index 1ffbbf5..0000000 --- a/scripts/ci/docker/setup_kvm.sh +++ /dev/null | |||
@@ -1,14 +0,0 @@ | |||
1 | #!/usr/bin/env bash | ||
2 | |||
3 | # This script makes the gid of the 'kvm' group to match the group | ||
4 | # owner of '/dev/kvm' | ||
5 | # | ||
6 | # These two are not guaranteed to match when a docker image starts | ||
7 | # with access to '/dev/kvm' that comes from the host | ||
8 | |||
9 | set -euo pipefail | ||
10 | |||
11 | kvm_gid=$(stat -c "%g" /dev/kvm) | ||
12 | groupmod -g "$kvm_gid" kvm | ||
13 | usermod -a -G kvm bitbake | ||
14 | ln -s /bin/true /usr/bin/kvm-ok | ||
diff --git a/scripts/ci/gitlab/checkout.yml b/scripts/ci/gitlab/checkout.yml deleted file mode 100644 index 28c9177..0000000 --- a/scripts/ci/gitlab/checkout.yml +++ /dev/null | |||
@@ -1,16 +0,0 @@ | |||
1 | .bb_checkout: | ||
2 | # parameters: | ||
3 | # - MANIFEST | ||
4 | # - REMOTE_SOURCE | ||
5 | # - CURRENT_PROJECT (will be symlinked instead of pulled) | ||
6 | |||
7 | image: $BITBAKE_CHECKOUT_IMAGE | ||
8 | cache: | ||
9 | paths: | ||
10 | - updater-repo | ||
11 | artifacts: | ||
12 | expire_in: "1 day" | ||
13 | paths: | ||
14 | - updater-repo | ||
15 | script: | ||
16 | - /scripts/checkout-oe.sh | ||
diff --git a/scripts/ci/gitlab/docker.yml b/scripts/ci/gitlab/docker.yml deleted file mode 100644 index ba488c6..0000000 --- a/scripts/ci/gitlab/docker.yml +++ /dev/null | |||
@@ -1,37 +0,0 @@ | |||
1 | .bb_docker_local: | ||
2 | # intended to be run on meta-updater's master branch which contains the | ||
3 | # reference docker files | ||
4 | # parameters: | ||
5 | # - BITBAKE_IMAGE | ||
6 | # - BITBAKE_CHECKOUT_IMAGE | ||
7 | # - BITBKAE_IMAGE_MASTER | ||
8 | # - BITBAKE_CHECKOUT_IMAGE_MASTER | ||
9 | image: docker:18 | ||
10 | stage: docker | ||
11 | services: | ||
12 | - docker:18-dind | ||
13 | script: | ||
14 | - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" | ||
15 | |||
16 | - docker pull "$BITBAKE_IMAGE" || docker pull "$BITBKAE_IMAGE_MASTER" || true | ||
17 | - docker build --pull --cache-from "$BITBKAE_IMAGE_MASTER" --cache-from "$BITBAKE_IMAGE" -f ./scripts/ci/Dockerfile.bitbake -t "$BITBAKE_IMAGE" ./scripts/ci | ||
18 | - docker push "$BITBAKE_IMAGE" | ||
19 | |||
20 | - docker pull "$BITBAKE_CHECKOUT_IMAGE" || docker pull "$BITBAKE_CHECKOUT_IMAGE_MASTER" || true | ||
21 | - docker build --pull --cache-from "$BITBAKE_CHECKOUT_IMAGE_MASTER" --cache-from "$BITBAKE_CHECKOUT_IMAGE" -f ./scripts/ci/Dockerfile.checkout -t "$BITBAKE_CHECKOUT_IMAGE" ./scripts/ci | ||
22 | - docker push "$BITBAKE_CHECKOUT_IMAGE" | ||
23 | |||
24 | .bb_docker_remote: | ||
25 | # intended to be run on other branches and repos: just pulls the last master image | ||
26 | # parameters: | ||
27 | # - BITBAKE_IMAGE | ||
28 | # - BITBAKE_CHECKOUT_IMAGE | ||
29 | image: docker:18 | ||
30 | stage: docker | ||
31 | services: | ||
32 | - docker:18-dind | ||
33 | script: | ||
34 | - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" | ||
35 | |||
36 | - docker pull "$BITBAKE_IMAGE" | ||
37 | - docker pull "$BITBAKE_CHECKOUT_IMAGE" | ||
diff --git a/scripts/ci/gitlab/tests.yml b/scripts/ci/gitlab/tests.yml deleted file mode 100644 index 4175412..0000000 --- a/scripts/ci/gitlab/tests.yml +++ /dev/null | |||
@@ -1,35 +0,0 @@ | |||
1 | .bitbake: | ||
2 | # parameters: | ||
3 | # - TEST_BUILD_DIR | ||
4 | # - TEST_MACHINE (defaults to qemux86-64) | ||
5 | # - BITBAKE_TARGETS | ||
6 | image: $BITBAKE_IMAGE | ||
7 | dependencies: | ||
8 | - Checkout | ||
9 | tags: | ||
10 | - bitbake | ||
11 | script: | ||
12 | - /scripts/configure.sh | ||
13 | - /scripts/build.sh $BITBAKE_TARGETS | ||
14 | |||
15 | .oe-selftest: | ||
16 | # parameters: | ||
17 | # - TEST_BUILD_DIR | ||
18 | # - TEST_MACHINE (defaults to qemux86-64) | ||
19 | # - OE_SELFTESTS | ||
20 | image: $BITBAKE_IMAGE | ||
21 | dependencies: | ||
22 | - Checkout | ||
23 | tags: | ||
24 | - bitbake | ||
25 | variables: | ||
26 | TEST_AKTUALIZR_CREDENTIALS: $CI_PROJECT_DIR/credentials.zip | ||
27 | script: | ||
28 | - aws s3 cp s3://ota-gitlab-ci/hereotaconnect_prod.zip credentials.zip | ||
29 | - sudo /usr/local/bin/setup_kvm.sh | ||
30 | - | | ||
31 | # sg is needed after adding bitbake to the kvm group (see setup_kvm.sh) | ||
32 | sg kvm << EOS | ||
33 | /scripts/configure.sh | ||
34 | /scripts/oe-selftest.sh $OE_SELFTESTS | ||
35 | EOS | ||