From 7dc8262c88cd40687fdf52770cf581c2b1c1e796 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Wed, 20 Jun 2018 17:11:32 +0200 Subject: Refactor some of the Jenkins pipeline stuff Some options looked deprecated --- scripts/ci/Jenkinsfile | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/ci/Jenkinsfile b/scripts/ci/Jenkinsfile index 83bd49c..2a9f5cf 100644 --- a/scripts/ci/Jenkinsfile +++ b/scripts/ci/Jenkinsfile @@ -3,7 +3,7 @@ pipeline { environment { TEST_LOCAL_CONF_APPEND = 'scripts/ci/local.conf.append' TEST_AKTUALIZR_DIR = 'aktualizr' - TEST_AKTUALIZR_BRANCH = 'origin/master' + TEST_AKTUALIZR_BRANCH = 'aktualizr/master' } stages { stage('checkout') { @@ -11,14 +11,19 @@ pipeline { label 'bitbake' } steps { - dir('aktualizr') { - checkout([$class: 'GitSCM', - userRemoteConfigs: [[url: 'https://github.com/advancedtelematic/aktualizr']], - branches: [[name: '*/master']], - changelog: true, - poll: true, - ]) - } + checkout([$class: 'GitSCM', + userRemoteConfigs: [ + [url: 'https://github.com/advancedtelematic/aktualizr', name: 'aktualizr'] + ], + branches: [[name: 'refs/heads/master']], + extensions: [ + [$class: 'DisableRemotePoll'], + [$class: 'PruneStaleBranch'], + [$class: 'RelativeTargetDirectory', + relativeTargetDir: 'aktualizr' + ] + ], + ]) checkout([$class: 'RepoScm', manifestRepositoryUrl: 'https://github.com/advancedtelematic/updater-repo', -- cgit v1.2.3-54-g00ecf From 991c2752340bfd8140e6254ca1e9cd3c48bceff7 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Thu, 21 Jun 2018 16:21:52 +0200 Subject: Try to bitbake on CI inside docker --- scripts/ci/Dockerfile.bitbake | 37 +++++++++++++++++++++++++++++++++++++ scripts/ci/Jenkinsfile | 9 +++++---- 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 scripts/ci/Dockerfile.bitbake (limited to 'scripts') diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake new file mode 100644 index 0000000..984f421 --- /dev/null +++ b/scripts/ci/Dockerfile.bitbake @@ -0,0 +1,37 @@ +FROM debian:stable +LABEL Description="Image for bitbaking" + +RUN sed -i 's#deb http://deb.debian.org/debian stable main#deb http://deb.debian.org/debian stable main contrib#g' /etc/apt/sources.list +RUN sed -i 's#deb http://deb.debian.org/debian stable-updates main#deb http://deb.debian.org/debian stable-updates main contrib#g' /etc/apt/sources.list +RUN apt-get update -q && apt-get install -qy \ + build-essential \ + bzip2 \ + chrpath \ + cpio \ + default-jre \ + diffstat \ + gawk \ + gcc-multilib \ + git-core \ + iputils-ping \ + iproute \ + libpython-dev \ + libsdl1.2-dev \ + locales \ + procps \ + python \ + python3 \ + python3-pexpect \ + python3-requests \ + qemu \ + socat \ + texinfo \ + unzip \ + wget \ + xterm \ + xz-utils + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen +ENV LC_ALL="en_US.UTF-8" +ENV LANG="en_US.UTF-8" +ENV LANGUAGE="en_US.UTF-8" diff --git a/scripts/ci/Jenkinsfile b/scripts/ci/Jenkinsfile index 2a9f5cf..fd0bc3b 100644 --- a/scripts/ci/Jenkinsfile +++ b/scripts/ci/Jenkinsfile @@ -4,12 +4,11 @@ pipeline { TEST_LOCAL_CONF_APPEND = 'scripts/ci/local.conf.append' TEST_AKTUALIZR_DIR = 'aktualizr' TEST_AKTUALIZR_BRANCH = 'aktualizr/master' + TEST_BITBAKE_COMMON_DIR = "${env.HOME}/bitbake-common" } stages { stage('checkout') { - agent { - label 'bitbake' - } + agent any steps { checkout([$class: 'GitSCM', userRemoteConfigs: [ @@ -53,7 +52,9 @@ pipeline { } stage('build-core-image-minimal') { agent { - label 'bitbake' + dockerfile { + filename 'scripts/ci/Dockerfile.bitbake' + } } steps { sh 'scripts/ci/configure.sh' -- cgit v1.2.3-54-g00ecf From d25358cf0127f769c6d2b3b73b1a50acae78cee9 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Fri, 22 Jun 2018 14:52:25 +0200 Subject: Use plain bash instead of hacky sed for CI config files --- scripts/ci/Jenkinsfile | 1 - scripts/ci/configure.sh | 20 ++++++++++---------- scripts/ci/local.conf.append | 5 ----- 3 files changed, 10 insertions(+), 16 deletions(-) delete mode 100644 scripts/ci/local.conf.append (limited to 'scripts') diff --git a/scripts/ci/Jenkinsfile b/scripts/ci/Jenkinsfile index fd0bc3b..84eebca 100644 --- a/scripts/ci/Jenkinsfile +++ b/scripts/ci/Jenkinsfile @@ -1,7 +1,6 @@ pipeline { agent none environment { - TEST_LOCAL_CONF_APPEND = 'scripts/ci/local.conf.append' TEST_AKTUALIZR_DIR = 'aktualizr' TEST_AKTUALIZR_BRANCH = 'aktualizr/master' TEST_BITBAKE_COMMON_DIR = "${env.HOME}/bitbake-common" diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh index 36ed059..cf42825 100755 --- a/scripts/ci/configure.sh +++ b/scripts/ci/configure.sh @@ -8,11 +8,10 @@ TEST_BUILD_DIR=${TEST_BUILD_DIR:-build} TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo} TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.} -TEST_LOCAL_CONF_APPEND=${TEST_LOCAL_CONF_APPEND:-} TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master} TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="${TEST_AKTUALIZR_DIR}/.git" git rev-parse "${TEST_AKTUALIZR_BRANCH}")} -# remove existing local.conf, keep +# move existing conf directory to backup, before generating a new one rm -rf "${TEST_BUILD_DIR}/conf.old" || true mv "${TEST_BUILD_DIR}/conf" "${TEST_BUILD_DIR}/conf.old" || true @@ -23,11 +22,12 @@ echo ">> Running envsetup.sh" . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}" ) -if [[ -n $TEST_LOCAL_CONF_APPEND ]]; then - echo ">> Appending to local.conf" - REMOTE_AKTUALIZR_BRANCH=$(sed 's#^[^/]*/##g' <<< "$TEST_AKTUALIZR_BRANCH") - cat "$TEST_LOCAL_CONF_APPEND" | \ - sed "s/\$/$TEST_AKTUALIZR_REV/g" | \ - sed "s/\$/$REMOTE_AKTUALIZR_BRANCH/g" \ - >> "${TEST_BUILD_DIR}/conf/local.conf" -fi +echo ">> Set aktualizr branch in bitbake's config" + +cat << EOF > "${TEST_BUILD_DIR}/conf/site.conf" +SANITY_TESTED_DISTROS = "" +SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV" +SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}" +BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH" +BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}" +EOF diff --git a/scripts/ci/local.conf.append b/scripts/ci/local.conf.append deleted file mode 100644 index 350e466..0000000 --- a/scripts/ci/local.conf.append +++ /dev/null @@ -1,5 +0,0 @@ -SANITY_TESTED_DISTROS = "" -SRCREV_pn-aktualizr = "$" -SRCREV_pn-aktualizr-native = "${SRCREV_pn-aktualizr}" -BRANCH_pn-aktualizr = "$" -BRANCH_pn-aktualizr-native = "${BRANCH_pn-aktualizr}" -- cgit v1.2.3-54-g00ecf From a3c960df21c5f820cd7aa592a449be667f65cf86 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Fri, 22 Jun 2018 15:09:57 +0200 Subject: Add caching to CI bitbaking Also fix various issues with environment variables and file transfers to Docker --- scripts/ci/Jenkinsfile | 9 +++++++-- scripts/ci/configure.sh | 25 +++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/ci/Jenkinsfile b/scripts/ci/Jenkinsfile index 84eebca..803664f 100644 --- a/scripts/ci/Jenkinsfile +++ b/scripts/ci/Jenkinsfile @@ -1,9 +1,10 @@ pipeline { agent none environment { + TEST_AKTUALIZR_REMOTE = 'aktualizr' TEST_AKTUALIZR_DIR = 'aktualizr' - TEST_AKTUALIZR_BRANCH = 'aktualizr/master' - TEST_BITBAKE_COMMON_DIR = "${env.HOME}/bitbake-common" + TEST_AKTUALIZR_BRANCH = 'master' + TEST_BITBAKE_COMMON_DIR = "/opt/jenkins/bitbake-common" } stages { stage('checkout') { @@ -41,6 +42,9 @@ pipeline { showAllChanges: false, ]) + // ignore bitbake build directories in docker + sh 'echo \'build*\' > .dockerignore' + // override meta-updater commit with currently tested branch sh ''' META_UPDATER_COMMIT=$(git rev-parse HEAD) @@ -53,6 +57,7 @@ pipeline { agent { dockerfile { filename 'scripts/ci/Dockerfile.bitbake' + args '-v /opt/jenkins/bitbake-common:/opt/jenkins/bitbake-common' } } steps { diff --git a/scripts/ci/configure.sh b/scripts/ci/configure.sh index cf42825..1e87a7b 100755 --- a/scripts/ci/configure.sh +++ b/scripts/ci/configure.sh @@ -9,7 +9,8 @@ TEST_REPO_DIR=${TEST_REPO_DIR:-updater-repo} TEST_AKTUALIZR_DIR=${TEST_AKTUALIZR_DIR:-.} TEST_AKTUALIZR_BRANCH=${TEST_AKTUALIZR_BRANCH:-master} -TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="${TEST_AKTUALIZR_DIR}/.git" git rev-parse "${TEST_AKTUALIZR_BRANCH}")} +TEST_AKTUALIZR_REV=${TEST_AKTUALIZR_REV:-$(GIT_DIR="$TEST_AKTUALIZR_DIR/.git" git rev-parse "$TEST_AKTUALIZR_REMOTE/$TEST_AKTUALIZR_BRANCH")} +TEST_BITBAKE_COMMON_DIR=${TEST_BITBAKE_COMMON_DIR:-} # move existing conf directory to backup, before generating a new one rm -rf "${TEST_BUILD_DIR}/conf.old" || true @@ -22,12 +23,32 @@ echo ">> Running envsetup.sh" . "${TEST_REPO_DIR}/meta-updater/scripts/envsetup.sh" "${TEST_MACHINE}" "${TEST_BUILD_DIR}" ) -echo ">> Set aktualizr branch in bitbake's config" +set +x +echo ">> Set common bitbake config options" cat << EOF > "${TEST_BUILD_DIR}/conf/site.conf" SANITY_TESTED_DISTROS = "" +SSTATE_MIRRORS ?= "file://.* https://bitbake-cache.atsgarage.com/PATH;downloadfilename=PATH" +IMAGE_FEATURES += "ssh-server-openssh" + +EOF + +echo ">> Set aktualizr branch in bitbake's config" +cat << EOF >> "${TEST_BUILD_DIR}/conf/site.conf" SRCREV_pn-aktualizr = "$TEST_AKTUALIZR_REV" SRCREV_pn-aktualizr-native = "\${SRCREV_pn-aktualizr}" BRANCH_pn-aktualizr = "$TEST_AKTUALIZR_BRANCH" BRANCH_pn-aktualizr-native = "\${BRANCH_pn-aktualizr}" + +EOF + +if [[ -n $TEST_BITBAKE_COMMON_DIR ]]; then + echo ">> Set caching" + SSTATE_DIR="$TEST_BITBAKE_COMMON_DIR/sstate-cache" + DL_DIR="$TEST_BITBAKE_COMMON_DIR/downloads" + mkdir -p "$SSTATE_DIR" "$DL_DIR" + cat << EOF >> "${TEST_BUILD_DIR}/conf/site.conf" +SSTATE_DIR = "$SSTATE_DIR" +DL_DIR = "$DL_DIR" EOF +fi -- cgit v1.2.3-54-g00ecf From 9278ad8c3322b9a3ec713ba86006e901bd824486 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 25 Jun 2018 14:03:59 +0200 Subject: Small doc about setting up Jenkins for meta-updater --- scripts/ci/README.adoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 scripts/ci/README.adoc (limited to 'scripts') diff --git a/scripts/ci/README.adoc b/scripts/ci/README.adoc new file mode 100644 index 0000000..222982b --- /dev/null +++ b/scripts/ci/README.adoc @@ -0,0 +1,14 @@ += Jenkins setup for running meta-updater CI + +As bitbake is quite resource-hungry, there are some special steps that are +needed to run Jenkins CI tasks: + +- docker should be installed and the `jenkins` unix user should belong to + the `docker` group +- `/opt/jenkins` should exist and have `jenkins:jenkins` permissions, it + will be mapped as a volume on the same location in the docker build + container + +Note that for nodes running Jenkins slaves as a docker container, the +`/opt/jenkins` directory must exist on the host system as well, with +permissions matching the user and groupd ids in Jenkins' docker -- cgit v1.2.3-54-g00ecf From e27183cfb12e56a6a81aa6fd5bb870ff2fe57fda Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Mon, 25 Jun 2018 14:11:41 +0200 Subject: Rename Jenkinsfile to Jenkinsfile.bleeding + small doc comment --- scripts/ci/Jenkinsfile | 71 --------------------------------------- scripts/ci/Jenkinsfile.bleeding | 74 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 71 deletions(-) delete mode 100644 scripts/ci/Jenkinsfile create mode 100644 scripts/ci/Jenkinsfile.bleeding (limited to 'scripts') diff --git a/scripts/ci/Jenkinsfile b/scripts/ci/Jenkinsfile deleted file mode 100644 index 803664f..0000000 --- a/scripts/ci/Jenkinsfile +++ /dev/null @@ -1,71 +0,0 @@ -pipeline { - agent none - environment { - TEST_AKTUALIZR_REMOTE = 'aktualizr' - TEST_AKTUALIZR_DIR = 'aktualizr' - TEST_AKTUALIZR_BRANCH = 'master' - TEST_BITBAKE_COMMON_DIR = "/opt/jenkins/bitbake-common" - } - stages { - stage('checkout') { - agent any - steps { - checkout([$class: 'GitSCM', - userRemoteConfigs: [ - [url: 'https://github.com/advancedtelematic/aktualizr', name: 'aktualizr'] - ], - branches: [[name: 'refs/heads/master']], - extensions: [ - [$class: 'DisableRemotePoll'], - [$class: 'PruneStaleBranch'], - [$class: 'RelativeTargetDirectory', - relativeTargetDir: 'aktualizr' - ] - ], - ]) - - checkout([$class: 'RepoScm', - manifestRepositoryUrl: 'https://github.com/advancedtelematic/updater-repo', - manifestBranch: null, - manifestFile: 'master.xml', - manifestGroup: null, - mirrorDir: null, - jobs: 0, - depth: 0, - localManifest: null, - destinationDir: 'updater-repo', - repoUrl: null, - currentBranch: false, - resetFirst: true, - quiet: false, - trace: false, - showAllChanges: false, - ]) - - // ignore bitbake build directories in docker - sh 'echo \'build*\' > .dockerignore' - - // override meta-updater commit with currently tested branch - sh ''' - META_UPDATER_COMMIT=$(git rev-parse HEAD) - cd updater-repo/meta-updater - git checkout $META_UPDATER_COMMIT - ''' - } - } - stage('build-core-image-minimal') { - agent { - dockerfile { - filename 'scripts/ci/Dockerfile.bitbake' - args '-v /opt/jenkins/bitbake-common:/opt/jenkins/bitbake-common' - } - } - steps { - sh 'scripts/ci/configure.sh' - - sh 'scripts/ci/build.sh core-image-minimal' - } - } - } -} -// vim: set ft=groovy tabstop=2 shiftwidth=2 expandtab: diff --git a/scripts/ci/Jenkinsfile.bleeding b/scripts/ci/Jenkinsfile.bleeding new file mode 100644 index 0000000..6d0f1e7 --- /dev/null +++ b/scripts/ci/Jenkinsfile.bleeding @@ -0,0 +1,74 @@ +// This CI setup checks out aktualizr, meta-updater and updater-repo and builds +// master branches whenever a change is pushed to any of these + +pipeline { + agent none + environment { + TEST_AKTUALIZR_REMOTE = 'aktualizr' + TEST_AKTUALIZR_DIR = 'aktualizr' + TEST_AKTUALIZR_BRANCH = 'master' + TEST_BITBAKE_COMMON_DIR = "/opt/jenkins/bitbake-common" + } + stages { + stage('checkout') { + agent any + steps { + checkout([$class: 'GitSCM', + userRemoteConfigs: [ + [url: 'https://github.com/advancedtelematic/aktualizr', name: 'aktualizr'] + ], + branches: [[name: 'refs/heads/master']], + extensions: [ + [$class: 'DisableRemotePoll'], + [$class: 'PruneStaleBranch'], + [$class: 'RelativeTargetDirectory', + relativeTargetDir: 'aktualizr' + ] + ], + ]) + + checkout([$class: 'RepoScm', + manifestRepositoryUrl: 'https://github.com/advancedtelematic/updater-repo', + manifestBranch: null, + manifestFile: 'master.xml', + manifestGroup: null, + mirrorDir: null, + jobs: 0, + depth: 0, + localManifest: null, + destinationDir: 'updater-repo', + repoUrl: null, + currentBranch: false, + resetFirst: true, + quiet: false, + trace: false, + showAllChanges: false, + ]) + + // ignore bitbake build directories in docker + sh 'echo \'build*\' > .dockerignore' + + // override meta-updater commit with currently tested branch + sh ''' + META_UPDATER_COMMIT=$(git rev-parse HEAD) + cd updater-repo/meta-updater + git checkout $META_UPDATER_COMMIT + ''' + } + } + stage('build-core-image-minimal') { + agent { + dockerfile { + filename 'scripts/ci/Dockerfile.bitbake' + args '-v /opt/jenkins/bitbake-common:/opt/jenkins/bitbake-common' + } + } + steps { + sh 'scripts/ci/configure.sh' + + sh 'scripts/ci/build.sh core-image-minimal' + } + } + } +} +// vim: set ft=groovy tabstop=2 shiftwidth=2 expandtab: -- cgit v1.2.3-54-g00ecf From ea7cf6a4a8cc4dcd04dac6fcdd6faf606f56e115 Mon Sep 17 00:00:00 2001 From: Laurent Bonnans Date: Wed, 27 Jun 2018 16:01:22 +0200 Subject: Remove python3-requests from Dockerfile.bitbake --- scripts/ci/Dockerfile.bitbake | 1 - 1 file changed, 1 deletion(-) (limited to 'scripts') diff --git a/scripts/ci/Dockerfile.bitbake b/scripts/ci/Dockerfile.bitbake index 984f421..4dfafec 100644 --- a/scripts/ci/Dockerfile.bitbake +++ b/scripts/ci/Dockerfile.bitbake @@ -22,7 +22,6 @@ RUN apt-get update -q && apt-get install -qy \ python \ python3 \ python3-pexpect \ - python3-requests \ qemu \ socat \ texinfo \ -- cgit v1.2.3-54-g00ecf