Fixes #20275: Dockerify Rust tests

luxpal · Nov 22, 2021 · b2ec51b · b2ec51b
1 parent 4e8d305
commit b2ec51b
Show file tree

Hide file tree

Showing 20 changed files with 198 additions and 170 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1,14 +1,21 @@
 @Library('slack-notification')
 import org.gradiant.jenkins.slack.SlackNotifier
 
+// uid of the jenkins user of the docker runners
+def user_id = "1007"
+
 pipeline {
     agent none
 
     stages {
         stage('Tests') {
             parallel {
                 stage('shell') {
-                    agent { label 'script' }
+                    agent { 
+                        dockerfile { 
+                            filename 'ci/shellcheck.Dockerfile'
+                        }
+                    }
                     steps {
                         sh script: './qa-test --shell', label: 'shell scripts lint'
                     }
@@ -25,7 +32,11 @@ pipeline {
                     }
                 }
                 stage('python') {
-                    agent { label 'script' }
+                    agent { 
+                        dockerfile { 
+                            filename 'ci/pylint.Dockerfile'
+                        }
+                    }
                     steps {
                         sh script: './qa-test --python', label: 'python scripts lint'
                     }
@@ -64,11 +75,9 @@ pipeline {
                     agent { 
                         dockerfile { 
                             filename 'api-doc/Dockerfile'
-                            // To get the jenkins user inside of the container
-                            args '-v /etc/passwd:/etc/passwd:ro'
+                            additionalBuildArgs  '--build-arg USER_ID='+user_id
                         }
                     }
-
                     stages {
                         stage('api-doc-test') {
                             when {
@@ -127,7 +136,6 @@ pipeline {
                             args '-v /etc/passwd:/etc/passwd:ro --tmpfs /srv/jenkins/.local:exec'
                         }
                     }
-
                     steps {
                         dir ('relay/sources') {
                             sh script: 'make check', label: 'rudder-pkg tests'
@@ -146,20 +154,22 @@ pipeline {
                     }
                 }
                 stage('webapp') {
-                    agent { label 'scala' }
-
+                    agent {
+                        dockerfile {
+                            filename 'webapp/sources/Dockerfile'
+                            additionalBuildArgs '--build-arg USER_ID='+user_id
+                            // we don't share elm folder as it is may break with concurrent builds
+                            // set same timezone as some tests rely on it
+                            // and share maven cache
+                            args '-v /etc/timezone:/etc/timezone:ro -v /srv/cache/maven:/home/jenkins/.m2'
+                        }
+                    }
                     stages {
                         stage('webapp-test') {
-                            when { changeRequest() }
                             steps {
                                 sh script: 'webapp/sources/rudder/rudder-core/src/test/resources/hooks.d/test-hooks.sh', label: "hooks tests"
                                 dir('webapp/sources') {
-                                    withMaven(maven: "latest",
-                                              // don't archive jars
-                                              options: [artifactsPublisher(disabled: true)]
-                                    ) {
-                                        sh script: 'mvn clean test -Dmaven.test.postgres=false', label: "webapp tests"
-                                    }
+                                    sh script: 'mvn clean test --batch-mode -Dmaven.test.postgres=false', label: "webapp tests"
                                 }
                             }
                             post {
@@ -178,12 +188,12 @@ pipeline {
                             steps {
                                 sh script: 'webapp/sources/rudder/rudder-core/src/test/resources/hooks.d/test-hooks.sh', label: "hooks tests"
                                 dir('webapp/sources') {
-                                    withMaven(maven: "latest",
-                                              globalMavenSettingsConfig: "1bfa2e1a-afda-4cb4-8568-236c44b94dbf",
+                                    withMaven(globalMavenSettingsConfig: "1bfa2e1a-afda-4cb4-8568-236c44b94dbf",
                                               // don't archive jars
                                               options: [artifactsPublisher(disabled: true)]
                                     ) {
-                                        sh script: 'mvn --update-snapshots clean package deploy', label: "webapp deploy"
+                                        // we need to use $MVN_COMMAND to get the settings file path
+                                        sh script: '$MVN_CMD --update-snapshots clean package deploy', label: "webapp deploy"
                                     }
                                 }
                             }
@@ -201,80 +211,59 @@ pipeline {
                         }
                     }
                 }
-                // No parallelism inside this stage
-                stage('rust') {
-                    agent { label 'rust' }
-
-                    // sccache server is run via systemd on the builder
-                    // because of https://github.com/mozilla/sccache/blob/master/docs/Jenkins.md
-
+                stage('relayd') {
+                    // we need to use a script for side container currently
+                    agent { label 'docker' }
                     environment {
-                        PATH = "${env.HOME}/.cargo/bin:${env.PATH}"
+                        POSTGRES_PASSWORD = 'PASSWORD'
+                        POSTGRES_DB       = 'rudder'
+                        POSTGRES_USER     = 'rudderreports'
                     }
-
-                    stages {
-                        // No built-in support for Rust tooling in Jenkins, let's do it ourselves
-                        stage('rust-tools') {
-                            steps {
-                                // System dependencies: libssl-dev pkg-config
-                                sh script: 'make -f rust.makefile setup', label: "Setup build tools"
-                            }
-
-                            post {
-                                always {
-                                    script {
-                                        new SlackNotifier().notifyResult("rust-team")
-                                    }
-                                }
-                            }
-                        }
-                        stage('relayd') {
-                            environment {
-                                RUSTC_WRAPPER = "sccache"
-                            }
-                            steps {
-                                // System dependencies: libpq-dev postgresql
-                                dir('relay/sources/relayd') {
-                                    // lock the database to avoid race conditions between parallel tests
-                                    lock('test-relayd-postgresql') {
+                    steps {
+                        script {
+                            docker.image('postgres:11-bullseye').withRun('-e POSTGRES_USER=${POSTGRES_USER} -e POSTGRES_PASSWORD=${POSTGRES_PASSWORD} -e POSTGRES_DB=${POSTGRES_DB}', '-c listen_addresses="*"') { c ->
+                                docker.build('relayd', '-f relay/sources/relayd/Dockerfile --build-arg USER_ID='+user_id+' --pull .')
+                                      .inside("-v /srv/cache/cargo:/usr/local/cargo/registry -v /srv/cache/sccache:/home/jenkins/.cache/sccache --link=${c.id}:postgres") {
+                                    dir('relay/sources/relayd') {
+                                        sh script: "PGPASSWORD=${POSTGRES_PASSWORD} psql -U ${POSTGRES_USER} -h postgres -d ${POSTGRES_DB} -a -f tools/create-database.sql", label: 'provision database'
                                         sh script: 'make check', label: 'relayd tests'
                                     }
-                                    sh script: 'make clean', label: 'relayd clean'
-                                }
-                            }
-                            post {
-                                always {
-                                    // linters results
-                                    recordIssues enabledForFailure: true, id: 'relayd', name: 'cargo relayd', sourceDirectory: 'relay/sources/relayd', sourceCodeEncoding: 'UTF-8',
-                                                 tool: cargo(pattern: 'relay/sources/relayd/target/cargo-clippy.json', reportEncoding: 'UTF-8', id: 'relayd', name: 'cargo relayd')
-
-                                    script {
-                                        new SlackNotifier().notifyResult("rust-team")
-                                    }
                                 }
-
-                            }
+                            }    
                         }
-                        stage('language') {
-                            environment {
-                                RUSTC_WRAPPER = "sccache"
-                            }
-                            steps {
-                                dir('rudder-lang') {
-                                    sh script: 'make check', label: 'language tests'
-                                    sh script: 'make clean', label: 'language clean'
-                                }
+                    }
+                    post {
+                        always {
+                            // linters results
+                            recordIssues enabledForFailure: true, id: 'relayd', name: 'cargo relayd', sourceDirectory: 'relay/sources/relayd', sourceCodeEncoding: 'UTF-8',
+                                         tool: cargo(pattern: 'relay/sources/relayd/target/cargo-clippy.json', reportEncoding: 'UTF-8', id: 'relayd', name: 'cargo relayd')
+                            script {
+                                new SlackNotifier().notifyResult("rust-team")
                             }
-                            post {
-                                always {
-                                    // linters results
-                                    recordIssues enabledForFailure: true, id: 'language', name: 'cargo language', sourceDirectory: 'rudder-lang', sourceCodeEncoding: 'UTF-8',
-                                                 tool: cargo(pattern: 'rudder-lang/target/cargo-clippy.json', reportEncoding: 'UTF-8', id: 'language', name: 'cargo language')
-
-                                    script {
-                                        new SlackNotifier().notifyResult("rust-team")
-                                    }
-                                }
+                        }
+                    }
+                }
+                stage('language') {
+                    agent { 
+                        dockerfile { 
+                            filename 'rudder-lang/Dockerfile'
+                            additionalBuildArgs  '--build-arg USER_ID='+user_id
+                            // mount cache
+                            args '-v /srv/cache/cargo:/usr/local/cargo/registry -v /srv/cache/sccache:/home/jenkins/.cache/sccache'
+                        }
+                    }
+                    steps {
+                        dir('rudder-lang') {
+                            sh script: 'make check', label: 'language tests'
+                        }
+                    }
+                    post {
+                        always {
+                            // linters results
+                            recordIssues enabledForFailure: true, id: 'language', name: 'cargo language', sourceDirectory: 'rudder-lang', sourceCodeEncoding: 'UTF-8',
+                                         tool: cargo(pattern: 'rudder-lang/target/cargo-clippy.json', reportEncoding: 'UTF-8', id: 'language', name: 'cargo language')
+                            script {
+                                new SlackNotifier().notifyResult("rust-team")
                             }
                         }
                     }

diff --git a/Jenkinsfile-security b/Jenkinsfile-security
@@ -3,23 +3,28 @@ import org.gradiant.jenkins.slack.SlackNotifier
 
 // Check vulns in dependencies on repo branches
 
+// uid of the jenkins user of the docker runners
+def user_id = "1007"
+
 pipeline {
     agent none
     triggers { cron('@daily') }
 
     stages {
         stage('deps-webapp') {
             when { not { changeRequest() } }
-            agent { label 'scala' }
+            agent {
+                dockerfile {
+                    filename 'webapp/sources/Dockerfile'
+                    additionalBuildArgs  '--build-arg USER_ID='+user_id
+                    // and share maven cache
+                    args '-v /srv/cache/maven:/home/jenkins/.m2'
+                }
+            }
             steps {
                 dir('webapp/sources') {
-                    withMaven(maven: "latest",
-                        // don't archive jars
-                        options: [artifactsPublisher(disabled: true)]
-                    ) {
-                        sh script: 'mvn -DfailBuildOnCVSS=7 -DcveValidForHours=48 -DsuppressionFiles=dependency-check-suppression.xml org.owasp:dependency-check-maven:6.1.6:aggregate', label: "check webapp dependencies"
-                        sh script: 'mvn license:aggregate-third-party-report', label: 'list webapp dependencies'
-                    }
+                    sh script: 'mvn --batch-mode -DfailBuildOnCVSS=7 -DcveValidForHours=48 -DsuppressionFiles=dependency-check-suppression.xml org.owasp:dependency-check-maven:6.1.6:aggregate', label: "check webapp dependencies"
+                    sh script: 'mvn --batch-mode license:aggregate-third-party-report', label: 'list webapp dependencies'
                 }
             }
             post {
@@ -33,10 +38,14 @@ pipeline {
         }
         stage('deps-relayd') {
             when { not { changeRequest() } }
-            agent { label 'rust' }
-            environment {
-                PATH = "${env.HOME}/.cargo/bin:${env.PATH}"
-            }
+            agent { 
+                dockerfile { 
+                    filename 'relay/sources/relayd/Dockerfile'
+                    additionalBuildArgs  '--build-arg USER_ID='+user_id
+                    // mount cache
+                    args '-v /srv/cache/cargo:/usr/local/cargo/registry'
+                }
+            }            
             steps {
                 dir('relay/sources/relayd') {
                     sh script: 'mkdir -p target', label: 'create target directory'
@@ -55,9 +64,13 @@ pipeline {
         }
         stage('deps-language') {
             when { not { changeRequest() } }
-            agent { label 'rust' }
-            environment {
-                PATH = "${env.HOME}/.cargo/bin:${env.PATH}"
+            agent { 
+                dockerfile { 
+                    filename 'rudder-lang/Dockerfile'
+                    additionalBuildArgs  '--build-arg USER_ID='+user_id
+                    // mount cache
+                    args '-v /srv/cache/cargo:/usr/local/cargo/registry'
+                }
             }
             steps {
                 dir('rudder-lang') {

diff --git a/api-doc/Dockerfile b/api-doc/Dockerfile
@@ -1,5 +1,9 @@
 FROM node:16-bullseye
 ENV NODE_ENV=production
 
+ARG USER_ID=1000
+COPY ci/user.sh .
+RUN ./user.sh $USER_ID
+
 RUN npm install --global @redocly/[email protected] [email protected]
 RUN apt-get update && apt-get install -y rsync python3-yaml && rm -rf /var/lib/apt/lists/*
diff --git a/ci/pylint.Dockerfile b/ci/pylint.Dockerfile
@@ -0,0 +1,3 @@
+FROM python
+
+RUN pip install pylint
diff --git a/ci/rust.sh b/ci/rust.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# used to setup Rust build containers
+
+rustup component add clippy
+
+# Install pre-compiled sccache
+SCCACHE_VER=0.2.15
+wget --quiet https://github.com/mozilla/sccache/releases/download/v$SCCACHE_VER/sccache-v$SCCACHE_VER-x86_64-unknown-linux-musl.tar.gz
+tar -xf sccache-v$SCCACHE_VER-x86_64-unknown-linux-musl.tar.gz
+chmod +x sccache-v$SCCACHE_VER-x86_64-unknown-linux-musl/sccache
+mv sccache-v$SCCACHE_VER-x86_64-unknown-linux-musl/sccache /usr/local/bin/
+
+# Install pre-compiled cargo-deny
+DENY_VER=0.10.1
+wget --quiet https://github.com/EmbarkStudios/cargo-deny/releases/download/$DENY_VER/cargo-deny-$DENY_VER-x86_64-unknown-linux-musl.tar.gz
+tar -xf cargo-deny-$DENY_VER-x86_64-unknown-linux-musl.tar.gz
+mv cargo-deny-$DENY_VER-x86_64-unknown-linux-musl/cargo-deny /usr/local/bin/
diff --git a/ci/shellcheck.Dockerfile b/ci/shellcheck.Dockerfile
@@ -0,0 +1,3 @@
+FROM debian:11
+
+RUN apt-get update && apt-get install shellcheck
diff --git a/ci/user.sh b/ci/user.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+# Jenkins user with correct uid
+USER_ID=$1
+mkdir /home/jenkins
+useradd -r -u $USER_ID -d /home/jenkins jenkins
+chown jenkins /home/jenkins
diff --git a/relay/sources/relayd/Dockerfile b/relay/sources/relayd/Dockerfile
@@ -0,0 +1,12 @@
+FROM rust:1.42.0-buster
+
+ARG USER_ID=1000
+COPY ci/user.sh .
+RUN ./user.sh $USER_ID
+
+COPY ci/rust.sh .
+RUN ./rust.sh
+
+ENV RUSTC_WRAPPER="sccache"
+
+RUN apt-get update && apt-get install -y pkg-config libpq-dev postgresql-client
diff --git a/relay/sources/relayd/Makefile b/relay/sources/relayd/Makefile
@@ -6,7 +6,7 @@ include ../../../rust.makefile
 # relayd tools, for general relay build see ../Makefile
 
 yum-dependencies:
-	$(YUM) pkgconfig postgresql-devel openssl-devel
+	$(YUM) pkgconfig postgresql-devel
 
 apt-dependencies:
-	$(APT) pkg-config libpq-dev libssl-dev
+	$(APT) pkg-config libpq-dev
diff --git a/relay/sources/relayd/README.md b/relay/sources/relayd/README.md
@@ -6,10 +6,7 @@
 
 Runtime dependencies are:
 
-* openssl
 * libpq
-* zlib
-* lzma
 
 To install build dependencies on Debian/Ubuntu:
 
@@ -31,8 +28,6 @@ To install:
 make DESTDIR=/target/directory install
 ```
 
-This project requires at least Rust 1.34.
-
 ## Development database
 
 Schema for the database is in: `webapp/sources/rudder/rudder-core/src/main/resources/reportsSchema.sql`
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		FROM debian:11

		RUN apt-get update && apt-get install shellcheck