Skip to content

Commit

Permalink
GitHub actions (OHDSI#1690)
Browse files Browse the repository at this point in the history
* Puts Maven Central as the first repository

This speeds up the build A LOT when none of the dependencies have been
cached yet (e.g. when performing a docker build). Reason: without putting this repository first, all
dependencies are first searched in all of the specified repositories and
only afterwards the central repository is tried. Because most
dependencies are not present in the specified repositories, still
central needs to be queried afterwards. When putting the central
repository first, this reduces time to retrieve dependencies in two
ways:

1. The central repository has a very low latency, much lower than the
other repositories. If a dependency can be fetched from here, it will be
faster than when it is fetched from another one.
2. For most dependencies, it reduces the number of repositories queried
from 7 to 1.

* Enabled github actions

* Use dedicated docker profile to avoid long command line

* Backport github action fixes

* Fix typo

* Remove unsupported platform

* Only use Maven central in the webapi-docker profile
  • Loading branch information
blootsvoets authored Feb 2, 2021
1 parent bc33776 commit d920b6a
Show file tree
Hide file tree
Showing 6 changed files with 311 additions and 10 deletions.
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/target
/.idea
/src/test
.git
5 changes: 5 additions & 0 deletions .github/ISSUE_TEMPLATE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
### Expected behavior

### Actual behavior

### Steps to reproduce behavior
130 changes: 130 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
# Continuous integration
name: CI

# Run in master and dev branches and in all pull requests to those branches
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]

env:
DOCKER_IMAGE: ohdsi/webapi

jobs:
# Build and test the code
build:
# The type of runner that the job will run on
runs-on: ubuntu-latest

env:
MAVEN_PROFILE: webapi-postgresql

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2

- uses: actions/setup-java@v1
with:
java-version: 8

- name: Maven cache
uses: actions/[email protected]
with:
# Cache gradle directories
path: ~/.m2
# Key for restoring and saving the cache
key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }}
restoreKey: ${{ runner.os }}-maven-

- name: Build code
run: mvn -B -DskipTests -P${{ env.MAVEN_PROFILE }} package

- name: Test
run: mvn -B -DskipUnitTests=false -P${{ env.MAVEN_PROFILE }} test

# Check that the docker image builds correctly
# Push to ohdsi/atlas:master for commits on master.
docker:
# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2

- name: Cache Docker layers
uses: actions/cache@v2
with:
path: /tmp/.buildx-cache
key: ${{ runner.os }}-buildx-${{ github.sha }}
restore-keys: |
${{ runner.os }}-buildx-
# Add Docker labels and tags
- name: Docker meta
id: docker_meta
uses: crazy-max/ghaction-docker-meta@v1
with:
images: ${{ env.DOCKER_IMAGE }}

# Setup docker build environment
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Set build parameters
id: build_params
run: |
echo "::set-output name=sha8::${GITHUB_SHA::8}"
if [ ${{ github.event_name == 'pull_request' }} ]; then
echo "::set-output name=push::false"
echo "::set-output name=load::true"
echo "::set-output name=platforms::linux/amd64"
else
echo "::set-output name=push::true"
echo "::set-output name=load::false"
echo "::set-output name=platforms::linux/amd64,linux/arm64"
fi
- name: Login to DockerHub
uses: docker/login-action@v1
if: steps.build_params.outputs.push == 'true'
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
with:
context: ./
file: ./Dockerfile
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
platforms: ${{ steps.build_params.outputs.platforms }}
push: ${{ steps.build_params.outputs.push }}
load: ${{ steps.build_params.outputs.load }}
build-args: |
GIT_BRANCH=${{ steps.docker_meta.outputs.version }}
GIT_COMMIT_ID_ABBREV=${{ steps.build_params.outputs.sha8 }}
tags: ${{ steps.docker_meta.outputs.tags }}
# Use runtime labels from docker_meta as well as fixed labels
labels: |
${{ steps.docker_meta.outputs.labels }}
maintainer=Joris Borgdorff <[email protected]>, Lee Evans - www.ltscomputingllc.com
org.opencontainers.image.authors=Joris Borgdorff <[email protected]>, Lee Evans - www.ltscomputingllc.com
org.opencontainers.image.vendor=OHDSI
org.opencontainers.image.licenses=Apache-2.0
# If the image was pushed, we need to pull it again to inspect it
- name: Pull image
if: steps.build_params.outputs.push == 'true'
run: docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}

- name: Inspect image
run: |
docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
107 changes: 107 additions & 0 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# Create release files
name: Release

on:
release:
types: [published]

env:
DOCKER_IMAGE: ohdsi/webapi

jobs:
upload:
env:
MAVEN_PROFILE: webapi-postgresql

# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2

- uses: actions/setup-java@v1
with:
java-version: 8

- name: Maven cache
uses: actions/[email protected]
with:
# Cache gradle directories
path: ~/.m2
# Key for restoring and saving the cache
key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml') }}
restoreKey: ${{ runner.os }}-maven-

- name: Build code
run: mvn -B -DskipTests -P${{ env.MAVEN_PROFILE }} package

# Upload it to GitHub
- name: Upload to GitHub
uses: AButler/[email protected]
with:
files: 'target/WebAPI.war'
repo-token: ${{ secrets.GITHUB_TOKEN }}

# Build and push tagged release docker image to
# ohdsi/atlas:<version> and ohdsi/atlas:latest.
docker:
# The type of runner that the job will run on
runs-on: ubuntu-latest

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
- uses: actions/checkout@v2

# Add Docker labels and tags
- name: Docker meta
id: docker_meta
uses: crazy-max/ghaction-docker-meta@v1
with:
images: ${{ env.DOCKER_IMAGE }}
tag-match: v(.*)
tag-match-group: 1

# Setup docker build environment
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

- name: Set build parameters
id: build_params
run: |
echo "::set-output name=sha8::${GITHUB_SHA::8}"
- name: Build and push
id: docker_build
uses: docker/build-push-action@v2
with:
context: ./
file: ./Dockerfile
# Allow running the image on the architectures supported by nginx-unprivileged:alpine.
platforms: linux/amd64,linux/arm64
push: true
build-args: |
GIT_BRANCH=${{ steps.docker_meta.outputs.version }}
GIT_COMMIT_ID_ABBREV=${{ steps.build_params.outputs.sha8 }}
tags: ${{ steps.docker_meta.outputs.tags }}
# Use runtime labels from docker_meta as well as fixed labels
labels: |
${{ steps.docker_meta.outputs.labels }}
maintainer=Joris Borgdorff <[email protected]>, Lee Evans - www.ltscomputingllc.com
org.opencontainers.image.authors=Joris Borgdorff <[email protected]>, Lee Evans - www.ltscomputingllc.com
org.opencontainers.image.vendor=OHDSI
org.opencontainers.image.licenses=Apache-2.0
- name: Inspect image
run: |
docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
21 changes: 15 additions & 6 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,19 +1,28 @@
FROM maven:3.6-jdk-8 as builder
FROM maven:3.6-jdk-11 as builder

WORKDIR /code

ARG MAVEN_PROFILE=webapi-postgresql
ARG MAVEN_PROFILE=webapi-docker

# Download dependencies
COPY pom.xml /code/
COPY .git /code/.git
COPY src /code/src
RUN mkdir .git \
&& mvn package \
-P${MAVEN_PROFILE}

ARG GIT_BRANCH=unknown
ARG GIT_COMMIT_ID_ABBREV=unknown

# Compile code and repackage it
RUN mvn package -P${MAVEN_PROFILE} -DskipTests \
COPY src /code/src
RUN mvn package \
-Dgit.branch=${GIT_BRANCH} \
-Dgit.commit.id.abbrev=${GIT_COMMIT_ID_ABBREV} \
-P${MAVEN_PROFILE} \
&& mkdir war \
&& mv target/WebAPI.war war \
&& cd war \
&& unzip WebAPI.war \
&& jar -xf WebAPI.war \
&& rm WebAPI.war

# OHDSI WebAPI and ATLAS web application running as a Spring Boot application with Java 11
Expand Down
57 changes: 53 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
<start-class>org.ohdsi.webapi.WebApi</start-class>
<skipUnitTests>true</skipUnitTests>
<skipITtests>true</skipITtests>
<miredot.phase>package</miredot.phase>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>

<!-- Primary DataSource -->
<!-- default sql server - TODO How about making postgres the default (e.g. in line with open source tech)? -alexfranken -->
Expand Down Expand Up @@ -379,8 +382,6 @@
<version>3.1</version>
<inherited>true</inherited>
<configuration>
<source>1.8</source>
<target>1.8</target>
<compilerArgs>
<!-- Prevents recompilation due to missing package-info.class, see MCOMPILER-205 -->
<arg>-Xpkginfo:always</arg>
Expand All @@ -390,7 +391,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-war-plugin</artifactId>
<version>2.6</version>
<version>3.3.1</version>
<configuration>
<failOnMissingWebXml>false</failOnMissingWebXml>
<attachClasses>true</attachClasses>
Expand Down Expand Up @@ -423,7 +424,7 @@
<version>2.2</version>
<executions>
<execution>
<phase>package</phase>
<phase>${miredot.phase}</phase>
<goals>
<goal>restdoc</goal>
</goals>
Expand Down Expand Up @@ -474,6 +475,10 @@
</repositories>

<pluginRepositories>
<pluginRepository>
<id>central</id>
<url>https://repo.maven.apache.org/maven2</url>
</pluginRepository>
<pluginRepository>
<id>miredot</id>
<name>MireDot Releases</name>
Expand Down Expand Up @@ -1148,6 +1153,50 @@
lower(email) = lower(?)</security.db.datasource.authenticationQuery>
</properties>
</profile>
<profile>
<id>webapi-docker</id>
<properties>
<git.branch>unknown</git.branch>
<git.commit.id.abbrev>unknown</git.commit.id.abbrev>
<maven.gitcommitid.skip>true</maven.gitcommitid.skip>
<miredot.phase>none</miredot.phase>
<skipTests>true</skipTests>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<datasource.driverClassName>org.postgresql.Driver</datasource.driverClassName>
<datasource.url>jdbc:postgresql://54.209.111.128:5432/vocabularyv5</datasource.url>
<datasource.username>USER</datasource.username>
<datasource.password>PASS</datasource.password>
<datasource.dialect>postgresql</datasource.dialect>
<datasource.ohdsi.schema>ohdsi</datasource.ohdsi.schema>
<flyway.datasource.driverClassName>${datasource.driverClassName}</flyway.datasource.driverClassName>
<flyway.datasource.url>${datasource.url}</flyway.datasource.url>
<flyway.datasource.username>userWithWritesToOhdsiSchema</flyway.datasource.username>
<flyway.datasource.password>PASS</flyway.datasource.password>
<flyway.schemas>${datasource.ohdsi.schema}</flyway.schemas>
<flyway.placeholders.ohdsiSchema>${datasource.ohdsi.schema}</flyway.placeholders.ohdsiSchema>
<flyway.locations>classpath:db/migration/postgresql</flyway.locations>
<spring.batch.repository.tableprefix>${datasource.ohdsi.schema}.BATCH_</spring.batch.repository.tableprefix>
<hibernate.dialect>org.hibernate.dialect.PostgreSQL9Dialect</hibernate.dialect>
<security.db.datasource.url>${datasource.url}</security.db.datasource.url>
<security.db.datasource.driverClassName>${datasource.driverClassName}</security.db.datasource.driverClassName>
<security.db.datasource.username>${datasource.username}</security.db.datasource.username>
<security.db.datasource.password>${datasource.password}</security.db.datasource.password>
<security.db.datasource.authenticationQuery>select password from ${security.db.datasource.schema}.users_data where \
lower(email) = lower(?)</security.db.datasource.authenticationQuery>
</properties>
<repositories>
<repository>
<id>central</id>
<url>https://repo.maven.apache.org/maven2</url>
</repository>
<repository>
<id>ohdsi</id>
<name>repo.ohdsi.org</name>
<url>http://repo.ohdsi.org:8085/nexus/content/groups/public</url>
</repository>
</repositories>
</profile>
<profile>
<id>webapi-mssql</id>
<properties>
Expand Down

0 comments on commit d920b6a

Please sign in to comment.