.gitlab-ci.yml

image: registry.gitlab.isc.org/isc-projects/stork/ci-base:latest

# Only create pipelines for merge requests and pushes/merges to the main branch
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'

variables:
  POSTGRES_ADDR: postgres:5432
  POSTGRES_DB: storktest
  POSTGRES_USER: storktest
  POSTGRES_PASSWORD: storktest
  STORK_DATABASE_MAINTENANCE_NAME: "postgres"
  STORK_DATABASE_MAINTENANCE_USER_NAME: "storktest"
  STORK_DATABASE_MAINTENANCE_PASSWORD: "storktest"

  # this is used to build docker images of stork (using DIND)
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: ""  # this is workaround for TLS problem https://about.gitlab.com/blog/2019/07/31/docker-in-docker-with-docker-19-dot-03/

  # Force Unicode
  LC_ALL: C.UTF-8
  LANG: C.UTF-8

# stages order
stages:
  - checks
  - system_tests
  - build
  - deploy

# cache
cache:
  key: one-shared-key-2
  paths:
  - webui/node_modules/
  - tools/

### build jobs ###

# common parts used in build jobs
.base_build_debian:
  stage: checks
  rules:
    - when: always
  tags:
    - linux
    - amd64
    - ssd
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
    - apt-get update
    - apt-get install -y ruby-dev make python3-venv
    - rake prepare
    - rake prepare:deps

.base_build_ubi:
  stage: checks
  rules:
    - when: always
  cache:
    key: one-shared-key-rpm-2
    paths:
    - webui/node_modules/
    - tools/
  image: registry.gitlab.isc.org/isc-projects/stork/pkgs-redhat-ubi8:latest
  tags:
    - linux
    - amd64
    - ssd
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
    - rake prepare
    - rake prepare:deps

lint_backend:
  extends: .base_build_debian
  script:
    - rake lint:backend

lint_python_pylint:
  extends: .base_build_debian
  script:
    - rake lint:python:pylint

lint_python_flake8:
  extends: .base_build_debian
  script:
    - rake lint:python:flake8

unittest_backend:
  extends: .base_build_debian
  services:
    - name: registry.gitlab.isc.org/isc-projects/stork/ci-postgres:11
      alias: postgres
  script:
    - echo 'postgres:*:*:storktest:storktest' > ~/.pgpass
    - chmod 600 ~/.pgpass
    - rake unittest:backend

lint_ui:
  extends: .base_build_debian
  script:
    - rake lint:ui

unittest_ui:
  extends: .base_build_debian
  script:
    - rake unittest:ui

build_ui:
  extends: .base_build_debian
  script:
    - rake build:ui
  artifacts:
    name: "ui"
    expire_in: 1 week
    paths:
      - webui/dist/stork/

build_backend:
  extends: .base_build_debian
  script:
    - rake build:backend
  artifacts:
    name: "backend"
    expire_in: 1 week
    paths:
      - backend/cmd/stork-agent/stork-agent
      - backend/cmd/stork-server/stork-server
      - backend/cmd/stork-db-migrate/stork-db-migrate

danger:
  extends: .base_build_debian
  stage: checks
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'
  tags:
    - linux
    - amd64
    - ssd
  before_script:
    - export CI_MERGE_REQUEST_ID=$(git ls-remote -q origin merge-requests\*\head | grep $CI_COMMIT_SHA | sed 's/.*refs\/merge-requests\/\([0-9]*\)\/head/\1/g')
    - export CI_PROJECT_PATH=$CI_PROJECT_ID #some version of gitlab has problems with searching by project path
    - export DANGER_GITLAB_HOST=gitlab.isc.org
    - export DANGER_GITLAB_API_BASE_URL=https://gitlab.isc.org/api/v4
  script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
    - rake lint:git

lint_shell:
  extends: .base_build_debian
  script:
    - rake lint:shell

tarball:
  stage: build
  rules:
    - when: always
  tags:
    - linux
    - amd64
    - ssd
  script:
    - rake release:tarball
  artifacts:
    paths:
      - stork-*.tar.gz
    expire_in: 1 week


### build rpm & deb packages ###

build_debs:
  extends: .base_build_debian
  stage: build
  script:
    - rake build:agent_pkg
    - rake build:server_pkg
    - cp dist/pkgs/*.deb ./
  artifacts:
    paths:
      - isc-stork-*.deb
    expire_in: 1 hour

build_rpms:
  extends: .base_build_ubi
  stage: build
  script:
    - rake build:agent_pkg
    - rake build:server_pkg
    - cp dist/pkgs/*.rpm ./
  artifacts:
    paths:
      - isc-stork-*.rpm
    expire_in: 1 hour

build_apks:
  stage: build
  rules:
    - when: always
  cache:
    key: one-shared-key-apk-1
    paths:
    - webui/node_modules/
    - tools/
  image: golang:1.18-alpine3.15
  tags:
    - linux
    - amd64
    - ssd
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
    # The initially installed TAR causes FPM to fail. It must be reinstalled: https://github.com/jordansissel/fpm/issues/1375#issuecomment-317571946
    - apk add --no-cache ruby-rake ruby-dev openjdk11-jre-headless python3 nodejs npm protoc~3.18.1 make gcc musl-dev tar
    - rake prepare:dist
    - rake prepare:deps
  script:
    - rake build:agent_pkg
    - rake build:server_pkg
    - cp dist/pkgs/*.apk ./
  artifacts:
    paths:
      - isc-stork-*.apk
    expire_in: 1 hour

packages:
  stage: build
  rules:
    - when: on_success
  needs:
    - build_debs
    - build_rpms
    - build_apks
  script:
    - ":"
  artifacts:
    paths:
      - isc-stork-*
    expire_in: 1 week

### install packages ###

install_debs:
  tags:
    - linux
    - amd64
    - ssd
  stage: build
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
    - apt-get update
    - apt-get install -y man
    # Ubuntu has dpkg configured to ignore man files by default.
    - rm /etc/dpkg/dpkg.cfg.d/excludes
  script:
    - "dpkg -i isc-stork-*.deb"
    - "stork-agent --version"
    - "stork-server --version"
    - "stork-tool --version"
    - "dpkg -i isc-stork-*.deb"
    - "stork-agent --version"
    - "stork-server --version"
    - "stork-tool --version"
    - "man stork-agent > /dev/null"
    - "man stork-server > /dev/null"
    - "man stork-tool > /dev/null"
    - "dpkg -r isc-stork-agent"
    - "dpkg -r isc-stork-server"
  needs:
    - build_debs

install_rpms:
  image: registry.gitlab.isc.org/isc-projects/stork/pkgs-redhat-ubi8:latest
  tags:
    - linux
    - amd64
    - ssd
  stage: build
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
    - yum install -y man
  script:
    - "rpm -i isc-stork-*.rpm"
    - "stork-agent --version"
    - "stork-server --version"
    - "stork-tool --version"
    - "rpm -U --force isc-stork-*.rpm"
    - "stork-agent --version"
    - "stork-server --version"
    - "stork-tool --version"
    - "man stork-agent > /dev/null"
    - "man stork-server > /dev/null"
    - "man stork-tool > /dev/null"
    - "rpm -e isc-stork-agent"
    - "rpm -e isc-stork-server"
  needs:
    - build_rpms

install_apks:
  image: alpine:3.15
  tags:
    - linux
    - amd64
    - ssd
  stage: build
  script:
    - "apk add --no-cache mandoc"
    - "apk add --no-cache --allow-untrusted isc-stork-*.apk"
    - "stork-agent --version"
    - "stork-server --version"
    - "stork-tool --version"
    - "apk add --no-cache --allow-untrusted isc-stork-*.apk"
    - "stork-agent --version"
    - "stork-server --version"
    - "stork-tool --version"
    - "man stork-agent > /dev/null"
    - "man stork-server > /dev/null"
    - "man stork-tool > /dev/null"
    - "apk del isc-stork-agent"
    - "apk del isc-stork-server"
  needs:
    - build_apks

### system testing ###

system_testing:
  stage: system_tests
  allow_failure: false
  tags:
    - linux
    - docker
    - amd64
    - ssd
  services:
    - docker:dind
  rules:
    - when: on_success
  image: docker/compose:1.29.2
  cache:
    key: system-tests-key-2
    paths:
    - tools/
  before_script:
    - ip -6 route del default
    - apk update
    - apk add --no-cache openjdk11-jre-headless python3 openssl ruby-rake nodejs npm
  variables:
    # Alpine uses the libc-musl library, which isn't compatible with the NodeJS
    # binary fetched by the Rake script. Additionally, the binaries for alpine
    # are missing on the NodeJS page. We cannot use the image based on Debian
    # because it is out-of-date.
    USE_SYSTEM_NODEJS: "true"
    # It must be an alias of the docker:dind service.
    DEFAULT_MAPPED_ADDRESS: "docker"
  # Don't block if the artifacts from the previous stages expired.
  dependencies: []
  script:
    # - rake system_tests_ui
    - rake systemtest
  artifacts:
    paths:
      - tests/system/test-results/**/*.log
    expire_in: 1 week
    when: always

### upload release notes and tarball to repo.isc.org ###

upload_to_repo:
  stage: deploy
  when: manual
  allow_failure: true
  tags:
    - linux
    - amd64
    - ssd
    - stork-repo
  needs:
    - tarball
  script:
    - eval $(ssh-agent -s)
    - echo "${REPO_SSH_PRIVATE_KEY}" | base64 -d | ssh-add -
    - mkdir ~/.ssh
    - ssh-keyscan -4 repo.isc.org >> ~/.ssh/known_hosts
    - rake release:notes
    - rake release:tarball:upload HOST=storkpush@repo.isc.org TARGET=/data/shared/sweng/stork/releases
  artifacts:
    name: release-notes
    expire_in: 1 week
    paths:
      - Stork-*-ReleaseNotes.txt

### demo deploy jobs ###

deploy_demo:
  stage: deploy
  cache: []
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'
      when: always
  allow_failure: true
  tags:
    - linux
    - amd64
    - ssd
  needs:
    - danger
  before_script:
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY_STORK" | base64 -d | ssh-add -
  script:
    # Print Docker info
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker info
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker version
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker-compose version
    # Stop all containers
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org "docker ps -q | xargs docker kill || true"
    # Force network cleanup and to make sure we have clean state do restart docker service
    # They can be replaced with docker-compose down after migration from the old demo.
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker container prune -f
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker network prune -f
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker volume prune -f
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org sudo systemctl restart docker
    # Copy source code
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org rm -rf deploy/*
    - scp -o StrictHostKeyChecking=no -r $PWD/* jenkins@stork.lab.isc.org:~/deploy
    # Build & Start Docker containers
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org CS_REPO_ACCESS_TOKEN=$CLOUDSMITH_REPO_ACCESS_TOKEN deploy/stork-demo.sh --no-prompt

### upload packages to cloudsmith.io ###

upload_packages:
  stage: deploy
  rules:
    - when: manual
  allow_failure: true
  image: registry.gitlab.isc.org/isc-projects/stork/pkgs-cloudsmith:latest
  tags:
    - linux
    - docker
    - amd64
    - ssd
  needs:
    - packages
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
  script:
    - export LANG='en_US.UTF-8' LC_ALL='en_US.UTF-8'
    - rake release:packages:upload REPO=stork

upload_test_packages:
  stage: deploy
  rules:
    - when: manual
  allow_failure: true
  image: registry.gitlab.isc.org/isc-projects/stork/pkgs-cloudsmith:latest
  tags:
    - linux
    - docker
    - amd64
    - ssd
  needs:
    - packages
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
  script:
    - export LANG='en_US.UTF-8' LC_ALL='en_US.UTF-8'
    - rake release:packages:upload REPO=stork-testing