CI refactor for staging enviroment

n1cks0n · Oct 28, 2021 · 86eb051 · 86eb051
1 parent 859278b
commit 86eb051
Show file tree

Hide file tree

Showing 14 changed files with 166 additions and 113 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -31,6 +31,9 @@ before_script:
 
 include:
   - local: .gitlab/ci/fetch-secrets.yaml
+  - local: .gitlab/ci/.rules-conditions.yml
+
+
   - local: .gitlab/ci/testing/.pretesting-gitlab-ci.yml
   - local: .gitlab/ci/testing/.nix-testing-gitlab-ci.yml
   - local: .gitlab/ci/testing/.venv-testing-gitlab-ci.yml
@@ -46,6 +49,7 @@ include:
   - local: .gitlab/ci/deploy/deploy-gitlab-ci.yml
 
   - local: .gitlab/ci/publish/.set-tag-gitlab-ci.yml
+  - local: .gitlab/ci/publish/.mirror-to-github-gitlab-ci.yml
   - local: .gitlab/ci/publish/.docker-publish-gitlab-ci.yml
 
   - template: Security/Secret-Detection.gitlab-ci.yml

diff --git a/.gitlab/ci/.rules-conditions.yml b/.gitlab/ci/.rules-conditions.yml
@@ -0,0 +1,52 @@
+# When should run
+.dev-staging-master:
+  rules:
+    - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|staging|dev)$/'
+      when: on_success
+
+.on-master-staging:
+  rules:
+    - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|staging)$/'
+      when: on_success
+
+.on-master:
+  rules:
+    - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
+      when: on_success
+
+.on-community-tag:
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^white-v[0-9.]+$/'
+      when: on_success
+
+.pipeline-control-test:
+  rules:
+    - if: $FULL_TEST || $DAILY_TEST
+      when: on_success
+
+.be-built:
+  rules:
+    - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
+      when: on_success
+
+.be-uploaded:
+  rules:
+    - if: '$BUILD_TEST || $FULL_TEST || $DAILY_TEST'
+      when: on_success
+
+
+# Ignore
+.ignore-on-tag:
+  rules:
+    - if: '$CI_COMMIT_TAG'
+      when: never
+
+.ignore-on-master:
+  rules:
+    - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
+      when: never
+
+.ignore-on-build:
+  rules:
+    - if: $BUILD_TEST
+      when: never
diff --git a/.gitlab/ci/build-ci/.build-gitlab-ci.yml b/.gitlab/ci/build-ci/.build-gitlab-ci.yml
@@ -37,10 +37,8 @@ generate_deb:
             - "faraday-server_amd64.deb"
         expire_in: 15 days
     rules:
-      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
-        when: on_success
-      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
-        when: on_success
+      - !reference [.on-master-staging, rules]
+      - !reference [.be-built, rules]
       - when: never
 
 
@@ -91,10 +89,8 @@ generate_rpm:
             - "faraday-server_amd64.rpm"
         expire_in: 15 days
     rules:
-      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
-        when: on_success
-      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
-        when: on_success
+      - !reference [.on-master-staging, rules]
+      - !reference [.be-built, rules]
       - when: never
 
 generate_docker_tar_gz:
@@ -115,7 +111,6 @@ generate_docker_tar_gz:
       paths:
         - faraday-server-docker.tar.gz
     rules:
-        - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
-          when: on_success
-        - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
-          when: on_success
+      - !reference [.on-master-staging, rules]
+      - !reference [.be-built, rules]
+      - when: never
diff --git a/.gitlab/ci/build-ci/.prebuild-gitlab-ci.yml b/.gitlab/ci/build-ci/.prebuild-gitlab-ci.yml
@@ -20,10 +20,8 @@ generate_build_file:
         - "/usr/bin/rsync -aq --exclude 'faraday_copy' --exclude '.cache' . faraday_copy"
         - "/bin/tar -zcf faraday.tar.gz faraday_copy"
     rules:
-      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
-        when: on_success
-      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
-        when: on_success
+      - !reference [.on-master-staging, rules]
+      - !reference [.be-built, rules]
       - when: never
     artifacts:
         name: 'faraday'
@@ -55,8 +53,6 @@ build_nix_python3:
       - py3.tar
     expire_in: 15 days # in the future we don't need to expire this.
   rules:
-    - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
-      when: on_success
-    - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
-      when: on_success
-    - when: never
+      - !reference [.on-master-staging, rules]
+      - !reference [.be-built, rules]
+      - when: never
diff --git a/.gitlab/ci/build-ci/.testing-gitlab-ci.yml b/.gitlab/ci/build-ci/.testing-gitlab-ci.yml
@@ -23,8 +23,6 @@ smoke_test_deb:
         - kill $(cat ~faraday/.faraday/faraday-server-port-5985.pid)
         - jobs
     rules:
-      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
-        when: on_success
-      - if: '$CI_COMMIT_TAG || $BUILD_TEST || $FULL_TEST || $DAILY_TEST'
-        when: on_success
+      - !reference [.on-master-staging, rules]
+      - !reference [.be-built, rules]
       - when: never
diff --git a/.gitlab/ci/publish/.docker-publish-gitlab-ci.yml b/.gitlab/ci/publish/.docker-publish-gitlab-ci.yml
@@ -14,8 +14,7 @@ docker-publish-dev:
     - docker image tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$VERSION
     - docker push "$CI_REGISTRY_IMAGE"
   rules:
-    - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
-      when: on_success
+    - !reference [ .on-master, rules ]
   needs: # dev won't wait for any previous stage, it will deploy instantly, and
       # then run tests in docker image (To be done)
     - job: generate_docker_tar_gz
@@ -35,7 +34,6 @@ docker-publish-prod:
     - docker image tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$VERSION
     - docker push $CI_REGISTRY_IMAGE:$VERSION
   rules:
-    - if: '$CI_COMMIT_TAG =~ /^white-v[0-9.]+$/'
-      when: on_success
+    - !reference [ .on-community-tag, rules ]
   dependencies: # prod will wait for any previous stage
     - generate_docker_tar_gz
diff --git a/.gitlab/ci/publish/.mirror-to-github-gitlab-ci.yml b/.gitlab/ci/publish/.mirror-to-github-gitlab-ci.yml
@@ -0,0 +1,37 @@
+update_github:
+    image: python:3
+    stage: publish
+    script:
+      - git remote set-url github https://${GH_USER}:${GH_TOKEN}@github.com/infobyte/faraday.git
+      - git push github $CI_COMMIT_REF_NAME:$DESTINY_BRANCH
+    rules:
+      - if: '$CI_COMMIT_REF_NAME == "white/master"'
+        variables:
+          DESTINY_BRANCH: master
+        when: on_success
+      - if: '$CI_COMMIT_REF_NAME == "white/staging"'
+        variables:
+          DESTINY_BRANCH: staging
+        when: on_success
+    tags:
+        - faradaytests
+
+
+tag_on_github:
+    image: python:3
+    stage: publish
+    script:
+      - git remote set-url github https://${GH_USER}:${GH_TOKEN}@github.com/infobyte/faraday.git
+      - export FARADAY_VERSION=$(eval $IMAGE_TAG)
+      - CHANGELOG/check_pre_tag.py
+      - git push github $CI_COMMIT_TAG:master
+      - git tag v$FARADAY_VERSION -m "$(cat CHANGELOG/$FARADAY_VERSION/community.md)"
+      - git push github v$FARADAY_VERSION
+      - scripts/github_release.py --deb-file ./faraday-server_amd64.deb --rpm-file ./faraday-server_amd64.rpm
+    rules:
+      - !reference [ .on-community-tag, rules ]
+    dependencies:
+      - generate_deb
+      - generate_rpm
+    tags:
+      - faradaytests
diff --git a/.gitlab/ci/publish/.set-tag-gitlab-ci.yml b/.gitlab/ci/publish/.set-tag-gitlab-ci.yml
@@ -1,22 +1,3 @@
-tag_on_github:
-    image: python:3
-    stage: publish
-    before_script:
-    script:
-      - git remote set-url github https://${GH_USER}:${GH_TOKEN}@github.com/infobyte/faraday.git
-      - export FARADAY_VERSION=$(eval $IMAGE_TAG)
-      - CHANGELOG/check_pre_tag.py
-      - git push github $CI_COMMIT_TAG:master
-      - git tag v$FARADAY_VERSION -m "$(cat CHANGELOG/$FARADAY_VERSION/white.md)"
-      - git push github v$FARADAY_VERSION
-      - scripts/github_release.py --deb-file ./faraday-server_amd64.deb --rpm-file ./faraday-server_amd64.rpm
-    rules:
-      - if: '$CI_COMMIT_TAG =~ /^white-v[0-9.]+$/'
-        when: on_success
-    dependencies:
-      - generate_deb
-      - generate_rpm
-
 publish_pypi:
     image: python:3
     stage: publish
@@ -27,5 +8,4 @@ publish_pypi:
       - python setup.py sdist bdist_wheel
       - twine upload -u $PYPI_USER -p $PYPI_PASS dist/* --verbose
     rules:
-      - if: '$CI_COMMIT_TAG =~ /^white-v[0-9.]+$/'
-        when: on_success
+      - !reference [ .on-community-tag, rules ]
diff --git a/.gitlab/ci/testing/.hypothesis-nix-gitlab-ci.yml b/.gitlab/ci/testing/.hypothesis-nix-gitlab-ci.yml
@@ -20,6 +20,7 @@ test_hypothesis:
         - mkdir run_from
         - nix-shell --command "pytest tests -v --cov=faraday/server/api --disable-warnings --connection-string=postgresql+psycopg2://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres/$POSTGRES_DB -m hypothesis"
     rules:
-      - if: '$HYPO_TEST || $FULL_TEST || $DAILY_TEST'
+      - if: $HYPO_TEST
         when: on_success
+      - !reference [.pipeline-control-test, rules]
       - when: never
diff --git a/.gitlab/ci/testing/.nix-testing-gitlab-ci.yml b/.gitlab/ci/testing/.nix-testing-gitlab-ci.yml
@@ -20,12 +20,10 @@ pylint:
             - pylint.svg
             - pylint3.svg
     rules:
-        - if: $BUILD_TEST
-          when: never
-        - if: '$CI_COMMIT_TAG'
-          when: never
-        - if: '$FULL_TEST || $DAILY_TEST'
-          when: on_success
+        - !reference [.ignore-on-build, rules]
+        - !reference [.ignore-on-tag, rules]
+        - !reference [.ignore-on-master, rules]
+        - !reference [.pipeline-control-test, rules]
         - when: on_success
 
 .postgresql_test_nix_base:
@@ -57,13 +55,10 @@ pylint:
         artifacts: false
   # Speed up tests
     rules:
-      - if: $BUILD_TEST
-        when: never
-      - if: '$FULL_TEST || $DAILY_TEST'
-        when: on_success
-      - if: '$CI_COMMIT_TAG'
-        when: never
-      - when: on_success
+        - !reference [.ignore-on-build, rules]
+        - !reference [.ignore-on-tag, rules]
+        - !reference [.pipeline-control-test, rules]
+        - when: on_success
 
 .sqlite_test_nix_base:
     tags:
@@ -93,12 +88,9 @@ pylint:
       - job: build_and_push_to_cachix
         artifacts: false
     rules:
-        - if: $BUILD_TEST
-          when: never
-        - if: '$CI_COMMIT_TAG'
-          when: never
-        - if: '$FULL_TEST || $DAILY_TEST'
-          when: on_success
+        - !reference [.ignore-on-build, rules]
+        - !reference [.ignore-on-tag, rules]
+        - !reference [.pipeline-control-test, rules]
         - when: on_success
 
 sqlite_test_nix:

diff --git a/.gitlab/ci/testing/.posttesting-gitlab-ci.yml b/.gitlab/ci/testing/.posttesting-gitlab-ci.yml
@@ -8,7 +8,15 @@ agent_integration_and_db_regression:
       strategy: depend
     rules:
       - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master)$/'
+        variables:
+          DISPATCHER_REF: master
+        when: on_success
+      - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(staging)$/'
+        variables:
+          DISPATCHER_REF: staging
         when: on_success
       - if: '$INTEGRATION || $FULL_TEST || $DAILY_TEST'
+        variables:
+          DISPATCHER_REF: staging
         when: on_success
       - when: never
diff --git a/.gitlab/ci/testing/.pretesting-gitlab-ci.yml b/.gitlab/ci/testing/.pretesting-gitlab-ci.yml
@@ -9,10 +9,8 @@ merge_conflict_check:
         - git config --global user.name "Mergerbot"
         - python3 scripts/merge-conflict-detector.py
     rules:
-        - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|dev)$/'
-          when: on_success
-        - if: '$CI_COMMIT_TAG'
-          when: never
+        - !reference [.dev-staging-master, rules]
+        - !reference [.ignore-on-tag, rules]
         - when: never
 
 sanity_check:
@@ -24,10 +22,8 @@ sanity_check:
         - bash scripts/sanity_check_commit.sh
         - scripts/sanity_check_file.py --mode=ls
     rules:
-        - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|dev)$/'
-          when: on_success
-        - if: '$CI_COMMIT_TAG'
-          when: never
+        - !reference [.dev-staging-master, rules]
+        - !reference [.ignore-on-tag, rules]
         - when: never
 
 migration_sanity_check:
@@ -41,10 +37,8 @@ migration_sanity_check:
         - cd faraday
         - $(alembic branches)
     rules:
-        - if: '$CI_COMMIT_REF_NAME =~ /^.*\/(master|dev)$/'
-          when: always
-        - if: '$CI_COMMIT_TAG'
-          when: never
+        - !reference [.dev-staging-master, rules]
+        - !reference [.ignore-on-tag, rules]
         - when: never
 
 bandit:
@@ -60,8 +54,7 @@ bandit:
       - "bandit -r ${CI_PROJECT_DIR}/faraday --format custom --skip B101 --msg-template \
           \"{abspath}:{line}: {test_id}[bandit]: {severity}: {msg}\""
     rules:
-        - if: '$CI_COMMIT_TAG'
-          when: never
+        - !reference [.ignore-on-tag, rules]
         - when: on_success
 
 build_and_push_to_cachix:
@@ -84,9 +77,7 @@ build_and_push_to_cachix:
         - nix-build | cachix push faradaysec
         - ./scripts/check-closure-size ./result
     rules:
-      - if: '$FULL_TEST || $DAILY_TEST'
-        when: on_success
-      - when: always
+      - when: on_success
 
 flake8:
     image: python:3
@@ -95,8 +86,7 @@ flake8:
       - pip install flake8
       - flake8 .
     rules:
-        - if: '$CI_COMMIT_TAG'
-          when: never
+        - !reference [.ignore-on-tag, rules]
         - when: on_success
 
 no-format-str:
@@ -106,6 +96,6 @@ no-format-str:
       - pip install flynt
       - flynt -df faraday tests
     rules:
-        - if: '$CI_COMMIT_TAG'
-          when: never
+        - !reference [.ignore-on-tag, rules]
+        - !reference [.ignore-on-master, rules]
         - when: on_success