test-universal-doc_id-kfp.yml

#
# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
#
name: Test KFP - transforms/universal/doc_id

on:
    workflow_dispatch:
    push:
        branches:
            - "dev"
            - "releases/**"
        tags:
            - "*"
        paths:
            - ".make.*"
            - "transforms/.make.workflow"
            - "transforms/universal/doc_id/**"
            - "!kfp/**" # This is tested in separate workflow
            - "!data-processing-lib/**" # This is tested in separate workflow
            - "!**.md"
            - "!**/doc/**"
            - "!**/images/**"
            - "!**.gitignore"
    pull_request:
        branches:
            - "dev"
            - "releases/**"
        paths:
            - ".make.*"
            - "transforms/.make.workflow"
            - "transforms/universal/doc_id/**"
            - "!data-processing-lib/**" # This is tested in separate workflow
            - "!kfp/**" # This is tested in separate workflow
            - "!**.md"
            - "!**/doc/**"
            - "!**/images/**"
            - "!**.gitignore"

# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
concurrency:
    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
    cancel-in-progress: true

jobs:
    test-kfp-v1:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Free up space in github runner
              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
              run: |
                  df -h
                  sudo rm -rf "/usr/local/share/boost"
                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                  df -h
            - name: Test KFP libs (shared and v1) and run a workflow
              timeout-minutes: 120
              run: |
                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
                      export REPOROOT=$PWD
                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                      source $K8S_SETUP_SCRIPTS/requirements.env
                      export PATH=$PATH:/tmp/
                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
                      chmod 777 /tmp/kind
                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
                      chmod 700 /tmp/get_helm.sh
                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
                      chmod 777 /tmp/helm
                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
                      chmod 777 /tmp/kubectl
                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
                      chmod +x /tmp/mc
                      export DEPLOY_KUBEFLOW=1
                      make -C $K8S_SETUP_SCRIPTS setup
                      make -C kfp/kfp_support_lib test
                      make -C transforms/universal/doc_id workflow-build
                      source $K8S_SETUP_SCRIPTS/common.sh
                      make -C transforms/universal/doc_id workflow-test
                      echo "Run transforms/universal/doc_id completed"
                   else
                      echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray"
                   fi

    test-kfp-v2:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Free up space in github runner
              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
              run: |
                  df -h
                  sudo rm -rf "/usr/local/share/boost"
                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
                  df -h
            - name: Test KFP libs (shared and v2) and run a workflow
              timeout-minutes: 120
              run: |
                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
                      export REPOROOT=$PWD
                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
                      source $K8S_SETUP_SCRIPTS/requirements.env
                      export PATH=$PATH:/tmp/
                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
                      chmod 777 /tmp/kind
                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
                      chmod 700 /tmp/get_helm.sh
                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
                      chmod 777 /tmp/helm
                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
                      chmod 777 /tmp/kubectl
                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
                      chmod +x /tmp/mc
                      export DEPLOY_KUBEFLOW=1
                      export KFPv2=1
                      make -C $K8S_SETUP_SCRIPTS setup
                      make -C kfp/kfp_support_lib test
                      make -C transforms/universal/doc_id workflow-build
                      source $K8S_SETUP_SCRIPTS/common.sh
                      make -C transforms/universal/doc_id workflow-test
                      echo "Run transforms/universal/doc_id completed"
                  else
                      echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray"
                  fi