Merge branch 'dev' into make-mods

pankajskku · Aug 18, 2024 · 483a756 · 483a756
2 parents d2c6102 + 3d99b3b
commit 483a756
Show file tree

Hide file tree

Showing 162 changed files with 9,878 additions and 1,299 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -7,33 +7,33 @@ on:
  - "dev"
  - "releases/**"
  tags:
- - '*'
+ - "*"
  pull_request:
  branches:
  - "dev"
  - "releases/**"
 jobs:
  check_if_push_images:
- # check whether the Docker images should be pushed to the remote repository
- # The images are pushed if it is a merge to dev branch or a new tag is created.
- # The latter being part of the release process.
- # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
+  # check whether the Docker images should be pushed to the remote repository
+  # The images are pushed if it is a merge to dev branch or a new tag is created.
+  # The latter being part of the release process.
+  # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
  runs-on: ubuntu-22.04
  outputs:
  publish_images: ${{ steps.version.outputs.publish_images }}
  steps:
  - id: version
  run: |
- publish_images='false'
- if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ;
- then
- publish_images='true'
- fi
- if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ;
- then
- publish_images='true'
- fi
- echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
+  publish_images='false'
+  if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ;
+  then
+  publish_images='true'
+  fi
+  if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ;
+  then
+  publish_images='true'
+  fi
+  echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
  test-make:
  runs-on: ubuntu-22.04
  steps:
@@ -216,8 +216,8 @@ jobs:
  if: needs.check_if_push_images.outputs.publish_images == 'true'
  runs-on: ubuntu-22.04
  env:
- DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
- DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
+  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+  DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
  timeout-minutes: 30
  steps:
  - name: Checkout
@@ -233,23 +233,24 @@ jobs:
  df -h
  - name: Test Code Transform Images
  run: |
- make -C data-processing-lib/spark DOCKER=docker image 
- - name: Print space
- # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+ make -C data-processing-lib/spark DOCKER=docker image
+ - name:
+ Print space
+ # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
  run: |
-  df -h
-  docker images
+ df -h
+ docker images
  - name: Publish images
  if: needs.check_if_push_images.outputs.publish_images == 'true'
  run: |
-  make -C data-processing-lib/spark publish-image
+ make -C data-processing-lib/spark publish-image
  test-code-images:
  needs: [check_if_push_images]
  runs-on: ubuntu-22.04
  timeout-minutes: 30
  env:
- DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
- DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
+  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+  DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
  steps:
  - name: Checkout
  uses: actions/checkout@v4
@@ -266,22 +267,23 @@ jobs:
  run: |
  make -C data-processing-lib DOCKER=docker image
  make -C transforms/code DOCKER=docker test-image
- - name: Print space
- # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+ - name:
+ Print space
+ # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
  run: |
-  df -h
-  docker images
+ df -h
+ docker images
  - name: Publish images
  if: needs.check_if_push_images.outputs.publish_images == 'true'
  run: |
-  make -C transforms/code publish
+ make -C transforms/code publish
  test-language-images:
  needs: [check_if_push_images]
  runs-on: ubuntu-22.04
  timeout-minutes: 120
  env:
- DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
- DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
+  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+  DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
  steps:
  - name: Checkout
  uses: actions/checkout@v4
@@ -304,14 +306,14 @@ jobs:
  - name: Publish images
  if: needs.check_if_push_images.outputs.publish_images == 'true'
  run: make -C transforms/language publish
- 
+
  test-universal-images:
  needs: [check_if_push_images]
  runs-on: ubuntu-22.04
  timeout-minutes: 120
  env:
- DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
- DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
+  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+  DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
  steps:
  - name: Checkout
  uses: actions/checkout@v4
@@ -329,40 +331,41 @@ jobs:
  run: |
  make -C data-processing-lib/spark DOCKER=docker image 
  make -C transforms/universal DOCKER=docker test-image
- - name: Print space
- # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+ - name:
+ Print space
+ # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
  run: |
-  df -h
-  docker images
+ df -h
+ docker images
  - name: Publish images
  if: needs.check_if_push_images.outputs.publish_images == 'true'
  run: make -C transforms/universal publish
  build-kfp-components:
- needs: [check_if_push_images]
- runs-on: ubuntu-22.04
- timeout-minutes: 30
- env:
- DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
- DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
- steps:
- - name: Checkout
- uses: actions/checkout@v4
- - name: Free up space in github runner
- # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
- run: |
- df -h
- sudo rm -rf "/usr/local/share/boost"
- sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
- sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
- df -h
- - name: Build 
- run: |
- make -C kfp/kfp_ray_components DOCKER=docker image
- make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image
- - name: Publish images
- if: needs.check_if_push_images.outputs.publish_images == 'true'
- run: make -C kfp/kfp_ray_components publish
+  needs: [check_if_push_images]
+  runs-on: ubuntu-22.04
+  timeout-minutes: 30
+  env:
+  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+  DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }}
+  steps:
+  - name: Checkout
+  uses: actions/checkout@v4
+  - name: Free up space in github runner
+  # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+  run: |
+  df -h
+  sudo rm -rf "/usr/local/share/boost"
+  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+  df -h
+  - name: Build
+  run: |
+  make -C kfp/kfp_ray_components DOCKER=docker image
+  make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image
+  - name: Publish images
+  if: needs.check_if_push_images.outputs.publish_images == 'true'
+  run: make -C kfp/kfp_ray_components publish
  test-tool-images:
  runs-on: ubuntu-22.04
  timeout-minutes: 30
@@ -371,4 +374,4 @@ jobs:
  uses: actions/checkout@v4
  - name: Build and Test Tool images
  run: |
- make -C tools/ingest2parquet DOCKER=docker test-image
+ make -C tools/ingest2parquet DOCKER=docker test-image
diff --git a/.github/workflows/workflow-manual-run.yml b/.github/workflows/workflow-manual-run.yml
@@ -52,9 +52,11 @@ jobs:
  curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
  chmod +x /tmp/mc
  make -C $K8S_SETUP_SCRIPTS setup
- make -C $WORKFLOW_PATH workflow-test
  debug_mode=$(echo "$DEBUG_MODE" | tr '[:upper:]' '[:lower:]')
  if [ "$debug_mode" == "true" ]; then
+ make -C $WORKFLOW_PATH workflow-test || true
  kubectl describe pod -A
  kubectl get pods -A
+ else
+ make -C $WORKFLOW_PATH workflow-test
  fi