feat: download openmldb spark distribution in java cicd (4paradigm#1111)

Add script to download openmldb spark distribution Download and extract spark files in cicd of java This is useful for unit tests of TaskManager which requires TaskManager with Spark to run test cases.
filaone · Jan 19, 2022 · c4e9050 · c4e9050
1 parent 59d4a89
commit c4e9050
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 0 deletions.
diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml
@@ -116,6 +116,7 @@ jobs:
  SQL_JAVASDK_ENABLE: ON
  OPENMLDB_BUILD_TARGET: 'cp_native_so openmldb'
  MAVEN_OPTS: -Duser.home=/github/home
+ SPARK_HOME: /tmp/spark/
  steps:
  - uses: actions/checkout@v2
 
@@ -166,6 +167,7 @@ jobs:
  - name: start services
  run: |
  sh steps/ut_zookeeper.sh start
+ sh steps/download_openmldb_spark.sh $SPARK_HOME
  cd onebox && sh start_onebox.sh && cd - || exit
 
  - name: run java modules smoke test

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -24,6 +24,7 @@ jobs:
  SQL_JAVASDK_ENABLE: ON
  TESTING_ENABLE: ON
  NPROC: 8
+ SPARK_HOME: /tmp/spark/
  steps:
  - uses: actions/checkout@v2
  with:
@@ -48,6 +49,7 @@ jobs:
  - name: start service
  run: |
  sh steps/ut_zookeeper.sh start
+ sh steps/download_openmldb_spark.sh $SPARK_HOME
  cd onebox
  bash start_onebox.sh
 

diff --git a/steps/download_openmldb_spark.sh b/steps/download_openmldb_spark.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -xe
+
+SPARK_HOME_PATH=$1
+
+# Download OpenMLDB Spark distribution
+curl -L -o spark-3.0.0-bin-openmldbspark.tgz https://github.com/4paradigm/spark/releases/download/v3.0.0-openmldb0.4.0/spark-3.0.0-bin-openmldbspark.tgz
+tar xzf ./spark-3.0.0-bin-openmldbspark.tgz
+
+# Move Spark files to $SPARK_HOME
+mv ./spark-3.0.0-bin-openmldbspark/ "$SPARK_HOME_PATH"
+
+rm ./spark-3.0.0-bin-openmldbspark.tgz