[FLINK-20835][test] Setup Azure build for running core and flink-test…

…s for fine-grained resource management This closes apache#14647
hsaputra · Feb 4, 2021 · 6423817 · 6423817
1 parent 05072ef
commit 6423817
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 2 deletions.
diff --git a/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/ClusterOptions.java
@@ -141,7 +141,8 @@ public static boolean isFineGrainedResourceManagementEnabled(Configuration confi
         // the feature base on the declarative protocol. We would be able to support both protocols
         // and no longer need this binding after FLINK-20838.
         return isDeclarativeResourceManagementEnabled(configuration)
-                && configuration.get(ENABLE_FINE_GRAINED_RESOURCE_MANAGEMENT);
+                && (configuration.get(ENABLE_FINE_GRAINED_RESOURCE_MANAGEMENT)
+                        || System.getProperties().containsKey("flink.tests.enable-fine-grained"));
     }
 
     /** The mode of how to handle user code attempting to exit JVM. */

diff --git a/tools/azure-pipelines/jobs-template.yml b/tools/azure-pipelines/jobs-template.yml
@@ -105,6 +105,8 @@ jobs:
         module: misc
       legacy_slot_management:
         module: legacy_slot_management
+      finegrained_resource_management:
+        module: finegrained_resource_management
   steps:
   # if on Azure, free up disk space
   - script: ./tools/azure-pipelines/free_disk_space.sh

diff --git a/tools/ci/stage.sh b/tools/ci/stage.sh
@@ -28,6 +28,7 @@ STAGE_TESTS="tests"
 STAGE_MISC="misc"
 STAGE_CLEANUP="cleanup"
 STAGE_LEGACY_SLOT_MANAGEMENT="legacy_slot_management"
+STAGE_FINEGRAINED_RESOURCE_MANAGEMENT="finegrained_resource_management"
 
 MODULES_CORE="\
 flink-annotations,\
@@ -125,6 +126,8 @@ flink-tests"
 
 MODULES_LEGACY_SLOT_MANAGEMENT=${MODULES_CORE},${MODULES_TESTS}
 
+MODULES_FINEGRAINED_RESOURCE_MANAGEMENT=${MODULES_CORE},${MODULES_TESTS}
+
 # we can only build the Scala Shell when building for Scala 2.11
 if [[ $PROFILE == *"scala-2.11"* ]]; then
     MODULES_CORE="$MODULES_CORE,flink-scala-shell"
@@ -164,6 +167,9 @@ function get_compile_modules_for_stage() {
         (${STAGE_LEGACY_SLOT_MANAGEMENT})
             echo "-pl $MODULES_LEGACY_SLOT_MANAGEMENT -am"
         ;;
+        (${STAGE_FINEGRAINED_RESOURCE_MANAGEMENT})
+            echo "-pl $MODULES_FINEGRAINED_RESOURCE_MANAGEMENT -am"
+        ;;
     esac
 }
 
@@ -183,6 +189,7 @@ function get_test_modules_for_stage() {
     local negated_tests=\!${MODULES_TESTS//,/,\!}
     local modules_misc="$negated_core,$negated_libraries,$negated_blink_planner,$negated_connectors,$negated_kafka_gelly,$negated_tests"
     local modules_legacy_slot_management=$MODULES_LEGACY_SLOT_MANAGEMENT
+    local modules_finegrained_resource_management=$MODULES_FINEGRAINED_RESOURCE_MANAGEMENT
 
     case ${stage} in
         (${STAGE_CORE})
@@ -208,6 +215,9 @@ function get_test_modules_for_stage() {
         ;;
         (${STAGE_LEGACY_SLOT_MANAGEMENT})
             echo "-pl $modules_legacy_slot_management"
-        ::
+        ;;
+        (${STAGE_FINEGRAINED_RESOURCE_MANAGEMENT})
+            echo "-pl $modules_finegrained_resource_management"
+        ;;
     esac
 }
diff --git a/tools/ci/test_controller.sh b/tools/ci/test_controller.sh
@@ -108,6 +108,9 @@ else
 	if [ $STAGE = $STAGE_LEGACY_SLOT_MANAGEMENT ]; then
 		MVN_TEST_OPTIONS="$MVN_TEST_OPTIONS -Dflink.tests.disable-declarative"
 	fi
+	if [ $STAGE = $STAGE_FINEGRAINED_RESOURCE_MANAGEMENT ]; then
+	  MVN_TEST_OPTIONS="$MVN_TEST_OPTIONS -Dflink.tests.enable-fine-grained"
+	fi
 	MVN_TEST_MODULES=$(get_test_modules_for_stage ${STAGE})
 
 	run_with_watchdog "run_mvn $MVN_COMMON_OPTIONS $MVN_TEST_OPTIONS $PROFILE $MVN_TEST_MODULES verify" $CALLBACK_ON_TIMEOUT