Increase dynamic filtering limits

txn2 · Oct 16, 2020 · 0d1b31d · 0d1b31d
1 parent 36d2ab3
commit 0d1b31d
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 18 deletions.
diff --git a/presto-main/src/main/java/io/prestosql/execution/DynamicFilterConfig.java b/presto-main/src/main/java/io/prestosql/execution/DynamicFilterConfig.java
@@ -37,19 +37,19 @@ public class DynamicFilterConfig
     private boolean enableLargeDynamicFilters;
     private int serviceThreadCount = 2;
 
-    private int smallBroadcastMaxDistinctValuesPerDriver = 100;
-    private DataSize smallBroadcastMaxSizePerDriver = DataSize.of(10, KILOBYTE);
-    private int smallBroadcastRangeRowLimitPerDriver;
-    private int smallPartitionedMaxDistinctValuesPerDriver = 10;
+    private int smallBroadcastMaxDistinctValuesPerDriver = 200;
+    private DataSize smallBroadcastMaxSizePerDriver = DataSize.of(20, KILOBYTE);
+    private int smallBroadcastRangeRowLimitPerDriver = 400;
+    private int smallPartitionedMaxDistinctValuesPerDriver = 20;
     private DataSize smallPartitionedMaxSizePerDriver = DataSize.of(10, KILOBYTE);
-    private int smallPartitionedRangeRowLimitPerDriver;
+    private int smallPartitionedRangeRowLimitPerDriver = 100;
 
     private int largeBroadcastMaxDistinctValuesPerDriver = 5_000;
     private DataSize largeBroadcastMaxSizePerDriver = DataSize.of(500, KILOBYTE);
-    private int largeBroadcastRangeRowLimitPerDriver;
+    private int largeBroadcastRangeRowLimitPerDriver = 10_000;
     private int largePartitionedMaxDistinctValuesPerDriver = 500;
     private DataSize largePartitionedMaxSizePerDriver = DataSize.of(50, KILOBYTE);
-    private int largePartitionedRangeRowLimitPerDriver;
+    private int largePartitionedRangeRowLimitPerDriver = 1_000;
 
     public boolean isEnableDynamicFiltering()
     {

diff --git a/presto-main/src/test/java/io/prestosql/execution/TestDynamicFilterConfig.java b/presto-main/src/test/java/io/prestosql/execution/TestDynamicFilterConfig.java
@@ -33,18 +33,18 @@ public void testDefaults()
                 .setEnableDynamicFiltering(true)
                 .setEnableLargeDynamicFilters(false)
                 .setServiceThreadCount(2)
-                .setSmallBroadcastMaxDistinctValuesPerDriver(100)
-                .setSmallBroadcastMaxSizePerDriver(DataSize.of(10, KILOBYTE))
-                .setSmallBroadcastRangeRowLimitPerDriver(0)
-                .setSmallPartitionedMaxDistinctValuesPerDriver(10)
+                .setSmallBroadcastMaxDistinctValuesPerDriver(200)
+                .setSmallBroadcastMaxSizePerDriver(DataSize.of(20, KILOBYTE))
+                .setSmallBroadcastRangeRowLimitPerDriver(400)
+                .setSmallPartitionedMaxDistinctValuesPerDriver(20)
                 .setSmallPartitionedMaxSizePerDriver(DataSize.of(10, KILOBYTE))
-                .setSmallPartitionedRangeRowLimitPerDriver(0)
+                .setSmallPartitionedRangeRowLimitPerDriver(100)
                 .setLargeBroadcastMaxDistinctValuesPerDriver(5000)
                 .setLargeBroadcastMaxSizePerDriver(DataSize.of(500, KILOBYTE))
-                .setLargeBroadcastRangeRowLimitPerDriver(0)
+                .setLargeBroadcastRangeRowLimitPerDriver(10_000)
                 .setLargePartitionedMaxDistinctValuesPerDriver(500)
                 .setLargePartitionedMaxSizePerDriver(DataSize.of(50, KILOBYTE))
-                .setLargePartitionedRangeRowLimitPerDriver(0));
+                .setLargePartitionedRangeRowLimitPerDriver(1_000));
     }
 
     @Test

diff --git a/presto-memory/src/test/java/io/prestosql/plugin/memory/TestMemorySmoke.java b/presto-memory/src/test/java/io/prestosql/plugin/memory/TestMemorySmoke.java
@@ -57,8 +57,10 @@ protected QueryRunner createQueryRunner()
             throws Exception
     {
         return MemoryQueryRunner.createQueryRunner(
-                // Reduced broadcast join limit for large DF to make withLargeDynamicFilters use range DF collection
+                // Adjust DF limits to test edge cases
                 ImmutableMap.of(
+                        "dynamic-filtering.small-broadcast.max-distinct-values-per-driver", "100",
+                        "dynamic-filtering.small-broadcast.range-row-limit-per-driver", "100",
                         "dynamic-filtering.large-broadcast.max-distinct-values-per-driver", "100",
                         "dynamic-filtering.large-broadcast.range-row-limit-per-driver", "100000"));
     }

diff --git a/presto-tests/src/test/java/io/prestosql/execution/TestCoordinatorDynamicFiltering.java b/presto-tests/src/test/java/io/prestosql/execution/TestCoordinatorDynamicFiltering.java
@@ -37,7 +37,9 @@
 import io.prestosql.spi.connector.ConnectorTransactionHandle;
 import io.prestosql.spi.connector.DynamicFilter;
 import io.prestosql.spi.connector.EmptyPageSource;
+import io.prestosql.spi.predicate.Domain;
 import io.prestosql.spi.predicate.TupleDomain;
+import io.prestosql.spi.predicate.ValueSet;
 import io.prestosql.spi.transaction.IsolationLevel;
 import io.prestosql.split.EmptySplit;
 import io.prestosql.testing.AbstractTestQueryFramework;
@@ -60,6 +62,7 @@
 import static io.prestosql.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
 import static io.prestosql.SystemSessionProperties.TASK_CONCURRENCY;
 import static io.prestosql.spi.predicate.Domain.singleValue;
+import static io.prestosql.spi.predicate.Range.range;
 import static io.prestosql.spi.type.BigintType.BIGINT;
 import static io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType.BROADCAST;
 import static io.prestosql.sql.analyzer.FeaturesConfig.JoinDistributionType.PARTITIONED;
@@ -102,6 +105,9 @@ protected QueryRunner createQueryRunner()
                 .setSystemProperty(JOIN_DISTRIBUTION_TYPE, PARTITIONED.name())
                 .build();
         return DistributedQueryRunner.builder(session)
+                .setExtraProperties(ImmutableMap.of(
+                        // keep limits lower to test edge cases
+                        "dynamic-filtering.small-partitioned.max-distinct-values-per-driver", "10"))
                 .build();
     }
 
@@ -165,7 +171,9 @@ public void testJoinWithNonSelectiveBuildSide()
     {
         assertQueryDynamicFilters(
                 "SELECT * FROM lineitem JOIN tpch.tiny.supplier ON lineitem.suppkey = supplier.suppkey",
-                TupleDomain.all());
+                TupleDomain.withColumnDomains(ImmutableMap.of(
+                        SUPP_KEY_HANDLE,
+                        Domain.create(ValueSet.ofRanges(range(BIGINT, 1L, true, 100L, true)), false))));
     }
 
     @Test(timeOut = 30_000)
@@ -195,7 +203,9 @@ public void testRightJoinWithNonSelectiveBuildSide()
     {
         assertQueryDynamicFilters(
                 "SELECT * FROM lineitem RIGHT JOIN tpch.tiny.supplier ON lineitem.suppkey = supplier.suppkey",
-                TupleDomain.all());
+                TupleDomain.withColumnDomains(ImmutableMap.of(
+                        SUPP_KEY_HANDLE,
+                        Domain.create(ValueSet.ofRanges(range(BIGINT, 1L, true, 100L, true)), false))));
     }
 
     @Test(timeOut = 30_000)
@@ -268,7 +278,9 @@ public void testSemiJoinWithNonSelectiveBuildSide()
     {
         assertQueryDynamicFilters(
                 "SELECT * FROM lineitem WHERE lineitem.suppkey IN (SELECT supplier.suppkey FROM tpch.tiny.supplier)",
-                TupleDomain.all());
+                TupleDomain.withColumnDomains(ImmutableMap.of(
+                        SUPP_KEY_HANDLE,
+                        Domain.create(ValueSet.ofRanges(range(BIGINT, 1L, true, 100L, true)), false))));
     }
 
     @Test(timeOut = 30_000)