Move iceberg session properties to hive common module

mxmarkovics · Jan 5, 2024 · 0d09809 · 0d09809
1 parent 5a70da7
commit 0d09809
Show file tree

Hide file tree

Showing 48 changed files with 875 additions and 1,113 deletions.
diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaConnector.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaConnector.java
@@ -16,6 +16,7 @@
 import com.facebook.airlift.bootstrap.LifeCycleManager;
 import com.facebook.airlift.log.Logger;
 import com.facebook.presto.delta.rule.DeltaPlanOptimizerProvider;
+import com.facebook.presto.hive.HiveCommonSessionProperties;
 import com.facebook.presto.spi.connector.Connector;
 import com.facebook.presto.spi.connector.ConnectorMetadata;
 import com.facebook.presto.spi.connector.ConnectorPageSourceProvider;
@@ -30,6 +31,7 @@
 
 import javax.inject.Inject;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import static com.facebook.presto.delta.DeltaTransactionHandle.INSTANCE;
@@ -44,6 +46,7 @@ public class DeltaConnector
     private final DeltaMetadata metadata;
     private final DeltaSplitManager splitManager;
     private final DeltaSessionProperties sessionProperties;
+    private final HiveCommonSessionProperties hiveCommonSessionProperties;
     private final DeltaPageSourceProvider pageSourceProvider;
     private final DeltaPlanOptimizerProvider planOptimizerProvider;
     private final DeltaTableProperties deltaTableProperties;
@@ -54,6 +57,7 @@ public DeltaConnector(
             DeltaMetadata metadata,
             DeltaSplitManager splitManager,
             DeltaSessionProperties sessionProperties,
+            HiveCommonSessionProperties hiveCommonSessionProperties,
             DeltaPageSourceProvider pageSourceProvider,
             DeltaPlanOptimizerProvider planOptimizerProvider,
             DeltaTableProperties deltaTableProperties)
@@ -62,6 +66,7 @@ public DeltaConnector(
         this.metadata = requireNonNull(metadata, "metadata is null");
         this.splitManager = requireNonNull(splitManager, "splitManager is null");
         this.sessionProperties = requireNonNull(sessionProperties, "sessionProperties is null");
+        this.hiveCommonSessionProperties = requireNonNull(hiveCommonSessionProperties, "hiveCommonSessionProperties is null");
         this.pageSourceProvider = requireNonNull(pageSourceProvider, "pageSourceProvider is null");
         this.planOptimizerProvider = requireNonNull(planOptimizerProvider, "planOptimizerProvider is null");
         this.deltaTableProperties = requireNonNull(deltaTableProperties, "tableProperties is null");
@@ -94,7 +99,9 @@ public ConnectorPageSourceProvider getPageSourceProvider()
     @Override
     public List<PropertyMetadata<?>> getSessionProperties()
     {
-        return sessionProperties.getSessionProperties();
+        List<PropertyMetadata<?>> allSessionProperties = new ArrayList<>(sessionProperties.getSessionProperties());
+        allSessionProperties.addAll(hiveCommonSessionProperties.getSessionProperties());
+        return allSessionProperties;
     }
 
     @Override

diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaConnectorFactory.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaConnectorFactory.java
@@ -16,6 +16,7 @@
 import com.facebook.airlift.bootstrap.Bootstrap;
 import com.facebook.airlift.json.JsonModule;
 import com.facebook.presto.cache.CachingModule;
+import com.facebook.presto.hive.HiveCommonModule;
 import com.facebook.presto.hive.authentication.HiveAuthenticationModule;
 import com.facebook.presto.hive.gcs.HiveGcsModule;
 import com.facebook.presto.hive.metastore.HiveMetastoreModule;
@@ -61,6 +62,7 @@ public Connector create(String catalogName, Map<String, String> requiredConfig,
                     new HiveGcsModule(),
                     new HiveAuthenticationModule(),
                     new HiveMetastoreModule(catalogName, Optional.empty()),
+                    new HiveCommonModule(),
                     binder -> {
                         binder.bind(RowExpressionService.class).toInstance(context.getRowExpressionService());
                     });

diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaPageSourceProvider.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaPageSourceProvider.java
@@ -84,12 +84,12 @@
 import static com.facebook.presto.delta.DeltaErrorCode.DELTA_CANNOT_OPEN_SPLIT;
 import static com.facebook.presto.delta.DeltaErrorCode.DELTA_MISSING_DATA;
 import static com.facebook.presto.delta.DeltaErrorCode.DELTA_PARQUET_SCHEMA_MISMATCH;
-import static com.facebook.presto.delta.DeltaSessionProperties.getParquetMaxReadBlockSize;
-import static com.facebook.presto.delta.DeltaSessionProperties.getReadNullMaskedParquetEncryptedValue;
-import static com.facebook.presto.delta.DeltaSessionProperties.isParquetBatchReaderVerificationEnabled;
-import static com.facebook.presto.delta.DeltaSessionProperties.isParquetBatchReadsEnabled;
 import static com.facebook.presto.delta.DeltaTypeUtils.convertPartitionValue;
 import static com.facebook.presto.hive.CacheQuota.NO_CACHE_CONSTRAINTS;
+import static com.facebook.presto.hive.HiveCommonSessionProperties.getParquetMaxReadBlockSize;
+import static com.facebook.presto.hive.HiveCommonSessionProperties.getReadNullMaskedParquetEncryptedValue;
+import static com.facebook.presto.hive.HiveCommonSessionProperties.isParquetBatchReaderVerificationEnabled;
+import static com.facebook.presto.hive.HiveCommonSessionProperties.isParquetBatchReadsEnabled;
 import static com.facebook.presto.hive.parquet.HdfsParquetDataSource.buildHdfsParquetDataSource;
 import static com.facebook.presto.hive.parquet.ParquetPageSourceFactory.checkSchemaMatch;
 import static com.facebook.presto.hive.parquet.ParquetPageSourceFactory.createDecryptor;

diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaSessionProperties.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaSessionProperties.java
@@ -14,78 +14,39 @@
 package com.facebook.presto.delta;
 
 import com.facebook.presto.cache.CacheConfig;
-import com.facebook.presto.hive.HiveClientConfig;
 import com.facebook.presto.spi.ConnectorSession;
-import com.facebook.presto.spi.schedule.NodeSelectionStrategy;
 import com.facebook.presto.spi.session.PropertyMetadata;
 import com.google.common.collect.ImmutableList;
-import io.airlift.units.DataSize;
 
 import javax.inject.Inject;
 
 import java.util.List;
 
-import static com.facebook.presto.common.type.VarcharType.VARCHAR;
-import static com.facebook.presto.hive.HiveSessionProperties.dataSizeSessionProperty;
 import static com.facebook.presto.spi.session.PropertyMetadata.booleanProperty;
 
 public final class DeltaSessionProperties
 {
     private static final String CACHE_ENABLED = "cache_enabled";
-    private static final String PARQUET_MAX_READ_BLOCK_SIZE = "parquet_max_read_block_size";
-    private static final String PARQUET_BATCH_READ_OPTIMIZATION_ENABLED = "parquet_batch_read_optimization_enabled";
-    private static final String PARQUET_BATCH_READER_VERIFICATION_ENABLED = "parquet_batch_reader_verification_enabled";
     public static final String PARQUET_DEREFERENCE_PUSHDOWN_ENABLED = "parquet_dereference_pushdown_enabled";
-    public static final String READ_MASKED_VALUE_ENABLED = "read_null_masked_parquet_encrypted_value_enabled";
-    private static final String NODE_SELECTION_STRATEGY = "node_selection_strategy";
 
     private final List<PropertyMetadata<?>> sessionProperties;
 
     @Inject
     public DeltaSessionProperties(
             DeltaConfig deltaConfigConfig,
-            HiveClientConfig hiveClientConfig,
             CacheConfig cacheConfig)
     {
         sessionProperties = ImmutableList.of(
                 booleanProperty(
+                        // required by presto-hive module, might be removed in future
                         CACHE_ENABLED,
                         "Enable cache for Delta tables",
                         cacheConfig.isCachingEnabled(),
                         false),
-                new PropertyMetadata<>(
-                        NODE_SELECTION_STRATEGY,
-                        "Node affinity selection strategy",
-                        VARCHAR,
-                        NodeSelectionStrategy.class,
-                        hiveClientConfig.getNodeSelectionStrategy(),
-                        false,
-                        value -> NodeSelectionStrategy.valueOf(((String) value).toUpperCase()),
-                        NodeSelectionStrategy::toString),
-                dataSizeSessionProperty(
-                        PARQUET_MAX_READ_BLOCK_SIZE,
-                        "Parquet: Maximum size of a block to read",
-                        hiveClientConfig.getParquetMaxReadBlockSize(),
-                        false),
-                booleanProperty(
-                        PARQUET_BATCH_READ_OPTIMIZATION_ENABLED,
-                        "Is Parquet batch read optimization enabled",
-                        hiveClientConfig.isParquetBatchReadOptimizationEnabled(),
-                        false),
-                booleanProperty(
-                        PARQUET_BATCH_READER_VERIFICATION_ENABLED,
-                        "Is Parquet batch reader verification enabled? This is for testing purposes only, not to be used in production",
-                        hiveClientConfig.isParquetBatchReaderVerificationEnabled(),
-                        false),
                 booleanProperty(
                         PARQUET_DEREFERENCE_PUSHDOWN_ENABLED,
                         "Is dereference pushdown expression pushdown into Parquet reader enabled?",
                         deltaConfigConfig.isParquetDereferencePushdownEnabled(),
-                        false),
-                booleanProperty(
-                        READ_MASKED_VALUE_ENABLED,
-                        "Return null when access is denied for an encrypted parquet column",
-                        hiveClientConfig.getReadNullMaskedParquetEncryptedValue(),
                         false));
     }
 
@@ -94,38 +55,8 @@ public List<PropertyMetadata<?>> getSessionProperties()
         return sessionProperties;
     }
 
-    public static boolean isCacheEnabled(ConnectorSession session)
-    {
-        return session.getProperty(CACHE_ENABLED, Boolean.class);
-    }
-
-    public static NodeSelectionStrategy getNodeSelectionStrategy(ConnectorSession session)
-    {
-        return session.getProperty(NODE_SELECTION_STRATEGY, NodeSelectionStrategy.class);
-    }
-
-    public static DataSize getParquetMaxReadBlockSize(ConnectorSession session)
-    {
-        return session.getProperty(PARQUET_MAX_READ_BLOCK_SIZE, DataSize.class);
-    }
-
-    public static boolean isParquetBatchReadsEnabled(ConnectorSession session)
-    {
-        return session.getProperty(PARQUET_BATCH_READ_OPTIMIZATION_ENABLED, Boolean.class);
-    }
-
-    public static boolean isParquetBatchReaderVerificationEnabled(ConnectorSession session)
-    {
-        return session.getProperty(PARQUET_BATCH_READER_VERIFICATION_ENABLED, Boolean.class);
-    }
-
     public static boolean isParquetDereferencePushdownEnabled(ConnectorSession session)
     {
         return session.getProperty(PARQUET_DEREFERENCE_PUSHDOWN_ENABLED, Boolean.class);
     }
-
-    public static boolean getReadNullMaskedParquetEncryptedValue(ConnectorSession session)
-    {
-        return session.getProperty(READ_MASKED_VALUE_ENABLED, Boolean.class);
-    }
 }
diff --git a/presto-delta/src/main/java/com/facebook/presto/delta/DeltaSplitManager.java b/presto-delta/src/main/java/com/facebook/presto/delta/DeltaSplitManager.java
@@ -35,7 +35,7 @@
 import java.util.Map.Entry;
 import java.util.concurrent.CompletableFuture;
 
-import static com.facebook.presto.delta.DeltaSessionProperties.getNodeSelectionStrategy;
+import static com.facebook.presto.hive.HiveCommonSessionProperties.getNodeSelectionStrategy;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.CompletableFuture.completedFuture;
 import static java.util.stream.Collectors.toMap;

diff --git a/presto-hive-common/pom.xml b/presto-hive-common/pom.xml
@@ -39,16 +39,31 @@
             <artifactId>presto-spi</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-orc</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.facebook.presto.hadoop</groupId>
             <artifactId>hadoop-apache2</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>units</artifactId>