Fix handling of missing columns in Parquet Reader

soscpd · Feb 16, 2016 · 40b024a · 40b024a
1 parent 251f3ef
commit 40b024a
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSource.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSource.java
@@ -51,6 +51,7 @@
 import static com.facebook.presto.hive.HiveUtil.bigintPartitionKey;
 import static com.facebook.presto.hive.HiveUtil.booleanPartitionKey;
 import static com.facebook.presto.hive.HiveUtil.doublePartitionKey;
+import static com.facebook.presto.hive.parquet.ParquetTypeUtils.getParquetType;
 import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
@@ -84,14 +85,16 @@ class ParquetPageSource
 
     public ParquetPageSource(
             ParquetReader parquetReader,
+            MessageType fileSchema,
             MessageType requestedSchema,
             Path path,
             long totalBytes,
             Properties splitSchema,
             List<HiveColumnHandle> columns,
             List<HivePartitionKey> partitionKeys,
             TupleDomain<HiveColumnHandle> effectivePredicate,
-            TypeManager typeManager)
+            TypeManager typeManager,
+            boolean useParquetColumnNames)
     {
         requireNonNull(path, "path is null");
         checkArgument(totalBytes >= 0, "totalBytes is negative");
@@ -173,6 +176,13 @@ else if (type.equals(VarcharType.VARCHAR)) {
 
                 constantBlocks[columnIndex] = blockBuilder.build();
             }
+            else if (getParquetType(column, fileSchema, useParquetColumnNames) == null) {
+                BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), MAX_VECTOR_LENGTH);
+                for (int i = 0; i < MAX_VECTOR_LENGTH; i++) {
+                    blockBuilder.appendNull();
+                }
+                constantBlocks[columnIndex] = blockBuilder.build();
+            }
         }
         types = typesBuilder.build();
         columnNames = namesBuilder.build();

diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java b/presto-hive/src/main/java/com/facebook/presto/hive/parquet/ParquetPageSourceFactory.java
@@ -171,14 +171,16 @@ public static ParquetPageSource createParquetPageSource(
                     dataSource);
 
             return new ParquetPageSource(parquetReader,
+                    fileSchema,
                     requestedSchema,
                     path,
                     length,
                     schema,
                     columns,
                     partitionKeys,
                     effectivePredicate,
-                    typeManager);
+                    typeManager,
+                    useParquetColumnNames);
         }
         catch (Exception e) {
             try {