Enable GenericIndexed V2 for built-in(druid-io managed) complex colum…

…ns. (apache#3987) * Enable GenericIndexed V2 for complex columns. * SerializerBuilder to use GenericColumnSerializer.
Gamehu · Mar 1, 2017 · 91344cb · 91344cb
1 parent a08660a
commit 91344cb
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 5 deletions.
diff --git a/...ain/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java b/...ain/java/io/druid/query/aggregation/datasketches/theta/SketchMergeComplexMetricSerde.java
@@ -21,12 +21,15 @@
 
 import com.yahoo.sketches.theta.Sketch;
 import io.druid.data.input.InputRow;
+import io.druid.segment.GenericColumnSerializer;
 import io.druid.segment.column.ColumnBuilder;
 import io.druid.segment.data.GenericIndexed;
+import io.druid.segment.data.IOPeon;
 import io.druid.segment.data.ObjectStrategy;
 import io.druid.segment.serde.ComplexColumnPartSupplier;
 import io.druid.segment.serde.ComplexMetricExtractor;
 import io.druid.segment.serde.ComplexMetricSerde;
+import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
 
 import java.nio.ByteBuffer;
 
@@ -76,4 +79,10 @@ public ObjectStrategy<Sketch> getObjectStrategy()
     return strategy;
   }
 
+  @Override
+  public GenericColumnSerializer getSerializer(IOPeon peon, String column)
+  {
+    return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy());
+  }
+
 }
diff --git a/.../src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java b/.../src/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramFoldingSerde.java
@@ -21,12 +21,15 @@
 
 import com.google.common.collect.Ordering;
 import io.druid.data.input.InputRow;
+import io.druid.segment.GenericColumnSerializer;
 import io.druid.segment.column.ColumnBuilder;
 import io.druid.segment.data.GenericIndexed;
+import io.druid.segment.data.IOPeon;
 import io.druid.segment.data.ObjectStrategy;
 import io.druid.segment.serde.ComplexColumnPartSupplier;
 import io.druid.segment.serde.ComplexMetricExtractor;
 import io.druid.segment.serde.ComplexMetricSerde;
+import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
 
 import java.nio.ByteBuffer;
 import java.util.Iterator;
@@ -98,6 +101,12 @@ public void deserializeColumn(
     columnBuilder.setComplexColumn(new ComplexColumnPartSupplier(getTypeName(), column));
   }
 
+  @Override
+  public GenericColumnSerializer getSerializer(IOPeon peon, String column)
+  {
+    return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy());
+  }
+
   public ObjectStrategy getObjectStrategy()
   {
     return new ObjectStrategy<ApproximateHistogram>()

diff --git a/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java b/extensions-core/stats/src/main/java/io/druid/query/aggregation/variance/VarianceSerde.java
@@ -21,12 +21,15 @@
 
 import com.google.common.collect.Ordering;
 import io.druid.data.input.InputRow;
+import io.druid.segment.GenericColumnSerializer;
 import io.druid.segment.column.ColumnBuilder;
 import io.druid.segment.data.GenericIndexed;
+import io.druid.segment.data.IOPeon;
 import io.druid.segment.data.ObjectStrategy;
 import io.druid.segment.serde.ComplexColumnPartSupplier;
 import io.druid.segment.serde.ComplexMetricExtractor;
 import io.druid.segment.serde.ComplexMetricSerde;
+import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
 
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -118,4 +121,11 @@ public int compare(VarianceAggregatorCollector o1, VarianceAggregatorCollector o
       }
     };
   }
+
+  @Override
+  public GenericColumnSerializer getSerializer(IOPeon peon, String column)
+  {
+    return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy());
+  }
+
 }
diff --git a/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java b/processing/src/main/java/io/druid/query/aggregation/hyperloglog/HyperUniquesSerde.java
@@ -21,14 +21,17 @@
 
 import com.google.common.collect.Ordering;
 import io.druid.data.input.InputRow;
-import io.druid.hll.HyperLogLogHash;
 import io.druid.hll.HyperLogLogCollector;
+import io.druid.hll.HyperLogLogHash;
+import io.druid.segment.GenericColumnSerializer;
 import io.druid.segment.column.ColumnBuilder;
 import io.druid.segment.data.GenericIndexed;
+import io.druid.segment.data.IOPeon;
 import io.druid.segment.data.ObjectStrategy;
 import io.druid.segment.serde.ComplexColumnPartSupplier;
 import io.druid.segment.serde.ComplexMetricExtractor;
 import io.druid.segment.serde.ComplexMetricSerde;
+import io.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer;
 
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -141,4 +144,11 @@ public int compare(HyperLogLogCollector o1, HyperLogLogCollector o2)
       }
     };
   }
+
+  @Override
+  public GenericColumnSerializer getSerializer(IOPeon peon, String column)
+  {
+    return LargeColumnSupportedComplexColumnSerializer.create(peon, column, this.getObjectStrategy());
+  }
+
 }
diff --git a/processing/src/main/java/io/druid/segment/IndexMergerV9.java b/processing/src/main/java/io/druid/segment/IndexMergerV9.java
@@ -48,7 +48,6 @@
 import io.druid.segment.data.TmpFileIOPeon;
 import io.druid.segment.loading.MMappedQueryableSegmentizerFactory;
 import io.druid.segment.serde.ComplexColumnPartSerde;
-import io.druid.segment.serde.ComplexColumnSerializer;
 import io.druid.segment.serde.ComplexMetricSerde;
 import io.druid.segment.serde.ComplexMetrics;
 import io.druid.segment.serde.FloatGenericColumnPartSerde;
@@ -367,7 +366,7 @@ private void makeMetricsColumns(
           builder.setValueType(ValueType.COMPLEX);
           builder.addSerde(
               ComplexColumnPartSerde.serializerBuilder().withTypeName(typeName)
-                                    .withDelegate((ComplexColumnSerializer) writer)
+                                    .withDelegate(writer)
                                     .build()
           );
           break;

diff --git a/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java
@@ -22,6 +22,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.druid.java.util.common.io.smoosh.FileSmoosher;
+import io.druid.segment.GenericColumnSerializer;
 import io.druid.segment.column.ColumnBuilder;
 import io.druid.segment.column.ColumnConfig;
 import io.druid.segment.data.GenericIndexed;
@@ -92,15 +93,15 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo
   public static class SerializerBuilder
   {
     private String typeName = null;
-    private ComplexColumnSerializer delegate = null;
+    private GenericColumnSerializer delegate = null;
 
     public SerializerBuilder withTypeName(final String typeName)
     {
       this.typeName = typeName;
       return this;
     }
 
-    public SerializerBuilder withDelegate(final ComplexColumnSerializer delegate)
+    public SerializerBuilder withDelegate(final GenericColumnSerializer delegate)
     {
       this.delegate = delegate;
       return this;