Add PostAggregators to generator cache keys for top-n queries (apache…

…#3899) * Add PostAggregators to generator cache keys for top-n queries * Add tests for strings * Remove debug comments * Add type keys and list sizes to cache key * Make post aggregators used for sort are considered for cache key generation * Use assertArrayEquals() * Improve findPostAggregatorsForSort() * Address comments * fix test failure * address comments
Gamehu · Feb 13, 2017 · 991e285 · 991e285
1 parent 8e31a46
commit 991e285
Show file tree

Hide file tree

Showing 49 changed files with 1,256 additions and 315 deletions.
diff --git a/common/src/main/java/io/druid/common/utils/StringUtils.java b/common/src/main/java/io/druid/common/utils/StringUtils.java
@@ -23,7 +23,7 @@
  */
 public class StringUtils extends io.druid.java.util.common.StringUtils
 {
-  private static final byte[] EMPTY_BYTES = new byte[0];
+  public static final byte[] EMPTY_BYTES = new byte[0];
 
   // should be used only for estimation
   // returns the same result with StringUtils.fromUtf8(value).length for valid string values

diff --git a/...main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java b/...main/java/io/druid/query/aggregation/datasketches/theta/SketchEstimatePostAggregator.java
@@ -26,6 +26,8 @@
 import com.google.common.collect.Sets;
 import com.google.common.primitives.Doubles;
 import io.druid.query.aggregation.PostAggregator;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Comparator;
 import java.util.Map;
@@ -147,4 +149,12 @@ public int hashCode()
     result = 31 * result + (errorBoundsStdDev != null ? errorBoundsStdDev.hashCode() : 0);
     return result;
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    final CacheKeyBuilder builder = new CacheKeyBuilder(PostAggregatorIds.DATA_SKETCHES_SKETCH_ESTIMATE)
+        .appendCacheable(field);
+    return errorBoundsStdDev == null ? builder.build() : builder.appendInt(errorBoundsStdDev).build();
+  }
 }
diff --git a/.../src/main/java/io/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java b/.../src/main/java/io/druid/query/aggregation/datasketches/theta/SketchSetPostAggregator.java
@@ -24,8 +24,9 @@
 import com.google.common.collect.Sets;
 import com.yahoo.sketches.Util;
 import io.druid.java.util.common.IAE;
-import io.druid.java.util.common.logger.Logger;
 import io.druid.query.aggregation.PostAggregator;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Comparator;
 import java.util.List;
@@ -34,9 +35,6 @@
 
 public class SketchSetPostAggregator implements PostAggregator
 {
-
-  private static final Logger LOG = new Logger(SketchSetPostAggregator.class);
-
   private final String name;
   private final List<PostAggregator> fields;
   private final SketchHolder.Func func;
@@ -163,4 +161,33 @@ public int hashCode()
     result = 31 * result + maxSketchSize;
     return result;
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    final CacheKeyBuilder builder = new CacheKeyBuilder(PostAggregatorIds.DATA_SKETCHES_SKETCH_SET)
+        .appendString(getFunc())
+        .appendInt(maxSketchSize);
+
+    if (preserveFieldOrderInCacheKey(func)) {
+      builder.appendCacheables(fields);
+    } else {
+      builder.appendCacheablesIgnoringOrder(fields);
+    }
+
+    return builder.build();
+  }
+
+  private static boolean preserveFieldOrderInCacheKey(SketchHolder.Func func)
+  {
+    switch (func) {
+      case NOT:
+        return true;
+      case UNION:
+      case INTERSECT:
+        return false;
+      default:
+        throw new IAE(func.name());
+    }
+  }
 }
diff --git a/...rc/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregator.java b/...rc/main/java/io/druid/query/aggregation/histogram/ApproximateHistogramPostAggregator.java
@@ -29,8 +29,8 @@ public abstract class ApproximateHistogramPostAggregator implements PostAggregat
 {
   private static final Comparator COMPARATOR = ApproximateHistogramAggregator.COMPARATOR;
 
-  private final String name;
-  private final String fieldName;
+  protected final String name;
+  protected final String fieldName;
 
   public ApproximateHistogramPostAggregator(
       String name,

diff --git a/...e/histogram/src/main/java/io/druid/query/aggregation/histogram/BucketsPostAggregator.java b/...e/histogram/src/main/java/io/druid/query/aggregation/histogram/BucketsPostAggregator.java
@@ -23,8 +23,9 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Sets;
-
 import io.druid.java.util.common.IAE;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Map;
 import java.util.Set;
@@ -35,8 +36,6 @@ public class BucketsPostAggregator extends ApproximateHistogramPostAggregator
   private final float bucketSize;
   private final float offset;
 
-  private String fieldName;
-
   @JsonCreator
   public BucketsPostAggregator(
       @JsonProperty("name") String name,
@@ -51,7 +50,6 @@ public BucketsPostAggregator(
       throw new IAE("Illegal bucketSize [%s], must be > 0", this.bucketSize);
     }
     this.offset = offset;
-    this.fieldName = fieldName;
   }
 
   @Override
@@ -63,7 +61,7 @@ public Set<String> getDependentFields()
   @Override
   public Object compute(Map<String, Object> values)
   {
-    ApproximateHistogram ah = (ApproximateHistogram) values.get(this.getFieldName());
+    ApproximateHistogram ah = (ApproximateHistogram) values.get(fieldName);
     return ah.toHistogram(bucketSize, offset);
   }
 
@@ -89,4 +87,14 @@ public String toString()
            ", offset=" + this.getOffset() +
            '}';
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return new CacheKeyBuilder(PostAggregatorIds.HISTOGRAM_BUCKETS)
+        .appendString(fieldName)
+        .appendFloat(bucketSize)
+        .appendFloat(offset)
+        .build();
+  }
 }
diff --git a/...ogram/src/main/java/io/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java b/...ogram/src/main/java/io/druid/query/aggregation/histogram/CustomBucketsPostAggregator.java
@@ -23,6 +23,8 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Sets;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -55,7 +57,7 @@ public Set<String> getDependentFields()
   @Override
   public Object compute(Map<String, Object> values)
   {
-    ApproximateHistogram ah = (ApproximateHistogram) values.get(this.getFieldName());
+    ApproximateHistogram ah = (ApproximateHistogram) values.get(fieldName);
     return ah.toHistogram(breaks);
   }
 
@@ -74,4 +76,13 @@ public String toString()
            ", breaks=" + Arrays.toString(this.getBreaks()) +
            '}';
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return new CacheKeyBuilder(PostAggregatorIds.HISTOGRAM_CUSTOM_BUCKETS)
+        .appendString(fieldName)
+        .appendFloatArray(breaks)
+        .build();
+  }
 }
diff --git a/...togram/src/main/java/io/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java b/...togram/src/main/java/io/druid/query/aggregation/histogram/EqualBucketsPostAggregator.java
@@ -23,8 +23,9 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Sets;
-
 import io.druid.java.util.common.IAE;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Map;
 import java.util.Set;
@@ -33,7 +34,6 @@
 public class EqualBucketsPostAggregator extends ApproximateHistogramPostAggregator
 {
   private final int numBuckets;
-  private String fieldName;
 
   @JsonCreator
   public EqualBucketsPostAggregator(
@@ -47,7 +47,6 @@ public EqualBucketsPostAggregator(
     if (this.numBuckets <= 1) {
       throw new IAE("Illegal number of buckets[%s], must be > 1", this.numBuckets);
     }
-    this.fieldName = fieldName;
   }
 
   @Override
@@ -59,7 +58,7 @@ public Set<String> getDependentFields()
   @Override
   public Object compute(Map<String, Object> values)
   {
-    ApproximateHistogram ah = (ApproximateHistogram) values.get(this.getFieldName());
+    ApproximateHistogram ah = (ApproximateHistogram) values.get(fieldName);
     return ah.toHistogram(numBuckets);
   }
 
@@ -73,9 +72,18 @@ public int getNumBuckets()
   public String toString()
   {
     return "EqualBucketsPostAggregator{" +
-           "name='" + this.getName() + '\'' +
-           ", fieldName='" + this.getFieldName() + '\'' +
-           ", numBuckets=" + this.getNumBuckets() +
+           "name='" + name + '\'' +
+           ", fieldName='" + fieldName + '\'' +
+           ", numBuckets=" + numBuckets +
            '}';
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return new CacheKeyBuilder(PostAggregatorIds.HISTOGRAM_EQUAL_BUCKETS)
+        .appendString(fieldName)
+        .appendInt(numBuckets)
+        .build();
+  }
 }
diff --git a/...-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MaxPostAggregator.java b/...-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MaxPostAggregator.java
@@ -23,6 +23,8 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Sets;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Comparator;
 import java.util.Map;
@@ -40,16 +42,13 @@ public int compare(Object o, Object o1)
     }
   };
 
-  private String fieldName;
-
   @JsonCreator
   public MaxPostAggregator(
       @JsonProperty("name") String name,
       @JsonProperty("fieldName") String fieldName
   )
   {
     super(name, fieldName);
-    this.fieldName = fieldName;
   }
 
   @Override
@@ -67,7 +66,7 @@ public Set<String> getDependentFields()
   @Override
   public Object compute(Map<String, Object> values)
   {
-    final ApproximateHistogram ah = (ApproximateHistogram) values.get(this.getFieldName());
+    final ApproximateHistogram ah = (ApproximateHistogram) values.get(fieldName);
     return ah.getMax();
   }
 
@@ -78,4 +77,12 @@ public String toString()
            "fieldName='" + fieldName + '\'' +
            '}';
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return new CacheKeyBuilder(PostAggregatorIds.HISTOGRAM_MAX)
+        .appendString(fieldName)
+        .build();
+  }
 }
diff --git a/...-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MinPostAggregator.java b/...-core/histogram/src/main/java/io/druid/query/aggregation/histogram/MinPostAggregator.java
@@ -23,6 +23,8 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Sets;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Comparator;
 import java.util.Map;
@@ -40,16 +42,13 @@ public int compare(Object o, Object o1)
     }
   };
 
-  private String fieldName;
-
   @JsonCreator
   public MinPostAggregator(
       @JsonProperty("name") String name,
       @JsonProperty("fieldName") String fieldName
   )
   {
     super(name, fieldName);
-    this.fieldName = fieldName;
   }
 
   @Override
@@ -67,7 +66,7 @@ public Set<String> getDependentFields()
   @Override
   public Object compute(Map<String, Object> values)
   {
-    final ApproximateHistogram ah = (ApproximateHistogram) values.get(this.getFieldName());
+    final ApproximateHistogram ah = (ApproximateHistogram) values.get(fieldName);
     return ah.getMin();
   }
 
@@ -78,4 +77,12 @@ public String toString()
            "fieldName='" + fieldName + '\'' +
            '}';
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return new CacheKeyBuilder(PostAggregatorIds.HISTOGRAM_MIN)
+        .appendString(fieldName)
+        .build();
+  }
 }
diff --git a/.../histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilePostAggregator.java b/.../histogram/src/main/java/io/druid/query/aggregation/histogram/QuantilePostAggregator.java
@@ -23,8 +23,9 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Sets;
-
 import io.druid.java.util.common.IAE;
+import io.druid.query.aggregation.post.PostAggregatorIds;
+import io.druid.query.cache.CacheKeyBuilder;
 
 import java.util.Comparator;
 import java.util.Map;
@@ -43,7 +44,7 @@ public int compare(Object o, Object o1)
   };
 
   private final float probability;
-  private String fieldName;
+  private final String fieldName;
 
   @JsonCreator
   public QuantilePostAggregator(
@@ -76,8 +77,8 @@ public Set<String> getDependentFields()
   @Override
   public Object compute(Map<String, Object> values)
   {
-    final ApproximateHistogram ah = (ApproximateHistogram) values.get(this.getFieldName());
-    return ah.getQuantiles(new float[]{this.getProbability()})[0];
+    final ApproximateHistogram ah = (ApproximateHistogram) values.get(fieldName);
+    return ah.getQuantiles(new float[]{probability})[0];
   }
 
   @JsonProperty
@@ -120,4 +121,13 @@ public String toString()
            ", fieldName='" + fieldName + '\'' +
            '}';
   }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return new CacheKeyBuilder(PostAggregatorIds.HISTOGRAM_QUANTILE)
+        .appendString(fieldName)
+        .appendFloat(probability)
+        .build();
+  }
 }