Ability to filter on virtual columns. (apache#3942)

This didn't need much other than having BitmapIndexSelector return null from various methods to trigger cursor based filtering.
vogali · Feb 22, 2017 · a47206e · a47206e
1 parent f910c05
commit a47206e
Show file tree

Hide file tree

Showing 8 changed files with 127 additions and 5 deletions.
diff --git a/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java b/processing/src/main/java/io/druid/query/search/search/AutoStrategy.java
@@ -25,6 +25,7 @@
 import io.druid.segment.ColumnSelectorBitmapIndexSelector;
 import io.druid.segment.QueryableIndex;
 import io.druid.segment.Segment;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.BitmapIndex;
 import io.druid.segment.column.Column;
 
@@ -54,6 +55,7 @@ public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment seg
     if (index != null) {
       final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
           index.getBitmapFactoryForDimensions(),
+          VirtualColumns.EMPTY,
           index
       );
 

diff --git a/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java b/processing/src/main/java/io/druid/query/search/search/UseIndexesStrategy.java
@@ -37,6 +37,7 @@
 import io.druid.segment.QueryableIndex;
 import io.druid.segment.Segment;
 import io.druid.segment.StorageAdapter;
+import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.BitmapIndex;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ColumnCapabilities;
@@ -80,6 +81,7 @@ public List<SearchQueryExecutor> getExecutionPlan(SearchQuery query, Segment seg
       if (bitmapSuppDims.size() > 0) {
         final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
             index.getBitmapFactoryForDimensions(),
+            VirtualColumns.EMPTY,
             index
         );
 
@@ -153,6 +155,7 @@ static ImmutableBitmap makeTimeFilteredBitmap(
     } else {
       final BitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
           index.getBitmapFactoryForDimensions(),
+          VirtualColumns.EMPTY,
           index
       );
       Preconditions.checkArgument(filter.supportsBitmapIndex(selector), "filter[%s] should support bitmap", filter);

diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java
@@ -26,6 +26,7 @@
 import io.druid.query.filter.BitmapIndexSelector;
 import io.druid.segment.column.BitmapIndex;
 import io.druid.segment.column.Column;
+import io.druid.segment.column.ColumnCapabilities;
 import io.druid.segment.column.DictionaryEncodedColumn;
 import io.druid.segment.column.GenericColumn;
 import io.druid.segment.column.ValueType;
@@ -40,20 +41,28 @@
 public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector
 {
   private final BitmapFactory bitmapFactory;
+  private final VirtualColumns virtualColumns;
   private final ColumnSelector index;
 
   public ColumnSelectorBitmapIndexSelector(
       final BitmapFactory bitmapFactory,
+      final VirtualColumns virtualColumns,
       final ColumnSelector index
   )
   {
     this.bitmapFactory = bitmapFactory;
+    this.virtualColumns = virtualColumns;
     this.index = index;
   }
 
   @Override
   public Indexed<String> getDimensionValues(String dimension)
   {
+    if (isFilterableVirtualColumn(dimension)) {
+      // Virtual columns don't have dictionaries or indexes.
+      return null;
+    }
+
     final Column columnDesc = index.getColumn(dimension);
     if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) {
       return null;
@@ -110,6 +119,11 @@ public BitmapFactory getBitmapFactory()
   @Override
   public BitmapIndex getBitmapIndex(String dimension)
   {
+    if (isFilterableVirtualColumn(dimension)) {
+      // Virtual columns don't have dictionaries or indexes.
+      return null;
+    }
+
     final Column column = index.getColumn(dimension);
     if (column == null || !columnSupportsFiltering(column)) {
       // for missing columns and columns with types that do not support filtering,
@@ -172,6 +186,11 @@ public ImmutableBitmap getBitmap(int idx)
   @Override
   public ImmutableBitmap getBitmapIndex(String dimension, String value)
   {
+    if (isFilterableVirtualColumn(dimension)) {
+      // Virtual columns don't have dictionaries or indexes.
+      return null;
+    }
+
     final Column column = index.getColumn(dimension);
     if (column == null || !columnSupportsFiltering(column)) {
       if (Strings.isNullOrEmpty(value)) {
@@ -192,6 +211,10 @@ public ImmutableBitmap getBitmapIndex(String dimension, String value)
   @Override
   public ImmutableRTree getSpatialIndex(String dimension)
   {
+    if (isFilterableVirtualColumn(dimension)) {
+      return new ImmutableRTree();
+    }
+
     final Column column = index.getColumn(dimension);
     if (column == null || !column.getCapabilities().hasSpatialIndexes()) {
       return new ImmutableRTree();
@@ -200,6 +223,16 @@ public ImmutableRTree getSpatialIndex(String dimension)
     return column.getSpatialIndex().getRTree();
   }
 
+  private boolean isFilterableVirtualColumn(final String columnName)
+  {
+    final ColumnCapabilities columnCapabilities = virtualColumns.getColumnCapabilities(columnName);
+    if (columnCapabilities == null) {
+      return false;
+    } else {
+      return Filters.FILTERABLE_TYPES.contains(columnCapabilities.getType());
+    }
+  }
+
   private static boolean columnSupportsFiltering(Column column)
   {
     ValueType columnType = column.getCapabilities().getType();

diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java
@@ -226,6 +226,7 @@ public Sequence<Cursor> makeCursors(
 
     final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
         index.getBitmapFactoryForDimensions(),
+        virtualColumns,
         index
     );
 

diff --git a/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java b/processing/src/test/java/io/druid/query/select/SelectQueryRunnerTest.java
@@ -42,13 +42,16 @@
 import io.druid.query.extraction.JavaScriptExtractionFn;
 import io.druid.query.extraction.MapLookupExtractor;
 import io.druid.query.filter.AndDimFilter;
+import io.druid.query.filter.BoundDimFilter;
 import io.druid.query.filter.DimFilter;
 import io.druid.query.filter.SelectorDimFilter;
 import io.druid.query.lookup.LookupExtractionFn;
+import io.druid.query.ordering.StringComparators;
 import io.druid.query.spec.LegacySegmentSpec;
 import io.druid.query.spec.QuerySegmentSpec;
 import io.druid.segment.column.Column;
 import io.druid.segment.column.ValueType;
+import io.druid.segment.virtual.ExpressionVirtualColumn;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 import org.junit.Assert;
@@ -479,6 +482,59 @@ public void testFullOnSelectWithFilter()
     }
   }
 
+  @Test
+  public void testFullOnSelectWithFilterOnVirtualColumn()
+  {
+    SelectQuery query = newTestQuery()
+        .intervals("2011-01-13/2011-01-14")
+        .filters(
+            new AndDimFilter(
+                Arrays.asList(
+                    new SelectorDimFilter(QueryRunnerTestHelper.marketDimension, "spot", null),
+                    new BoundDimFilter("expr", "11.1", null, false, false, null, null, StringComparators.NUMERIC)
+                )
+            )
+        )
+        .granularity(QueryRunnerTestHelper.allGran)
+        .dimensionSpecs(DefaultDimensionSpec.toSpec(QueryRunnerTestHelper.qualityDimension))
+        .metrics(Lists.<String>newArrayList(QueryRunnerTestHelper.indexMetric))
+        .pagingSpec(new PagingSpec(null, 10, true))
+        .virtualColumns(new ExpressionVirtualColumn("expr", "index / 10.0"))
+        .build();
+
+    HashMap<String, Object> context = new HashMap<String, Object>();
+    Iterable<Result<SelectResultValue>> results = Sequences.toList(
+        runner.run(query, context),
+        Lists.<Result<SelectResultValue>>newArrayList()
+    );
+
+    final List<List<Map<String, Object>>> events = toEvents(
+        new String[]{
+            EventHolder.timestampKey + ":TIME",
+            null,
+            QueryRunnerTestHelper.qualityDimension + ":STRING",
+            null,
+            null,
+            QueryRunnerTestHelper.indexMetric + ":FLOAT"
+        },
+        // filtered values with all granularity
+        new String[]{
+            "2011-01-13T00:00:00.000Z	spot	health	preferred	hpreferred	114.947403",
+            "2011-01-13T00:00:00.000Z	spot	technology	preferred	tpreferred	111.356672"
+        }
+    );
+
+    PagingOffset offset = query.getPagingOffset(QueryRunnerTestHelper.segmentId);
+    List<Result<SelectResultValue>> expectedResults = toExpected(
+        events,
+        Lists.newArrayList("quality"),
+        Lists.<String>newArrayList("index"),
+        offset.startOffset(),
+        offset.threshold()
+    );
+    verify(expectedResults, results);
+  }
+
   @Test
   public void testSelectWithFilterLookupExtractionFn () {
 

diff --git a/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BaseFilterTest.java
@@ -54,6 +54,7 @@
 import io.druid.segment.QueryableIndexStorageAdapter;
 import io.druid.segment.StorageAdapter;
 import io.druid.segment.TestHelper;
+import io.druid.segment.VirtualColumn;
 import io.druid.segment.VirtualColumns;
 import io.druid.segment.column.ValueType;
 import io.druid.segment.data.BitmapSerdeFactory;
@@ -62,6 +63,7 @@
 import io.druid.segment.data.RoaringBitmapSerdeFactory;
 import io.druid.segment.incremental.IncrementalIndex;
 import io.druid.segment.incremental.IncrementalIndexStorageAdapter;
+import io.druid.segment.virtual.ExpressionVirtualColumn;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.Before;
@@ -79,6 +81,12 @@
 
 public abstract class BaseFilterTest
 {
+  private static final VirtualColumns VIRTUAL_COLUMNS = VirtualColumns.create(
+      ImmutableList.<VirtualColumn>of(
+          new ExpressionVirtualColumn("expr", "1.0 + 0.1")
+      )
+  );
+
   @Rule
   public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
@@ -291,15 +299,13 @@ private DimFilter maybeOptimize(final DimFilter dimFilter)
 
   private Sequence<Cursor> makeCursorSequence(final Filter filter)
   {
-    final Sequence<Cursor> cursors = adapter.makeCursors(
+    return adapter.makeCursors(
         filter,
         new Interval(JodaUtils.MIN_INSTANT, JodaUtils.MAX_INSTANT),
-        VirtualColumns.EMPTY,
+        VIRTUAL_COLUMNS,
         QueryGranularities.ALL,
         false
     );
-
-    return cursors;
   }
 
   /**
@@ -444,7 +450,7 @@ private List<String> selectColumnValuesMatchingFilterUsingRowBasedColumnSelector
     // Perform test
     final SettableSupplier<InputRow> rowSupplier = new SettableSupplier<>();
     final ValueMatcher matcher = makeFilter(filter).makeMatcher(
-        RowBasedColumnSelectorFactory.create(rowSupplier, rowSignature)
+        VIRTUAL_COLUMNS.wrap(RowBasedColumnSelectorFactory.create(rowSupplier, rowSignature))
     );
     final List<String> values = Lists.newArrayList();
     for (InputRow row : rows) {

diff --git a/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java b/processing/src/test/java/io/druid/segment/filter/BoundFilterTest.java
@@ -354,6 +354,20 @@ public void testNumericMatchTooStrict()
     );
   }
 
+  @Test
+  public void testNumericMatchVirtualColumn()
+  {
+    assertFilterMatches(
+        new BoundDimFilter("expr", "1", "2", false, false, false, null, StringComparators.NUMERIC),
+        ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7")
+    );
+
+    assertFilterMatches(
+        new BoundDimFilter("expr", "2", "3", false, false, false, null, StringComparators.NUMERIC),
+        ImmutableList.<String>of()
+    );
+  }
+
   @Test
   public void testNumericMatchExactlySingleValue()
   {

diff --git a/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SelectorFilterTest.java
@@ -144,6 +144,13 @@ public void testMissingColumnNotSpecifiedInDimensionList()
     assertFilterMatches(new SelectorDimFilter("dim4", "c", null), ImmutableList.<String>of());
   }
 
+  @Test
+  public void testExpressionVirtualColumn()
+  {
+    assertFilterMatches(new SelectorDimFilter("expr", "1.1", null), ImmutableList.of("0", "1", "2", "3", "4", "5"));
+    assertFilterMatches(new SelectorDimFilter("expr", "1.2", null), ImmutableList.<String>of());
+  }
+
   @Test
   public void testSelectorWithLookupExtractionFn()
   {
-Original file line number
+Diff line change
@@ Expand Up / @@ -226,6 +226,7 @@ public Sequence<Cursor> makeCursors( @@
         final ColumnSelectorBitmapIndexSelector selector = new ColumnSelectorBitmapIndexSelector(
             index.getBitmapFactoryForDimensions(),
+            virtualColumns,
             index
         );
@@ Expand Down @@