diff --git a/NOTICE b/NOTICE index 2c1e672e8e2e..1f91649e5559 100644 --- a/NOTICE +++ b/NOTICE @@ -36,3 +36,11 @@ This product contains a modified version of TestNG 6.8.7 * http://testng.org/license/ (Apache License, Version 2.0) * HOMEPAGE: * http://testng.org/ + +This product contains a modified version of Metamarkets bytebuffer-collections library + * LICENSE: + * https://github.com/metamx/bytebuffer-collections/blob/master/LICENSE (Apache License, Version 2.0) + * HOMEPAGE: + * https://github.com/metamx/bytebuffer-collections + * COMMIT TAG: + * https://github.com/metamx/bytebuffer-collections/commit/3d1e7c8 diff --git a/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java index fef3d6a8b7ab..adba83188fe7 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/BoundFilterBenchmark.java @@ -19,14 +19,32 @@ package io.druid.benchmark; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; -import com.metamx.collections.spatial.ImmutableRTree; + +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.BoundDimFilter; import io.druid.query.ordering.StringComparators; @@ -38,22 +56,6 @@ import io.druid.segment.filter.BoundFilter; import io.druid.segment.serde.BitmapIndexColumnPartSupplier; import it.uniroma3.mat.extendedset.intset.ConciseSetUtils; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; @State(Scope.Benchmark) @Fork(value = 1) diff --git a/benchmarks/src/main/java/io/druid/benchmark/ConciseComplementBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/ConciseComplementBenchmark.java index dc69035acf96..feb8f0f4ba8f 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/ConciseComplementBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/ConciseComplementBenchmark.java @@ -20,7 +20,8 @@ package io.druid.benchmark; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import java.util.concurrent.TimeUnit; + import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Mode; @@ -30,7 +31,7 @@ import org.openjdk.jmh.annotations.State; import org.openjdk.jmh.infra.Blackhole; -import java.util.concurrent.TimeUnit; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; @State(Scope.Benchmark) public class ConciseComplementBenchmark diff --git a/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java index 424c003e3691..48799cd01676 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/DimensionPredicateFilterBenchmark.java @@ -23,11 +23,11 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.collect.FluentIterable; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.DruidLongPredicate; import io.druid.query.filter.DruidPredicateFactory; diff --git a/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java b/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java index def4f51f50ef..a5d0e54b2101 100644 --- a/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java +++ b/benchmarks/src/main/java/io/druid/benchmark/LikeFilterBenchmark.java @@ -21,11 +21,11 @@ import com.google.common.base.Function; import com.google.common.collect.FluentIterable; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.BoundDimFilter; import io.druid.query.filter.Filter; diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html new file mode 100755 index 000000000000..eba65d3d3c18 --- /dev/null +++ b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.html @@ -0,0 +1,145 @@ + + + + + + + Benchmark results for methods in class RangeBitmapBenchmarkTest + + + + + + + + + + +
+

Benchmark results for methods in class RangeBitmapBenchmarkTest

+ +
+
+ +
+ + +
+
+ + + + diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp new file mode 100755 index 000000000000..17306131437b --- /dev/null +++ b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.RangeBitmapBenchmarkTest.jsonp @@ -0,0 +1,22 @@ +receiveJsonpData({ +"cols": [ +{"label": "Run", "type": "string"}, +{"label": "Custom key", "type": "string"}, +{"label": "Timestamp", "type": "string"}, +{"label": "timeConciseUnion", "type": "string"} , +{"label": "timeGenericConciseIntersection", "type": "string"} , +{"label": "timeGenericConciseUnion", "type": "string"} , +{"label": "timeGenericRoaringIntersection", "type": "string"} , +{"label": "timeGenericRoaringUnion", "type": "string"} , +{"label": "timeImmutableRoaringUnion", "type": "string"} , +{"label": "timeOffheapConciseUnion", "type": "string"} , +{"label": "timeOffheapRoaringUnion", "type": "string"} , +{"label": "timeRoaringUnion", "type": "string"} ], +"rows": [ +{"c": [{"v": "5"}, {"v": "0.00001"}, {"v": "2014-11-04 14:03:04.268"}, {"v": 80.304}, {"v": 79.758}, {"v": 65.896}, {"v": 0.008}, {"v": 0.278}, {"v": 0.596}, {"v": 70.89}, {"v": 0.275}, {"v": 0.202}]}, +{"c": [{"v": "1"}, {"v": "0.0001"}, {"v": "2014-11-04 13:32:21.752"}, {"v": 30.843}, {"v": 30.863}, {"v": 32.306}, {"v": 0.012}, {"v": 0.272}, {"v": 0.546}, {"v": 32.727}, {"v": 0.327}, {"v": 0.158}]}, +{"c": [{"v": "2"}, {"v": "0.0010"}, {"v": "2014-11-04 13:41:55.608"}, {"v": 3.801}, {"v": 3.441}, {"v": 3.421}, {"v": 0.019}, {"v": 0.272}, {"v": 0.524}, {"v": 3.76}, {"v": 0.271}, {"v": 0.171}]}, +{"c": [{"v": "3"}, {"v": "0.0100"}, {"v": "2014-11-04 13:45:36.077"}, {"v": 0.341}, {"v": 0.541}, {"v": 0.628}, {"v": 0.025}, {"v": 0.276}, {"v": 0.576}, {"v": 0.352}, {"v": 0.276}, {"v": 0.263}]}, +{"c": [{"v": "4"}, {"v": "0.1000"}, {"v": "2014-11-04 13:49:27.509"}, {"v": 0.051}, {"v": 0.062}, {"v": 0.046}, {"v": 0.039}, {"v": 0.295}, {"v": 0.47}, {"v": 0.045}, {"v": 0.299}, {"v": 0.181}]}, +{"c": [{"v": "38"}, {"v": "0.25000"}, {"v": "2014-11-04 15:22:42.446"}, {"v": 0.036}, {"v": 0.041}, {"v": 0.04}, {"v": 0.052}, {"v": 0.179}, {"v": 0.391}, {"v": 0.042}, {"v": 0.18}, {"v": 0.105}]} +]}); diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html new file mode 100755 index 000000000000..a6dc08ad64fd --- /dev/null +++ b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.html @@ -0,0 +1,145 @@ + + + + + + + Benchmark results for methods in class UniformBitmapBenchmarkTest + + + + + + + + + + +
+

Benchmark results for methods in class UniformBitmapBenchmarkTest

+ +
+
+ +
+ + +
+
+ + + + diff --git a/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp new file mode 100755 index 000000000000..1194215e2289 --- /dev/null +++ b/bytebuffer-collections/benchmarks/io.druid.collections.bitmap.UniformBitmapBenchmarkTest.jsonp @@ -0,0 +1,23 @@ +receiveJsonpData({ +"cols": [ +{"label": "Run", "type": "string"}, +{"label": "Custom key", "type": "string"}, +{"label": "Timestamp", "type": "string"}, +{"label": "timeConciseUnion", "type": "string"} , +{"label": "timeGenericConciseIntersection", "type": "string"} , +{"label": "timeGenericConciseUnion", "type": "string"} , +{"label": "timeGenericRoaringIntersection", "type": "string"} , +{"label": "timeGenericRoaringUnion", "type": "string"} , +{"label": "timeImmutableRoaringUnion", "type": "string"} , +{"label": "timeOffheapConciseUnion", "type": "string"} , +{"label": "timeOffheapRoaringUnion", "type": "string"} , +{"label": "timeRoaringUnion", "type": "string"} ], +"rows": [ +{"c": [{"v": "6"}, {"v": "0.0001"}, {"v": "2014-11-04 11:41:24.142"}, {"v": 1.099}, {"v": 0.401}, {"v": 1.391}, {"v": 0.02}, {"v": 0.131}, {"v": 0.104}, {"v": 1.171}, {"v": 0.132}, {"v": 0.091}]}, +{"c": [{"v": "5"}, {"v": "0.0010"}, {"v": "2014-11-04 11:37:12.305"}, {"v": 6.989}, {"v": 0.595}, {"v": 7.4}, {"v": 0.026}, {"v": 0.144}, {"v": 0.098}, {"v": 7.95}, {"v": 0.139}, {"v": 0.066}]}, +{"c": [{"v": "4"}, {"v": "0.0100"}, {"v": "2014-11-04 11:23:42.26"}, {"v": 50.259}, {"v": 4.768}, {"v": 51.716}, {"v": 0.053}, {"v": 0.563}, {"v": 0.223}, {"v": 54.117}, {"v": 0.59}, {"v": 0.175}]}, +{"c": [{"v": "8"}, {"v": "0.1000"}, {"v": "2014-11-04 12:19:56.926"}, {"v": 64.505}, {"v": 23.741}, {"v": 63.565}, {"v": 0.031}, {"v": 0.353}, {"v": 0.528}, {"v": 59.636}, {"v": 0.352}, {"v": 0.155}]}, +{"c": [{"v": "7"}, {"v": "0.2500"}, {"v": "2014-11-04 11:52:42.488"}, {"v": 67.57}, {"v": 64.276}, {"v": 60.747}, {"v": 0.021}, {"v": 0.275}, {"v": 0.523}, {"v": 69.835}, {"v": 0.251}, {"v": 0.178}]}, +{"c": [{"v": "2"}, {"v": "0.5000"}, {"v": "2014-11-04 10:19:33.921"}, {"v": 66.058}, {"v": 67.714}, {"v": 64.162}, {"v": 0.026}, {"v": 0.264}, {"v": 0.541}, {"v": 66.445}, {"v": 0.281}, {"v": 0.168}]}, +{"c": [{"v": "3"}, {"v": "0.7500"}, {"v": "2014-11-04 10:44:45.546"}, {"v": 65.028}, {"v": 70.115}, {"v": 63.475}, {"v": 0.027}, {"v": 0.284}, {"v": 0.574}, {"v": 68.909}, {"v": 0.295}, {"v": 0.195}]} +]}); diff --git a/bytebuffer-collections/pom.xml b/bytebuffer-collections/pom.xml new file mode 100755 index 000000000000..7a11fed8b8e1 --- /dev/null +++ b/bytebuffer-collections/pom.xml @@ -0,0 +1,135 @@ + + + + + 4.0.0 + + + io.druid + druid + 0.9.3-SNAPSHOT + + + bytebuffer-collections + bytebuffer-collections + ByteBuffer Collections + + + + com.metamx + extendedset + 1.3.10 + + + com.google.guava + guava + 16.0.1 + + + com.fasterxml.jackson.core + jackson-annotations + 2.4.6 + + + com.fasterxml.jackson.core + jackson-core + 2.4.6 + + + com.fasterxml.jackson.core + jackson-databind + 2.4.6 + + + org.roaringbitmap + RoaringBitmap + 0.5.18 + + + + + junit + junit + test + + + org.easymock + easymock + 3.0 + test + + + com.carrotsearch + junit-benchmarks + 0.7.2 + test + + + com.h2database + h2 + 1.4.182 + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + io.druid.test.annotation.Benchmark + + + + + + + + benchmark + + + + maven-surefire-plugin + + -server -Xms3G -Xmx3G -Djub.consumers=CONSOLE,H2 -Djub.db.file=benchmarks/benchmarks + io.druid.test.annotation.Benchmark + io.druid.test.annotation.Dummy + + + + + + + diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/IntegerSet.java b/bytebuffer-collections/src/main/java/io/druid/collections/IntegerSet.java new file mode 100755 index 000000000000..0509f30fdc64 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/IntegerSet.java @@ -0,0 +1,204 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections; + +import com.google.common.collect.Sets; +import io.druid.collections.bitmap.MutableBitmap; +import org.roaringbitmap.IntIterator; + +import java.util.Collection; +import java.util.Iterator; +import java.util.Set; + +/** + * + */ +public class IntegerSet implements Set +{ + private final MutableBitmap mutableBitmap; + + private IntegerSet(MutableBitmap mutableBitmap) + { + this.mutableBitmap = mutableBitmap; + } + + public static IntegerSet wrap(MutableBitmap mutableBitmap) + { + return new IntegerSet(mutableBitmap); + } + + @Override + public int size() + { + return this.mutableBitmap.size(); + } + + @Override + public boolean isEmpty() + { + return this.mutableBitmap.isEmpty(); + } + + @Override + public boolean contains(Object o) + { + if (o instanceof Integer) { + return mutableBitmap.get((Integer) o); + } else if (o instanceof Long) { + return this.contains(((Long) o).intValue()); + } + return false; + } + + @Override + public Iterator iterator() + { + return new BitSetIterator(mutableBitmap); + } + + @Override + public Object[] toArray() + { + Integer[] retval = new Integer[mutableBitmap.size()]; + int pos = 0; + for (Integer i : this) { + retval[pos++] = i; + } + return retval; + } + + @Override + public T[] toArray(T[] a) + { + return Sets.newHashSet(this).toArray(a); + } + + @Override + public boolean add(Integer integer) + { + if (null == integer) { + throw new NullPointerException("BitSet cannot contain null values"); + } + if (integer < 0) { + throw new IllegalArgumentException("Only positive integers or zero can be added"); + } + boolean isSet = mutableBitmap.get(integer); + mutableBitmap.add(integer.intValue()); + return !isSet; + } + + @Override + public boolean remove(Object o) + { + if (o == null) { + throw new NullPointerException("BitSet cannot contain null values"); + } + if (o instanceof Integer) { + Integer integer = (Integer) o; + boolean isSet = mutableBitmap.get(integer); + mutableBitmap.remove(integer); + return isSet; + } else { + throw new ClassCastException("Cannot remove non Integer from integer BitSet"); + } + } + + @Override + public boolean containsAll(Collection c) + { + Iterator it = c.iterator(); + while (it.hasNext()) { + if (!this.contains(it.next())) { + return false; + } + } + return true; + } + + @Override + public boolean addAll(Collection c) + { + boolean setChanged = false; + for (Integer i : c) { + if (!this.contains(i)) { + setChanged = true; + this.add(i); + } + } + return setChanged; + } + + @Override + public boolean retainAll(Collection c) + { + // Stub + throw new UnsupportedOperationException("Cannot retainAll ona an IntegerSet"); + } + + @Override + public boolean removeAll(Collection c) + { + Iterator it = c.iterator(); + boolean changed = false; + while (it.hasNext()) { + Integer val = (Integer) it.next(); + changed = remove(val) || changed; + } + return changed; + } + + @Override + public void clear() + { + mutableBitmap.clear(); + } + + public static class BitSetIterator implements Iterator + { + private final IntIterator intIt; + private final MutableBitmap bitSet; + private Integer prior = null; + + public BitSetIterator(MutableBitmap bitSet) + { + this.intIt = bitSet.iterator(); + this.bitSet = bitSet; + } + + @Override + public boolean hasNext() + { + return intIt.hasNext(); + } + + @Override + public Integer next() + { + prior = intIt.next(); + return prior; + } + + @Override + public void remove() + { + bitSet.remove(prior); + } + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java new file mode 100755 index 000000000000..acb59511109e --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitSetBitmapFactory.java @@ -0,0 +1,98 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.nio.ByteBuffer; +import java.util.BitSet; + +/** + * BitSetBitmapFactory implements BitmapFactory as a wrapper for java.util.BitSet + */ +public class BitSetBitmapFactory implements BitmapFactory +{ + @Override + public MutableBitmap makeEmptyMutableBitmap() + { + return new WrappedBitSetBitmap(); + } + + @Override + public ImmutableBitmap makeEmptyImmutableBitmap() + { + return makeEmptyMutableBitmap(); + } + + @Override + public ImmutableBitmap makeImmutableBitmap(MutableBitmap mutableBitmap) + { + return mutableBitmap; + } + + @Override + public ImmutableBitmap mapImmutableBitmap(ByteBuffer b) + { + return new WrappedBitSetBitmap(BitSet.valueOf(b.array())); + } + + @Override + public ImmutableBitmap union(Iterable b) + { + WrappedBitSetBitmap newSet = null; + for (ImmutableBitmap bm : b) { + if (null == newSet) { + newSet = new WrappedBitSetBitmap(((WrappedBitSetBitmap) bm).cloneBitSet()); + } else { + newSet.union(bm); + } + } + return newSet; + } + + @Override + public ImmutableBitmap intersection(Iterable b) + { + + WrappedBitSetBitmap newSet = null; + for (ImmutableBitmap bm : b) { + if (null == newSet) { + newSet = new WrappedBitSetBitmap(((WrappedBitSetBitmap) bm).cloneBitSet()); + } else { + newSet.intersection(bm); + } + } + return newSet; + } + + @Override + public ImmutableBitmap complement(ImmutableBitmap b) + { + BitSet bitSet = ((WrappedBitSetBitmap) b).cloneBitSet(); + bitSet.flip(0, bitSet.size()); + return new WrappedBitSetBitmap(bitSet); + } + + @Override + public ImmutableBitmap complement( + ImmutableBitmap b, int length + ) + { + return null; + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitmapFactory.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitmapFactory.java new file mode 100755 index 000000000000..21c034a133a9 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/BitmapFactory.java @@ -0,0 +1,83 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.nio.ByteBuffer; + +public interface BitmapFactory +{ + /** + * Create a new empty bitmap + * + * @return the new bitmap + */ + public MutableBitmap makeEmptyMutableBitmap(); + + public ImmutableBitmap makeEmptyImmutableBitmap(); + + public ImmutableBitmap makeImmutableBitmap(MutableBitmap mutableBitmap); + + /** + * Given a ByteBuffer pointing at a serialized version of a bitmap, + * instantiate an immutable mapped bitmap. + * + * When using RoaringBitmap (with the RoaringBitmapFactory class), it is not + * necessary for b.limit() to indicate the end of the serialized content + * whereas it is critical to set b.limit() appropriately with ConciseSet (with + * the ConciseBitmapFactory). + * + * @param b the input byte buffer + * + * @return the new bitmap + */ + public ImmutableBitmap mapImmutableBitmap(ByteBuffer b); + + /** + * Compute the union (bitwise-OR) of a set of bitmaps. They are assumed to be + * instances of of the proper WrappedConciseBitmap otherwise a ClassCastException + * is thrown. + * + * @param b input ImmutableGenericBitmap objects + * + * @return the union. + * + * @throws ClassCastException if one of the ImmutableGenericBitmap objects if not an instance + * of WrappedImmutableConciseBitmap + */ + public ImmutableBitmap union(Iterable b); + + /** + * Compute the intersection (bitwise-AND) of a set of bitmaps. They are assumed to be + * instances of of the proper WrappedConciseBitmap otherwise a ClassCastException + * is thrown. + * + * @param b input ImmutableGenericBitmap objects + * + * @return the union. + * + * @throws ClassCastException if one of the ImmutableGenericBitmap objects if not an instance + * of WrappedImmutableConciseBitmap + */ + public ImmutableBitmap intersection(Iterable b); + + public ImmutableBitmap complement(ImmutableBitmap b); + + public ImmutableBitmap complement(ImmutableBitmap b, int length); +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java new file mode 100755 index 000000000000..679f53316828 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ConciseBitmapFactory.java @@ -0,0 +1,138 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.nio.ByteBuffer; +import java.util.Iterator; + +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +/** + * As the name suggests, this class instantiates bitmaps of the types + * WrappedConciseBitmap and WrappedImmutableConciseBitmap. + */ +public class ConciseBitmapFactory implements BitmapFactory +{ + private static final ImmutableConciseSet EMPTY_IMMUTABLE_BITMAP = new ImmutableConciseSet(); + private static final WrappedImmutableConciseBitmap WRAPPED_IMMUTABLE_CONCISE_BITMAP = + new WrappedImmutableConciseBitmap(EMPTY_IMMUTABLE_BITMAP); + + private static Iterable unwrap( + final Iterable b + ) + { + return new Iterable() + { + @Override + public Iterator iterator() + { + final Iterator i = b.iterator(); + return new Iterator() + { + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNext() + { + return i.hasNext(); + } + + @Override + public ImmutableConciseSet next() + { + final WrappedImmutableConciseBitmap wrappedBitmap = (WrappedImmutableConciseBitmap) i.next(); + + if (wrappedBitmap == null) { + return EMPTY_IMMUTABLE_BITMAP; + } + + return wrappedBitmap.getBitmap(); + } + }; + } + }; + } + + @Override + public MutableBitmap makeEmptyMutableBitmap() + { + return new WrappedConciseBitmap(); + } + + @Override + public ImmutableBitmap makeEmptyImmutableBitmap() + { + return WRAPPED_IMMUTABLE_CONCISE_BITMAP; + } + + @Override + public ImmutableBitmap makeImmutableBitmap(MutableBitmap mutableBitmap) + { + if (!(mutableBitmap instanceof WrappedConciseBitmap)) { + throw new IllegalStateException(String.format("Cannot convert [%s]", mutableBitmap.getClass())); + } + return new WrappedImmutableConciseBitmap( + ImmutableConciseSet.newImmutableFromMutable( + ((WrappedConciseBitmap) mutableBitmap).getBitmap() + ) + ); + } + + @Override + public ImmutableBitmap mapImmutableBitmap(ByteBuffer b) + { + return new WrappedImmutableConciseBitmap(b); + } + + @Override + public ImmutableBitmap union(Iterable b) + throws ClassCastException + { + return new WrappedImmutableConciseBitmap(ImmutableConciseSet.union(unwrap(b))); + } + + @Override + public ImmutableBitmap intersection(Iterable b) + throws ClassCastException + { + return new WrappedImmutableConciseBitmap(ImmutableConciseSet.intersection(unwrap(b))); + } + + @Override + public ImmutableBitmap complement(ImmutableBitmap b) + { + return new WrappedImmutableConciseBitmap(ImmutableConciseSet.complement(((WrappedImmutableConciseBitmap) b).getBitmap())); + } + + @Override + public ImmutableBitmap complement(ImmutableBitmap b, int length) + { + return new WrappedImmutableConciseBitmap( + ImmutableConciseSet.complement( + ((WrappedImmutableConciseBitmap) b).getBitmap(), + length + ) + ); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java new file mode 100755 index 000000000000..7438aafff121 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/ImmutableBitmap.java @@ -0,0 +1,84 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import org.roaringbitmap.IntIterator; + +/** + * This class is meant to represent a simple wrapper around an immutable bitmap + * class. + */ +public interface ImmutableBitmap +{ + /** + * @return an iterator over the set bits of this bitmap + */ + public IntIterator iterator(); + + /** + * @return The number of bits set to true in this bitmap + */ + public int size(); + + public byte[] toBytes(); + + public int compareTo(ImmutableBitmap other); + + /** + * @return True if this bitmap is empty (contains no set bit) + */ + public boolean isEmpty(); + + /** + * Returns true if the bit at position value is set + * + * @param value the position to check + * + * @return true if bit is set + */ + public boolean get(int value); + + /** + * Compute the bitwise-or of this bitmap with another bitmap. A new bitmap is generated. + * + * Note that the other bitmap should be of the same class instance. + * + * @param otherBitmap other bitmap + */ + public ImmutableBitmap union(ImmutableBitmap otherBitmap); + + /** + * Compute the bitwise-and of this bitmap with another bitmap. A new bitmap is generated. + * + * Note that the other bitmap should be of the same class instance. + * + * @param otherBitmap other bitmap + */ + public ImmutableBitmap intersection(ImmutableBitmap otherBitmap); + + /** + * Compute the bitwise-andNot of this bitmap with another bitmap. A new bitmap is generated. + * + * Note that the other bitmap should be of the same class instance. + * + * @param otherBitmap other bitmap + */ + public ImmutableBitmap difference(ImmutableBitmap otherBitmap); +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/MutableBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/MutableBitmap.java new file mode 100755 index 000000000000..5f39cddeebe1 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/MutableBitmap.java @@ -0,0 +1,111 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.nio.ByteBuffer; + +/** + * This class is meant to represent a simple wrapper around a bitmap class. + */ +public interface MutableBitmap extends ImmutableBitmap +{ + /** + * Empties the content of this bitmap. + */ + public void clear(); + + /** + * Compute the bitwise-or of this bitmap with another bitmap. The current + * bitmap is modified whereas the other bitmap is left intact. + * + * Note that the other bitmap should be of the same class instance. + * + * @param mutableBitmap other bitmap + */ + public void or(MutableBitmap mutableBitmap); + + /** + * Compute the bitwise-and of this bitmap with another bitmap. The current + * bitmap is modified whereas the other bitmap is left intact. + * + * Note that the other bitmap should be of the same class instance. + * + * @param mutableBitmap other bitmap + */ + public void and(MutableBitmap mutableBitmap); + + + /** + * Compute the bitwise-xor of this bitmap with another bitmap. The current + * bitmap is modified whereas the other bitmap is left intact. + * + * Note that the other bitmap should be of the same class instance. + * + * @param mutableBitmap other bitmap + */ + public void xor(MutableBitmap mutableBitmap); + + /** + * Compute the bitwise-andNot of this bitmap with another bitmap. The current + * bitmap is modified whereas the other bitmap is left intact. + * + * Note that the other bitmap should be of the same class instance. + * + * @param mutableBitmap other bitmap + */ + public void andNot(MutableBitmap mutableBitmap); + + /** + * Return the size in bytes for the purpose of serialization to a ByteBuffer. + * Note that this is distinct from the memory usage. + * + * @return the total set in bytes + */ + public int getSizeInBytes(); + + /** + * Add the specified integer to the bitmap. This is equivalent to setting the + * ith bit to the value 1. + * + * @param entry integer to be added + */ + public void add(int entry); + + /** + * Remove the specified integer to the bitmap. This is equivalent to setting the + * ith bit to the value 1. + * + * @param entry integer to be remove + */ + public void remove(int entry); + + /** + * Write out a serialized (Immutable) version of the bitmap to the ByteBuffer. We preprend + * the serialized bitmap with a 4-byte int indicating the size in bytes. Thus + * getSizeInBytes() + 4 bytes are written. + * + * (These 4 bytes are required by ConciseSet but not by RoaringBitmap. + * Nevertheless, we always write them for the sake of simplicity, even if it + * wastes 4 bytes in some instances.) + * + * @param buffer where we write + */ + public void serialize(ByteBuffer buffer); +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java new file mode 100755 index 000000000000..0aa86e05c362 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/RoaringBitmapFactory.java @@ -0,0 +1,179 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import com.google.common.base.Throwables; +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.buffer.BufferFastAggregation; +import org.roaringbitmap.buffer.ImmutableRoaringBitmap; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import java.util.Iterator; + +/** + * As the name suggests, this class instantiates bitmaps of the types + * WrappedRoaringBitmap and WrappedImmutableRoaringBitmap. + */ +public class RoaringBitmapFactory implements BitmapFactory +{ + static final boolean DEFAULT_COMPRESS_RUN_ON_SERIALIZATION = false; + private static final ImmutableRoaringBitmap EMPTY_IMMUTABLE_BITMAP; + + static { + try { + final RoaringBitmap roaringBitmap = new RoaringBitmap(); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + roaringBitmap.serialize(new DataOutputStream(out)); + final byte[] bytes = out.toByteArray(); + + ByteBuffer buf = ByteBuffer.wrap(bytes); + EMPTY_IMMUTABLE_BITMAP = new ImmutableRoaringBitmap(buf); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + private static final WrappedImmutableRoaringBitmap WRAPPED_IMMUTABLE_ROARING_BITMAP = + new WrappedImmutableRoaringBitmap(EMPTY_IMMUTABLE_BITMAP); + + private final boolean compressRunOnSerialization; + + public RoaringBitmapFactory() + { + this(DEFAULT_COMPRESS_RUN_ON_SERIALIZATION); + } + + public RoaringBitmapFactory(boolean compressRunOnSerialization) + { + this.compressRunOnSerialization = compressRunOnSerialization; + } + + private static Iterable unwrap( + final Iterable b + ) + { + return new Iterable() + { + @Override + public Iterator iterator() + { + final Iterator i = b.iterator(); + return new Iterator() + { + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + + @Override + public boolean hasNext() + { + return i.hasNext(); + } + + @Override + public ImmutableRoaringBitmap next() + { + WrappedImmutableRoaringBitmap wrappedBitmap = (WrappedImmutableRoaringBitmap) i.next(); + + if (wrappedBitmap == null) { + return EMPTY_IMMUTABLE_BITMAP; + } + + return wrappedBitmap.getBitmap(); + } + }; + } + }; + } + + @Override + public MutableBitmap makeEmptyMutableBitmap() + { + return new WrappedRoaringBitmap(compressRunOnSerialization); + } + + @Override + public ImmutableBitmap makeEmptyImmutableBitmap() + { + return WRAPPED_IMMUTABLE_ROARING_BITMAP; + } + + @Override + public ImmutableBitmap makeImmutableBitmap(MutableBitmap mutableBitmap) + { + if (!(mutableBitmap instanceof WrappedRoaringBitmap)) { + throw new IllegalStateException(String.format("Cannot convert [%s]", mutableBitmap.getClass())); + } + try { + return ((WrappedRoaringBitmap) mutableBitmap).toImmutableBitmap(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @Override + public ImmutableBitmap mapImmutableBitmap(ByteBuffer b) + { + return new WrappedImmutableRoaringBitmap(b); + } + + @Override + public ImmutableBitmap union(Iterable b) + { + return new WrappedImmutableRoaringBitmap(ImmutableRoaringBitmap.or(unwrap(b).iterator())); + } + + @Override + public ImmutableBitmap intersection(Iterable b) + { + return new WrappedImmutableRoaringBitmap(BufferFastAggregation.and(unwrap(b).iterator())); + } + + @Override + public ImmutableBitmap complement(ImmutableBitmap b) + { + return new WrappedImmutableRoaringBitmap( + ImmutableRoaringBitmap.flip( + ((WrappedImmutableRoaringBitmap) b).getBitmap(), + 0, + b.size() + ) + ); + } + + @Override + public ImmutableBitmap complement( + ImmutableBitmap b, int length + ) + { + return new WrappedImmutableRoaringBitmap( + ImmutableRoaringBitmap.flip( + ((WrappedImmutableRoaringBitmap) b).getBitmap(), + 0, + length + ) + ); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java new file mode 100755 index 000000000000..3781f335aafa --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedBitSetBitmap.java @@ -0,0 +1,149 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.nio.ByteBuffer; +import java.util.BitSet; + +/** + * WrappedBitSetBitmap implements MutableBitmap for java.util.BitSet + */ +public class WrappedBitSetBitmap extends WrappedImmutableBitSetBitmap implements MutableBitmap +{ + + public WrappedBitSetBitmap() + { + super(); + } + + public WrappedBitSetBitmap(BitSet bitSet) + { + super(bitSet); + } + + public WrappedBitSetBitmap(ByteBuffer byteBuffer) + { + super(byteBuffer); + } + + protected BitSet cloneBitSet() + { + return (BitSet) bitmap.clone(); + } + + @Override + public void clear() + { + bitmap.clear(); + } + + @Override + public void or(MutableBitmap mutableBitmap) + { + if (mutableBitmap instanceof WrappedBitSetBitmap) { + WrappedBitSetBitmap bitSet = (WrappedBitSetBitmap) mutableBitmap; + this.bitmap.or(bitSet.bitmap); + } else { + throw new IllegalArgumentException( + String.format( + "Unknown class type: %s expected %s", + mutableBitmap.getClass().getCanonicalName(), + WrappedBitSetBitmap.class.getCanonicalName() + ) + ); + } + } + + @Override + public void and(MutableBitmap mutableBitmap) + { + if (mutableBitmap instanceof WrappedBitSetBitmap) { + WrappedBitSetBitmap bitSet = (WrappedBitSetBitmap) mutableBitmap; + this.bitmap.and(bitSet.bitmap); + } else { + throw new IllegalArgumentException( + String.format( + "Unknown class type: %s expected %s", + mutableBitmap.getClass().getCanonicalName(), + WrappedBitSetBitmap.class.getCanonicalName() + ) + ); + } + } + + @Override + public void xor(MutableBitmap mutableBitmap) + { + if (mutableBitmap instanceof WrappedBitSetBitmap) { + WrappedBitSetBitmap bitSet = (WrappedBitSetBitmap) mutableBitmap; + this.bitmap.xor(bitSet.bitmap); + } else { + throw new IllegalArgumentException( + String.format( + "Unknown class type: %s expected %s", + mutableBitmap.getClass().getCanonicalName(), + WrappedBitSetBitmap.class.getCanonicalName() + ) + ); + } + } + + @Override + public void andNot(MutableBitmap mutableBitmap) + { + if (mutableBitmap instanceof WrappedBitSetBitmap) { + WrappedBitSetBitmap bitSet = (WrappedBitSetBitmap) mutableBitmap; + this.bitmap.andNot(bitSet.bitmap); + } else { + throw new IllegalArgumentException( + String.format( + "Unknown class type: %s expected %s", + mutableBitmap.getClass().getCanonicalName(), + WrappedBitSetBitmap.class.getCanonicalName() + ) + ); + } + } + + @Override + public int getSizeInBytes() + { + // BitSet.size() returns the size in *bits* + return this.bitmap.size() / Byte.SIZE; + } + + @Override + public void add(int entry) + { + this.bitmap.set(entry); + } + + @Override + public void remove(int entry) + { + this.bitmap.clear(entry); + } + + @Override + public void serialize(ByteBuffer buffer) + { + buffer.put(this.bitmap.toByteArray()); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java new file mode 100755 index 000000000000..6fe730ce0f19 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseBitmap.java @@ -0,0 +1,210 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.nio.ByteBuffer; + +import org.roaringbitmap.IntIterator; + +import com.google.common.primitives.Ints; + +import it.uniroma3.mat.extendedset.intset.ConciseSet; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import it.uniroma3.mat.extendedset.intset.IntSet; + +public class WrappedConciseBitmap implements MutableBitmap +{ + /** + * Underlying bitmap. + */ + private ConciseSet bitmap; + + /** + * Create a new WrappedConciseBitmap wrapping an empty ConciseSet + */ + public WrappedConciseBitmap() + { + this.bitmap = new ConciseSet(); + } + + /** + * Create a bitmap wrappign the given bitmap + * + * @param conciseSet bitmap to be wrapped + */ + public WrappedConciseBitmap(ConciseSet conciseSet) + { + this.bitmap = conciseSet; + } + + ConciseSet getBitmap() + { + return bitmap; + } + + @Override + public byte[] toBytes() + { + return ImmutableConciseSet.newImmutableFromMutable(bitmap).toBytes(); + } + + @Override + public int compareTo(ImmutableBitmap other) + { + return bitmap.compareTo(((WrappedConciseBitmap) other).getBitmap()); + } + + @Override + public void clear() + { + bitmap.clear(); + } + + @Override + public void or(MutableBitmap mutableBitmap) + { + WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap; + ConciseSet unwrappedOtherBitmap = other.bitmap; + bitmap.addAll(unwrappedOtherBitmap); + } + + @Override + public void and(MutableBitmap mutableBitmap) + { + WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap; + ConciseSet unwrappedOtherBitmap = other.bitmap; + bitmap = bitmap.intersection(unwrappedOtherBitmap); + } + + @Override + public void xor(MutableBitmap mutableBitmap) + { + WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap; + ConciseSet unwrappedOtherBitmap = other.bitmap; + bitmap = bitmap.symmetricDifference(unwrappedOtherBitmap); + } + + @Override + public void andNot(MutableBitmap mutableBitmap) + { + WrappedConciseBitmap other = (WrappedConciseBitmap) mutableBitmap; + ConciseSet unwrappedOtherBitmap = other.bitmap; + bitmap = bitmap.difference(unwrappedOtherBitmap); + } + + @Override + public int getSizeInBytes() + { + return bitmap.getWords().length * Ints.BYTES; + } + + @Override + public void add(int entry) + { + bitmap.add(entry); + } + + @Override + public int size() + { + return bitmap.size(); + } + + @Override + public void serialize(ByteBuffer buffer) + { + buffer.put(toBytes()); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + bitmap.toString(); + } + + @Override + public void remove(int entry) + { + bitmap.remove(entry); + } + + @Override + public IntIterator iterator() + { + final IntSet.IntIterator i = bitmap.iterator(); + return new IntIterator() + { + @Override + public IntIterator clone() + { + throw new UnsupportedOperationException("clone is not supported on ConciseSet iterator"); + } + + @Override + public boolean hasNext() + { + return i.hasNext(); + } + + @Override + public int next() + { + return i.next(); + } + + }; + } + + @Override + public boolean isEmpty() + { + return bitmap.size() == 0; + } + + @Override + public ImmutableBitmap union(ImmutableBitmap otherBitmap) + { + WrappedConciseBitmap other = (WrappedConciseBitmap) otherBitmap; + ConciseSet unwrappedOtherBitmap = other.bitmap; + return new WrappedConciseBitmap(bitmap.clone().union(unwrappedOtherBitmap)); + } + + @Override + public ImmutableBitmap intersection(ImmutableBitmap otherBitmap) + { + WrappedConciseBitmap other = (WrappedConciseBitmap) otherBitmap; + ConciseSet unwrappedOtherBitmap = other.bitmap; + return new WrappedConciseBitmap(bitmap.clone().intersection(unwrappedOtherBitmap)); + } + + @Override + public ImmutableBitmap difference(ImmutableBitmap otherBitmap) + { + WrappedConciseBitmap other = (WrappedConciseBitmap) otherBitmap; + ConciseSet unwrappedOtherBitmap = other.bitmap; + return new WrappedConciseBitmap(bitmap.clone().difference(unwrappedOtherBitmap)); + } + + @Override + public boolean get(int value) + { + return bitmap.contains(value); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseIntIterator.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseIntIterator.java new file mode 100755 index 000000000000..c357cefdebd2 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedConciseIntIterator.java @@ -0,0 +1,54 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import org.roaringbitmap.IntIterator; + +import it.uniroma3.mat.extendedset.intset.IntSet; + +/** + */ +public class WrappedConciseIntIterator implements IntIterator +{ + private final IntSet.IntIterator itr; + + public WrappedConciseIntIterator(IntSet.IntIterator itr) + { + this.itr = itr; + } + + @Override + public boolean hasNext() + { + return itr.hasNext(); + } + + @Override + public int next() + { + return itr.next(); + } + + @Override + public IntIterator clone() + { + return new WrappedConciseIntIterator(itr.clone()); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java new file mode 100755 index 000000000000..633a1de063ab --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableBitSetBitmap.java @@ -0,0 +1,144 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import org.roaringbitmap.IntIterator; + +import java.nio.ByteBuffer; +import java.util.BitSet; + +/** + * WrappedImmutableBitSetBitmap implements ImmutableBitmap for java.util.BitSet + */ +public class WrappedImmutableBitSetBitmap implements ImmutableBitmap +{ + protected final BitSet bitmap; + + public WrappedImmutableBitSetBitmap(BitSet bitmap) + { + this.bitmap = bitmap; + } + + public WrappedImmutableBitSetBitmap() + { + this(new BitSet()); + } + + // WARNING: the current implementation of BitSet (1.7) copies the contents of ByteBuffer to + // on heap! + // TODO: make a new BitSet implementation which can use ByteBuffers properly. + public WrappedImmutableBitSetBitmap(ByteBuffer byteBuffer) + { + this(BitSet.valueOf(byteBuffer)); + } + + @Override + public IntIterator iterator() + { + return new BitSetIterator(); + } + + @Override + public boolean get(int value) + { + return bitmap.get(value); + } + + @Override + public int size() + { + return bitmap.cardinality(); + } + + @Override + public byte[] toBytes() + { + return bitmap.toByteArray(); + } + + @Override + public int compareTo(ImmutableBitmap other) + { + // TODO: find out what this is supposed to even do + BitSet otherSet = ((WrappedImmutableBitSetBitmap) other).bitmap; + int lengthCompare = Integer.compare(otherSet.length(), bitmap.length()); + if (lengthCompare != 0) { + return lengthCompare; + } + return Integer.compare(otherSet.nextSetBit(0), bitmap.nextSetBit(0)); + } + + @Override + public boolean isEmpty() + { + return bitmap.isEmpty(); + } + + @Override + public ImmutableBitmap union(ImmutableBitmap otherBitmap) + { + WrappedBitSetBitmap retval = new WrappedBitSetBitmap((BitSet) bitmap.clone()); + retval.or((WrappedBitSetBitmap) otherBitmap); + return retval; + } + + @Override + public ImmutableBitmap intersection(ImmutableBitmap otherBitmap) + { + WrappedBitSetBitmap retval = new WrappedBitSetBitmap((BitSet) bitmap.clone()); + retval.and((WrappedBitSetBitmap) otherBitmap); + return retval; + } + + @Override + public ImmutableBitmap difference(ImmutableBitmap otherBitmap) + { + WrappedBitSetBitmap retval = new WrappedBitSetBitmap((BitSet) bitmap.clone()); + retval.andNot((WrappedBitSetBitmap) otherBitmap); + return retval; + } + + private class BitSetIterator implements IntIterator + { + private int pos = -1; + + @Override + public boolean hasNext() + { + return bitmap.nextSetBit(pos + 1) >= 0; + } + + @Override + public int next() + { + pos = bitmap.nextSetBit(pos + 1); + return pos; + } + + @Override + public IntIterator clone() + { + BitSetIterator newIt = new BitSetIterator(); + newIt.pos = pos; + return newIt; + } + } + +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java new file mode 100755 index 000000000000..5fe4515b89fc --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableConciseBitmap.java @@ -0,0 +1,147 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + + +import java.nio.ByteBuffer; + +import org.roaringbitmap.IntIterator; + +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import it.uniroma3.mat.extendedset.intset.IntSet; + +public class WrappedImmutableConciseBitmap implements ImmutableBitmap +{ + /** + * Underlying bitmap. + */ + private final ImmutableConciseSet bitmap; + + public WrappedImmutableConciseBitmap(ByteBuffer byteBuffer) + { + this.bitmap = new ImmutableConciseSet(byteBuffer.asReadOnlyBuffer()); + } + + /** + * Wrap an ImmutableConciseSet + * + * @param immutableConciseSet bitmap to be wrapped + */ + public WrappedImmutableConciseBitmap(ImmutableConciseSet immutableConciseSet) + { + this.bitmap = immutableConciseSet; + } + + public ImmutableConciseSet getBitmap() + { + return bitmap; + } + + @Override + public boolean get(int value) + { + return bitmap.contains(value); + } + + @Override + public byte[] toBytes() + { + return bitmap.toBytes(); + } + + @Override + public int compareTo(ImmutableBitmap other) + { + return bitmap.compareTo(((WrappedImmutableConciseBitmap) other).getBitmap()); + } + + @Override + public String toString() + { + return getClass().getSimpleName() + bitmap.toString(); + } + + @Override + public IntIterator iterator() + { + final IntSet.IntIterator i = bitmap.iterator(); + return new IntIterator() + { + @Override + public IntIterator clone() + { + return new WrappedConciseIntIterator(i.clone()); + } + + @Override + public boolean hasNext() + { + return i.hasNext(); + } + + @Override + public int next() + { + return i.next(); + } + }; + } + + @Override + public int size() + { + return bitmap.size(); + } + + @Override + public boolean isEmpty() + { + return bitmap.size() == 0; + } + + @Override + public ImmutableBitmap union(ImmutableBitmap otherBitmap) + { + WrappedImmutableConciseBitmap other = (WrappedImmutableConciseBitmap) otherBitmap; + ImmutableConciseSet unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableConciseBitmap(ImmutableConciseSet.union(bitmap, unwrappedOtherBitmap)); + } + + @Override + public ImmutableBitmap intersection(ImmutableBitmap otherBitmap) + { + WrappedImmutableConciseBitmap other = (WrappedImmutableConciseBitmap) otherBitmap; + ImmutableConciseSet unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableConciseBitmap(ImmutableConciseSet.intersection(bitmap, unwrappedOtherBitmap)); + } + + @Override + public ImmutableBitmap difference(ImmutableBitmap otherBitmap) + { + WrappedImmutableConciseBitmap other = (WrappedImmutableConciseBitmap) otherBitmap; + ImmutableConciseSet unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableConciseBitmap( + ImmutableConciseSet.intersection( + bitmap, + ImmutableConciseSet.complement(unwrappedOtherBitmap) + ) + ); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java new file mode 100755 index 000000000000..9b2210ab115d --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedImmutableRoaringBitmap.java @@ -0,0 +1,129 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import com.google.common.base.Throwables; +import org.roaringbitmap.IntIterator; +import org.roaringbitmap.buffer.ImmutableRoaringBitmap; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; + +public class WrappedImmutableRoaringBitmap implements ImmutableBitmap +{ + /** + * Underlying bitmap. + */ + private final ImmutableRoaringBitmap bitmap; + + protected WrappedImmutableRoaringBitmap(ByteBuffer byteBuffer) + { + this.bitmap = new ImmutableRoaringBitmap(byteBuffer.asReadOnlyBuffer()); + } + + /** + * Wrap an ImmutableRoaringBitmap + * + * @param immutableRoaringBitmap bitmap to be wrapped + */ + public WrappedImmutableRoaringBitmap(ImmutableRoaringBitmap immutableRoaringBitmap) + { + this.bitmap = immutableRoaringBitmap; + } + + public ImmutableRoaringBitmap getBitmap() + { + return bitmap; + } + + @Override + public byte[] toBytes() + { + try { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + bitmap.serialize(new DataOutputStream(out)); + return out.toByteArray(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @Override + public int compareTo(ImmutableBitmap other) + { + return 0; + } + + @Override + public String toString() + { + return getClass().getSimpleName() + bitmap.toString(); + } + + @Override + public IntIterator iterator() + { + return bitmap.getIntIterator(); + } + + @Override + public int size() + { + return bitmap.getCardinality(); + } + + @Override + public boolean isEmpty() + { + return bitmap.isEmpty(); + } + + @Override + public ImmutableBitmap union(ImmutableBitmap otherBitmap) + { + WrappedImmutableRoaringBitmap other = (WrappedImmutableRoaringBitmap) otherBitmap; + ImmutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableRoaringBitmap(ImmutableRoaringBitmap.or(bitmap, unwrappedOtherBitmap)); + } + + @Override + public boolean get(int value) + { + return bitmap.contains(value); + } + + @Override + public ImmutableBitmap intersection(ImmutableBitmap otherBitmap) + { + WrappedImmutableRoaringBitmap other = (WrappedImmutableRoaringBitmap) otherBitmap; + ImmutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableRoaringBitmap(ImmutableRoaringBitmap.and(bitmap, unwrappedOtherBitmap)); + } + + @Override + public ImmutableBitmap difference(ImmutableBitmap otherBitmap) + { + WrappedImmutableRoaringBitmap other = (WrappedImmutableRoaringBitmap) otherBitmap; + ImmutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableRoaringBitmap(ImmutableRoaringBitmap.andNot(bitmap, unwrappedOtherBitmap)); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java new file mode 100755 index 000000000000..eeb7bdb0e260 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/bitmap/WrappedRoaringBitmap.java @@ -0,0 +1,265 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import com.google.common.base.Throwables; +import org.roaringbitmap.IntIterator; +import org.roaringbitmap.RoaringBitmap; +import org.roaringbitmap.buffer.MutableRoaringBitmap; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public class WrappedRoaringBitmap implements MutableBitmap +{ + // attempt to compress long runs prior to serialization (requires RoaringBitmap version 0.5 or better) + // this may improve compression greatly in some cases at the expense of slower serialization + // in the worst case. + private final boolean compressRunOnSerialization; + /** + * Underlying bitmap. + */ + private MutableRoaringBitmap bitmap; + + /** + * Creates a new WrappedRoaringBitmap wrapping an empty MutableRoaringBitmap + */ + public WrappedRoaringBitmap() + { + this(RoaringBitmapFactory.DEFAULT_COMPRESS_RUN_ON_SERIALIZATION); + } + + /** + * Creates a new WrappedRoaringBitmap wrapping an empty MutableRoaringBitmap + * + * @param compressRunOnSerialization indicates whether to call {@link RoaringBitmap#runOptimize()} before serializing + */ + public WrappedRoaringBitmap(boolean compressRunOnSerialization) + { + this.bitmap = new MutableRoaringBitmap(); + this.compressRunOnSerialization = compressRunOnSerialization; + } + + ImmutableBitmap toImmutableBitmap() + { + MutableRoaringBitmap mrb = bitmap.clone(); + if (compressRunOnSerialization) { + mrb.runOptimize(); + } + return new WrappedImmutableRoaringBitmap(mrb); + } + + @Override + public byte[] toBytes() + { + try { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + if (compressRunOnSerialization) { + bitmap.runOptimize(); + } + bitmap.serialize(new DataOutputStream(out)); + return out.toByteArray(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + @Override + public int compareTo(ImmutableBitmap other) + { + return 0; + } + + @Override + public void clear() + { + this.bitmap.clear(); + } + + @Override + public void or(MutableBitmap mutableBitmap) + { + WrappedRoaringBitmap other = (WrappedRoaringBitmap) mutableBitmap; + MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + bitmap.or(unwrappedOtherBitmap); + } + + @Override + public void and(MutableBitmap mutableBitmap) + { + WrappedRoaringBitmap other = (WrappedRoaringBitmap) mutableBitmap; + MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + bitmap.and(unwrappedOtherBitmap); + } + + + @Override + public void andNot(MutableBitmap mutableBitmap) + { + WrappedRoaringBitmap other = (WrappedRoaringBitmap) mutableBitmap; + MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + bitmap.andNot(unwrappedOtherBitmap); + } + + + @Override + public void xor(MutableBitmap mutableBitmap) + { + WrappedRoaringBitmap other = (WrappedRoaringBitmap) mutableBitmap; + MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + bitmap.xor(unwrappedOtherBitmap); + } + + @Override + public int getSizeInBytes() + { + if (compressRunOnSerialization) { + bitmap.runOptimize(); + } + return bitmap.serializedSizeInBytes(); + } + + @Override + public void add(int entry) + { + bitmap.add(entry); + } + + @Override + public int size() + { + return bitmap.getCardinality(); + } + + @Override + public void serialize(ByteBuffer buffer) + { + if (compressRunOnSerialization) { + bitmap.runOptimize(); + } + try { + bitmap.serialize( + new DataOutputStream( + new OutputStream() + { + ByteBuffer mBB; + + OutputStream init(ByteBuffer mbb) + { + mBB = mbb; + return this; + } + + @Override + public void close() + { + // unnecessary + } + + @Override + public void flush() + { + // unnecessary + } + + @Override + public void write(int b) + { + mBB.put((byte) b); + } + + @Override + public void write(byte[] b) + { + mBB.put(b); + } + + @Override + public void write(byte[] b, int off, int l) + { + mBB.put(b, off, l); + } + }.init(buffer) + ) + ); + } + catch (IOException e) { + e.printStackTrace(); // impossible in theory + } + } + + @Override + public String toString() + { + return getClass().getSimpleName() + bitmap.toString(); + } + + @Override + public void remove(int entry) + { + bitmap.remove(entry); + } + + @Override + public IntIterator iterator() + { + return bitmap.getIntIterator(); + } + + @Override + public boolean isEmpty() + { + return bitmap.isEmpty(); + } + + @Override + public ImmutableBitmap union(ImmutableBitmap otherBitmap) + { + WrappedRoaringBitmap other = (WrappedRoaringBitmap) otherBitmap; + MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableRoaringBitmap(MutableRoaringBitmap.or(bitmap, unwrappedOtherBitmap)); + } + + @Override + public ImmutableBitmap intersection(ImmutableBitmap otherBitmap) + { + WrappedRoaringBitmap other = (WrappedRoaringBitmap) otherBitmap; + MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableRoaringBitmap(MutableRoaringBitmap.and(bitmap, unwrappedOtherBitmap)); + } + + @Override + public ImmutableBitmap difference(ImmutableBitmap otherBitmap) + { + WrappedRoaringBitmap other = (WrappedRoaringBitmap) otherBitmap; + MutableRoaringBitmap unwrappedOtherBitmap = other.bitmap; + return new WrappedImmutableRoaringBitmap(MutableRoaringBitmap.andNot(bitmap, unwrappedOtherBitmap)); + } + + @Override + public boolean get(int value) + { + return bitmap.contains(value); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java new file mode 100755 index 000000000000..6f5cf938fff0 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableNode.java @@ -0,0 +1,226 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; + +import java.nio.ByteBuffer; +import java.util.Iterator; + +/** + * Byte layout: + * Header + * 0 to 1 : the MSB is a boolean flag for isLeaf, the next 15 bits represent the number of children of a node + * Body + * 2 to 2 + numDims * Floats.BYTES : minCoordinates + * 2 + numDims * Floats.BYTES to 2 + 2 * numDims * Floats.BYTES : maxCoordinates + * concise set + * rest (children) : Every 4 bytes is storing an offset representing the position of a child. + * + * The child offset is an offset from the initialOffset + */ +public class ImmutableNode +{ + public static final int HEADER_NUM_BYTES = 2; + + private final int numDims; + private final int initialOffset; + private final int offsetFromInitial; + + private final short numChildren; + private final boolean isLeaf; + private final int childrenOffset; + + private final ByteBuffer data; + + private final BitmapFactory bitmapFactory; + + public ImmutableNode( + int numDims, + int initialOffset, + int offsetFromInitial, + ByteBuffer data, + BitmapFactory bitmapFactory + ) + { + this.bitmapFactory = bitmapFactory; + this.numDims = numDims; + this.initialOffset = initialOffset; + this.offsetFromInitial = offsetFromInitial; + short header = data.getShort(initialOffset + offsetFromInitial); + this.isLeaf = (header & 0x8000) != 0; + this.numChildren = (short) (header & 0x7FFF); + final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Floats.BYTES; + int bitmapSize = data.getInt(sizePosition); + this.childrenOffset = initialOffset + + offsetFromInitial + + HEADER_NUM_BYTES + + 2 * numDims * Floats.BYTES + + Ints.BYTES + + bitmapSize; + + this.data = data; + } + + public ImmutableNode( + int numDims, + int initialOffset, + int offsetFromInitial, + short numChildren, + boolean leaf, + ByteBuffer data, + BitmapFactory bitmapFactory + ) + { + this.bitmapFactory = bitmapFactory; + this.numDims = numDims; + this.initialOffset = initialOffset; + this.offsetFromInitial = offsetFromInitial; + this.numChildren = numChildren; + this.isLeaf = leaf; + final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Floats.BYTES; + int bitmapSize = data.getInt(sizePosition); + this.childrenOffset = initialOffset + + offsetFromInitial + + HEADER_NUM_BYTES + + 2 * numDims * Floats.BYTES + + Ints.BYTES + + bitmapSize; + + this.data = data; + } + + public BitmapFactory getBitmapFactory() + { + return bitmapFactory; + } + + public int getInitialOffset() + { + return initialOffset; + } + + public int getOffsetFromInitial() + { + return offsetFromInitial; + } + + public int getNumDims() + { + return numDims; + } + + public int getNumChildren() + { + return numChildren; + } + + public boolean isLeaf() + { + return isLeaf; + } + + public float[] getMinCoordinates() + { + return getCoords(initialOffset + offsetFromInitial + HEADER_NUM_BYTES); + } + + public float[] getMaxCoordinates() + { + return getCoords(initialOffset + offsetFromInitial + HEADER_NUM_BYTES + numDims * Floats.BYTES); + } + + public ImmutableBitmap getImmutableBitmap() + { + final int sizePosition = initialOffset + offsetFromInitial + HEADER_NUM_BYTES + 2 * numDims * Floats.BYTES; + int numBytes = data.getInt(sizePosition); + data.position(sizePosition + Ints.BYTES); + ByteBuffer tmpBuffer = data.slice(); + tmpBuffer.limit(numBytes); + return bitmapFactory.mapImmutableBitmap(tmpBuffer.asReadOnlyBuffer()); + } + + public Iterable getChildren() + { + return new Iterable() + { + @Override + public Iterator iterator() + { + return new Iterator() + { + private volatile int count = 0; + + @Override + public boolean hasNext() + { + return (count < numChildren); + } + + @Override + public ImmutableNode next() + { + if (isLeaf) { + return new ImmutablePoint( + numDims, + initialOffset, + data.getInt(childrenOffset + (count++) * Ints.BYTES), + data, + bitmapFactory + ); + } + return new ImmutableNode( + numDims, + initialOffset, + data.getInt(childrenOffset + (count++) * Ints.BYTES), + data, + bitmapFactory + ); + } + + @Override + public void remove() + { + throw new UnsupportedOperationException(); + } + }; + } + }; + } + + public ByteBuffer getData() + { + return data; + } + + private float[] getCoords(int offset) + { + final float[] retVal = new float[numDims]; + + final ByteBuffer readOnlyBuffer = data.asReadOnlyBuffer(); + readOnlyBuffer.position(offset); + readOnlyBuffer.asFloatBuffer().get(retVal); + + return retVal; + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java new file mode 100755 index 000000000000..1ee07020571f --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutablePoint.java @@ -0,0 +1,64 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import io.druid.collections.bitmap.BitmapFactory; + +import java.nio.ByteBuffer; + +public class ImmutablePoint extends ImmutableNode +{ + public ImmutablePoint( + int numDims, + int initialOffset, + int offsetFromInitial, + ByteBuffer data, + BitmapFactory bitmapFactory + ) + { + super(numDims, initialOffset, offsetFromInitial, (short) 0, true, data, bitmapFactory); + } + + public ImmutablePoint(ImmutableNode node) + { + super( + node.getNumDims(), + node.getInitialOffset(), + node.getOffsetFromInitial(), + (short) 0, + true, + node.getData(), + node.getBitmapFactory() + ); + } + + public float[] getCoords() + { + return super.getMinCoordinates(); + } + + @Override + public Iterable getChildren() + { + // should never get here + throw new UnsupportedOperationException(); + } + +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java new file mode 100755 index 000000000000..9da5957170bc --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/ImmutableRTree.java @@ -0,0 +1,144 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.search.Bound; +import io.druid.collections.spatial.search.GutmanSearchStrategy; +import io.druid.collections.spatial.search.SearchStrategy; + +import java.nio.ByteBuffer; + +/** + * An immutable representation of an {@link RTree} for spatial indexing. + */ +public class ImmutableRTree +{ + private static byte VERSION = 0x0; + private final int numDims; + private final ImmutableNode root; + private final ByteBuffer data; + private final SearchStrategy defaultSearchStrategy = new GutmanSearchStrategy(); + + public ImmutableRTree() + { + this.numDims = 0; + this.data = ByteBuffer.wrap(new byte[]{}); + this.root = null; + } + + public ImmutableRTree(ByteBuffer data, BitmapFactory bitmapFactory) + { + final int initPosition = data.position(); + Preconditions.checkArgument(data.get(0) == VERSION, "Mismatching versions"); + this.numDims = data.getInt(1 + initPosition) & 0x7FFF; + this.data = data; + this.root = new ImmutableNode(numDims, initPosition, 1 + Ints.BYTES, data, bitmapFactory); + } + + public static ImmutableRTree newImmutableFromMutable(RTree rTree) + { + if (rTree.getSize() == 0) { + return new ImmutableRTree(); + } + + ByteBuffer buffer = ByteBuffer.wrap(new byte[calcNumBytes(rTree)]); + + buffer.put(VERSION); + buffer.putInt(rTree.getNumDims()); + rTree.getRoot().storeInByteBuffer(buffer, buffer.position()); + buffer.position(0); + return new ImmutableRTree(buffer.asReadOnlyBuffer(), rTree.getBitmapFactory()); + } + + private static int calcNumBytes(RTree tree) + { + int total = 1 + Ints.BYTES; // VERSION and numDims + + total += calcNodeBytes(tree.getRoot()); + + return total; + } + + private static int calcNodeBytes(Node node) + { + int total = 0; + + // find size of this node + total += node.getSizeInBytes(); + + // recursively find sizes of child nodes + for (Node child : node.getChildren()) { + if (node.isLeaf()) { + total += child.getSizeInBytes(); + } else { + total += calcNodeBytes(child); + } + } + + return total; + } + + public int size() + { + return data.capacity(); + } + + public ImmutableNode getRoot() + { + return root; + } + + public int getNumDims() + { + return numDims; + } + + public Iterable search(Bound bound) + { + return search(defaultSearchStrategy, bound); + } + + public Iterable search(SearchStrategy strategy, Bound bound) + { + if (bound.getNumDims() == numDims) { + return strategy.search(root, bound); + } else { + // If the dimension counts don't match (for example, if this is called on a blank `new ImmutableRTree()`) + return ImmutableList.of(); + } + } + + public byte[] toBytes() + { + ByteBuffer buf = ByteBuffer.allocate(data.capacity()); + buf.put(data.asReadOnlyBuffer()); + return buf.array(); + } + + public int compareTo(ImmutableRTree other) + { + return this.data.compareTo(other.data); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Node.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Node.java new file mode 100755 index 000000000000..1206083799c1 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Node.java @@ -0,0 +1,235 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +/** + */ +public class Node +{ + private final float[] minCoordinates; + private final float[] maxCoordinates; + + private final List children; + private final boolean isLeaf; + private final MutableBitmap bitmap; + + private Node parent; + + public Node(float[] minCoordinates, float[] maxCoordinates, boolean isLeaf, BitmapFactory bitmapFactory) + { + this( + minCoordinates, + maxCoordinates, + Lists.newArrayList(), + isLeaf, + null, + bitmapFactory.makeEmptyMutableBitmap() + ); + } + + public Node( + float[] minCoordinates, + float[] maxCoordinates, + List children, + boolean isLeaf, + Node parent, + MutableBitmap bitmap + ) + { + Preconditions.checkArgument(minCoordinates.length == maxCoordinates.length); + + this.minCoordinates = minCoordinates; + this.maxCoordinates = maxCoordinates; + this.children = children; + for (Node child : children) { + child.setParent(this); + } + this.isLeaf = isLeaf; + this.bitmap = bitmap; + this.parent = parent; + } + + public int getNumDims() + { + return minCoordinates.length; + } + + public float[] getMinCoordinates() + { + return minCoordinates; + } + + public float[] getMaxCoordinates() + { + return maxCoordinates; + } + + public Node getParent() + { + return parent; + } + + private void setParent(Node p) + { + parent = p; + } + + public void addChild(Node node) + { + node.setParent(this); + children.add(node); + } + + public List getChildren() + { + return children; + } + + public boolean isLeaf() + { + return isLeaf; + } + + public double getArea() + { + return calculateArea(); + } + + public boolean contains(Node other) + { + Preconditions.checkArgument(getNumDims() == other.getNumDims()); + + for (int i = 0; i < getNumDims(); i++) { + if (other.getMinCoordinates()[i] < minCoordinates[i] || other.getMaxCoordinates()[i] > maxCoordinates[i]) { + return false; + } + } + return true; + } + + public boolean contains(float[] coords) + { + Preconditions.checkArgument(getNumDims() == coords.length); + + for (int i = 0; i < getNumDims(); i++) { + if (coords[i] < minCoordinates[i] || coords[i] > maxCoordinates[i]) { + return false; + } + } + return true; + } + + public boolean enclose() + { + boolean retVal = false; + float[] minCoords = new float[getNumDims()]; + Arrays.fill(minCoords, Float.MAX_VALUE); + float[] maxCoords = new float[getNumDims()]; + Arrays.fill(maxCoords, -Float.MAX_VALUE); + + for (Node child : getChildren()) { + for (int i = 0; i < getNumDims(); i++) { + minCoords[i] = Math.min(child.getMinCoordinates()[i], minCoords[i]); + maxCoords[i] = Math.max(child.getMaxCoordinates()[i], maxCoords[i]); + } + } + + if (!Arrays.equals(minCoords, minCoordinates)) { + System.arraycopy(minCoords, 0, minCoordinates, 0, minCoordinates.length); + retVal = true; + } + if (!Arrays.equals(maxCoords, maxCoordinates)) { + System.arraycopy(maxCoords, 0, maxCoordinates, 0, maxCoordinates.length); + retVal = true; + } + + return retVal; + } + + public MutableBitmap getBitmap() + { + return bitmap; + } + + public void addToBitmapIndex(Node node) + { + bitmap.or(node.getBitmap()); + } + + public void clear() + { + children.clear(); + bitmap.clear(); + } + + public int getSizeInBytes() + { + return ImmutableNode.HEADER_NUM_BYTES + + 2 * getNumDims() * Floats.BYTES + + Ints.BYTES // size of the set + + bitmap.getSizeInBytes() + + getChildren().size() * Ints.BYTES; + } + + public int storeInByteBuffer(ByteBuffer buffer, int position) + { + buffer.position(position); + buffer.putShort((short) (((isLeaf ? 0x1 : 0x0) << 15) | getChildren().size())); + for (float v : getMinCoordinates()) { + buffer.putFloat(v); + } + for (float v : getMaxCoordinates()) { + buffer.putFloat(v); + } + byte[] bytes = bitmap.toBytes(); + buffer.putInt(bytes.length); + buffer.put(bytes); + + int pos = buffer.position(); + int childStartOffset = pos + getChildren().size() * Ints.BYTES; + for (Node child : getChildren()) { + buffer.putInt(pos, childStartOffset); + childStartOffset = child.storeInByteBuffer(buffer, childStartOffset); + pos += Ints.BYTES; + } + + return childStartOffset; + } + + private double calculateArea() + { + double area = 1.0; + for (int i = 0; i < minCoordinates.length; i++) { + area *= (maxCoordinates[i] - minCoordinates[i]); + } + return area; + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Point.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Point.java new file mode 100755 index 000000000000..ffb878236941 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/Point.java @@ -0,0 +1,113 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import com.google.common.collect.Lists; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; + +import java.util.Arrays; +import java.util.List; + +/** + */ +public class Point extends Node +{ + private final float[] coords; + private final MutableBitmap bitmap; + + public Point(float[] coords, int entry, BitmapFactory bitmapFactory) + { + super( + coords, + Arrays.copyOf(coords, coords.length), + Lists.newArrayList(), + true, + null, + makeBitmap(entry, bitmapFactory) + ); + + this.coords = coords; + this.bitmap = bitmapFactory.makeEmptyMutableBitmap(); + this.bitmap.add(entry); + } + + public Point(float[] coords, MutableBitmap entry) + { + super(coords, Arrays.copyOf(coords, coords.length), Lists.newArrayList(), true, null, entry); + + this.coords = coords; + this.bitmap = entry; + } + + private static MutableBitmap makeBitmap(int entry, BitmapFactory bitmapFactory) + { + MutableBitmap retVal = bitmapFactory.makeEmptyMutableBitmap(); + retVal.add(entry); + return retVal; + } + + public float[] getCoords() + { + return coords; + } + + @Override + public MutableBitmap getBitmap() + { + return bitmap; + } + + @Override + public void addChild(Node node) + { + throw new UnsupportedOperationException(); + } + + @Override + public List getChildren() + { + return Lists.newArrayList(); + } + + @Override + public boolean isLeaf() + { + return true; + } + + @Override + public double getArea() + { + return 0; + } + + @Override + public boolean contains(Node other) + { + return false; + } + + @Override + public boolean enclose() + { + return false; + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTree.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTree.java new file mode 100755 index 000000000000..68398aa69bec --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTree.java @@ -0,0 +1,245 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import com.google.common.base.Preconditions; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; +import io.druid.collections.spatial.split.SplitStrategy; + +import java.util.Arrays; + +/** + * This RTree has been optimized to work with bitmap inverted indexes. + * + * This code will probably make a lot more sense if you read: + * http://www.sai.msu.su/~megera/postgres/gist/papers/gutman-rtree.pdf + */ +public class RTree +{ + private final int numDims; + private final SplitStrategy splitStrategy; + private final BitmapFactory bitmapFactory; + private Node root; + private volatile int size; + + public RTree(BitmapFactory bitmapFactory) + { + this(0, new LinearGutmanSplitStrategy(0, 0, bitmapFactory), bitmapFactory); + } + + public RTree(int numDims, SplitStrategy splitStrategy, BitmapFactory bitmapFactory) + { + this.numDims = numDims; + this.splitStrategy = splitStrategy; + this.bitmapFactory = bitmapFactory; + this.root = buildRoot(true); + } + + public BitmapFactory getBitmapFactory() + { + return bitmapFactory; + } + + /** + * This description is from the original paper. + * + * Algorithm Insert: Insert a new index entry E into an R-tree. + * + * I1. [Find position for new record]. Invoke {@link #chooseLeaf(Node, Point)} to select + * a leaf node L in which to place E. + * + * I2. [Add records to leaf node]. If L has room for another entry, install E. Otherwise invoke + * {@link SplitStrategy} split methods to obtain L and LL containing E and all the old entries of L. + * + * I3. [Propagate changes upward]. Invoke {@link #adjustTree(Node, Node)} on L, also passing LL if a split was + * performed. + * + * I4. [Grow tree taller]. If node split propagation caused the root to split, create a new record whose + * children are the two resulting nodes. + * + * @param coords - the coordinates of the entry + * @param entry - the integer to insert + */ + public void insert(float[] coords, int entry) + { + Preconditions.checkArgument(coords.length == numDims); + insertInner(new Point(coords, entry, bitmapFactory)); + } + + public void insert(float[] coords, MutableBitmap entry) + { + Preconditions.checkArgument(coords.length == numDims); + insertInner(new Point(coords, entry)); + } + + /** + * Not yet implemented. + * + * @param coords - the coordinates of the entry + * @param entry - the integer to insert + * + * @return - whether the operation completed successfully + */ + public boolean delete(double[] coords, int entry) + { + throw new UnsupportedOperationException(); + } + + public int getSize() + { + return size; + } + + public int getNumDims() + { + return numDims; + } + + public SplitStrategy getSplitStrategy() + { + return splitStrategy; + } + + public Node getRoot() + { + return root; + } + + private Node buildRoot(boolean isLeaf) + { + float[] initMinCoords = new float[numDims]; + float[] initMaxCoords = new float[numDims]; + Arrays.fill(initMinCoords, -Float.MAX_VALUE); + Arrays.fill(initMaxCoords, Float.MAX_VALUE); + + return new Node(initMinCoords, initMaxCoords, isLeaf, bitmapFactory); + } + + private void insertInner(Point point) + { + Node node = chooseLeaf(root, point); + node.addChild(point); + + if (splitStrategy.needToSplit(node)) { + Node[] groups = splitStrategy.split(node); + adjustTree(groups[0], groups[1]); + } else { + adjustTree(node, null); + } + + size++; + } + + + /** + * This description is from the original paper. + * + * Algorithm ChooseLeaf. Select a leaf node in which to place a new index entry E. + * + * CL1. [Initialize]. Set N to be the root node. + * + * CL2. [Leaf check]. If N is a leaf, return N. + * + * CL3. [Choose subtree]. If N is not a leaf, let F be the entry in N whose rectangle + * FI needs least enlargement to include EI. Resolve ties by choosing the entry with the rectangle + * of smallest area. + * + * CL4. [Descend until a leaf is reached]. Set N to be the child node pointed to by Fp and repeated from CL2. + * + * @param node - current node to evaluate + * @param point - point to insert + * + * @return - leafNode where point can be inserted + */ + private Node chooseLeaf(Node node, Point point) + { + node.addToBitmapIndex(point); + + if (node.isLeaf()) { + return node; + } + + double minCost = Double.MAX_VALUE; + Node optimal = node.getChildren().get(0); + for (Node child : node.getChildren()) { + double cost = RTreeUtils.getExpansionCost(child, point); + if (cost < minCost) { + minCost = cost; + optimal = child; + } else if (cost == minCost) { + // Resolve ties by choosing the entry with the rectangle of smallest area + if (child.getArea() < optimal.getArea()) { + optimal = child; + } + } + } + + return chooseLeaf(optimal, point); + } + + /** + * This description is from the original paper. + * + * AT1. [Initialize]. Set N=L. If L was split previously, set NN to be the resulting second node. + * + * AT2. [Check if done]. If N is the root, stop. + * + * AT3. [Adjust covering rectangle in parent entry]. Let P be the parent node of N, and let Ev(N)I be N's entry in P. + * Adjust Ev(N)I so that it tightly encloses all entry rectangles in N. + * + * AT4. [Propagate node split upward]. If N has a partner NN resulting from an earlier split, create a new entry + * Ev(NN) with Ev(NN)p pointing to NN and Ev(NN)I enclosing all rectangles in NN. Add Ev(NN) to p is there is room. + * Otherwise, invoke {@link SplitStrategy} split to product p and pp containing Ev(NN) and all p's old entries. + * + * @param n - first node to adjust + * @param nn - optional second node to adjust + */ + private void adjustTree(Node n, Node nn) + { + // special case for root + if (n == root) { + if (nn != null) { + root = buildRoot(false); + root.addChild(n); + root.addChild(nn); + } + root.enclose(); + return; + } + + boolean updateParent = n.enclose(); + + if (nn != null) { + nn.enclose(); + updateParent = true; + + if (splitStrategy.needToSplit(n.getParent())) { + Node[] groups = splitStrategy.split(n.getParent()); + adjustTree(groups[0], groups[1]); + } + } + + if (n.getParent() != null && updateParent) { + adjustTree(n.getParent(), null); + } + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTreeUtils.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTreeUtils.java new file mode 100755 index 000000000000..e5c00b642850 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/RTreeUtils.java @@ -0,0 +1,252 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; + +/** + */ +public class RTreeUtils +{ + private static ObjectMapper jsonMapper = new ObjectMapper(); + + public static double getEnclosingArea(Node a, Node b) + { + Preconditions.checkArgument(a.getNumDims() == b.getNumDims()); + + double[] minCoords = new double[a.getNumDims()]; + double[] maxCoords = new double[a.getNumDims()]; + + for (int i = 0; i < minCoords.length; i++) { + minCoords[i] = Math.min(a.getMinCoordinates()[i], b.getMinCoordinates()[i]); + maxCoords[i] = Math.max(a.getMaxCoordinates()[i], b.getMaxCoordinates()[i]); + } + + double area = 1.0; + for (int i = 0; i < minCoords.length; i++) { + area *= (maxCoords[i] - minCoords[i]); + } + + return area; + } + + public static double getExpansionCost(Node node, Point point) + { + Preconditions.checkArgument(node.getNumDims() == point.getNumDims()); + + if (node.contains(point.getCoords())) { + return 0; + } + + double expanded = 1.0; + for (int i = 0; i < node.getNumDims(); i++) { + double min = Math.min(point.getCoords()[i], node.getMinCoordinates()[i]); + double max = Math.max(point.getCoords()[i], node.getMinCoordinates()[i]); + expanded *= (max - min); + } + + return (expanded - node.getArea()); + } + + public static void enclose(Node[] nodes) + { + for (Node node : nodes) { + node.enclose(); + } + } + + public static Iterable getBitmaps(ImmutableRTree tree) + { + return depthFirstSearch(tree.getRoot()); + } + + public static Iterable depthFirstSearch(ImmutableNode node) + { + if (node.isLeaf()) { + return Iterables.transform( + node.getChildren(), + new Function() + { + @Override + public ImmutablePoint apply(ImmutableNode tNode) + { + return new ImmutablePoint(tNode); + } + } + ); + } else { + return Iterables.concat( + Iterables.transform( + + node.getChildren(), + new Function>() + { + @Override + public Iterable apply(ImmutableNode child) + { + return depthFirstSearch(child); + } + } + ) + ); + } + } + + public static void print(RTree tree) + { + System.out.printf("numDims : %d%n", tree.getNumDims()); + try { + printRTreeNode(tree.getRoot(), 0); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + public static void print(ImmutableRTree tree) + { + System.out.printf("numDims : %d%n", tree.getNumDims()); + try { + printNode(tree.getRoot(), 0); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + public static void printRTreeNode(Node node, int level) throws Exception + { + System.out.printf( + "%sminCoords: %s, maxCoords: %s, numChildren: %d, isLeaf:%s%n", + makeDashes(level), + jsonMapper.writeValueAsString(node.getMinCoordinates()), + jsonMapper.writeValueAsString( + node.getMaxCoordinates() + ), + node.getChildren().size(), + node.isLeaf() + ); + if (node.isLeaf()) { + for (Node child : node.getChildren()) { + Point point = (Point) (child); + System.out + .printf( + "%scoords: %s, conciseSet: %s%n", + makeDashes(level), + jsonMapper.writeValueAsString(point.getCoords()), + point.getBitmap() + ); + } + } else { + level++; + for (Node child : node.getChildren()) { + printRTreeNode(child, level); + } + } + } + + public static boolean verifyEnclose(Node node) + { + for (Node child : node.getChildren()) { + for (int i = 0; i < node.getNumDims(); i++) { + if (child.getMinCoordinates()[i] < node.getMinCoordinates()[i] + || child.getMaxCoordinates()[i] > node.getMaxCoordinates()[i]) { + return false; + } + } + } + + if (!node.isLeaf()) { + for (Node child : node.getChildren()) { + if (!verifyEnclose(child)) { + return false; + } + } + } + + return true; + } + + public static boolean verifyEnclose(ImmutableNode node) + { + for (ImmutableNode child : node.getChildren()) { + for (int i = 0; i < node.getNumDims(); i++) { + if (child.getMinCoordinates()[i] < node.getMinCoordinates()[i] + || child.getMaxCoordinates()[i] > node.getMaxCoordinates()[i]) { + return false; + } + } + } + + if (!node.isLeaf()) { + for (ImmutableNode child : node.getChildren()) { + if (!verifyEnclose(child)) { + return false; + } + } + } + + return true; + } + + private static void printNode(ImmutableNode node, int level) throws Exception + { + System.out.printf( + "%sminCoords: %s, maxCoords: %s, numChildren: %d, isLeaf: %s%n", + makeDashes(level), + jsonMapper.writeValueAsString(node.getMinCoordinates()), + jsonMapper.writeValueAsString( + node.getMaxCoordinates() + ), + node.getNumChildren(), + node.isLeaf() + ); + if (node.isLeaf()) { + for (ImmutableNode immutableNode : node.getChildren()) { + ImmutablePoint point = new ImmutablePoint(immutableNode); + System.out + .printf( + "%scoords: %s, conciseSet: %s%n", + makeDashes(level), + jsonMapper.writeValueAsString(point.getCoords()), + point.getImmutableBitmap() + ); + } + } else { + level++; + for (ImmutableNode immutableNode : node.getChildren()) { + printNode(immutableNode, level); + } + } + } + + private static String makeDashes(int level) + { + String retVal = ""; + for (int i = 0; i < level; i++) { + retVal += "-"; + } + return retVal; + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/Bound.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/Bound.java new file mode 100755 index 000000000000..17d8934234bf --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/Bound.java @@ -0,0 +1,48 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.druid.collections.spatial.ImmutableNode; +import io.druid.collections.spatial.ImmutablePoint; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "rectangular", value = RectangularBound.class), + @JsonSubTypes.Type(name = "radius", value = RadiusBound.class), + @JsonSubTypes.Type(name = "polygon", value = PolygonBound.class) +}) +public interface Bound +{ + public int getLimit(); + + public int getNumDims(); + + public boolean overlaps(ImmutableNode node); + + public boolean contains(float[] coords); + + public Iterable filter(Iterable points); + + public byte[] getCacheKey(); +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java new file mode 100755 index 000000000000..6dc635f139bb --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/GutmanSearchStrategy.java @@ -0,0 +1,210 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.ImmutableNode; +import io.druid.collections.spatial.ImmutablePoint; + +/** + */ +public class GutmanSearchStrategy implements SearchStrategy +{ + @Override + public Iterable search(ImmutableNode node, Bound bound) + { + if (bound.getLimit() > 0) { + return Iterables.transform( + breadthFirstSearch(node, bound), + new Function() + { + @Override + public ImmutableBitmap apply(ImmutableNode immutableNode) + { + return immutableNode.getImmutableBitmap(); + } + } + ); + } + + return Iterables.transform( + depthFirstSearch(node, bound), + new Function() + { + @Override + public ImmutableBitmap apply(ImmutablePoint immutablePoint) + { + return immutablePoint.getImmutableBitmap(); + } + } + ); + } + + public Iterable depthFirstSearch(ImmutableNode node, final Bound bound) + { + if (node.isLeaf()) { + return bound.filter( + Iterables.transform( + node.getChildren(), + new Function() + { + @Override + public ImmutablePoint apply(ImmutableNode tNode) + { + return new ImmutablePoint(tNode); + } + } + ) + ); + } else { + return Iterables.concat( + Iterables.transform( + Iterables.filter( + node.getChildren(), + new Predicate() + { + @Override + public boolean apply(ImmutableNode child) + { + return bound.overlaps(child); + } + } + ), + new Function>() + { + @Override + public Iterable apply(ImmutableNode child) + { + return depthFirstSearch(child, bound); + } + } + ) + ); + } + } + + public Iterable breadthFirstSearch( + ImmutableNode node, + final Bound bound + ) + { + if (node.isLeaf()) { + return Iterables.filter( + node.getChildren(), + new Predicate() + { + @Override + public boolean apply(ImmutableNode immutableNode) + { + return bound.contains(immutableNode.getMinCoordinates()); + } + } + ); + } + return breadthFirstSearch(node.getChildren(), bound, 0); + } + + public Iterable breadthFirstSearch( + Iterable nodes, + final Bound bound, + int total + ) + { + Iterable points = Iterables.concat( + Iterables.transform( + Iterables.filter( + nodes, + new Predicate() + { + @Override + public boolean apply(ImmutableNode immutableNode) + { + return immutableNode.isLeaf(); + } + } + ), + new Function>() + { + @Override + public Iterable apply(ImmutableNode immutableNode) + { + return Iterables.filter( + immutableNode.getChildren(), + new Predicate() + { + @Override + public boolean apply(ImmutableNode immutableNode) + { + return bound.contains(immutableNode.getMinCoordinates()); + } + } + ); + } + } + ) + ); + + Iterable overlappingNodes = Iterables.filter( + nodes, + new Predicate() + { + @Override + public boolean apply(ImmutableNode immutableNode) + { + return !immutableNode.isLeaf() && bound.overlaps(immutableNode); + } + } + ); + + int totalPoints = Iterables.size(points); + int totalOverlap = Iterables.size(overlappingNodes); + + if (totalOverlap == 0 || (totalPoints + totalOverlap + total) >= bound.getLimit()) { + return Iterables.concat( + points, + overlappingNodes + ); + } else { + return Iterables.concat( + points, + breadthFirstSearch( + Iterables.concat( + Iterables.transform( + overlappingNodes, + new Function>() + { + @Override + public Iterable apply(ImmutableNode immutableNode) + { + return immutableNode.getChildren(); + } + } + ) + ), + bound, + totalPoints + ) + ); + } + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/PolygonBound.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/PolygonBound.java new file mode 100755 index 000000000000..6bc9bb26eedd --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/PolygonBound.java @@ -0,0 +1,176 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import io.druid.collections.spatial.ImmutablePoint; + +import java.nio.ByteBuffer; + +/** + */ +public class PolygonBound extends RectangularBound +{ + private static final byte CACHE_TYPE_ID = 0x02; + + private final float[] abscissa; + private final float[] ordinate; + + private PolygonBound(float[] abscissa, float[] ordinate, int limit) + { + super(getMinCoords(abscissa, ordinate), getMaxCoords(abscissa, ordinate), limit); + this.abscissa = abscissa; + this.ordinate = ordinate; + } + + private static float[] getMinCoords(float[] abscissa, float[] ordinate) + { + float[] retVal = new float[2]; + retVal[0] = abscissa[0]; + retVal[1] = ordinate[0]; + + for (int i = 1; i < abscissa.length; i++) { + if (abscissa[i] < retVal[0]) { + retVal[0] = abscissa[i]; + } + if (ordinate[i] < retVal[1]) { + retVal[1] = ordinate[i]; + } + } + return retVal; + } + + private static float[] getMaxCoords(float[] abscissa, float[] ordinate) + { + float[] retVal = new float[2]; + retVal[0] = abscissa[0]; + retVal[1] = ordinate[0]; + for (int i = 1; i < abscissa.length; i++) { + if (abscissa[i] > retVal[0]) { + retVal[0] = abscissa[i]; + } + if (ordinate[i] > retVal[1]) { + retVal[1] = ordinate[i]; + } + } + return retVal; + } + + /** + * abscissa and ordinate contain the coordinates of polygon. + * abscissa[i] is the horizontal coordinate for the i'th corner of the polygon, + * and ordinate[i] is the vertical coordinate for the i'th corner. + * The polygon must have more than 2 corners, so the length of abscissa or ordinate must be equal or greater than 3. + * + * if the polygon is a rectangular, which corners are {0.0, 0.0}, {0.0, 1.0}, {1.0, 1.0}, {1.0, 0.0}, + * the abscissa should be {0.0, 0.0, 1.0, 1.0} and ordinate should be {0.0, 1.0, 1.0, 0.0} + */ + @JsonCreator + public static PolygonBound from( + @JsonProperty("abscissa") float[] abscissa, + @JsonProperty("ordinate") float[] ordinate, + @JsonProperty("limit") int limit + ) + { + Preconditions.checkArgument(abscissa.length == ordinate.length); //abscissa and ordinate should be the same length + Preconditions.checkArgument(abscissa.length > 2); //a polygon should have more than 2 corners + return new PolygonBound(abscissa, ordinate, limit); + } + + public static PolygonBound from(float[] abscissa, float[] ordinate) + { + return PolygonBound.from(abscissa, ordinate, 0); + } + + @JsonProperty + public float[] getOrdinate() + { + return ordinate; + } + + @JsonProperty + public float[] getAbscissa() + { + return abscissa; + } + + @Override + public boolean contains(float[] coords) + { + int polyCorners = abscissa.length; + int j = polyCorners - 1; + boolean oddNodes = false; + for (int i = 0; i < polyCorners; i++) { + if ((ordinate[i] < coords[1] && ordinate[j] >= coords[1] + || ordinate[j] < coords[1] && ordinate[i] >= coords[1]) + && (abscissa[i] <= coords[0] || abscissa[j] <= coords[0])) { + if (abscissa[i] + (coords[1] - ordinate[i]) / (ordinate[j] - ordinate[i]) * (abscissa[j] - abscissa[i]) + < coords[0]) { + oddNodes = !oddNodes; + } + } + j = i; + } + return oddNodes; + } + + @Override + public Iterable filter(Iterable points) + { + return Iterables.filter( + points, + new Predicate() + { + @Override + public boolean apply(ImmutablePoint immutablePoint) + { + return contains(immutablePoint.getCoords()); + } + } + ); + } + + @Override + public byte[] getCacheKey() + { + ByteBuffer abscissaBuffer = ByteBuffer.allocate(abscissa.length * Floats.BYTES); + abscissaBuffer.asFloatBuffer().put(abscissa); + final byte[] abscissaCacheKey = abscissaBuffer.array(); + + ByteBuffer ordinateBuffer = ByteBuffer.allocate(ordinate.length * Floats.BYTES); + ordinateBuffer.asFloatBuffer().put(ordinate); + final byte[] ordinateCacheKey = ordinateBuffer.array(); + + final ByteBuffer cacheKey = ByteBuffer + .allocate(1 + abscissaCacheKey.length + ordinateCacheKey.length + Ints.BYTES) + .put(abscissaCacheKey) + .put(ordinateCacheKey) + .putInt(getLimit()) + .put(CACHE_TYPE_ID); + + return cacheKey.array(); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RadiusBound.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RadiusBound.java new file mode 100755 index 000000000000..cb905062d61f --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RadiusBound.java @@ -0,0 +1,132 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import io.druid.collections.spatial.ImmutablePoint; + +import java.nio.ByteBuffer; + +/** + */ +public class RadiusBound extends RectangularBound +{ + private static final byte CACHE_TYPE_ID = 0x01; + private final float[] coords; + private final float radius; + + @JsonCreator + public RadiusBound( + @JsonProperty("coords") float[] coords, + @JsonProperty("radius") float radius, + @JsonProperty("limit") int limit + ) + { + super(getMinCoords(coords, radius), getMaxCoords(coords, radius), limit); + + this.coords = coords; + this.radius = radius; + } + + public RadiusBound( + float[] coords, + float radius + ) + { + this(coords, radius, 0); + } + + private static float[] getMinCoords(float[] coords, float radius) + { + float[] retVal = new float[coords.length]; + for (int i = 0; i < coords.length; i++) { + retVal[i] = coords[i] - radius; + } + return retVal; + } + + private static float[] getMaxCoords(float[] coords, float radius) + { + float[] retVal = new float[coords.length]; + for (int i = 0; i < coords.length; i++) { + retVal[i] = coords[i] + radius; + } + return retVal; + } + + @JsonProperty + public float[] getCoords() + { + return coords; + } + + @JsonProperty + public float getRadius() + { + return radius; + } + + @Override + public boolean contains(float[] otherCoords) + { + double total = 0.0; + for (int i = 0; i < coords.length; i++) { + total += Math.pow(otherCoords[i] - coords[i], 2); + } + + return (total <= Math.pow(radius, 2)); + } + + @Override + public Iterable filter(Iterable points) + { + return Iterables.filter( + points, + new Predicate() + { + @Override + public boolean apply(ImmutablePoint point) + { + return contains(point.getCoords()); + } + } + ); + } + + @Override + public byte[] getCacheKey() + { + final ByteBuffer minCoordsBuffer = ByteBuffer.allocate(coords.length * Floats.BYTES); + minCoordsBuffer.asFloatBuffer().put(coords); + final byte[] minCoordsCacheKey = minCoordsBuffer.array(); + final ByteBuffer cacheKey = ByteBuffer + .allocate(1 + minCoordsCacheKey.length + Ints.BYTES + Floats.BYTES) + .put(minCoordsCacheKey) + .putFloat(radius) + .putInt(getLimit()) + .put(CACHE_TYPE_ID); + return cacheKey.array(); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RectangularBound.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RectangularBound.java new file mode 100755 index 000000000000..0d6ab38f7818 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/RectangularBound.java @@ -0,0 +1,155 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.primitives.Floats; +import com.google.common.primitives.Ints; +import io.druid.collections.spatial.ImmutableNode; +import io.druid.collections.spatial.ImmutablePoint; + +import java.nio.ByteBuffer; + +/** + */ +public class RectangularBound implements Bound +{ + private static final byte CACHE_TYPE_ID = 0x0; + + private final float[] minCoords; + private final float[] maxCoords; + private final int limit; + private final int numDims; + + @JsonCreator + public RectangularBound( + @JsonProperty("minCoords") float[] minCoords, + @JsonProperty("maxCoords") float[] maxCoords, + @JsonProperty("limit") int limit + ) + { + Preconditions.checkArgument(minCoords.length == maxCoords.length); + + this.numDims = minCoords.length; + + this.minCoords = minCoords; + this.maxCoords = maxCoords; + this.limit = limit; + } + + public RectangularBound( + float[] minCoords, + float[] maxCoords + ) + { + this(minCoords, maxCoords, 0); + } + + @JsonProperty + public float[] getMinCoords() + { + return minCoords; + } + + @JsonProperty + public float[] getMaxCoords() + { + return maxCoords; + } + + @JsonProperty + public int getLimit() + { + return limit; + } + + @Override + public int getNumDims() + { + return numDims; + } + + @Override + public boolean overlaps(ImmutableNode node) + { + final float[] nodeMinCoords = node.getMinCoordinates(); + final float[] nodeMaxCoords = node.getMaxCoordinates(); + + for (int i = 0; i < numDims; i++) { + if (nodeMaxCoords[i] < minCoords[i] || nodeMinCoords[i] > maxCoords[i]) { + return false; + } + } + + return true; + } + + @Override + public boolean contains(float[] coords) + { + for (int i = 0; i < numDims; i++) { + if (coords[i] < minCoords[i] || coords[i] > maxCoords[i]) { + return false; + } + } + + return true; + } + + @Override + public Iterable filter(Iterable points) + { + return Iterables.filter( + points, + new Predicate() + { + @Override + public boolean apply(ImmutablePoint immutablePoint) + { + return contains(immutablePoint.getCoords()); + } + } + ); + } + + @Override + public byte[] getCacheKey() + { + ByteBuffer minCoordsBuffer = ByteBuffer.allocate(minCoords.length * Floats.BYTES); + minCoordsBuffer.asFloatBuffer().put(minCoords); + final byte[] minCoordsCacheKey = minCoordsBuffer.array(); + + ByteBuffer maxCoordsBuffer = ByteBuffer.allocate(maxCoords.length * Floats.BYTES); + maxCoordsBuffer.asFloatBuffer().put(maxCoords); + final byte[] maxCoordsCacheKey = maxCoordsBuffer.array(); + + final ByteBuffer cacheKey = ByteBuffer + .allocate(1 + minCoordsCacheKey.length + maxCoordsCacheKey.length + Ints.BYTES) + .put(minCoordsCacheKey) + .put(maxCoordsCacheKey) + .putInt(limit) + .put(CACHE_TYPE_ID); + return cacheKey.array(); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java new file mode 100755 index 000000000000..c018a5a92972 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/search/SearchStrategy.java @@ -0,0 +1,32 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.ImmutableNode; +//import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + + +/** + */ +public interface SearchStrategy +{ + public Iterable search(ImmutableNode node, Bound bound); +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java new file mode 100755 index 000000000000..fb425f46e100 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/GutmanSplitStrategy.java @@ -0,0 +1,135 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.split; + +import com.google.common.collect.Lists; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.spatial.Node; +import io.druid.collections.spatial.RTreeUtils; + +import java.util.Arrays; +import java.util.List; + +/** + */ +public abstract class GutmanSplitStrategy implements SplitStrategy +{ + private final int minNumChildren; + private final int maxNumChildren; + private final BitmapFactory bf; + + protected GutmanSplitStrategy(int minNumChildren, int maxNumChildren, BitmapFactory b) + { + this.minNumChildren = minNumChildren; + this.maxNumChildren = maxNumChildren; + this.bf = b; + } + + @Override + public boolean needToSplit(Node node) + { + return (node.getChildren().size() > maxNumChildren); + } + + /** + * This algorithm is from the original paper. + * + * Algorithm Split. Divide a set of M+1 index entries into two groups. + * + * S1. [Pick first entry for each group]. Apply Algorithm {@link #pickSeeds(java.util.List)} to choose + * two entries to be the first elements of the groups. Assign each to a group. + * + * S2. [Check if done]. If all entries have been assigned, stop. If one group has so few entries that all the rest + * must be assigned to it in order for it to have the minimum number m, assign them and stop. + * + * S3. [Select entry to assign]. Invoke Algorithm {@link #pickNext(java.util.List, Node[])} + * to choose the next entry to assign. Add it to the group whose covering rectangle will have to be enlarged least to + * accommodate it. Resolve ties by adding the entry to the group smaller area, then to the one with fewer entries, then + * to either. Repeat from S2. + */ + @Override + public Node[] split(Node node) + { + List children = Lists.newArrayList(node.getChildren()); + Node[] seeds = pickSeeds(children); + + node.clear(); + node.addChild(seeds[0]); + node.addToBitmapIndex(seeds[0]); + + Node group1 = new Node( + Arrays.copyOf(seeds[1].getMinCoordinates(), seeds[1].getMinCoordinates().length), + Arrays.copyOf(seeds[1].getMaxCoordinates(), seeds[1].getMaxCoordinates().length), + Lists.newArrayList(seeds[1]), + node.isLeaf(), + node.getParent(), + bf.makeEmptyMutableBitmap() + ); + group1.addToBitmapIndex(seeds[1]); + if (node.getParent() != null) { + node.getParent().addChild(group1); + } + Node[] groups = new Node[]{ + node, group1 + }; + + RTreeUtils.enclose(groups); + + while (!children.isEmpty()) { + for (Node group : groups) { + if (group.getChildren().size() + children.size() <= minNumChildren) { + for (Node child : children) { + group.addToBitmapIndex(child); + group.addChild(child); + } + RTreeUtils.enclose(groups); + return groups; + } + } + + Node nextToAssign = pickNext(children, groups); + double group0ExpandedArea = RTreeUtils.getEnclosingArea(groups[0], nextToAssign); + double group1ExpandedArea = RTreeUtils.getEnclosingArea(groups[1], nextToAssign); + + Node optimal; + if (group0ExpandedArea < group1ExpandedArea) { + optimal = groups[0]; + } else if (group0ExpandedArea == group1ExpandedArea) { + if (groups[0].getArea() < groups[1].getArea()) { + optimal = groups[0]; + } else { + optimal = groups[1]; + } + } else { + optimal = groups[1]; + } + + optimal.addToBitmapIndex(nextToAssign); + optimal.addChild(nextToAssign); + optimal.enclose(); + } + + return groups; + } + + public abstract Node[] pickSeeds(List nodes); + + public abstract Node pickNext(List nodes, Node[] groups); +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java new file mode 100755 index 000000000000..a193f466cff5 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategy.java @@ -0,0 +1,119 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.split; + +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.spatial.Node; + +import java.util.List; + +/** + */ +public class LinearGutmanSplitStrategy extends GutmanSplitStrategy +{ + public LinearGutmanSplitStrategy(int minNumChildren, int maxNumChildren, BitmapFactory bf) + { + super(minNumChildren, maxNumChildren, bf); + } + + /** + * This algorithm is from the original paper. + * + * Algorithm LinearPickSeeds. Select two entries to be the first elements of the groups. + * + * LPS1. [Find extreme rectangles along all dimensions]. Along each dimension, find the entry whose rectangle has + * the highest low side, and the one with the lowest high side. Record the separation. + * + * LPS2. [Adjust for shape of the rectangle cluster]. Normalize the separations by dividing by the width of the + * entire set along the corresponding dimension. + * + * LPS3. [Select the most extreme pair]. Choose the pair with the greatest normalized separation along any dimension. + * + * @param nodes - nodes to choose from + * + * @return - two groups representing the seeds + */ + @Override + public Node[] pickSeeds(List nodes) + { + int[] optimalIndices = new int[2]; + int numDims = nodes.get(0).getNumDims(); + + double bestNormalized = 0.0; + for (int i = 0; i < numDims; i++) { + float minCoord = Float.MAX_VALUE; + float maxCoord = -Float.MAX_VALUE; + float highestLowSide = -Float.MAX_VALUE; + float lowestHighside = Float.MAX_VALUE; + int highestLowSideIndex = 0; + int lowestHighSideIndex = 0; + + int counter = 0; + for (Node node : nodes) { + minCoord = Math.min(minCoord, node.getMinCoordinates()[i]); + maxCoord = Math.max(maxCoord, node.getMaxCoordinates()[i]); + + if (node.getMinCoordinates()[i] > highestLowSide) { + highestLowSide = node.getMinCoordinates()[i]; + highestLowSideIndex = counter; + } + if (node.getMaxCoordinates()[i] < lowestHighside) { + lowestHighside = node.getMaxCoordinates()[i]; + lowestHighSideIndex = counter; + } + + counter++; + } + double normalizedSeparation = (highestLowSideIndex == lowestHighSideIndex) ? -1.0 : + Math.abs((highestLowSide - lowestHighside) / (maxCoord - minCoord)); + if (normalizedSeparation > bestNormalized) { + optimalIndices[0] = highestLowSideIndex; + optimalIndices[1] = lowestHighSideIndex; + bestNormalized = normalizedSeparation; + } + } + + // Didn't actually find anything, just return first 2 children + if (bestNormalized == 0) { + optimalIndices[0] = 0; + optimalIndices[1] = 1; + } + + int indexToRemove1 = Math.min(optimalIndices[0], optimalIndices[1]); + int indexToRemove2 = Math.max(optimalIndices[0], optimalIndices[1]); + return new Node[]{nodes.remove(indexToRemove1), nodes.remove(indexToRemove2 - 1)}; + } + + /** + * This algorithm is from the original paper. + * + * Algorithm LinearPickNext. PickNext simply choose any of the remaining entries. + * + * @param nodes - remaining nodes + * @param groups - the left and right groups + * + * @return - the optimal selected node + */ + @Override + public Node pickNext(List nodes, Node[] groups) + { + return nodes.remove(0); + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/QuadraticGutmanSplitStrategy.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/QuadraticGutmanSplitStrategy.java new file mode 100755 index 000000000000..444fc5ecaeb0 --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/QuadraticGutmanSplitStrategy.java @@ -0,0 +1,83 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.split; + +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.spatial.Node; +import io.druid.collections.spatial.RTreeUtils; + +import java.util.List; + +/** + */ +public class QuadraticGutmanSplitStrategy extends GutmanSplitStrategy +{ + public QuadraticGutmanSplitStrategy(int minNumChildren, int maxNumChildren, BitmapFactory bf) + { + super(minNumChildren, maxNumChildren, bf); + } + + @Override + public Node[] pickSeeds(List nodes) + { + double highestCost = Double.MIN_VALUE; + int[] highestCostIndices = new int[2]; + + for (int i = 0; i < nodes.size() - 1; i++) { + for (int j = i + 1; j < nodes.size(); j++) { + double cost = RTreeUtils.getEnclosingArea(nodes.get(i), nodes.get(j)) - + nodes.get(i).getArea() - nodes.get(j).getArea(); + if (cost > highestCost) { + highestCost = cost; + highestCostIndices[0] = i; + highestCostIndices[1] = j; + } + } + } + + return new Node[]{nodes.remove(highestCostIndices[0]), nodes.remove(highestCostIndices[1] - 1)}; + } + + @Override + public Node pickNext(List nodes, Node[] groups) + { + double highestCost = Double.MIN_VALUE; + Node costlyNode = null; + int counter = 0; + int index = -1; + for (Node node : nodes) { + double group0Cost = RTreeUtils.getEnclosingArea(node, groups[0]); + double group1Cost = RTreeUtils.getEnclosingArea(node, groups[1]); + double cost = Math.abs(group0Cost - group1Cost); + if (cost > highestCost) { + highestCost = cost; + costlyNode = node; + index = counter; + } + counter++; + } + + if (costlyNode != null) { + nodes.remove(index); + } + + return costlyNode; + } +} diff --git a/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java new file mode 100755 index 000000000000..5b1f8a46c77d --- /dev/null +++ b/bytebuffer-collections/src/main/java/io/druid/collections/spatial/split/SplitStrategy.java @@ -0,0 +1,31 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.split; + +import io.druid.collections.spatial.Node; + +/** + */ +public interface SplitStrategy +{ + public boolean needToSplit(Node node); + + public Node[] split(Node node); +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/IntSetTestUtility.java b/bytebuffer-collections/src/test/java/io/druid/collections/IntSetTestUtility.java new file mode 100755 index 000000000000..ac2a54ab2d1d --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/IntSetTestUtility.java @@ -0,0 +1,114 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import org.roaringbitmap.IntIterator; + +import java.util.BitSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +/** + * + */ +public class IntSetTestUtility +{ + + private static Set setBits = Sets.newTreeSet(Lists.newArrayList(1, 2, 3, 5, 8, 13, 21)); + + public static Set getSetBits() + { + return Sets.newTreeSet(setBits); + } + + public static final BitSet createSimpleBitSet(Set setBits) + { + BitSet retval = new BitSet(); + for (int i : setBits) { + retval.set(i); + } + return retval; + } + + public static final void addAllToMutable(MutableBitmap mutableBitmap, Iterable intSet) + { + for (Integer integer : intSet) { + mutableBitmap.add(integer); + } + } + + public static Boolean equalSets(Set s1, ImmutableBitmap s2) + { + Set s3 = new HashSet<>(); + for (Integer i : new IntIt(s2.iterator())) { + s3.add(i); + } + return Sets.difference(s1, s3).isEmpty(); + } + + private static class IntIt implements Iterable + { + private final Iterator intIter; + + public IntIt(IntIterator intIt) + { + this.intIter = new IntIter(intIt); + } + + @Override + public Iterator iterator() + { + return intIter; + } + + private static class IntIter implements Iterator + { + private final IntIterator intIt; + + public IntIter(IntIterator intIt) + { + this.intIt = intIt; + } + + @Override + public boolean hasNext() + { + return intIt.hasNext(); + } + + @Override + public Integer next() + { + return intIt.next(); + } + + @Override + public void remove() + { + throw new UnsupportedOperationException("Cannot remove ints from int iterator"); + } + } + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/TestIntegerSet.java b/bytebuffer-collections/src/test/java/io/druid/collections/TestIntegerSet.java new file mode 100755 index 000000000000..a837bd972b26 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/TestIntegerSet.java @@ -0,0 +1,242 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.WrappedBitSetBitmap; +import io.druid.collections.bitmap.WrappedConciseBitmap; +import io.druid.collections.bitmap.WrappedRoaringBitmap; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Set; + +/** + * + */ +public class TestIntegerSet +{ + private static Iterable> clazzes = Lists.newArrayList( + WrappedBitSetBitmap.class, + WrappedConciseBitmap.class, + WrappedRoaringBitmap.class + ); + + @Test + public void testSimpleSet() + { + WrappedBitSetBitmap wrappedBitSetBitmapBitSet = new WrappedBitSetBitmap(); + IntSetTestUtility.addAllToMutable(wrappedBitSetBitmapBitSet, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitSetBitmapBitSet); + + Assert.assertTrue(Sets.difference(integerSet, IntSetTestUtility.getSetBits()).isEmpty()); + } + + @Test + public void testSimpleAdd() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Set set = IntSetTestUtility.getSetBits(); + set.add(999); + integerSet.add(999); + + Assert.assertTrue(Sets.difference(integerSet, set).isEmpty()); + + integerSet.add(58577); + Assert.assertFalse(Sets.difference(integerSet, set).isEmpty()); + } + } + + @Test + public void testContainsAll() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Set set = IntSetTestUtility.getSetBits(); + Assert.assertTrue(integerSet.containsAll(set)); + + set.add(999); + Assert.assertFalse(integerSet.containsAll(set)); + } + } + + @Test + public void testRemoveEverything() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Set set = IntSetTestUtility.getSetBits(); + + integerSet.removeAll(set); + boolean isEmpty = integerSet.isEmpty(); + Assert.assertTrue(isEmpty); + } + } + + @Test + public void testRemoveOneThing() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Set set = IntSetTestUtility.getSetBits(); + + integerSet.remove(1); + set.remove(1); + + Assert.assertTrue(Sets.difference(set, integerSet).isEmpty()); + } + } + + + @Test + public void testIsEmpty() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Assert.assertFalse(integerSet.isEmpty()); + + integerSet.clear(); + + Assert.assertTrue(integerSet.isEmpty()); + + integerSet.add(1); + Assert.assertFalse(integerSet.isEmpty()); + } + } + + @Test + public void testSize() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Set set = IntSetTestUtility.getSetBits(); + + Assert.assertEquals(set.size(), integerSet.size()); + } + } + + + @Test + public void testRetainAll() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Set set = IntSetTestUtility.getSetBits(); + + set.remove(1); + set.add(9999); + + boolean threwError = false; + try { + integerSet.retainAll(set); + } + catch (UnsupportedOperationException ex) { + threwError = true; + } + Assert.assertTrue(threwError); + } + } + + @Test + public void testIntOverflow() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + Exception e = null; + try { + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + integerSet.add(Integer.MAX_VALUE + 1); + } + catch (java.lang.IllegalArgumentException ex) { + e = ex; + } + Assert.assertNotNull(e); + } + } + + @Test + public void testToArray() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + Exception e = null; + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + Set set = Sets.newHashSet((Integer[]) integerSet.toArray()); + Assert.assertTrue(Sets.difference(integerSet, set).isEmpty()); + } + } + + + @Test + public void testToSmallArray() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + Exception e = null; + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + Set set = Sets.newHashSet((Integer[]) integerSet.toArray(new Integer[0])); + Assert.assertTrue(Sets.difference(integerSet, set).isEmpty()); + } + } + + + @Test + public void testToBigArray() throws IllegalAccessException, InstantiationException + { + for (Class clazz : clazzes) { + Exception e = null; + MutableBitmap wrappedBitmap = clazz.newInstance(); + IntSetTestUtility.addAllToMutable(wrappedBitmap, IntSetTestUtility.getSetBits()); + IntegerSet integerSet = IntegerSet.wrap(wrappedBitmap); + + Integer[] bigArray = new Integer[1024]; + integerSet.toArray(bigArray); + Set set = Sets.newHashSet(bigArray); + Assert.assertTrue(Sets.difference(integerSet, set).isEmpty()); + } + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java new file mode 100755 index 000000000000..3366319adaaf --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/BitmapBenchmark.java @@ -0,0 +1,199 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Random; + +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; +import org.roaringbitmap.buffer.BufferFastAggregation; +import org.roaringbitmap.buffer.ImmutableRoaringBitmap; +import org.roaringbitmap.buffer.MutableRoaringBitmap; + +import com.carrotsearch.junitbenchmarks.BenchmarkOptions; +import com.carrotsearch.junitbenchmarks.BenchmarkRule; +import com.carrotsearch.junitbenchmarks.Clock; +import com.google.common.collect.Lists; + +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + + +@BenchmarkOptions(clock = Clock.NANO_TIME, benchmarkRounds = 50) +public class BitmapBenchmark +{ + public static final int LENGTH = 500_000; + public static final int SIZE = 10_000; + final static ImmutableConciseSet concise[] = new ImmutableConciseSet[SIZE]; + final static ImmutableConciseSet offheapConcise[] = new ImmutableConciseSet[SIZE]; + final static ImmutableRoaringBitmap roaring[] = new ImmutableRoaringBitmap[SIZE]; + final static ImmutableRoaringBitmap immutableRoaring[] = new ImmutableRoaringBitmap[SIZE]; + final static ImmutableRoaringBitmap offheapRoaring[] = new ImmutableRoaringBitmap[SIZE]; + final static ImmutableBitmap genericConcise[] = new ImmutableBitmap[SIZE]; + final static ImmutableBitmap genericRoaring[] = new ImmutableBitmap[SIZE]; + final static ConciseBitmapFactory conciseFactory = new ConciseBitmapFactory(); + final static RoaringBitmapFactory roaringFactory = new RoaringBitmapFactory(); + static Random rand = new Random(0); + static long totalConciseBytes = 0; + static long totalRoaringBytes = 0; + static long conciseCount = 0; + static long roaringCount = 0; + static long unionCount = 0; + static long minIntersection = 0; + @Rule + public TestRule benchmarkRun = new BenchmarkRule(); + + protected static ImmutableConciseSet makeOffheapConcise(ImmutableConciseSet concise) + { + final byte[] bytes = concise.toBytes(); + totalConciseBytes += bytes.length; + conciseCount++; + final ByteBuffer buf = ByteBuffer.allocateDirect(bytes.length).put(bytes); + buf.rewind(); + return new ImmutableConciseSet(buf); + } + + protected static ImmutableRoaringBitmap writeImmutable(MutableRoaringBitmap r, ByteBuffer buf) throws IOException + { + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + r.serialize(new DataOutputStream(out)); + final byte[] bytes = out.toByteArray(); + Assert.assertEquals(buf.remaining(), bytes.length); + buf.put(bytes); + buf.rewind(); + return new ImmutableRoaringBitmap(buf.asReadOnlyBuffer()); + } + + protected static void reset() + { + conciseCount = 0; + roaringCount = 0; + totalConciseBytes = 0; + totalRoaringBytes = 0; + unionCount = 0; + minIntersection = 0; + rand = new Random(0); + } + + protected static void printSizeStats(double density, String name) + { + System.out.println(""); + System.out.println("## " + name); + System.out.println(""); + System.out.printf(" d = %06.5f | Concise | Roaring" + System.lineSeparator(), density); + System.out.println("-------------|---------|---------"); + System.out.printf("Count | %5d | %5d " + System.lineSeparator(), conciseCount, roaringCount); + System.out.printf( + "Average size | %5d | %5d " + System.lineSeparator(), + totalConciseBytes / conciseCount, + totalRoaringBytes / roaringCount + ); + System.out.println("-------------|---------|---------"); + System.out.println(""); + System.out.flush(); + } + + protected static ImmutableRoaringBitmap makeOffheapRoaring(MutableRoaringBitmap r) throws IOException + { + final int size = r.serializedSizeInBytes(); + final ByteBuffer buf = ByteBuffer.allocateDirect(size); + totalRoaringBytes += size; + roaringCount++; + return writeImmutable(r, buf); + } + + protected static ImmutableRoaringBitmap makeImmutableRoaring(MutableRoaringBitmap r) throws IOException + { + final ByteBuffer buf = ByteBuffer.allocate(r.serializedSizeInBytes()); + return writeImmutable(r, buf); + } + + @Test + @BenchmarkOptions(warmupRounds = 1, benchmarkRounds = 2) + public void timeConciseUnion() throws Exception + { + ImmutableConciseSet union = ImmutableConciseSet.union(concise); + Assert.assertEquals(unionCount, union.size()); + } + + @Test + @BenchmarkOptions(warmupRounds = 1, benchmarkRounds = 2) + public void timeOffheapConciseUnion() throws Exception + { + ImmutableConciseSet union = ImmutableConciseSet.union(offheapConcise); + Assert.assertEquals(unionCount, union.size()); + } + + @Test + @BenchmarkOptions(warmupRounds = 1, benchmarkRounds = 2) + public void timeGenericConciseUnion() throws Exception + { + ImmutableBitmap union = conciseFactory.union(Lists.newArrayList(genericConcise)); + Assert.assertEquals(unionCount, union.size()); + } + + @Test + @BenchmarkOptions(warmupRounds = 1, benchmarkRounds = 5) + public void timeGenericConciseIntersection() throws Exception + { + ImmutableBitmap intersection = conciseFactory.intersection(Lists.newArrayList(genericConcise)); + Assert.assertTrue(intersection.size() >= minIntersection); + } + + @Test + public void timeRoaringUnion() throws Exception + { + ImmutableRoaringBitmap union = BufferFastAggregation.horizontal_or(Lists.newArrayList(roaring).iterator()); + Assert.assertEquals(unionCount, union.getCardinality()); + } + + @Test + public void timeImmutableRoaringUnion() throws Exception + { + ImmutableRoaringBitmap union = BufferFastAggregation.horizontal_or(Lists.newArrayList(immutableRoaring).iterator()); + Assert.assertEquals(unionCount, union.getCardinality()); + } + + @Test + public void timeOffheapRoaringUnion() throws Exception + { + ImmutableRoaringBitmap union = BufferFastAggregation.horizontal_or(Lists.newArrayList(offheapRoaring).iterator()); + Assert.assertEquals(unionCount, union.getCardinality()); + } + + @Test + public void timeGenericRoaringUnion() throws Exception + { + ImmutableBitmap union = roaringFactory.union(Lists.newArrayList(genericRoaring)); + Assert.assertEquals(unionCount, union.size()); + } + + @Test + public void timeGenericRoaringIntersection() throws Exception + { + ImmutableBitmap intersection = roaringFactory.intersection(Lists.newArrayList(genericRoaring)); + Assert.assertTrue(intersection.size() >= minIntersection); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java new file mode 100755 index 000000000000..7ab0ce5301e1 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/ConciseBitmapFactoryTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.util.Arrays; +import java.util.Set; + +import org.junit.Test; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +import it.uniroma3.mat.extendedset.intset.ConciseSet; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import junit.framework.Assert; + +public class ConciseBitmapFactoryTest +{ + @Test + public void testUnwrapWithNull() throws Exception + { + ConciseBitmapFactory factory = new ConciseBitmapFactory(); + + ImmutableBitmap bitmap = factory.union( + Iterables.transform( + Lists.newArrayList(new WrappedConciseBitmap()), + new Function() + { + @Override + public ImmutableBitmap apply(WrappedConciseBitmap input) + { + return null; + } + } + ) + ); + + Assert.assertEquals(0, bitmap.size()); + } + + @Test + public void testUnwrapMerge() throws Exception + { + ConciseBitmapFactory factory = new ConciseBitmapFactory(); + + WrappedConciseBitmap set = new WrappedConciseBitmap(); + set.add(1); + set.add(3); + set.add(5); + + ImmutableBitmap bitmap = factory.union( + Arrays.asList( + factory.makeImmutableBitmap(set), + null + ) + ); + + Assert.assertEquals(3, bitmap.size()); + } + + @Test + public void testGetOutOfBounds() + { + final ConciseSet conciseSet = new ConciseSet(); + final Set ints = ImmutableSet.of(0, 4, 9); + for (int i : ints) { + conciseSet.add(i); + } + final ImmutableBitmap immutableBitmap = new WrappedImmutableConciseBitmap( + ImmutableConciseSet.newImmutableFromMutable(conciseSet)); + final MutableBitmap mutableBitmap = new WrappedConciseBitmap(conciseSet); + for (int i = 0; i < 10; ++i) { + Assert.assertEquals(Integer.toString(i), ints.contains(i), mutableBitmap.get(i)); + Assert.assertEquals(Integer.toString(i), ints.contains(i), immutableBitmap.get(i)); + } + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java new file mode 100755 index 000000000000..84c170556cd5 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RangeBitmapBenchmarkTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.util.BitSet; + +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.roaringbitmap.buffer.MutableRoaringBitmap; + +import com.carrotsearch.junitbenchmarks.annotation.BenchmarkHistoryChart; +import com.carrotsearch.junitbenchmarks.annotation.LabelType; + +import io.druid.test.annotation.Benchmark; +import it.uniroma3.mat.extendedset.intset.ConciseSet; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +@Category({Benchmark.class}) +@BenchmarkHistoryChart(labelWith = LabelType.CUSTOM_KEY, maxRuns = 20) +public class RangeBitmapBenchmarkTest extends BitmapBenchmark +{ + + public static final double DENSITY = 0.001; + public static final int MIN_INTERSECT = 50; + + @BeforeClass + public static void prepareRandomRanges() throws Exception + { + System.setProperty("jub.customkey", String.format("%06.5f", DENSITY)); + reset(); + + final BitSet expectedUnion = new BitSet(); + for (int i = 0; i < SIZE; ++i) { + ConciseSet c = new ConciseSet(); + MutableRoaringBitmap r = new MutableRoaringBitmap(); + { + int k = 0; + boolean fill = true; + while (k < LENGTH) { + int runLength = (int) (LENGTH * DENSITY) + rand.nextInt((int) (LENGTH * DENSITY)); + for (int j = k; fill && j < LENGTH && j < k + runLength; ++j) { + c.add(j); + r.add(j); + expectedUnion.set(j); + } + k += runLength; + fill = !fill; + } + } + minIntersection = MIN_INTERSECT; + for (int k = LENGTH / 2; k < LENGTH / 2 + minIntersection; ++k) { + c.add(k); + r.add(k); + expectedUnion.set(k); + } + concise[i] = ImmutableConciseSet.newImmutableFromMutable(c); + offheapConcise[i] = makeOffheapConcise(concise[i]); + roaring[i] = r; + immutableRoaring[i] = makeImmutableRoaring(r); + offheapRoaring[i] = makeOffheapRoaring(r); + genericConcise[i] = new WrappedImmutableConciseBitmap(offheapConcise[i]); + genericRoaring[i] = new WrappedImmutableRoaringBitmap(offheapRoaring[i]); + } + unionCount = expectedUnion.cardinality(); + printSizeStats(DENSITY, "Random Alternating Bitmap"); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java new file mode 100755 index 000000000000..1cfa59e7f5e4 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/RoaringBitmapFactoryTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import com.google.common.base.Function; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Test; +import org.roaringbitmap.IntIterator; + +import java.util.Arrays; + +public class RoaringBitmapFactoryTest +{ + + // testing https://github.com/metamx/bytebuffer-collections/issues/26 + @Test + public void testIssue26() throws Exception + { + checkEmptyComplement(new ConciseBitmapFactory()); + checkEmptyComplement(new RoaringBitmapFactory()); + } + + // used by issue 26 + private void checkEmptyComplement(BitmapFactory bitmapFactory) throws Exception + { + int numRow = 5104234; + ImmutableBitmap bitmap = bitmapFactory.complement(bitmapFactory.makeEmptyImmutableBitmap(), numRow); + ImmutableBitmap notBitmap = bitmapFactory.complement(bitmap, numRow); + Assert.assertTrue(notBitmap.size() == 0); + Assert.assertTrue(notBitmap.isEmpty()); + IntIterator intIter = notBitmap.iterator(); + Assert.assertFalse(intIter.hasNext()); + } + + @Test + public void testUnwrapWithNull() throws Exception + { + RoaringBitmapFactory factory = new RoaringBitmapFactory(); + + ImmutableBitmap bitmap = factory.union( + Iterables.transform( + Lists.newArrayList(new WrappedRoaringBitmap()), + new Function() + { + @Override + public ImmutableBitmap apply(WrappedRoaringBitmap input) + { + return null; + } + } + ) + ); + + Assert.assertEquals(0, bitmap.size()); + } + + @Test + public void testUnwrapMerge() throws Exception + { + RoaringBitmapFactory factory = new RoaringBitmapFactory(); + + WrappedRoaringBitmap set = new WrappedRoaringBitmap(); + set.add(1); + set.add(3); + set.add(5); + + ImmutableBitmap bitmap = factory.union( + Arrays.asList( + factory.makeImmutableBitmap(set), + null + ) + ); + + Assert.assertEquals(3, bitmap.size()); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java new file mode 100755 index 000000000000..a88e3133fd5e --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/UniformBitmapBenchmarkTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import java.util.BitSet; + +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; +import org.roaringbitmap.buffer.MutableRoaringBitmap; + +import com.carrotsearch.junitbenchmarks.annotation.BenchmarkHistoryChart; +import com.carrotsearch.junitbenchmarks.annotation.LabelType; + +import io.druid.test.annotation.Benchmark; +import it.uniroma3.mat.extendedset.intset.ConciseSet; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +@Category({Benchmark.class}) +@BenchmarkHistoryChart(labelWith = LabelType.CUSTOM_KEY, maxRuns = 20) +public class UniformBitmapBenchmarkTest extends BitmapBenchmark +{ + + public static final double DENSITY = 0.01; + public static final int MIN_INTERSECT = 50; + + @BeforeClass + public static void prepareMostlyUniform() throws Exception + { + System.setProperty("jub.customkey", String.format("%05.4f", DENSITY)); + reset(); + + final BitSet expectedUnion = new BitSet(); + final int[] knownTrue = new int[MIN_INTERSECT]; + for (int i = 0; i < knownTrue.length; ++i) { + knownTrue[i] = rand.nextInt(LENGTH); + } + for (int i = 0; i < SIZE; ++i) { + ConciseSet c = new ConciseSet(); + MutableRoaringBitmap r = new MutableRoaringBitmap(); + for (int k = 0; k < LENGTH; ++k) { + if (rand.nextDouble() < DENSITY) { + c.add(k); + r.add(k); + expectedUnion.set(k); + } + } + for (int k : knownTrue) { + c.add(k); + r.add(k); + expectedUnion.set(k); + } + concise[i] = ImmutableConciseSet.newImmutableFromMutable(c); + offheapConcise[i] = makeOffheapConcise(concise[i]); + roaring[i] = r; + immutableRoaring[i] = makeImmutableRoaring(r); + offheapRoaring[i] = makeOffheapRoaring(r); + genericConcise[i] = new WrappedImmutableConciseBitmap(offheapConcise[i]); + genericRoaring[i] = new WrappedImmutableRoaringBitmap(offheapRoaring[i]); + } + unionCount = expectedUnion.cardinality(); + minIntersection = knownTrue.length; + printSizeStats(DENSITY, "Uniform Bitmap"); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java new file mode 100755 index 000000000000..59af18ddf26e --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedBitSetBitmapBitSetTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import com.google.common.collect.Sets; + +import io.druid.collections.IntSetTestUtility; + +import org.junit.Assert; +import org.junit.Test; +import org.roaringbitmap.IntIterator; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.BitSet; +import java.util.Set; + +/** + * + */ +public class WrappedBitSetBitmapBitSetTest +{ + + private static final WrappedBitSetBitmap defaultBitSet() + { + return new WrappedBitSetBitmap(IntSetTestUtility.createSimpleBitSet(IntSetTestUtility.getSetBits())); + } + + @Test + public void testIterator() + { + WrappedBitSetBitmap bitSet = new WrappedBitSetBitmap(); + for (int i : IntSetTestUtility.getSetBits()) { + bitSet.add(i); + } + IntIterator intIt = bitSet.iterator(); + for (int i : IntSetTestUtility.getSetBits()) { + Assert.assertTrue(intIt.hasNext()); + Assert.assertEquals(i, intIt.next()); + } + } + + @Test + public void testSize() + { + BitSet bitSet = IntSetTestUtility.createSimpleBitSet(IntSetTestUtility.getSetBits()); + WrappedBitSetBitmap wrappedBitSetBitmapBitSet = new WrappedBitSetBitmap(bitSet); + Assert.assertEquals(bitSet.cardinality(), wrappedBitSetBitmapBitSet.size()); + } + + @Test + public void testOffHeap() + { + ByteBuffer buffer = ByteBuffer.allocateDirect(Long.SIZE * 100 / 8).order(ByteOrder.LITTLE_ENDIAN); + BitSet testSet = BitSet.valueOf(buffer); + testSet.set(1); + WrappedImmutableBitSetBitmap bitMap = new WrappedImmutableBitSetBitmap(testSet); + Assert.assertTrue(bitMap.get(1)); + testSet.set(2); + Assert.assertTrue(bitMap.get(2)); + } + + @Test + public void testSimpleBitSet() + { + WrappedBitSetBitmap bitSet = new WrappedBitSetBitmap(IntSetTestUtility.createSimpleBitSet(IntSetTestUtility.getSetBits())); + Assert.assertTrue(IntSetTestUtility.equalSets(IntSetTestUtility.getSetBits(), bitSet)); + } + + @Test + public void testUnion() + { + WrappedBitSetBitmap bitSet = new WrappedBitSetBitmap(IntSetTestUtility.createSimpleBitSet(IntSetTestUtility.getSetBits())); + + Set extraBits = Sets.newHashSet(6, 9); + WrappedBitSetBitmap bitExtraSet = new WrappedBitSetBitmap(IntSetTestUtility.createSimpleBitSet(extraBits)); + + Set union = Sets.union(extraBits, IntSetTestUtility.getSetBits()); + + Assert.assertTrue(IntSetTestUtility.equalSets(union, (WrappedBitSetBitmap) bitSet.union(bitExtraSet))); + } + + @Test + public void testIntersection() + { + WrappedBitSetBitmap bitSet = new WrappedBitSetBitmap(IntSetTestUtility.createSimpleBitSet(IntSetTestUtility.getSetBits())); + + Set extraBits = Sets.newHashSet(1, 2, 3, 4, 5, 6, 7, 8); + WrappedBitSetBitmap bitExtraSet = new WrappedBitSetBitmap(IntSetTestUtility.createSimpleBitSet(extraBits)); + + Set intersection = Sets.intersection(extraBits, IntSetTestUtility.getSetBits()); + + Assert.assertTrue(IntSetTestUtility.equalSets( + intersection, + (WrappedBitSetBitmap) bitSet.intersection(bitExtraSet) + )); + } + + @Test + public void testAnd() + { + WrappedBitSetBitmap bitSet = defaultBitSet(); + WrappedBitSetBitmap bitSet2 = defaultBitSet(); + Set defaultBitSet = IntSetTestUtility.getSetBits(); + bitSet.remove(1); + bitSet2.remove(2); + + bitSet.and(bitSet2); + + defaultBitSet.remove(1); + defaultBitSet.remove(2); + + Assert.assertTrue(IntSetTestUtility.equalSets(defaultBitSet, bitSet)); + } + + + @Test + public void testOr() + { + WrappedBitSetBitmap bitSet = defaultBitSet(); + WrappedBitSetBitmap bitSet2 = defaultBitSet(); + Set defaultBitSet = IntSetTestUtility.getSetBits(); + bitSet.remove(1); + bitSet2.remove(2); + + bitSet.or(bitSet2); + + Assert.assertTrue(IntSetTestUtility.equalSets(defaultBitSet, bitSet)); + } + + @Test + public void testAndNot() + { + WrappedBitSetBitmap bitSet = defaultBitSet(); + WrappedBitSetBitmap bitSet2 = defaultBitSet(); + Set defaultBitSet = Sets.newHashSet(); + bitSet.remove(1); + bitSet2.remove(2); + + bitSet.andNot(bitSet2); + + defaultBitSet.add(2); + + Assert.assertTrue(IntSetTestUtility.equalSets(defaultBitSet, bitSet)); + } + + + @Test + public void testSerialize() + { + WrappedBitSetBitmap bitSet = defaultBitSet(); + Set defaultBitSet = IntSetTestUtility.getSetBits(); + byte[] buffer = new byte[bitSet.getSizeInBytes()]; + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); + bitSet.serialize(byteBuffer); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java new file mode 100755 index 000000000000..402a494c90c5 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/bitmap/WrappedRoaringBitmapTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.bitmap; + +import junit.framework.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +@RunWith(Parameterized.class) +public class WrappedRoaringBitmapTest +{ + private final RoaringBitmapFactory factory; + + public WrappedRoaringBitmapTest(RoaringBitmapFactory factory) + { + this.factory = factory; + } + + @Parameterized.Parameters + public static List factoryClasses() + { + return Arrays.asList( + (RoaringBitmapFactory[]) Arrays.asList( + new RoaringBitmapFactory(false) + ).toArray(), + (RoaringBitmapFactory[]) Arrays.asList( + new RoaringBitmapFactory(true) + ).toArray() + ); + } + + private WrappedRoaringBitmap createWrappedRoaringBitmap() + { + WrappedRoaringBitmap set = (WrappedRoaringBitmap) factory.makeEmptyMutableBitmap(); + set.add(1); + set.add(3); + set.add(5); + set.add(7); + set.add(9); + return set; + } + + @Test + public void testSerialize() + { + WrappedRoaringBitmap set = createWrappedRoaringBitmap(); + + byte[] buffer = new byte[set.getSizeInBytes()]; + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer); + set.serialize(byteBuffer); + byteBuffer.flip(); + ImmutableBitmap immutableBitmap = new RoaringBitmapFactory().mapImmutableBitmap(byteBuffer); + Assert.assertEquals(5, immutableBitmap.size()); + } + + @Test + public void testToByteArray() + { + WrappedRoaringBitmap set = createWrappedRoaringBitmap(); + ImmutableBitmap immutableBitmap = new RoaringBitmapFactory().mapImmutableBitmap(ByteBuffer.wrap(set.toBytes())); + Assert.assertEquals(5, immutableBitmap.size()); + } + +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java new file mode 100755 index 000000000000..4ae8bc744316 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/ImmutableRTreeTest.java @@ -0,0 +1,651 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.spatial.search.PolygonBound; +import io.druid.collections.spatial.search.RadiusBound; +import io.druid.collections.spatial.search.RectangularBound; +import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; +import junit.framework.Assert; +import org.junit.Test; +import org.roaringbitmap.IntIterator; + +import java.nio.ByteBuffer; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** + */ +public class ImmutableRTreeTest +{ + @Test + public void testToAndFromByteBuffer() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{1, 1}, 2); + tree.insert(new float[]{2, 2}, 3); + tree.insert(new float[]{3, 3}, 4); + tree.insert(new float[]{4, 4}, 5); + + ImmutableRTree firstTree = ImmutableRTree.newImmutableFromMutable(tree); + ByteBuffer buffer = ByteBuffer.wrap(firstTree.toBytes()); + ImmutableRTree secondTree = new ImmutableRTree(buffer, bf); + Iterable points = secondTree.search(new RadiusBound(new float[]{0, 0}, 10)); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testToAndFromByteBufferRoaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{1, 1}, 2); + tree.insert(new float[]{2, 2}, 3); + tree.insert(new float[]{3, 3}, 4); + tree.insert(new float[]{4, 4}, 5); + + ImmutableRTree firstTree = ImmutableRTree.newImmutableFromMutable(tree); + ByteBuffer buffer = ByteBuffer.wrap(firstTree.toBytes()); + ImmutableRTree secondTree = new ImmutableRTree(buffer, bf); + Iterable points = secondTree.search(new RadiusBound(new float[]{0, 0}, 10)); + ImmutableBitmap finalSet = bf.union(points); + + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchNoSplit() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{10, 10}, 10); + tree.insert(new float[]{1, 3}, 2); + tree.insert(new float[]{27, 34}, 20); + tree.insert(new float[]{106, 19}, 30); + tree.insert(new float[]{4, 2}, 3); + tree.insert(new float[]{5, 0}, 4); + tree.insert(new float[]{4, 72}, 40); + tree.insert(new float[]{-4, -3}, 5); + tree.insert(new float[]{119, -78}, 50); + + Assert.assertEquals(tree.getRoot().getChildren().size(), 10); + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(new RadiusBound(new float[]{0, 0}, 5)); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchNoSplitRoaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{10, 10}, 10); + tree.insert(new float[]{1, 3}, 2); + tree.insert(new float[]{27, 34}, 20); + tree.insert(new float[]{106, 19}, 30); + tree.insert(new float[]{4, 2}, 3); + tree.insert(new float[]{5, 0}, 4); + tree.insert(new float[]{4, 72}, 40); + tree.insert(new float[]{-4, -3}, 5); + tree.insert(new float[]{119, -78}, 50); + + Assert.assertEquals(tree.getRoot().getChildren().size(), 10); + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(new RadiusBound(new float[]{0, 0}, 5)); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchWithSplit() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{1, 3}, 2); + tree.insert(new float[]{4, 2}, 3); + tree.insert(new float[]{5, 0}, 4); + tree.insert(new float[]{-4, -3}, 5); + + Random rand = new Random(); + for (int i = 0; i < 95; i++) { + tree.insert( + new float[]{(float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(new RadiusBound(new float[]{0, 0}, 5)); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchWithSplitRoaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{1, 3}, 2); + tree.insert(new float[]{4, 2}, 3); + tree.insert(new float[]{5, 0}, 4); + tree.insert(new float[]{-4, -3}, 5); + + Random rand = new Random(); + for (int i = 0; i < 95; i++) { + tree.insert( + new float[]{(float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(new RadiusBound(new float[]{0, 0}, 5)); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + + @Test + public void testSearchWithSplit2() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0.0f, 0.0f}, 0); + tree.insert(new float[]{1.0f, 3.0f}, 1); + tree.insert(new float[]{4.0f, 2.0f}, 2); + tree.insert(new float[]{7.0f, 3.0f}, 3); + tree.insert(new float[]{8.0f, 6.0f}, 4); + + Random rand = new Random(); + for (int i = 5; i < 5000; i++) { + tree.insert( + new float[]{(float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search( + new RectangularBound( + new float[]{0, 0}, + new float[]{9, 9} + ) + ); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(0, 1, 2, 3, 4); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchWithSplit2Roaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0.0f, 0.0f}, 0); + tree.insert(new float[]{1.0f, 3.0f}, 1); + tree.insert(new float[]{4.0f, 2.0f}, 2); + tree.insert(new float[]{7.0f, 3.0f}, 3); + tree.insert(new float[]{8.0f, 6.0f}, 4); + + Random rand = new Random(); + for (int i = 5; i < 5000; i++) { + tree.insert( + new float[]{(float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search( + new RectangularBound( + new float[]{0, 0}, + new float[]{9, 9} + ) + ); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(0, 1, 2, 3, 4); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchWithSplit3() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0.0f, 0.0f}, 0); + tree.insert(new float[]{1.0f, 3.0f}, 1); + tree.insert(new float[]{4.0f, 2.0f}, 2); + tree.insert(new float[]{7.0f, 3.0f}, 3); + tree.insert(new float[]{8.0f, 6.0f}, 4); + + Random rand = new Random(); + for (int i = 5; i < 5000; i++) { + tree.insert( + new float[]{(float) (rand.nextFloat() * 10 + 10.0), (float) (rand.nextFloat() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search( + new RadiusBound(new float[]{0.0f, 0.0f}, 5) + ); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 3); + + Set expected = Sets.newHashSet(0, 1, 2); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchWithSplit3Roaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0.0f, 0.0f}, 0); + tree.insert(new float[]{1.0f, 3.0f}, 1); + tree.insert(new float[]{4.0f, 2.0f}, 2); + tree.insert(new float[]{7.0f, 3.0f}, 3); + tree.insert(new float[]{8.0f, 6.0f}, 4); + + Random rand = new Random(); + for (int i = 5; i < 5000; i++) { + tree.insert( + new float[]{(float) (rand.nextFloat() * 10 + 10.0), (float) (rand.nextFloat() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search( + new RadiusBound(new float[]{0.0f, 0.0f}, 5) + ); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 3); + + Set expected = Sets.newHashSet(0, 1, 2); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchWithSplit4() + { + BitmapFactory bf = new ConciseBitmapFactory(); + //RTree tree = new RTree(2, new QuadraticGutmanSplitStrategy(0, 100, bf), bf); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + Random rand = new Random(); + + int outPolygon = 0, inPolygon = 0; + for (; inPolygon < 500; ) { + double abscissa = rand.nextDouble() * 5; + double ordinate = rand.nextDouble() * 4; + + if (abscissa < 1 || abscissa > 4 || ordinate < 1 || ordinate > 3 || abscissa < 2 && ordinate > 2) { + tree.insert( + new float[]{(float) abscissa, (float) ordinate}, + outPolygon + 500 + ); + outPolygon++; + } else if (abscissa > 1 && abscissa < 4 && ordinate > 1 && ordinate < 2 + || abscissa > 2 && abscissa < 4 && ordinate >= 2 && ordinate < 3) { + tree.insert( + new float[]{(float) abscissa, (float) ordinate}, + inPolygon + ); + inPolygon++; + } + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(PolygonBound.from( + new float[]{1.0f, 1.0f, 2.0f, 2.0f, 4.0f, 4.0f}, + new float[]{1.0f, 2.0f, 2.0f, 3.0f, 3.0f, 1.0f} + )); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() == 500); + + Set expected = Sets.newHashSet(); + for (int i = 0; i < 500; i++) { + expected.add(i); + } + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testSearchWithSplit4Roaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + Random rand = new Random(); + + int outPolygon = 0, inPolygon = 0; + for (; inPolygon < 500; ) { + double abscissa = rand.nextDouble() * 5; + double ordinate = rand.nextDouble() * 4; + + if (abscissa < 1 || abscissa > 4 || ordinate < 1 || ordinate > 3 || abscissa < 2 && ordinate > 2) { + tree.insert( + new float[]{(float) abscissa, (float) ordinate}, + outPolygon + 500 + ); + outPolygon++; + } else if (abscissa > 1 && abscissa < 4 && ordinate > 1 && ordinate < 2 + || abscissa > 2 && abscissa < 4 && ordinate >= 2 && ordinate < 3) { + tree.insert( + new float[]{(float) abscissa, (float) ordinate}, + inPolygon + ); + inPolygon++; + } + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(PolygonBound.from( + new float[]{1.0f, 1.0f, 2.0f, 2.0f, 4.0f, 4.0f}, + new float[]{1.0f, 2.0f, 2.0f, 3.0f, 3.0f, 1.0f} + )); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() == 500); + + Set expected = Sets.newHashSet(); + for (int i = 0; i < 500; i++) { + expected.add(i); + } + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + @Test + public void testEmptyConciseSet() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0.0f, 0.0f}, bf.makeEmptyMutableBitmap()); + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search( + new RadiusBound(new float[]{0.0f, 0.0f}, 5) + ); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertEquals(finalSet.size(), 0); + } + + @Test + public void testEmptyRoaringBitmap() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0.0f, 0.0f}, bf.makeEmptyMutableBitmap()); + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search( + new RadiusBound(new float[]{0.0f, 0.0f}, 5) + ); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertEquals(finalSet.size(), 0); + Assert.assertTrue(finalSet.isEmpty()); + } + + @Test + public void testSearchWithSplitLimitedBound() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{1, 3}, 2); + tree.insert(new float[]{4, 2}, 3); + tree.insert(new float[]{5, 0}, 4); + tree.insert(new float[]{-4, -3}, 5); + + Random rand = new Random(); + for (int i = 0; i < 4995; i++) { + tree.insert( + new float[]{(float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(new RadiusBound(new float[]{0, 0}, 5, 2)); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + + @Test + public void testSearchWithSplitLimitedBoundRoaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + tree.insert(new float[]{0, 0}, 1); + tree.insert(new float[]{1, 3}, 2); + tree.insert(new float[]{4, 2}, 3); + tree.insert(new float[]{5, 0}, 4); + tree.insert(new float[]{-4, -3}, 5); + + Random rand = new Random(); + for (int i = 0; i < 4995; i++) { + tree.insert( + new float[]{(float) (rand.nextDouble() * 10 + 10.0), (float) (rand.nextDouble() * 10 + 10.0)}, + i + ); + } + + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + Iterable points = searchTree.search(new RadiusBound(new float[]{0, 0}, 5, 2)); + ImmutableBitmap finalSet = bf.union(points); + Assert.assertTrue(finalSet.size() >= 5); + + Set expected = Sets.newHashSet(1, 2, 3, 4, 5); + IntIterator iter = finalSet.iterator(); + while (iter.hasNext()) { + Assert.assertTrue(expected.contains(iter.next())); + } + } + + //@Test + public void showBenchmarks() + { + final int start = 1; + final int factor = 10; + final int end = 10000000; + final int radius = 10; + + for (int numPoints = start; numPoints <= end; numPoints *= factor) { + try { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + + Stopwatch stopwatch = Stopwatch.createStarted(); + Random rand = new Random(); + for (int i = 0; i < numPoints; i++) { + tree.insert(new float[]{(float) (rand.nextDouble() * 100), (float) (rand.nextDouble() * 100)}, i); + } + long stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + System.out.printf("[%,d]: insert = %,d ms%n", numPoints, stop); + + stopwatch.reset().start(); + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + System.out.printf("[%,d]: size = %,d bytes%n", numPoints, searchTree.toBytes().length); + System.out.printf("[%,d]: buildImmutable = %,d ms%n", numPoints, stop); + + stopwatch.reset().start(); + + Iterable points = searchTree.search(new RadiusBound(new float[]{50, 50}, radius)); + + Iterables.size(points); + stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + System.out.printf("[%,d]: search = %,dms%n", numPoints, stop); + + stopwatch.reset().start(); + + ImmutableBitmap finalSet = bf.union(points); + + stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + System.out.printf("[%,d]: union of %,d points in %,d ms%n", numPoints, finalSet.size(), stop); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + } + + //@Test + public void showBenchmarksBoundWithLimits() + { + //final int start = 1; + final int start = 10000000; + final int factor = 10; + final int end = 10000000; + //final int end = 10; + + for (int numPoints = start; numPoints <= end; numPoints *= factor) { + try { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + + Stopwatch stopwatch = Stopwatch.createStarted(); + Random rand = new Random(); + for (int i = 0; i < numPoints; i++) { + tree.insert(new float[]{(float) (rand.nextDouble() * 100), (float) (rand.nextDouble() * 100)}, i); + } + long stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + System.out.printf("[%,d]: insert = %,d ms%n", numPoints, stop); + + stopwatch.reset().start(); + ImmutableRTree searchTree = ImmutableRTree.newImmutableFromMutable(tree); + stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + System.out.printf("[%,d]: size = %,d bytes%n", numPoints, searchTree.toBytes().length); + System.out.printf("[%,d]: buildImmutable = %,d ms%n", numPoints, stop); + + stopwatch.reset().start(); + + Iterable points = searchTree.search( + new RectangularBound( + new float[]{40, 40}, + new float[]{60, 60}, + 100 + ) + ); + + Iterables.size(points); + stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + System.out.printf("[%,d]: search = %,dms%n", numPoints, stop); + + stopwatch.reset().start(); + + ImmutableBitmap finalSet = bf.union(points); + + stop = stopwatch.elapsed(TimeUnit.MILLISECONDS); + System.out.printf("[%,d]: union of %,d points in %,d ms%n", numPoints, finalSet.size(), stop); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/RTreeTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/RTreeTest.java new file mode 100755 index 000000000000..06986d8f7d1b --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/RTreeTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial; + +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; +import junit.framework.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Random; + +/** + */ +public class RTreeTest +{ + private RTree tree; + private RTree roaringtree; + + @Before + public void setUp() throws Exception + { + BitmapFactory bf = new ConciseBitmapFactory(); + tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + BitmapFactory rbf = new RoaringBitmapFactory(); + roaringtree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, rbf), rbf); + + } + + @Test + public void testInsertNoSplit() + { + float[] elem = new float[]{5, 5}; + tree.insert(elem, 1); + Assert.assertTrue(Arrays.equals(elem, tree.getRoot().getMinCoordinates())); + Assert.assertTrue(Arrays.equals(elem, tree.getRoot().getMaxCoordinates())); + + tree.insert(new float[]{6, 7}, 2); + tree.insert(new float[]{1, 3}, 3); + tree.insert(new float[]{10, 4}, 4); + tree.insert(new float[]{8, 2}, 5); + + Assert.assertEquals(tree.getRoot().getChildren().size(), 5); + + float[] expectedMin = new float[]{1, 2}; + float[] expectedMax = new float[]{10, 7}; + + Assert.assertTrue(Arrays.equals(expectedMin, tree.getRoot().getMinCoordinates())); + Assert.assertTrue(Arrays.equals(expectedMax, tree.getRoot().getMaxCoordinates())); + Assert.assertEquals(tree.getRoot().getArea(), 45.0d); + } + + @Test + public void testInsertDuplicatesNoSplit() + { + tree.insert(new float[]{1, 1}, 1); + tree.insert(new float[]{1, 1}, 1); + tree.insert(new float[]{1, 1}, 1); + + Assert.assertEquals(tree.getRoot().getChildren().size(), 3); + } + + @Test + public void testInsertDuplicatesNoSplitRoaring() + { + roaringtree.insert(new float[]{1, 1}, 1); + roaringtree.insert(new float[]{1, 1}, 1); + roaringtree.insert(new float[]{1, 1}, 1); + + Assert.assertEquals(roaringtree.getRoot().getChildren().size(), 3); + } + + + @Test + public void testSplitOccurs() + { + Random rand = new Random(); + for (int i = 0; i < 100; i++) { + tree.insert(new float[]{rand.nextFloat(), rand.nextFloat()}, i); + } + + Assert.assertTrue(tree.getRoot().getChildren().size() > 1); + } + + @Test + public void testSplitOccursRoaring() + { + Random rand = new Random(); + for (int i = 0; i < 100; i++) { + roaringtree.insert(new float[]{rand.nextFloat(), rand.nextFloat()}, i); + } + + Assert.assertTrue(roaringtree.getRoot().getChildren().size() > 1); + } + +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java new file mode 100755 index 000000000000..73ac85e65802 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/PolygonBoundTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +public class PolygonBoundTest +{ + @Test + public void testCacheKey() + { + Assert.assertArrayEquals( + PolygonBound.from(new float[]{1F, 2F, 3F}, new float[]{0F, 2F, 0F}, 1).getCacheKey(), + PolygonBound.from(new float[]{1F, 2F, 3F}, new float[]{0F, 2F, 0F}, 1).getCacheKey() + ); + Assert.assertFalse(Arrays.equals( + PolygonBound.from(new float[]{1F, 2F, 3F}, new float[]{0F, 2F, 0F}, 1).getCacheKey(), + PolygonBound.from(new float[]{1F, 2F, 3F}, new float[]{0F, 2F, 1F}, 1).getCacheKey() + )); + Assert.assertFalse(Arrays.equals( + PolygonBound.from(new float[]{1F, 2F, 3F}, new float[]{0F, 2F, 0F}, 1).getCacheKey(), + PolygonBound.from(new float[]{1F, 2F, 2F}, new float[]{0F, 2F, 0F}, 1).getCacheKey() + )); + Assert.assertFalse(Arrays.equals( + PolygonBound.from(new float[]{1F, 2F, 3F}, new float[]{0F, 2F, 0F}, 1).getCacheKey(), + PolygonBound.from(new float[]{1F, 2F, 3F}, new float[]{0F, 2F, 0F}, 2).getCacheKey() + )); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java new file mode 100755 index 000000000000..46309bbcc9f2 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RadiusBoundTest.java @@ -0,0 +1,51 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +public class RadiusBoundTest +{ + @Test + public void testCacheKey() + { + final float[] coords0 = new float[]{1.0F, 2.0F}; + final float[] coords1 = new float[]{1.1F, 2.1F}; + Assert.assertArrayEquals( + new RadiusBound(coords0, 3.0F, 10).getCacheKey(), + new RadiusBound(coords0, 3.0F, 10).getCacheKey() + ); + Assert.assertFalse(Arrays.equals( + new RadiusBound(coords0, 3.0F, 10).getCacheKey(), + new RadiusBound(coords1, 3.0F, 10).getCacheKey() + )); + Assert.assertFalse(Arrays.equals( + new RadiusBound(coords0, 3.0F, 10).getCacheKey(), + new RadiusBound(coords0, 3.1F, 10).getCacheKey() + )); + Assert.assertFalse(Arrays.equals( + new RadiusBound(coords0, 3.0F, 10).getCacheKey(), + new RadiusBound(coords0, 3.0F, 9).getCacheKey() + )); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java new file mode 100755 index 000000000000..025618be8cdd --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/search/RectangularBoundTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.search; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +public class RectangularBoundTest +{ + @Test + public void testCacheKey() + { + Assert.assertArrayEquals( + new RectangularBound(new float[]{1F, 1F}, new float[]{2F, 2F}, 1).getCacheKey(), + new RectangularBound(new float[]{1F, 1F}, new float[]{2F, 2F}, 1).getCacheKey() + ); + Assert.assertFalse(Arrays.equals( + new RectangularBound(new float[]{1F, 1F}, new float[]{2F, 2F}, 1).getCacheKey(), + new RectangularBound(new float[]{1F, 1F}, new float[]{2F, 3F}, 1).getCacheKey() + )); + Assert.assertFalse(Arrays.equals( + new RectangularBound(new float[]{1F, 1F}, new float[]{2F, 2F}, 1).getCacheKey(), + new RectangularBound(new float[]{1F, 0F}, new float[]{2F, 2F}, 1).getCacheKey() + )); + Assert.assertFalse(Arrays.equals( + new RectangularBound(new float[]{1F, 1F}, new float[]{2F, 2F}, 1).getCacheKey(), + new RectangularBound(new float[]{1F, 1F}, new float[]{2F, 2F}, 2).getCacheKey() + )); + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java new file mode 100755 index 000000000000..5caeba4e6f13 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/collections/spatial/split/LinearGutmanSplitStrategyTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.collections.spatial.split; + +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.spatial.Node; +import io.druid.collections.spatial.Point; +import io.druid.collections.spatial.RTree; +import junit.framework.Assert; +import org.junit.Test; + +import java.util.Random; + +/** + */ +public class LinearGutmanSplitStrategyTest +{ + @Test + public void testPickSeeds() throws Exception + { + BitmapFactory bf = new ConciseBitmapFactory(); + LinearGutmanSplitStrategy strategy = new LinearGutmanSplitStrategy(0, 50, bf); + Node node = new Node(new float[2], new float[2], true, bf); + + node.addChild(new Point(new float[]{3, 7}, 1, bf)); + node.addChild(new Point(new float[]{1, 6}, 1, bf)); + node.addChild(new Point(new float[]{9, 8}, 1, bf)); + node.addChild(new Point(new float[]{2, 5}, 1, bf)); + node.addChild(new Point(new float[]{4, 4}, 1, bf)); + node.enclose(); + + Node[] groups = strategy.split(node); + Assert.assertEquals(groups[0].getMinCoordinates()[0], 1.0f); + Assert.assertEquals(groups[0].getMinCoordinates()[1], 4.0f); + Assert.assertEquals(groups[1].getMinCoordinates()[0], 9.0f); + Assert.assertEquals(groups[1].getMinCoordinates()[1], 8.0f); + } + + @Test + public void testPickSeedsRoaring() throws Exception + { + BitmapFactory bf = new RoaringBitmapFactory(); + LinearGutmanSplitStrategy strategy = new LinearGutmanSplitStrategy(0, 50, bf); + Node node = new Node(new float[2], new float[2], true, bf); + + node.addChild(new Point(new float[]{3, 7}, 1, bf)); + node.addChild(new Point(new float[]{1, 6}, 1, bf)); + node.addChild(new Point(new float[]{9, 8}, 1, bf)); + node.addChild(new Point(new float[]{2, 5}, 1, bf)); + node.addChild(new Point(new float[]{4, 4}, 1, bf)); + node.enclose(); + + Node[] groups = strategy.split(node); + Assert.assertEquals(groups[0].getMinCoordinates()[0], 1.0f); + Assert.assertEquals(groups[0].getMinCoordinates()[1], 4.0f); + Assert.assertEquals(groups[1].getMinCoordinates()[0], 9.0f); + Assert.assertEquals(groups[1].getMinCoordinates()[1], 8.0f); + } + + + @Test + public void testNumChildrenSize() + { + BitmapFactory bf = new ConciseBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + Random rand = new Random(); + for (int i = 0; i < 100; i++) { + tree.insert(new float[]{rand.nextFloat(), rand.nextFloat()}, i); + } + + Assert.assertTrue(getNumPoints(tree.getRoot()) >= tree.getSize()); + } + + @Test + public void testNumChildrenSizeRoaring() + { + BitmapFactory bf = new RoaringBitmapFactory(); + RTree tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bf), bf); + Random rand = new Random(); + for (int i = 0; i < 100; i++) { + tree.insert(new float[]{rand.nextFloat(), rand.nextFloat()}, i); + } + + Assert.assertTrue(getNumPoints(tree.getRoot()) >= tree.getSize()); + } + + private int getNumPoints(Node node) + { + int total = 0; + if (node.isLeaf()) { + total += node.getChildren().size(); + } else { + for (Node child : node.getChildren()) { + total += getNumPoints(child); + } + } + return total; + } +} diff --git a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java b/bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java new file mode 100755 index 000000000000..2fa616eef401 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/test/annotation/Benchmark.java @@ -0,0 +1,24 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.test.annotation; + +public interface Benchmark +{ +} diff --git a/bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java b/bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java new file mode 100755 index 000000000000..ae1d36869320 --- /dev/null +++ b/bytebuffer-collections/src/test/java/io/druid/test/annotation/Dummy.java @@ -0,0 +1,24 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.test.annotation; + +public interface Dummy +{ +} diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/BitMapFactory.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/BitMapFactory.java index 3c517cd4ae07..8a76bf4bc171 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/BitMapFactory.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/BitMapFactory.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; /** */ diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/ConciseBitMapFactory.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/ConciseBitMapFactory.java index e9b2d49b7b93..b87f6ef28b48 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/ConciseBitMapFactory.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/ConciseBitMapFactory.java @@ -19,9 +19,9 @@ package io.druid.query.aggregation.distinctcount; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ConciseBitmapFactory; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; public class ConciseBitMapFactory implements BitMapFactory { diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java index 54f7f5744daa..880ab0b2aae7 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountAggregator.java @@ -19,7 +19,7 @@ package io.druid.query.aggregation.distinctcount; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.query.aggregation.Aggregator; import io.druid.segment.DimensionSelector; diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java index a90dd6f2dcca..809cd0ac796f 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/DistinctCountBufferAggregator.java @@ -19,8 +19,8 @@ package io.druid.query.aggregation.distinctcount; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.WrappedRoaringBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.WrappedRoaringBitmap; import io.druid.query.aggregation.BufferAggregator; import io.druid.segment.DimensionSelector; diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/JavaBitMapFactory.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/JavaBitMapFactory.java index 5c1f2dc41160..f8d2cc8b7412 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/JavaBitMapFactory.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/JavaBitMapFactory.java @@ -19,9 +19,9 @@ package io.druid.query.aggregation.distinctcount; -import com.metamx.collections.bitmap.BitSetBitmapFactory; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.BitSetBitmapFactory; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; public class JavaBitMapFactory implements BitMapFactory { diff --git a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/RoaringBitMapFactory.java b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/RoaringBitMapFactory.java index 0da6cc04e17e..a3ed78a83b58 100644 --- a/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/RoaringBitMapFactory.java +++ b/extensions-contrib/distinctcount/src/main/java/io/druid/query/aggregation/distinctcount/RoaringBitMapFactory.java @@ -19,9 +19,9 @@ package io.druid.query.aggregation.distinctcount; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; public class RoaringBitMapFactory implements BitMapFactory { diff --git a/pom.xml b/pom.xml index a0a1b45e2160..ad2814b77097 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ benchmarks aws-common java-util + bytebuffer-collections extensions-core/avro-extensions extensions-core/datasketches diff --git a/processing/pom.xml b/processing/pom.xml index 43c2b11fa54a..624fecd46544 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -36,8 +36,9 @@ ${project.parent.version} - com.metamx + io.druid bytebuffer-collections + ${project.parent.version} it.unimi.dsi diff --git a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java index 4675876d5e26..fc3bac6c2911 100644 --- a/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java @@ -19,9 +19,9 @@ package io.druid.query.filter; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.Indexed; diff --git a/processing/src/main/java/io/druid/query/filter/Filter.java b/processing/src/main/java/io/druid/query/filter/Filter.java index 1a52237981ff..26d634ded491 100644 --- a/processing/src/main/java/io/druid/query/filter/Filter.java +++ b/processing/src/main/java/io/druid/query/filter/Filter.java @@ -19,7 +19,7 @@ package io.druid.query.filter; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; /** */ diff --git a/processing/src/main/java/io/druid/query/filter/RowOffsetMatcherFactory.java b/processing/src/main/java/io/druid/query/filter/RowOffsetMatcherFactory.java index 0de44c34d087..6c2b4411cda7 100644 --- a/processing/src/main/java/io/druid/query/filter/RowOffsetMatcherFactory.java +++ b/processing/src/main/java/io/druid/query/filter/RowOffsetMatcherFactory.java @@ -19,7 +19,7 @@ package io.druid.query.filter; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; /** */ diff --git a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java index 5d9f67ffbc1d..7555c93e9df8 100644 --- a/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.RangeSet; -import com.metamx.collections.spatial.search.Bound; +import io.druid.collections.spatial.search.Bound; import io.druid.java.util.common.StringUtils; import io.druid.segment.filter.SpatialFilter; diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java index 5d84c7b45331..93bdb20db995 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -25,9 +25,9 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import com.metamx.emitter.EmittingLogger; import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; diff --git a/processing/src/main/java/io/druid/segment/BitmapOffset.java b/processing/src/main/java/io/druid/segment/BitmapOffset.java index 1c41ee557182..5e4ae18309ba 100644 --- a/processing/src/main/java/io/druid/segment/BitmapOffset.java +++ b/processing/src/main/java/io/druid/segment/BitmapOffset.java @@ -19,10 +19,10 @@ package io.druid.segment; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.WrappedImmutableRoaringBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.WrappedImmutableRoaringBitmap; import io.druid.segment.data.Offset; import io.druid.segment.data.RoaringBitmapSerdeFactory; import org.roaringbitmap.IntIterator; diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java index 6b7c31c7c8ee..9b9d5c2c9f4f 100644 --- a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -20,9 +20,9 @@ package io.druid.segment; import com.google.common.base.Strings; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.query.filter.BitmapIndexSelector; import io.druid.segment.column.BitmapIndex; import io.druid.segment.column.Column; diff --git a/processing/src/main/java/io/druid/segment/DimensionIndexer.java b/processing/src/main/java/io/druid/segment/DimensionIndexer.java index 4ac8672632fc..82d8021bfefa 100644 --- a/processing/src/main/java/io/druid/segment/DimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/DimensionIndexer.java @@ -19,8 +19,8 @@ package io.druid.segment; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.query.dimension.DimensionSpec; import io.druid.query.filter.DruidPredicateFactory; import io.druid.query.filter.ValueMatcher; diff --git a/processing/src/main/java/io/druid/segment/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java index 7872af8b1744..80c9b69dd930 100644 --- a/processing/src/main/java/io/druid/segment/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -36,11 +36,11 @@ import com.google.common.io.Files; import com.google.common.primitives.Ints; import com.google.inject.Inject; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ConciseBitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.spatial.ImmutableRTree; import com.metamx.emitter.EmittingLogger; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.IAE; diff --git a/processing/src/main/java/io/druid/segment/MMappedIndex.java b/processing/src/main/java/io/druid/segment/MMappedIndex.java index 7297458d1fd3..d7d08183648d 100644 --- a/processing/src/main/java/io/druid/segment/MMappedIndex.java +++ b/processing/src/main/java/io/druid/segment/MMappedIndex.java @@ -19,8 +19,8 @@ package io.druid.segment; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.java.util.common.logger.Logger; import io.druid.segment.data.CompressedLongsIndexedSupplier; diff --git a/processing/src/main/java/io/druid/segment/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java index c18913946223..31492b18cc43 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -19,7 +19,7 @@ package io.druid.segment; -import com.metamx.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.BitmapFactory; import io.druid.segment.data.Indexed; import org.joda.time.Interval; diff --git a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index fc1d898aa12c..38dbe383c617 100644 --- a/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -29,7 +29,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Closer; -import com.metamx.collections.bitmap.ImmutableBitmap; + +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.granularity.QueryGranularity; import io.druid.java.util.common.guava.Sequence; import io.druid.java.util.common.guava.Sequences; diff --git a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index 76261309299b..f91fa7aa0291 100644 --- a/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; -import com.metamx.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.BitmapFactory; import io.druid.java.util.common.io.smoosh.SmooshedFileMapper; import io.druid.segment.column.Column; import io.druid.segment.column.ColumnCapabilities; diff --git a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java index 31866ad449e1..5c94ed307fc7 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionIndexer.java @@ -25,9 +25,10 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.MutableBitmap; + import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.query.dimension.DimensionSpec; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.DruidPredicateFactory; diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java index e916588a0dff..e55e999da80f 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerLegacy.java @@ -25,10 +25,10 @@ import com.google.common.io.Files; import com.google.common.io.OutputSupplier; import com.google.common.primitives.Ints; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.spatial.ImmutableRTree; -import com.metamx.collections.spatial.RTree; -import com.metamx.collections.spatial.split.LinearGutmanSplitStrategy; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.spatial.ImmutableRTree; +import io.druid.collections.spatial.RTree; +import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; import io.druid.common.guava.FileOutputSupplier; import io.druid.common.utils.SerializerUtils; import io.druid.java.util.common.ByteBufferUtils; diff --git a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java index 595b4c78a118..eee670ac65d7 100644 --- a/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java +++ b/processing/src/main/java/io/druid/segment/StringDimensionMergerV9.java @@ -25,12 +25,12 @@ import com.google.common.io.ByteStreams; import com.google.common.io.Closer; import com.google.common.io.Files; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.spatial.ImmutableRTree; -import com.metamx.collections.spatial.RTree; -import com.metamx.collections.spatial.split.LinearGutmanSplitStrategy; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.spatial.ImmutableRTree; +import io.druid.collections.spatial.RTree; +import io.druid.collections.spatial.split.LinearGutmanSplitStrategy; import io.druid.java.util.common.ByteBufferUtils; import io.druid.java.util.common.ISE; import io.druid.java.util.common.logger.Logger; diff --git a/processing/src/main/java/io/druid/segment/column/BitmapIndex.java b/processing/src/main/java/io/druid/segment/column/BitmapIndex.java index c64ecfe6f46e..b28b66a0c562 100644 --- a/processing/src/main/java/io/druid/segment/column/BitmapIndex.java +++ b/processing/src/main/java/io/druid/segment/column/BitmapIndex.java @@ -19,8 +19,8 @@ package io.druid.segment.column; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; /** */ diff --git a/processing/src/main/java/io/druid/segment/column/SpatialIndex.java b/processing/src/main/java/io/druid/segment/column/SpatialIndex.java index a905936ca08f..54b5887752be 100644 --- a/processing/src/main/java/io/druid/segment/column/SpatialIndex.java +++ b/processing/src/main/java/io/druid/segment/column/SpatialIndex.java @@ -19,7 +19,7 @@ package io.druid.segment.column; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.spatial.ImmutableRTree; /** */ diff --git a/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java b/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java index 40d84efb8ccb..2c78d39429cf 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapCompressedIndexedInts.java @@ -20,7 +20,7 @@ package io.druid.segment.data; import com.google.common.collect.Ordering; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.IntIteratorUtils; import it.unimi.dsi.fastutil.ints.IntIterator; diff --git a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java index cbf7dbddb5f9..a7c6fe6bcfb2 100644 --- a/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/BitmapSerdeFactory.java @@ -21,8 +21,8 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; /** */ diff --git a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java index 3442f4edca89..c5ef97f594d2 100644 --- a/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseBitmapSerdeFactory.java @@ -19,14 +19,15 @@ package io.druid.segment.data; +import java.nio.ByteBuffer; + import com.google.common.collect.Ordering; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ConciseBitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.WrappedImmutableConciseBitmap; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; -import java.nio.ByteBuffer; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.WrappedImmutableConciseBitmap; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; /** */ diff --git a/processing/src/main/java/io/druid/segment/data/IndexedRTree.java b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java index ce6f778dc1c9..ba1ea9123aa0 100644 --- a/processing/src/main/java/io/druid/segment/data/IndexedRTree.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java @@ -20,8 +20,8 @@ package io.druid.segment.data; import com.google.common.collect.Ordering; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.spatial.ImmutableRTree; import java.nio.ByteBuffer; diff --git a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java index 86e6a0d02173..9c2ae1d21a85 100644 --- a/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java +++ b/processing/src/main/java/io/druid/segment/data/RoaringBitmapSerdeFactory.java @@ -22,10 +22,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Ordering; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; -import com.metamx.collections.bitmap.WrappedImmutableRoaringBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.bitmap.WrappedImmutableRoaringBitmap; import org.roaringbitmap.buffer.ImmutableRoaringBitmap; import java.nio.ByteBuffer; diff --git a/processing/src/main/java/io/druid/segment/filter/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java index 778a83e3cfcd..5365c151a80a 100644 --- a/processing/src/main/java/io/druid/segment/filter/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -21,7 +21,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.BooleanFilter; import io.druid.query.filter.Filter; diff --git a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java index 946050f3f324..5318d21700d5 100644 --- a/processing/src/main/java/io/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/BoundFilter.java @@ -21,7 +21,7 @@ import com.google.common.base.Predicate; import com.google.common.base.Supplier; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.BoundDimFilter; diff --git a/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java index 1443ef9f4b6e..c3146038ea4d 100644 --- a/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java @@ -21,7 +21,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.DruidLongPredicate; diff --git a/processing/src/main/java/io/druid/segment/filter/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java index e322480592b8..0123d8df1b70 100644 --- a/processing/src/main/java/io/druid/segment/filter/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -24,7 +24,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.common.guava.GuavaUtils; import io.druid.java.util.common.guava.FunctionalIterable; import io.druid.query.Query; diff --git a/processing/src/main/java/io/druid/segment/filter/InFilter.java b/processing/src/main/java/io/druid/segment/filter/InFilter.java index 0b34c6fddaa1..f650260b024b 100644 --- a/processing/src/main/java/io/druid/segment/filter/InFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/InFilter.java @@ -24,7 +24,7 @@ import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.Iterables; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.DruidLongPredicate; diff --git a/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java index 760d3a98b2ce..85084e82f44b 100644 --- a/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java @@ -20,7 +20,7 @@ package io.druid.segment.filter; import com.google.common.base.Predicate; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.query.filter.JavaScriptDimFilter; diff --git a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java index 7814eb72453a..00ba94728549 100644 --- a/processing/src/main/java/io/druid/segment/filter/LikeFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/LikeFilter.java @@ -20,7 +20,8 @@ package io.druid.segment.filter; import com.google.common.base.Strings; -import com.metamx.collections.bitmap.ImmutableBitmap; + +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; diff --git a/processing/src/main/java/io/druid/segment/filter/NotFilter.java b/processing/src/main/java/io/druid/segment/filter/NotFilter.java index 190b727b32d4..0e3c73118242 100644 --- a/processing/src/main/java/io/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/NotFilter.java @@ -19,7 +19,7 @@ package io.druid.segment.filter; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; diff --git a/processing/src/main/java/io/druid/segment/filter/OrFilter.java b/processing/src/main/java/io/druid/segment/filter/OrFilter.java index f44fd3b8af52..ba288b381645 100644 --- a/processing/src/main/java/io/druid/segment/filter/OrFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/OrFilter.java @@ -21,7 +21,7 @@ import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.BooleanFilter; import io.druid.query.filter.Filter; diff --git a/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java b/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java index d7c9d102c782..77b98fd40b67 100644 --- a/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java @@ -19,7 +19,7 @@ package io.druid.segment.filter; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.Filter; import io.druid.query.filter.ValueMatcher; diff --git a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java index 988f61eccb89..ff28649aac8a 100644 --- a/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java @@ -20,8 +20,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.spatial.search.Bound; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.search.Bound; import io.druid.query.filter.BitmapIndexSelector; import io.druid.query.filter.DruidLongPredicate; import io.druid.query.filter.DruidPredicateFactory; diff --git a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java index 95f3e4f12905..ebd008a75d68 100644 --- a/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java @@ -22,8 +22,8 @@ import com.google.common.base.Function; import com.google.common.collect.Iterators; import com.google.common.collect.Maps; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.logger.Logger; import io.druid.segment.DimensionHandler; import io.druid.segment.DimensionIndexer; diff --git a/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java b/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java index ab162f535201..ea204ced0d11 100644 --- a/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java @@ -20,8 +20,8 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; import io.druid.segment.column.BitmapIndex; import io.druid.segment.data.GenericIndexed; diff --git a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 39b7c7c5bfcb..b6398fc6de29 100644 --- a/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -23,8 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Ints; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.java.util.common.IAE; import io.druid.segment.CompressedVSizeIndexedSupplier; import io.druid.segment.CompressedVSizeIndexedV3Supplier; diff --git a/processing/src/main/java/io/druid/segment/serde/SpatialIndexColumnPartSupplier.java b/processing/src/main/java/io/druid/segment/serde/SpatialIndexColumnPartSupplier.java index f0351990dd1a..04435448ae7c 100644 --- a/processing/src/main/java/io/druid/segment/serde/SpatialIndexColumnPartSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/SpatialIndexColumnPartSupplier.java @@ -19,7 +19,7 @@ package io.druid.segment.serde; import com.google.common.base.Supplier; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.segment.column.SpatialIndex; /** diff --git a/processing/src/test/java/io/druid/segment/BitmapOffsetTest.java b/processing/src/test/java/io/druid/segment/BitmapOffsetTest.java index 38b65fcdc596..dd8161023229 100644 --- a/processing/src/test/java/io/druid/segment/BitmapOffsetTest.java +++ b/processing/src/test/java/io/druid/segment/BitmapOffsetTest.java @@ -23,11 +23,11 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; -import com.metamx.collections.bitmap.BitSetBitmapFactory; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ConciseBitmapFactory; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.bitmap.BitSetBitmapFactory; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; import io.druid.segment.data.Offset; import org.junit.Assert; import org.junit.Test; diff --git a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java index 2885be71c21b..8d21f034e9b3 100644 --- a/processing/src/test/java/io/druid/segment/EmptyIndexTest.java +++ b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java @@ -21,7 +21,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; import io.druid.granularity.QueryGranularities; import io.druid.query.aggregation.AggregatorFactory; import io.druid.segment.column.Column; diff --git a/processing/src/test/java/io/druid/segment/IndexMergerTest.java b/processing/src/test/java/io/druid/segment/IndexMergerTest.java index a6651c09d8fc..85aef5928e41 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTest.java @@ -27,8 +27,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; -import com.metamx.collections.bitmap.RoaringBitmapFactory; + import io.druid.data.input.InputRow; +import io.druid.collections.bitmap.RoaringBitmapFactory; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionSchema; import io.druid.data.input.impl.DimensionSchema.MultiValueHandling; diff --git a/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java b/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java index f4a3186f69d0..ee9db667a339 100644 --- a/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerV9WithSpatialIndexTest.java @@ -23,8 +23,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.collections.spatial.search.RadiusBound; -import com.metamx.collections.spatial.search.RectangularBound; +import io.druid.collections.spatial.search.RadiusBound; +import io.druid.collections.spatial.search.RectangularBound; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.SpatialDimensionSchema; diff --git a/processing/src/test/java/io/druid/segment/data/BitmapCreationBenchmark.java b/processing/src/test/java/io/druid/segment/data/BitmapCreationBenchmark.java index 26dcb8bc8fb2..c4de414e4d42 100644 --- a/processing/src/test/java/io/druid/segment/data/BitmapCreationBenchmark.java +++ b/processing/src/test/java/io/druid/segment/data/BitmapCreationBenchmark.java @@ -20,9 +20,9 @@ import com.carrotsearch.junitbenchmarks.AbstractBenchmark; import com.carrotsearch.junitbenchmarks.BenchmarkOptions; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; import io.druid.java.util.common.logger.Logger; import org.junit.AfterClass; import org.junit.Assert; diff --git a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java index c45f773bfbe9..bd1432c825e6 100644 --- a/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/ExtractionDimFilterTest.java @@ -21,12 +21,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ConciseBitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.MutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; -import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.MutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.spatial.ImmutableRTree; import io.druid.query.extraction.DimExtractionFn; import io.druid.query.extraction.ExtractionFn; import io.druid.query.filter.BitmapIndexSelector; diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java index 591056cecb8d..c1a46038aa1e 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java @@ -23,8 +23,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.metamx.collections.spatial.search.RadiusBound; -import com.metamx.collections.spatial.search.RectangularBound; +import io.druid.collections.spatial.search.RadiusBound; +import io.druid.collections.spatial.search.RectangularBound; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.SpatialDimensionSchema; diff --git a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java index 3458559a100a..bc4f283b42c9 100644 --- a/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java @@ -23,8 +23,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.collections.spatial.search.RadiusBound; -import com.metamx.collections.spatial.search.RectangularBound; +import io.druid.collections.spatial.search.RadiusBound; +import io.druid.collections.spatial.search.RectangularBound; import io.druid.data.input.MapBasedInputRow; import io.druid.data.input.impl.DimensionsSpec; import io.druid.data.input.impl.SpatialDimensionSchema; diff --git a/services/src/main/java/io/druid/cli/DumpSegment.java b/services/src/main/java/io/druid/cli/DumpSegment.java index ec144a3a708a..5420270a7cb6 100644 --- a/services/src/main/java/io/druid/cli/DumpSegment.java +++ b/services/src/main/java/io/druid/cli/DumpSegment.java @@ -35,10 +35,10 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.name.Names; -import com.metamx.collections.bitmap.BitmapFactory; -import com.metamx.collections.bitmap.ConciseBitmapFactory; -import com.metamx.collections.bitmap.ImmutableBitmap; -import com.metamx.collections.bitmap.RoaringBitmapFactory; +import io.druid.collections.bitmap.BitmapFactory; +import io.druid.collections.bitmap.ConciseBitmapFactory; +import io.druid.collections.bitmap.ImmutableBitmap; +import io.druid.collections.bitmap.RoaringBitmapFactory; import io.airlift.airline.Command; import io.airlift.airline.Option;