Skip to content

Commit

Permalink
eagerly allocate the intermediate computation buffers (apache#3628)
Browse files Browse the repository at this point in the history
  • Loading branch information
himanshug authored and fjy committed Oct 31, 2016
1 parent 9f5c895 commit 32c5494
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ public void setup() throws IOException

StupidPool<ByteBuffer> bufferPool = new StupidPool<>(
new OffheapBufferGenerator("compute", 250_000_000),
0,
Integer.MAX_VALUE
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ public void setup() throws IOException
}

factory = new TopNQueryRunnerFactory(
new StupidPool<>(new OffheapBufferGenerator("compute", 250000000), Integer.MAX_VALUE),
new StupidPool<>(new OffheapBufferGenerator("compute", 250000000), 0, Integer.MAX_VALUE),
new TopNQueryQueryToolChest(new TopNQueryConfig(), QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator()),
QueryBenchmarkUtil.NOOP_QUERYWATCHER
);
Expand Down
16 changes: 13 additions & 3 deletions common/src/main/java/io/druid/collections/StupidPool.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

package io.druid.collections;

import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;

import io.druid.java.util.common.ISE;
import io.druid.java.util.common.logger.Logger;

Expand All @@ -45,17 +45,27 @@ public StupidPool(
Supplier<T> generator
)
{
this.generator = generator;
this.objectsCacheMaxCount = Integer.MAX_VALUE;
this(generator, 0, Integer.MAX_VALUE);
}

public StupidPool(
Supplier<T> generator,
int initCount,
int objectsCacheMaxCount
)
{
Preconditions.checkArgument(
initCount <= objectsCacheMaxCount,
"initCount[%s] must be less/equal to objectsCacheMaxCount[%s]",
initCount,
objectsCacheMaxCount
);
this.generator = generator;
this.objectsCacheMaxCount = objectsCacheMaxCount;

for (int i = 0; i < initCount; i++) {
objects.add(generator.get());
}
}

public ResourceHolder<T> take()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ public StupidPool<ByteBuffer> getIntermediateResultsPool(DruidProcessingConfig c
verifyDirectMemory(config);
return new StupidPool<>(
new OffheapBufferGenerator("intermediate processing", config.intermediateComputeSizeBytes()),
config.getNumThreads(),
config.poolCacheMaxCount()
);
}
Expand Down

0 comments on commit 32c5494

Please sign in to comment.