Skip to content

Commit

Permalink
New property for each metric that tells the StatsDEmitter to convert …
Browse files Browse the repository at this point in the history
…metric values from range 0-1 to 0-100. This (apache#3936)

prevents rates and percentages expressed as Doubles (0.xx) from being rounded down to 0.
  • Loading branch information
michaelschiff authored and fjy committed Feb 16, 2017
1 parent ca6053d commit e5fb0e1
Show file tree
Hide file tree
Showing 8 changed files with 190 additions and 53 deletions.
4 changes: 3 additions & 1 deletion docs/content/development/extensions-contrib/statsd.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,12 @@ All the configuration parameters for the StatsD emitter are under `druid.emitter

Each metric sent to StatsD must specify a type, one of `[timer, counter, guage]`. StatsD Emitter expects this mapping to
be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric.
StatsD expects that metric values be integers. Druid emits some metrics with values between the range 0 and 1. To accommodate these metrics they are converted
into the range 0 to 100. This conversion can be enabled by setting the optional "convertRange" field true in the JSON mapping file.
If the user does not specify their own JSON file, a default mapping is used. All
metrics are expected to be mapped. Metrics which are not mapped will log an error.
StatsD metric path is organized using the following schema:
`<druid metric name> : { "dimensions" : <dimension list>, "type" : <StatsD type>}`
`<druid metric name> : { "dimensions" : <dimension list>, "type" : <StatsD type>, "convertRange" : true/false}`
e.g.
`query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"}`

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,12 @@ public DimensionConverter(ObjectMapper mapper, String dimensionMapPath)
metricMap = readMap(mapper, dimensionMapPath);
}

public StatsDMetric.Type addFilteredUserDims(String service, String metric, Map<String, Object> userDims, ImmutableList.Builder<String> builder)
public StatsDMetric addFilteredUserDims(
String service,
String metric,
Map<String, Object> userDims,
ImmutableList.Builder<String> builder
)
{
/*
Find the metric in the map. If we cant find it try to look it up prefixed by the service name.
Expand All @@ -64,7 +69,7 @@ public StatsDMetric.Type addFilteredUserDims(String service, String metric, Map<
builder.add(userDims.get(dim).toString());
}
}
return statsDMetric.type;
return statsDMetric;
} else {
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import com.timgroup.statsd.NonBlockingStatsDClient;
import com.timgroup.statsd.StatsDClient;
import com.timgroup.statsd.StatsDClientErrorHandler;

import io.druid.java.util.common.logger.Logger;

import java.io.IOException;
Expand All @@ -47,28 +46,36 @@ public class StatsDEmitter implements Emitter
private final StatsDEmitterConfig config;
private final DimensionConverter converter;

public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper) {
this.config = config;
this.converter = new DimensionConverter(mapper, config.getDimensionMapPath());
statsd = new NonBlockingStatsDClient(
config.getPrefix(),
config.getHostname(),
config.getPort(),
new StatsDClientErrorHandler()
{
private int exceptionCount = 0;
@Override
public void handle(Exception exception)
{
if (exceptionCount % 1000 == 0) {
log.error(exception, "Error sending metric to StatsD.");
}
exceptionCount += 1;
}
}
public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper)
{
this(config, mapper,
new NonBlockingStatsDClient(
config.getPrefix(),
config.getHostname(),
config.getPort(),
new StatsDClientErrorHandler()
{
private int exceptionCount = 0;

@Override
public void handle(Exception exception)
{
if (exceptionCount % 1000 == 0) {
log.error(exception, "Error sending metric to StatsD.");
}
exceptionCount += 1;
}
}
)
);
}

public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper, StatsDClient client)
{
this.config = config;
this.converter = new DimensionConverter(mapper, config.getDimensionMapPath());
this.statsd = client;
}

@Override
public void start() {}
Expand All @@ -91,28 +98,29 @@ public void emit(Event event)
nameBuilder.add(service);
nameBuilder.add(metric);

StatsDMetric.Type metricType = converter.addFilteredUserDims(service, metric, userDims, nameBuilder);
StatsDMetric statsDMetric = converter.addFilteredUserDims(service, metric, userDims, nameBuilder);

if (metricType != null) {
if (statsDMetric != null) {

String fullName = Joiner.on(config.getSeparator())
.join(nameBuilder.build())
.replaceAll(DRUID_METRIC_SEPARATOR, config.getSeparator())
.replaceAll(STATSD_SEPARATOR, config.getSeparator());

switch (metricType) {
long val = statsDMetric.convertRange ? Math.round(value.doubleValue() * 100) : value.longValue();
switch (statsDMetric.type) {
case count:
statsd.count(fullName, value.longValue());
statsd.count(fullName, val);
break;
case timer:
statsd.time(fullName, value.longValue());
statsd.time(fullName, val);
break;
case gauge:
statsd.gauge(fullName, value.longValue());
statsd.gauge(fullName, val);
break;
}
} else {
log.error("Metric=[%s] has no StatsD type mapping", metric);
log.error("Metric=[%s] has no StatsD type mapping", statsDMetric);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,10 @@
public class StatsDEmitterModule implements DruidModule
{
private static final String EMITTER_TYPE = "statsd";

@Override
public List<? extends Module> getJacksonModules() {
public List<? extends Module> getJacksonModules()
{
return Collections.EMPTY_LIST;
}

Expand All @@ -51,7 +53,8 @@ public void configure(Binder binder)
@Provides
@ManageLifecycle
@Named(EMITTER_TYPE)
public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper){
public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper)
{
return new StatsDEmitter(config, mapper);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,20 +25,27 @@
import java.util.SortedSet;

/**
*/
public class StatsDMetric {
*/
public class StatsDMetric
{
public final SortedSet<String> dimensions;
public final Type type;
public final boolean convertRange;

@JsonCreator
public StatsDMetric(
@JsonProperty("dimensions") SortedSet<String> dimensions,
@JsonProperty("type") Type type)
@JsonProperty("type") Type type,
@JsonProperty("convertRange") boolean convertRange
)
{
this.dimensions = dimensions;
this.type = type;
this.convertRange = convertRange;
}

public enum Type {
public enum Type
{
count, gauge, timer
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"query/cache/delta/hits" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/misses" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/evictions" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count", "convertRange" : true },
"query/cache/delta/averageBytes" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/timeouts" : { "dimensions" : [], "type" : "count" },
"query/cache/delta/errors" : { "dimensions" : [], "type" : "count" },
Expand All @@ -25,7 +25,7 @@
"query/cache/total/hits" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/misses" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/evictions" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge", "convertRange" : true },
"query/cache/total/averageBytes" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/timeouts" : { "dimensions" : [], "type" : "gauge" },
"query/cache/total/errors" : { "dimensions" : [], "type" : "gauge" },
Expand Down Expand Up @@ -65,7 +65,7 @@

"segment/max" : { "dimensions" : [], "type" : "gauge"},
"segment/used" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
"segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" },
"segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge", "convertRange" : true },

"jvm/pool/committed" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
"jvm/pool/init" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" },
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,27 +35,28 @@ public class DimensionConverterTest
public void testConvert() throws Exception
{
DimensionConverter dimensionConverter = new DimensionConverter(new ObjectMapper(), null);
ServiceMetricEvent event = new ServiceMetricEvent.Builder().setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1");
ServiceMetricEvent event = new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1");

ImmutableList.Builder<String> actual = new ImmutableList.Builder<>();
StatsDMetric.Type type = dimensionConverter.addFilteredUserDims(
StatsDMetric statsDMetric = dimensionConverter.addFilteredUserDims(
event.getService(),
event.getMetric(),
event.getUserDims(),
actual
);
assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, type);
assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, statsDMetric.type);
ImmutableList.Builder<String> expected = new ImmutableList.Builder<>();
expected.add("data-source");
expected.add("groupBy");
Expand Down
111 changes: 111 additions & 0 deletions extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import com.fasterxml.jackson.databind.ObjectMapper;
import com.metamx.emitter.service.ServiceMetricEvent;
import com.timgroup.statsd.StatsDClient;
import io.druid.emitter.statsd.StatsDEmitter;
import io.druid.emitter.statsd.StatsDEmitterConfig;

import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;

import org.joda.time.DateTime;
import org.junit.Test;

/**
*/
public class StatsDEmitterTest
{
@Test
public void testConvertRange()
{
StatsDClient client = createMock(StatsDClient.class);
StatsDEmitter emitter = new StatsDEmitter(
new StatsDEmitterConfig("localhost", 8888, null, null, null, null),
new ObjectMapper(),
client
);
client.gauge("broker.query.cache.total.hitRate", 54);
replay(client);
emitter.emit(new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.build(new DateTime(), "query/cache/total/hitRate", 0.54)
.build("broker", "brokerHost1")
);
verify(client);
}

@Test
public void testNoConvertRange()
{
StatsDClient client = createMock(StatsDClient.class);
StatsDEmitter emitter = new StatsDEmitter(
new StatsDEmitterConfig("localhost", 8888, null, null, null, null),
new ObjectMapper(),
client
);
client.time("broker.query.time.data-source.groupBy", 10);
replay(client);
emitter.emit(new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1")
);
verify(client);
}

@Test
public void testConfigOptions()
{
StatsDClient client = createMock(StatsDClient.class);
StatsDEmitter emitter = new StatsDEmitter(
new StatsDEmitterConfig("localhost", 8888, null, "#", true, null),
new ObjectMapper(),
client
);
client.time("brokerHost1#broker#query#time#data-source#groupBy", 10);
replay(client);
emitter.emit(new ServiceMetricEvent.Builder()
.setDimension("dataSource", "data-source")
.setDimension("type", "groupBy")
.setDimension("interval", "2013/2015")
.setDimension("some_random_dim1", "random_dim_value1")
.setDimension("some_random_dim2", "random_dim_value2")
.setDimension("hasFilters", "no")
.setDimension("duration", "P1D")
.setDimension("remoteAddress", "194.0.90.2")
.setDimension("id", "ID")
.setDimension("context", "{context}")
.build(new DateTime(), "query/time", 10)
.build("broker", "brokerHost1")
);
verify(client);
}
}

0 comments on commit e5fb0e1

Please sign in to comment.