From e5fb0e1ff5ee915a5bb21fe0803ebce5abeabcb1 Mon Sep 17 00:00:00 2001 From: michaelschiff Date: Thu, 16 Feb 2017 13:55:56 -0800 Subject: [PATCH] New property for each metric that tells the StatsDEmitter to convert metric values from range 0-1 to 0-100. This (#3936) prevents rates and percentages expressed as Doubles (0.xx) from being rounded down to 0. --- .../development/extensions-contrib/statsd.md | 4 +- .../emitter/statsd/DimensionConverter.java | 9 +- .../druid/emitter/statsd/StatsDEmitter.java | 62 +++++----- .../emitter/statsd/StatsDEmitterModule.java | 7 +- .../io/druid/emitter/statsd/StatsDMetric.java | 15 ++- .../resources/defaultMetricDimensions.json | 6 +- .../src/test/java/DimensionConverterTest.java | 29 ++--- .../src/test/java/StatsDEmitterTest.java | 111 ++++++++++++++++++ 8 files changed, 190 insertions(+), 53 deletions(-) create mode 100644 extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java diff --git a/docs/content/development/extensions-contrib/statsd.md b/docs/content/development/extensions-contrib/statsd.md index 25d41b38afb9..8894ec87a6f8 100644 --- a/docs/content/development/extensions-contrib/statsd.md +++ b/docs/content/development/extensions-contrib/statsd.md @@ -29,10 +29,12 @@ All the configuration parameters for the StatsD emitter are under `druid.emitter Each metric sent to StatsD must specify a type, one of `[timer, counter, guage]`. StatsD Emitter expects this mapping to be provided as a JSON file. Additionally, this mapping specifies which dimensions should be included for each metric. +StatsD expects that metric values be integers. Druid emits some metrics with values between the range 0 and 1. To accommodate these metrics they are converted +into the range 0 to 100. This conversion can be enabled by setting the optional "convertRange" field true in the JSON mapping file. If the user does not specify their own JSON file, a default mapping is used. All metrics are expected to be mapped. Metrics which are not mapped will log an error. StatsD metric path is organized using the following schema: -` : { "dimensions" : , "type" : }` +` : { "dimensions" : , "type" : , "convertRange" : true/false}` e.g. `query/time" : { "dimensions" : ["dataSource", "type"], "type" : "timer"}` diff --git a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/DimensionConverter.java b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/DimensionConverter.java index 7279799b6e3c..ed3521f14181 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/DimensionConverter.java +++ b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/DimensionConverter.java @@ -46,7 +46,12 @@ public DimensionConverter(ObjectMapper mapper, String dimensionMapPath) metricMap = readMap(mapper, dimensionMapPath); } - public StatsDMetric.Type addFilteredUserDims(String service, String metric, Map userDims, ImmutableList.Builder builder) + public StatsDMetric addFilteredUserDims( + String service, + String metric, + Map userDims, + ImmutableList.Builder builder + ) { /* Find the metric in the map. If we cant find it try to look it up prefixed by the service name. @@ -64,7 +69,7 @@ public StatsDMetric.Type addFilteredUserDims(String service, String metric, Map< builder.add(userDims.get(dim).toString()); } } - return statsDMetric.type; + return statsDMetric; } else { return null; } diff --git a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java index 4b44f0a71a37..416f28f162ce 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java +++ b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitter.java @@ -28,7 +28,6 @@ import com.timgroup.statsd.NonBlockingStatsDClient; import com.timgroup.statsd.StatsDClient; import com.timgroup.statsd.StatsDClientErrorHandler; - import io.druid.java.util.common.logger.Logger; import java.io.IOException; @@ -47,28 +46,36 @@ public class StatsDEmitter implements Emitter private final StatsDEmitterConfig config; private final DimensionConverter converter; - public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper) { - this.config = config; - this.converter = new DimensionConverter(mapper, config.getDimensionMapPath()); - statsd = new NonBlockingStatsDClient( - config.getPrefix(), - config.getHostname(), - config.getPort(), - new StatsDClientErrorHandler() - { - private int exceptionCount = 0; - @Override - public void handle(Exception exception) - { - if (exceptionCount % 1000 == 0) { - log.error(exception, "Error sending metric to StatsD."); - } - exceptionCount += 1; - } - } + public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper) + { + this(config, mapper, + new NonBlockingStatsDClient( + config.getPrefix(), + config.getHostname(), + config.getPort(), + new StatsDClientErrorHandler() + { + private int exceptionCount = 0; + + @Override + public void handle(Exception exception) + { + if (exceptionCount % 1000 == 0) { + log.error(exception, "Error sending metric to StatsD."); + } + exceptionCount += 1; + } + } + ) ); } + public StatsDEmitter(StatsDEmitterConfig config, ObjectMapper mapper, StatsDClient client) + { + this.config = config; + this.converter = new DimensionConverter(mapper, config.getDimensionMapPath()); + this.statsd = client; + } @Override public void start() {} @@ -91,28 +98,29 @@ public void emit(Event event) nameBuilder.add(service); nameBuilder.add(metric); - StatsDMetric.Type metricType = converter.addFilteredUserDims(service, metric, userDims, nameBuilder); + StatsDMetric statsDMetric = converter.addFilteredUserDims(service, metric, userDims, nameBuilder); - if (metricType != null) { + if (statsDMetric != null) { String fullName = Joiner.on(config.getSeparator()) .join(nameBuilder.build()) .replaceAll(DRUID_METRIC_SEPARATOR, config.getSeparator()) .replaceAll(STATSD_SEPARATOR, config.getSeparator()); - switch (metricType) { + long val = statsDMetric.convertRange ? Math.round(value.doubleValue() * 100) : value.longValue(); + switch (statsDMetric.type) { case count: - statsd.count(fullName, value.longValue()); + statsd.count(fullName, val); break; case timer: - statsd.time(fullName, value.longValue()); + statsd.time(fullName, val); break; case gauge: - statsd.gauge(fullName, value.longValue()); + statsd.gauge(fullName, val); break; } } else { - log.error("Metric=[%s] has no StatsD type mapping", metric); + log.error("Metric=[%s] has no StatsD type mapping", statsDMetric); } } } diff --git a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java index 66e52863b329..ad0b8f8cd75c 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java +++ b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDEmitterModule.java @@ -37,8 +37,10 @@ public class StatsDEmitterModule implements DruidModule { private static final String EMITTER_TYPE = "statsd"; + @Override - public List getJacksonModules() { + public List getJacksonModules() + { return Collections.EMPTY_LIST; } @@ -51,7 +53,8 @@ public void configure(Binder binder) @Provides @ManageLifecycle @Named(EMITTER_TYPE) - public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper){ + public Emitter getEmitter(StatsDEmitterConfig config, ObjectMapper mapper) + { return new StatsDEmitter(config, mapper); } } diff --git a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDMetric.java b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDMetric.java index 2a557ed31c0f..2a469296fd89 100644 --- a/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDMetric.java +++ b/extensions-contrib/statsd-emitter/src/main/java/io/druid/emitter/statsd/StatsDMetric.java @@ -25,20 +25,27 @@ import java.util.SortedSet; /** -*/ -public class StatsDMetric { + */ +public class StatsDMetric +{ public final SortedSet dimensions; public final Type type; + public final boolean convertRange; + @JsonCreator public StatsDMetric( @JsonProperty("dimensions") SortedSet dimensions, - @JsonProperty("type") Type type) + @JsonProperty("type") Type type, + @JsonProperty("convertRange") boolean convertRange + ) { this.dimensions = dimensions; this.type = type; + this.convertRange = convertRange; } - public enum Type { + public enum Type + { count, gauge, timer } } diff --git a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json index a452ec6ea308..1f9d6d631904 100644 --- a/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json +++ b/extensions-contrib/statsd-emitter/src/main/resources/defaultMetricDimensions.json @@ -15,7 +15,7 @@ "query/cache/delta/hits" : { "dimensions" : [], "type" : "count" }, "query/cache/delta/misses" : { "dimensions" : [], "type" : "count" }, "query/cache/delta/evictions" : { "dimensions" : [], "type" : "count" }, - "query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count" }, + "query/cache/delta/hitRate" : { "dimensions" : [], "type" : "count", "convertRange" : true }, "query/cache/delta/averageBytes" : { "dimensions" : [], "type" : "count" }, "query/cache/delta/timeouts" : { "dimensions" : [], "type" : "count" }, "query/cache/delta/errors" : { "dimensions" : [], "type" : "count" }, @@ -25,7 +25,7 @@ "query/cache/total/hits" : { "dimensions" : [], "type" : "gauge" }, "query/cache/total/misses" : { "dimensions" : [], "type" : "gauge" }, "query/cache/total/evictions" : { "dimensions" : [], "type" : "gauge" }, - "query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge" }, + "query/cache/total/hitRate" : { "dimensions" : [], "type" : "gauge", "convertRange" : true }, "query/cache/total/averageBytes" : { "dimensions" : [], "type" : "gauge" }, "query/cache/total/timeouts" : { "dimensions" : [], "type" : "gauge" }, "query/cache/total/errors" : { "dimensions" : [], "type" : "gauge" }, @@ -65,7 +65,7 @@ "segment/max" : { "dimensions" : [], "type" : "gauge"}, "segment/used" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" }, - "segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge" }, + "segment/usedPercent" : { "dimensions" : ["dataSource", "tier", "priority"], "type" : "gauge", "convertRange" : true }, "jvm/pool/committed" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" }, "jvm/pool/init" : { "dimensions" : ["poolKind", "poolName"], "type" : "gauge" }, diff --git a/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java b/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java index 130f328d65c5..9ede60772607 100644 --- a/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java +++ b/extensions-contrib/statsd-emitter/src/test/java/DimensionConverterTest.java @@ -35,27 +35,28 @@ public class DimensionConverterTest public void testConvert() throws Exception { DimensionConverter dimensionConverter = new DimensionConverter(new ObjectMapper(), null); - ServiceMetricEvent event = new ServiceMetricEvent.Builder().setDimension("dataSource", "data-source") - .setDimension("type", "groupBy") - .setDimension("interval", "2013/2015") - .setDimension("some_random_dim1", "random_dim_value1") - .setDimension("some_random_dim2", "random_dim_value2") - .setDimension("hasFilters", "no") - .setDimension("duration", "P1D") - .setDimension("remoteAddress", "194.0.90.2") - .setDimension("id", "ID") - .setDimension("context", "{context}") - .build(new DateTime(), "query/time", 10) - .build("broker", "brokerHost1"); + ServiceMetricEvent event = new ServiceMetricEvent.Builder() + .setDimension("dataSource", "data-source") + .setDimension("type", "groupBy") + .setDimension("interval", "2013/2015") + .setDimension("some_random_dim1", "random_dim_value1") + .setDimension("some_random_dim2", "random_dim_value2") + .setDimension("hasFilters", "no") + .setDimension("duration", "P1D") + .setDimension("remoteAddress", "194.0.90.2") + .setDimension("id", "ID") + .setDimension("context", "{context}") + .build(new DateTime(), "query/time", 10) + .build("broker", "brokerHost1"); ImmutableList.Builder actual = new ImmutableList.Builder<>(); - StatsDMetric.Type type = dimensionConverter.addFilteredUserDims( + StatsDMetric statsDMetric = dimensionConverter.addFilteredUserDims( event.getService(), event.getMetric(), event.getUserDims(), actual ); - assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, type); + assertEquals("correct StatsDMetric.Type", StatsDMetric.Type.timer, statsDMetric.type); ImmutableList.Builder expected = new ImmutableList.Builder<>(); expected.add("data-source"); expected.add("groupBy"); diff --git a/extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java b/extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java new file mode 100644 index 000000000000..a6286e46504d --- /dev/null +++ b/extensions-contrib/statsd-emitter/src/test/java/StatsDEmitterTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.metamx.emitter.service.ServiceMetricEvent; +import com.timgroup.statsd.StatsDClient; +import io.druid.emitter.statsd.StatsDEmitter; +import io.druid.emitter.statsd.StatsDEmitterConfig; + +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import org.joda.time.DateTime; +import org.junit.Test; + +/** + */ +public class StatsDEmitterTest +{ + @Test + public void testConvertRange() + { + StatsDClient client = createMock(StatsDClient.class); + StatsDEmitter emitter = new StatsDEmitter( + new StatsDEmitterConfig("localhost", 8888, null, null, null, null), + new ObjectMapper(), + client + ); + client.gauge("broker.query.cache.total.hitRate", 54); + replay(client); + emitter.emit(new ServiceMetricEvent.Builder() + .setDimension("dataSource", "data-source") + .build(new DateTime(), "query/cache/total/hitRate", 0.54) + .build("broker", "brokerHost1") + ); + verify(client); + } + + @Test + public void testNoConvertRange() + { + StatsDClient client = createMock(StatsDClient.class); + StatsDEmitter emitter = new StatsDEmitter( + new StatsDEmitterConfig("localhost", 8888, null, null, null, null), + new ObjectMapper(), + client + ); + client.time("broker.query.time.data-source.groupBy", 10); + replay(client); + emitter.emit(new ServiceMetricEvent.Builder() + .setDimension("dataSource", "data-source") + .setDimension("type", "groupBy") + .setDimension("interval", "2013/2015") + .setDimension("some_random_dim1", "random_dim_value1") + .setDimension("some_random_dim2", "random_dim_value2") + .setDimension("hasFilters", "no") + .setDimension("duration", "P1D") + .setDimension("remoteAddress", "194.0.90.2") + .setDimension("id", "ID") + .setDimension("context", "{context}") + .build(new DateTime(), "query/time", 10) + .build("broker", "brokerHost1") + ); + verify(client); + } + + @Test + public void testConfigOptions() + { + StatsDClient client = createMock(StatsDClient.class); + StatsDEmitter emitter = new StatsDEmitter( + new StatsDEmitterConfig("localhost", 8888, null, "#", true, null), + new ObjectMapper(), + client + ); + client.time("brokerHost1#broker#query#time#data-source#groupBy", 10); + replay(client); + emitter.emit(new ServiceMetricEvent.Builder() + .setDimension("dataSource", "data-source") + .setDimension("type", "groupBy") + .setDimension("interval", "2013/2015") + .setDimension("some_random_dim1", "random_dim_value1") + .setDimension("some_random_dim2", "random_dim_value2") + .setDimension("hasFilters", "no") + .setDimension("duration", "P1D") + .setDimension("remoteAddress", "194.0.90.2") + .setDimension("id", "ID") + .setDimension("context", "{context}") + .build(new DateTime(), "query/time", 10) + .build("broker", "brokerHost1") + ); + verify(client); + } +}