Merge pull request #2700 from palantir/fix/cassandra-metrics

[QoS] Feature/automatic-throttling-using-cassandra-metrics
bkrieger · Jan 12, 2018 · 706fbde · 706fbde
2 parents e5c1e8c + 11c6946
commit 706fbde
Show file tree

Hide file tree

Showing 74 changed files with 2,450 additions and 168 deletions.
diff --git a/atlasdb-api/build.gradle b/atlasdb-api/build.gradle
@@ -14,6 +14,7 @@ dependencies {
  compile group: 'javax.validation', name: 'validation-api'
 
  compile group: 'com.palantir.remoting-api', name: 'ssl-config'
+ compile group: 'com.palantir.remoting3', name: 'jaxrs-clients'
 
  processor group: 'org.immutables', name: 'value'
 

diff --git a/atlasdb-api/versions.lock b/atlasdb-api/versions.lock
@@ -83,7 +83,10 @@
  "com.google.code.findbugs:jsr305": {
  "locked": "3.0.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-commons"
+ "com.palantir.atlasdb:atlasdb-commons",
+ "com.palantir.remoting-api:errors",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils"
  ]
  },
  "com.google.guava:guava": {
@@ -254,6 +257,21 @@
  "com.palantir.remoting3:okhttp-clients"
  ]
  },
+ "com.squareup.okhttp3:okhttp": {
+ "locked": "3.8.1",
+ "transitive": [
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
+ ]
+ },
+ "com.squareup.okio:okio": {
+ "locked": "1.13.0",
+ "transitive": [
+ "com.squareup.okhttp3:okhttp"
+ ]
+ },
  "io.dropwizard.metrics:metrics-core": {
  "locked": "3.2.3",
  "transitive": [
@@ -394,7 +412,10 @@
  "com.google.code.findbugs:jsr305": {
  "locked": "3.0.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-commons"
+ "com.palantir.atlasdb:atlasdb-commons",
+ "com.palantir.remoting-api:errors",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils"
  ]
  },
  "com.google.guava:guava": {
@@ -565,6 +586,21 @@
  "com.palantir.remoting3:okhttp-clients"
  ]
  },
+ "com.squareup.okhttp3:okhttp": {
+ "locked": "3.8.1",
+ "transitive": [
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
+ ]
+ },
+ "com.squareup.okio:okio": {
+ "locked": "1.13.0",
+ "transitive": [
+ "com.squareup.okhttp3:okhttp"
+ ]
+ },
  "io.dropwizard.metrics:metrics-core": {
  "locked": "3.2.3",
  "transitive": [

diff --git a/atlasdb-cassandra/versions.lock b/atlasdb-cassandra/versions.lock
@@ -116,7 +116,10 @@
  "com.google.code.findbugs:jsr305": {
  "locked": "3.0.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-commons"
+ "com.palantir.atlasdb:atlasdb-commons",
+ "com.palantir.remoting-api:errors",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils"
  ]
  },
  "com.google.guava:guava": {
@@ -320,6 +323,7 @@
  "com.palantir.remoting3:jaxrs-clients": {
  "locked": "3.5.1",
  "transitive": [
+ "com.palantir.atlasdb:atlasdb-api",
  "com.palantir.atlasdb:qos-service-api"
  ]
  },
@@ -442,6 +446,21 @@
  "com.palantir.remoting3:okhttp-clients"
  ]
  },
+ "com.squareup.okhttp3:okhttp": {
+ "locked": "3.8.1",
+ "transitive": [
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
+ ]
+ },
+ "com.squareup.okio:okio": {
+ "locked": "1.13.0",
+ "transitive": [
+ "com.squareup.okhttp3:okhttp"
+ ]
+ },
  "com.squareup:javapoet": {
  "locked": "1.9.0",
  "transitive": [
@@ -748,7 +767,10 @@
  "com.google.code.findbugs:jsr305": {
  "locked": "3.0.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-commons"
+ "com.palantir.atlasdb:atlasdb-commons",
+ "com.palantir.remoting-api:errors",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils"
  ]
  },
  "com.google.guava:guava": {
@@ -952,6 +974,7 @@
  "com.palantir.remoting3:jaxrs-clients": {
  "locked": "3.5.1",
  "transitive": [
+ "com.palantir.atlasdb:atlasdb-api",
  "com.palantir.atlasdb:qos-service-api"
  ]
  },
@@ -1074,6 +1097,21 @@
  "com.palantir.remoting3:okhttp-clients"
  ]
  },
+ "com.squareup.okhttp3:okhttp": {
+ "locked": "3.8.1",
+ "transitive": [
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
+ ]
+ },
+ "com.squareup.okio:okio": {
+ "locked": "1.13.0",
+ "transitive": [
+ "com.squareup.okhttp3:okhttp"
+ ]
+ },
  "com.squareup:javapoet": {
  "locked": "1.9.0",
  "transitive": [

diff --git a/atlasdb-cli-distribution/versions.lock b/atlasdb-cli-distribution/versions.lock
@@ -243,6 +243,8 @@
  "com.palantir.atlasdb:atlasdb-commons",
  "com.palantir.remoting-api:errors",
  "com.palantir.remoting2:error-handling",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils",
  "io.dropwizard:dropwizard-util"
  ]
  },
@@ -680,6 +682,7 @@
  "com.palantir.remoting3:jaxrs-clients": {
  "locked": "3.5.1",
  "transitive": [
+ "com.palantir.atlasdb:atlasdb-api",
  "com.palantir.atlasdb:qos-service-api"
  ]
  },
@@ -828,7 +831,11 @@
  "com.squareup.okhttp3:okhttp": {
  "locked": "3.8.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-feign"
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.atlasdb:atlasdb-feign",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
  ]
  },
  "com.squareup.okio:okio": {

diff --git a/atlasdb-cli/versions.lock b/atlasdb-cli/versions.lock
@@ -226,6 +226,8 @@
  "com.palantir.atlasdb:atlasdb-commons",
  "com.palantir.remoting-api:errors",
  "com.palantir.remoting2:error-handling",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils",
  "io.dropwizard:dropwizard-util"
  ]
  },
@@ -596,6 +598,7 @@
  "com.palantir.remoting3:jaxrs-clients": {
  "locked": "3.5.1",
  "transitive": [
+ "com.palantir.atlasdb:atlasdb-api",
  "com.palantir.atlasdb:qos-service-api"
  ]
  },
@@ -744,7 +747,11 @@
  "com.squareup.okhttp3:okhttp": {
  "locked": "3.8.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-feign"
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.atlasdb:atlasdb-feign",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
  ]
  },
  "com.squareup.okio:okio": {
@@ -1239,6 +1246,8 @@
  "com.palantir.atlasdb:atlasdb-commons",
  "com.palantir.remoting-api:errors",
  "com.palantir.remoting2:error-handling",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils",
  "io.dropwizard:dropwizard-util"
  ]
  },
@@ -1606,6 +1615,7 @@
  "com.palantir.remoting3:jaxrs-clients": {
  "locked": "3.5.1",
  "transitive": [
+ "com.palantir.atlasdb:atlasdb-api",
  "com.palantir.atlasdb:qos-service-api"
  ]
  },
@@ -1754,7 +1764,11 @@
  "com.squareup.okhttp3:okhttp": {
  "locked": "3.8.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-feign"
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.atlasdb:atlasdb-feign",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
  ]
  },
  "com.squareup.okio:okio": {

diff --git a/atlasdb-client/versions.lock b/atlasdb-client/versions.lock
@@ -93,7 +93,10 @@
  "com.google.code.findbugs:jsr305": {
  "locked": "3.0.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-commons"
+ "com.palantir.atlasdb:atlasdb-commons",
+ "com.palantir.remoting-api:errors",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils"
  ]
  },
  "com.google.guava:guava": {
@@ -240,6 +243,7 @@
  "com.palantir.remoting3:jaxrs-clients": {
  "locked": "3.5.1",
  "transitive": [
+ "com.palantir.atlasdb:atlasdb-api",
  "com.palantir.atlasdb:qos-service-api"
  ]
  },
@@ -352,6 +356,21 @@
  "com.palantir.remoting3:okhttp-clients"
  ]
  },
+ "com.squareup.okhttp3:okhttp": {
+ "locked": "3.8.1",
+ "transitive": [
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
+ ]
+ },
+ "com.squareup.okio:okio": {
+ "locked": "1.13.0",
+ "transitive": [
+ "com.squareup.okhttp3:okhttp"
+ ]
+ },
  "com.squareup:javapoet": {
  "locked": "1.9.0",
  "requested": "1.9.0"
@@ -535,7 +554,10 @@
  "com.google.code.findbugs:jsr305": {
  "locked": "3.0.1",
  "transitive": [
- "com.palantir.atlasdb:atlasdb-commons"
+ "com.palantir.atlasdb:atlasdb-commons",
+ "com.palantir.remoting-api:errors",
+ "com.palantir.remoting3:jaxrs-clients",
+ "com.palantir.remoting3:refresh-utils"
  ]
  },
  "com.google.guava:guava": {
@@ -682,6 +704,7 @@
  "com.palantir.remoting3:jaxrs-clients": {
  "locked": "3.5.1",
  "transitive": [
+ "com.palantir.atlasdb:atlasdb-api",
  "com.palantir.atlasdb:qos-service-api"
  ]
  },
@@ -794,6 +817,21 @@
  "com.palantir.remoting3:okhttp-clients"
  ]
  },
+ "com.squareup.okhttp3:okhttp": {
+ "locked": "3.8.1",
+ "transitive": [
+ "com.netflix.feign:feign-okhttp",
+ "com.palantir.remoting3:okhttp-clients",
+ "com.palantir.remoting3:tracing-okhttp3",
+ "com.squareup.okhttp3:logging-interceptor"
+ ]
+ },
+ "com.squareup.okio:okio": {
+ "locked": "1.13.0",
+ "transitive": [
+ "com.squareup.okhttp3:okhttp"
+ ]
+ },
  "com.squareup:javapoet": {
  "locked": "1.9.0",
  "requested": "1.9.0"

diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/TransactionManagers.java
@@ -70,7 +70,9 @@
 import com.palantir.atlasdb.persistentlock.NoOpPersistentLockService;
 import com.palantir.atlasdb.persistentlock.PersistentLockService;
 import com.palantir.atlasdb.qos.QosClient;
+import com.palantir.atlasdb.qos.QosService;
 import com.palantir.atlasdb.qos.client.AtlasDbQosClient;
+import com.palantir.atlasdb.qos.config.ImmutableQosLimitsConfig;
 import com.palantir.atlasdb.qos.config.QosClientConfig;
 import com.palantir.atlasdb.qos.ratelimit.QosRateLimiters;
 import com.palantir.atlasdb.schema.generated.SweepTableFactory;
@@ -113,6 +115,9 @@
 import com.palantir.lock.impl.LegacyTimelockService;
 import com.palantir.lock.impl.LockServiceImpl;
 import com.palantir.lock.v2.TimelockService;
+import com.palantir.remoting.api.config.service.ServiceConfiguration;
+import com.palantir.remoting3.clients.ClientConfigurations;
+import com.palantir.remoting3.jaxrs.JaxRsClient;
 import com.palantir.timestamp.TimestampService;
 import com.palantir.timestamp.TimestampStoreInvalidator;
 import com.palantir.tritium.metrics.registry.DefaultTaggedMetricRegistry;
@@ -170,6 +175,7 @@ public static ImmutableTransactionManagers.ConfigBuildStage builder() {
 
  /**
  * Accepts a single {@link Schema}.
+ *
  * @see TransactionManagers#createInMemory(Set)
  */
  public static SerializableTransactionManager createInMemory(Schema schema) {
@@ -299,9 +305,26 @@ public SerializableTransactionManager serializable() {
  }
 
  private QosClient getQosClient(Supplier<QosClientConfig> config) {
- QosRateLimiters rateLimiters = QosRateLimiters.create(
- JavaSuppliers.compose(conf -> conf.maxBackoffSleepTime().toMilliseconds(), config),
- JavaSuppliers.compose(QosClientConfig::limits, config));
+ Optional<ServiceConfiguration> qosServiceConfig = config.get().qosService();
+ QosRateLimiters rateLimiters;
+ if (qosServiceConfig.isPresent()) {
+ QosService qosService = JaxRsClient.create(QosService.class, userAgent(),
+ ClientConfigurations.of(qosServiceConfig.get()));
+ rateLimiters = QosRateLimiters.create(
+ JavaSuppliers.compose(conf -> conf.maxBackoffSleepTime().toMilliseconds(), config),
+ () -> {
+ long readLimit = qosService.readLimit(config().getNamespaceString());
+ long writeLimit = qosService.writeLimit(config().getNamespaceString());
+ return ImmutableQosLimitsConfig.builder()
+ .readBytesPerSecond(readLimit)
+ .writeBytesPerSecond(writeLimit)
+ .build();
+ });
+ } else {
+ rateLimiters = QosRateLimiters.create(
+ JavaSuppliers.compose(conf -> conf.maxBackoffSleepTime().toMilliseconds(), config),
+ JavaSuppliers.compose(QosClientConfig::limits, config));
+ }
  return AtlasDbQosClient.create(rateLimiters);
  }