Merge branch 'develop' into dependency-lock

.gitignore

@@ -30,6 +30,7 @@ generated_testSrc/
 .factorypath
 out/
 atlasdb-cli/var/data/
+*.hprof
 
 # Common text-editor temp files
 # emacs

atlasdb-api/src/main/java/com/palantir/atlasdb/transaction/api/TransactionManager.java

@@ -120,4 +120,10 @@ <T, E extends Exception> T runTaskThrowOnConflict(TransactionTask<T, E> task)
  * @throws IllegalStateException if the transaction manager has been closed.
  */
  long getUnreadableTimestamp();
+
+ /**
+ * Clear the timestamp cache. This is mostly useful for tests that perform operations that would invalidate
+ * the cache, although this can also be used to free up some memory.
+ */
+ void clearTimestampCache();
 }

atlasdb-cassandra-integration-tests/src/test/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPoolIntegrationTest.java

@@ -129,6 +129,8 @@ public void testIsRetriableException() {
  public void testIsRetriableWithBackoffException() {
  assertTrue(CassandraClientPool.isRetriableWithBackoffException(new NoSuchElementException()));
  assertTrue(CassandraClientPool.isRetriableWithBackoffException(new UnavailableException()));
+ assertTrue(CassandraClientPool.isRetriableWithBackoffException(
+ new TTransportException(new SocketTimeoutException())));
  assertTrue(CassandraClientPool.isRetriableWithBackoffException(
  new TTransportException(new UnavailableException())));
  }

atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/OneNodeDownGetTest.java

@@ -112,14 +112,14 @@ public void getRangeOfTimestampsThrows() {
  ClosableIterator<RowResult<Set<Long>>> it = OneNodeDownTestSuite.db.getRangeOfTimestamps(
  OneNodeDownTestSuite.TEST_TABLE, range, Long.MAX_VALUE);
  assertThatThrownBy(() -> it.next())
- .isInstanceOf(InsufficientConsistencyException.class)
+ .isExactlyInstanceOf(InsufficientConsistencyException.class)
  .hasMessage("This operation requires all Cassandra nodes to be up and available.");
  }
 
  @Test
  public void getAllTimestampsThrows() {
  assertThatThrownBy(() -> OneNodeDownTestSuite.db.getAllTimestamps(OneNodeDownTestSuite.TEST_TABLE,
  ImmutableSet.of(OneNodeDownTestSuite.CELL_1_1), Long.MAX_VALUE))
- .isInstanceOf(PalantirRuntimeException.class);
+ .isExactlyInstanceOf(PalantirRuntimeException.class);
  }
 }

atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/OneNodeDownMetadataTest.java

@@ -51,7 +51,10 @@ public void putMetadataForTableThrows() {
  TableMetadata newTableMetadata = new TableMetadata(new NameMetadataDescription(),
  new ColumnMetadataDescription(), ConflictHandler.IGNORE_ALL);
  assertThatThrownBy(() -> OneNodeDownTestSuite.db.putMetadataForTable(OneNodeDownTestSuite.TEST_TABLE,
- newTableMetadata.persistToBytes())).isInstanceOf(IllegalStateException.class);
+ newTableMetadata.persistToBytes())).isExactlyInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("At schema version UNREACHABLE");
+
+ canGetMetadataForTable();
  }
 
  @Test
@@ -60,6 +63,9 @@ public void putMetadataForTablesThrows() {
  new ColumnMetadataDescription(), ConflictHandler.IGNORE_ALL);
  assertThatThrownBy(() -> OneNodeDownTestSuite.db.putMetadataForTables(
  ImmutableMap.of(OneNodeDownTestSuite.TEST_TABLE, newTableMetadata.persistToBytes())))
- .isInstanceOf(IllegalStateException.class);
+ .isExactlyInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("At schema version UNREACHABLE");
+
+ canGetMetadataForTable();
  }
 }

atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/OneNodeDownPutTest.java

@@ -15,6 +15,7 @@
  */
 package com.palantir.cassandra.multinode;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertEquals;
 
@@ -74,8 +75,13 @@ public void canPutUnlessExists() {
  @Test
  public void putUnlessExistsThrowsOnExists() {
  assertThatThrownBy(() -> OneNodeDownTestSuite.db.putUnlessExists(OneNodeDownTestSuite.TEST_TABLE,
- ImmutableMap.of(OneNodeDownTestSuite.CELL_1_1, OneNodeDownTestSuite.DEFAULT_CONTENTS))).isInstanceOf(
- KeyAlreadyExistsException.class);
+ ImmutableMap.of(OneNodeDownTestSuite.CELL_1_1, OneNodeDownTestSuite.DEFAULT_CONTENTS)))
+ .isInstanceOf(KeyAlreadyExistsException.class);
+
+ Map<Cell, Value> result = OneNodeDownTestSuite.db.get(OneNodeDownTestSuite.TEST_TABLE,
+ ImmutableMap.of(OneNodeDownTestSuite.CELL_1_1, AtlasDbConstants.TRANSACTION_TS));
+ assertThat(Value.create(OneNodeDownTestSuite.DEFAULT_CONTENTS, AtlasDbConstants.TRANSACTION_TS))
+ .isNotEqualTo(result.get(OneNodeDownTestSuite.CELL_1_1));
  }
 
  @Test

atlasdb-cassandra-multinode-tests/src/test/java/com/palantir/cassandra/multinode/OneNodeDownTableManipulationTest.java

@@ -15,6 +15,7 @@
  */
 package com.palantir.cassandra.multinode;
 
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import org.junit.Test;
@@ -31,29 +32,48 @@ public class OneNodeDownTableManipulationTest {
 
  @Test
  public void createTableThrows() {
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames()).doesNotContain(NEW_TABLE);
  assertThatThrownBy(
  () -> OneNodeDownTestSuite.db.createTable(NEW_TABLE, AtlasDbConstants.GENERIC_TABLE_METADATA))
- .isExactlyInstanceOf(IllegalStateException.class);
+ .isExactlyInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("At schema version UNREACHABLE");
+ // This documents and verifies the current behaviour, creating the table in spite of the exception
+ // Seems to be inconsistent with the API
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames()).contains(NEW_TABLE);
  }
 
  @Test
  public void createTablesThrows() {
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames()).doesNotContain(NEW_TABLE2);
  assertThatThrownBy(() -> OneNodeDownTestSuite.db.createTables(
  ImmutableMap.of(NEW_TABLE2, AtlasDbConstants.GENERIC_TABLE_METADATA)))
- .isExactlyInstanceOf(IllegalStateException.class);
+ .isExactlyInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("At schema version UNREACHABLE");
+ // This documents and verifies the current behaviour, creating the table in spite of the exception
+ // Seems to be inconsistent with the API
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames()).contains(NEW_TABLE2);
  }
 
  @Test
  public void dropTableThrows() {
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames()).contains(OneNodeDownTestSuite.TEST_TABLE_TO_DROP);
  assertThatThrownBy(() -> OneNodeDownTestSuite.db.dropTable(OneNodeDownTestSuite.TEST_TABLE_TO_DROP))
  .isExactlyInstanceOf(IllegalStateException.class);
+ // This documents and verifies the current behaviour, dropping the table in spite of the exception
+ // Seems to be inconsistent with the API
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames()).doesNotContain(OneNodeDownTestSuite.TEST_TABLE_TO_DROP);
  }
 
  @Test
  public void dropTablesThrows() {
- ImmutableSet<TableReference> tablesToDrop = ImmutableSet.of(OneNodeDownTestSuite.TEST_TABLE_TO_DROP_2);
- assertThatThrownBy(() -> OneNodeDownTestSuite.db.dropTables(tablesToDrop))
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames()).contains(OneNodeDownTestSuite.TEST_TABLE_TO_DROP_2);
+ assertThatThrownBy(() -> OneNodeDownTestSuite.db.dropTables(
+ ImmutableSet.of(OneNodeDownTestSuite.TEST_TABLE_TO_DROP_2)))
  .isExactlyInstanceOf(IllegalStateException.class);
+ // This documents and verifies the current behaviour, dropping the table in spite of the exception
+ // Seems to be inconsistent with the API
+ assertThat(OneNodeDownTestSuite.db.getAllTableNames())
+ .doesNotContain(OneNodeDownTestSuite.TEST_TABLE_TO_DROP_2);
  }
 
  @Test

atlasdb-cassandra/src/main/java/com/palantir/atlasdb/cassandra/CassandraAtlasDbFactory.java

@@ -25,9 +25,9 @@
 import com.palantir.atlasdb.spi.AtlasDbFactory;
 import com.palantir.atlasdb.spi.KeyValueServiceConfig;
 import com.palantir.atlasdb.versions.AtlasDbVersion;
-import com.palantir.timestamp.DebugLogger;
 import com.palantir.timestamp.PersistentTimestampService;
 import com.palantir.timestamp.TimestampService;
+import com.palantir.util.DebugLogger;
 
 @AutoService(AtlasDbFactory.class)
 public class CassandraAtlasDbFactory implements AtlasDbFactory {

atlasdb-cassandra/src/main/java/com/palantir/atlasdb/cassandra/CassandraKeyValueServiceConfig.java

@@ -44,7 +44,7 @@ public abstract class CassandraKeyValueServiceConfig implements KeyValueServiceC
 
  @Value.Default
  public int poolSize() {
- return 20;
+ return 30;
  }
 
  /**
@@ -54,7 +54,7 @@ public int poolSize() {
  */
  @Value.Default
  public int maxConnectionBurstSize() {
- return 5 * poolSize();
+ return 100;
  }
 
  /**
@@ -208,7 +208,7 @@ protected final void check() {
  }
  Preconditions.checkNotNull(keyspace(), "'keyspace' must be specified");
  double evictionCheckProportion = proportionConnectionsToCheckPerEvictionRun();
- Preconditions.checkArgument(evictionCheckProportion > 0 && evictionCheckProportion <= 1,
- "'proportionConnectionsToCheckPerEvictionRun' must be between 0 and 1");
+ Preconditions.checkArgument(evictionCheckProportion > 0.01 && evictionCheckProportion <= 1,
+ "'proportionConnectionsToCheckPerEvictionRun' must be between 0.01 and 1");
  }
 }

atlasdb-cassandra/src/main/java/com/palantir/atlasdb/keyvalue/cassandra/CassandraClientPool.java

@@ -518,7 +518,8 @@ public <V, K extends Exception> V runWithRetryOnHost(
  if (isRetriableWithBackoffException(e)) {
  log.warn("Retrying with backoff a query intended for host {}.", hostPool.getHost(), e);
  try {
- Thread.sleep(numTries * 1000);
+ // And value between -500 and +500ms to backoff to better spread load on failover
+ Thread.sleep(numTries * 1000 + (ThreadLocalRandom.current().nextInt(1000) - 500));
  } catch (InterruptedException i) {
  throw new RuntimeException(i);
  }
@@ -653,6 +654,8 @@ static boolean isRetriableWithBackoffException(Throwable ex) {
  && (ex instanceof NoSuchElementException
  // remote cassandra node couldn't talk to enough other remote cassandra nodes to answer
  || ex instanceof UnavailableException
+ // tcp socket timeout, possibly indicating network flake, long GC, or restarting server
+ || isConnectionException(ex)
  || isRetriableWithBackoffException(ex.getCause()));
  }