storage: import storage-service v1.76

build.gradle

@@ -62,6 +62,9 @@ subprojects {
         liquibaseVersion = '3.10.3'
         curve25519Vesrion = '0.5.0'
         zkgroupVersion = '0.8.2'
+        syslog4jVersion = '0.9.30'
+        bitableVersion = '1.27.3'
+        bigtableEmulatorVersion = '0.138.4'
     }
 
     tasks.withType(JavaCompile) {

settings.gradle

@@ -4,5 +4,6 @@ include(':websocket-resources')
 include(':redis-dispatch')
 include(':service')
 include(':gcm-sender-async')
+include(':storage-service')
 
 project(':service').name = 'whisper-service'

storage-service/build.gradle

@@ -0,0 +1,34 @@
+plugins {
+    id 'application'
+    id 'com.google.protobuf'
+    id 'com.google.cloud.tools.jib'
+}
+
+application {
+    getMainClass().set 'org.signal.storageservice.StorageService'
+}
+
+dependencies {
+    implementation "io.dropwizard:dropwizard-core:${dropwizardVersion}"
+    implementation "io.dropwizard:dropwizard-auth:${dropwizardVersion}"
+    implementation "com.google.cloud:google-cloud-bigtable:${bitableVersion}"
+    implementation "org.syslog4j:syslog4j:${syslog4jVersion}"
+    implementation "org.signal:zkgroup-java:${zkgroupVersion}"
+    implementation "net.logstash.logback:logstash-logback-encoder:${logstashLogbackVersion}"
+    implementation "org.coursera:dropwizard-metrics-datadog:${dropwizardMetricsDatadogVersion}"
+    runtimeOnly "org.glassfish.jaxb:jaxb-runtime:${jaxbVersion}"
+    testImplementation "io.dropwizard:dropwizard-testing:${dropwizardVersion}"
+    testImplementation "org.mockito:mockito-core:${mockitoVersion}"
+    testImplementation "com.google.cloud:google-cloud-bigtable-emulator:${bigtableEmulatorVersion}"
+    testImplementation "org.assertj:assertj-core:${assertjVersion}"
+    testImplementation "org.junit.jupiter:junit-jupiter:${junitJupiterVersion}"
+}
+
+jib {
+    from {
+        image = 'eclipse-temurin:11-jre@sha256:fec2ed05a90d99ad0c8af17438d0db38765a278e0190e8af82c7c7f7a5c84ce2'
+    }
+    to {
+        image = 'sweetlies-storage-service:latest'
+    }
+}

storage-service/src/main/java/org/signal/storageservice/StorageService.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2020-2021 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.storageservice;
+
+import static com.codahale.metrics.MetricRegistry.name;
+
+import com.codahale.metrics.SharedMetricRegistries;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.PropertyAccessor;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
+import com.google.cloud.bigtable.data.v2.BigtableDataClient;
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import io.dropwizard.Application;
+import io.dropwizard.auth.AuthFilter;
+import io.dropwizard.auth.PolymorphicAuthDynamicFeature;
+import io.dropwizard.auth.PolymorphicAuthValueFactoryProvider;
+import io.dropwizard.auth.basic.BasicCredentialAuthFilter;
+import io.dropwizard.auth.basic.BasicCredentials;
+import io.dropwizard.setup.Bootstrap;
+import io.dropwizard.setup.Environment;
+import java.time.Clock;
+import java.util.List;
+import org.signal.storageservice.auth.ExternalGroupCredentialGenerator;
+import org.signal.storageservice.auth.ExternalServiceCredentialValidator;
+import org.signal.storageservice.auth.GroupUser;
+import org.signal.storageservice.auth.GroupUserAuthenticator;
+import org.signal.storageservice.auth.User;
+import org.signal.storageservice.auth.UserAuthenticator;
+import org.signal.storageservice.controllers.BackupsController;
+import org.signal.storageservice.controllers.GroupsController;
+import org.signal.storageservice.controllers.HealthCheckController;
+import org.signal.storageservice.controllers.StorageController;
+import org.signal.storageservice.metrics.CpuUsageGauge;
+import org.signal.storageservice.metrics.FileDescriptorGauge;
+import org.signal.storageservice.metrics.FreeMemoryGauge;
+import org.signal.storageservice.metrics.NetworkReceivedGauge;
+import org.signal.storageservice.metrics.NetworkSentGauge;
+import org.signal.storageservice.metrics.StorageMetrics;
+import org.signal.storageservice.providers.CompletionExceptionMapper;
+import org.signal.storageservice.providers.InvalidProtocolBufferExceptionMapper;
+import org.signal.storageservice.providers.ProtocolBufferMessageBodyProvider;
+import org.signal.storageservice.providers.ProtocolBufferValidationErrorMessageBodyWriter;
+import org.signal.storageservice.s3.PolicySigner;
+import org.signal.storageservice.s3.PostPolicyGenerator;
+import org.signal.storageservice.storage.BackupsManager;
+import org.signal.storageservice.storage.GroupsManager;
+import org.signal.storageservice.storage.StorageManager;
+import org.signal.storageservice.util.UncaughtExceptionHandler;
+import org.signal.zkgroup.ServerSecretParams;
+import org.signal.zkgroup.auth.ServerZkAuthOperations;
+
+public class StorageService extends Application<StorageServiceConfiguration> {
+
+  @Override
+  public void initialize(Bootstrap<StorageServiceConfiguration> bootstrap) { }
+
+  @Override
+  public void run(StorageServiceConfiguration config, Environment environment) throws Exception {
+    SharedMetricRegistries.add(StorageMetrics.NAME, environment.metrics());
+
+    UncaughtExceptionHandler.register();
+
+    BigtableTableAdminSettings bigtableTableAdminSettings = BigtableTableAdminSettings.newBuilder()
+                                                                                      .setProjectId(config.getBigTableConfiguration().getProjectId())
+                                                                                      .setInstanceId(config.getBigTableConfiguration().getInstanceId())
+                                                                                      .build();
+    BigtableTableAdminClient bigtableTableAdminClient = BigtableTableAdminClient.create(bigtableTableAdminSettings);
+
+    BigtableDataSettings bigtableDataSettings = BigtableDataSettings.newBuilder()
+                                                                    .setProjectId(config.getBigTableConfiguration().getProjectId())
+                                                                    .setInstanceId(config.getBigTableConfiguration().getInstanceId())
+                                                                    .build();
+    BigtableDataClient bigtableDataClient = BigtableDataClient.create(bigtableDataSettings);
+    ServerSecretParams serverSecretParams = new ServerSecretParams(config.getZkConfiguration().getServerSecret());
+    StorageManager     storageManager     = new StorageManager(bigtableDataClient, config.getBigTableConfiguration().getContactManifestsTableId(), config.getBigTableConfiguration().getContactsTableId());
+    GroupsManager      groupsManager      = new GroupsManager(bigtableDataClient, config.getBigTableConfiguration().getGroupsTableId(), config.getBigTableConfiguration().getGroupLogsTableId());
+    BackupsManager backupsManager = new BackupsManager(bigtableTableAdminClient, config.getBigTableConfiguration().getClusterId(), List.of(
+            config.getBigTableConfiguration().getContactManifestsTableId(),
+            config.getBigTableConfiguration().getContactsTableId(),
+            config.getBigTableConfiguration().getGroupLogsTableId(),
+            config.getBigTableConfiguration().getGroupsTableId()));
+
+    environment.getObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    environment.getObjectMapper().setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.NONE);
+    environment.getObjectMapper().setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
+
+    environment.jersey().register(ProtocolBufferMessageBodyProvider.class);
+    environment.jersey().register(ProtocolBufferValidationErrorMessageBodyWriter.class);
+    environment.jersey().register(InvalidProtocolBufferExceptionMapper.class);
+    environment.jersey().register(CompletionExceptionMapper.class);
+
+    UserAuthenticator      userAuthenticator      = new UserAuthenticator(new ExternalServiceCredentialValidator(config.getAuthenticationConfiguration().getKey()));
+    GroupUserAuthenticator groupUserAuthenticator = new GroupUserAuthenticator(new ServerZkAuthOperations(serverSecretParams));
+    ExternalGroupCredentialGenerator externalGroupCredentialGenerator = new ExternalGroupCredentialGenerator(
+        config.getGroupConfiguration().getExternalServiceSecret(), Clock.systemUTC());
+
+    AuthFilter<BasicCredentials, User>      userAuthFilter      = new BasicCredentialAuthFilter.Builder<User>().setAuthenticator(userAuthenticator).buildAuthFilter();
+    AuthFilter<BasicCredentials, GroupUser> groupUserAuthFilter = new BasicCredentialAuthFilter.Builder<GroupUser>().setAuthenticator(groupUserAuthenticator).buildAuthFilter();
+
+    PolicySigner        policySigner        = new PolicySigner(config.getCdnConfiguration().getAccessSecret(), config.getCdnConfiguration().getRegion());
+    PostPolicyGenerator postPolicyGenerator = new PostPolicyGenerator(config.getCdnConfiguration().getRegion(), config.getCdnConfiguration().getBucket(), config.getCdnConfiguration().getAccessKey());
+
+    environment.jersey().register(new PolymorphicAuthDynamicFeature<>(ImmutableMap.of(User.class, userAuthFilter, GroupUser.class, groupUserAuthFilter)));
+    environment.jersey().register(new PolymorphicAuthValueFactoryProvider.Binder<>(ImmutableSet.of(User.class, GroupUser.class)));
+
+    environment.jersey().register(new HealthCheckController());
+    environment.jersey().register(new BackupsController(backupsManager));
+    environment.jersey().register(new StorageController(storageManager));
+    environment.jersey().register(new GroupsController(groupsManager, serverSecretParams, policySigner, postPolicyGenerator, config.getGroupConfiguration(), externalGroupCredentialGenerator));
+
+    environment.metrics().register(name(CpuUsageGauge.class, "cpu"), new CpuUsageGauge());
+    environment.metrics().register(name(FreeMemoryGauge.class, "free_memory"), new FreeMemoryGauge());
+    environment.metrics().register(name(NetworkSentGauge.class, "bytes_sent"), new NetworkSentGauge());
+    environment.metrics().register(name(NetworkReceivedGauge.class, "bytes_received"), new NetworkReceivedGauge());
+    environment.metrics().register(name(FileDescriptorGauge.class, "fd_count"), new FileDescriptorGauge());
+  }
+
+  public static void main(String[] argv) throws Exception {
+    new StorageService().run(argv);
+  }
+}

storage-service/src/main/java/org/signal/storageservice/StorageServiceConfiguration.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2020 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.storageservice;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.dropwizard.Configuration;
+import org.signal.storageservice.configuration.AuthenticationConfiguration;
+import org.signal.storageservice.configuration.BigTableConfiguration;
+import org.signal.storageservice.configuration.CdnConfiguration;
+import org.signal.storageservice.configuration.GroupConfiguration;
+import org.signal.storageservice.configuration.ZkConfiguration;
+
+import javax.validation.Valid;
+import javax.validation.constraints.NotNull;
+
+public class StorageServiceConfiguration extends Configuration {
+
+  @JsonProperty
+  @Valid
+  @NotNull
+  private BigTableConfiguration bigtable;
+
+  @JsonProperty
+  @Valid
+  @NotNull
+  private AuthenticationConfiguration authentication;
+
+  @JsonProperty
+  @Valid
+  @NotNull
+  private ZkConfiguration zkConfig;
+
+  @JsonProperty
+  @Valid
+  @NotNull
+  private CdnConfiguration cdn;
+
+  @JsonProperty
+  @Valid
+  @NotNull
+  private GroupConfiguration group;
+
+  public BigTableConfiguration getBigTableConfiguration() {
+    return bigtable;
+  }
+
+  public AuthenticationConfiguration getAuthenticationConfiguration() {
+    return authentication;
+  }
+
+  public ZkConfiguration getZkConfiguration() {
+    return zkConfig;
+  }
+
+  public CdnConfiguration getCdnConfiguration() {
+    return cdn;
+  }
+
+  public GroupConfiguration getGroupConfiguration() {
+    return group;
+  }
+}

storage-service/src/main/java/org/signal/storageservice/auth/ExternalGroupCredentialGenerator.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2020 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.signal.storageservice.auth;
+
+import com.google.protobuf.ByteString;
+import org.apache.commons.codec.binary.Hex;
+import org.signal.storageservice.util.Util;
+
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+import java.nio.charset.StandardCharsets;
+import java.security.InvalidKeyException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Clock;
+
+public class ExternalGroupCredentialGenerator {
+
+  private final byte[] key;
+  private final Clock clock;
+
+  public ExternalGroupCredentialGenerator(byte[] key, Clock clock) {
+    this.key = key;
+    this.clock = clock;
+  }
+
+  public String generateFor(ByteString uuidCiphertext, ByteString groupId, boolean isAllowedToInitiateGroupCall) {
+    final MessageDigest digest = getDigestInstance();
+    final long currentTimeSeconds = clock.millis() / 1000;
+    String encodedData =
+        "2:"
+        + Hex.encodeHexString(digest.digest(uuidCiphertext.toByteArray())) + ":"
+        + Hex.encodeHexString(groupId.toByteArray()) + ":"
+        + currentTimeSeconds + ":"
+        + (isAllowedToInitiateGroupCall ? "1" : "0");
+    String truncatedHmac = Hex.encodeHexString(
+        Util.truncate(getHmac(key, encodedData.getBytes(StandardCharsets.UTF_8)), 10));
+
+    return encodedData + ":" + truncatedHmac;
+  }
+
+  private Mac getMacInstance() {
+    try {
+      return Mac.getInstance("HmacSHA256");
+    } catch (NoSuchAlgorithmException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  private MessageDigest getDigestInstance() {
+    try {
+      return MessageDigest.getInstance("SHA-256");
+    } catch (NoSuchAlgorithmException e) {
+      throw new AssertionError(e);
+    }
+  }
+
+  private byte[] getHmac(byte[] key, byte[] input) {
+    try {
+      final Mac mac = getMacInstance();
+      mac.init(new SecretKeySpec(key, "HmacSHA256"));
+      return mac.doFinal(input);
+    } catch (InvalidKeyException e) {
+      throw new AssertionError(e);
+    }
+  }
+}