Make s3 work better (apache#3898)

Igosuki · Feb 2, 2017 · a73f1c9 · a73f1c9
1 parent e6b95e8
commit a73f1c9
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 21 deletions.
diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java
@@ -27,7 +27,6 @@
 import com.google.inject.Provides;
 import com.google.inject.multibindings.MapBinder;
 import io.druid.common.aws.AWSCredentialsConfig;
-import io.druid.common.aws.AWSCredentialsUtils;
 import io.druid.data.SearchableVersionedDataFinder;
 import io.druid.guice.Binders;
 import io.druid.guice.JsonConfigProvider;
@@ -43,6 +42,7 @@
 public class S3StorageDruidModule implements DruidModule
 {
   public static final String SCHEME = "s3_zip";
+
   @Override
   public List<? extends Module> getJacksonModules()
   {
@@ -85,7 +85,10 @@ public void configure(Binder binder)
     Binders.dataSegmentPullerBinder(binder).addBinding(SCHEME).to(S3DataSegmentPuller.class).in(LazySingleton.class);
     Binders.dataSegmentKillerBinder(binder).addBinding(SCHEME).to(S3DataSegmentKiller.class).in(LazySingleton.class);
     Binders.dataSegmentMoverBinder(binder).addBinding(SCHEME).to(S3DataSegmentMover.class).in(LazySingleton.class);
-    Binders.dataSegmentArchiverBinder(binder).addBinding(SCHEME).to(S3DataSegmentArchiver.class).in(LazySingleton.class);
+    Binders.dataSegmentArchiverBinder(binder)
+           .addBinding(SCHEME)
+           .to(S3DataSegmentArchiver.class)
+           .in(LazySingleton.class);
     Binders.dataSegmentPusherBinder(binder).addBinding("s3").to(S3DataSegmentPusher.class).in(LazySingleton.class);
     Binders.dataSegmentFinderBinder(binder).addBinding("s3").to(S3DataSegmentFinder.class).in(LazySingleton.class);
     JsonConfigProvider.bind(binder, "druid.storage", S3DataSegmentPusherConfig.class);
@@ -96,18 +99,11 @@ public void configure(Binder binder)
     binder.bind(S3TaskLogs.class).in(LazySingleton.class);
   }
 
-  @Provides
-  @LazySingleton
-  public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config)
-  {
-    return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config);
-  }
-
   @Provides
   @LazySingleton
   public RestS3Service getRestS3Service(AWSCredentialsProvider provider)
   {
-    if(provider.getCredentials() instanceof com.amazonaws.auth.AWSSessionCredentials) {
+    if (provider.getCredentials() instanceof com.amazonaws.auth.AWSSessionCredentials) {
       return new RestS3Service(new AWSSessionCredentialsAdapter(provider));
     } else {
       return new RestS3Service(new AWSCredentials(

diff --git a/...ions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java b/...ions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java
@@ -23,6 +23,7 @@
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSSessionCredentials;
 import io.druid.common.aws.AWSCredentialsConfig;
+import io.druid.guice.AWSModule;
 import org.easymock.EasyMock;
 import org.junit.Rule;
 import org.junit.Test;
@@ -35,32 +36,34 @@
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
-public class TestAWSCredentialsProvider {
-  @Test
-  public void testWithFixedAWSKeys() {
-    S3StorageDruidModule module = new S3StorageDruidModule();
+public class TestAWSCredentialsProvider
+{
+  private final AWSModule awsModule = new AWSModule();
+  private final S3StorageDruidModule s3Module = new S3StorageDruidModule();
 
+  @Test
+  public void testWithFixedAWSKeys()
+  {
     AWSCredentialsConfig config = EasyMock.createMock(AWSCredentialsConfig.class);
     EasyMock.expect(config.getAccessKey()).andReturn("accessKeySample").atLeastOnce();
     EasyMock.expect(config.getSecretKey()).andReturn("secretKeySample").atLeastOnce();
     EasyMock.replay(config);
 
-    AWSCredentialsProvider provider = module.getAWSCredentialsProvider(config);
+    AWSCredentialsProvider provider = awsModule.getAWSCredentialsProvider(config);
     AWSCredentials credentials = provider.getCredentials();
     assertEquals(credentials.getAWSAccessKeyId(), "accessKeySample");
     assertEquals(credentials.getAWSSecretKey(), "secretKeySample");
 
     // try to create
-    module.getRestS3Service(provider);
+    s3Module.getRestS3Service(provider);
   }
 
   @Rule
   public TemporaryFolder folder = new TemporaryFolder();
 
   @Test
-  public void testWithFileSessionCredentials() throws IOException {
-    S3StorageDruidModule module = new S3StorageDruidModule();
-
+  public void testWithFileSessionCredentials() throws IOException
+  {
     AWSCredentialsConfig config = EasyMock.createMock(AWSCredentialsConfig.class);
     EasyMock.expect(config.getAccessKey()).andReturn("");
     EasyMock.expect(config.getSecretKey()).andReturn("");
@@ -71,7 +74,7 @@ public void testWithFileSessionCredentials() throws IOException {
     EasyMock.expect(config.getFileSessionCredentials()).andReturn(file.getAbsolutePath()).atLeastOnce();
     EasyMock.replay(config);
 
-    AWSCredentialsProvider provider = module.getAWSCredentialsProvider(config);
+    AWSCredentialsProvider provider = awsModule.getAWSCredentialsProvider(config);
     AWSCredentials credentials = provider.getCredentials();
     assertTrue(credentials instanceof AWSSessionCredentials);
     AWSSessionCredentials sessionCredentials = (AWSSessionCredentials) credentials;
@@ -80,6 +83,6 @@ public void testWithFileSessionCredentials() throws IOException {
     assertEquals(sessionCredentials.getSessionToken(), "sessionTokenSample");
 
     // try to create
-    module.getRestS3Service(provider);
+    s3Module.getRestS3Service(provider);
   }
 }