From a73f1c9c70bb11ec0b1297f9354f6450eefdfc0f Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Thu, 2 Feb 2017 10:04:30 -0800 Subject: [PATCH] Make s3 work better (#3898) --- .../storage/s3/S3StorageDruidModule.java | 16 +++++------- .../s3/TestAWSCredentialsProvider.java | 25 +++++++++++-------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java index 2d97f6ee2b90..301aae0494a8 100644 --- a/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java +++ b/extensions-core/s3-extensions/src/main/java/io/druid/storage/s3/S3StorageDruidModule.java @@ -27,7 +27,6 @@ import com.google.inject.Provides; import com.google.inject.multibindings.MapBinder; import io.druid.common.aws.AWSCredentialsConfig; -import io.druid.common.aws.AWSCredentialsUtils; import io.druid.data.SearchableVersionedDataFinder; import io.druid.guice.Binders; import io.druid.guice.JsonConfigProvider; @@ -43,6 +42,7 @@ public class S3StorageDruidModule implements DruidModule { public static final String SCHEME = "s3_zip"; + @Override public List getJacksonModules() { @@ -85,7 +85,10 @@ public void configure(Binder binder) Binders.dataSegmentPullerBinder(binder).addBinding(SCHEME).to(S3DataSegmentPuller.class).in(LazySingleton.class); Binders.dataSegmentKillerBinder(binder).addBinding(SCHEME).to(S3DataSegmentKiller.class).in(LazySingleton.class); Binders.dataSegmentMoverBinder(binder).addBinding(SCHEME).to(S3DataSegmentMover.class).in(LazySingleton.class); - Binders.dataSegmentArchiverBinder(binder).addBinding(SCHEME).to(S3DataSegmentArchiver.class).in(LazySingleton.class); + Binders.dataSegmentArchiverBinder(binder) + .addBinding(SCHEME) + .to(S3DataSegmentArchiver.class) + .in(LazySingleton.class); Binders.dataSegmentPusherBinder(binder).addBinding("s3").to(S3DataSegmentPusher.class).in(LazySingleton.class); Binders.dataSegmentFinderBinder(binder).addBinding("s3").to(S3DataSegmentFinder.class).in(LazySingleton.class); JsonConfigProvider.bind(binder, "druid.storage", S3DataSegmentPusherConfig.class); @@ -96,18 +99,11 @@ public void configure(Binder binder) binder.bind(S3TaskLogs.class).in(LazySingleton.class); } - @Provides - @LazySingleton - public AWSCredentialsProvider getAWSCredentialsProvider(final AWSCredentialsConfig config) - { - return AWSCredentialsUtils.defaultAWSCredentialsProviderChain(config); - } - @Provides @LazySingleton public RestS3Service getRestS3Service(AWSCredentialsProvider provider) { - if(provider.getCredentials() instanceof com.amazonaws.auth.AWSSessionCredentials) { + if (provider.getCredentials() instanceof com.amazonaws.auth.AWSSessionCredentials) { return new RestS3Service(new AWSSessionCredentialsAdapter(provider)); } else { return new RestS3Service(new AWSCredentials( diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java index 418e2f1acb4a..f38574fc82bf 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/TestAWSCredentialsProvider.java @@ -23,6 +23,7 @@ import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSSessionCredentials; import io.druid.common.aws.AWSCredentialsConfig; +import io.druid.guice.AWSModule; import org.easymock.EasyMock; import org.junit.Rule; import org.junit.Test; @@ -35,32 +36,34 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -public class TestAWSCredentialsProvider { - @Test - public void testWithFixedAWSKeys() { - S3StorageDruidModule module = new S3StorageDruidModule(); +public class TestAWSCredentialsProvider +{ + private final AWSModule awsModule = new AWSModule(); + private final S3StorageDruidModule s3Module = new S3StorageDruidModule(); + @Test + public void testWithFixedAWSKeys() + { AWSCredentialsConfig config = EasyMock.createMock(AWSCredentialsConfig.class); EasyMock.expect(config.getAccessKey()).andReturn("accessKeySample").atLeastOnce(); EasyMock.expect(config.getSecretKey()).andReturn("secretKeySample").atLeastOnce(); EasyMock.replay(config); - AWSCredentialsProvider provider = module.getAWSCredentialsProvider(config); + AWSCredentialsProvider provider = awsModule.getAWSCredentialsProvider(config); AWSCredentials credentials = provider.getCredentials(); assertEquals(credentials.getAWSAccessKeyId(), "accessKeySample"); assertEquals(credentials.getAWSSecretKey(), "secretKeySample"); // try to create - module.getRestS3Service(provider); + s3Module.getRestS3Service(provider); } @Rule public TemporaryFolder folder = new TemporaryFolder(); @Test - public void testWithFileSessionCredentials() throws IOException { - S3StorageDruidModule module = new S3StorageDruidModule(); - + public void testWithFileSessionCredentials() throws IOException + { AWSCredentialsConfig config = EasyMock.createMock(AWSCredentialsConfig.class); EasyMock.expect(config.getAccessKey()).andReturn(""); EasyMock.expect(config.getSecretKey()).andReturn(""); @@ -71,7 +74,7 @@ public void testWithFileSessionCredentials() throws IOException { EasyMock.expect(config.getFileSessionCredentials()).andReturn(file.getAbsolutePath()).atLeastOnce(); EasyMock.replay(config); - AWSCredentialsProvider provider = module.getAWSCredentialsProvider(config); + AWSCredentialsProvider provider = awsModule.getAWSCredentialsProvider(config); AWSCredentials credentials = provider.getCredentials(); assertTrue(credentials instanceof AWSSessionCredentials); AWSSessionCredentials sessionCredentials = (AWSSessionCredentials) credentials; @@ -80,6 +83,6 @@ public void testWithFileSessionCredentials() throws IOException { assertEquals(sessionCredentials.getSessionToken(), "sessionTokenSample"); // try to create - module.getRestS3Service(provider); + s3Module.getRestS3Service(provider); } }