Skip to content

Commit

Permalink
[FLINK-10869] [tests] Update all S3 tests to use new test credentials
Browse files Browse the repository at this point in the history
  • Loading branch information
@StephanEwen
StephanEwen committed Nov 16, 2018
1 parent 5e6502d commit d617c05
Show file tree
Hide file tree
Showing 9 changed files with 76 additions and 235 deletions.
17 changes: 5 additions & 12 deletions ...s-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemBehaviorITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import org.apache.flink.core.fs.FileSystemBehaviorTestSuite;
import org.apache.flink.core.fs.FileSystemKind;
import org.apache.flink.core.fs.Path;
import org.apache.flink.testutils.s3.S3TestCredentials;

import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.BeforeClass;

import java.io.IOException;
Expand All @@ -36,24 +36,17 @@
*/
public class HadoopS3FileSystemBehaviorITCase extends FileSystemBehaviorTestSuite {

private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET");

private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID();

private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY");
private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY");

@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
// check whether credentials exist
Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null);
Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null);
Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null);
S3TestCredentials.assumeCredentialsAvailable();

// initialize configuration with valid credentials
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());
FileSystem.initialize(conf);
}

Expand All @@ -69,7 +62,7 @@ public FileSystem getFileSystem() throws Exception {

@Override
public Path getBasePath() throws Exception {
return new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
return new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR);
}

@Override
Expand Down
44 changes: 19 additions & 25 deletions ...ink-s3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3FileSystemITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.FileSystem.WriteMode;
import org.apache.flink.core.fs.Path;
import org.apache.flink.testutils.s3.S3TestCredentials;
import org.apache.flink.util.TestLogger;

import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand Down Expand Up @@ -69,10 +69,6 @@ public static List<String> parameters() {

private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID();

private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET");
private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY");
private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY");

/**
* Will be updated by {@link #checkCredentialsAndSetup()} if the test is not skipped.
*/
Expand All @@ -81,18 +77,16 @@ public static List<String> parameters() {
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
// check whether credentials exist
Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null);
Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null);
Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null);
S3TestCredentials.assumeCredentialsAvailable();

// initialize configuration with valid credentials
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());
FileSystem.initialize(conf);

// check for uniqueness of the test directory
final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
final Path directory = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR);
final FileSystem fs = directory.getFileSystem();

// directory must not yet exist
Expand All @@ -108,11 +102,11 @@ public static void cleanUp() throws IOException, InterruptedException {
final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs
// initialize configuration with valid credentials
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());
FileSystem.initialize(conf);

final Path directory = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
final Path directory = new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR);
final FileSystem fs = directory.getFileSystem();

// clean up
Expand All @@ -128,7 +122,7 @@ public static void cleanUp() throws IOException, InterruptedException {
}

private String getBasePath() {
return scheme + "://" + BUCKET + '/' + TEST_DATA_DIR + "/" + scheme;
return S3TestCredentials.getTestBucketUriWithScheme(scheme) + TEST_DATA_DIR + '/' + scheme;
}

@Test
Expand All @@ -138,8 +132,8 @@ public void testConfigKeysForwarding() throws Exception {
// standard Hadoop-style credential keys
{
Configuration conf = new Configuration();
conf.setString("fs.s3a.access.key", ACCESS_KEY);
conf.setString("fs.s3a.secret.key", SECRET_KEY);
conf.setString("fs.s3a.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("fs.s3a.secret.key", S3TestCredentials.getS3SecretKey());

FileSystem.initialize(conf);
path.getFileSystem();
Expand All @@ -148,8 +142,8 @@ public void testConfigKeysForwarding() throws Exception {
// shortened Hadoop-style credential keys
{
Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());

FileSystem.initialize(conf);
path.getFileSystem();
Expand All @@ -158,8 +152,8 @@ public void testConfigKeysForwarding() throws Exception {
// shortened Presto-style credential keys
{
Configuration conf = new Configuration();
conf.setString("s3.access-key", ACCESS_KEY);
conf.setString("s3.secret-key", SECRET_KEY);
conf.setString("s3.access-key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret-key", S3TestCredentials.getS3SecretKey());

FileSystem.initialize(conf);
path.getFileSystem();
Expand All @@ -170,8 +164,8 @@ public void testConfigKeysForwarding() throws Exception {
public void testSimpleFileWriteAndRead() throws Exception {
final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());

final String testLine = "Hello Upload!";

Expand Down Expand Up @@ -208,8 +202,8 @@ public void testSimpleFileWriteAndRead() throws Exception {
public void testDirectoryListing() throws Exception {
final long deadline = System.nanoTime() + 30_000_000_000L; // 30 secs
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());

FileSystem.initialize(conf);

Expand Down
20 changes: 7 additions & 13 deletions ...op/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterExceptionTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,12 @@
import org.apache.flink.core.fs.RecoverableFsDataOutputStream;
import org.apache.flink.core.fs.RecoverableWriter;
import org.apache.flink.fs.s3.common.FlinkS3FileSystem;
import org.apache.flink.testutils.s3.S3TestCredentials;
import org.apache.flink.util.StringUtils;
import org.apache.flink.util.TestLogger;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
Expand All @@ -55,20 +55,14 @@ public class HadoopS3RecoverableWriterExceptionTest extends TestLogger {

// ----------------------- S3 general configuration -----------------------

private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY");
private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY");
private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET");

private static final long PART_UPLOAD_MIN_SIZE_VALUE = 7L << 20;
private static final int MAX_CONCURRENT_UPLOADS_VALUE = 2;

// ----------------------- Test Specific configuration -----------------------

private static final Random RND = new Random();

private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID();

private static final Path basePath = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
private static Path basePath;

private static FlinkS3FileSystem fileSystem;

Expand All @@ -89,14 +83,14 @@ public class HadoopS3RecoverableWriterExceptionTest extends TestLogger {
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
// check whether credentials exist
Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null);
Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null);
Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null);
S3TestCredentials.assumeCredentialsAvailable();

basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID());

// initialize configuration with valid credentials
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());

conf.setLong(PART_UPLOAD_MIN_SIZE, PART_UPLOAD_MIN_SIZE_VALUE);
conf.setInteger(MAX_CONCURRENT_UPLOADS, MAX_CONCURRENT_UPLOADS_VALUE);
Expand Down
20 changes: 7 additions & 13 deletions ...3-fs-hadoop/src/test/java/org/apache/flink/fs/s3hadoop/HadoopS3RecoverableWriterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,14 @@
import org.apache.flink.core.fs.RecoverableWriter;
import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.fs.s3.common.FlinkS3FileSystem;
import org.apache.flink.testutils.s3.S3TestCredentials;
import org.apache.flink.util.MathUtils;
import org.apache.flink.util.StringUtils;
import org.apache.flink.util.TestLogger;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
Expand Down Expand Up @@ -63,20 +63,14 @@ public class HadoopS3RecoverableWriterTest extends TestLogger {

// ----------------------- S3 general configuration -----------------------

private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY");
private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY");
private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET");

private static final long PART_UPLOAD_MIN_SIZE_VALUE = 7L << 20;
private static final int MAX_CONCURRENT_UPLOADS_VALUE = 2;

// ----------------------- Test Specific configuration -----------------------

private static final Random RND = new Random();

private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID();

private static final Path basePath = new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
private static Path basePath;

private static FlinkS3FileSystem fileSystem;

Expand All @@ -101,14 +95,14 @@ public class HadoopS3RecoverableWriterTest extends TestLogger {
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
// check whether credentials exist
Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null);
Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null);
Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null);
S3TestCredentials.assumeCredentialsAvailable();

basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID());

// initialize configuration with valid credentials
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());

conf.setLong(PART_UPLOAD_MIN_SIZE, PART_UPLOAD_MIN_SIZE_VALUE);
conf.setInteger(MAX_CONCURRENT_UPLOADS, MAX_CONCURRENT_UPLOADS_VALUE);
Expand Down
17 changes: 5 additions & 12 deletions ...s-presto/src/test/java/org/apache/flink/fs/s3presto/PrestoS3FileSystemBehaviorITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import org.apache.flink.core.fs.FileSystemBehaviorTestSuite;
import org.apache.flink.core.fs.FileSystemKind;
import org.apache.flink.core.fs.Path;
import org.apache.flink.testutils.s3.S3TestCredentials;

import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.BeforeClass;

import java.io.IOException;
Expand All @@ -36,24 +36,17 @@
*/
public class PrestoS3FileSystemBehaviorITCase extends FileSystemBehaviorTestSuite {

private static final String BUCKET = System.getenv("ARTIFACTS_AWS_BUCKET");

private static final String TEST_DATA_DIR = "tests-" + UUID.randomUUID();

private static final String ACCESS_KEY = System.getenv("ARTIFACTS_AWS_ACCESS_KEY");
private static final String SECRET_KEY = System.getenv("ARTIFACTS_AWS_SECRET_KEY");

@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
// check whether credentials exist
Assume.assumeTrue("AWS S3 bucket not configured, skipping test...", BUCKET != null);
Assume.assumeTrue("AWS S3 access key not configured, skipping test...", ACCESS_KEY != null);
Assume.assumeTrue("AWS S3 secret key not configured, skipping test...", SECRET_KEY != null);
S3TestCredentials.assumeCredentialsAvailable();

// initialize configuration with valid credentials
final Configuration conf = new Configuration();
conf.setString("s3.access.key", ACCESS_KEY);
conf.setString("s3.secret.key", SECRET_KEY);
conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());
FileSystem.initialize(conf);
}

Expand All @@ -69,7 +62,7 @@ public FileSystem getFileSystem() throws Exception {

@Override
public Path getBasePath() throws Exception {
return new Path("s3://" + BUCKET + '/' + TEST_DATA_DIR);
return new Path(S3TestCredentials.getTestBucketUri() + TEST_DATA_DIR);
}

@Override
Expand Down
Loading

0 comments on commit d617c05

Please sign in to comment.