Merge pull request localstack#22 from atlassian/feat/junit-test

Add JUnit integration + example test class

.travis.yml

@@ -11,6 +11,11 @@ addons:
   apt:
     packages:
     - oracle-java8-installer
+    - oracle-java8-set-default
+
+env:
+  global:
+    - JAVA_HOME=/usr/lib/jvm/java-8-oracle
 
 install:
   - travis_wait 20 make install

Makefile

@@ -28,6 +28,7 @@ compile:           ## Compile Java code (KCL library utils)
 	echo "Compiling"
 	$(VENV_RUN); python -c 'from localstack.utils.kinesis import kclipy_helper; print kclipy_helper.get_kcl_classpath()'
 	javac -cp $(shell $(VENV_RUN); python -c 'from localstack.utils.kinesis import kclipy_helper; print kclipy_helper.get_kcl_classpath()') localstack/utils/kinesis/java/com/atlassian/*.java
+	(test ! -e ext/java || cd ext/java && mvn -DskipTests package)
 	# TODO enable once we want to support Java-based Lambdas
 	# (cd localstack/mock && mvn package)
 
@@ -39,7 +40,7 @@ coveralls:         ## Publish coveralls metrics
 	($(VENV_RUN); coveralls)
 
 infra:             ## Manually start the local infrastructure for testing
-	($(VENV_RUN); localstack/mock/infra.py)
+	$(VENV_RUN); exec localstack/mock/infra.py
 
 web:               ## Start web application (dashboard)
 	($(VENV_RUN); bin/localstack web --port=8081)

README.md

@@ -68,9 +68,17 @@ missing functionality on top of them:
 * `npm` (node.js package manager)
 * `java`/`javac` (Java runtime environment and compiler)
 
-## Installation
+## Installing
 
-To install the tool and all its dependencies, run the following command:
+The easiest way to install *LocalStack* is via `pip`:
+
+```
+pip install localstack
+```
+
+## Developing
+
+If you pull the repo in order to extend/modify LocalStack, run this command to install all dependencies:
 
 ```
 make install
@@ -131,6 +139,27 @@ def my_app_test():
 
 See the example test file `tests/test_integration.py` for more details.
 
+## Integration with Java/JUnit
+
+In order to use *LocalStack* with Java, the project ships with a simple JUnit runner. Take a look
+at the example JUnit test in `ext/java`. When you run the test, all dependencies are automatically
+downloaded and installed to a temporary directory in your system.
+
+```
+@RunWith(LocalstackTestRunner.class)
+public class MyCloudAppTest {
+
+  @Test
+  public void testLocalS3API() {
+    AmazonS3 s3 = new AmazonS3Client(...);
+    s3.setEndpoint(LocalstackTestRunner.getEndpointS3());
+    List<Bucket> buckets = s3.listBuckets();
+    ...
+  }
+
+}
+```
+
 ## Web Dashboard
 
 The projects also comes with a simple Web dashboard that allows to view the
@@ -143,6 +172,8 @@ make web
 
 ## Change Log
 
+* v0.3.0: Add simple integration for JUnit; improve process signal handling
+* v0.2.11: Refactored the AWS assume role function
 * v0.2.10: Added AWS assume role functionality.
 * v0.2.9: Kinesis error response formatting
 * v0.2.7: Throw Kinesis errors randomly

ext/java/.gitignore

@@ -0,0 +1 @@
+/target/

ext/java/pom.xml

@@ -0,0 +1,55 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<groupId>com.atlassian</groupId>
+	<artifactId>localstack-utils</artifactId>
+	<packaging>jar</packaging>
+	<version>1.0-SNAPSHOT</version>
+	<name>localstack-utils</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>com.amazonaws</groupId>
+			<artifactId>aws-lambda-java-core</artifactId>
+			<version>1.1.0</version>
+		</dependency>
+		<dependency>
+			<groupId>com.amazonaws</groupId>
+			<artifactId>aws-lambda-java-events</artifactId>
+			<version>1.3.0</version>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>4.12</version>
+		</dependency>
+		<dependency>
+			<groupId>com.amazonaws</groupId>
+			<artifactId>aws-java-sdk</artifactId>
+			<version>1.11.86</version>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<version>2.3</version>
+				<configuration>
+					<createDependencyReducedPom>false</createDependencyReducedPom>
+				</configuration>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+</project>

localstack/mock/src/main/java/com/atlassian/LambdaContext.java → ext/java/src/main/java/com/atlassian/localstack/LambdaContext.java

@@ -1,4 +1,4 @@
-package com.atlassian;
+package com.atlassian.localstack;
 
 import java.util.logging.Level;
 import java.util.logging.Logger;

localstack/mock/src/main/java/com/atlassian/LambdaExecutor.java → ext/java/src/main/java/com/atlassian/localstack/LambdaExecutor.java

@@ -1,4 +1,4 @@
-package com.atlassian;
+package com.atlassian.localstack;
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
@@ -21,6 +21,11 @@
 import com.amazonaws.services.lambda.runtime.events.KinesisEvent.Record;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+/**
+ * TODO: Support for AWS Lambda functions written in Java is work in progress.
+ *
+ * @author Waldemar Hummer
+ */
 public class LambdaExecutor {
 
 	@SuppressWarnings("unchecked")
@@ -32,8 +37,8 @@ public static void main(String[] args) throws Exception {
 			if(test) {
 				final String testFile = "/tmp/test.event.kinesis.json";
 				String content = "{\"records\": ["
-						+ "{\"kinesis\": " +
-						+ "{}" +
+						+ "{\"kinesis\": "
+						+ "{}"
 						+ "}"
 						+ "]}";
 				writeFile(testFile, content);
@@ -53,6 +58,7 @@ public void run() {
 		KinesisEvent event = new KinesisEvent();
 		ObjectMapper reader = new ObjectMapper();
 		String fileContent = readFile(args[1]);
+		@SuppressWarnings("deprecation")
 		Map<String,Object> map = reader.reader(Map.class).readValue(fileContent);
 		List<Map<String,Object>> records = (List<Map<String, Object>>) get(map, "Records");
 		event.setRecords(new LinkedList<KinesisEvent.KinesisEventRecord>());

ext/java/src/main/java/com/atlassian/localstack/LocalstackTestRunner.java

@@ -0,0 +1,190 @@
+package com.atlassian.localstack;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.InitializationError;
+
+import com.amazonaws.util.IOUtils;
+
+/**
+ * Simple JUnit test runner that automatically downloads, installs, starts,
+ * and stops the LocalStack local cloud infrastructure components.
+ *
+ * Should work cross-OS, however has been only tested under Unix (Linux/MacOS).
+ *
+ * @author Waldemar Hummer
+ */
+public class LocalstackTestRunner extends BlockJUnit4ClassRunner {
+
+	private static final AtomicReference<Process> INFRA_STARTED = new AtomicReference<Process>();
+	private static String CONFIG_FILE_CONTENT = "";
+
+	private static final String INFRA_READY_MARKER = "Ready.";
+	private static final String TMP_INSTALL_DIR = System.getProperty("java.io.tmpdir") +
+			File.separator + "localstack_install_dir";
+	private static final String ADDITIONAL_PATH = "/usr/local/bin/";
+	private static final String LOCALHOST = "localhost";
+	private static final String LOCALSTACK_REPO_URL = "https://github.com/atlassian/localstack";
+
+	private static final Logger LOG = Logger.getLogger(LocalstackTestRunner.class.getName());
+
+	public LocalstackTestRunner(Class<?> klass) throws InitializationError {
+		super(klass);
+	}
+
+	/* SERVICE ENDPOINTS */
+
+	public static String getEndpointS3() {
+		ensureInstallation();
+		return getEndpoint("s3");
+	}
+
+	public static String getEndpointKinesis() {
+		ensureInstallation();
+		return getEndpoint("kinesis");
+	}
+
+	public static String getEndpointLambda() {
+		ensureInstallation();
+		return getEndpoint("lambda");
+	}
+
+	public static String getEndpointDynamoDB() {
+		ensureInstallation();
+		return getEndpoint("dynamodb");
+	}
+
+	public static String getEndpointDynamoDBStreams() {
+		ensureInstallation();
+		return getEndpoint("dynamodbstreams");
+	}
+
+	public static String getEndpointAPIGateway() {
+		ensureInstallation();
+		return getEndpoint("apigateway");
+	}
+
+	public static String getEndpointElasticsearch() {
+		ensureInstallation();
+		return getEndpoint("elasticsearch");
+	}
+
+	public static String getEndpointFirehose() {
+		ensureInstallation();
+		return getEndpoint("firehose");
+	}
+
+	public static String getEndpointSNS() {
+		ensureInstallation();
+		return getEndpoint("sns");
+	}
+
+	public static String getEndpointSQS() {
+		ensureInstallation();
+		return getEndpoint("sns");
+	}
+
+	/* UTILITY METHODS */
+
+	@Override
+	public void run(RunNotifier notifier) {
+		setupInfrastructure();
+		super.run(notifier);
+	}
+
+	private static void ensureInstallation() {
+		File dir = new File(TMP_INSTALL_DIR);
+		if(!dir.exists()) {
+			LOG.info("Installing LocalStack to temporary directory (this might take a while): " + TMP_INSTALL_DIR);
+			exec("git clone " + LOCALSTACK_REPO_URL + " " + TMP_INSTALL_DIR);
+			exec("cd " + TMP_INSTALL_DIR + "; make install");
+		}
+	}
+
+	private static void killProcess(Process p) {
+		p.destroy();
+		p.destroyForcibly();
+	}
+
+	private static String getEndpoint(String service) {
+		ensureInstallation();
+		String regex = ".*DEFAULT_PORT_" + service.toUpperCase() + "\\s*=\\s*([0-9]+).*";
+		String port = Pattern.compile(regex, Pattern.DOTALL | Pattern.MULTILINE).matcher(CONFIG_FILE_CONTENT).replaceAll("$1");
+		return "http://" + LOCALHOST + ":" + port + "/";
+	}
+
+	private static Process exec(String cmd) {
+		return exec(cmd, true);
+	}
+
+	private static Process exec(String cmd, boolean wait) {
+		try {
+			Map<String, String> env = System.getenv();
+			final Process p = Runtime.getRuntime().exec(new String[]{"/bin/bash", "-c", cmd},
+					new String[]{"PATH=" + ADDITIONAL_PATH + ":" + env.get("PATH")});
+			if (wait) {
+				int code = p.waitFor();
+				if(code != 0) {
+					String stderr = IOUtils.toString(p.getErrorStream());
+					String stdout = IOUtils.toString(p.getInputStream());
+					throw new IllegalStateException("Failed to run command '" + cmd + "', return code " + code +
+							".\nSTDOUT: " + stdout + "\nSTDERR: " + stderr);
+				}
+			} else {
+				/* make sure we destroy the process on JVM shutdown */
+				Runtime.getRuntime().addShutdownHook(new Thread() {
+					public void run() {
+						killProcess(p);
+					}
+				});
+			}
+			return p;
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	private void setupInfrastructure() {
+		synchronized (INFRA_STARTED) {
+			ensureInstallation();
+			if(INFRA_STARTED.get() != null) return;
+			String cmd = "cd " + TMP_INSTALL_DIR + "; exec make infra";
+			Process proc;
+			try {
+				proc = exec(cmd, false);
+				BufferedReader r1 = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+				String line;
+				LOG.info("Waiting for infrastructure to be spun up");
+				while((line = r1.readLine()) != null) {
+					if(INFRA_READY_MARKER.equals(line)) {
+						break;
+					}
+				}
+				/* read contents of LocalStack config file */
+				String configFile = TMP_INSTALL_DIR + File.separator + "localstack" +  File.separator + "constants.py";
+				CONFIG_FILE_CONTENT = IOUtils.toString(new FileInputStream(configFile));
+				INFRA_STARTED.set(proc);
+			} catch (IOException e) {
+				throw new RuntimeException(e);
+			}
+		}
+	}
+
+	public static void teardownInfrastructure() {
+		Process proc = INFRA_STARTED.get();
+		if(proc == null) {
+			return;
+		}
+		killProcess(proc);
+	}
+}