Merge pull request #659 from xschildw/dev-plfm-7068

PLFM-7068: setup markdown-it lambda

src/main/java/org/sagebionetworks/template/Constants.java

@@ -67,7 +67,6 @@ public class Constants {
  public static final String PROPERTY_KEY_VPC_SUBNET_PREFIX = "org.sagebionetworks.vpc.subnet.prefix";
  public static final String PROPERTY_KEY_COLORS = "org.sagebionetworks.vpc.colors.csv";
  public static final String PROPERTY_KEY_VPC_PEERING_ACCEPT_ROLE_ARN = "org.sagebionetworks.vpc.peering.accept.role.arn";
- public static final String PROPERTY_KEY_OLD_VPC_ID = "org.sagebionetworks.vpc.old.vpc.id";
  public static final String PROPERTY_KEY_OLD_VPC_CIDR = "org.sagebionetworks.vpc.old.vpc.cidr";
  // repo
  public static final String PROPERTY_KEY_STACK = "org.sagebionetworks.stack";
@@ -120,6 +119,8 @@ public class Constants {
  public static final String PROPERTY_KEY_RDS_REPO_SNAPSHOT_IDENTIFIER = "org.sagebionetworks.repo.snapshot.identifier";
  public static final String PROPERTY_KEY_RDS_TABLES_SNAPSHOT_IDENTIFIERS = "org.sagebionetworks.tables.snapshot.identifiers";
  public static final String PROPERTY_KEY_LAMBDA_VIRUS_SCANNER_ARTIFACT_URL = "org.sagebionetworks.lambda.virusscanner.artifactUrl";
+ public static final String PROPERTY_KEY_LAMBDA_MARKDOWNIT_ARTIFACT_URL = "org.sagebionetworks.lambda.markdownit.artifactUrl";
+ public static final String PROPERTY_KEY_LAMBDA_ARTIFACT_BUCKET = "org.sagebionetworks.lambda.artifact.bucket";
  public static final String NOSNAPSHOT = "NOSNAPSHOT"; // value to indicate a snapshot is not used to init a stack
 
  public static final String PROPERTY_KEY_ENABLE_RDS_ENHANCED_MONITORING = "org.sagebionetworks.enable.rds.enhanced.monitoring";
@@ -168,6 +169,8 @@ public class Constants {
  public static final String TEMPLATE_DATAWAREHOUSE = "templates/datawarehouse/datawarehouse-template.json.vpt";
  public static final String TEMPLATE_S3_BUCKET_POLICY = "templates/s3/s3-bucket-policy.json.vpt";
 
+ public static final String TEMPLATE_MARKDOWNIT_API_VTP = "templates/markdownit/markdown-it-api.json.vtp";
+
  public static final int JSON_INDENT = 5;
 
  /*
@@ -274,6 +277,7 @@ public class Constants {
  public static final String MACHINE_TYPES = "machineTypes";
  public static final String POOL_TYPES = "poolTypes";
 
+
  /**
  * Create a camel case name from dash-separated-name. Given 'foo-bar' will
  * return 'FooBar'

src/main/java/org/sagebionetworks/template/TemplateGuiceModule.java

@@ -65,6 +65,8 @@
 import org.sagebionetworks.template.ip.address.IpAddressPoolBuilderImpl;
 import org.sagebionetworks.template.jobs.AsynchAdminJobExecutor;
 import org.sagebionetworks.template.jobs.AsynchAdminJobExecutorImpl;
+import org.sagebionetworks.template.markdownit.MarkDownItLambdaBuilder;
+import org.sagebionetworks.template.markdownit.MarkDownItLambdaBuilderImpl;
 import org.sagebionetworks.template.nlb.BindNetworkLoadBalancerBuilder;
 import org.sagebionetworks.template.nlb.BindNetworkLoadBalancerBuilderImpl;
 import org.sagebionetworks.template.nlb.NetworkLoadBalancerBuilder;
@@ -179,6 +181,7 @@ protected void configure() {
  bind(ExpiredStackTeardown.class).to(ExpiredStackTeardownImpl.class);
  bind(DataWarehouseBuilder.class).to(DataWarehouseBuilderImpl.class);
  bind(BackfillDataWarehouseBuilder.class).to(BackfillDataWarehouseBuilderImpl.class);
+ bind(MarkDownItLambdaBuilder.class).to(MarkDownItLambdaBuilderImpl.class);
 
  Multibinder<VelocityContextProvider> velocityContextProviderMultibinder = Multibinder.newSetBinder(binder(), VelocityContextProvider.class);
 
@@ -361,4 +364,5 @@ public S3TransferManagerFactory provideS3TransferManagerFactory(AmazonS3 s3Clien
  public DataWarehouseConfig dataWarehouseConfigProvider() throws IOException {
  return new DataWarehouseConfigValidator(loadFromJsonFile(DATAWAREHOUSE_CONFIG_FILE, DataWarehouseConfig.class)).validate();
  }
+
 }

src/main/java/org/sagebionetworks/template/markdownit/MarkDownItLambdaBuilder.java

@@ -0,0 +1,6 @@
+package org.sagebionetworks.template.markdownit;
+
+public interface MarkDownItLambdaBuilder {
+ public void buildMarkDownItLambda();
+
+}

src/main/java/org/sagebionetworks/template/markdownit/MarkDownItLambdaBuilderImpl.java

@@ -0,0 +1,112 @@
+package org.sagebionetworks.template.markdownit;
+
+import com.amazonaws.services.cloudformation.model.Stack;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.google.inject.Inject;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.velocity.Template;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.sagebionetworks.template.CloudFormationClient;
+import org.sagebionetworks.template.Constants;
+import org.sagebionetworks.template.CreateOrUpdateStackRequest;
+import org.sagebionetworks.template.StackTagsProvider;
+import org.sagebionetworks.template.config.RepoConfiguration;
+import org.sagebionetworks.template.utils.ArtifactDownload;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.util.Optional;
+
+import static org.sagebionetworks.template.Constants.CAPABILITY_NAMED_IAM;
+import static org.sagebionetworks.template.Constants.PROPERTY_KEY_LAMBDA_ARTIFACT_BUCKET;
+import static org.sagebionetworks.template.Constants.PROPERTY_KEY_LAMBDA_MARKDOWNIT_ARTIFACT_URL;
+import static org.sagebionetworks.template.Constants.PROPERTY_KEY_STACK;
+
+public class MarkDownItLambdaBuilderImpl implements MarkDownItLambdaBuilder {
+
+ private static final Logger LOGGER = LogManager.getLogger(MarkDownItLambdaBuilderImpl.class);
+
+ private RepoConfiguration config;
+
+ private ArtifactDownload downloader;
+
+ private CloudFormationClient cloudFormationClient;
+
+ private StackTagsProvider tagsProvider;
+
+ private AmazonS3 s3Client;
+
+ private VelocityEngine velocityEngine;
+
+ @Inject
+ public MarkDownItLambdaBuilderImpl(RepoConfiguration config,
+ ArtifactDownload downloader, CloudFormationClient cloudFormationClient,
+ StackTagsProvider tagsProvider, AmazonS3 s3Client,
+ VelocityEngine velocityEngine) {
+ this.config = config;
+ this.downloader = downloader;
+ this.cloudFormationClient = cloudFormationClient;
+ this.tagsProvider = tagsProvider;
+ this.s3Client = s3Client;
+ this.velocityEngine = velocityEngine;
+
+ }
+
+ @Override
+ public void buildMarkDownItLambda() {
+
+ String stack = config.getProperty(PROPERTY_KEY_STACK);
+ String artifactBucket = config.getProperty(PROPERTY_KEY_LAMBDA_ARTIFACT_BUCKET);
+ String lambdaSourceArtifactUrl = config.getProperty(PROPERTY_KEY_LAMBDA_MARKDOWNIT_ARTIFACT_URL);
+ String lambdaArtifactKey = String.format("artifacts/markdown-it/%s", FilenameUtils.getName(lambdaSourceArtifactUrl));
+
+ // Download from jfrog and upload to S3
+ File artifact = downloader.downloadFile(lambdaSourceArtifactUrl);
+ try {
+ s3Client.putObject(artifactBucket, lambdaArtifactKey, artifact);
+ } finally {
+ artifact.delete();
+ }
+
+ buildMarkDownItLambdaStack(stack, artifactBucket, lambdaArtifactKey);
+
+ }
+
+ private Optional<Stack> buildMarkDownItLambdaStack(String stack, String artifactBucket, String artifactKey) {
+
+ String stackName = String.format("%s-markdown-it-function", stack);
+
+ // Setup context
+ VelocityContext context = new VelocityContext();
+ context.put("lambdaArtifactBucket", artifactBucket);
+ context.put("lambdaArtifactKey", artifactKey);
+
+ // Generate template
+ Template template = velocityEngine.getTemplate(Constants.TEMPLATE_MARKDOWNIT_API_VTP);
+ StringWriter stringWriter = new StringWriter();
+ template.merge(context, stringWriter);
+ String resultJSON = stringWriter.toString();
+ LOGGER.info(resultJSON);
+
+ // Create stack
+ CreateOrUpdateStackRequest req = new CreateOrUpdateStackRequest()
+ .withStackName(stackName)
+ .withTemplateBody(resultJSON)
+ .withTags(tagsProvider.getStackTags())
+ .withCapabilities(CAPABILITY_NAMED_IAM);
+ cloudFormationClient.createOrUpdateStack(req);
+
+ try {
+ cloudFormationClient.waitForStackToComplete(stackName);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return Optional.of(cloudFormationClient.describeStack(stackName).orElseThrow(()->new IllegalStateException("Stack does not exist: "+stackName)));
+
+ }
+}

src/main/java/org/sagebionetworks/template/markdownit/MarkDownItLambdaBuilderMain.java

@@ -0,0 +1,15 @@
+package org.sagebionetworks.template.markdownit;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.sagebionetworks.template.TemplateGuiceModule;
+
+
+public class MarkDownItLambdaBuilderMain {
+
+ public static void main(String[] args) throws InterruptedException {
+ Injector injector = Guice.createInjector(new TemplateGuiceModule());
+ MarkDownItLambdaBuilder builder = injector.getInstance(MarkDownItLambdaBuilder.class);
+ builder.buildMarkDownItLambda();
+ }
+}

src/main/resources/templates/markdownit/markdown-it-api.json.vtp

@@ -0,0 +1,64 @@
+{
+ "Resources": {
+ "mdlambdaServiceRole": {
+ "Type": "AWS::IAM::Role",
+ "Properties": {
+ "AssumeRolePolicyDocument": {
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Effect": "Allow",
+ "Principal": {
+ "Service": "lambda.amazonaws.com"
+ }
+ }
+ ],
+ "Version": "2012-10-17"
+ },
+ "ManagedPolicyArns": [
+ #[[{ "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" }]]#
+ ]
+ }
+ },
+ "mdlambda": {
+ "Type": "AWS::Lambda::Function",
+ "Properties": {
+ "Code": {
+ "S3Bucket": "${lambdaArtifactBucket}",
+ "S3Key": "${lambdaArtifactKey}"
+ },
+ "Handler": "index.handler",
+ "Role": { "Fn::GetAtt": [ "mdlambdaServiceRole", "Arn" ] },
+ "Runtime": "nodejs20.x",
+ "Timeout": 10
+ }
+ },
+ "mdlambdaFunctionUrl": {
+ "Type": "AWS::Lambda::Url",
+ "Properties": {
+ "AuthType": "NONE",
+ "TargetFunctionArn": { "Ref": "mdlambda" },
+ "Cors": {
+ "AllowOrigins": ["*"],
+ "AllowMethods": ["GET", "POST"]
+ }
+ }
+ },
+ "mdlambdaFunctionUrlPermission": {
+ "Type": "AWS::Lambda::Permission",
+ "Properties": {
+ "Action": "lambda:InvokeFunctionUrl",
+ "FunctionName": { "Ref": "mdlambda" },
+ "Principal": "*",
+ "FunctionUrlAuthType": "NONE"
+ }
+ }
+ },
+ "Outputs": {
+ "LambdaFunctionUrl": {
+ "Value": { "Ref": "mdlambdaFunctionUrl" },
+ "Description": "The URL endpoint for the Lambda function"
+ }
+ }
+
+}

src/test/java/org/sagebionetworks/template/markdownit/MarkDownItLambdaBuilderImplTest.java

@@ -0,0 +1,136 @@
+package org.sagebionetworks.template.markdownit;
+
+import com.amazonaws.services.cloudformation.model.Output;
+import com.amazonaws.services.cloudformation.model.Stack;
+import com.amazonaws.services.s3.AmazonS3;
+import org.apache.velocity.Template;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.json.JSONObject;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.sagebionetworks.template.CloudFormationClient;
+import org.sagebionetworks.template.CreateOrUpdateStackRequest;
+import org.sagebionetworks.template.StackTagsProvider;
+import org.sagebionetworks.template.TemplateGuiceModule;
+import org.sagebionetworks.template.config.RepoConfiguration;
+import org.sagebionetworks.template.utils.ArtifactDownload;
+
+import java.io.File;
+import java.io.StringWriter;
+import java.util.Collections;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.sagebionetworks.template.Constants.CAPABILITY_NAMED_IAM;
+import static org.sagebionetworks.template.Constants.JSON_INDENT;
+import static org.sagebionetworks.template.Constants.PROPERTY_KEY_LAMBDA_ARTIFACT_BUCKET;
+import static org.sagebionetworks.template.Constants.PROPERTY_KEY_LAMBDA_MARKDOWNIT_ARTIFACT_URL;
+import static org.sagebionetworks.template.Constants.PROPERTY_KEY_STACK;
+
+@ExtendWith(MockitoExtension.class)
+public class MarkDownItLambdaBuilderImplTest {
+
+ @Mock
+ RepoConfiguration mockConfig;
+ @Mock
+ ArtifactDownload mockDownloader;
+
+ @Mock
+ CloudFormationClient mockCloudFormationClient;
+
+ @Mock
+ StackTagsProvider mockTagsProvider;
+
+ @Mock
+ AmazonS3 mockS3Client;
+
+ VelocityEngine velocityEngine;
+
+ @Mock
+ File mockFile;
+
+
+ private String stack;
+
+ @BeforeEach
+ public void before() {
+ stack = "dev";
+ when(mockConfig.getProperty(PROPERTY_KEY_STACK)).thenReturn(stack);
+ when(mockConfig.getProperty(PROPERTY_KEY_LAMBDA_ARTIFACT_BUCKET)).thenReturn("lambda.sagebase.org");
+ when(mockConfig.getProperty(PROPERTY_KEY_LAMBDA_MARKDOWNIT_ARTIFACT_URL)).thenReturn("https://sagebionetworks.jfrog.io/lambda/org/sagebase/markdownit/markdownit.zip");
+ }
+
+ @Test
+ public void testBuildMarkDownItLambda() throws Exception {
+
+ velocityEngine = new TemplateGuiceModule().velocityEngineProvider();
+
+ MarkDownItLambdaBuilder builder = new MarkDownItLambdaBuilderImpl(
+ mockConfig,
+ mockDownloader,
+ mockCloudFormationClient,
+ mockTagsProvider,
+ mockS3Client,
+ velocityEngine);
+
+
+ when(mockDownloader.downloadFile(any())).thenReturn(mockFile);
+
+ when(mockTagsProvider.getStackTags()).thenReturn(Collections.emptyList());
+
+ Stack markdownItLambdaStack = new Stack();
+
+ when(mockCloudFormationClient.describeStack(any())).thenReturn(Optional.of(markdownItLambdaStack));
+
+ String expectedBucket = "lambda.sagebase.org";
+ String expectedKey = "artifacts/markdown-it/markdownit.zip";
+
+ // call under test
+ builder.buildMarkDownItLambda();
+
+ verify(mockDownloader).downloadFile("https://sagebionetworks.jfrog.io/lambda/org/sagebase/markdownit/markdownit.zip");
+ verify(mockS3Client).putObject(expectedBucket, expectedKey, mockFile);
+
+ verify(mockFile).delete();
+
+ ArgumentCaptor<CreateOrUpdateStackRequest> argCaptorCreateOrUpdateStack = ArgumentCaptor.forClass(CreateOrUpdateStackRequest.class);
+ ArgumentCaptor<String> argCaptorWaitForStack = ArgumentCaptor.forClass(String.class);
+ ArgumentCaptor<String> argCaptorDescribeStack = ArgumentCaptor.forClass(String.class);
+
+ verify(mockCloudFormationClient, times(1)).createOrUpdateStack(argCaptorCreateOrUpdateStack.capture());
+ verify(mockCloudFormationClient, times(1)).waitForStackToComplete(argCaptorWaitForStack.capture());
+ verify(mockCloudFormationClient, times(1)).describeStack(argCaptorDescribeStack.capture());
+
+ CreateOrUpdateStackRequest request = argCaptorCreateOrUpdateStack.getValue();
+ assertEquals("dev-markdown-it-function", request.getStackName());
+ assertTrue(request.getTags().isEmpty());
+ assertEquals(1, request.getCapabilities().length);
+ assertEquals(CAPABILITY_NAMED_IAM, request.getCapabilities()[0]);
+ assertNotNull(request.getTemplateBody());
+
+ JSONObject templateJson = new JSONObject(request.getTemplateBody());
+ System.out.println(request.getTemplateBody());
+ JSONObject resources = templateJson.getJSONObject("Resources");
+ assertTrue(resources.has("mdlambdaServiceRole"));
+ assertTrue(resources.has("mdlambda"));
+ assertTrue(resources.has("mdlambdaFunctionUrl"));
+
+ assertEquals("dev-markdown-it-function", argCaptorWaitForStack.getValue());
+ assertEquals("dev-markdown-it-function", argCaptorDescribeStack.getValue());
+
+ }
+
+}