feat(ecs): Override Task Def Artifacts (spinnaker#4117)

Co-authored-by: allisaurus <[email protected]>

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/ecs/EcsServerGroupCreator.groovy

@@ -18,19 +18,40 @@ package com.netflix.spinnaker.orca.clouddriver.tasks.providers.ecs
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.netflix.spinnaker.kork.artifacts.model.Artifact
+import com.netflix.spinnaker.kork.exceptions.ConfigurationException
 import com.netflix.spinnaker.orca.api.pipeline.models.StageExecution
+import com.netflix.spinnaker.orca.clouddriver.OortService
 import com.netflix.spinnaker.orca.clouddriver.tasks.servergroup.ServerGroupCreator
 import com.netflix.spinnaker.orca.kato.tasks.DeploymentDetailsAware
 import com.netflix.spinnaker.orca.pipeline.model.DockerTrigger
 import com.netflix.spinnaker.orca.api.pipeline.models.ExecutionType
 import com.netflix.spinnaker.orca.pipeline.util.ArtifactUtils
 import groovy.util.logging.Slf4j
+import org.springframework.beans.factory.annotation.Autowired
+import retrofit.client.Response
+
 import javax.annotation.Nullable
 import org.springframework.stereotype.Component
+import com.fasterxml.jackson.core.type.TypeReference
+import com.google.common.collect.ImmutableMap
+import com.netflix.spinnaker.orca.pipeline.expressions.PipelineExpressionEvaluator
+import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor
+import org.yaml.snakeyaml.Yaml
+import org.yaml.snakeyaml.constructor.SafeConstructor
+
+import java.time.Duration
+import java.util.function.Supplier
+import java.util.stream.StreamSupport
+import com.netflix.spinnaker.kork.core.RetrySupport
+import com.netflix.spinnaker.orca.jackson.OrcaObjectMapper
 
 @Slf4j
 @Component
 class EcsServerGroupCreator implements ServerGroupCreator, DeploymentDetailsAware {
+  final private OortService oortService
+  private final ContextParameterProcessor contextParameterProcessor
+  private static final ThreadLocal<Yaml> yamlParser =
+      ThreadLocal.withInitial({ -> new Yaml(new SafeConstructor()) })
 
   final String cloudProvider = "ecs"
   final boolean katoResultExpected = false
@@ -39,9 +60,15 @@ class EcsServerGroupCreator implements ServerGroupCreator, DeploymentDetailsAwar
 
   final ObjectMapper mapper = new ObjectMapper()
   final ArtifactUtils artifactUtils
+  private static final ObjectMapper objectMapper = OrcaObjectMapper.getInstance()
+  private final RetrySupport retrySupport
 
-  EcsServerGroupCreator(ArtifactUtils artifactUtils) {
+  @Autowired
+  EcsServerGroupCreator(ArtifactUtils artifactUtils, OortService oort, ContextParameterProcessor contextParameterProcessor, RetrySupport retrySupport) {
     this.artifactUtils = artifactUtils
+    this.oortService = oort
+    this.contextParameterProcessor = contextParameterProcessor
+    this.retrySupport = retrySupport
   }
 
   @Override
@@ -57,6 +84,22 @@ class EcsServerGroupCreator implements ServerGroupCreator, DeploymentDetailsAwar
     if (operation.useTaskDefinitionArtifact) {
       if (operation.taskDefinitionArtifact) {
         operation.resolvedTaskDefinitionArtifact = getTaskDefArtifact(stage, operation.taskDefinitionArtifact)
+        if (operation.evaluateTaskDefinitionArtifactExpressions) {
+          Iterable<Object> rawArtifact =
+              retrySupport.retry(
+                  fetchAndParseArtifact(operation.resolvedTaskDefinitionArtifact), 10, Duration.ofMillis(200), true)
+
+          List<Map<Object, Object>> unevaluatedArtifact =
+              StreamSupport.stream(rawArtifact.spliterator(), false)
+                  .filter(Objects.&nonNull)
+                  .map(this.&coerceArtifactToList)
+                  .collect()
+                  .flatten()
+
+          Map<Object, Object> evaluatedArtifact = getSpelEvaluatedArtifact(unevaluatedArtifact, stage)
+
+          operation.spelProcessedTaskDefinitionArtifact = evaluatedArtifact
+        }
       } else {
         throw new IllegalStateException("No task definition artifact found in context for operation.")
       }
@@ -86,6 +129,47 @@ class EcsServerGroupCreator implements ServerGroupCreator, DeploymentDetailsAwar
     return [[(ServerGroupCreator.OPERATION): operation]]
   }
 
+  private Supplier<Iterable<Object>> fetchAndParseArtifact(Artifact artifact) {
+    return { ->
+      Response artifactText = oortService.fetchArtifact(artifact)
+      try {
+        if(artifactText != null && artifactText.getBody() != null){
+          return yamlParser.get().loadAll(artifactText.getBody().in());
+        } else{
+          throw new ConfigurationException("Invalid artifact configuration or task definition artifact is null")
+        }
+      } catch (Exception e) {
+        throw new IllegalStateException(e);
+      }
+    }
+  }
+
+  private List<Map<Object, Object>> coerceArtifactToList(Object artifact) {
+    Map<Object, Object> singleArtifact =
+        objectMapper.convertValue(artifact, new TypeReference<Map<Object, Object>>() {})
+    return List.of(singleArtifact)
+  }
+
+  private Map<Object, Object> getSpelEvaluatedArtifact(
+      List<Map<Object, Object>> unevaluatedArtifact, StageExecution stage) {
+    Map<String, Object> processorInput = ImmutableMap.of("artifact", unevaluatedArtifact);
+
+    Map<String, Object> processorResult =
+        contextParameterProcessor.process(
+            processorInput,
+            contextParameterProcessor.buildExecutionContext(stage),
+            true)
+
+    if ((boolean) stage.getContext().getOrDefault("failOnFailedExpressions", false)
+        && processorResult.containsKey(PipelineExpressionEvaluator.SUMMARY)) {
+      throw new IllegalStateException(
+          String.format(
+              "Failure evaluating Artifact expressions: %s",
+              processorResult.get(PipelineExpressionEvaluator.SUMMARY)))
+    }
+    return processorResult.get("artifact").get(0)
+  }
+
   static String buildImageId(Object registry, Object repo, Object tag) {
     if (registry) {
       return "$registry/$repo:$tag"

orca-clouddriver/src/test/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/ecs/EcsServerGroupCreatorSpec.groovy

@@ -16,6 +16,10 @@
 package com.netflix.spinnaker.orca.clouddriver.tasks.providers.ecs
 
 import com.google.common.collect.Maps
+import com.netflix.spinnaker.kork.core.RetrySupport
+import com.netflix.spinnaker.orca.clouddriver.OortService
+import com.netflix.spinnaker.orca.pipeline.util.ContextParameterProcessor
+import retrofit.mime.TypedString
 import spock.lang.Specification
 import spock.lang.Subject
 import com.netflix.spinnaker.kork.artifacts.model.Artifact
@@ -29,6 +33,8 @@ class EcsServerGroupCreatorSpec extends Specification {
   @Subject
   ArtifactUtils mockResolver
   EcsServerGroupCreator creator
+  OortService oortService = Mock()
+  ContextParameterProcessor contextParameterProcessor = new ContextParameterProcessor()
   def stage = stage {}
 
   def deployConfig = [
@@ -38,7 +44,7 @@ class EcsServerGroupCreatorSpec extends Specification {
 
   def setup() {
     mockResolver = Stub(ArtifactUtils)
-    creator = new EcsServerGroupCreator(mockResolver)
+    creator = new EcsServerGroupCreator(mockResolver, oortService, contextParameterProcessor, new RetrySupport())
     stage.execution.stages.add(stage)
     stage.context = deployConfig
   }
@@ -185,4 +191,235 @@ class EcsServerGroupCreatorSpec extends Specification {
       it.containsKey("createServerGroup")
     }.createServerGroup == expected
   }
+
+  def "creates operation from downloaded and SpEL processed artifact with no parameter value available"() {
+    given:
+    def testArtifactId = createTaskdefArtifactId()
+    def taskDefArtifact = createTaskDefArtifact(testArtifactId)
+
+    Artifact resolvedArtifact = createResolvedArtifact()
+    mockResolver.getBoundArtifactForStage(stage, testArtifactId, null) >> resolvedArtifact
+    def (testReg,testRepo,testTag) = ["myregistry.io","myrepo","latest"]
+    def testDescription = createTestDescription(testReg, testRepo, testTag)
+    def testMappings = []
+    def map1 = createMap1(testDescription)
+    def map2 = createMap2(testDescription)
+    testMappings.add(map1)
+    testMappings.add(map2)
+
+    // add inputs to stage context
+    stage.execution = new PipelineExecutionImpl(ExecutionType.PIPELINE, 'ecs')
+    stage.context.useTaskDefinitionArtifact = true
+    stage.context.evaluateTaskDefinitionArtifactExpressions = true
+    stage.context.taskDefinitionArtifact = taskDefArtifact
+    stage.context.containerMappings = testMappings
+    stage.execution.trigger.parameters.put("notFound", "noValue")
+
+    when:
+    def operations = creator.getOperations(stage)
+
+    then:
+
+    1 * oortService.fetchArtifact(*_) >> new retrofit.client.Response('http://oort.com', 200, 'Okay', [], new TypedString(response))
+    0 * oortService._
+    operations[0].createServerGroup.spelProcessedTaskDefinitionArtifact.toString().equals(expected)
+
+    where:
+    response                                               | expected
+    '{"foo": "${ parameters[\'tg\'] ?: \'noValue\' }"}'    | "[foo:noValue]"
+  }
+
+  def "creates operation from downloaded and SpEL processed artifact"() {
+    given:
+    def testArtifactId = createTaskdefArtifactId()
+    def taskDefArtifact = createTaskDefArtifact(testArtifactId)
+
+    Artifact resolvedArtifact = createResolvedArtifact()
+    mockResolver.getBoundArtifactForStage(stage, testArtifactId, null) >> resolvedArtifact
+    def (testReg,testRepo,testTag) = ["myregistry.io","myrepo","latest"]
+    def testDescription = createTestDescription(testReg, testRepo, testTag)
+    def testMappings = []
+    def map1 = createMap1(testDescription)
+    def map2 = createMap2(testDescription)
+    testMappings.add(map1)
+    testMappings.add(map2)
+
+    // add inputs to stage context
+    stage.execution = new PipelineExecutionImpl(ExecutionType.PIPELINE, 'ecs')
+    stage.context.useTaskDefinitionArtifact = true
+    stage.context.evaluateTaskDefinitionArtifactExpressions = true
+    stage.context.taskDefinitionArtifact = taskDefArtifact
+    stage.context.containerMappings = testMappings
+    stage.execution.trigger.parameters.put("tg", "bar")
+
+    when:
+    def operations = creator.getOperations(stage)
+
+    then:
+
+    1 * oortService.fetchArtifact(*_) >> new retrofit.client.Response('http://oort.com', 200, 'Okay', [], new TypedString(response))
+    0 * oortService._
+    operations[0].createServerGroup.spelProcessedTaskDefinitionArtifact.toString().equals(expected)
+
+    where:
+    response                                               | expected
+    '{"foo": "${ parameters[\'tg\'] }"}'                   | "[foo:bar]"
+    '{"foo": "${ #toInt(\'43\') }"}'                       | "[foo:43]"
+  }
+
+  def "creates operation when evaluateTaskDefinitionArtifactExpressions flag is falsy"() {
+    given:
+    // define artifact inputs
+    def testArtifactId = createTaskdefArtifactId()
+    def taskDefArtifact = createTaskDefArtifact(testArtifactId)
+
+    Artifact resolvedArtifact = createResolvedArtifact()
+    mockResolver.getBoundArtifactForStage(stage, testArtifactId, null) >> resolvedArtifact
+    def (testReg,testRepo,testTag) = ["myregistry.io","myrepo","latest"]
+    def testDescription = createTestDescription(testReg, testRepo, testTag)
+    def testMappings = []
+    def map1 = createMap1(testDescription)
+    def map2 = createMap2(testDescription)
+    testMappings.add(map1)
+    testMappings.add(map2)
+
+    def containerToImageMap = [
+      web: "$testReg/$testRepo:$testTag",
+      logs: "$testReg/$testRepo:$testTag"
+    ]
+
+    // add inputs to stage context
+    stage.execution = new PipelineExecutionImpl(ExecutionType.PIPELINE, 'ecs')
+    stage.context.useTaskDefinitionArtifact = true
+    stage.context.taskDefinitionArtifact = taskDefArtifact
+    stage.context.containerMappings = testMappings
+    stage.context.evaluateTaskDefinitionArtifactExpressions = false
+
+    def expected = Maps.newHashMap(deployConfig)
+    expected.resolvedTaskDefinitionArtifact = resolvedArtifact
+    expected.containerToImageMap = containerToImageMap
+
+    when:
+    def operations = creator.getOperations(stage)
+
+    then:
+    operations.find {
+      it.containsKey("createServerGroup")
+    }.createServerGroup == expected
+  }
+
+  def "creates operation when evaluateTaskDefinitionArtifactExpressions is truthy but there is no expression"() {
+    given:
+    def testArtifactId = createTaskdefArtifactId()
+    def taskDefArtifact = createTaskDefArtifact(testArtifactId)
+
+    Artifact resolvedArtifact = createResolvedArtifact()
+    mockResolver.getBoundArtifactForStage(stage, testArtifactId, null) >> resolvedArtifact
+    def (testReg,testRepo,testTag) = ["myregistry.io","myrepo","latest"]
+    def testDescription = createTestDescription(testReg, testRepo, testTag)
+    def testMappings = []
+    def map1 = createMap1(testDescription)
+    def map2 = createMap2(testDescription)
+    testMappings.add(map1)
+    testMappings.add(map2)
+
+    // add inputs to stage context
+    stage.execution = new PipelineExecutionImpl(ExecutionType.PIPELINE, 'ecs')
+    stage.context.useTaskDefinitionArtifact = true
+    stage.context.evaluateTaskDefinitionArtifactExpressions = true
+    stage.context.taskDefinitionArtifact = taskDefArtifact
+    stage.context.containerMappings = testMappings
+
+    when:
+    def operations = creator.getOperations(stage)
+
+    then:
+
+    1 * oortService.fetchArtifact(*_) >> new retrofit.client.Response('http://oort.com', 200, 'Okay', [], new TypedString(response))
+    0 * oortService._
+    operations[0].createServerGroup.spelProcessedTaskDefinitionArtifact.toString().equals(expected)
+
+    where:
+    response                                               | expected
+    '{"foo": "bar"}'                                       | "[foo:bar]"
+  }
+
+  def "creates operation when evaluateTaskDefinitionArtifactExpressions is truthy with environment variables"() {
+    given:
+    def testArtifactId = createTaskdefArtifactId()
+    def taskDefArtifact = createTaskDefArtifact(testArtifactId)
+
+    Artifact resolvedArtifact = createResolvedArtifact()
+    mockResolver.getBoundArtifactForStage(stage, testArtifactId, null) >> resolvedArtifact
+    def (testReg,testRepo,testTag) = ["myregistry.io","myrepo","latest"]
+    def testDescription = createTestDescription(testReg, testRepo, testTag)
+    def testMappings = []
+    def map1 = createMap1(testDescription)
+    def map2 = createMap2(testDescription)
+    testMappings.add(map1)
+    testMappings.add(map2)
+
+    // add inputs to stage context
+    stage.execution = new PipelineExecutionImpl(ExecutionType.PIPELINE, 'ecs')
+    stage.context.useTaskDefinitionArtifact = true
+    stage.context.evaluateTaskDefinitionArtifactExpressions = true
+    stage.context.taskDefinitionArtifact = taskDefArtifact
+    stage.context.containerMappings = testMappings
+    stage.execution.trigger.parameters.put("tg", "bar")
+    stage.execution.trigger.parameters.put("ENV1", "bar")
+    stage.context.name = "Deploy"
+
+    when:
+    def operations = creator.getOperations(stage)
+
+    then:
+
+    1 * oortService.fetchArtifact(*_) >> new retrofit.client.Response('http://oort.com', 200, 'Okay', [], new TypedString(response))
+    0 * oortService._
+    operations[0].createServerGroup.spelProcessedTaskDefinitionArtifact.toString().equals(expected)
+
+    where:
+    response                                               | expected
+    '{"foo": "${ENV1}.b.${ENV2}"}'                         | "[foo:\${ENV1}.b.\${ENV2}]"
+    '{"foo": "${ parameters[\'tg\'] }"}'                   | "[foo:bar]"
+    '{"foo": "${ #toInt(\'80\') }"}'                       | "[foo:80]"
+  }
+
+
+  def createTaskdefArtifactId(){
+    return "aaaa-bbbb-cccc-dddd"
+  }
+
+  def createTaskDefArtifact(String testArtifactId){
+    return [
+        artifactId: testArtifactId
+    ]
+  }
+
+  def createResolvedArtifact(){
+    return Artifact.builder().type('s3/object').name('s3://testfile.json').build()
+  }
+
+  def createTestDescription(String testReg, String testRepo, String testTag){
+    return [
+        fromTrigger: "true",
+        registry: testReg,
+        repository: testRepo,
+        tag: testTag
+    ]
+  }
+
+  def createMap1(testDescription){
+    return [
+        containerName: "web",
+        imageDescription: testDescription
+    ]
+  }
+
+  def createMap2(testDescription){
+    return [
+        containerName: "logs",
+        imageDescription: testDescription
+    ]
+  }
 }