Merge branch 'master' of github.com:spinnaker/orca

orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/api/BakeRequest.groovy

@@ -28,7 +28,7 @@ import groovy.transform.Immutable
 @Immutable(copyWith = true)
 @CompileStatic
 class BakeRequest {
-  static final Default = new BakeRequest(System.getProperty("user.name"), null, null, null, null, null, CloudProviderType.aws, Label.release, "ubuntu", null, null, null, null, null, null, null, false, null, null)
+  static final Default = new BakeRequest(System.getProperty("user.name"), null, null, null, null, null, CloudProviderType.aws, Label.release, "ubuntu", null, null, null, null, null, null, null, null, null)
 
   String user
   @JsonProperty("package") String packageName
@@ -46,13 +46,12 @@ class BakeRequest {
   Boolean enhancedNetworking
   String amiName
   String amiSuffix
-  Boolean allowMissingPackageInstallation
 
   String templateFileName
   Map extendedAttributes
 
   static enum CloudProviderType {
-    aws, docker, gce
+    aws, azure, docker, gce
   }
 
   static enum Label {

orca-bakery/src/main/groovy/com/netflix/spinnaker/orca/bakery/tasks/CreateBakeTask.groovy

@@ -55,6 +55,9 @@ class CreateBakeTask implements RetryableTask {
   @Value('${bakery.propagateCloudProviderType:false}')
   boolean propagateCloudProviderType
 
+  @Value('${bakery.allowMissingPackageInstallation:false}')
+  boolean allowMissingPackageInstallation
+
   @Override
   TaskResult execute(Stage stage) {
     String region = stage.context.region
@@ -132,16 +135,14 @@ class CreateBakeTask implements RetryableTask {
                                               extractBuildDetails,
                                               false /* extractVersion */,
                                               mapper)
-    Map requestMap = packageInfo.findTargetPackage()
+
+    Map requestMap = packageInfo.findTargetPackage(allowMissingPackageInstallation)
     BakeRequest bakeRequest = mapper.convertValue(requestMap, BakeRequest)
 
     if (!propagateCloudProviderType) {
       bakeRequest = bakeRequest.copyWith(cloudProviderType: null)
     }
 
-    // The allowMissingPackageInstallation only affect Orca behavior. We don't want to propagate it to any bakery.
-    bakeRequest = bakeRequest.copyWith(allowMissingPackageInstallation: null)
-
     if (!roscoApisEnabled) {
       bakeRequest = bakeRequest.copyWith(templateFileName: null, extendedAttributes: null)
     }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/providers/aws/ApplySourceServerGroupCapacityTask.groovy

@@ -58,6 +58,13 @@ class ApplySourceServerGroupCapacityTask extends AbstractServerGroupTask {
         getCloudProvider(ancestorDeployStage)
       ).get()
 
+      def minCapacity = Math.min(
+        ancestorDeployStageData.sourceServerGroupCapacitySnapshot.min as Long,
+        deployServerGroup.capacity.min as Long
+      )
+
+      log.info("Restoring capacity of ${ancestorDeployStageData.region}/${deployServerGroup.name} to ${minCapacity} (currentMin: ${deployServerGroup.capacity.min}, snapshotMin: ${ancestorDeployStageData.sourceServerGroupCapacitySnapshot.min})")
+
       return [
         credentials    : getCredentials(stage),
         regions        : [ancestorDeployStageData.region],
@@ -66,7 +73,7 @@ class ApplySourceServerGroupCapacityTask extends AbstractServerGroupTask {
         serverGroupName: deployServerGroup.name,
         capacity       : deployServerGroup.capacity + [
           // only update the min capacity, desired + max should be inherited from the current server group
-          min: ancestorDeployStageData.sourceServerGroupCapacitySnapshot.min
+          min: minCapacity
         ]
       ]
     } catch (Exception e) {

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/servergroup/strategies/AbstractDeployStrategyStage.groovy

@@ -130,7 +130,12 @@ abstract class AbstractDeployStrategyStage extends AbstractCloudProviderAwareSta
     static CleanupConfig fromStage(Stage stage) {
       def stageData = stage.mapTo(StageData)
       def loc = TargetServerGroup.Support.locationFromStageData(stageData)
-      new CleanupConfig(stageData.account, stageData.cluster, stageData.cloudProvider, loc)
+      new CleanupConfig(
+          account: stageData.account,
+          cluster: stageData.cluster,
+          cloudProvider: stageData.cloudProvider,
+          location: loc
+      )
     }
   }
 }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/servergroup/strategies/RedBlackStrategy.groovy

@@ -61,18 +61,18 @@ class RedBlackStrategy implements Strategy {
       LinearStage.injectAfter(stage, "shrinkCluster", shrinkClusterStage, shrinkContext)
     }
 
+    LinearStage.injectAfter(stage, "disableCluster", disableClusterStage, baseContext + [
+        remainingEnabledServerGroups: 1,
+        preferLargerOverNewer       : false
+    ])
+
     if (stageData.scaleDown) {
       def scaleDown = baseContext + [
-          allowScaleDownActive         : true,
-          remainingFullSizeServerGroups: 1,
-          preferLargerOverNewer        : false
+        allowScaleDownActive         : false,
+        remainingFullSizeServerGroups: 1,
+        preferLargerOverNewer        : false
       ]
       LinearStage.injectAfter(stage, "scaleDown", scaleDownClusterStage, scaleDown)
     }
-
-    LinearStage.injectAfter(stage, "disableCluster", disableClusterStage, baseContext + [
-        remainingEnabledServerGroups: 1,
-        preferLargerOverNewer       : false
-    ])
   }
 }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/servergroup/support/TargetServerGroup.groovy

@@ -55,14 +55,18 @@ class TargetServerGroup {
     ]
 
     def loc = getLocation()
-    if (loc.type == Location.Type.REGION) {
-      op.region = loc.value
-    } else if (loc.type == Location.Type.ZONE) {
-      op.zone = loc.value
-    } else if (loc.type == Location.Type.NAMESPACE) {
-      op.namespace = loc.value
-    } else {
-      throw new IllegalStateException("unsupported location type $loc.type")
+    switch (loc.type) {
+      case (Location.Type.NAMESPACE):
+        op.namespace = loc.value
+        break
+      case (Location.Type.REGION):
+        op.region = loc.value
+        break
+      case (Location.Type.ZONE):
+        op.zone = loc.value
+        break
+      default:
+        throw new IllegalStateException("unsupported location type $loc.type")
     }
     return op
   }
@@ -73,30 +77,30 @@ class TargetServerGroup {
   }
 
   public static class Support {
-    static Location resolveLocation(String cloudProvider, String zone, String namespace, String region) {
-      if (cloudProvider == "gce" && zone) {
-        return Location.zone(zone)
-      } else if (namespace) {
+    static Location resolveLocation(String namespace, String region, String zone) {
+      if (namespace) {
         return Location.namespace(namespace)
       } else if (region) {
         return Location.region(region)
+      } else if (zone) {
+        return Location.zone(zone)
       } else {
-        throw new IllegalArgumentException("No known location type provided. Must be `region`, `zone` or `namespace`.")
+        throw new IllegalArgumentException("No known location type provided. Must be `namespace`, `region` or `zone`.")
       }
     }
 
     static Location locationFromServerGroup(Map<String, Object> serverGroup, Location.Type exactLocationType) {
       switch (exactLocationType) {
-        case (Location.Type.ZONE):
-          return Location.zone(serverGroup.zone)
         case (Location.Type.NAMESPACE):
           return Location.namespace(serverGroup.namespace)
         case (Location.Type.REGION):
           return Location.region(serverGroup.region)
+        case (Location.Type.ZONE):
+          return Location.zone(serverGroup.zone)
       }
 
       try {
-        return resolveLocation(serverGroup.type, serverGroup.zone, serverGroup.namespace, serverGroup.region)
+        return resolveLocation(serverGroup.namespace, serverGroup.region, serverGroup.zone)
       } catch (e) {
         throw new IllegalArgumentException("Incorrect location specified for ${serverGroup.serverGroupName ?: serverGroup.name}: ${e.message}")
       }
@@ -112,7 +116,7 @@ class TargetServerGroup {
     static Location locationFromStageData(StageData stageData) {
       try {
         List zones = stageData.availabilityZones?.values()?.flatten()?.toArray()
-        return resolveLocation(stageData.cloudProvider, zones?.get(0), stageData.namespace, stageData.region)
+        return resolveLocation(stageData.namespace, stageData.region, zones?.get(0))
       } catch (e) {
         throw new IllegalArgumentException("Incorrect location specified for ${stageData}: ${e.message}")
       }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/cluster/AbstractClusterWideClouddriverTask.groovy

@@ -108,13 +108,17 @@ abstract class AbstractClusterWideClouddriverTask extends AbstractCloudProviderA
     }
 
     def locations =
-      stage.context.regions
-      ? stage.context.regions.collect { new Location(type: Location.Type.REGION, value: it) }
-      : stage.context.zones
-        ? stage.context.zones.collect { new Location(type: Location.Type.ZONE, value: it) }
-        : stage.context.region
-          ? [new Location(type: Location.Type.REGION, value: stage.context.region)]
-          : []
+      stage.context.namespaces
+      ? stage.context.namespaces.collect { new Location(type: Location.Type.NAMESPACE, value: it) }
+      : stage.context.regions
+        ? stage.context.regions.collect { new Location(type: Location.Type.REGION, value: it) }
+        : stage.context.zones
+          ? stage.context.zones.collect { new Location(type: Location.Type.ZONE, value: it) }
+          : stage.context.namespace
+            ? [new Location(type: Location.Type.NAMESPACE, value: stage.context.namespace)]
+            : stage.context.region
+              ? [new Location(type: Location.Type.REGION, value: stage.context.region)]
+              : []
 
     Location.Type exactLocationType = locations?.getAt(0)?.type
 

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/aws/AmazonServerGroupCreator.groovy

@@ -124,17 +124,15 @@ class AmazonServerGroupCreator implements ServerGroupCreator, DeploymentDetailsA
 
   def allowLaunchOperations(Map createServerGroupOp) {
     def ops = []
-    if (createServerGroupOp.credentials != defaultBakeAccount) {
-      if (createServerGroupOp.availabilityZones) {
-        ops.addAll(createServerGroupOp.availabilityZones.collect { String region, List<String> azs ->
-          [account    : createServerGroupOp.credentials,
-           credentials: defaultBakeAccount,
-           region     : region,
-           amiName    : createServerGroupOp.amiName]
-        })
-
-        log.info("Generated `allowLaunchDescriptions` (allowLaunchDescriptions: ${ops})")
-      }
+    if (createServerGroupOp.availabilityZones && createServerGroupOp.credentials != defaultBakeAccount) {
+      ops.addAll(createServerGroupOp.availabilityZones.collect { String region, List<String> azs ->
+        [account    : createServerGroupOp.credentials,
+         credentials: defaultBakeAccount,
+         region     : region,
+         amiName    : createServerGroupOp.amiName]
+      })
+
+      log.info("Generated `allowLaunchDescriptions` (allowLaunchDescriptions: ${ops})")
     }
     return ops
   }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/azure/AzureServerGroupCreator.groovy

@@ -17,13 +17,14 @@
 package com.netflix.spinnaker.orca.clouddriver.tasks.providers.azure
 
 import com.netflix.spinnaker.orca.clouddriver.tasks.servergroup.ServerGroupCreator
+import com.netflix.spinnaker.orca.kato.tasks.DeploymentDetailsAware
 import com.netflix.spinnaker.orca.pipeline.model.Stage
 import groovy.util.logging.Slf4j
 import org.springframework.stereotype.Component
 
 @Slf4j
 @Component
-class AzureServerGroupCreator implements ServerGroupCreator {
+class AzureServerGroupCreator implements ServerGroupCreator, DeploymentDetailsAware {
 
   boolean katoResultExpected = false
   String cloudProvider = "azure"
@@ -38,6 +39,14 @@ class AzureServerGroupCreator implements ServerGroupCreator {
       operation.credentials = operation.account
     }
 
+    def bakeStage = getPreviousStageWithImage(stage, operation.region)
+
+    if (bakeStage) {
+      operation.image.isCustom = true
+      operation.image.uri = bakeStage.context?.imageId
+      operation.image.ostype = bakeStage.context?.osType
+    }
+
     return [[(ServerGroupCreator.OPERATION): operation]]
   }
 }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/providers/kubernetes/KubernetesContainerFinder.groovy

@@ -45,14 +45,9 @@ class KubernetesContainerFinder {
         if (stage.execution instanceof Pipeline) {
           Map trigger = ((Pipeline) stage.execution).trigger
 
-          def matchingTag = trigger?.buildInfo?.taggedImages?.findResult { info ->
-            if (container.imageDescription.registry == info.getAt("registry") &&
-                container.imageDescription.repository == info.getAt("repository")) {
-              return info.tag
-            }
+          if (trigger?.registry == container.imageDescription.registry && trigger?.repository == container.imageDescription.repository) {
+            container.imageDescription.tag = trigger.tag
           }
-
-          container.imageDescription.tag = matchingTag
         }
 
         if (!container.imageDescription.tag) {

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/servergroup/CloneServerGroupTask.groovy

@@ -24,6 +24,7 @@ import com.netflix.spinnaker.orca.TaskResult
 import com.netflix.spinnaker.orca.clouddriver.KatoService
 import com.netflix.spinnaker.orca.clouddriver.tasks.AbstractCloudProviderAwareTask
 import com.netflix.spinnaker.orca.clouddriver.utils.HealthHelper
+import com.netflix.spinnaker.orca.kato.tasks.DeploymentDetailsAware
 import com.netflix.spinnaker.orca.pipeline.model.Stage
 import groovy.util.logging.Slf4j
 import org.springframework.beans.factory.annotation.Autowired
@@ -32,7 +33,7 @@ import org.springframework.stereotype.Component
 
 @Slf4j
 @Component
-class CloneServerGroupTask extends AbstractCloudProviderAwareTask implements Task {
+class CloneServerGroupTask extends AbstractCloudProviderAwareTask implements Task, DeploymentDetailsAware {
 
   @Autowired
   KatoService kato
@@ -47,8 +48,17 @@ class CloneServerGroupTask extends AbstractCloudProviderAwareTask implements Tas
   TaskResult execute(Stage stage) {
     def operation = [:]
     operation.putAll(stage.context)
-    operation.amiName = operation.amiName ?: stage.preceding("bake")?.context?.amiName as String
-    operation.imageId = operation.imageId ?: stage.preceding("bake")?.context?.imageId as String
+    String targetRegion = operation.region ?: operation.availabilityZones?.keySet()?.getAt(0) ?: operation.source?.region
+    withImageFromPrecedingStage(stage, targetRegion) {
+      operation.amiName = operation.amiName ?: it.amiName
+      operation.imageId = operation.imageId ?: it.imageId
+    }
+
+    withImageFromDeploymentDetails(stage, targetRegion) {
+      operation.amiName = operation.amiName ?: it.amiName
+      operation.imageId = operation.imageId ?: it.imageId
+    }
+
     String cloudProvider = getCloudProvider(stage)
     String credentials = getCredentials(stage)
     def taskId = kato.requestOperations(cloudProvider, getDescriptions(operation)).toBlocking().first()
@@ -77,7 +87,8 @@ class CloneServerGroupTask extends AbstractCloudProviderAwareTask implements Tas
     // NFLX bakes images in their test account. This rigmarole is to allow the prod account access to that image.
     if (getCloudProvider(operation) == "aws" && // the operation is a clone of stage.context.
         operation.credentials != defaultBakeAccount &&
-        operation.availabilityZones) {
+        operation.availabilityZones &&
+        operation.amiName) {
       def allowLaunchDescriptions = operation.availabilityZones.collect { String region, List<String> azs ->
         [
           allowLaunchDescription: [

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/kato/pipeline/ParallelDeployStage.groovy

@@ -135,6 +135,10 @@ class ParallelDeployStage extends ParallelStage {
             }
           }
         }
+
+        // Avoid passing 'stageEnabled' configuration on to the deploy stage in a strategy pipeline
+        cluster.remove("stageEnabled")
+
         stage.context.clusters = [cluster as Map<String, Object>]
       }
     }

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/kato/pipeline/QuickPatchStage.groovy

@@ -60,6 +60,9 @@ class QuickPatchStage extends LinearStage {
   @Value('${bakery.roscoApisEnabled:false}')
   boolean roscoApisEnabled
 
+  @Value('${bakery.allowMissingPackageInstallation:false}')
+  boolean allowMissingPackageInstallation
+
   @Autowired
   ObjectMapper objectMapper
 
@@ -96,7 +99,7 @@ class QuickPatchStage extends LinearStage {
                                               true /* extractBuildDetails */,
                                               true /* extractVersion */,
                                               objectMapper)
-    String version = stage.context?.patchVersion ?:  packageInfo.findTargetPackage()?.packageVersion
+    String version = stage.context?.patchVersion ?:  packageInfo.findTargetPackage(allowMissingPackageInstallation)?.packageVersion
 
     stage.context.put("version", version) // so the ui can display the discovered package version and we can verify for skipUpToDate
     def instances = getInstancesForCluster(stage)

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/kato/pipeline/strategy/DeployStrategyStage.groovy

@@ -126,19 +126,20 @@ abstract class DeployStrategyStage extends AbstractCloudProviderAwareStage {
       injectAfter(stage, "shrinkCluster", shrinkClusterStage, shrinkContext)
     }
 
+    injectAfter(stage, "disableCluster", disableClusterStage, baseContext + [
+      remainingEnabledServerGroups: 1,
+      preferLargerOverNewer: false
+    ])
+
     if (stageData.scaleDown) {
       def scaleDown = baseContext + [
-        allowScaleDownActive: true,
+        allowScaleDownActive: false,
         remainingFullSizeServerGroups: 1,
         preferLargerOverNewer: false
       ]
       injectAfter(stage, "scaleDown", scaleDownClusterStage, scaleDown)
     }
 
-    injectAfter(stage, "disableCluster", disableClusterStage, baseContext + [
-      remainingEnabledServerGroups: 1,
-      preferLargerOverNewer: false
-    ])
   }
 
   protected void composeRollingPushFlow(Stage stage) {