feat(jobs): preconfigured run job stage (spinnaker#2540)

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/config/ClouddriverJobConfiguration.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.clouddriver.config;
+
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+
+@EnableConfigurationProperties(JobConfigurationProperties.class)
+public class ClouddriverJobConfiguration {}

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/config/JobConfigurationProperties.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.clouddriver.config;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import java.util.List;
+
+@ConfigurationProperties("job")
+@Data
+public class JobConfigurationProperties {
+  List<PreconfiguredJobStageProperties> preconfigured;
+}

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/config/PreconfiguredJobStageParameter.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.clouddriver.config;
+
+import com.netflix.spinnaker.orca.config.PreconfiguredStageParameter;
+import lombok.Data;
+
+@Data
+public class PreconfiguredJobStageParameter extends PreconfiguredStageParameter {
+
+  String mapping;
+
+}

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/config/PreconfiguredJobStageProperties.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.clouddriver.config;
+
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Data
+
+public class PreconfiguredJobStageProperties {
+
+  // Fields are public as job stages use reflection to access these directly from outside the class
+  public boolean enabled = true;
+  public String label;
+  public String description;
+  public String type;
+  public List<PreconfiguredJobStageParameter> parameters;
+  public boolean waitForCompletion = true;
+  public String cloudProvider;
+  public String credentials;
+  public String region;
+  public String propertyFile;
+  public Map<String, Object> cluster = new HashMap<>();
+
+}

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/exception/PreconfiguredJobNotFoundException.groovy

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2017 Schibsted ASA.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ *
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.clouddriver.exception
+
+class PreconfiguredJobNotFoundException extends RuntimeException {
+  PreconfiguredJobNotFoundException(String jobKey) {
+    super("Could not find a stage named '$jobKey'")
+  }
+}

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/pipeline/job/PreconfiguredJobStage.groovy

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.clouddriver.pipeline.job
+
+
+import com.netflix.spinnaker.orca.clouddriver.config.PreconfiguredJobStageProperties
+import com.netflix.spinnaker.orca.clouddriver.exception.PreconfiguredJobNotFoundException
+import com.netflix.spinnaker.orca.clouddriver.service.JobService
+import com.netflix.spinnaker.orca.pipeline.TaskNode
+import com.netflix.spinnaker.orca.pipeline.model.Stage
+import org.springframework.beans.factory.annotation.Autowired
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.stereotype.Component
+
+@Component
+@ConditionalOnProperty("job.preconfigured")
+class PreconfiguredJobStage extends RunJobStage {
+
+  @Autowired
+  private JobService jobService
+
+  def fields = PreconfiguredJobStageProperties.declaredFields.findAll {
+    !it.synthetic && !['props', 'enabled', 'label', 'description', 'type', 'parameters'].contains(it.name)
+  }.collect { it.name }
+
+  @Override
+  void taskGraph(Stage stage, TaskNode.Builder builder) {
+    def preconfiguredJob = jobService.getPreconfiguredStages().find { stage.type == it.type }
+
+    if (!preconfiguredJob) {
+      throw new PreconfiguredJobNotFoundException((String) stage.type)
+    }
+
+    stage.setContext(overrideIfNotSetInContextAndOverrideDefault(stage.context, preconfiguredJob))
+    super.taskGraph(stage, builder)
+  }
+
+  private Map<String, Object> overrideIfNotSetInContextAndOverrideDefault(Map<String, Object> context, PreconfiguredJobStageProperties preconfiguredWebhook) {
+    fields.each {
+      if (context[it] == null || preconfiguredWebhook[it] != null) {
+        context[it] = preconfiguredWebhook[it]
+      }
+    }
+    preconfiguredWebhook.parameters.each { defaults ->
+      if (defaults.defaultValue != null) {
+        Eval.xy(context, defaults.defaultValue, "x.${defaults.mapping} = y.toString()")
+      }
+    }
+    if (context.parameters) {
+      context.parameters.each { k, v ->
+        def parameterDefinition = preconfiguredWebhook.parameters.find { it.name == k }
+        if (parameterDefinition) {
+          Eval.xy(context, v, "x.${parameterDefinition.mapping} = y.toString()")
+        }
+      }
+      context.remove("parameters")
+    }
+    return context
+  }
+
+}

orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/service/JobService.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License")
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.clouddriver.service;
+
+import com.netflix.spinnaker.orca.clouddriver.config.JobConfigurationProperties;
+import com.netflix.spinnaker.orca.clouddriver.config.PreconfiguredJobStageProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Service
+@ConditionalOnProperty("job.preconfigured")
+public class JobService {
+
+  @Autowired JobConfigurationProperties jobConfigurationProperties;
+
+  @Autowired
+  List<PreconfiguredJobStageProperties> getPreconfiguredStages() {
+    return jobConfigurationProperties.getPreconfigured().stream().filter(it -> it.enabled == true).collect(Collectors.toList());
+  }
+
+}

orca-core/src/main/java/com/netflix/spinnaker/orca/config/PreconfiguredStageParameter.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2017 Schibsted ASA.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.orca.config;
+
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class PreconfiguredStageParameter {
+  private String name;
+  private String label;
+  private String defaultValue;
+  private String description;
+  private ParameterType type = ParameterType.string;
+  private int order;
+
+  public
+  enum ParameterType {
+    string
+  }
+}

orca-web/config/orca.yml

@@ -82,3 +82,27 @@ logging:
 #      description: This is also a webhook stage, but it has no properties set
 #      type: anotherCustomWebhook
 #      enabled: false
+
+# This configuration lets you configure runJob stages that will appear as native stages in the Deck UI.
+# Properties that are set here will not be displayed in the GUI
+# job:
+#  preconfigured:
+#    - label: Some runJob
+#      description: This is a runJob stage, but it appears as a native stage in Spinnaker
+#      type: customRunJob # Should be unique
+#      cloudProvider: titus
+#      cluster:
+#        imageId: alpine:latest
+#      credentials: test
+#      region: us-east-1
+#      enabled: true # default true
+#      waitForCompletion: true
+#      parameters:
+#      - name: source
+#        mapping: cluster.env.source
+#        defaultValue: mysource
+#        order: 0
+#      - name: destination
+#        mapping: cluster.env.destination
+#        defaultValue: final
+#        order: 1

orca-web/src/main/groovy/com/netflix/spinnaker/orca/Main.groovy

@@ -24,6 +24,7 @@ import com.netflix.spinnaker.kork.PlatformComponents
 import com.netflix.spinnaker.orca.applications.config.ApplicationConfig
 import com.netflix.spinnaker.orca.bakery.config.BakeryConfiguration
 import com.netflix.spinnaker.orca.clouddriver.config.CloudDriverConfiguration
+import com.netflix.spinnaker.orca.clouddriver.config.ClouddriverJobConfiguration
 import com.netflix.spinnaker.orca.config.KeelConfiguration
 import com.netflix.spinnaker.orca.config.OrcaConfiguration
 import com.netflix.spinnaker.orca.config.PipelineTemplateConfiguration
@@ -66,6 +67,7 @@ import org.springframework.scheduling.annotation.EnableAsync
   Front50Configuration,
   FlexConfiguration,
   CloudDriverConfiguration,
+  ClouddriverJobConfiguration,
   IgorConfiguration,
   DiscoveryPollingConfiguration,
   TomcatConfiguration,

orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/OperationsController.groovy

@@ -22,6 +22,7 @@ import com.netflix.spinnaker.fiat.shared.FiatService
 import com.netflix.spinnaker.fiat.shared.FiatStatus
 import com.netflix.spinnaker.kork.web.exceptions.InvalidRequestException
 import com.netflix.spinnaker.kork.web.exceptions.ValidationException
+import com.netflix.spinnaker.orca.clouddriver.service.JobService
 import com.netflix.spinnaker.orca.extensionpoint.pipeline.PipelinePreprocessor
 import com.netflix.spinnaker.orca.igor.BuildService
 import com.netflix.spinnaker.orca.pipeline.ExecutionLauncher
@@ -73,6 +74,9 @@ class OperationsController {
   @Autowired(required = false)
   WebhookService webhookService
 
+  @Autowired(required = false)
+  JobService jobService
+
   @Autowired(required = false)
   ArtifactResolver artifactResolver
 
@@ -254,6 +258,22 @@ class OperationsController {
     }
   }
 
+  @RequestMapping(value = "/jobs/preconfigured")
+  List<Map<String, Object>> preconfiguredJob() {
+    if (!jobService) {
+      return []
+    }
+    return jobService?.getPreconfiguredStages().collect{
+      [ label: it.label,
+        description: it.description,
+        type: it.type,
+        waitForCompletion: it.waitForCompletion,
+        noUserConfigurableFields: true,
+        parameters: it.parameters,
+      ]
+    }
+  }
+
   private void convertLinearToParallel(Map<String, Serializable> pipelineConfig) {
     def stages = (List<Map<String, Object>>) pipelineConfig.stages
     stages.eachWithIndex { Map<String, Object> stage, int index ->