feat(delivery): upsert and delete delivery configs through orca (spin…

…naker#2672)
nscmcgrath · Feb 22, 2019 · 95339db · 95339db
1 parent 83c8d11
commit 95339db
Show file tree

Hide file tree

Showing 7 changed files with 316 additions and 30 deletions.
diff --git a/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/Front50Service.groovy b/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/Front50Service.groovy
@@ -18,6 +18,7 @@ package com.netflix.spinnaker.orca.front50
 import com.netflix.spinnaker.fiat.model.resources.ServiceAccount
 import com.netflix.spinnaker.orca.front50.model.Application
 import com.netflix.spinnaker.orca.front50.model.ApplicationNotifications
+import com.netflix.spinnaker.orca.front50.model.DeliveryConfig
 import com.netflix.spinnaker.orca.front50.model.Front50Credential
 import retrofit.client.Response
 import retrofit.http.*
@@ -129,6 +130,18 @@ interface Front50Service {
   @POST("/serviceAccounts")
   Response saveServiceAccount(@Body ServiceAccount serviceAccount)
 
+  @GET("/deliveries/{id}")
+  DeliveryConfig getDeliveryConfig(@Path("id") String id)
+
+  @POST("/deliveries")
+  DeliveryConfig createDeliveryConfig(@Body DeliveryConfig deliveryConfig)
+
+  @PUT("/deliveries/{id}")
+  DeliveryConfig updateDeliveryConfig(@Path("id") String id, @Body DeliveryConfig deliveryConfig)
+
+  @DELETE("/applications/{application}/deliveries/{id}")
+  Response deleteDeliveryConfig(@Path("application") String application, @Path("id") String id)
+
   static class Project {
     String id
     String name

diff --git a/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/model/DeliveryConfig.java b/orca-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/model/DeliveryConfig.java
@@ -0,0 +1,32 @@
+package com.netflix.spinnaker.orca.front50.model;
+
+import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import lombok.Data;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Data
+public class DeliveryConfig {
+  private String id;
+  private String application;
+  private Long lastModified;
+  private Long createTs;
+  private String lastModifiedBy;
+  private List<Map<String,Object>> deliveryArtifacts;
+  private List<Map<String,Object>> deliveryEnvironments;
+
+  private Map<String,Object> details = new HashMap<>();
+
+  @JsonAnyGetter
+  Map<String,Object> details() {
+    return details;
+  }
+
+  @JsonAnySetter
+  void set(String name, Object value) {
+    details.put(name, value);
+  }
+}
diff --git a/...rc/main/groovy/com/netflix/spinnaker/orca/front50/pipeline/DeleteDeliveryConfigStage.java b/...rc/main/groovy/com/netflix/spinnaker/orca/front50/pipeline/DeleteDeliveryConfigStage.java
@@ -0,0 +1,18 @@
+package com.netflix.spinnaker.orca.front50.pipeline;
+
+import com.netflix.spinnaker.orca.front50.tasks.DeleteDeliveryConfigTask;
+import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
+import com.netflix.spinnaker.orca.pipeline.TaskNode;
+import com.netflix.spinnaker.orca.pipeline.model.Stage;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Nonnull;
+
+@Component
+public class DeleteDeliveryConfigStage implements StageDefinitionBuilder {
+  @Override
+  public void taskGraph(@Nonnull Stage stage, @Nonnull TaskNode.Builder builder) {
+    builder
+      .withTask("deleteDeliveryConfig", DeleteDeliveryConfigTask.class);
+  }
+}
diff --git a/...rc/main/groovy/com/netflix/spinnaker/orca/front50/pipeline/UpsertDeliveryConfigStage.java b/...rc/main/groovy/com/netflix/spinnaker/orca/front50/pipeline/UpsertDeliveryConfigStage.java
@@ -0,0 +1,20 @@
+package com.netflix.spinnaker.orca.front50.pipeline;
+
+import com.netflix.spinnaker.orca.front50.tasks.MonitorFront50Task;
+import com.netflix.spinnaker.orca.front50.tasks.UpsertDeliveryConfigTask;
+import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
+import com.netflix.spinnaker.orca.pipeline.TaskNode;
+import com.netflix.spinnaker.orca.pipeline.model.Stage;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Nonnull;
+
+@Component
+public class UpsertDeliveryConfigStage implements StageDefinitionBuilder {
+  @Override
+  public void taskGraph(@Nonnull Stage stage, @Nonnull TaskNode.Builder builder) {
+    builder
+      .withTask("upsertDeliveryConfig", UpsertDeliveryConfigTask.class)
+      .withTask("monitorUpsert", MonitorFront50Task.class);
+  }
+}
diff --git a/...50/src/main/groovy/com/netflix/spinnaker/orca/front50/tasks/DeleteDeliveryConfigTask.java b/...50/src/main/groovy/com/netflix/spinnaker/orca/front50/tasks/DeleteDeliveryConfigTask.java
@@ -0,0 +1,79 @@
+package com.netflix.spinnaker.orca.front50.tasks;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.netflix.spinnaker.orca.ExecutionStatus;
+import com.netflix.spinnaker.orca.Task;
+import com.netflix.spinnaker.orca.TaskResult;
+import com.netflix.spinnaker.orca.front50.Front50Service;
+import com.netflix.spinnaker.orca.front50.model.DeliveryConfig;
+import com.netflix.spinnaker.orca.pipeline.model.Stage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import retrofit.RetrofitError;
+
+import javax.annotation.Nonnull;
+import java.util.Arrays;
+import java.util.Optional;
+
+@Component
+public class DeleteDeliveryConfigTask implements Task {
+
+  private Logger log = LoggerFactory.getLogger(getClass());
+
+  private Front50Service front50Service;
+  private ObjectMapper objectMapper;
+
+  @Autowired
+  public DeleteDeliveryConfigTask(Front50Service front50Service, ObjectMapper objectMapper) {
+    this.front50Service = front50Service;
+    this.objectMapper = objectMapper;
+  }
+
+  @Nonnull
+  @Override
+  public TaskResult execute(@Nonnull Stage stage) {
+    StageData stageData = stage.mapTo(StageData.class);
+
+    if (stageData.deliveryConfigId == null) {
+      throw new IllegalArgumentException("Key 'deliveryConfigId' must be provided.");
+    }
+
+    Optional<DeliveryConfig> config = getDeliveryConfig(stageData.deliveryConfigId);
+
+    if (!config.isPresent()) {
+      log.debug("Config {} does not exist, considering deletion successful.", stageData.deliveryConfigId);
+      return new TaskResult(ExecutionStatus.SUCCEEDED);
+    }
+
+    try {
+      log.debug("Deleting delivery config: " + objectMapper.writeValueAsString(config.get()));
+    } catch (JsonProcessingException e) {
+      // ignore
+    }
+
+    front50Service.deleteDeliveryConfig(config.get().getApplication(), stageData.deliveryConfigId);
+
+    return new TaskResult(ExecutionStatus.SUCCEEDED);
+  }
+
+  public Optional<DeliveryConfig> getDeliveryConfig(String id) {
+    try {
+      DeliveryConfig deliveryConfig = front50Service.getDeliveryConfig(id);
+      return Optional.of(deliveryConfig);
+    } catch (RetrofitError e) {
+      //ignore an unknown (404) or unauthorized (403, 401)
+      if (e.getResponse() != null && Arrays.asList(404, 403, 401).contains(e.getResponse().getStatus())) {
+        return Optional.empty();
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  private static class StageData {
+    public String deliveryConfigId;
+  }
+}
diff --git a/...-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/tasks/MonitorFront50Task.java b/...-front50/src/main/groovy/com/netflix/spinnaker/orca/front50/tasks/MonitorFront50Task.java
@@ -17,22 +17,27 @@
 package com.netflix.spinnaker.orca.front50.tasks;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.netflix.spinnaker.orca.ExecutionStatus;
 import com.netflix.spinnaker.orca.RetryableTask;
 import com.netflix.spinnaker.orca.TaskResult;
 import com.netflix.spinnaker.orca.front50.Front50Service;
+import com.netflix.spinnaker.orca.front50.model.DeliveryConfig;
 import com.netflix.spinnaker.orca.pipeline.model.Stage;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
+import retrofit.RetrofitError;
 
 import javax.annotation.Nonnull;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 @Component
 public class MonitorFront50Task implements RetryableTask {
@@ -42,11 +47,15 @@ public class MonitorFront50Task implements RetryableTask {
   private final int successThreshold;
   private final int gracePeriodMs;
 
+  private final ObjectMapper objectMapper;
+
   @Autowired
   public MonitorFront50Task(Optional<Front50Service> front50Service,
+                            ObjectMapper objectMapper,
                             @Value("${tasks.monitorFront50Task.successThreshold:0}") int successThreshold,
                             @Value("${tasks.monitorFront50Task.gracePeriodMs:5000}") int gracePeriodMs) {
     this.front50Service = front50Service.orElse(null);
+    this.objectMapper = objectMapper;
     this.successThreshold = successThreshold;
 
     // some storage providers round the last modified time to the nearest second, this allows for a configurable
@@ -78,33 +87,7 @@ public TaskResult execute(@Nonnull Stage stage) {
     StageData stageData = stage.mapTo(StageData.class);
     if (stageData.pipelineId != null) {
       try {
-        /*
-         * Some storage services (notably S3) are eventually consistent when versioning is enabled.
-         *
-         * This "dirty hack" attempts to ensure that each underlying instance of Front50 has cached an _updated copy_
-         * of the modified resource.
-         *
-         * It does so by making multiple requests (currently only applies to pipelines) with the expectation that they
-         * will round-robin across all instances of Front50.
-         */
-        for (int i = 0; i < successThreshold; i++) {
-          Optional<Map<String, Object>> pipeline = getPipeline(stageData.pipelineId);
-          if (!pipeline.isPresent()) {
-            return TaskResult.RUNNING;
-          }
-
-          Long lastModifiedTime = Long.valueOf(pipeline.get().get("updateTs").toString());
-          if (lastModifiedTime < (stage.getStartTime() - gracePeriodMs)) {
-            return TaskResult.RUNNING;
-          }
-
-          try {
-            // small delay between verification attempts
-            Thread.sleep(1000);
-          } catch (InterruptedException ignored) {}
-        }
-
-        return TaskResult.SUCCEEDED;
+        return monitor(this::getPipeline, stageData.pipelineId, stage.getStartTime());
       } catch (Exception e) {
         log.error(
           "Unable to verify that pipeline has been updated (executionId: {}, pipeline: {})",
@@ -114,22 +97,85 @@ public TaskResult execute(@Nonnull Stage stage) {
         );
         return TaskResult.RUNNING;
       }
+    } else if (stageData.deliveryConfig != null) {
+      String deliveryConfigId = stageData.deliveryConfig.getId();
+      try {
+        return monitor(this::getDeliveryConfig, deliveryConfigId, stage.getStartTime());
+      } catch (Exception e) {
+        log.error(
+          "Unable to verify that delivery config has been updated (executionId: {}, configId: {})",
+          stage.getExecution().getId(),
+          deliveryConfigId,
+          e
+        );
+        return TaskResult.RUNNING;
+      }
     } else {
       log.warn(
-        "No pipeline id found, unable to verify that pipeline has been updated (executionId: {}, pipeline: {})",
-        stage.getExecution().getId(),
-        stageData.pipelineName
+        "No id found, unable to verify that the object has been updated (executionId: {})",
+        stage.getExecution().getId()
       );
     }
 
     return new TaskResult(ExecutionStatus.SUCCEEDED);
   }
 
+  private TaskResult monitor(Function<String, Optional<Map<String,Object>>> getObjectFunction, String id, Long startTime) {
+    /*
+     * Some storage services (notably S3) are eventually consistent when versioning is enabled.
+     *
+     * This "dirty hack" attempts to ensure that each underlying instance of Front50 has cached an _updated copy_
+     * of the modified resource.
+     *
+     * It does so by making multiple requests (currently only applies to pipelines) with the expectation that they
+     * will round-robin across all instances of Front50.
+     */
+    for (int i = 0; i < successThreshold; i++) {
+      Optional<Map<String, Object>> object = getObjectFunction.apply(id);
+      if (!object.isPresent()) {
+        return TaskResult.RUNNING;
+      }
+
+      Long lastModifiedTime;
+      if (object.get().containsKey("updateTs")) {
+        lastModifiedTime = Long.valueOf(object.get().get("updateTs").toString());
+      } else {
+        lastModifiedTime = Long.valueOf(object.get().get("lastModified").toString());
+      }
+
+      if (lastModifiedTime < (startTime - gracePeriodMs)) {
+        return TaskResult.RUNNING;
+      }
+
+      try {
+        // small delay between verification attempts
+        Thread.sleep(1000);
+      } catch (InterruptedException ignored) {}
+    }
+
+    return TaskResult.SUCCEEDED;
+  }
+
   private Optional<Map<String, Object>> getPipeline(String id) {
     List<Map<String, Object>> pipelines = front50Service.getPipelineHistory(id, 1);
     return pipelines.isEmpty() ? Optional.empty() : Optional.of(pipelines.get(0));
   }
 
+  @SuppressWarnings("unchecked")
+  private Optional<Map<String, Object>> getDeliveryConfig(String id) {
+    try {
+      DeliveryConfig deliveryConfig = front50Service.getDeliveryConfig(id);
+      return Optional.of(objectMapper.convertValue(deliveryConfig, Map.class));
+    } catch (RetrofitError e) {
+      //ignore an unknown (404) or unauthorized (403, 401)
+      if (e.getResponse() != null && Arrays.asList(404, 403, 401).contains(e.getResponse().getStatus())) {
+        return Optional.empty();
+      } else {
+        throw e;
+      }
+    }
+  }
+
   private static class StageData {
     public String application;
 
@@ -138,5 +184,7 @@ private static class StageData {
 
     @JsonProperty("pipeline.name")
     public String pipelineName;
+
+    public DeliveryConfig deliveryConfig;
   }
 }