Skip to content

Commit

Permalink
feat(judges): First pass at support for exercising pairs of judges on…
Browse files Browse the repository at this point in the history
… canned data. (spinnaker#408)
  • Loading branch information
Matt Duftler authored Oct 25, 2018
1 parent e53f299 commit 9fb40f4
Show file tree
Hide file tree
Showing 6 changed files with 337 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ public CanaryExecutionStatusResponse fromExecution(String unresolvedStorageAccou
}

// Some older (stored) results have the execution request only in the judge context.
public String getCanaryExectutionRequestFromJudgeContext(Execution pipeline) {
public String getCanaryExecutionRequestFromJudgeContext(Execution pipeline) {
Stage contextStage = pipeline.getStages().stream()
.filter(stage -> stage.getRefId().equals(CanaryStageNames.REFID_JUDGE))
.findFirst()
Expand All @@ -225,7 +225,7 @@ public CanaryExecutionRequest getCanaryExecutionRequest(Execution pipeline) {

String canaryExecutionRequestJSON = (String)context.get("canaryExecutionRequest");
if (canaryExecutionRequestJSON == null) {
canaryExecutionRequestJSON = getCanaryExectutionRequestFromJudgeContext(pipeline);
canaryExecutionRequestJSON = getCanaryExecutionRequestFromJudgeContext(pipeline);
}
if (canaryExecutionRequestJSON == null) {
return null;
Expand Down Expand Up @@ -439,6 +439,109 @@ public CanaryExecutionResponse buildExecution(String application,
return CanaryExecutionResponse.builder().canaryExecutionId(pipeline.getId()).build();
}

public CanaryExecutionResponse buildJudgeComparisonExecution(String application,
String parentPipelineExecutionId,
@NotNull String canaryConfigId,
@NotNull CanaryConfig canaryConfig,
String overrideCanaryJudge1,
String overrideCanaryJudge2,
String metricSetPairListId,
Double passThreshold,
Double marginalThreshold,
String resolvedConfigurationAccountName,
@NotNull String resolvedStorageAccountName) throws JsonProcessingException {
if (StringUtils.isEmpty(application)) {
application = "kayenta-" + currentInstanceId;
}

if (StringUtils.isEmpty(parentPipelineExecutionId)) {
parentPipelineExecutionId = "no-parent-pipeline-execution";
}

canaryConfig = QueryConfigUtils.escapeTemplates(canaryConfig);

HashMap<String, Object> setupCanaryContext =
Maps.newHashMap(
new ImmutableMap.Builder<String, Object>()
.put("refId", CanaryStageNames.REFID_SET_CONTEXT)
.put("user", "[anonymous]")
.put("application", application)
.put("parentPipelineExecutionId", parentPipelineExecutionId)
.put("storageAccountName", resolvedStorageAccountName)
.put("canaryConfig", canaryConfig)
.build());
if (resolvedConfigurationAccountName != null) {
setupCanaryContext.put("configurationAccountName", resolvedConfigurationAccountName);
}
if (canaryConfigId != null) {
setupCanaryContext.put("canaryConfigId", canaryConfigId);
}

Map<String, Object> canaryJudgeContext1 =
Maps.newHashMap(
new ImmutableMap.Builder<String, Object>()
.put("refId", CanaryStageNames.REFID_JUDGE)
.put("requisiteStageRefIds", Collections.singletonList(CanaryStageNames.REFID_SET_CONTEXT))
.put("user", "[anonymous]")
.put("storageAccountName", resolvedStorageAccountName)
.put("metricSetPairListId", metricSetPairListId)
.put("orchestratorScoreThresholds", CanaryClassifierThresholdsConfig.builder().pass(passThreshold).marginal(marginalThreshold).build())
.build());
if (StringUtils.isNotEmpty(overrideCanaryJudge1)) {
canaryJudgeContext1.put("overrideJudgeName", overrideCanaryJudge1);
}

Map<String, Object> canaryJudgeContext2 =
Maps.newHashMap(
new ImmutableMap.Builder<String, Object>()
.put("refId", CanaryStageNames.REFID_JUDGE + "-2")
.put("requisiteStageRefIds", Collections.singletonList(CanaryStageNames.REFID_SET_CONTEXT))
.put("user", "[anonymous]")
.put("storageAccountName", resolvedStorageAccountName)
.put("metricSetPairListId", metricSetPairListId)
.put("orchestratorScoreThresholds", CanaryClassifierThresholdsConfig.builder().pass(passThreshold).marginal(marginalThreshold).build())
.build());
if (StringUtils.isNotEmpty(overrideCanaryJudge2)) {
canaryJudgeContext2.put("overrideJudgeName", overrideCanaryJudge2);
}

Map<String, Object> compareJudgeResultsContext =
Maps.newHashMap(
new ImmutableMap.Builder<String, Object>()
.put("refId", "compareJudgeResults")
.put("requisiteStageRefIds", Arrays.asList(new String[]{CanaryStageNames.REFID_JUDGE,
CanaryStageNames.REFID_JUDGE + "-2"}))
.put("user", "[anonymous]")
.put("storageAccountName", resolvedStorageAccountName)
.put("judge1Result", "${ #stage('Perform Analysis with Judge 1')['context']['result']}")
.put("judge2Result", "${ #stage('Perform Analysis with Judge 2')['context']['result']}")
.build());

String canaryPipelineConfigId = application + "-standard-canary-pipeline";
PipelineBuilder pipelineBuilder =
new PipelineBuilder(application)
.withName("Standard Canary Pipeline")
.withPipelineConfigId(canaryPipelineConfigId)
.withStage("setupCanary", "Setup Canary", setupCanaryContext)
.withStage("canaryJudge", "Perform Analysis with Judge 1", canaryJudgeContext1)
.withStage("canaryJudge", "Perform Analysis with Judge 2", canaryJudgeContext2)
.withStage("compareJudgeResults", "Compare Judge Results", compareJudgeResultsContext);

Execution pipeline = pipelineBuilder
.withLimitConcurrent(false)
.build();

executionRepository.store(pipeline);

try {
executionLauncher.start(pipeline);
} catch (Throwable t) {
handleStartupFailure(pipeline, t);
}

return CanaryExecutionResponse.builder().canaryExecutionId(pipeline.getId()).build();
}

private void handleStartupFailure(Execution execution, Throwable failure) {
final String canceledBy = "system";
final String reason = "Failed on startup: " + failure.getMessage();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,13 @@
import com.netflix.spinnaker.orca.ExecutionStatus;
import com.netflix.spinnaker.orca.RetryableTask;
import com.netflix.spinnaker.orca.TaskResult;
import com.netflix.spinnaker.orca.pipeline.model.Execution;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.Nonnull;
import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -104,7 +102,8 @@ public TaskResult execute(@Nonnull Stage stage) {
CanaryJudge canaryJudge = null;

if (canaryJudgeConfig != null) {
String judgeName = canaryJudgeConfig.getName();
String overrideJudgeName = (String)context.get("overrideJudgeName");
String judgeName = StringUtils.isNotEmpty(overrideJudgeName) ? overrideJudgeName : canaryJudgeConfig.getName();

if (!StringUtils.isEmpty(judgeName)) {
canaryJudge =
Expand All @@ -127,7 +126,7 @@ public TaskResult execute(@Nonnull Stage stage) {

CanaryResult canaryResult = CanaryResult.builder()
.judgeResult(result)
.canaryDuration(canaryExecutionRequest.calculateDuration())
.canaryDuration(canaryExecutionRequest != null ? canaryExecutionRequest.calculateDuration() : null)
.build();

storageService.storeObject(resolvedStorageAccountName, ObjectType.CANARY_RESULT, canaryJudgeResultId, canaryResult);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Copyright 2018 Google, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.kayenta.canary.orca;

import com.netflix.spinnaker.orca.pipeline.StageDefinitionBuilder;
import com.netflix.spinnaker.orca.pipeline.TaskNode;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;

import javax.annotation.Nonnull;

@Component
public class CompareJudgeResultsStage {

@Bean
StageDefinitionBuilder compareJudgeResultsStageBuilder(){
return new StageDefinitionBuilder() {
@Override
public void taskGraph(@Nonnull Stage stage, @Nonnull TaskNode.Builder builder) {
builder.withTask("compareJudgeResults", CompareJudgeResultsTask.class);
}

@Nonnull
@Override
public String getType() {
return "compareJudgeResults";
}
};
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Copyright 2018 Google, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.kayenta.canary.orca;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.netflix.kayenta.canary.CanaryJudge;
import com.netflix.kayenta.canary.ExecutionMapper;
import com.netflix.kayenta.security.AccountCredentialsRepository;
import com.netflix.kayenta.storage.StorageServiceRepository;
import com.netflix.spinnaker.orca.ExecutionStatus;
import com.netflix.spinnaker.orca.RetryableTask;
import com.netflix.spinnaker.orca.TaskResult;
import com.netflix.spinnaker.orca.pipeline.model.Stage;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import javax.annotation.Nonnull;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.Map;

@Slf4j
@Component
public class CompareJudgeResultsTask implements RetryableTask {

private final AccountCredentialsRepository accountCredentialsRepository;
private final StorageServiceRepository storageServiceRepository;
private final List<CanaryJudge> canaryJudges;
private final ObjectMapper objectMapper;
private final ExecutionMapper executionMapper;

@Autowired
public CompareJudgeResultsTask(AccountCredentialsRepository accountCredentialsRepository,
StorageServiceRepository storageServiceRepository,
List<CanaryJudge> canaryJudges,
ObjectMapper kayentaObjectMapper,
ExecutionMapper executionMapper) {
this.accountCredentialsRepository = accountCredentialsRepository;
this.storageServiceRepository = storageServiceRepository;
this.canaryJudges = canaryJudges;
this.objectMapper = kayentaObjectMapper;
this.executionMapper = executionMapper;
}

@Override
public long getBackoffPeriod() {
// TODO(duftler): Externalize this configuration.
return Duration.ofSeconds(2).toMillis();
}

@Override
public long getTimeout() {
// TODO(duftler): Externalize this configuration.
return Duration.ofMinutes(2).toMillis();
}

@Nonnull
@Override
public TaskResult execute(@Nonnull Stage stage) {
Map<String, Object> context = stage.getContext();
Map judge1Result = (Map)context.get("judge1Result");
Map judge2Result = (Map)context.get("judge2Result");

// TODO: Now that the plumbing works, perform some kind of actual comparison.
Map<String, Map> comparisonResult =
ImmutableMap.<String, Map>builder()
.put("judge1Result", judge1Result)
.put("judge2Result", judge2Result)
.build();
Map<String, Map> outputs = Collections.singletonMap("comparisonResult", comparisonResult);

return new TaskResult(ExecutionStatus.SUCCEEDED, Collections.emptyMap(), outputs);
}
}
Loading

0 comments on commit 9fb40f4

Please sign in to comment.