feat(monitoring): Allow for executions.running to be tracked for a …

…specific set of applications (spinnaker#2220)

Calculating this metric for orchestrations can be a somewhat heavy
operation and has been left as an explicit opt-in.

```
pollers:
  enhancedMonitoring:
    enabled: true
    intervalMs: 60000
    applications:
      - my_application_foo
      - my_application_bar
```

In such a configuration, the # of `RUNNING` orchestrations will be
determined every 60s and emitted as the `executions.running` metric.

orca-web/src/main/groovy/com/netflix/spinnaker/config/EnhancedMonitoringConfiguration.java

@@ -0,0 +1,91 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.config;
+
+import com.google.common.collect.Lists;
+import com.netflix.spectator.api.Id;
+import com.netflix.spectator.api.Registry;
+import com.netflix.spinnaker.orca.ExecutionStatus;
+import com.netflix.spinnaker.orca.pipeline.model.Execution;
+import com.netflix.spinnaker.orca.pipeline.persistence.ExecutionRepository;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.Scheduled;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+@Configuration
+@EnableConfigurationProperties(EnhancedMonitoringConfigurationProperties.class)
+@ConditionalOnExpression(value = "${pollers.enhancedMonitoring.enabled:false}")
+public class EnhancedMonitoringConfiguration {
+  private final Logger log = LoggerFactory.getLogger(getClass());
+
+  private final Registry registry;
+  private final ExecutionRepository executionRepository;
+  private final EnhancedMonitoringConfigurationProperties configuration;
+
+  private final Map<String, AtomicLong> orchestrationCountPerApplication = new HashMap<>();
+
+  @Autowired
+  public EnhancedMonitoringConfiguration(Registry registry,
+                                         ExecutionRepository executionRepository,
+                                         EnhancedMonitoringConfigurationProperties configuration) {
+    this.registry = registry;
+    this.executionRepository = executionRepository;
+    this.configuration = configuration;
+
+    Id runningOrchestrationsId = registry
+      .createId("executions.running")
+      .withTag("executionType", "Orchestration"); // similar to what MetricsTagHelper is doing
+
+    for (String application : configuration.getApplications()) {
+      Id applicationSpecificId = runningOrchestrationsId.withTag("application", application);
+      orchestrationCountPerApplication.put(
+        application,
+        registry.gauge(applicationSpecificId, new AtomicLong(0))
+      );
+    }
+  }
+
+  @Scheduled(fixedDelayString = "${pollers.enhancedMonitoring.intervalMs:60000}")
+  void refresh() {
+    log.info("Refreshing Running Orchestration Counts ({})", orchestrationCountPerApplication);
+
+    for (String application : configuration.getApplications()) {
+      try {
+        List<Execution> executions = Lists.newArrayList(
+          executionRepository.retrieveOrchestrationsForApplication(
+            application,
+            new ExecutionRepository.ExecutionCriteria().setStatuses(ExecutionStatus.RUNNING)
+          )
+        );
+        orchestrationCountPerApplication.get(application).set(executions.size());
+      } catch (Exception e) {
+        log.error("Unable to refresh running orchestration count (application: {})", application, e);
+      }
+    }
+
+    log.info("Refreshed Running Orchestration Counts ({})", orchestrationCountPerApplication);
+  }
+}

orca-web/src/main/groovy/com/netflix/spinnaker/config/EnhancedMonitoringConfigurationProperties.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+import java.util.List;
+
+@ConfigurationProperties(prefix = "pollers.enhancedMonitoring")
+public class EnhancedMonitoringConfigurationProperties {
+  private List<String> applications;
+
+  public List<String> getApplications() {
+    return applications;
+  }
+
+  public void setApplications(List<String> applications) {
+    this.applications = applications;
+  }
+}

orca-web/src/main/groovy/com/netflix/spinnaker/orca/controllers/TaskController.groovy

@@ -16,6 +16,8 @@
 
 package com.netflix.spinnaker.orca.controllers
 
+import groovy.util.logging.Slf4j
+
 import com.netflix.spinnaker.orca.ExecutionStatus
 import com.netflix.spinnaker.orca.front50.Front50Service
 import com.netflix.spinnaker.orca.model.OrchestrationViewModel
@@ -50,6 +52,7 @@ import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.
 import static com.netflix.spinnaker.orca.pipeline.model.Execution.ExecutionType.PIPELINE
 import static java.time.ZoneOffset.UTC
 
+@Slf4j
 @RestController
 class TaskController {
   @Autowired(required = false)