Skip to content

Commit

Permalink
Add Servo dependency. Add JMX/Servo metrics monitoring for monkey run…
Browse files Browse the repository at this point in the history 
…s, marked resources, unmarked resources, cleaned resources, and failed to clean resources.
  • Loading branch information
@ebukoski
ebukoski committed Nov 2, 2015
1 parent ae4ed80 commit 992a449
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 5 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ dependencies {
compile 'org.apache.jclouds.api:ec2:1.9.0'
compile 'org.apache.jclouds.provider:aws-ec2:1.9.0'
compile 'ch.qos.logback:logback-classic:1.0.13'

compile 'com.netflix.servo:servo-core:0.9.4'

testCompile 'org.testng:testng:6.3.1'
testCompile 'org.mockito:mockito-core:1.8.5'
Expand Down
16 changes: 16 additions & 0 deletions src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,15 @@

import java.util.Collection;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.netflix.servo.annotations.DataSourceType;
import com.netflix.servo.annotations.Monitor;
import com.netflix.servo.monitor.Monitors;
import com.netflix.simianarmy.MonkeyCalendar;
import com.netflix.simianarmy.MonkeyConfiguration;
import com.netflix.simianarmy.MonkeyRecorder;
Expand Down Expand Up @@ -60,6 +64,9 @@ public class BasicJanitorMonkey extends JanitorMonkey {
private final MonkeyRecorder recorder;

private final MonkeyCalendar calendar;

/** Keep track of the number of monkey runs */
protected final AtomicLong monkeyRuns = new AtomicLong(0);

/**
* Instantiates a new basic janitor monkey.
Expand All @@ -78,6 +85,9 @@ public BasicJanitorMonkey(Context ctx) {
resourceTracker = ctx.resourceTracker();
recorder = ctx.recorder();
calendar = ctx.calendar();

// register this janitor with servo
Monitors.registerObject("simianarmy.janitor", this);
}

/** {@inheritDoc} */
Expand All @@ -90,6 +100,7 @@ public void doMonkeyBusiness() {
return;
} else {
LOGGER.info(String.format("Marking resources with %d janitors.", janitors.size()));
monkeyRuns.incrementAndGet();
for (AbstractJanitor janitor : janitors) {
LOGGER.info(String.format("Running janitor for region %s", janitor.getRegion()));
janitor.markResources();
Expand Down Expand Up @@ -221,4 +232,9 @@ private boolean isJanitorMonkeyEnabled() {
LOGGER.info("JanitorMonkey disabled, set {}=true", prop);
return false;
}

@Monitor(name="runs", type=DataSourceType.COUNTER)
public AtomicLong getMonkeyRuns() {
return monkeyRuns;
}
}
58 changes: 57 additions & 1 deletion src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
package com.netflix.simianarmy.janitor;

import com.google.common.collect.Maps;
import com.netflix.servo.annotations.DataSourceType;
import com.netflix.servo.annotations.Monitor;
import com.netflix.servo.annotations.MonitorTags;
import com.netflix.simianarmy.MonkeyCalendar;
import com.netflix.simianarmy.MonkeyConfiguration;
import com.netflix.simianarmy.MonkeyRecorder;
Expand All @@ -41,6 +44,11 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;

import com.netflix.servo.monitor.Monitors;
import com.netflix.servo.tag.BasicTagList;
import com.netflix.servo.tag.TagList;

/**
* An abstract implementation of Janitor. It marks resources that the rule engine considers
Expand All @@ -55,12 +63,28 @@ public abstract class AbstractJanitor implements Janitor {
/** The Constant LOGGER. */
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJanitor.class);

/** Keep track of the number of cleaned resources */
protected final AtomicLong cleanedResourcesCount = new AtomicLong(0);

/** Keep track of the number of marked resources */
protected final AtomicLong markedResourcesCount = new AtomicLong(0);

/** Keep track of the number of failed to clean resources */
protected final AtomicLong failedToCleanResourcesCount = new AtomicLong(0);

/** Keep track of the number of unmarked resources */
protected final AtomicLong unmarkedResourcesCount = new AtomicLong(0);

/** Tags to attach to servo metrics */
@MonitorTags
protected TagList tags;

private final String region;
/** The region the janitor is running in. */
public String getRegion() {
return region;
}

/**
* The rule engine used to decide if a resource should be a cleanup
* candidate.
Expand Down Expand Up @@ -183,6 +207,13 @@ public AbstractJanitor(Context ctx, ResourceType resourceType) {
Validate.notNull(resourceType);
// recorder could be null and no events are recorded when it is.
this.recorder = ctx.recorder();

// setup servo tags, currently just tag each published metric with the region
this.tags = BasicTagList.of("simianarmy.janitor.region", ctx.region());

// register this janitor with servo
String monitorObjName = String.format("simianarmy.janitor.%s.%s", this.resourceType.name(), this.region);
Monitors.registerObject(monitorObjName, this);
}

@Override
Expand Down Expand Up @@ -222,6 +253,7 @@ public void markResources() {
recorder.recordEvent(evt);
}
resourceTracker.addOrUpdate(resource);
markedResourcesCount.incrementAndGet();
postMark(resource);
} else {
LOGGER.info(String.format(
Expand All @@ -247,6 +279,7 @@ public void markResources() {
resource.getId()));
}
unmarkedResources.add(resource);
unmarkedResourcesCount.incrementAndGet();
}
}

Expand Down Expand Up @@ -291,13 +324,15 @@ public void cleanupResources() {
recorder.recordEvent(evt);
}
cleanup(markedResource);
cleanedResourcesCount.incrementAndGet();
markedResource.setActualTerminationTime(now);
markedResource.setState(Resource.CleanupState.JANITOR_TERMINATED);
resourceTracker.addOrUpdate(markedResource);
} catch (Exception e) {
LOGGER.error(String.format("Failed to clean up the resource %s of type %s.",
markedResource.getId(), markedResource.getResourceType().name()), e);
failedToCleanResources.add(markedResource);
failedToCleanResourcesCount.incrementAndGet();
continue;
}
postCleanup(markedResource);
Expand Down Expand Up @@ -392,7 +427,28 @@ private void unmarkUserTerminatedResources(
markedResource.getId()));
}
unmarkedResources.add(markedResource);
unmarkedResourcesCount.incrementAndGet();
}
}
}

@Monitor(name="cleanedResourcesCount", type=DataSourceType.COUNTER)
long getResourcesCleanedCount() {
return cleanedResourcesCount.get();
}

@Monitor(name="markedResourcesCount", type=DataSourceType.COUNTER)
long getMarkedResourcesCount() {
return markedResourcesCount.get();
}

@Monitor(name="failedToCleanResourcesCount", type=DataSourceType.COUNTER)
long getFailedToCleanResourcesCount() {
return failedToCleanResourcesCount.get();
}

@Monitor(name="unmarkedResourcesCount", type=DataSourceType.COUNTER)
long getUnmarkedResourcesCount() {
return unmarkedResourcesCount.get();
}
}
35 changes: 32 additions & 3 deletions src/test/java/com/netflix/simianarmy/janitor/TestAbstractJanitor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,10 @@ public static void testJanitor() {
Assert.assertEquals(janitor.cleanedResourceIds.size(), n / 2);
for (int i = 1; i <= n; i += 2) {
Assert.assertTrue(janitor.cleanedResourceIds.contains(String.valueOf(i)));
}
}
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}

@Test
Expand Down Expand Up @@ -196,7 +199,11 @@ public static void testJanitorWithOptedOutResources() {
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(resourceTracker.getResources(
TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(),
2);
2);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
Assert.assertEquals(janitor.getUnmarkedResourcesCount(), 3);
}

@Test
Expand All @@ -222,7 +229,10 @@ public static void testJanitorWithCleanupFailure() {

janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - 1);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 1);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 1);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 1);
}

@Test
Expand Down Expand Up @@ -271,6 +281,10 @@ public static void testJanitorWithUnmarking() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), n / 2);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
Assert.assertEquals(janitor.getUnmarkedResourcesCount(), n/6);
}


Expand Down Expand Up @@ -310,6 +324,9 @@ public static void testJanitorWithFutureTerminationTime() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), 0);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}


Expand Down Expand Up @@ -346,6 +363,9 @@ public static void testJanitorWithoutNotification() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), 0);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}

@Test
Expand All @@ -370,6 +390,8 @@ public static void testLeashedJanitorForMarking() {
n);
janitor.markResources();
Assert.assertEquals(janitor.getMarkedResources().size(), n / 2);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());

// No resource is really changed in tracker
Assert.assertEquals(resourceTracker.getResources(
Expand Down Expand Up @@ -415,6 +437,9 @@ public static void testJanitorWithoutHoldingOffCleanup() {
TestResourceType.TEST_RESOURCE_TYPE, CleanupState.JANITOR_TERMINATED, TEST_REGION).size(),
n);
Assert.assertEquals(janitor.cleanedResourceIds.size(), n);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
}

@Test
Expand Down Expand Up @@ -461,6 +486,10 @@ public static void testJanitorWithUnmarkingUserTerminated() {
janitor.cleanupResources();
Assert.assertEquals(janitor.getCleanedResources().size(), n / 2 - n / 3 + n / 6);
Assert.assertEquals(janitor.getFailedToCleanResources().size(), 0);
Assert.assertEquals(janitor.getResourcesCleanedCount(), janitor.cleanedResourceIds.size());
Assert.assertEquals(janitor.getMarkedResourcesCount(), janitor.markedResourceIds.size());
Assert.assertEquals(janitor.getFailedToCleanResourcesCount(), 0);
Assert.assertEquals(janitor.getUnmarkedResourcesCount(), n / 3);
}
}

Expand Down

0 comments on commit 992a449

Please sign in to comment.