bugfix: fix thread pool full experiment cannot be stopped when uninstall directly

Author: xcaspar

chaosblade-exec-bootstrap/chaosblade-exec-bootstrap-jvmsandbox/src/main/java/com/alibaba/chaosblade/exec/bootstrap/jvmsandbox/SandboxModule.java

@@ -87,9 +87,10 @@ public void onLoad() throws Throwable {
     @Override
     public void onUnload() throws Throwable {
         LOGGER.info("unload chaosblade module");
-        watchIds.clear();
         dispatchService.unload();
         ManagerFactory.unload();
+        watchIds.clear();
+        LOGGER.info("unload chaosblade module successfully");
     }
 
     @Override

chaosblade-exec-common/src/main/java/com/alibaba/chaosblade/exec/common/center/DefaultStatusManager.java

@@ -17,7 +17,9 @@
 package com.alibaba.chaosblade.exec.common.center;
 
 import java.util.Collections;
+import java.util.Enumeration;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -165,6 +167,16 @@ public StatusMetric getStatusMetricByUid(String uid) {
         return metricMap.get(identifier);
     }
 
+    @Override
+    public Set<String> getAllUids() {
+        Enumeration<String> keys = experiments.keys();
+        HashSet<String> uids = new HashSet<String>();
+        while (keys.hasMoreElements()) {
+            uids.add(keys.nextElement());
+        }
+        return uids;
+    }
+
     @Override
     public void load() {
         closed = false;

chaosblade-exec-common/src/main/java/com/alibaba/chaosblade/exec/common/center/ManagerFactory.java

@@ -56,8 +56,8 @@ public static void load() {
      * Close manager service
      */
     public static void unload() {
+        statusManager.unload();
         modelSpecManager.unload();
         listenerManager.unload();
-        statusManager.unload();
     }
 }

chaosblade-exec-common/src/main/java/com/alibaba/chaosblade/exec/common/center/StatusManager.java

@@ -18,6 +18,7 @@
 
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import com.alibaba.chaosblade.exec.common.model.Model;
 
@@ -80,4 +81,9 @@ public interface StatusManager extends ManagerService {
      */
     StatusMetric getStatusMetricByUid(String uid);
 
+    /**
+     * @return all running experiment uids
+     */
+    Set<String> getAllUids();
+
 }

chaosblade-exec-common/src/main/java/com/alibaba/chaosblade/exec/common/transport/Response.java

@@ -146,8 +146,8 @@ public enum Code {
         NOT_FOUND(404, "request handler not found"),
         ILLEGAL_PARAMETER(405, "illegal parameter"),
         DUPLICATE_INJECTION(406, "duplicate injection"),
-        SERVER_ERROR(500, "server error");
-
+        SERVER_ERROR(500, "server error"),
+        ILLEGAL_STATE(504, "illegal state");
         private int code;
         private String msg;
 

chaosblade-exec-service/src/main/java/com/alibaba/chaosblade/exec/service/handler/CreateHandler.java

@@ -44,6 +44,7 @@ public class CreateHandler implements RequestHandler {
 
     private ModelSpecManager modelSpecManager;
     private StatusManager statusManager;
+    private volatile boolean unloaded;
 
     public CreateHandler() {
         this.modelSpecManager = ManagerFactory.getModelSpecManager();
@@ -63,6 +64,9 @@ public String getHandlerName() {
      */
     @Override
     public Response handle(Request request) {
+        if (unloaded) {
+            return Response.ofFailure(Code.ILLEGAL_STATE, "the agent is uninstalling");
+        }
         // check necessary arguments
         String suid = request.getParam("suid");
         if (StringUtil.isBlank(suid)) {
@@ -98,6 +102,11 @@ public Response handle(Request request) {
         return handleInjection(suid, model, modelSpec);
     }
 
+    @Override
+    public void unload() {
+        unloaded = true;
+    }
+
     /**
      * Set log level to debug if debug parameter is true
      *

chaosblade-exec-service/src/main/java/com/alibaba/chaosblade/exec/service/handler/DefaultDispatchService.java

@@ -16,6 +16,8 @@
 
 package com.alibaba.chaosblade.exec.service.handler;
 
+import java.util.Map.Entry;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.alibaba.chaosblade.exec.common.transport.Request;
@@ -71,6 +73,10 @@ public void load() {
 
     @Override
     public void unload() {
+        Set<Entry<String, RequestHandler>> entries = handles.entrySet();
+        for (Entry<String, RequestHandler> entry : entries) {
+            entry.getValue().unload();
+        }
         handles.clear();
     }
 }

chaosblade-exec-service/src/main/java/com/alibaba/chaosblade/exec/service/handler/DestroyHandler.java

@@ -16,6 +16,8 @@
 
 package com.alibaba.chaosblade.exec.service.handler;
 
+import java.util.Set;
+
 import com.alibaba.chaosblade.exec.common.center.ManagerFactory;
 import com.alibaba.chaosblade.exec.common.center.ModelSpecManager;
 import com.alibaba.chaosblade.exec.common.center.StatusManager;
@@ -28,11 +30,14 @@
 import com.alibaba.chaosblade.exec.common.transport.Response.Code;
 import com.alibaba.chaosblade.exec.common.util.StringUtil;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * @author Changjun Xiao
  */
 public class DestroyHandler implements RequestHandler {
-
+    private static final Logger LOGGER = LoggerFactory.getLogger(DestroyHandler.class);
     private ModelSpecManager modelSpecManager;
     private StatusManager statusManager;
 
@@ -52,6 +57,10 @@ public Response handle(Request request) {
         if (StringUtil.isBlank(uid)) {
             return Response.ofFailure(Code.ILLEGAL_PARAMETER, "less experiment uid");
         }
+        return destroy(uid);
+    }
+
+    private Response destroy(String uid) {
         Model model = statusManager.removeExp(uid);
 
         if (model == null) {
@@ -64,11 +73,24 @@ public Response handle(Request request) {
         try {
             applyPreDestroyInjectionModelHandler(uid, modelSpec, model);
         } catch (ExperimentException ex) {
-            return Response.ofFailure(Response.Code.SERVER_ERROR, ex.getMessage());
+            return Response.ofFailure(Code.SERVER_ERROR, ex.getMessage());
         }
         return Response.ofSuccess("success");
     }
 
+    @Override
+    public void unload() {
+        Set<String> uids = statusManager.getAllUids();
+        for (String uid : uids) {
+            Response response = destroy(uid);
+            if (response.isSuccess()) {
+                LOGGER.debug("destroy {} successfully when unload", uid);
+            } else {
+                LOGGER.warn("destroy {} failed because of {} when unload", uid, response.getError());
+            }
+        }
+    }
+
     private void applyPreDestroyInjectionModelHandler(String uid, ModelSpec modelSpec, Model model)
         throws ExperimentException {
         if (modelSpec instanceof PreDestroyInjectionModelHandler) {

chaosblade-exec-service/src/main/java/com/alibaba/chaosblade/exec/service/handler/RequestHandler.java

@@ -26,4 +26,6 @@ public interface RequestHandler {
     String getHandlerName();
 
     Response handle(Request paramRequest);
+
+    void unload();
 }

chaosblade-exec-service/src/main/java/com/alibaba/chaosblade/exec/service/handler/StatusHandler.java

@@ -51,4 +51,8 @@ public Response handle(Request request) {
         }
         return Response.ofSuccess(String.valueOf(statusMetric.getCount()));
     }
+
+    @Override
+    public void unload() {
+    }
 }