Merge pull request wildfly#10673 from rhusar/WFLY-9449v2

kabir · web-flow · commit d1337a2f0704 · 2017-11-30T13:00:11.000Z
WFLY-9449 SingletonPartitionTestCase can fail intermittently
diff --git a/testsuite/integration/clustering/src/test/java/org/jboss/as/test/clustering/cluster/singleton/SingletonPartitionTestCase.java b/testsuite/integration/clustering/src/test/java/org/jboss/as/test/clustering/cluster/singleton/SingletonPartitionTestCase.java
@@ -49,6 +49,7 @@
 import org.jboss.as.test.clustering.cluster.singleton.service.NodeService;
 import org.jboss.as.test.clustering.cluster.singleton.service.NodeServiceServlet;
 import org.jboss.as.test.http.util.TestHttpClientUtils;
+import org.jboss.as.test.shared.TimeoutUtil;
 import org.jboss.msc.service.ServiceActivator;
 import org.jboss.shrinkwrap.api.Archive;
 import org.jboss.shrinkwrap.api.ShrinkWrap;
@@ -74,7 +75,8 @@
 @RunAsClient
 public class SingletonPartitionTestCase extends ClusterAbstractTestCase {
 
-    private static final long TOPOLOGY_CHANGE_TIMEOUT = 120000; // maximum time in ms to wait for cluster topology change
+    private static final long TOPOLOGY_CHANGE_TIMEOUT = TimeoutUtil.adjust(120_000); // maximum time in ms to wait for cluster topology change
+    private static final int SERVICE_TIMEOUT = TimeoutUtil.adjust(5_000); // it takes a little extra time after merge for the singleton service to migrate
     private static final String CONTAINER = "server";
 
     @Deployment(name = DEPLOYMENT_1, managed = false, testable = false)
@@ -126,7 +128,7 @@ public void testSingletonService(
 
         waitForView(baseURL1, NODE_1, NODE_2);
         waitForView(baseURL2, NODE_1, NODE_2);
-        Thread.sleep(5000); // it takes a little extra time after merge for the singleton service to migrate
+        Thread.sleep(SERVICE_TIMEOUT);
 
 
         // check service A
@@ -142,7 +144,7 @@ public void testSingletonService(
         partition(true, baseURL1, baseURL2);
         waitForView(baseURL1, NODE_1);
         waitForView(baseURL2, NODE_2);
-        Thread.sleep(5000);
+        Thread.sleep(SERVICE_TIMEOUT);
 
         // check service A
         checkSingletonNode(serviceANode1Uri, NODE_1);
@@ -157,7 +159,7 @@ public void testSingletonService(
         partition(false, baseURL1, baseURL2);
         waitForView(baseURL1, NODE_1, NODE_2);
         waitForView(baseURL2, NODE_1, NODE_2);
-        Thread.sleep(5000);
+        Thread.sleep(SERVICE_TIMEOUT);
 
         // check service A
         checkSingletonNode(serviceANode1Uri, SingletonServiceActivator.SERVICE_A_PREFERRED_NODE);
@@ -172,7 +174,7 @@ public void testSingletonService(
         partition(true, baseURL1, baseURL2);
         waitForView(baseURL1, NODE_1);
         waitForView(baseURL2, NODE_2);
-        Thread.sleep(5000);
+        Thread.sleep(SERVICE_TIMEOUT);
 
         // check service A
         checkSingletonNode(serviceANode1Uri, NODE_1);
@@ -184,10 +186,10 @@ public void testSingletonService(
 
     @Override
     public void afterTestMethod() {
-        // Stop the container to ensure there aren't any remnants since the test operates on a live JGroups channel
-        stop(CONTAINERS);
-
         super.afterTestMethod();
+
+        // Stop the container to ensure there aren't any remnants since the test operates on a live JGroups channels
+        stop(CONTAINERS);
     }
 
     private static void checkSingletonNode(URI serviceUri, String expectedProviderNode) throws IOException {
diff --git a/testsuite/integration/clustering/src/test/java/org/jboss/as/test/clustering/cluster/singleton/partition/PartitionerServlet.java b/testsuite/integration/clustering/src/test/java/org/jboss/as/test/clustering/cluster/singleton/partition/PartitionerServlet.java
@@ -26,6 +26,8 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
 
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;
@@ -36,25 +38,36 @@
 import org.jboss.as.server.CurrentServiceContainer;
 import org.jboss.msc.service.ServiceController;
 import org.jboss.msc.service.ServiceName;
+import org.jgroups.Address;
 import org.jgroups.Channel;
+import org.jgroups.Event;
 import org.jgroups.View;
 import org.jgroups.protocols.DISCARD;
 import org.jgroups.protocols.TP;
 import org.jgroups.protocols.pbcast.GMS;
 import org.jgroups.stack.ProtocolStack;
 
 /**
- * Servlet used to simulate network partitions. Responds to {@code /partitioner?partition=true} by inserting DISCARD protocol to the
- * stack and {@code /partitioner?partition=false} by removing previously inserted DISCARD protocol.
+ * Servlet used to simulate network partitions. Responds to {@code /partitioner?partition=true} by inserting DISCARD protocol to the stack
+ * and installing new view directly on the {@link GMS} and responds to {@code /partitioner?partition=false} by removing previously inserted
+ * {@link DISCARD} protocol and passing MERGE event up the stack.
+ * <p>
+ * Note that while it would be desirable for the tests to leave splitting and merging to the servers themselves, this is not practical in a
+ * test suite. While FD/VERIFY_SUSPECT/GMS can be be configured to detect partitions quickly the MERGE3 handles merges within randomized
+ * intervals and uses unreliable channel which can easily take several minutes for merge to actually happen. Also, speeds up the test
+ * significantly.
  *
  * @author Radoslav Husar
  */
 @WebServlet(urlPatterns = {PartitionerServlet.SERVLET_PATH})
 public class PartitionerServlet extends HttpServlet {
 
+    private static final long VIEWS_TIMEOUT = 3_000;
     public static final String SERVLET_PATH = "partitioner";
     private static final String PARTITION = "partition";
 
+    private static Map<Address, View> mergeViews;
+
     public static URI createURI(URL baseURL, boolean partition) throws URISyntaxException {
         return baseURL.toURI().resolve(SERVLET_PATH + '?' + PARTITION + '=' + partition);
     }
@@ -71,17 +84,35 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t
             Channel channel = service.awaitValue();
 
             if (partition) {
+                // Store views for future merge event
+                GMS gms = (GMS) channel.getProtocolStack().findProtocol(GMS.class);
+                mergeViews = new HashMap<>();
+                channel.getView().getMembers().forEach(
+                        address -> mergeViews.put(address, View.create(address, gms.getViewId().getId() + 1, address))
+                );
+                // Wait a few seconds to ensure everyone stored a full view
+                Thread.sleep(VIEWS_TIMEOUT);
+
                 // Simulate partitions by injecting DISCARD protocol
                 DISCARD discard = new DISCARD();
                 discard.setDiscardAll(true);
                 channel.getProtocolStack().insertProtocol(discard, ProtocolStack.ABOVE, TP.class);
 
                 // Speed up partitioning
-                GMS gms = (GMS) channel.getProtocolStack().findProtocol(GMS.class);
                 View view = View.create(channel.getAddress(), gms.getViewId().getId() + 1, channel.getAddress());
                 gms.installView(view);
             } else {
                 channel.getProtocolStack().removeProtocol(DISCARD.class);
+                // Wait a few seconds for the other node to remove DISCARD so it does not discard our MERGE request
+                Thread.sleep(VIEWS_TIMEOUT);
+
+                // Since the coordinator is determined by ordering the address in org.jgroups.protocols.pbcast.Merger#determineMergeLeader
+                // let just all nodes send the merge..
+                this.log("Passing event up the stack: " + new Event(Event.MERGE, mergeViews));
+
+                GMS gms = (GMS) channel.getProtocolStack().findProtocol(GMS.class);
+                gms.up(new Event(Event.MERGE, mergeViews));
+                mergeViews = null;
             }
         } catch (Exception e) {
             throw new ServletException(e);
diff --git a/testsuite/integration/elytron/pom.xml b/testsuite/integration/elytron/pom.xml
@@ -55,18 +55,11 @@
             <artifactId>jboss-logmanager</artifactId>
             <scope>test</scope>
         </dependency>
-
         <dependency>
             <groupId>org.wildfly</groupId>
             <artifactId>wildfly-testsuite-shared</artifactId>
             <scope>test</scope>
         </dependency>
-
-        <dependency>
-            <groupId>org.jboss.arquillian.junit</groupId>
-            <artifactId>arquillian-junit-container</artifactId>
-            <scope>test</scope>
-        </dependency>
         <dependency>
             <groupId>org.wildfly.arquillian</groupId>
             <artifactId>wildfly-arquillian-container-managed</artifactId>
diff --git a/testsuite/integration/pom.xml b/testsuite/integration/pom.xml
@@ -371,6 +371,13 @@
 
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-antrun-plugin</artifactId>
+                <configuration>
+                    <skip>${skip.antrun}</skip>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 
diff --git a/testsuite/pom.xml b/testsuite/pom.xml
@@ -80,6 +80,8 @@
         <!-- This project's root dir. -->
         <jbossas.project.dir>${jbossas.ts.dir}/..</jbossas.project.dir>
 
+        <!-- Skips profile generation; to be used with running single test with -Dtest=... -->
+        <skip.antrun>false</skip.antrun>
 
         <!-- Used to provide an absolute location for the distribution under test. -->
         <!-- This value is overridden in modules with the correct relative pathname. -->
