Merge pull request Netflix#697 from mattrjacobs/fallback-happening-on…

…-main-thread Missing execution event for FALLBACK_REJECTION
eason-li · Feb 24, 2015 · e56d4b9 · e56d4b9
2 parents a329f13 + c75b8a2
commit e56d4b9
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 1 deletion.
diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java b/hystrix-core/src/main/java/com/netflix/hystrix/AbstractCommand.java
@@ -720,7 +720,7 @@ public void call() {
                         });
             } else {
                 metrics.markFallbackRejection();
-
+                executionResult = executionResult.addEvents(HystrixEventType.FALLBACK_REJECTION);
                 logger.debug("HystrixCommand Fallback Rejection."); // debug only since we're throwing the exception and someone higher will do something with it
                 // if we couldn't acquire a permit, we "fail fast" by throwing an exception
                 return Observable.error(new HystrixRuntimeException(FailureType.REJECTED_SEMAPHORE_FALLBACK, this.getClass(), getLogMessagePrefix() + " fallback execution rejected.", null, null));

diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/HystrixCommandTest.java
@@ -3634,6 +3634,50 @@ public void run() {
         assertEquals(0, circuitBreaker.metrics.getCumulativeCount(HystrixRollingNumberEvent.RESPONSE_FROM_CACHE));
     }
 
+    static class EventCommand extends HystrixCommand {
+        public EventCommand() {
+            super(Setter.withGroupKey(HystrixCommandGroupKey.Factory.asKey("eventGroup")).andCommandPropertiesDefaults(new HystrixCommandProperties.Setter().withFallbackIsolationSemaphoreMaxConcurrentRequests(3)));
+        }
+
+        @Override
+        protected String run() throws Exception {
+            System.out.println(Thread.currentThread().getName() + " : In run()");
+            throw new RuntimeException("run_exception");
+        }
+
+        @Override
+        public String getFallback() {
+            try {
+                System.out.println(Thread.currentThread().getName() + " : In fallback => " + getExecutionEvents());
+                Thread.sleep(30000L);
+            } catch (InterruptedException e) {
+                System.out.println(Thread.currentThread().getName() + " : Interruption occurred");
+            }
+            System.out.println(Thread.currentThread().getName() + " : CMD Success Result");
+            return "fallback";
+        }
+    }
+
+    //if I set fallback semaphore to same as threadpool (10), I set up a race.
+    //instead, I set fallback sempahore to much less (3).  This should guarantee that all fallbacks only happen in the threadpool, and main thread does not block
+    @Test(timeout=5000)
+    public void testFallbackRejection() throws InterruptedException, ExecutionException {
+        for (int i = 0; i < 1000; i++) {
+            EventCommand cmd = new EventCommand();
+
+            try {
+                if (i == 500) {
+                    Thread.sleep(100L);
+                }
+                cmd.queue();
+                System.out.println("queued: " + i);
+            } catch (Exception e) {
+                System.out.println("Fail Fast on queue() : " + cmd.getExecutionEvents());
+
+            }
+        }
+    }
+
     @Test
     public void testNonBlockingCommandQueueFiresTimeout() { //see https://github.com/Netflix/Hystrix/issues/514
         final TestHystrixCommand<?> cmd = getCommand(ExecutionIsolationStrategy.THREAD, AbstractTestHystrixCommand.ExecutionResult.SUCCESS, 200, AbstractTestHystrixCommand.FallbackResult.SUCCESS, 50);