Merge pull request cpputest#1272 from nokia/fail-for-no-run

Return failure if no tests are run or ignored
syncclk · Dec 27, 2019 · 6547d2c · 6547d2c
2 parents bc26541 + 4163c84
commit 6547d2c
Show file tree

Hide file tree

Showing 6 changed files with 91 additions and 16 deletions.
diff --git a/include/CppUTest/TestResult.h b/include/CppUTest/TestResult.h
@@ -85,6 +85,11 @@ class TestResult
         return failureCount_;
     }
 
+    bool isFailure() const
+    {
+        return (getFailureCount() != 0) || (getRunCount() + getIgnoredCount() == 0);
+    }
+
     long getTotalExecutionTime() const;
     void setTotalExecutionTime(long exTime);
 

diff --git a/src/CppUTest/CommandLineTestRunner.cpp b/src/CppUTest/CommandLineTestRunner.cpp
@@ -113,8 +113,9 @@ int CommandLineTestRunner::runAllTests()
 {
     initializeTestRun();
     int loopCount = 0;
-    int failureCount = 0;
-    int repeat_ = arguments_->getRepeatCount();
+    int failedTestCount = 0;
+    int failedExecutionCount = 0;
+    int repeatCount = arguments_->getRepeatCount();
 
     if (arguments_->isListingTestGroupNames())
     {
@@ -139,17 +140,20 @@ int CommandLineTestRunner::runAllTests()
         output_->print("\n");
         srand(seed);
     }
-    while (loopCount++ < repeat_) {
+    while (loopCount++ < repeatCount) {
         if (shuffleEnabled)
         {
             registry_->shuffleRunOrder(rand_);
         }
-        output_->printTestRun(loopCount, repeat_);
+        output_->printTestRun(loopCount, repeatCount);
         TestResult tr(*output_);
         registry_->runAllTests(tr);
-        failureCount += tr.getFailureCount();
+        failedTestCount += tr.getFailureCount();
+        if (tr.isFailure()) {
+            failedExecutionCount++;
+        }
     }
-    return failureCount;
+    return failedTestCount != 0 ? failedTestCount : failedExecutionCount;
 }
 
 TestOutput* CommandLineTestRunner::createTeamCityOutput()

diff --git a/src/CppUTest/TestOutput.cpp b/src/CppUTest/TestOutput.cpp
@@ -145,14 +145,20 @@ void TestOutput::printCurrentGroupEnded(const TestResult& /*res*/)
 void TestOutput::printTestsEnded(const TestResult& result)
 {
     print("\n");
-    const bool anyTestFailed = result.getFailureCount() > 0;
-    if (anyTestFailed) {
+    const bool isFailure = result.isFailure();
+    const int failureCount = result.getFailureCount();
+    if (isFailure) {
         if (color_) {
             print("\033[31;1m");
         }
         print("Errors (");
-        print(result.getFailureCount());
-        print(" failures, ");
+        if (failureCount > 0) {
+            print(failureCount);
+            print(" failures, ");
+        }
+        else {
+            print("ran nothing, ");
+        }
     }
     else {
         if (color_) {
@@ -170,7 +176,7 @@ void TestOutput::printTestsEnded(const TestResult& result)
     print(" ignored, ");
     print(result.getFilteredOutCount());
     print(" filtered out, ");
-    if (shuffleSeed_ != SHUFFLE_DISABLED && (verbose_ || anyTestFailed)) {
+    if (shuffleSeed_ != SHUFFLE_DISABLED && (verbose_ || isFailure)) {
         print("shuffle seed was: ");
         print(shuffleSeed_);
         print(", ");
@@ -180,6 +186,10 @@ void TestOutput::printTestsEnded(const TestResult& result)
     if (color_) {
         print("\033[m");
     }
+    if (isFailure && failureCount == 0) {
+        print("\nNote: test run failed because no tests were run or ignored. Assuming something went wrong. "
+              "This often happens because of linking errors or typos in test filter.");
+    }
     print("\n\n");
 
     dotCount_ = 0;

diff --git a/tests/CppUTest/CommandLineTestRunnerTest.cpp b/tests/CppUTest/CommandLineTestRunnerTest.cpp
@@ -159,14 +159,14 @@ TEST(CommandLineTestRunner, ReturnsZeroWhenNoErrors)
     LONGS_EQUAL(0, returned);
 }
 
-TEST(CommandLineTestRunner, ReturnsZeroWhenNoTestsMatchProvidedFilter)
+TEST(CommandLineTestRunner, ReturnsOneWhenNoTestsMatchProvidedFilter)
 {
     const char* argv[] = { "tests.exe", "-g", "NoSuchGroup"};
 
     CommandLineTestRunnerWithStringBufferOutput commandLineTestRunner(3, argv, &registry);
     int returned = commandLineTestRunner.runAllTestsMain();
 
-    LONGS_EQUAL(0, returned);
+    LONGS_EQUAL(1, returned);
 }
 
 TEST(CommandLineTestRunner, TeamcityOutputEnabled)

diff --git a/tests/CppUTest/TestOutputTest.cpp b/tests/CppUTest/TestOutputTest.cpp
@@ -76,6 +76,12 @@ TEST_GROUP(TestOutput)
         delete f3;
         delete result;
     }
+
+    void runOneTest()
+    {
+        result->countTest();
+        result->countRun();
+    }
 };
 
 TEST(TestOutput, PrintConstCharStar)
@@ -120,16 +126,18 @@ TEST(TestOutput, PrintTestALot)
 TEST(TestOutput, PrintTestALotAndSimulateRepeatRun)
 {
     for (int i = 0; i < 60; ++i) {
+        runOneTest();
         printer->printCurrentTestEnded(*result);
     }
 
     printer->printTestsEnded(*result);
 
     for (int i = 0; i < 60; ++i) {
+        runOneTest();
         printer->printCurrentTestEnded(*result);
     }
     STRCMP_EQUAL("..................................................\n.........." \
-        "\nOK (0 tests, 0 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)\n\n" \
+        "\nOK (60 tests, 60 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)\n\n" \
         "..................................................\n..........", mock->getOutput().asCharString());
 }
 
@@ -164,17 +172,21 @@ TEST(TestOutput, PrintTestVerboseEnded)
 TEST(TestOutput, printColorWithSuccess)
 {
     mock->color();
+    runOneTest();
     printer->printTestsEnded(*result);
-    STRCMP_EQUAL("\n\033[32;1mOK (0 tests, 0 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)\033[m\n\n", mock->getOutput().asCharString());
+    STRCMP_EQUAL("\n\033[32;1mOK (1 tests, 1 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)\033[m\n\n",
+        mock->getOutput().asCharString());
 }
 
 TEST(TestOutput, printColorWithFailures)
 {
     mock->color();
+    runOneTest();
     result->addFailure(*f);
     printer->flush();
     printer->printTestsEnded(*result);
-    STRCMP_EQUAL("\n\033[31;1mErrors (1 failures, 0 tests, 0 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)\033[m\n\n", mock->getOutput().asCharString());
+    STRCMP_EQUAL("\n\033[31;1mErrors (1 failures, 1 tests, 1 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)"
+                 "\033[m\n\n", mock->getOutput().asCharString());
 }
 
 TEST(TestOutput, PrintTestRun)
@@ -250,6 +262,17 @@ TEST(TestOutput, printTestsEndedWithFailures)
     STRCMP_EQUAL("\nErrors (1 failures, 0 tests, 0 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)\n\n", mock->getOutput().asCharString());
 }
 
+TEST(TestOutput, printTestsEndedWithNoTestsRunOrIgnored)
+{
+    result->countTest();
+    printer->flush();
+    printer->printTestsEnded(*result);
+    STRCMP_EQUAL("\nErrors (ran nothing, 1 tests, 0 ran, 0 checks, 0 ignored, 0 filtered out, 10 ms)\n"
+                 "Note: test run failed because no tests were run or ignored. Assuming something went wrong. "
+                 "This often happens because of linking errors or typos in test filter.\n\n",
+        mock->getOutput().asCharString());
+}
+
 class CompositeTestOutputTestStringBufferTestOutput : public StringBufferTestOutput
 {
   public:

diff --git a/tests/CppUTest/TestResultTest.cpp b/tests/CppUTest/TestResultTest.cpp
@@ -64,3 +64,36 @@ TEST(TestResult, TestEndedWillPrintResultsAndExecutionTime)
     res->testsEnded();
     CHECK(mock->getOutput().contains("10 ms"));
 }
+
+TEST(TestResult, ResultIsOkIfTestIsRunWithNoFailures)
+{
+    res->countTest();
+    res->countRun();
+    CHECK_FALSE(res->isFailure());
+}
+
+TEST(TestResult, ResultIsOkIfTestIsIgnored)
+{
+    res->countTest();
+    res->countIgnored();
+    CHECK_FALSE(res->isFailure());
+}
+
+TEST(TestResult, ResultIsNotOkIfFailures)
+{
+    res->countTest();
+    res->countRun();
+    res->addFailure(TestFailure(UtestShell::getCurrent(), StringFrom("dummy message")));
+    CHECK_TRUE(res->isFailure());
+}
+
+TEST(TestResult, ResultIsNotOkIfNoTestsAtAll)
+{
+    CHECK_TRUE(res->isFailure());
+}
+
+TEST(TestResult, ResultIsNotOkIfNoTestsRunOrIgnored)
+{
+    res->countTest();
+    CHECK_TRUE(res->isFailure());
+}