Shuffle test execution order with -s

Shuffling is off by default. -s enables shuffling with random seed derived from current time. The seed is printed before executing tests. A specific test execution order can be reproduced by setting the seed explicitly with -s <seed>. Seed values 0 and 1 are reserved values and cannot be set. If -v (verbose) is enabled, the seed is also printed after test execution. This might be useful in case tests produce console output which could cause the seed printed at the beginning to be lost. Test order is reshuffled between repeats (-r). Co-authored-by: Risto-Matti Vuonnala <[email protected]>
jakubmiernik · Jan 11, 2019 · aa46da5 · aa46da5
1 parent 30e58f0
commit aa46da5
Show file tree

Hide file tree

Showing 12 changed files with 179 additions and 9 deletions.
diff --git a/.gitignore b/.gitignore
@@ -80,6 +80,10 @@ missing
 test-driver
 platforms/iar/settings
 cpputest_*.xml
+generated/
+*.deps/
+*.dirstamp
+*.trs
 
 #IAR automatically generated files
 *.dep

diff --git a/include/CppUTest/CommandLineArguments.h b/include/CppUTest/CommandLineArguments.h
@@ -47,6 +47,7 @@ class CommandLineArguments
     bool isListingTestGroupAndCaseNames() const;
     bool isRunIgnored() const;
     int getRepeatCount() const;
+    unsigned int getShuffle() const;
     const TestFilter* getGroupFilters() const;
     const TestFilter* getNameFilters() const;
     bool isJUnitOutput() const;
@@ -72,13 +73,15 @@ class CommandLineArguments
     bool listTestGroupAndCaseNames_;
     bool runIgnored_;
     int repeat_;
+    unsigned int shuffle_; // 0: shuffling disabled, 1: random seed, other values: specific random seed
     TestFilter* groupFilters_;
     TestFilter* nameFilters_;
     OutputType outputType_;
     SimpleString packageName_;
 
     SimpleString getParameterField(int ac, const char *const *av, int& i, const SimpleString& parameterName);
     void SetRepeatCount(int ac, const char *const *av, int& index);
+    void SetShuffle(int ac, const char *const *av, int& index);
     void AddGroupFilter(int ac, const char *const *av, int& index);
     void AddStrictGroupFilter(int ac, const char *const *av, int& index);
     void AddExcludeGroupFilter(int ac, const char *const *av, int& index);

diff --git a/include/CppUTest/StandardCLibrary.h b/include/CppUTest/StandardCLibrary.h
@@ -15,6 +15,8 @@
 #ifdef __cplusplus
  #if CPPUTEST_USE_STD_CPP_LIB
   #include <cstdlib>
+  #include <vector>
+  #include <algorithm>
  #endif
 #endif
 

diff --git a/include/CppUTest/TestOutput.h b/include/CppUTest/TestOutput.h
@@ -58,6 +58,7 @@ class TestOutput
 
     virtual void verbose();
     virtual void color();
+    virtual void setShuffleSeed(unsigned int);
     virtual void printBuffer(const char*)=0;
     virtual void print(const char*);
     virtual void print(long);
@@ -91,6 +92,7 @@ class TestOutput
     int dotCount_;
     bool verbose_;
     bool color_;
+    unsigned int shuffleSeed_;
     const char* progressIndication_;
 
     static WorkingEnvironment workingEnvironment_;

diff --git a/include/CppUTest/TestRegistry.h b/include/CppUTest/TestRegistry.h
@@ -50,6 +50,7 @@ class TestRegistry
     virtual void unDoLastAddTest();
     virtual int countTests();
     virtual void runAllTests(TestResult& result);
+    virtual void shuffleRunOrder();
     virtual void listTestGroupNames(TestResult& result);
     virtual void listTestGroupAndCaseNames(TestResult& result);
     virtual void setNameFilters(const TestFilter* filters);

diff --git a/include/CppUTest/TestResult.h b/include/CppUTest/TestResult.h
@@ -57,6 +57,7 @@ class TestResult
     virtual void countCheck();
     virtual void countFilteredOut();
     virtual void countIgnored();
+    virtual void setShuffleSeed(unsigned int);
     virtual void addFailure(const TestFailure& failure);
     virtual void print(const char* text);
 
@@ -84,6 +85,10 @@ class TestResult
     {
         return failureCount_;
     }
+    unsigned int getShuffleSeed() const
+    {
+        return shuffleSeed_;
+    }
 
     long getTotalExecutionTime() const;
     void setTotalExecutionTime(long exTime);
@@ -99,6 +104,7 @@ class TestResult
     int failureCount_;
     int filteredOutCount_;
     int ignoredCount_;
+    unsigned int shuffleSeed_;
     long totalExecutionTime_;
     long timeStarted_;
     long currentTestTimeStarted_;

diff --git a/src/CppUTest/CommandLineArguments.cpp b/src/CppUTest/CommandLineArguments.cpp
@@ -30,7 +30,7 @@
 #include "CppUTest/PlatformSpecificFunctions.h"
 
 CommandLineArguments::CommandLineArguments(int ac, const char *const *av) :
-    ac_(ac), av_(av), verbose_(false), color_(false), runTestsAsSeperateProcess_(false), listTestGroupNames_(false), listTestGroupAndCaseNames_(false), runIgnored_(false), repeat_(1), groupFilters_(NULLPTR), nameFilters_(NULLPTR), outputType_(OUTPUT_ECLIPSE)
+    ac_(ac), av_(av), verbose_(false), color_(false), runTestsAsSeperateProcess_(false), listTestGroupNames_(false), listTestGroupAndCaseNames_(false), runIgnored_(false), repeat_(1), shuffle_(0), groupFilters_(NULLPTR), nameFilters_(NULLPTR), outputType_(OUTPUT_ECLIPSE)
 {
 }
 
@@ -69,6 +69,7 @@ bool CommandLineArguments::parse(TestPlugin* plugin)
         else if (argument.startsWith("-sn")) AddStrictNameFilter(ac_, av_, i);
         else if (argument.startsWith("-xn")) AddExcludeNameFilter(ac_, av_, i);
         else if (argument.startsWith("-xsn")) AddExcludeStrictNameFilter(ac_, av_, i);
+        else if (argument.startsWith("-s")) SetShuffle(ac_, av_, i);
         else if (argument.startsWith("TEST(")) AddTestToRunBasedOnVerboseOutput(ac_, av_, i, "TEST(");
         else if (argument.startsWith("IGNORE_TEST(")) AddTestToRunBasedOnVerboseOutput(ac_, av_, i, "IGNORE_TEST(");
         else if (argument.startsWith("-o")) correctParameters = SetOutputType(ac_, av_, i);
@@ -85,7 +86,7 @@ bool CommandLineArguments::parse(TestPlugin* plugin)
 
 const char* CommandLineArguments::usage() const
 {
-    return "usage [-v] [-c] [-p] [-lg] [-ln] [-ri] [-r#] [-g|sg|xg|xsg groupName]... [-n|sn|xn|xsn testName]... [\"TEST(groupName, testName)\"]... [-o{normal, junit, teamcity}] [-k packageName]\n";
+    return "usage [-v] [-c] [-p] [-lg] [-ln] [-ri] [-r#] [-g|sg|xg|xsg groupName]... [-n|sn|xn|xsn testName]... [-s [randomizerSeed]] [\"TEST(groupName, testName)\"]... [-o{normal, junit, teamcity}] [-k packageName]\n";
 }
 
 bool CommandLineArguments::isVerbose() const
@@ -124,6 +125,11 @@ int CommandLineArguments::getRepeatCount() const
     return repeat_;
 }
 
+unsigned int CommandLineArguments::getShuffle() const
+{
+    return shuffle_;
+}
+
 const TestFilter* CommandLineArguments::getGroupFilters() const
 {
     return groupFilters_;
@@ -149,6 +155,21 @@ void CommandLineArguments::SetRepeatCount(int ac, const char *const *av, int& i)
 
 }
 
+void CommandLineArguments::SetShuffle(int ac, const char * const *av, int& i)
+{
+    shuffle_ = 1;
+    SimpleString shuffleParameter(av[i]);
+    if (shuffleParameter.size() > 2) {
+        shuffle_ = SimpleString::AtoI(av[i] + 2);
+    } else if (i + 1 < ac) {
+        const int parsed = SimpleString::AtoI(av[i + 1]);
+        if (parsed != 0) {
+            i++;
+            shuffle_ = parsed;
+        }
+    }
+}
+
 SimpleString CommandLineArguments::getParameterField(int ac, const char * const *av, int& i, const SimpleString& parameterName)
 {
     size_t parameterLength = parameterName.size();

diff --git a/src/CppUTest/CommandLineTestRunner.cpp b/src/CppUTest/CommandLineTestRunner.cpp
@@ -29,6 +29,7 @@
 #include "CppUTest/CommandLineTestRunner.h"
 #include "CppUTest/TestOutput.h"
 #include "CppUTest/JUnitTestOutput.h"
+#include "CppUTest/PlatformSpecificFunctions.h"
 #include "CppUTest/TeamCityTestOutput.h"
 #include "CppUTest/TestRegistry.h"
 
@@ -88,13 +89,25 @@ void CommandLineTestRunner::initializeTestRun()
 {
     registry_->setGroupFilters(arguments_->getGroupFilters());
     registry_->setNameFilters(arguments_->getNameFilters());
-	
+
     if (arguments_->isVerbose()) output_->verbose();
     if (arguments_->isColor()) output_->color();
     if (arguments_->runTestsInSeperateProcess()) registry_->setRunTestsInSeperateProcess();
     if (arguments_->isRunIgnored()) registry_->setRunIgnored();
 }
 
+static unsigned int getSeed(unsigned int shuffleArg)
+{
+    if (shuffleArg > 1) return shuffleArg;
+
+    const unsigned int generatedSeed = GetPlatformSpecificTimeInMillis();
+
+    // do not allow seed values 0 or 1 because they cannot be given as cmd line arguments
+    if (generatedSeed > 1) return generatedSeed;
+
+    return 2;
+}
+
 int CommandLineTestRunner::runAllTests()
 {
     initializeTestRun();
@@ -115,14 +128,32 @@ int CommandLineTestRunner::runAllTests()
         registry_->listTestGroupAndCaseNames(tr);
         return 0;
     }
-
+    const bool shuffleEnabled = arguments_->getShuffle() != 0;
+    unsigned int seed = 0;
+    if (shuffleEnabled)
+    {
+        seed = getSeed(arguments_->getShuffle());
+        output_->print("Test order shuffling enabled, seed: ");
+        output_->print(seed);
+        output_->print("\n");
+        std::srand(seed);
+    }
     while (loopCount++ < repeat_) {
+        if (shuffleEnabled)
+        {
+            registry_->shuffleRunOrder();
+        }
         output_->printTestRun(loopCount, repeat_);
         TestResult tr(*output_);
         registry_->runAllTests(tr);
         failureCount += tr.getFailureCount();
     }
-
+    if (shuffleEnabled && arguments_->isVerbose())
+    {
+        output_->print("Random seed was: ");
+        output_->print(seed);
+        output_->print("\n");
+    }
     return failureCount;
 }
 

diff --git a/src/CppUTest/TestOutput.cpp b/src/CppUTest/TestOutput.cpp
@@ -45,7 +45,7 @@ TestOutput::WorkingEnvironment TestOutput::getWorkingEnvironment()
 
 
 TestOutput::TestOutput() :
-    dotCount_(0), verbose_(false), color_(false), progressIndication_(".")
+    dotCount_(0), verbose_(false), color_(false), shuffleSeed_(0), progressIndication_(".")
 {
 }
 
@@ -63,6 +63,11 @@ void TestOutput::color()
     color_ = true;
 }
 
+void TestOutput::setShuffleSeed(unsigned int shuffleSeed)
+{
+    shuffleSeed_ = shuffleSeed;
+}
+
 void TestOutput::print(const char* str)
 {
     printBuffer(str);
@@ -162,6 +167,9 @@ void TestOutput::printTestsEnded(const TestResult& result)
     print(" checks, ");
     print(result.getIgnoredCount());
     print(" ignored, ");
+    if (shuffleSeed_ != 0) {
+         printf("0x%08X seed, ", shuffleSeed_);
+    }
     print(result.getFilteredOutCount());
     print(" filtered out, ");
     print(result.getTotalExecutionTime());

diff --git a/src/CppUTest/TestRegistry.cpp b/src/CppUTest/TestRegistry.cpp
@@ -227,6 +227,23 @@ UtestShell* TestRegistry::getFirstTest()
     return tests_;
 }
 
+void TestRegistry::shuffleRunOrder()
+{
+    std::vector<UtestShell *> tests;
+    for (UtestShell *test = tests_; test != NULL; test = test->getNext()) {
+        tests.push_back(test);
+    }
+    std::random_shuffle(tests.begin(), tests.end());
+
+    // Store shuffled vector back to linked list
+    UtestShell *prev = NULL;
+    for (size_t i = 0; i < tests.size(); ++i)
+    {
+        prev = tests[i]->addTest(prev);
+    }
+    tests_ = prev;
+}
+
 UtestShell* TestRegistry::getTestWithNext(UtestShell* test)
 {
     UtestShell* current = tests_;

diff --git a/src/CppUTest/TestResult.cpp b/src/CppUTest/TestResult.cpp
@@ -32,7 +32,7 @@
 #include "CppUTest/PlatformSpecificFunctions.h"
 
 TestResult::TestResult(TestOutput& p) :
-    output_(p), testCount_(0), runCount_(0), checkCount_(0), failureCount_(0), filteredOutCount_(0), ignoredCount_(0), totalExecutionTime_(0), timeStarted_(0), currentTestTimeStarted_(0),
+    output_(p), testCount_(0), runCount_(0), checkCount_(0), failureCount_(0), filteredOutCount_(0), ignoredCount_(0), shuffleSeed_(0), totalExecutionTime_(0), timeStarted_(0), currentTestTimeStarted_(0),
             currentTestTotalExecutionTime_(0), currentGroupTimeStarted_(0), currentGroupTotalExecutionTime_(0)
 {
 }
@@ -76,6 +76,10 @@ void TestResult::addFailure(const TestFailure& failure)
     output_.printFailure(failure);
     failureCount_++;
 }
+void TestResult::setShuffleSeed(unsigned int shuffleSeed)
+{
+    shuffleSeed_ = shuffleSeed;
+}
 
 void TestResult::countTest()
 {