Add a few improvements to the baseline testing framework

1) Add a QBASELINETEST_MAIN macro as a replacement for QTEST_MAIN, relieving the clients of reproducing the kludgy workaround 2) Add a -server command line option to baseline tests, as an alternative to specifying the server in an environment variable 3) Fix command line parsing so that it exits on syntax errors. Change-Id: I36f38267143a308e971e2e7b2fdbe4be44370043 Reviewed-by: Volker Hilsheimer <[email protected]>
Schaudge · Jun 10, 2024 · 1036e9d · 1036e9d
1 parent 2a0b907
commit 1036e9d
Show file tree

Hide file tree

Showing 8 changed files with 42 additions and 58 deletions.
diff --git a/tests/baseline/painting/tst_baseline_painting.cpp b/tests/baseline/painting/tst_baseline_painting.cpp
@@ -473,17 +473,6 @@ void tst_Lancelot::paint(QPaintDevice *device, GraphicsEngine engine, QImage::Fo
     p.end();
 }
 
-#define main _realmain
-QTEST_MAIN(tst_Lancelot)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Lancelot);
 
 #include "tst_baseline_painting.moc"
diff --git a/tests/baseline/shared/baselineprotocol.cpp b/tests/baseline/shared/baselineprotocol.cpp
@@ -261,18 +261,21 @@ bool BaselineProtocol::disconnect()
 }
 
 
-bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo& clientInfo)
+bool BaselineProtocol::connect(const QString &testCase, bool *dryrun, const PlatformInfo &clientInfo, const QString &server)
 {
     errMsg.clear();
-    QByteArray serverName(qgetenv("QT_LANCELOT_SERVER"));
-    if (serverName.isNull())
-        serverName = "lancelot.test.qt-project.org";
+    QString serverName = server;
+    if (serverName.isEmpty()) {
+        serverName = qEnvironmentVariable("QT_LANCELOT_SERVER");
+        if (serverName.isEmpty())
+            serverName = QStringLiteral("lancelot.test.qt-project.org");
+    }
 
     socket.connectToHost(serverName, ServerPort);
     if (!socket.waitForConnected(Timeout)) {
         QThread::sleep(std::chrono::seconds{3});  // Wait a bit and try again, the server might just be restarting
         if (!socket.waitForConnected(Timeout)) {
-            errMsg += QLS("TCP connectToHost failed. Host:") + QLS(serverName) + QLS(" port:") + QString::number(ServerPort);
+            errMsg += QLS("TCP connectToHost failed. Host:") + serverName + QLS(" port:") + QString::number(ServerPort);
             return false;
         }
     }

diff --git a/tests/baseline/shared/baselineprotocol.h b/tests/baseline/shared/baselineprotocol.h
@@ -115,7 +115,8 @@ class BaselineProtocol
     // For client:
 
     // For advanced client:
-    bool connect(const QString &testCase, bool *dryrun = nullptr, const PlatformInfo& clientInfo = PlatformInfo());
+    bool connect(const QString &testCase, bool *dryrun = nullptr,
+                 const PlatformInfo &clientInfo = PlatformInfo(), const QString &server = QString());
     bool disconnect();
     bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList);
     bool submitMatch(const ImageItem &item, QByteArray *serverMsg);

diff --git a/tests/baseline/shared/qbaselinetest.cpp b/tests/baseline/shared/qbaselinetest.cpp
@@ -11,6 +11,7 @@
 namespace QBaselineTest {
 
 static char *fargv[MAXCMDLINEARGS];
+static QString server;
 static bool simfail = false;
 static PlatformInfo customInfo;
 static bool customAutoModeSet = false;
@@ -33,6 +34,7 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         return;
 
     bool showHelp = false;
+    bool abortOnHelp = true;
 
     int fargc = 0;
     int numArgs = *argcp;
@@ -41,7 +43,16 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         QByteArray arg = (*argvp)[i];
         QByteArray nextArg = (i+1 < numArgs) ? (*argvp)[i+1] : nullptr;
 
-        if (arg == "-simfail") {
+        if (arg == "-server") {
+            i++;
+            if (!nextArg.isEmpty()) {
+                server = QString::fromLocal8Bit(nextArg);
+            } else {
+                qWarning() << "-server requires parameter";
+                showHelp = true;
+                break;
+            }
+        } else if (arg == "-simfail") {
             simfail = true;
         } else if (arg == "-fuzzlevel") {
             i++;
@@ -77,10 +88,13 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
             }
             customInfo.addOverride(key, value);
         } else {
-            if ( (arg == "-help") || (arg == "--help") )
+            if ( (arg == "-help") || (arg == "--help") ) {
                 showHelp = true;
+                abortOnHelp = false;
+            }
             if (fargc >= MAXCMDLINEARGS) {
                 qWarning() << "Too many command line arguments!";
+                showHelp = true;
                 break;
             }
             fargv[fargc++] = (*argvp)[i];
@@ -93,7 +107,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         // TBD: arrange for this to be printed *after* QTest's help
         QTextStream out(stdout);
         out << "\n Baseline testing (lancelot) options:\n";
-        out << " -simfail            : Force an image comparison mismatch. For testing purposes.\n";
+        out << " -server <host>      : Set the network baseline server to connect to.\n";
+        out << "                       The default is taken from the environment variable QT_LANCELOT_SERVER.\n";
+        out << " -simfail            : Force an image comparison mismatch. For development purposes.\n";
         out << " -fuzzlevel <int>    : Specify the percentage of fuzziness in comparison. Overrides server default. 0 means exact match.\n";
         out << " -auto               : Inform server that this run is done by a daemon, CI system or similar.\n";
         out << " -adhoc (default)    : The inverse of -auto; this run is done by human, e.g. for testing.\n";
@@ -104,6 +120,9 @@ void handleCmdLineArgs(int *argcp, char ***argvp)
         out << "                       for example: -compareto QtVersion=4.8.0\n";
         out << "                       Multiple -compareto client specifications may be given.\n";
         out << "\n";
+        out.flush();
+        if (abortOnHelp)
+            std::exit(1);
     }
 }
 
@@ -182,7 +201,7 @@ bool connect(QByteArray *msg, bool *error)
         return false;
     }
 
-    if (!proto.connect(testCase, &dryRunMode, clientInfo)) {
+    if (!proto.connect(testCase, &dryRunMode, clientInfo, server)) {
         *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
         *error = true;
         return false;

diff --git a/tests/baseline/shared/qbaselinetest.h b/tests/baseline/shared/qbaselinetest.h
@@ -66,4 +66,9 @@ do {\
         QSKIP("Blacklisted on baseline server.");\
 } while (0)
 
+#define QBASELINETEST_MAIN(TestObject) QTEST_MAIN_WRAPPER(TestObject, \
+    QHashSeed::setDeterministicGlobalSeed(); \
+    QBaselineTest::handleCmdLineArgs(&argc, &argv); \
+    QTEST_MAIN_SETUP())
+
 #endif // BASELINETEST_H
diff --git a/tests/baseline/stylesheet/tst_baseline_stylesheet.cpp b/tests/baseline/stylesheet/tst_baseline_stylesheet.cpp
@@ -224,17 +224,6 @@ void tst_Stylesheet::tst_QHeaderView()
     QBASELINE_TEST(takeSnapshot());
 }
 
-#define main _realmain
-QTEST_MAIN(tst_Stylesheet)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Stylesheet)
 
 #include "tst_baseline_stylesheet.moc"
diff --git a/tests/baseline/text/tst_baseline_text.cpp b/tests/baseline/text/tst_baseline_text.cpp
@@ -103,17 +103,6 @@ void tst_Text::tst_differentScriptsBackgrounds()
 }
 
 
-#define main _realmain
-QTEST_MAIN(tst_Text)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Text)
 
 #include "tst_baseline_text.moc"
diff --git a/tests/baseline/widgets/tst_baseline_widgets.cpp b/tests/baseline/widgets/tst_baseline_widgets.cpp
@@ -1275,17 +1275,6 @@ void tst_Widgets::tst_QLCDNumber()
     QBASELINE_CHECK_DEFERRED(takeSnapshot(), "lcdnumber");
 }
 
-#define main _realmain
-QTEST_MAIN(tst_Widgets)
-#undef main
-
-int main(int argc, char *argv[])
-{
-    // Avoid rendering variations caused by QHash randomization
-    QHashSeed::setDeterministicGlobalSeed();
-
-    QBaselineTest::handleCmdLineArgs(&argc, &argv);
-    return _realmain(argc, argv);
-}
+QBASELINETEST_MAIN(tst_Widgets)
 
 #include "tst_baseline_widgets.moc"