[IMP] tests, web: improve test_js end catching

Since odoo#99912 logging an error message doesn't always end qunit tests.
This was mainly to allow to failfast logging qunit errors earlier
without stopping the tests in order to test all qunit anyway.

The logic was to have an end message that stops the test.

Unfortunately some errors will prevent the qunit suite to start
and the test will wait a 1800 long timer. An example was because of
a Missing dependencies. https://runbot.odoo.com/runbot/build/19306352

This new approach will avoid to stop only if the message looks like a
qunit failure and the final message is not there (to be sure).

closes odoo#100238

Signed-off-by: Xavier Dollé (xdo) <[email protected]>

addons/web/tests/test_js.py

@@ -7,13 +7,27 @@
 RE_ONLY = re.compile(r'QUnit\.(only|debug)\(')
 
 
+def qunit_error_checker(message):
+    # We don't want to stop qunit if a qunit is breaking.
+
+    # '%s/%s test failed.' case: end message when all tests are finished
+    if  'tests failed.' in message:
+        return True
+
+    # "QUnit test failed" case: one qunit failed. don't stop in this case
+    if "QUnit test failed:" in message:
+        return False
+
+    return True  # in other cases, always stop (missing dependency, ...)
+
+
 @odoo.tests.tagged('post_install', '-at_install')
 class WebSuite(odoo.tests.HttpCase):
 
     @odoo.tests.no_retry
     def test_js(self):
         # webclient desktop test suite
-        self.browser_js('/web/tests?mod=web', "", "", login='admin', timeout=1800, failure_message=' tests failed.')
+        self.browser_js('/web/tests?mod=web', "", "", login='admin', timeout=1800, error_checker=qunit_error_checker)
 
     def test_check_suite(self):
         # verify no js test is using `QUnit.only` as it forbid any other test to be executed
@@ -45,4 +59,4 @@ class MobileWebSuite(odoo.tests.HttpCase):
 
     def test_mobile_js(self):
         # webclient mobile test suite
-        self.browser_js('/web/tests/mobile?mod=web', "", "", login='admin', timeout=1800, failure_message=' tests failed.')
+        self.browser_js('/web/tests/mobile?mod=web', "", "", login='admin', timeout=1800, error_checker=qunit_error_checker)

odoo/tests/common.py

@@ -960,7 +960,7 @@ def __init__(self, test_class):
         self._open_websocket()
         self._request_id = itertools.count()
         self._result = Future()
-        self.failure_message = ''
+        self.error_checker = None
         self.had_failure = False
         # maps request_id to Futures
         self._responses = {}
@@ -1278,7 +1278,7 @@ def _handle_console(self, type, args=None, stackTrace=None, **kw): # pylint: dis
 
         if log_type == 'error':
             self.had_failure = True
-            if self.failure_message in message:
+            if not self.error_checker or self.error_checker(message):
                 self.take_screenshot()
                 self._save_screencast()
                 try:
@@ -1476,8 +1476,8 @@ def _wait_ready(self, ready_code, timeout=60):
         self._logger.info('Ready code last try result: %s', result)
         return False
 
-    def _wait_code_ok(self, code, timeout, failure_message=''):
-        self.failure_message = failure_message
+    def _wait_code_ok(self, code, timeout, error_checker=None):
+        self.error_checker = error_checker
         self._logger.info('Evaluate test code "%s"', code)
         start = time.time()
         res = self._websocket_request('Runtime.evaluate', params={
@@ -1764,7 +1764,7 @@ def authenticate(self, user, password):
 
         return session
 
-    def browser_js(self, url_path, code, ready='', login=None, timeout=60, cookies=None, failure_message='', watch=False, **kw):
+    def browser_js(self, url_path, code, ready='', login=None, timeout=60, cookies=None, error_checker=None, watch=False, **kw):
         """ Test js code running in the browser
         - optionnally log as 'login'
         - load page given by url_path
@@ -1773,8 +1773,8 @@ def browser_js(self, url_path, code, ready='', login=None, timeout=60, cookies=N
         - open another chrome window to watch code execution if watch is True
 
         To signal success test do: console.log('test successful')
-        To signal test failure raise an exception or
-        call console.error with a message containing the failure_message
+        To signal test failure raise an exception or call console.error with a message.
+        Test will stop when a failure occurs if error_checker is not defined or returns True for this message
 
         """
         if not self.env.registry.loaded:
@@ -1822,7 +1822,7 @@ def browser_js(self, url_path, code, ready='', login=None, timeout=60, cookies=N
 
             error = False
             try:
-                self.browser._wait_code_ok(code, timeout, failure_message=failure_message)
+                self.browser._wait_code_ok(code, timeout, error_checker=error_checker)
             except ChromeBrowserException as chrome_browser_exception:
                 error = chrome_browser_exception
             if error:  # dont keep initial traceback, keep that outside of except