Bug 1812092 - Update startup tests to use a fixed current date. r=per…

…ftest-reviewers,AlexandruIonescu

Differential Revision: https://phabricator.services.mozilla.com/D173247

python/mozperftest/mozperftest/system/android_startup.py

@@ -4,7 +4,7 @@
 import re
 import statistics
 import time
-from datetime import date, datetime, timedelta
+from datetime import datetime, timedelta
 
 import mozdevice
 
@@ -24,6 +24,9 @@
 KEY_TEST_NAME = "test_name"
 
 MEASUREMENT_DATA = ["mean", "median", "standard_deviation"]
+OLD_VERSION_FOCUS_PAGE_START_LINE_COUNT = 3
+NEW_VERSION_FOCUS_PAGE_START_LINE_COUNT = 2
+STDOUT_LINE_COUNT = 2
 
 TEST_COLD_MAIN_FF = "cold_main_first_frame"
 TEST_COLD_MAIN_RESTORE = "cold_main_session_restore"
@@ -155,7 +158,6 @@ def __init__(self, env, mach_cmd):
         super(AndroidStartUp, self).__init__(env, mach_cmd)
         self.android_activity = None
         self.capture_logcat = self.capture_file = self.app_name = None
-        self.download_date = date.today()
         self.architecture = "arm64-v8a"
         self.device = mozdevice.ADBDevice(use_root=False)
 
@@ -317,11 +319,13 @@ def __get_page_start_datetime():
             )
             if is_old_version_of_focus:
                 assert (
-                    page_start_line_count == 3
+                    page_start_line_count
+                    == OLD_VERSION_FOCUS_PAGE_START_LINE_COUNT  # should be 3
                 ), page_start_assert_msg  # Lines: about:blank, target URL, target URL.
             else:
                 assert (
-                    page_start_line_count == 2
+                    page_start_line_count
+                    == NEW_VERSION_FOCUS_PAGE_START_LINE_COUNT  # Should be 2
                 ), page_start_assert_msg  # Lines: about:blank, target URL.
             return __line_to_datetime(
                 page_start_lines[1]
@@ -339,9 +343,8 @@ def get_warmup_delay_seconds(self):
         We've been told the start up cache is populated ~60s after first start up. As such,
         we should measure start up with the start up cache populated. If the
         args say we shouldn't wait, we only wait a short duration ~= visual completeness.
-        return 60 if self.startup_cache else 5
         """
-        return 1
+        return 60 if self.startup_cache else 5
 
     def get_start_cmd(self, test_name):
         intent_action_prefix = "android.intent.action.{}"
@@ -373,7 +376,7 @@ def get_component_name_for_intent(self, intent):
         )
         result_output = self.device.shell_output(resolve_component_args)
         stdout = result_output.splitlines()
-        if len(stdout) != 2:
+        if len(stdout) != STDOUT_LINE_COUNT:  # Should be 2
             raise AndroidStartUpMatchingError(f"expected 2 lines. Got: {stdout}")
         return stdout[1]
 

python/mozperftest/mozperftest/tests/test_android_startup.py

@@ -10,7 +10,7 @@
 import pytest
 import requests
 
-import mozperftest.system.android_startup as android_startup
+from mozperftest.system import android_startup
 from mozperftest.system.android_startup import (
     AndroidStartUpInstallError,
     AndroidStartUpMatchingError,
@@ -35,6 +35,7 @@
     "AndroidStartUp-product": "fenix",
     "AndroidStartUp-release-channel": "nightly",
     "apk_metadata": SAMPLE_APK_METADATA,
+    "test-date": "2023.01.01",
 }
 
 

testing/performance/hooks_android_startup.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import pathlib
+import re
+import subprocess
 from datetime import datetime
 
 import requests
@@ -13,33 +15,27 @@
     KEY_DATETIME,
     KEY_NAME,
     KEY_PRODUCT,
-    PROD_FOCUS,
 )
 
+HTTP_200_OKAY = 200
+
 
 def before_iterations(kw):
     product = kw["AndroidStartUp_product"]
-    download_date = datetime.today()
     architecture = "arm64-v8a"
+    commit_info = subprocess.getoutput("hg log -l 1")
+    commit_date = re.search(r"date:\s+([:\s\w]+)\s+", str(commit_info)).group(1)
+    download_date = datetime.strptime(commit_date, "%a %b %d %H:%M:%S %Y").strftime(
+        DATETIME_FORMAT
+    )
 
-    if product == PROD_FOCUS:
-        if download_date >= datetime(2023, 2, 17):
-            pass
-        elif download_date >= datetime(2022, 12, 15):
-            product += "-v3"
-        elif download_date >= datetime(2021, 11, 5):
-            product += "-v2"
-
-    # The above sections with v2, v3 are occurring because of change in task cluster indicies
-    # This is not expected to occur regularly, the new indicies contain both focus and fenix
-    # android components
-
-    nightly_url = BASE_URL_DICT[product + "-latest"].format(
-        date=download_date.strftime(DATETIME_FORMAT), architecture=architecture
+    nightly_url = BASE_URL_DICT[product].format(
+        date=download_date, architecture=architecture
     )
     filename = f"{product}_nightly_{architecture}.apk"
     print("Fetching {}...".format(filename), end="", flush=True)
     download_apk_as_date(nightly_url, download_date, filename)
+    print(f"Downloaded {product} for date: {download_date}")
 
     kw["apk_metadata"] = {
         KEY_NAME: filename,
@@ -52,7 +48,7 @@ def before_iterations(kw):
 
 def download_apk_as_date(nightly_url, download_date_string, filename):
     apk = requests.get(nightly_url)
-    if apk.status_code != 200:
+    if apk.status_code != HTTP_200_OKAY:
         raise Exception(
             f"Something went wrong downloading the apk check to make sure you have entered"
             f" a date that is valid and that the apk for the date you have requested("