wip

Signed-off-by: Hanno J. Gödecke <[email protected]>

e2e/config.js

@@ -1,5 +1,15 @@
 const OUTPUT_DIR = 'e2e/.results';
 
+/**
+ * @typedef TestConfig
+ * @property {string} name
+ */
+
+// add your test name here …
+const TEST_NAMES = {
+    AppStartTime: 'App start time',
+};
+
 module.exports = {
     APP_PACKAGE: 'com.expensify.chat',
 
@@ -22,6 +32,22 @@ module.exports = {
     // The file we write logs to
     LOG_FILE: `${OUTPUT_DIR}/debug.log`,
 
-    // The time in milliseconds after which a operation fails due to timeout
+    // The time in milliseconds after which an operation fails due to timeout
     INTERACTION_TIMEOUT: 30_000,
+
+    TEST_NAMES,
+
+    /**
+     * Add your test configurations here. At least,
+     * you need to add a name for your test.
+     *
+     * @type {Object.<string, TestConfig>}
+     */
+    TESTS_CONFIG: {
+        [TEST_NAMES.AppStartTime]: {
+            name: TEST_NAMES.AppStartTime,
+
+            // ... any additional config you might need
+        },
+    },
 };

e2e/server/index.js

@@ -89,9 +89,27 @@ const createServerInstance = () => {
         };
     };
 
+    let activeTestConfig;
+
+    /**
+     * @param {TestConfig} testConfig
+     */
+    const setTestConfig = (testConfig) => {
+        activeTestConfig = testConfig;
+    };
+
     const server = createServer(async (req, res) => {
         res.statusCode = 200;
         switch (req.url) {
+            case Routes.testConfig: {
+                if (activeTestConfig == null) {
+                    res.statusCode = 404;
+                    res.end('No test config available');
+                    return;
+                }
+                return res.end(JSON.stringify(activeTestConfig));
+            }
+
             case Routes.testResults: {
                 const data = await getPostJSONRequestData(req, res);
                 if (data == null) {
@@ -120,6 +138,7 @@ const createServerInstance = () => {
     });
 
     return {
+        setTestConfig,
         addTestResultListener,
         addTestDoneListener,
         start: () => new Promise(resolve => server.listen(PORT, resolve)),

e2e/server/routes.js

@@ -1,4 +1,10 @@
 module.exports = {
+    // The app calls this endpoint to know which test to run
+    testConfig: '/test_config',
+
+    // when running a test the app reports the results to this endpoint
     testResults: '/test_results',
+
+    // when the app is done running a test it calls this endpoint
     testDone: '/test_done',
 };

e2e/testRunner.js

@@ -14,6 +14,7 @@ const {
     OUTPUT_DIR,
     LOG_FILE,
     RUNS,
+    TESTS,
 } = require('./config');
 const compare = require('./compare/compare');
 const Logger = require('./utils/logger');
@@ -31,12 +32,6 @@ const args = process.argv.slice(2);
 
 const baselineBranch = process.env.baseline || DEFAULT_BASELINE_BRANCH;
 
-const TESTS = [
-    {
-        name: 'App start time',
-    },
-];
-
 // clear all files from previous jobs
 try {
     fs.rmSync(OUTPUT_DIR, {recursive: true, force: true});
@@ -98,6 +93,7 @@ const runTestsOnBranch = async (branch, baselineOrCompare) => {
     // run the tests
     for (const test of TESTS) {
         const testLog = Logger.progressInfo(`Running test '${test.name}'`);
+        server.setTestConfig(test);
 
         // We run each test multiple time to average out the results
         for (let i = 0; i < RUNS; i++) {

src/libs/E2E/client.js

@@ -31,7 +31,15 @@ const submitTestResults = testResult => fetch(`${SERVER_ADDRESS}${Routes.testRes
 
 const submitTestDone = () => fetch(`${SERVER_ADDRESS}${Routes.testDone}`);
 
+/**
+ * @returns {Promise<TestConfig>}
+ */
+const getTestConfig = () => fetch(`${SERVER_ADDRESS}${Routes.testConfig}`)
+    .then(res => res.json())
+    .then(config => config);
+
 export default {
     submitTestResults,
     submitTestDone,
+    getTestConfig,
 };

src/libs/E2E/reactNativeLaunchingTest.js

@@ -7,14 +7,17 @@
 // start the usual app
 import '../../../index';
 import Performance from '../Performance';
+import E2EConfig from '../../../e2e/config';
+import E2EClient from './client';
 
 console.debug('==========================');
 console.debug('==== Running e2e test ====');
 console.debug('==========================');
 
-const tests = [
-    require('./tests/appStartTimeTest.e2e').default,
-];
+// import your test here, define its name and config first in e2e/config.js
+const tests = {
+    [E2EConfig.TEST_NAMES.AppStartTime]: require('./tests/appStartTimeTest.e2e'),
+};
 
 // Once we receive the TII measurement we know that the app is initialized and ready to be used:
 const appReady = new Promise((resolve) => {
@@ -27,10 +30,21 @@ const appReady = new Promise((resolve) => {
     });
 });
 
+const testConfig = E2EClient.getTestConfig();
+
 console.debug('[E2E] App startup time test launched, waiting for app to become ready…');
-appReady.then(() => {
-    // TODO: when supporting multiple test cases, this file will decide which test to run.
-    //       The test cases will then be separated in a accompanying /tests folder.
-    tests[0]();
+testConfig.then((config) => {
+    const test = tests[config.name];
+    if (!test) {
+        // instead of throwing, report the error to the server, which is better for UX
+        return E2EClient.submitTestResults({
+            name: config.name,
+            error: `Test ${config.name} not found`,
+        });
+    }
+
+    appReady.then(() => {
+        test();
+    });
 });