Support parallel test execution (wix#609)

This PR brings DeviceRegistry, an entity which synchronizes device distribution across processes, enabling parallel test execution on any supported test runner.

.travis.yml

@@ -5,7 +5,7 @@ matrix:
   include:
     - language: objective-c
       os: osx
-      osx_image: xcode9.2
+      osx_image: xcode9.3
       env:
         - REACT_NATIVE_VERSION=0.53.3
       install:
@@ -14,7 +14,7 @@ matrix:
       - ./scripts/ci.ios.sh
     - language: objective-c
       os: osx
-      osx_image: xcode9.2
+      osx_image: xcode9.3
       env:
         - REACT_NATIVE_VERSION=0.51.1
       install:
@@ -42,7 +42,7 @@ matrix:
     # Example Projects
     - language: objective-c
       os: osx
-      osx_image: xcode9
+      osx_image: xcode9.3
       env:
         - REACT_NATIVE_VERSION=0.51.1
       install:

detox/local-cli/detox-test.js

@@ -3,8 +3,10 @@
 const program = require('commander');
 const path = require('path');
 const cp = require('child_process');
+
 const _ = require('lodash');
 const CustomError = require('../src/errors/CustomError');
+const environment = require('../src/utils/environment');
 const config = require(path.join(process.cwd(), 'package.json')).detox;
 
 class DetoxConfigError extends CustomError {}
@@ -38,6 +40,10 @@ program
     '[Android Only] Launch Emulator in headless mode. Useful when running on CI.')
   .parse(process.argv);
 
+
+clearDeviceRegistryLockFile();
+
+
 if (program.configuration) {
   if (!config.configurations[program.configuration]) {
     throw new DetoxConfigError(`Cannot determine configuration '${program.configuration}'. 
@@ -53,9 +59,6 @@ const runner = getConfigFor(['testRunner'], 'mocha');
 const runnerConfig = getConfigFor(['runnerConfig'], getDefaultRunnerConfig());
 const platform = (config.configurations[program.configuration].type).split('.')[0];
 
-run();
-
-
 if (typeof program.debugSynchronization === "boolean") {
   program.debugSynchronization = 3000;
 }
@@ -102,28 +105,32 @@ function runMocha() {
   const binPath = path.join('node_modules', '.bin', 'mocha');
   const command = `${binPath} ${testFolder} ${configFile} ${configuration} ${loglevel} ${cleanup} ${reuse} ${debugSynchronization} ${platformString} ${artifactsLocation} ${headless}`;
 
-  console.log(command);
   cp.execSync(command, {stdio: 'inherit'});
 }
 
 function runJest() {
+  const currentConfiguration = config.configurations && config.configurations[program.configuration];
+  const maxWorkers = currentConfiguration.maxWorkers || 1;
   const configFile = runnerConfig ? `--config=${runnerConfig}` : '';
-  const platform = program.platform ? `--testNamePattern='^((?!${getPlatformSpecificString(program.platform)}).)*$'` : '';
-  const binPath = path.join('node_modules', '.bin', 'jest');
+
   const platformString = platform ? `--testNamePattern='^((?!${getPlatformSpecificString(platform)}).)*$'` : '';
-  const command = `${binPath} ${testFolder} ${configFile} --runInBand ${platformString}`;
+  const binPath = path.join('node_modules', '.bin', 'jest');
+  const command = `${binPath} ${testFolder} ${configFile} --maxWorkers=${maxWorkers} ${platformString}`;
+  const env = Object.assign({}, process.env, {
+    configuration: program.configuration,
+    loglevel: program.loglevel,
+    cleanup: program.cleanup,
+    reuse: program.reuse,
+    debugSynchronization: program.debugSynchronization,
+    artifactsLocation: program.artifactsLocation,
+    headless: program.headless
+  });
+
   console.log(command);
+
   cp.execSync(command, {
     stdio: 'inherit',
-    env: Object.assign({}, process.env, {
-      configuration: program.configuration,
-      loglevel: program.loglevel,
-      cleanup: program.cleanup,
-      reuse: program.reuse,
-      debugSynchronization: program.debugSynchronization,
-      artifactsLocation: program.artifactsLocation,
-      headless: program.headless
-    })
+    env
   });
 }
 
@@ -154,10 +161,16 @@ function getPlatformSpecificString(platform) {
   return platformRevertString;
 }
 
+
+function clearDeviceRegistryLockFile() {
+  const fs = require('fs');
+  fs.writeFileSync(environment.getDeviceLockFilePath(), '[]');
+}
+
 function getDefaultConfiguration() {
   if (_.size(config.configurations) === 1) {
     return _.keys(config.configurations)[0];
   }
 }
 
-
+run();

detox/package.json

@@ -47,6 +47,7 @@
     "lodash": "^4.14.1",
     "minimist": "^1.2.0",
     "npmlog": "^4.0.2",
+    "proper-lockfile": "^3.0.2",
     "shell-utils": "^1.0.9",
     "tail": "^1.2.3",
     "telnet-client": "0.15.3",
@@ -74,6 +75,7 @@
       "debug.js",
       "src/ios/earlgreyapi",
       "src/android/espressoapi",
+      "appdatapath.js",
       ".test.js",
       ".mock.js"
     ],

detox/src/client/AsyncWebSocket.js

@@ -51,7 +51,7 @@ class AsyncWebSocket {
 
   async send(message, messageId) {
     if (!this.ws) {
-      throw new Error(`Can't send a message on a closed websocket, init the by calling 'open()'`);
+      throw new Error(`Can't send a message on a closed websocket, init the by calling 'open()'. Message:  ${JSON.stringify(message)}`);
     }
 
     return new Promise(async(resolve, reject) => {

detox/src/client/Client.js

@@ -36,7 +36,9 @@ class Client {
   async cleanup() {
     clearTimeout(this.slowInvocationStatusHandler);
     if (this.isConnected && !this.pandingAppCrash) {
-      await this.sendAction(new actions.Cleanup(this.successfulTestRun));
+      if(this.ws.isOpen()) {
+        await this.sendAction(new actions.Cleanup(this.successfulTestRun));
+      }
       this.isConnected = false;
     }
 
@@ -102,8 +104,10 @@ class Client {
 
   slowInvocationStatus() {
     return setTimeout(async () => {
-      const status = await this.currentStatus();
-      this.slowInvocationStatusHandler = this.slowInvocationStatus();
+      if (this.ws.isOpen()) {
+        const status = await this.currentStatus();
+        this.slowInvocationStatusHandler = this.slowInvocationStatus();
+      }
     }, this.slowInvocationTimeout);
   }
 }

detox/src/client/Client.test.js

@@ -10,10 +10,11 @@ describe('Client', () => {
   beforeEach(() => {
     jest.mock('npmlog');
     WebSocket = jest.mock('./AsyncWebSocket');
-    Client = require('./Client');
 
     jest.mock('../utils/argparse');
     argparse = require('../utils/argparse');
+
+    Client = require('./Client');
   });
 
   it(`reloadReactNative() - should receive ready from device and resolve`, async () => {
@@ -75,6 +76,17 @@ describe('Client', () => {
     expect(client.ws.send).not.toHaveBeenCalled();
   });
 
+  it(`cleanup() - if "connected" but ws is closed should do nothing`, async () => {
+    await connect();
+    client.ws.send.mockReturnValueOnce(response("ready", {}, 1));
+    await client.waitUntilReady();
+
+    client.ws.isOpen.mockReturnValue(false);
+    await client.cleanup();
+
+    expect(client.ws.send).toHaveBeenCalledTimes(2);
+  });
+
   it(`execute() - "invokeResult" on an invocation object should resolve`, async () => {
     await connect();
     client.ws.send.mockReturnValueOnce(response("invokeResult", {result: "(GREYElementInteraction)"}, 1));
@@ -85,26 +97,29 @@ describe('Client', () => {
     expect(client.ws.send).toHaveBeenCalledTimes(2);
   });
 
-  async function executeWithSlowInvocation(invocationTime) {
+  it(`execute() - fast invocation should not trigger "slowInvocationStatus"`, async () => {
     argparse.getArgValue.mockReturnValue(2); // set debug-slow-invocations
-
     await connect();
-
-    client.ws.send.mockReturnValueOnce(timeout(invocationTime).then(()=> response("invokeResult", {result:"(GREYElementInteraction)"}, 1)))
-                  .mockReturnValueOnce(response("currentStatusResult", {"state":"busy","resources":[{"name":"App State","info":{"prettyPrint":"Waiting for network requests to finish.","elements":["__NSCFLocalDataTask:0x7fc95d72b6c0"],"appState":"Waiting for network requests to finish."}},{"name":"Dispatch Queue","info":{"queue":"OS_dispatch_queue_main: com.apple.main-thread[0x10805ea80] = { xrefcnt = 0x80000000, refcnt = 0x80000000, target = com.apple.root.default-qos.overcommit[0x10805f1c0], width = 0x1, state = 0x000fffe000000403, in-flight = 0, thread = 0x403 }","prettyPrint":"com.apple.main-thread"}}]}, 2));
-
-    const call = invoke.call(invoke.IOS.Class('GREYMatchers'), 'matcherForAccessibilityLabel:', 'test');
-    await client.execute(call);
-  }
-
-  it(`execute() - fast invocation should not trigger "slowInvocationStatus"`, async () => {
     await executeWithSlowInvocation(1);
     expect(client.ws.send).toHaveBeenLastCalledWith({"params": {"args": ["test"], "method": "matcherForAccessibilityLabel:", "target": {"type": "Class", "value": "GREYMatchers"}}, "type": "invoke"}, undefined);
+    expect(client.ws.send).toHaveBeenCalledTimes(2);
   });
 
   it(`execute() - slow invocation should trigger "slowInvocationStatus:`, async () => {
+    argparse.getArgValue.mockReturnValue(2); // set debug-slow-invocations
+    await connect();
     await executeWithSlowInvocation(4);
     expect(client.ws.send).toHaveBeenLastCalledWith({"params": {}, "type": "currentStatus"}, undefined);
+    expect(client.ws.send).toHaveBeenCalledTimes(3);
+  });
+
+  it(`execute() - slow invocation should do nothing if ws was closed`, async () => {
+    argparse.getArgValue.mockReturnValue(2); // set debug-slow-invocations
+    await connect();
+    client.ws.isOpen.mockReturnValue(false);
+    await executeWithSlowInvocation(4);
+
+    expect(client.ws.send).toHaveBeenCalledTimes(2);
   });
 
   it(`execute() - "invokeResult" on an invocation function should resolve`, async () => {
@@ -185,6 +200,14 @@ describe('Client', () => {
       }));
   }
 
+  async function executeWithSlowInvocation(invocationTime) {
+    client.ws.send.mockReturnValueOnce(timeout(invocationTime).then(()=> response("invokeResult", {result:"(GREYElementInteraction)"}, 1)))
+          .mockReturnValueOnce(response("currentStatusResult", {"state":"busy","resources":[{"name":"App State","info":{"prettyPrint":"Waiting for network requests to finish.","elements":["__NSCFLocalDataTask:0x7fc95d72b6c0"],"appState":"Waiting for network requests to finish."}},{"name":"Dispatch Queue","info":{"queue":"OS_dispatch_queue_main: com.apple.main-thread[0x10805ea80] = { xrefcnt = 0x80000000, refcnt = 0x80000000, target = com.apple.root.default-qos.overcommit[0x10805f1c0], width = 0x1, state = 0x000fffe000000403, in-flight = 0, thread = 0x403 }","prettyPrint":"com.apple.main-thread"}}]}, 2));
+
+    const call = invoke.call(invoke.IOS.Class('GREYMatchers'), 'matcherForAccessibilityLabel:', 'test');
+    await client.execute(call);
+  }
+
   async function timeout(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
   }

detox/src/devices/AppleSimUtils.js

@@ -1,3 +1,4 @@
+const process = require('process');
 const _ = require('lodash');
 const exec = require('../utils/exec');
 const retry = require('../utils/retry');
@@ -20,22 +21,39 @@ class AppleSimUtils {
   }
 
   async findDeviceUDID(query) {
+    const udids = await this.findDevicesUDID(query);
+    return udids[0];
+  }
+
+  async findDevicesUDID(query) {
     const statusLogs = {
       trying: `Searching for device matching ${query}...`
     };
-    let correctQuery = this._correctQueryWithOS(query);
-    const response = await this._execAppleSimUtils({ args: `--list "${correctQuery}" --maxResults=1` }, statusLogs, 1);
+
+    let type;
+    let os;
+    if (_.includes(query, ',')) {
+      const parts = _.split(query, ',');
+      type = parts[0].trim();
+      os = parts[1].trim();
+    } else {
+      type = query;
+      const deviceInfo = await this.deviceTypeAndNewestRuntimeFor(query);
+      os = deviceInfo.newestRuntime.version;
+    }
+
+    const response = await this._execAppleSimUtils({ args: `--list --byType "${type}" --byOS "${os}"`}, statusLogs, 1);
     const parsed = this._parseResponseFromAppleSimUtils(response);
-    const udid = _.get(parsed, [0, 'udid']);
-    if (!udid) {
+    const udids = _.map(parsed, 'udid');
+    if (!udids || !udids.length || !udids[0]) {
       throw new Error(`Can't find a simulator to match with "${query}", run 'xcrun simctl list' to list your supported devices.
       It is advised to only state a device type, and not to state iOS version, e.g. "iPhone 7"`);
     }
-    return udid;
+    return udids;
   }
 
   async findDeviceByUDID(udid) {
-    const response = await this._execAppleSimUtils({ args: `--list` }, undefined, 1);
+    const response = await this._execAppleSimUtils({args: `--list --byId "${udid}"`}, undefined, 1);
     const parsed = this._parseResponseFromAppleSimUtils(response);
     const device = _.find(parsed, (device) => _.isEqual(device.udid, udid));
     if (!device) {
@@ -44,25 +62,35 @@ class AppleSimUtils {
     return device;
   }
 
-  async waitForDeviceState(udid, state) {
-    let device;
-    await retry({ retries: 10, interval: 1000 }, async () => {
-      device = await this.findDeviceByUDID(udid);
-      if (!_.isEqual(device.state, state)) {
-        throw new Error(`device is in state '${device.state}'`);
-      }
-    });
-    return device;
+  async boot(udid) {
+    if (!await this.isBooted(udid)) {
+      await this._bootDeviceByXcodeVersion(udid);
+    }
   }
 
-  async boot(udid) {
+  async isBooted(udid) {
     const device = await this.findDeviceByUDID(udid);
-    if (_.isEqual(device.state, 'Booted') || _.isEqual(device.state, 'Booting')) {
-      return false;
+    return (_.isEqual(device.state, 'Booted') || _.isEqual(device.state, 'Booting'));
+  }
+
+  async deviceTypeAndNewestRuntimeFor(name) {
+    const result = await this._execSimctl({ cmd: `list -j` });
+    const stdout = _.get(result, 'stdout');
+    const output = JSON.parse(stdout);
+    const deviceType = _.filter(output.devicetypes, { 'name': name})[0];
+    const newestRuntime = _.maxBy(output.runtimes, r => Number(r.version));
+    return { deviceType, newestRuntime };
+  }
+  async create(name) {
+    const deviceInfo = await this.deviceTypeAndNewestRuntimeFor(name);
+
+    if (deviceInfo.newestRuntime) {
+      const result = await this._execSimctl({cmd: `create "${name}-Detox" "${deviceInfo.deviceType.identifier}" "${deviceInfo.newestRuntime.identifier}"`});
+      const udid = _.get(result, 'stdout').trim();
+      return udid;
+    } else {
+      throw new Error(`Unable to create device. No runtime found for ${name}`);
     }
-    await this.waitForDeviceState(udid, 'Shutdown');
-    await this._bootDeviceByXcodeVersion(udid);
-    await this.waitForDeviceState(udid, 'Booted');
   }
 
   async install(udid, absPath) {
@@ -163,15 +191,6 @@ class AppleSimUtils {
     return await exec.execWithRetriesAndLogs(`/usr/bin/xcrun simctl ${cmd}`, undefined, statusLogs, retries);
   }
 
-  _correctQueryWithOS(query) {
-    let correctQuery = query;
-    if (_.includes(query, ',')) {
-      const parts = _.split(query, ',');
-      correctQuery = `${parts[0].trim()}, OS=${parts[1].trim()}`;
-    }
-    return correctQuery;
-  }
-
   _parseResponseFromAppleSimUtils(response) {
     let out = _.get(response, 'stdout');
     if (_.isEmpty(out)) {
@@ -200,6 +219,7 @@ class AppleSimUtils {
     } else {
       await this._bootDeviceMagically(udid);
     }
+    await this._execSimctl({ cmd: `bootstatus ${udid}`, retries: 1 });
   }
 
   async _bootDeviceMagically(udid) {
@@ -245,4 +265,4 @@ class LogsInfo {
   }
 }
 
-module.exports = AppleSimUtils;
+module.exports = AppleSimUtils;