Add support for testcheck.

Summary:
Adds support for property testing via Lee Byron's testcheck-js library. <s>This cannot merge until this merges: https://github.com/leebyron/jasmine-check/pull/5</s>

Please see the documentation for testcheck: https://github.com/leebyron/testcheck-js

(bypass-lint)
Closes jestjs#1083

Differential Revision: D3369819

fbshipit-source-id: d8eb3f24711346d47bd7bde06c7d3a1a1f70c2e3

.eslintrc

@@ -1,5 +1,9 @@
 {
   "extends": "./node_modules/fbjs-scripts/eslint/.eslintrc.js",
+  "globals": {
+    "check": true,
+    "gen": true,
+  },
   "parser": "babel-eslint",
   "plugins": [
     "babel"

CHANGELOG.md

@@ -1,5 +1,7 @@
 ## master
 
+* Add support for property testing via testcheck-js.
+
 ## jest-cli 12.1.1
 
 * Windows stability fixes.

docs/API.md

@@ -81,8 +81,10 @@ Jest uses Jasmine 2 by default. An introduction to Jasmine 2 can be found
 
   - `afterEach(fn)`
   - `beforeEach(fn)`
+  - [`check`](#property-testing)
   - `describe(name, fn)`
   - [`expect(value)`](#expect-value)
+  - [`gen`](#property-testing)
   - `it(name, fn)`
   - `fit(name, fn)` executes only this test. Useful when investigating a failure
   - [`jest`](#the-jest-object)
@@ -344,6 +346,49 @@ module should receive a mock implementation or not.
 Returns a mock module instead of the actual module, bypassing all checks on
 whether the module should be required normally or not.
 
+## Property testing
+
+Jest supports property testing with the
+[testcheck-js](https://github.com/leebyron/testcheck-js) library. The API is
+the same as that of [jasmine-check](https://github.com/leebyron/jasmine-check):
+
+### `check.it(name, [options], generators, fn)`
+Creates a property test. Test cases will be created by the given `generators`
+and passed as arguments to `fn`. If any test case fails, a shrunken failing
+value will be given in the test output. For example:
+
+```js
+check.it('can recover encoded URIs',
+  [gen.string],
+  s => expect(s).toBe(decodeURI(encodeURI(s))));
+```
+
+If `options` are provided, they override the corresponding command-line options.
+The possible options are:
+
+```
+{
+  times: number;   // The number of test cases to run. Default: 100.
+  maxSize: number; // The maximum size of sized data such as numbers
+                   // (their magnitude) or arrays (their length). This can be
+                   // overridden with `gen.resize`. Default: 200.
+  seed: number;    // The random number seed. Defaults to a random value.
+}
+```
+
+### `check.fit(name, [options], generators, fn)`
+
+Executes this test and skips all others. Like `fit`, but for property tests.
+
+### `check.xit(name, [options], generators, fn)`
+
+Skips this test. Like `xit`, but for property tests.
+
+### `gen`
+
+A library of generators for property tests. See the
+[`testcheck` documentation](https://github.com/leebyron/testcheck-js).
+
 ## Configuration
 
 Jest's configuration can be defined in the `package.json` file of your project

integration_tests/__tests__/snapshot-test.js

@@ -21,7 +21,7 @@ describe('Snapshot', () => {
 
   afterEach(() => {
     fs.unlinkSync(snapshotFile);
-    fs.rmdirSync(snapshotDir)
+    fs.rmdirSync(snapshotDir);
   });
 
   it('works as expected', () => {

integration_tests/__tests__/testcheck-test.js

@@ -0,0 +1,61 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+'use strict';
+
+jest.unmock('../runJest');
+
+const runJest = require('../runJest');
+
+describe('testcheck', () => {
+  it('works', () => {
+    const result = runJest.json('testcheck', ['testcheck-test.js']);
+    const json = result.json;
+
+    expect(json.numTotalTests).toBe(3);
+    expect(json.numPassedTests).toBe(1);
+    expect(json.numFailedTests).toBe(1);
+    expect(json.numPendingTests).toBe(1);
+  });
+
+  it('runs "fit" tests exclusively', () => {
+    const result = runJest.json('testcheck', ['testcheck-fit-test.js']);
+    const json = result.json;
+
+    expect(json.numTotalTests).toBe(3);
+    expect(json.numPassedTests).toBe(1);
+    expect(json.numFailedTests).toBe(0);
+    expect(json.numPendingTests).toBe(2);
+  });
+
+  it('merges in testcheck options', () => {
+    const result = runJest.json('testcheck', [
+      'testcheck-options-test.js',
+      '--testcheckSeed', '0',
+      '--testcheckTimes', '5',
+      '--testcheckMaxSize', '2',
+    ]);
+    const json = result.json;
+    const output = result.stderr.toString();
+
+    expect(json.numTotalTests).toBe(4);
+    expect(json.numPassedTests).toBe(4);
+
+    expect(output).toMatch(/An array: \[ 1, 9, -5, 4, 3, 8 \]/);
+    expect(output).toMatch(/runCountWithoutOverride: 5/);
+    expect(output).toMatch(/runCountWithOverride: 1/);
+  });
+
+  it('reports exceptions', () => {
+    const result = runJest('testcheck', ['testcheck-exceptions-test.js']);
+    expect(result.status).toBe(1);
+    expect(result.stdout.toString()).toMatch(
+      /Error: This error should be reported/
+    );
+  });
+});

integration_tests/coverage_report/__tests__/sum-test.js

@@ -15,4 +15,11 @@ describe('sum', () => {
   it('adds numbers', () => {
     expect(sum(1, 2)).toEqual(3);
   });
+  // Required for test coverage of jasmine-check-install.js:
+  check.it('generates numbers', {times: 1}, [gen.posInt], i => {
+    expect(i).toBeGreaterThan(-1);
+  });
+  check.it('generates numbers', [gen.posInt], i => {
+    expect(i).toBeGreaterThan(-1);
+  });
 });

integration_tests/testcheck/__tests__/testcheck-exceptions-test.js

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+'use strict';
+
+describe('testcheck-exceptions', () => {
+  check.it('reports exceptions', [], () => {
+    throw new Error('This error should be reported.');
+  });
+});

integration_tests/testcheck/__tests__/testcheck-fit-test.js

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+'use strict';
+
+describe('testcheck-fit', () => {
+  check.fit('runs tests with fit exclusively', [], () => {
+    expect(true).toBe(true);
+  });
+
+  check.it('should not run this test', [], () => {
+    expect(true).toBe(false);
+  });
+
+  it('should not run this test either', () => {
+    expect(true).toBe(false);
+  });
+});

integration_tests/testcheck/__tests__/testcheck-options-test.js

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+'use strict';
+
+let runCountWithoutOverride = 0;
+let runCountWithOverride = 0;
+
+describe('testcheck-options', () => {
+  check.it('runs things ten times by default', [], () => {
+    runCountWithoutOverride++;
+  });
+
+  check.it('allows overrides', {times: 1}, [], () => {
+    runCountWithOverride++;
+  });
+
+  check.it('generates correctly-sized arrays', [gen.array(gen.int)], a => {
+    expect(a.length).toBeLessThan(2);
+  });
+
+  check.it('has the right seed', [gen.resize(10, gen.array(gen.int))], a => {
+    console.log('An array:', a);
+  });
+
+  afterAll(() => {
+    console.log('runCountWithoutOverride:', runCountWithoutOverride);
+    console.log('runCountWithOverride:', runCountWithOverride);
+  });
+});

integration_tests/testcheck/__tests__/testcheck-test.js

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+'use strict';
+
+describe('testcheck', () => {
+  check.it('works', [gen.int], a => {
+    expect(a).toEqual(jasmine.any(Number));
+  });
+
+  check.it('reports failures', [], () => {
+    expect(true).toBe(false);
+  });
+
+  check.xit('skips tests', [], () => {
+    expect(true).toBe(false);
+  });
+});

integration_tests/testcheck/package.json

@@ -0,0 +1 @@
+{}

packages/jest-cli/src/cli/processArgs.js

@@ -201,6 +201,35 @@ function processArgs() {
       ),
       type: 'boolean',
     },
+    testcheckTimes: {
+      default: 100,
+      description: _wrapDesc(
+        'The number of test cases to generate for each testcheck test. ' +
+        'May be overridden for individual test cases using the options ' +
+        'argument of check.it.'
+      ),
+      type: 'number',
+    },
+    testcheckMaxSize: {
+      default: 200,
+      description: _wrapDesc(
+        'The maximum size of sized types, such as arrays and ints, to be ' +
+        'generated for testcheck tests. ' +
+        'May be overridden for individual test cases using the options ' +
+        'argument of check.it. ' +
+        'Generators can also be sized using gen.resize(n, anotherGenerator).'
+      ),
+      type: 'number',
+    },
+    testcheckSeed: {
+      default: undefined,
+      description: _wrapDesc(
+        'The seed for generating testcheck test cases. Defaults to random. ' +
+        'May be overridden for individual test cases using the options ' +
+        'argument of check.it.'
+      ),
+      type: 'number',
+    },
   };
 
   const argv = yargs

packages/jest-cli/src/config/read.js

@@ -63,6 +63,12 @@ function readConfig(argv, packageRoot) {
     }
     config.noStackTrace = argv.noStackTrace;
 
+    config.testcheckOptions = {
+      times: argv.testcheckTimes,
+      maxSize: argv.testcheckMaxSize,
+      seed: argv.testcheckSeed,
+    };
+
     return config;
   });
 }

packages/jest-jasmine2/package.json

@@ -9,6 +9,7 @@
   "main": "src/index.js",
   "dependencies": {
     "graceful-fs": "^4.1.3",
+    "jasmine-check": "^0.1.4",
     "jest-util": "^12.1.0",
     "jest-snapshot": "^12.1.0"
   },

packages/jest-jasmine2/src/index.js

@@ -91,6 +91,12 @@ function jasmine2(config, environment, moduleLoader, testPath) {
 
     jasminePit.install(environment.global);
 
+    // Jasmine-check must be installed from within the vm context because
+    // testcheck is compiled Clojurescript and is very delicate.
+    environment.global.jasmine.testcheckOptions = config.testcheckOptions;
+    moduleLoader.requireModule(require.resolve('./jasmine-check-install'));
+    delete environment.global.jasmine.testcheckOptions;
+
     if (config.setupTestFrameworkScriptFile) {
       moduleLoader.requireModule(config.setupTestFrameworkScriptFile);
     }

packages/jest-jasmine2/src/jasmine-check-install.js

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+'use strict';
+
+jest.unmock('jasmine-check');
+require('jasmine-check').install();
+
+// Replace the 'check' functions with ones that default to options from
+// the jest configuration:
+(() => {
+  const configOptions = global.jasmine.testcheckOptions;
+  const check = global.check;
+  const makeMergeOptions = (object, methodName) => {
+    const original = object[methodName];
+    object[methodName] = (specName, options, gens, propertyFn) => {
+      if (!propertyFn) {
+        propertyFn = gens;
+        gens = options;
+        options = {};
+      }
+      const mergedOptions = Object.assign({}, configOptions, options);
+      return original(specName, mergedOptions, gens, propertyFn);
+    };
+  };
+
+  makeMergeOptions(check, 'it');
+  makeMergeOptions(check, 'iit');
+  makeMergeOptions(check.it, 'only');
+  makeMergeOptions(check, 'fit');
+  makeMergeOptions(check, 'xit');
+})();