Snapshot failures (jestjs#1308)

* namespace snapshot test results

* Snapshot test results

packages/jest-cli/src/TestRunner.js

@@ -158,21 +158,42 @@ class TestRunner {
 
     const testRun = this._createTestRun(testPaths, onTestResult, onRunFailure);
 
+    const setSuccess = () => {
+      const anyTestFailures = !(
+        aggregatedResults.numFailedTests === 0 &&
+        aggregatedResults.numRuntimeErrorTestSuites === 0
+      );
+
+      const anyReporterErrors = this._dispatcher.hasErrors();
+
+      aggregatedResults.success = !(
+        anyTestFailures ||
+        aggregatedResults.snapshot.failure ||
+        anyReporterErrors
+      );
+    };
+
+    const updateSnapshotSummary = () => {
+      return snapshot.cleanup(this._hasteContext, config.updateSnapshot)
+        .then(status => {
+          aggregatedResults.snapshot.filesRemoved += status.filesRemoved;
+          aggregatedResults.snapshot.didUpdate = config.updateSnapshot;
+          aggregatedResults.snapshot.failure = !!(
+            !aggregatedResults.snapshot.didUpdate && (
+              aggregatedResults.snapshot.unchecked ||
+              aggregatedResults.snapshot.unmatched ||
+              aggregatedResults.snapshot.filesRemoved
+            )
+          );
+        });
+    };
+
     return testRun
-      .then(() => {
-        aggregatedResults.success =
-          aggregatedResults.numFailedTests === 0 &&
-          aggregatedResults.numRuntimeErrorTestSuites === 0;
-        return snapshot.cleanup(this._hasteContext, config.updateSnapshot)
-          .then(status => {
-            aggregatedResults.snapshotFilesRemoved = status.filesRemoved;
-            aggregatedResults.didUpdate = config.updateSnapshot;
-            this._dispatcher.onRunComplete(config, aggregatedResults);
-            aggregatedResults.success = !this._dispatcher.hasErrors();
-            return aggregatedResults;
-          });
-      })
-      .then(results => this._cacheTestResults(results).then(() => results));
+      .then(updateSnapshotSummary)
+      .then(() => this._dispatcher.onRunComplete(config, aggregatedResults))
+      .then(setSuccess)
+      .then(() => this._cacheTestResults(aggregatedResults))
+      .then(() => aggregatedResults);
   }
 
   _createTestRun(
@@ -259,7 +280,6 @@ class TestRunner {
 
 const createAggregatedResults = (numTotalTestSuites: number) => {
   return {
-    didUpdate: false,
     numFailedTests: 0,
     numFailedTestSuites: 0,
     numPassedTests: 0,
@@ -268,7 +288,21 @@ const createAggregatedResults = (numTotalTestSuites: number) => {
     numRuntimeErrorTestSuites: 0,
     numTotalTests: 0,
     numTotalTestSuites,
-    snapshotFilesRemoved: 0,
+    snapshot: {
+      added: 0,
+      didUpdate: false, // is set only after the full run
+      failure: false,
+      filesAdded: 0,
+      // combines individual test results + results after full run
+      filesRemoved: 0,
+      filesUnmatched: 0,
+      filesUpdated: 0,
+      matched: 0,
+      total: 0,
+      unchecked: 0,
+      unmatched: 0,
+      updated: 0,
+    },
     startTime: Date.now(),
     success: false,
     testResults: [],
@@ -293,9 +327,40 @@ const addResult = (
   } else {
     aggregatedResults.numPassedTestSuites++;
   }
+
+  // Snapshot data
+
+
+  if (testResult.snapshot.added) {
+    aggregatedResults.snapshot.filesAdded++;
+  }
+  if (testResult.snapshot.fileDeleted) {
+    aggregatedResults.snapshot.filesRemoved++;
+  }
+  if (testResult.snapshot.unmatched) {
+    aggregatedResults.snapshot.filesUnmatched++;
+  }
+  if (testResult.snapshot.updated) {
+    aggregatedResults.snapshot.filesUpdated++;
+  }
+  if (testResult.hasUncheckedKeys) {
+    aggregatedResults.snapshot.unchecked++;
+  }
+
+  aggregatedResults.snapshot.added += testResult.snapshot.added;
+  aggregatedResults.snapshot.matched += testResult.snapshot.matched;
+  aggregatedResults.snapshot.unmatched += testResult.snapshot.unmatched;
+  aggregatedResults.snapshot.updated += testResult.snapshot.updated;
+  aggregatedResults.snapshot.total +=
+    testResult.snapshot.added +
+    testResult.snapshot.matched +
+    testResult.snapshot.updated;
 };
 
-const buildFailureTestResult = (testPath: string, err: TestError) => {
+const buildFailureTestResult = (
+  testPath: string,
+  err: TestError
+): TestResult => {
   return {
     hasUncheckedKeys: false,
     numFailingTests: 1,
@@ -305,11 +370,13 @@ const buildFailureTestResult = (testPath: string, err: TestError) => {
       end: 0,
       start: 0,
     },
-    snapshotFileDeleted: false,
-    snapshotsAdded: 0,
-    snapshotsMatched: 0,
-    snapshotsUnmatched: 0,
-    snapshotsUpdated: 0,
+    snapshot: {
+      fileDeleted: false,
+      added: 0,
+      matched: 0,
+      unmatched: 0,
+      updated: 0,
+    },
     testExecError: err,
     testFilePath: testPath,
     testResults: [],

packages/jest-cli/src/lib/formatTestResults.js

@@ -13,7 +13,7 @@
 const utils = require('jest-util');
 
 import type {
-  AggregatedTestResults,
+  AggregatedResult,
   CodeCoverageFormatter,
   CodeCoverageReporter,
   TestResult,
@@ -59,7 +59,7 @@ const formatResult = (
 };
 
 function formatTestResults(
-  results: AggregatedTestResults,
+  results: AggregatedResult,
   config: Config,
   codeCoverageFormatter?: CodeCoverageFormatter,
   reporter?: CodeCoverageReporter,

packages/jest-cli/src/reporters/NotifyReporter.js

@@ -25,8 +25,11 @@ class NotifyReporter extends BaseReporter {
   onRunComplete(config: Config, result: AggregatedResult): void {
     let title;
     let message;
+    const success = result.numFailedTests === 0 &&
+      result.numRuntimeErrorTestSuites === 0;
 
-    if (result.success) {
+
+    if (success) {
       title = util.format('%d%% Passed', 100);
       message = util.format(
         (isDarwin ? '\u2705 ' : '') + '%d tests passed',

packages/jest-cli/src/reporters/SummaryReporter.js

@@ -9,25 +9,12 @@
  */
 'use strict';
 
-import type {AggregatedResult} from 'types/TestResult';
+import type {AggregatedResult, SnapshotSummary} from 'types/TestResult';
 import type {Config} from 'types/Config';
 
 const chalk = require('chalk');
 const BaseReporter = require('./BaseReporter');
 
-type SnapshotSummary = {
-  added: number,
-  didUpdate: boolean,
-  filesAdded: number,
-  filesRemoved: number,
-  filesUnmatched: number,
-  filesUpdated: number,
-  matched: number,
-  total: number,
-  unchecked: number,
-  unmatched: number,
-  updated: number,
-};
 
 const FAIL_COLOR = chalk.bold.red;
 const PASS_COLOR = chalk.bold.green;
@@ -50,16 +37,11 @@ class SummareReporter extends BaseReporter {
     const totalErrors = aggregatedResults.numRuntimeErrorTestSuites;
     const runTime = (Date.now() - aggregatedResults.startTime) / 1000;
 
-    const snapshots = this._getSnapshotSummary(aggregatedResults);
-    const snapshotFailure = !!(!snapshots.didUpdate && (
-      snapshots.unchecked ||
-      snapshots.unmatched ||
-      snapshots.filesRemoved
-    ));
+    const snapshots = aggregatedResults.snapshot;
 
     let results = '';
 
-    if (snapshotFailure) {
+    if (snapshots.failure) {
       results += FAIL_COLOR('snapshot failure') + ', ';
     }
 
@@ -87,56 +69,6 @@ class SummareReporter extends BaseReporter {
     this._printSummary(aggregatedResults);
     this._printSnapshotSummary(snapshots);
     this.log(results);
-
-    if (failedTests || totalErrors || snapshotFailure) {
-      this._setError(new Error('Some of the tests failed'));
-    }
-  }
-
-  _getSnapshotSummary(aggregatedResults: AggregatedResult): SnapshotSummary {
-    let added = 0;
-    let filesAdded = 0;
-    let filesRemoved = aggregatedResults.snapshotFilesRemoved;
-    let filesUnmatched = 0;
-    let filesUpdated = 0;
-    let matched = 0;
-    let unchecked = 0;
-    let unmatched = 0;
-    let updated = 0;
-    aggregatedResults.testResults.forEach(result => {
-      if (result.snapshotsAdded) {
-        filesAdded++;
-      }
-      if (result.snapshotFileDeleted) {
-        filesRemoved++;
-      }
-      if (result.snapshotsUnmatched) {
-        filesUnmatched++;
-      }
-      if (result.snapshotsUpdated) {
-        filesUpdated++;
-      }
-      if (result.hasUncheckedKeys) {
-        unchecked++;
-      }
-      added += result.snapshotsAdded;
-      matched += result.snapshotsMatched;
-      unmatched += result.snapshotsUnmatched;
-      updated += result.snapshotsUpdated;
-    });
-    return {
-      added,
-      didUpdate: aggregatedResults.didUpdate,
-      filesAdded,
-      filesRemoved,
-      filesUnmatched,
-      filesUpdated,
-      matched,
-      total: matched + added + updated,
-      unchecked,
-      unmatched,
-      updated,
-    };
   }
 
   _printSnapshotSummary(snapshots: SnapshotSummary) {

packages/jest-jasmine2/src/index.js

@@ -314,11 +314,11 @@ function jasmine2(
     const status = currentSnapshot.save(updateSnapshot);
 
     results.hasUncheckedKeys = !status.deleted && hasUncheckedKeys;
-    results.snapshotFileDeleted = status.deleted;
-    results.snapshotsAdded = snapshotState.added;
-    results.snapshotsMatched = snapshotState.matched;
-    results.snapshotsUnmatched = snapshotState.unmatched;
-    results.snapshotsUpdated = snapshotState.updated;
+    results.snapshot.fileDeleted = status.deleted;
+    results.snapshot.added = snapshotState.added;
+    results.snapshot.matched = snapshotState.matched;
+    results.snapshot.unmatched = snapshotState.unmatched;
+    results.snapshot.updated = snapshotState.updated;
     return results;
   });
 }

packages/jest-jasmine2/src/reporter.js

@@ -91,6 +91,7 @@ class Jasmine2Reporter {
       numPassingTests,
       numPendingTests,
       testResults,
+      snapshot: {},
     });
   }
 

types/TestResult.js

@@ -42,7 +42,6 @@ export type AssertionResult = {
 };
 
 export type AggregatedResult = {
-  didUpdate: boolean,
   numFailedTests: number,
   numFailedTestSuites: number,
   numPassedTests: number,
@@ -51,7 +50,7 @@ export type AggregatedResult = {
   numRuntimeErrorTestSuites: number,
   numTotalTests: number,
   numTotalTestSuites: number,
-  snapshotFilesRemoved: number,
+  snapshot: SnapshotSummary,
   startTime: number,
   success: boolean,
   testResults: Array<TestResult>,
@@ -75,30 +74,18 @@ export type TestResult = {
     end: Milliseconds,
     start: Milliseconds,
   },
-  snapshotFileDeleted: boolean,
-  snapshotsAdded: number,
-  snapshotsMatched: number,
-  snapshotsUnmatched: number,
-  snapshotsUpdated: number,
+  snapshot: {
+    fileDeleted: boolean,
+    added: number,
+    matched: number,
+    unmatched: number,
+    updated: number,
+  },
   testExecError: Error,
   testFilePath: string,
   testResults: Array<AssertionResult>,
 };
 
-export type AggregatedTestResults = {
-  success: ?boolean,
-  startTime: Date,
-  numTotalTestSuites: number,
-  numPassedTestSuites: number,
-  numFailedTestSuites: number,
-  numRuntimeErrorTestSuites: number,
-  numTotalTests: number,
-  numPassedTests: number,
-  numFailedTests: number,
-  numPendingTests: number,
-  testResults: Array<TestResult>,
-};
-
 export type CodeCoverageResult = {
   coveredSpans: Array<Object>,
   uncoveredSpans: Array<Object>,
@@ -111,3 +98,19 @@ export type CodeCoverageFormatter = (
   coverage: ?CodeCoverageResult,
   reporter?: CodeCoverageReporter,
 ) => ?Object;
+
+
+export type SnapshotSummary = {
+  added: number,
+  didUpdate: boolean,
+  failure: boolean,
+  filesAdded: number,
+  filesRemoved: number,
+  filesUnmatched: number,
+  filesUpdated: number,
+  matched: number,
+  total: number,
+  unchecked: number,
+  unmatched: number,
+  updated: number,
+};