Merge pull request snyk#706 from snyk/feat/prune-test-graph

feat: prune graph on test if asked

src/lib/errors/index.ts

@@ -12,3 +12,4 @@ export {FailedToGetVulnerabilitiesError} from './failed-to-get-vulnerabilities-e
 export {UnsupportedFeatureFlagError} from './unsupported-feature-flag-error';
 export {UnsupportedPackageManagerError} from './unsupported-package-manager-error';
 export {FailedToRunTestError} from './failed-to-run-test-error';
+export {TooManyVulnPaths} from './too-many-vuln-paths';

src/lib/errors/too-many-vuln-paths.ts

@@ -0,0 +1,14 @@
+import {CustomError} from './custom-error';
+
+export class TooManyVulnPaths extends CustomError {
+  private static ERROR_CODE: number = 413;
+  private static ERROR_STRING_CODE: string = 'TOO_MANY_VULN_PATHS';
+  private static ERROR_MESSAGE: string = 'Too many vulnerable paths to process the project';
+
+  constructor() {
+    super(TooManyVulnPaths.ERROR_MESSAGE);
+    this.code = TooManyVulnPaths.ERROR_CODE;
+    this.strCode = TooManyVulnPaths.ERROR_STRING_CODE;
+    this.userMessage = TooManyVulnPaths.ERROR_MESSAGE;
+  }
+}

src/lib/monitor.ts

@@ -12,7 +12,8 @@ import { SingleDepRootResult, DepTree, MonitorMeta, MonitorResult } from './type
 import * as projectMetadata from './project-metadata';
 import * as path from 'path';
 import {MonitorError, ConnectionTimeoutError} from './errors';
-import { SupportedPackageManagers, GRAPH_SUPPORTED_PACKAGE_MANAGERS } from './package-managers';
+import { countPathsToGraphRoot, pruneGraph } from './prune';
+import { GRAPH_SUPPORTED_PACKAGE_MANAGERS } from './package-managers';
 
 const debug = Debug('snyk');
 
@@ -332,44 +333,6 @@ export async function monitorGraph(
   });
 }
 
-function countPathsToGraphRoot(graph: depGraphLib.DepGraph): number {
-  return graph
-    .getPkgs()
-    .map((pkg) => graph.countPathsToRoot(pkg))
-    .reduce((acc, pathsToRoot) => acc + pathsToRoot, 0);
-}
-
-async function pruneGraph(
-    depGraph: depGraphLib.DepGraph,
-    packageManager: SupportedPackageManagers,
-  ): Promise<depGraphLib.DepGraph> {
-    try {
-    const threshold = config.PRUNE_DEPS_THRESHOLD; // Arbitrary threshold for maximum number of elements in the tree
-    const prunedTree: DepTree = (await depGraphLib.legacy.graphToDepTree(
-      depGraph,
-      packageManager,
-      { deduplicateWithinTopLevelDeps: true },
-    )) as DepTree;
-
-    const prunedGraph = await depGraphLib.legacy.depTreeToGraph(
-      prunedTree,
-      packageManager,
-    );
-    const count = countPathsToGraphRoot(prunedGraph);
-    debug('prunedPathsCount: ' +  count);
-
-    if (count < threshold) {
-      return prunedGraph;
-    }
-    debug('Too many vulnerable paths to monitor the project');
-    // TODO: create a custom error for this
-    throw new Error('Too many vulnerable paths to monitor the project');
-  } catch (e) {
-    debug('Failed to prune the graph, returning original:' + e);
-    return depGraph;
-  }
-}
-
 function pluckPolicies(pkg) {
   if (!pkg) {
     return null;

src/lib/prune.ts

@@ -0,0 +1,44 @@
+import * as _debug from 'debug';
+import { DepGraph, legacy } from '@snyk/dep-graph';
+
+import { DepTree } from './types';
+import * as config from './config';
+import { TooManyVulnPaths } from './errors';
+import { SupportedPackageManagers } from './package-managers';
+
+const debug = _debug('snyk:prune');
+
+const {depTreeToGraph, graphToDepTree} = legacy;
+
+export function countPathsToGraphRoot(graph: DepGraph): number {
+  return graph.getPkgs().reduce((acc, pkg) => acc + graph.countPathsToRoot(pkg), 0);
+}
+
+export async function pruneGraph(
+  depGraph: DepGraph,
+  packageManager: SupportedPackageManagers,
+): Promise<DepGraph> {
+  try {
+    // Arbitrary threshold for maximum number of elements in the tree
+    const threshold = config.PRUNE_DEPS_THRESHOLD;
+    const prunedTree = (await graphToDepTree(
+      depGraph,
+      packageManager,
+      { deduplicateWithinTopLevelDeps: true },
+    )) as DepTree;
+
+    const prunedGraph = await depTreeToGraph(prunedTree, packageManager);
+    const count = countPathsToGraphRoot(prunedGraph);
+    debug('prunedPathsCount: ' +  count);
+
+    if (count < threshold) {
+      return prunedGraph;
+    }
+
+    debug('Too many vulnerable paths to process the project');
+    throw new TooManyVulnPaths();
+  } catch (e) {
+    debug('Failed to prune the graph, returning original: ' + e);
+    return depGraph;
+  }
+}

src/lib/snyk-test/run-test.ts

@@ -25,6 +25,7 @@ import {
 } from '../errors';
 import { maybePrintDeps } from '../print-deps';
 import { SupportedPackageManagers } from '../package-managers';
+import { countPathsToGraphRoot, pruneGraph } from '../prune';
 
 // tslint:disable-next-line:no-var-requires
 const debug = require('debug')('snyk');
@@ -72,7 +73,6 @@ async function runTest(packageManager: SupportedPackageManagers,
       if (payload.body && payload.body.docker && payload.body.docker.dockerfilePackages) {
         dockerfilePackages = payload.body.docker.dockerfilePackages;
       }
-      await spinner.clear<void>(spinnerLbl)();
       await spinner(spinnerLbl);
       analytics.add('depGraph', !!depGraph);
       analytics.add('isDocker', !!(payload.body && payload.body.docker));
@@ -333,9 +333,22 @@ async function assembleLocalPayloads(root, options: Options & TestOptions): Prom
         // Graphs are more compact and robust representations.
         // Legacy parts of the code are still using trees, but will eventually be fully migrated.
         debug('converting dep-tree to dep-graph', {name: pkg.name, targetFile: depRoot.targetFile || options.file});
-        const depGraph = await depGraphLib.legacy.depTreeToGraph(
+        let depGraph = await depGraphLib.legacy.depTreeToGraph(
           pkg, options.packageManager);
+
         debug('done converting dep-tree to dep-graph', {uniquePkgsCount: depGraph.getPkgs().length});
+        if (options['prune-repeated-subdependencies']) {
+          debug('Trying to prune the graph');
+          const prePruneDepCount = countPathsToGraphRoot(depGraph);
+          debug('pre prunedPathsCount: ' +  prePruneDepCount);
+
+          depGraph = await pruneGraph(depGraph, options.packageManager);
+
+          analytics.add('prePrunedPathsCount', prePruneDepCount);
+          const postPruneDepCount = countPathsToGraphRoot(depGraph);
+          debug('post prunedPathsCount: ' +  postPruneDepCount);
+          analytics.add('postPrunedPathsCount', postPruneDepCount);
+        }
         body.depGraph = depGraph;
       }
 

src/lib/types.ts

@@ -62,6 +62,7 @@ export function isMultiResult(pet: SingleDepRootResult | MultiDepRootsResult): p
 export interface TestOptions {
   traverseNodeModules: boolean;
   interactive: boolean;
+  'prune-repeated-subdependencies'?: boolean;
 }
 export interface ProtectOptions {
   loose: boolean;