Improve how plugins are installed (#2102)

.gitignore

@@ -1,5 +1,4 @@
 node_modules
-!**/fixtures/**/node_modules
 research
 .DS_Store
 terraform
@@ -11,3 +10,5 @@ coverage
 .eslintcache
 .cache
 .vscode
+!**/fixtures/**/node_modules
+!**/fixtures/**/plugins_cache*/.netlify

packages/build/src/core/feature_flags.js

@@ -22,6 +22,8 @@ const DEFAULT_FEATURE_FLAGS = {
   service_buildbot_enable_deploy_server: false,
   // When `true`, the Deploy preview commenting core plugin is enabled
   dpc: false,
+  // When `true`, use the new plugins installation flow
+  new_plugins_install: false,
 }
 
 module.exports = { normalizeFeatureFlags, DEFAULT_FEATURE_FLAGS }

packages/build/src/install/main.js

@@ -12,9 +12,9 @@ const installDependencies = function ({ packageRoot, isLocal }) {
   return runCommand({ packageRoot, isLocal, type: 'install' })
 }
 
-// Add new Node.js dependencies
-const addDependencies = function ({ packageRoot, isLocal, packages }) {
-  return runCommand({ packageRoot, packages, isLocal, type: 'add' })
+// Add new Node.js dependencies, with exact semver ranges
+const addExactDependencies = function ({ packageRoot, isLocal, packages }) {
+  return runCommand({ packageRoot, packages, isLocal, type: 'addExact' })
 }
 
 const runCommand = async function ({ packageRoot, packages = [], isLocal, type }) {
@@ -39,7 +39,7 @@ const getCommand = async function ({ packageRoot, type, isLocal }) {
 
 const getManager = async function (type, packageRoot) {
   // `addDependencies()` always uses npm
-  if (type === 'add') {
+  if (type === 'addExact') {
     return 'npm'
   }
 
@@ -52,7 +52,7 @@ const getManager = async function (type, packageRoot) {
 
 const COMMANDS = {
   npm: {
-    add: ['npm', 'install', '--no-progress', '--no-audit', '--no-fund', '--no-save'],
+    addExact: ['npm', 'install', '--no-progress', '--no-audit', '--no-fund', '--save-exact'],
     install: ['npm', 'install', '--no-progress', '--no-audit', '--no-fund'],
   },
   yarn: {
@@ -82,4 +82,4 @@ const isNotNpmLogMessage = function (line) {
 }
 const NPM_LOG_MESSAGES = ['complete log of this run', '-debug.log']
 
-module.exports = { installDependencies, addDependencies }
+module.exports = { installDependencies, addExactDependencies }

packages/build/src/install/missing.js

@@ -8,7 +8,7 @@ const pathExists = require('path-exists')
 
 const { logInstallMissingPlugins, logConfigOnlyPlugins } = require('../log/messages/install')
 
-const { addDependencies } = require('./main')
+const { addExactDependencies } = require('./main')
 
 const pWriteFile = promisify(writeFile)
 
@@ -26,7 +26,7 @@ const installMissingPlugins = async function ({ pluginsOptions, autoPluginsDir,
   logInstallMissingPlugins(logs, packages)
 
   await createAutoPluginsDir(autoPluginsDir)
-  await addDependencies({ packageRoot: autoPluginsDir, isLocal: mode !== 'buildbot', packages })
+  await addExactDependencies({ packageRoot: autoPluginsDir, isLocal: mode !== 'buildbot', packages })
 }
 
 const getMissingPlugins = function (pluginsOptions) {
@@ -41,6 +41,7 @@ const getPackageName = function ({ packageName }) {
   return packageName
 }
 
+// We pin the version without using semver ranges ^ nor ~
 const getPackage = function ({ packageName, expectedVersion }) {
   return `${packageName}@${expectedVersion}`
 }
@@ -80,24 +81,30 @@ const AUTO_PLUGINS_PACKAGE_JSON = {
 }
 
 // External plugins must be installed either in the UI or in `package.json`.
-// A third way is deprecated: adding it to `netlify.toml` but not in
-// `package.json`. In that case, the plugin will be automatically installed
-// like UI plugins.
-// We still support this for backward compatibility but print a warning on each
-// build (even if the plugin was installed in a previous build).
-// We deprecate this third way because:
-//  - having fewer ways of installing plugins is simpler
-//  - using `package.json` is faster and more reliable
-const warnOnConfigOnlyPlugins = function ({ pluginsOptions, logs }) {
-  const packages = pluginsOptions.filter(isConfigOnlyPlugin).map(getPackageName)
+// A third way is supported but undocumented: adding it to `netlify.toml` but
+// not in `package.json`. In that case, the plugin will be automatically
+// installed.
+// This is only supported for plugins listed in `plugins.json`. Otherwise,
+// we should fail the build. At the moment, we only print a warning for
+// backward compatibility.
+const warnOnConfigOnlyPlugins = function ({ pluginsOptions, featureFlags, logs }) {
+  const packages = pluginsOptions
+    .filter(({ loadedFrom, origin, expectedVersion }) =>
+      isConfigOnlyPlugin({ loadedFrom, origin, expectedVersion, featureFlags }),
+    )
+    .map(getPackageName)
   if (packages.length === 0) {
     return
   }
 
   logConfigOnlyPlugins(logs, packages)
 }
 
-const isConfigOnlyPlugin = function ({ loadedFrom, origin }) {
+const isConfigOnlyPlugin = function ({ loadedFrom, origin, expectedVersion, featureFlags }) {
+  if (featureFlags.new_plugins_install) {
+    return expectedVersion === 'latest'
+  }
+
   return loadedFrom === 'auto_install' && origin === 'config'
 }
 

packages/build/src/plugins/expected_version.js

@@ -1,13 +1,65 @@
 'use strict'
 
-const addExpectedVersions = function ({ pluginsOptions }) {
+const { resolvePath } = require('../utils/resolve')
+
+const { getPluginsList } = require('./list')
+
+// When using plugins in our official list, those are installed in .netlify/plugins/
+// We ensure that the last version that's been approved is always the one being used.
+// We also ensure that the plugin is our official list.
+const addExpectedVersions = async function ({ pluginsOptions, autoPluginsDir, featureFlags, debug, logs, testOpts }) {
   if (!pluginsOptions.some(isAutoPlugin)) {
     return pluginsOptions
   }
 
+  if (featureFlags.new_plugins_install) {
+    const pluginsList = await getPluginsList({ debug, logs, testOpts })
+    return await Promise.all(
+      pluginsOptions.map((pluginOptions) => addExpectedVersion({ pluginsList, autoPluginsDir, pluginOptions })),
+    )
+  }
+
   return pluginsOptions.map(addExpectedVersionUnlessPath)
 }
 
+// Any `pluginOptions` with `expectedVersion` set will be automatically installed
+const addExpectedVersion = async function ({
+  pluginsList,
+  autoPluginsDir,
+  pluginOptions,
+  pluginOptions: { packageName, pluginPath, loadedFrom },
+}) {
+  if (!isAutoPlugin({ loadedFrom })) {
+    return pluginOptions
+  }
+
+  const expectedVersion = pluginsList[packageName]
+
+  // Plugins that are not in our official list can only be specified in
+  // `netlify.toml` providing they are also installed in the site's package.json.
+  // Otherwise, the build should fail. For backward compatibility, we only print
+  // a warning message at the moment.
+  if (expectedVersion === undefined) {
+    return { ...pluginOptions, expectedVersion: 'latest' }
+  }
+
+  // Plugin was not previously installed
+  if (pluginPath === undefined) {
+    return { ...pluginOptions, expectedVersion }
+  }
+
+  const packageJsonPath = await resolvePath(`${packageName}/package.json`, autoPluginsDir)
+  // eslint-disable-next-line node/global-require, import/no-dynamic-require
+  const { version } = require(packageJsonPath)
+
+  // Plugin was previously installed but a new version is available
+  if (version !== expectedVersion) {
+    return { ...pluginOptions, expectedVersion }
+  }
+
+  return pluginOptions
+}
+
 const isAutoPlugin = function ({ loadedFrom }) {
   return loadedFrom === 'auto_install'
 }

packages/build/src/plugins/list.js

@@ -45,8 +45,8 @@ const fetchPluginsList = async function ({ logs, pluginsListUrl }) {
 }
 
 const PLUGINS_LIST_URL = 'https://netlify-plugins.netlify.app/plugins.json'
-// 10 seconds HTTP request timeout
-const PLUGINS_LIST_TIMEOUT = 1e4
+// 1 minute HTTP request timeout
+const PLUGINS_LIST_TIMEOUT = 6e4
 
 const normalizePluginsList = function (pluginsList) {
   return fromEntries(pluginsList.map(normalizePluginItem))

packages/build/src/plugins/options.js

@@ -36,6 +36,7 @@ const tGetPluginsOptions = async function ({
     pluginsOptions,
     buildDir,
     mode,
+    featureFlags,
     logs,
     buildImagePluginsDir,
     debug,

packages/build/src/plugins/resolve.js

@@ -7,18 +7,17 @@ const { installMissingPlugins, warnOnConfigOnlyPlugins } = require('../install/m
 const { resolvePath, tryResolvePath } = require('../utils/resolve')
 
 const { addExpectedVersions } = require('./expected_version')
-const { getPluginsList } = require('./list.js')
 
 // Try to find plugins in four places, by priority order:
 //  - already loaded (core plugins)
 //  - local plugin
 //  - external plugin already installed in `node_modules`, most likely through `package.json`
-//  - cached in the build image
-//  - automatically installed by us (fallback)
+//  - automatically installed by us, to `.netlify/plugins/`
 const resolvePluginsPath = async function ({
   pluginsOptions,
   buildDir,
   mode,
+  featureFlags,
   logs,
   buildImagePluginsDir,
   debug,
@@ -27,13 +26,21 @@ const resolvePluginsPath = async function ({
   const autoPluginsDir = getAutoPluginsDir(buildDir)
   const pluginsOptionsA = await Promise.all(
     pluginsOptions.map((pluginOptions) =>
-      resolvePluginPath({ pluginOptions, buildDir, buildImagePluginsDir, debug, logs, testOpts }),
+      resolvePluginPath({ pluginOptions, buildDir, buildImagePluginsDir, featureFlags, autoPluginsDir }),
     ),
   )
-  const pluginsOptionsB = addExpectedVersions({ pluginsOptions: pluginsOptionsA })
+  const pluginsOptionsB = await addExpectedVersions({
+    pluginsOptions: pluginsOptionsA,
+    autoPluginsDir,
+    featureFlags,
+    debug,
+    logs,
+    testOpts,
+  })
   const pluginsOptionsC = await handleMissingPlugins({
     pluginsOptions: pluginsOptionsB,
     autoPluginsDir,
+    featureFlags,
     mode,
     logs,
   })
@@ -49,14 +56,14 @@ const getAutoPluginsDir = function (buildDir) {
 
 const AUTO_PLUGINS_DIR = '.netlify/plugins/'
 
+// eslint-disable-next-line complexity, max-statements
 const resolvePluginPath = async function ({
   pluginOptions,
   pluginOptions: { packageName, loadedFrom },
   buildDir,
   buildImagePluginsDir,
-  debug,
-  logs,
-  testOpts,
+  featureFlags,
+  autoPluginsDir,
 }) {
   // Core plugins
   if (loadedFrom !== undefined) {
@@ -78,17 +85,24 @@ const resolvePluginPath = async function ({
     return { ...pluginOptions, pluginPath: manualPath, loadedFrom: 'package.json' }
   }
 
-  await getPluginsList({ debug, logs, testOpts })
-
-  // Cached in the build image
-  const buildImagePath = await tryBuildImagePath({ packageName, buildDir, buildImagePluginsDir })
-  if (buildImagePath !== undefined) {
-    return { ...pluginOptions, pluginPath: buildImagePath, loadedFrom: 'image_cache' }
+  if (featureFlags.new_plugins_install) {
+    // Previously automatically installed
+    const { path: automaticPath } = await tryResolvePath(packageName, autoPluginsDir)
+    if (automaticPath !== undefined) {
+      return { ...pluginOptions, pluginPath: automaticPath, loadedFrom: 'auto_install' }
+    }
+  } else {
+    // Cached in the build image
+    const buildImagePath = await tryBuildImagePath({ packageName, buildDir, buildImagePluginsDir })
+    if (buildImagePath !== undefined) {
+      return { ...pluginOptions, pluginPath: buildImagePath, loadedFrom: 'image_cache' }
+    }
   }
 
   // Happens if the plugin:
   //  - name is mispelled
   //  - is not in our official list
+  //  - is in our official list but has not been installed by this site yet
   return { ...pluginOptions, loadedFrom: 'auto_install' }
 }
 
@@ -125,14 +139,14 @@ const tryBuildImagePath = async function ({ packageName, buildDir, buildImagePlu
   return resolvePath(buildImagePath, buildDir)
 }
 
-// Handle plugins that were neither local, in the build image cache nor in
-// node_modules. We automatically install those, with a warning.
-const handleMissingPlugins = async function ({ pluginsOptions, autoPluginsDir, mode, logs }) {
+// Install plugins from the official list that have not been previously installed.
+// Print a warning if they have not been installed through the UI.
+const handleMissingPlugins = async function ({ pluginsOptions, autoPluginsDir, featureFlags, mode, logs }) {
   await installMissingPlugins({ pluginsOptions, autoPluginsDir, mode, logs })
   const pluginsOptionsA = await Promise.all(
     pluginsOptions.map((pluginOptions) => resolveMissingPluginPath({ pluginOptions, autoPluginsDir })),
   )
-  warnOnConfigOnlyPlugins({ pluginsOptions: pluginsOptionsA, logs })
+  warnOnConfigOnlyPlugins({ pluginsOptions: pluginsOptionsA, featureFlags, logs })
   return pluginsOptionsA
 }
 

packages/build/tests/helpers/normalize.js

@@ -69,7 +69,7 @@ const NORMALIZE_REGEXPS = [
   [/ \.\.\.:443/g, ''],
   // List of available plugins from `plugins.json`.
   // That list changes all the time, so we need to remove it.
-  [/(Available plugins)[^]*Loading plugins/m, '$1'],
+  [/(Available plugins)[^]*(Missing|Loading|Installing) plugins/m, '$1'],
 ]
 
 module.exports = { normalizeOutput }

packages/build/tests/plugins/fixtures/invalid_package/netlify.toml

@@ -0,0 +1,2 @@
+[[plugins]]
+package = "power-cartesian-product"

packages/build/tests/plugins/fixtures/plugins_cache/.netlify/plugins/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "netlify-local-plugins",
+  "description": "This directory contains Build plugins that have been automatically installed by Netlify.",
+  "version": "1.0.0",
+  "private": true,
+  "author": "Netlify",
+  "license": "MIT",
+  "dependencies": {
+    "netlify-plugin-contextual-env": "0.3.0"
+  }
+}

packages/build/tests/plugins/fixtures/plugins_cache/netlify.toml

@@ -0,0 +1,2 @@
+[[plugins]]
+package = "netlify-plugin-contextual-env"

packages/build/tests/plugins/fixtures/plugins_cache/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "module_plugin",
+  "version": "0.0.1",
+  "description": "test",
+  "license": "MIT",
+  "repository": "test",
+  "dependencies": {}
+}