test(e2e): add base configuration and runner

.gitignore

@@ -4,3 +4,4 @@ node_modules
 coverage
 .nyc_output
 .rpt2_cache
+.env

e2e/browserstack-send-status.js

@@ -0,0 +1,87 @@
+const axios = require('axios')
+
+const BS_USER = process.env.BS_USER
+const BS_KEY = process.env.BS_KEY
+
+function getKey (client) {
+  // const { capabilities, currentTest } = client
+  // originally i wanted to use this but it turns out the information changes
+  // on the afterEach, making the key non valid. But because every environment
+  // runs on a different thread, the sessionMap object is only shared for a given
+  // environment, so only using the name of the test (currentTest.module) seems to be
+  // enough
+  // return `${capabilities.platform}::${capabilities.browserName}@${
+  //   capabilities.version
+  // } ${currentTest.module}: ${currentTest.name}`
+
+  return `${client.currentTest.module}: ${client.currentTest.name}`
+}
+
+function shouldSkipBrowserstackReporting (client) {
+  return !BS_USER || !BS_KEY || client.options.selenium_port !== 80
+}
+
+/**
+ * Generates an object with beforeEach and afterEach functions to be added
+ * to every test suite. It cannot be added globably because these must be
+ * executed before each test (instead of each test suite as it does in globalModules)
+ */
+module.exports = function sendStatus () {
+  const sessionMap = Object.create(null)
+
+  return {
+    beforeEach (browser, cb) {
+      // avoid running if missing credentials
+      if (shouldSkipBrowserstackReporting(this.client)) return cb()
+      // retrieve the session and save it to the map
+      const key = getKey(this.client)
+      browser.session(({ sessionId }) => {
+        sessionMap[key] = sessionId
+        cb()
+      })
+    },
+
+    afterEach (browser, cb) {
+      // avoid running if missing credentials
+      if (shouldSkipBrowserstackReporting(this.client)) return cb()
+      const key = getKey(this.client)
+      const { results } = this.client.currentTest
+      const sessionId = sessionMap[key]
+
+      if (!sessionId) {
+        console.warn('❌ Cannot find sessionId for ' + key)
+        return cb()
+      }
+
+      if (results.errors > 0 || results.failed > 0) {
+        const reason = results.lastError.message
+        console.log('Found failed test', reason)
+        axios
+          .put(
+            `https://api.browserstack.com/automate/sessions/${sessionId}.json`,
+            {
+              // change the name so it's easier to find
+              name: key,
+              status: 'failed',
+              reason
+            },
+            {
+              auth: {
+                username: BS_USER,
+                password: BS_KEY
+              }
+            }
+          )
+          .catch(err => {
+            console.log('❌ Failed changing status for sessions', err)
+          })
+          .then(() => {
+            console.log('✅ Sent for', sessionId)
+            cb()
+          })
+      } else {
+        cb()
+      }
+    }
+  }
+}

e2e/nightwatch.browserstack.js

@@ -0,0 +1,125 @@
+/**
+ * Running tests on remote browsers
+ */
+
+const BS_USER = process.env.BS_USER
+const BS_KEY = process.env.BS_KEY
+
+const nwConf = {
+  src_folders: ['test/e2e/specs'],
+  output_folder: 'test/e2e/reports',
+  custom_commands_path: ['node_modules/nightwatch-helpers/commands'],
+  custom_assertions_path: ['node_modules/nightwatch-helpers/assertions'],
+  // set to true when testing on multiple browsers (-e chrome,firefox) to display tests as they pass instead of waiting for everything to be finished
+  live_output: true,
+
+  // this couldn't work at the end, so we used ./browserstack-send-status.js
+  // globals_path: resolve(__dirname, './globalModules.js'),
+
+  selenium: {
+    start_process: false,
+    host: 'hub-cloud.browserstack.com',
+    port: 80
+  },
+
+  common_capabilities: {
+    'browserstack.user': BS_USER,
+    'browserstack.key': BS_KEY,
+    name: 'Bstack-[Nightwatch] Vue Router Parallel Test',
+    'browserstack.local': true,
+    'browserstack.video': false,
+    acceptSslCerts: true,
+    resolution: '1024x768'
+  },
+
+  test_settings: {
+    // default: {},
+
+    chrome: {
+      desiredCapabilities: {
+        browser: 'chrome'
+      }
+    },
+
+    chromeQt: {
+      desiredCapabilities: {
+        browser: 'chrome',
+        browser_version: '49.0'
+      }
+    },
+
+    firefox: {
+      desiredCapabilities: {
+        browser: 'firefox'
+      }
+    },
+
+    safari: {
+      desiredCapabilities: {
+        os: 'OS X',
+        os_version: 'Mojave',
+        browser: 'Safari',
+        browser_version: '12.0'
+      }
+    },
+
+    safari6: {
+      desiredCapabilities: {
+        os: 'OS X',
+        os_version: 'Lion',
+        browser: 'Safari',
+        browser_version: '6.0'
+      }
+    },
+
+    ie9: {
+      desiredCapabilities: {
+        browser: 'internet explorer',
+        browser_version: '9'
+        // name: 'Bstack-[Nightwatch] Vue Router',
+        // 'browserstack.video': true
+      }
+    },
+
+    ie: {
+      desiredCapabilities: {
+        browser: 'internet explorer',
+        browser_version: '11'
+        // name: 'Bstack-[Nightwatch] Vue Router',
+        // 'browserstack.video': true
+      }
+    },
+
+    android44: {
+      desiredCapabilities: {
+        device: 'Google Nexus 5',
+        realMobile: 'true',
+        os_version: '4.4'
+      }
+    },
+
+    ios7: {
+      desiredCapabilities: {
+        device: 'iPhone 7',
+        realMobile: 'true',
+        os_version: '10'
+      }
+    }
+  }
+}
+
+// Code to copy seleniumhost/port into test settings
+for (const setting in nwConf.test_settings) {
+  const config = nwConf.test_settings[setting]
+  config['selenium_host'] = nwConf.selenium.host
+  config['selenium_port'] = nwConf.selenium.port
+
+  // merge common_capabilities
+  for (const key in nwConf.common_capabilities) {
+    // fallback to common_capabilities
+    config['desiredCapabilities'][key] =
+      config['desiredCapabilities'][key] || nwConf.common_capabilities[key]
+  }
+}
+
+module.exports = nwConf

e2e/nightwatch.config.js

@@ -0,0 +1,76 @@
+// yarn nightwatch -e chrome,safari,firefox
+
+module.exports = {
+  src_folders: ['e2e/specs'],
+  output_folder: 'e2e/reports',
+  custom_commands_path: ['node_modules/nightwatch-helpers/commands'],
+  custom_assertions_path: ['node_modules/nightwatch-helpers/assertions'],
+  // set to true when testing on multiple browsers (-e chrome,firefox) to display tests as they pass instead of waiting for everything to be finished
+  live_output: false,
+
+  selenium: {
+    start_process: true,
+    server_path: require('selenium-server').path,
+    host: '127.0.0.1',
+    port: 4444,
+    cli_args: {
+      'webdriver.chrome.driver': require('chromedriver').path,
+      // 'webdriver.gecko.driver': require('geckodriver').path
+    },
+  },
+
+  test_settings: {
+    default: {
+      selenium_port: 4444,
+      selenium_host: 'localhost',
+      silent: true,
+      screenshots: {
+        enabled: true,
+        on_failure: true,
+        on_error: false,
+        path: 'e2e/screenshots',
+      },
+      desiredCapabilities: {
+        browserName: 'chrome',
+        acceptSslCerts: true,
+        chromeOptions: {
+          // https://github.com/nightwatchjs/nightwatch/releases/tag/v1.1.12
+          w3c: false,
+          args: ['window-size=1280,800', 'headless'],
+        },
+      },
+    },
+
+    chrome: {
+      desiredCapabilities: {
+        browserName: 'chrome',
+        acceptSslCerts: true,
+        chromeOptions: {
+          // https://github.com/nightwatchjs/nightwatch/releases/tag/v1.1.12
+          w3c: false,
+          args: ['window-size=1280,800'],
+        },
+      },
+    },
+
+    safari: {
+      desiredCapabilities: {
+        browserName: 'safari',
+        acceptSslCerts: true,
+      },
+    },
+
+    firefox: {
+      desiredCapabilities: {
+        browserName: 'firefox',
+        acceptSslCerts: true,
+      },
+    },
+
+    ie: {
+      desiredCapabilities: {
+        browser: 'internet explorer',
+      },
+    },
+  },
+}

e2e/reports/CHROME_77.0.3865.120_Mac_OS_X_basic.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuites errors="0"
+            failures="0"
+            tests="1">
+
+  <testsuite name="basic"
+    errors="0" failures="0" hostname="" id="" package="basic" skipped="0"
+    tests="1" time="0.6290" timestamp="">
+
+    <testcase name="basic" classname="basic" time="0.6290" assertions="7">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+    </testcase>
+
+
+
+
+
+  </testsuite>
+</testsuites>

e2e/reports/CHROME_77.0.3865.90_Mac_OS_X_basic.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuites errors="0"
+            failures="1"
+            tests="2">
+
+  <testsuite name="basic"
+    errors="0" failures="1" hostname="" id="" package="basic" skipped="0"
+    tests="2" time="2.193" timestamp="">
+
+    <testcase name="basic" classname="basic" time="1.054" assertions="1">  <failure message="Timed out while waiting for element &lt;#app&gt; to be present for 1000 milliseconds. - expected [0;32m&#34;visible&#34;[0m but got: [0;31m&#34;not found&#34;[0m">    at Object.basic (/Users/posva/vue-router-next/e2e/specs/basic.js:11:8)
+    at process._tickCallback (internal/process/next_tick.js:68:7)</failure>
+<system-out>[[ATTACHMENT|/Users/posva/vue-router-next/e2e/screenshots/basic/basic_FAILED_Oct-15-2019-185131-GMT+0200-(Central-European-Summer.png]]</system-out>
+    <failure message="    at Object.basic (/Users/posva/vue-router-next/e2e/specs/basic.js:11:8)">    at process._tickCallback (internal/process/next_tick.js:68:7)</failure>
+    </testcase>
+
+
+
+
+
+  </testsuite>
+</testsuites>