ci(docs-infra): test PR previews on CI (angular#25671)

The deployment of PR previews is triggered by the notification webhook
of the `aio_preview` CircleCI job (which creates and stores the build
artifacts).

This commit adds a new job (`test_aio_preview`), which waits for the
preview to be deployed (for PRs that do have a preview) and then runs
some tests against it (currently only PWA tests).

Fixes angular#23818

PR Close angular#25671

.circleci/config.yml

@@ -158,6 +158,21 @@ jobs:
           # `AIO_ARTIFACT_PATH` in `aio/aio-builds-setup/Dockerfile`
           destination: aio/dist/aio-snapshot.tgz
 
+  # This job should only be run on PR builds, where `CIRCLE_PR_NUMBER` is defined.
+  test_aio_preview:
+    <<: *job_defaults
+    steps:
+      - checkout:
+          <<: *post_checkout
+      - restore_cache:
+          key: *cache_key
+      - run: yarn install --cwd aio --frozen-lockfile --non-interactive
+      - run:
+          name: Wait for preview and run tests
+          command: |
+            source "./scripts/ci/env.sh" print
+            xvfb-run --auto-servernum node scripts/test-preview.js $CIRCLE_PR_NUMBER $CIRCLE_SHA1 $AIO_MIN_PWA_SCORE
+
   # This job exists only for backwards-compatibility with old scripts and tests
   # that rely on the pre-Bazel dist/packages-dist layout.
   # It duplicates some work with the job above: we build the bazel packages
@@ -257,6 +272,9 @@ workflows:
           filters:
             branches:
               only: /pull\/\d+/
+      - test_aio_preview:
+          requires:
+            - aio_preview
       - integration_test:
           requires:
             - build-packages-dist

aio/scripts/deploy-to-firebase.sh

@@ -104,5 +104,5 @@ fi
   firebase deploy --message "Commit: $TRAVIS_COMMIT" --non-interactive --token "$firebaseToken"
 
   # Run PWA-score tests
-  yarn test-pwa-score "$deployedUrl" "$MIN_PWA_SCORE"
+  yarn test-pwa-score "$deployedUrl" "$AIO_MIN_PWA_SCORE"
 )

aio/scripts/test-preview.js

@@ -0,0 +1,121 @@
+#!/usr/bin/env node
+
+/**
+ * Usage:
+ *   node scripts/test-preview <pr-number> <pr-last-sha> <min-pwa-score>
+ *
+ * Checks whether a PR will (eventually) have a (public) preview, waits for the preview to be
+ * available, and runs PWA tests against the preview.
+ *
+ * For PRs that are expected to have a preview, this script will fail if the preview is still not
+ * available after a pre-defined waiting period or if the PWA tests fail.
+ */
+
+// Imports
+const {spawn} = require('child_process');
+const {get: httpsGet} = require('https');
+const {relative} = require('path');
+
+// Input
+const [prNumber, prLastSha, minPwaScore] = validateArgs(process.argv.slice(2));
+
+// Variables
+const aioBuildsDomain = 'ngbuilds.io';
+const previewCheckInterval = 30000;
+const previewCheckAttempts = 10;
+
+const shortSha = prLastSha && prLastSha.slice(0, 7);
+const previewabilityCheckUrl = `https://${aioBuildsDomain}/can-have-public-preview/${prNumber}`;
+const previewUrl = `https://pr${prNumber}-${shortSha}.${aioBuildsDomain}/`;
+
+// Check whether the PR can have a (public) preview.
+get(previewabilityCheckUrl).
+  then(response => JSON.parse(response)).
+  then(({canHavePublicPreview, reason}) => {
+    // Nothing to do, if this PR can have no (public) preview.
+    if (canHavePublicPreview === false) {
+      reportNoPreview(reason);
+      return;
+    }
+
+    // There should be a preview. Wait for it to be available.
+    return poll(previewCheckInterval, previewCheckAttempts, () => get(previewUrl)).
+      // The preview is still not available after the specified waiting period.
+      catch(() => {
+        const totalSecs = Math.round((previewCheckInterval * previewCheckAttempts) / 1000);
+        throw new Error(`Preview still not available after ${totalSecs}s.`);
+      }).
+      // The preview is now available. Run the PWA tests.
+      then(() => runPwaTests());
+  }).
+  catch(onError);
+
+// Helpers
+function get(url) {
+  console.log(`GET ${url}`);
+  return new Promise((resolve, reject) => {
+    const onResponse = res => {
+      const statusCode = res.statusCode || -1;
+      const isSuccess = (200 <= statusCode) && (statusCode < 400);
+      let responseText = '';
+
+      res.
+        on('error', reject).
+        on('data', d => responseText += d).
+        on('end', () => isSuccess ?
+          resolve(responseText) :
+          reject(`Request to '${url}' failed (status: ${statusCode}): ${responseText}`));
+    };
+
+    httpsGet(url, onResponse).
+      on('error', reject);
+  });
+}
+
+function onError(err) {
+  console.error(err);
+  process.exit(1);
+}
+
+function poll(interval, attempts, checkCondition) {
+  return new Promise((resolve, reject) => {
+    if (!attempts) return reject();
+
+    checkCondition().
+      then(() => resolve()).
+      catch(() => wait(interval).
+        then(() => poll(interval, attempts - 1, checkCondition)).
+        then(resolve, reject));
+  });
+}
+
+function reportNoPreview(reason) {
+  console.log(`No (public) preview available. (Reason: ${reason})`);
+}
+
+function runPwaTests() {
+  return new Promise((resolve, reject) => {
+    const spawnOptions = {cwd: __dirname, stdio: 'inherit'};
+    spawn('yarn', ['test-pwa-score', previewUrl, minPwaScore], spawnOptions).
+      on('error', reject).
+      on('exit', code => (code === 0 ? resolve : reject)());
+  });
+}
+
+function validateArgs(args) {
+  if (args.length !== 3) {
+    const relativeScriptPath = relative('.', __filename.replace(/\.js$/, ''));
+    const usageCmd = `node ${relativeScriptPath} <pr-number> <pr-last-sha> <min-pwa-score>`;
+
+    return onError(
+      `Invalid number of arguments (expected 3, found ${args.length}).\n` +
+      `Usage: ${usageCmd}`);
+  }
+
+  return args;
+}
+
+function wait(delay) {
+  console.log(`Waiting ${delay}ms...`);
+  return new Promise(resolve => setTimeout(resolve, delay));
+}

scripts/ci/env.sh

@@ -40,6 +40,7 @@ setEnvVar CHROMIUM_VERSION 561733  # Chrome 68 linux stable, see https://www.chr
 setEnvVar CHROMEDRIVER_VERSION_ARG "--versions.chrome 2.41"
 setEnvVar SAUCE_CONNECT_VERSION 4.4.9
 setEnvVar ANGULAR_CLI_VERSION 1.6.3
+setEnvVar AIO_MIN_PWA_SCORE 95
 setEnvVar PROJECT_ROOT $(cd ${thisDir}/../..; pwd)
 
 if [[ ${TRAVIS:-} ]]; then
@@ -63,8 +64,6 @@ if [[ ${TRAVIS:-} ]]; then
       # Determine the current stable branch.
       readonly versionRe="^\s*([0-9]+\.[0-9]+)\.[0-9]+.*$"
       setEnvVar STABLE_BRANCH `npm info @angular/core dist-tags.latest | sed -r "s/$versionRe/\1.x/"`
-
-      setEnvVar MIN_PWA_SCORE 95
       ;;
   esac
 else