Merge pull request #210 from replayio/PRO-583

Our test reporter docs suggest using a deprecated `filter` option for advanced configurations. Instead we should be using the new advanced `upload` options supported by both Cypress and Playwright plugins.
replayio · Jul 5, 2024 · 59bbe22 · 59bbe22
2 parents 003470d + 840d2d2
commit 59bbe22
Show file tree

Hide file tree

Showing 2 changed files with 103 additions and 109 deletions.
diff --git a/README.md b/README.md
@@ -13,13 +13,15 @@ Docs are built with:
 ## Installation
 
 ```bash
-npm i
+pnpm install
 ```
 
 ## Local development
 
+Note that Node 18 is currently required to run this web server successfully.
+
 ```
-npm run dev
+pnpm run dev
 ```
 
 ## Issues

diff --git a/src/app/reference/ci-workflows/upload-strategies/page.md b/src/app/reference/ci-workflows/upload-strategies/page.md
@@ -1,34 +1,27 @@
 ---
 title: Upload strategies
-description: When you run tests and create recordings, they are stored locally. You can opt to upload them automatically or define your own uploading strategy. All uploaded recordings become accessible in the Replay App.
+description: When you run tests and create recordings, they are stored locally. You can choose which recordings get uploaded to Replay. All uploaded recordings become accessible in the Replay App.
 ---
 
 {% callout %}
-While uploading just failed test is good for saving resources, our recommendation is to upload both failed and passed tests so that you can compare them. This can be really useful for debugging purposes.
+While uploading just failed tests is good for saving resources, our recommendation is to upload both failed and passed tests so that you can compare them. This can be really useful for debugging purposes.
 {% /callout %}
 
-## Upload failed tests only
+## Upload all tests
 
-By default, all test replays are uploaded no matter the result. If you want to upload only the failed recordings, you can use the `filter` property in the plugin configuration:
+To upload all test replays no matter the result, set the `upload` option to true.
 
 {% tabs labels=["cypress", "playwright"] %}
 {% tab %}
 
-```js
+```js {% fileName="cypress.config.js" highlight=[7] %}
 export default defineConfig({
  e2e: {
  setupNodeEvents(cyOn, config) {
  const on = wrapOn(cyOn)
  replayPlugin(on, config, {
- upload: true,
  apiKey: process.env.REPLAY_API_KEY,
- filter: function (recording) {
- // upload runtime crashes and any failed tests
- return (
- recording.status === 'crashed' ||
- recording.metadata.test.result === 'failed'
- )
- },
+ upload: true,
  })
  return config
  },
@@ -39,21 +32,14 @@ export default defineConfig({
 {% /tab %}
 {% tab %}
 
-```js
+```js {% fileName="playwright.config.js" highlight=[7] %}
 import { replayDevices, replayReporter } from "@replayio/playwright";
 
 const config: PlaywrightTestConfig = {
  reporter: [
  replayReporter({
  apiKey: process.env.REPLAY_API_KEY,
  upload: true,
- filter: function (recording) {
- // upload runtime crashes and any failed tests
- return (
- recording.status === "crashed" ||
- recording.metadata.test.result === "failed"
- );
- },
  }),
  ["line"],
  ],
@@ -64,32 +50,29 @@ const config: PlaywrightTestConfig = {
  },
  ],
 };
+
 export default config;
 ```
 
 {% /tab %}
 {% /tabs %}
 
-## Upload failed and flaky Cypress tests
+## Upload failed tests only
+
+To upload recordings only for failed tests use `statusThreshold` option:
 
-By default, all test replays are uploaded no matter the result. If you want to upload failed and flaky tests, you can use the `filter` property in the plugin configuration:
+{% tabs labels=["cypress", "playwright"] %}
+{% tab %}
 
-```js
+```js {% fileName="cypress.config.js" highlight=[7,8,9] %}
 export default defineConfig({
  e2e: {
  setupNodeEvents(cyOn, config) {
  const on = wrapOn(cyOn)
  replayPlugin(on, config, {
- upload: true,
  apiKey: process.env.REPLAY_API_KEY,
- filter: function (recording) {
- // upload runtime crashes and recordings with any tests that failed
- return (
- recording.status === 'crashed' ||
- recording.metadata.test.tests.some(
- (test) => test.result === 'failed',
- )
- )
+ upload: {
+ statusThreshold: 'failed',
  },
  })
  return config
@@ -98,23 +81,52 @@ export default defineConfig({
 })
 ```
 
-## Upload only for the primary branch
+{% /tab %}
+{% tab %}
+
+```js {% fileName="playwright.config.js" highlight=[7,8,9] %}
+import { replayDevices, replayReporter } from "@replayio/playwright";
+
+const config: PlaywrightTestConfig = {
+ reporter: [
+ replayReporter({
+ apiKey: process.env.REPLAY_API_KEY,
+ upload: {
+ statusThreshold: "failed"
+ },
+ }),
+ ["line"],
+ ],
+ projects: [
+ {
+ name: "replay-chromium",
+ use: { ...replayDevices["Replay Chromium"] },
+ },
+ ],
+};
+
+export default config;
+```
+
+{% /tab %}
+{% /tabs %}
 
-The recording metadata includes some details about the source control including the repository and branch name which can also be used to filter your uploads. The example below uploads all recordings from the `main` branch:
+## Upload failed and flaky tests
+
+To upload recordings for failed and flaky tests use `statusThreshold` option:
 
 {% tabs labels=["cypress", "playwright"] %}
 {% tab %}
 
-```js
+```js {% fileName="cypress.config.js" highlight=[7,8,9] %}
 export default defineConfig({
  e2e: {
  setupNodeEvents(cyOn, config) {
  const on = wrapOn(cyOn)
  replayPlugin(on, config, {
- upload: true,
  apiKey: process.env.REPLAY_API_KEY,
- filter: function (recording) {
- return recording.metadata.source.branch === 'main'
+ upload: {
+ statusThreshold: 'failed-and-flaky',
  },
  })
  return config
@@ -126,16 +138,15 @@ export default defineConfig({
 {% /tab %}
 {% tab %}
 
-```js
+```js {% fileName="playwright.config.js" highlight=[7,8,9] %}
 import { replayDevices, replayReporter } from "@replayio/playwright";
 
 const config: PlaywrightTestConfig = {
  reporter: [
  replayReporter({
  apiKey: process.env.REPLAY_API_KEY,
- upload: true,
- filter: function (recording) {
- return recording.metadata.source.branch === "main";
+ upload: {
+ statusThreshold: "failed-and-flaky"
  },
  }),
  ["line"],
@@ -147,42 +158,35 @@ const config: PlaywrightTestConfig = {
  },
  ],
 };
+
 export default config;
 ```
 
 {% /tab %}
 {% /tabs %}
 
-## Upload some passing runs
+## Upload only for the primary branch
+
+Many CI providers provide an environment variable that references the current branch name.
+
+- [CircleCI](https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables): `$CIRCLE_BRANCH`
+- [GitLab](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html): `$CI_COMMIT_REF_NAME`
+- [Semaphore](https://docs.semaphoreci.com/ci-cd-environment/environment-variables/): `$SEMAPHORE_GIT_BRANCH`
+- [Travis](https://docs.travis-ci.com/user/environment-variables/#default-environment-variables): `$TRAVIS_BRANCH`
 
-If you've adopted one the configurations above but would also like to periodically upload all replays for a test run, you can add a condition to the filter that returns `true` for a given test run id. This is only one possible implementation of this approach and you're welcome to adopt others such as using external environment variables.
+GitHub [stores this value in a default variable](https://docs.github.com/en/actions/learn-github-actions/variables) named `GITHUB_BASE_REF` that can be passed along as [part of a Workflow](https://docs.github.com/en/actions/learn-github-actions/variables#defining-environment-variables-for-a-single-workflow) and then referenced in the test config like so:
 
 {% tabs labels=["cypress", "playwright"] %}
 {% tab %}
 
-```js
-const convertStringToInt = (string) =>
- string.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0)
-
+```js {% fileName="cypress.config.js" highlight=[7] %}
 export default defineConfig({
  e2e: {
  setupNodeEvents(cyOn, config) {
  const on = wrapOn(cyOn)
  replayPlugin(on, config, {
- upload: true,
  apiKey: process.env.REPLAY_API_KEY,
- filter: function (recording) {
- // randomly upload 10% of all test runs
- if (convertStringToInt(r.metadata.test.run.id) % 10 === 1) {
- return true
- }
-
- // upload runtime crashes and any failed tests
- return (
- recording.status === 'crashed' ||
- recording.metadata.test.result === 'failed'
- )
- },
+ upload: process.env.BRANCH_NAME === 'main',
  })
  return config
  },
@@ -193,29 +197,14 @@ export default defineConfig({
 {% /tab %}
 {% tab %}
 
-```js
+```js {% fileName="playwright.config.js" highlight=[7] %}
 import { replayDevices, replayReporter } from "@replayio/playwright";
 
-const convertStringToInt = string =>
- string.split("").reduce((acc, char) => acc + char.charCodeAt(0), 0);
-
 const config: PlaywrightTestConfig = {
  reporter: [
  replayReporter({
  apiKey: process.env.REPLAY_API_KEY,
- upload: true,
- filter: function (recording) {
- // randomly upload 10% of all test runs
- if (convertStringToInt(r.metadata.test.run.id) % 10 === 1) {
- return true;
- }
-
- // upload runtime crashes and any failed tests
- return (
- recording.status === "crashed" ||
- recording.metadata.test.result === "failed"
- );
- },
+ upload: process.env.BRANCH_NAME === "main"
  }),
  ["line"],
  ],
@@ -226,38 +215,41 @@ const config: PlaywrightTestConfig = {
  },
  ],
 };
+
 export default config;
 ```
 
 {% /tab %}
 {% /tabs %}
 
-## Using GitHub Action
-
-Alternatively, you can upload your replays in a separate step using our [GitHub upload action](https://github.com/replayio/action-upload). To filter which replays to upload, you can use [JSONata filtering functions](https://docs.jsonata.org/higher-order-functions#filter).
-
-```yml {% fileName=".github/workflows/e2e.yml" lineNumbers=true highlight=[19] %}
-name: Replay tests
-on:
- pull_request:
- push:
- branches: [main]
-jobs:
- cypress-run:
- runs-on: ubuntu-22.04
- steps:
- - name: Checkout
- uses: actions/checkout@v4
- - name: Install dependencies
- run: npm ci
- - name: Install Replay Chromium
- run: npx replayio install
- - name: Run Playwright tests with Replay Browser
- run: npx playwright test --project replay-chromium --reporter=@replayio/playwright/reporter,line
- - name: Upload replays
- if: ${{ always() }}
- uses: replayio/[email protected]
- with:
- api-key: ${{ secrets.REPLAY_API_KEY }}
- filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}
+## Reducing the number of uploaded recordings
+
+Use the advanced upload options to reduce the number of recordings that are uploaded. When this option is enabled, only one recording will be uploaded for any passing or failing test. For flaky tests, two recordings will be uploaded– the passing test and one of the failed attempts.
+
+```js {% fileName="playwright.config.js" highlight=[7,8,9] %}
+import { replayDevices, replayReporter } from "@replayio/playwright";
+
+const config: PlaywrightTestConfig = {
+ reporter: [
+ replayReporter({
+ apiKey: process.env.REPLAY_API_KEY,
+ upload: {
+ minimizeUploads: true,
+ },
+ }),
+ ["line"],
+ ],
+ projects: [
+ {
+ name: "replay-chromium",
+ use: { ...replayDevices["Replay Chromium"] },
+ },
+ ],
+};
+
+export default config;
 ```
+
+{% callout %}
+Note this option is only available for tests recorded with Playwright
+{% /callout %}