Add buildkite analytics test collection to react and react-native e2e…

… tests (bugsnag#2213) * Add analytics test collection to react-native tests * Add test collection for buildkite analytics to react-native-cli tests * Re-add maze-options in pipeline files * Use correct reports path * Use correct reports path * Remove BB only options from react-native builds
joycesmartbear · Oct 8, 2024 · 57f0114 · 57f0114
1 parent 456777b
commit 57f0114
Show file tree

Hide file tree

Showing 10 changed files with 140 additions and 0 deletions.
diff --git a/.buildkite/basic/react-native-android-pipeline.yml b/.buildkite/basic/react-native-android-pipeline.yml
@@ -73,6 +73,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true
@@ -106,6 +111,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true

diff --git a/.buildkite/basic/react-native-ios-pipeline.yml b/.buildkite/basic/react-native-ios-pipeline.yml
@@ -75,6 +75,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true
@@ -107,6 +112,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           RCT_NEW_ARCH_ENABLED: "1"
           RN_VERSION: "{{matrix}}"

diff --git a/.buildkite/full/react-native-android-pipeline.full.yml b/.buildkite/full/react-native-android-pipeline.full.yml
@@ -220,6 +220,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
         concurrency: 25
@@ -248,6 +253,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
           HERMES: "true"
@@ -277,6 +287,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
           HERMES: "true"
@@ -306,6 +321,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
           HERMES: "true"
@@ -333,6 +353,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true
@@ -370,6 +395,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true
@@ -405,6 +435,11 @@ steps:
               - --no-tunnel
               - --aws-public-ip
               - features/react-native-navigation.feature
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true
@@ -440,6 +475,11 @@ steps:
               - --no-tunnel
               - --aws-public-ip
               - features/react-native-navigation.feature
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true

diff --git a/.buildkite/full/react-native-cli-pipeline.full.yml b/.buildkite/full/react-native-cli-pipeline.full.yml
@@ -161,6 +161,11 @@ steps:
               - --no-tunnel
               - --aws-public-ip
               - features/run-app-tests
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_CLI_BUILDKITE_ANALYTICS_TOKEN"
         concurrency: 25
         concurrency_group: 'bitbar'
         concurrency_method: eager
@@ -187,6 +192,11 @@ steps:
               - --no-tunnel
               - --aws-public-ip
               - features/run-app-tests
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_CLI_BUILDKITE_ANALYTICS_TOKEN"
         concurrency: 25
         concurrency_group: 'bitbar'
         concurrency_method: eager
@@ -213,6 +223,11 @@ steps:
               - --no-tunnel
               - --aws-public-ip
               - features/run-app-tests
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_CLI_BUILDKITE_ANALYTICS_TOKEN"
         concurrency: 25
         concurrency_group: 'bitbar'
         concurrency_method: eager
@@ -237,6 +252,11 @@ steps:
               - --device=IOS_14
               - --a11y-locator
               - features/run-app-tests
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_CLI_BUILDKITE_ANALYTICS_TOKEN"
         concurrency: 5
         concurrency_group: "browserstack-app"
         concurrency_method: eager
@@ -258,6 +278,11 @@ steps:
               - --device=IOS_14
               - --a11y-locator
               - features/run-app-tests
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_CLI_BUILDKITE_ANALYTICS_TOKEN"
         concurrency: 5
         concurrency_group: "browserstack-app"
         concurrency_method: eager
@@ -279,6 +304,11 @@ steps:
               - --device=IOS_14
               - --a11y-locator
               - features/run-app-tests
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_CLI_BUILDKITE_ANALYTICS_TOKEN"
         concurrency: 5
         concurrency_group: "browserstack-app"
         concurrency_method: eager
diff --git a/.buildkite/full/react-native-ios-pipeline.full.yml b/.buildkite/full/react-native-ios-pipeline.full.yml
@@ -199,6 +199,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
         concurrency: 25
@@ -225,6 +230,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
           HERMES: "true"
@@ -252,6 +262,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
           HERMES: "true"
@@ -279,6 +294,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           SKIP_NAVIGATION_SCENARIOS: "true"
           HERMES: "true"
@@ -305,6 +325,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true
@@ -338,6 +363,11 @@ steps:
               - --fail-fast
               - --no-tunnel
               - --aws-public-ip
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         env:
           RCT_NEW_ARCH_ENABLED: "1"
           RN_VERSION: "{{matrix}}"
@@ -375,6 +405,11 @@ steps:
               - --no-tunnel
               - --aws-public-ip
               - features/react-native-navigation.feature
+          test-collector#v1.10.2:
+            files: "reports/TEST-*.xml"
+            format: "junit"
+            branch: "^main|next$$"
+            api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
         retry:
           manual:
             permit_on_passed: true
@@ -411,6 +446,11 @@ steps:
       #         - --no-tunnel
       #         - --aws-public-ip
       #         - features/react-native-navigation.feature
+      #     test-collector#v1.10.2:
+      #       files: "reports/TEST-*.xml"
+      #       format: "junit"
+      #       branch: "^main|next$$"
+      #       api-token-env-name: "REACT_NATIVE_BUILDKITE_ANALYTICS_TOKEN"
       #   env:
       #     RCT_NEW_ARCH_ENABLED: "1"
       #     RN_VERSION: "{{matrix}}"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -188,6 +188,7 @@ services:
       - ./test/react-native/features/:/app/features
       - ./test/react-native/maze_output:/app/maze_output
       - /var/run/docker.sock:/var/run/docker.sock
+      - ./reports/:/app/reports
 
   react-native-cli-maze-runner:
     image: 855461928731.dkr.ecr.us-west-1.amazonaws.com/maze-runner-releases:latest-v9-cli
@@ -207,6 +208,7 @@ services:
       - ./test/react-native-cli/features/:/app/features/
       - ./test/react-native-cli/maze_output:/app/maze_output
       - /var/run/docker.sock:/var/run/docker.sock
+      - ./reports/:/app/reports
 
   release:
     build:

diff --git a/test/react-native-cli/features/support/maze.all.cfg b/test/react-native-cli/features/support/maze.all.cfg
@@ -0,0 +1 @@
+--a11y-locator
diff --git a/test/react-native-cli/features/support/maze.buildkite.cfg b/test/react-native-cli/features/support/maze.buildkite.cfg
@@ -0,0 +1,3 @@
+--format=junit
+--out=reports
+--format=pretty
diff --git a/test/react-native/features/support/maze.all.cfg b/test/react-native/features/support/maze.all.cfg
@@ -0,0 +1 @@
+--a11y-locator
diff --git a/test/react-native/features/support/maze.buildkite.cfg b/test/react-native/features/support/maze.buildkite.cfg
@@ -0,0 +1,3 @@
+--format=junit
+--out=reports
+--format=pretty