Skip to content

Commit

Permalink
Merge pull request Expensify#29656 from margelo/osp/separate-test-runs
Browse files Browse the repository at this point in the history
[No QA]Separate e2e test runs
  • Loading branch information
AndrewGable authored Oct 23, 2023
2 parents af8d25f + 6c6061f commit c50054a
Show file tree
Hide file tree
Showing 13 changed files with 285 additions and 145 deletions.
89 changes: 60 additions & 29 deletions .github/workflows/e2ePerformanceTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ jobs:
steps:
- uses: actions/checkout@v3

- name: Setup Node
uses: Expensify/App/.github/actions/composite/setupNode@main

- name: Make zip directory for everything to send to AWS Device Farm
run: mkdir zip

Expand All @@ -137,7 +140,7 @@ jobs:

# The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it
- name: Rename baseline APK
run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk"
run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-main.apk"

- name: Download delta APK
uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b
Expand All @@ -147,7 +150,7 @@ jobs:
path: zip

- name: Rename delta APK
run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-compare.apk"
run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-delta.apk"

- name: Copy e2e code into zip folder
run: cp -r tests/e2e zip
Expand All @@ -162,44 +165,72 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_REGION: us-west-2

- name: Schedule AWS Device Farm test run
- name: Schedule AWS Device Farm test run on main branch
uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
id: schedule-awsdf-main
with:
name: App E2E Performance Regression Tests
project_arn: ${{ secrets.AWS_PROJECT_ARN }}
device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
app_file: zip/app-e2eRelease-baseline.apk
app_file: zip/app-e2eRelease-main.apk
app_type: ANDROID_APP
test_type: APPIUM_NODE
test_package_file: App.zip
test_package_type: APPIUM_NODE_TEST_PACKAGE
test_spec_file: tests/e2e/TestSpec.yml
test_spec_file: tests/e2e/TestSpecMain.yml
test_spec_type: APPIUM_NODE_TEST_SPEC
remote_src: false
file_artifacts: Customer Artifacts.zip
log_artifacts: debug.log
cleanup: true

- name: Unzip AWS Device Farm results
if: ${{ always() }}
run: unzip "Customer Artifacts.zip"

- name: Print AWS Device Farm run results
if: ${{ always() }}
run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/output.md"

- name: Print AWS Device Farm verbose run results
if: ${{ always() && runner.debug != null && fromJSON(runner.debug) }}
run: cat "./Host_Machine_Files/\$WORKING_DIRECTORY/debug.log"

# TODO: Once tests are more reliable we should uncomment this
# - name: Check if test failed, if so post the results and add the DeployBlocker label
# run: |
# if grep -q '🔴' ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md; then
# gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
# gh pr comment ${{ inputs.PR_NUMBER }} -F ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md
# gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
# else
# echo '✅ no performance regression detected'
# fi
# env:
# GITHUB_TOKEN: ${{ github.token }}
- name: Print logs if run failed
if: failure()
run: |
echo ${{ steps.schedule-awsdf-main.outputs.data }}
unzip "Customer Artifacts.zip" -d mainResults
cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log
- name: Unzip AWS Device Farm main results
run: unzip "Customer Artifacts.zip" -d mainResults

- name: Delete Customer Artifacts.zip
run: rm "Customer Artifacts.zip"

- name: Schedule AWS Device Farm test run on delta branch
uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b
with:
name: App E2E Performance Regression Tests
project_arn: ${{ secrets.AWS_PROJECT_ARN }}
device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }}
app_file: zip/app-e2eRelease-delta.apk
app_type: ANDROID_APP
test_type: APPIUM_NODE
test_package_file: App.zip
test_package_type: APPIUM_NODE_TEST_PACKAGE
test_spec_file: tests/e2e/TestSpecDelta.yml
test_spec_type: APPIUM_NODE_TEST_SPEC
remote_src: false
file_artifacts: Customer Artifacts.zip
cleanup: true

- name: Unzip AWS Device Farm delta results
run: unzip "Customer Artifacts.zip" -d deltaResults

- name: Compare results
run: node tests/e2e/merge.js --mainPath ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/main.json --deltaPath ./deltaResults//Host_Machine_Files/\$WORKING_DIRECTORY/delta.json --outputPath ./output.md

- name: Print results
run: cat "./output.md"

- name: Check if test failed, if so post the results and add the DeployBlocker label
run: |
if grep -q '🔴' ./output.md; then
gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash
gh pr comment ${{ inputs.PR_NUMBER }} -F ./output.md
gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker."
else
echo '✅ no performance regression detected'
fi
env:
GITHUB_TOKEN: ${{ github.token }}
11 changes: 8 additions & 3 deletions metro.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ require('dotenv').config();
const defaultConfig = getDefaultConfig(__dirname);

const isUsingMockAPI = process.env.E2E_TESTING === 'true';

if (isUsingMockAPI) {
// eslint-disable-next-line no-console
console.warn('⚠️ Using mock API');
console.log('⚠️⚠️⚠️⚠️ Using mock API ⚠️⚠️⚠️⚠️');
}

/**
Expand All @@ -25,10 +26,14 @@ const config = {
resolveRequest: (context, moduleName, platform) => {
const resolution = context.resolveRequest(context, moduleName, platform);
if (isUsingMockAPI && moduleName.includes('/API')) {
const originalPath = resolution.filePath;
const mockPath = originalPath.replace('src/libs/API.ts', 'src/libs/E2E/API.mock.js').replace('/src/libs/API.js/', 'src/libs/E2E/API.mock.js');
// eslint-disable-next-line no-console
console.log('⚠️⚠️⚠️⚠️ Replacing resolution path', originalPath, ' => ', mockPath);

return {
...resolution,
// TODO: Change API.mock.js extension once it is migrated to TypeScript
filePath: resolution.filePath.replace(/src\/libs\/API.js/, 'src/libs/E2E/API.mock.js'),
filePath: mockPath,
};
}
return resolution;
Expand Down
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@
"analyze-packages": "ANALYZE_BUNDLE=true webpack --config config/webpack/webpack.common.js --env envFile=.env.production",
"symbolicate:android": "npx metro-symbolicate android/app/build/generated/sourcemaps/react/release/index.android.bundle.map",
"symbolicate:ios": "npx metro-symbolicate main.jsbundle.map",
"test:e2e": "node tests/e2e/testRunner.js --development",
"test:e2e:main": "node tests/e2e/testRunner.js --development --branch main --skipCheckout",
"test:e2e:delta": "node tests/e2e/testRunner.js --development --branch main --label delta --skipCheckout",
"test:e2e:compare": "node tests/e2e/merge.js",
"gh-actions-unused-styles": "./.github/scripts/findUnusedKeys.sh",
"workflow-test": "./workflow_tests/scripts/runWorkflowTests.sh",
"workflow-test:generate": "node workflow_tests/utils/preGenerateTest.js"
Expand Down
26 changes: 26 additions & 0 deletions tests/e2e/TestSpecDelta.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
version: 0.1

phases:
install:
commands:
# Install correct version of node
- export NVM_DIR=$HOME/.nvm
- . $NVM_DIR/nvm.sh
- nvm install 16.15.1
- nvm use 16.15.1

# Reverse ports using AWS magic
- PORT=4723
- IP_ADDRESS=$(ip -4 addr show eth0 | grep -Po "(?<=inet\s)\d+(\.\d+){3}")
- reverse_values="{\"ip_address\":\"$IP_ADDRESS\",\"local_port\":\"$PORT\",\"remote_port\":\"$PORT\"}"
- "curl -H \"Content-Type: application/json\" -X POST -d \"$reverse_values\" http://localhost:31007/reverse_forward_tcp"
- adb reverse tcp:$PORT tcp:$PORT

test:
commands:
- cd zip
- npm install underscore
- node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout --label delta --appPath app-e2eRelease-delta.apk

artifacts:
- $WORKING_DIRECTORY
2 changes: 1 addition & 1 deletion tests/e2e/TestSpec.yml → tests/e2e/TestSpecMain.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ phases:
commands:
- cd zip
- npm install underscore
- node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout
- node e2e/testRunner.js -- --skipInstallDeps --buildMode "skip" --skipCheckout --branch main --appPath app-e2eRelease-main.apk

artifacts:
- $WORKING_DIRECTORY
5 changes: 2 additions & 3 deletions tests/e2e/compare/compare.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
const fs = require('fs/promises');
const fsSync = require('fs');
const _ = require('underscore');
const {OUTPUT_DIR} = require('../config');
const {computeProbability, computeZ} = require('./math');
const printToConsole = require('./output/console');
const writeToMarkdown = require('./output/markdown');
Expand Down Expand Up @@ -119,7 +118,7 @@ function compareResults(compareEntries, baselineEntries) {
};
}

module.exports = (baselineFile = `${OUTPUT_DIR}/baseline.json`, compareFile = `${OUTPUT_DIR}/compare.json`, outputFormat = 'all') => {
module.exports = (baselineFile, compareFile, outputFile, outputFormat = 'all') => {
const hasBaselineFile = fsSync.existsSync(baselineFile);
if (!hasBaselineFile) {
throw new Error(`Baseline results files "${baselineFile}" does not exists.`);
Expand All @@ -136,7 +135,7 @@ module.exports = (baselineFile = `${OUTPUT_DIR}/baseline.json`, compareFile = `$
printToConsole(outputData);
}
if (outputFormat === 'markdown' || outputFormat === 'all') {
return writeToMarkdown(`${OUTPUT_DIR}/output.md`, outputData);
return writeToMarkdown(outputFile, outputData);
}
});
});
Expand Down
10 changes: 2 additions & 8 deletions tests/e2e/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,18 @@ const TEST_NAMES = {
module.exports = {
APP_PACKAGE: 'com.expensify.chat.adhoc',

APP_PATHS: {
baseline: './app-e2eRelease-baseline.apk',
compare: './app-e2eRelease-compare.apk',
},
APP_PATH: './app-e2eRelease-main.apk',

ENTRY_FILE: 'src/libs/E2E/reactNativeLaunchingTest.js',

// The port of the testing server that communicates with the app
SERVER_PORT: 4723,

// The amount of times a test should be executed for average performance metrics
RUNS: 90,
RUNS: 60,

DEFAULT_BASELINE_BRANCH: 'main',

// The amount of runs that should happen without counting test results
WARM_UP_RUNS: 3,

OUTPUT_DIR,

// The file to write intermediate results to
Expand Down
9 changes: 2 additions & 7 deletions tests/e2e/config.local.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,5 @@
module.exports = {
APP_PACKAGE: 'com.expensify.chat.dev',

WARM_UP_RUNS: 1,
APP_PACKAGE: 'com.expensify.chat.adhoc',
APP_PATH: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
RUNS: 8,
APP_PATHS: {
baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk',
},
};
27 changes: 27 additions & 0 deletions tests/e2e/merge.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
const compare = require('./compare/compare');
const {OUTPUT_DIR} = require('./config');

const args = process.argv.slice(2);

let mainPath = `${OUTPUT_DIR}/main.json`;
if (args.includes('--mainPath')) {
mainPath = args[args.indexOf('--mainPath') + 1];
}

let deltaPath = `${OUTPUT_DIR}/delta.json`;
if (args.includes('--deltaPath')) {
deltaPath = args[args.indexOf('--deltaPath') + 1];
}

let outputPath = `${OUTPUT_DIR}/output.md`;
if (args.includes('--outputPath')) {
outputPath = args[args.indexOf('--outputPath') + 1];
}

async function run() {
await compare(mainPath, deltaPath, outputPath, 'all');

process.exit(0);
}

run();
Loading

0 comments on commit c50054a

Please sign in to comment.