Skip to content

Commit

Permalink
ci: disable cluster-test LBT
Browse files Browse the repository at this point in the history 
Closes: aptos-labs#9942
  • Loading branch information
@bmwill @bors-libra
bmwill authored and bors-libra committed Dec 4, 2021
1 parent 92859de commit bb9c7f7
Showing 1 changed file with 0 additions and 244 deletions.
244 changes: 0 additions & 244 deletions .github/workflows/ci-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,191 +312,6 @@ jobs:
with:
github-token: ${{secrets.GITHUB_TOKEN}}

land-blocking-test:
name: Run land blocking test
runs-on: self-hosted
needs: [prepare, build-images, need-base-images]
if: ${{ always() && needs.build-images.result=='success' && needs.prepare.outputs.test-land-blocking == 'true' }}
timeout-minutes: 45
steps:
- uses: actions/[email protected]
with:
# This ensures that the tip of the PR is checked out instead of the merge between the base ref and the tip
# On `push` this value will be empty and will "do-the-right-thing"
ref: ${{ github.event.pull_request.head.sha }}
- name: Launch cluster test
# NOTE Remember to update PR comment payload if cti cmd is updated.
run: |
set +e
date
export CTI_OUTPUT_LOG=$(mktemp)
echo "CTI_OUTPUT_LOG=$CTI_OUTPUT_LOG" >> $GITHUB_ENV
cmd=""
if ${{ needs.need-base-images.result!='success' }}; then
cmd="./scripts/cti --tag ${{ needs.build-images.outputs.head-tag }} --report report.json --suite land_blocking"
else
cmd="./scripts/cti --tag ${{ needs.need-base-images.outputs.prev-tag }} --cluster-test-tag ${{ needs.build-images.outputs.head-tag }} -E BATCH_SIZE=15 -E UPDATE_TO_TAG=${{ needs.build-images.outputs.head-tag }} --report report.json --suite land_blocking_compat"
fi
eval $cmd
ret=$?
echo "cti exit code: $ret"
echo "CTI_REPRO_CMD=$cmd" >> $GITHUB_ENV
echo "CTI_EXIT_CODE=$ret" >> $GITHUB_ENV
msg_text="*${{ github.job }}* job in ${{ github.workflow }} workflow failed for PR ${{ needs.prepare.outputs.changes-pull-request-number }}."
if [ -s "report.json" ]; then
echo "report.json start"
cat report.json
echo "report.json end"
msg_text="$msg_text Report:\n$(cat report.json)"
else
echo "report.json is empty or not found."
msg_text="$msg_text Report:\nEmpty"
ret=1
fi
if [ $ret -ne 0 ]; then
jq -n \
--arg msg "$msg_text" \
--arg url "https://github.com/${{ github.repository }}/actions/runs/${{github.run_id}}" \
--arg pr_url "https://github.com/${{ github.repository }}/pull/${{ needs.prepare.outputs.changes-pull-request-number }}" \
'{
"attachments": [
{
"text": $msg,
"actions": [
{
"type": "button",
"text": "Visit Job",
"url": $url
},
{
"type": "button",
"text": "Visit PR",
"url": $pr_url
}
]
}
]
}' > /tmp/payload
curl -X POST -H 'Content-type: application/json' -d @/tmp/payload ${{ secrets.WEBHOOK_FLAKY_LAND_BLOCKING_CT }}
fi
- name: Post test results on PR
uses: actions/[email protected]
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
// Find the number of the pull request that trigggers this push
let pr_num = ${{ needs.prepare.outputs.changes-pull-request-number }};
if (!pr_num) {
console.warn("Did not find pull request num in previous step");
console.log("GH event payload\n", context.payload);
return;
}
// Read and check cluster test results
let should_fail = false;
let env_vars = process.env;
let body = '';
const fsp = require('fs').promises;
try {
data = await fsp.readFile('report.json', 'utf-8');
var result = JSON.parse(data);
// TODO - set P/F based on metrics TPS, latency
body = `Cluster Test Result
\`\`\`
${result.text}
${result.links}
\`\`\`
`;
// Check CTI exit code for errors
if (parseInt(env_vars.CTI_EXIT_CODE) != 0) {
body += "\n :exclamation: Cluster Test failed - non-zero exit code for `cti` \n"
should_fail = true;
} else {
let tps = result.metrics.find(m => m.experiment == "all up" && m.metric == "avg_tps").value;
let min_tps = 1100;
if (tps < min_tps) {
body += "\n :exclamation: Performance regression is detected on this PR";
body += "\n TPS with PR: " + tps + ", this is lower then minimum allowed " + min_tps + " TPS.";
console.log(body);
should_fail = true;
}
}
} catch (err) {
if (err.code === 'ENOENT') {
body = "Cluster Test failed - no test report found.\n";
// Check Cluster Test output log for infra error
try {
cti_log = await fsp.readFile(env_vars.CTI_OUTPUT_LOG, 'utf-8');
let re = /.*(^Failed\sto\s.*\"Service\sUnavailable.\sPlease\stry\sagain\slater\.\".*)/;
if (re.test(cti_log)) {
let match = re.exec(cti_log);
body += " There was service infra error:\n";
body += `
${match[1]}
`
+ "\n"
;
body += "To retry, comment your PR with `/land`.";
body += " If that doesn't trigger re-run, amend and push again.";
}
} catch (err) {
console.error("Failed to check infra error in CT output log.\n", err);
}
} else {
body = "Cluster Test runner failed.";
console.error(err);
}
body += " See https://github.com/diem/diem/actions/runs/${{github.run_id}}";
// Post comment on PR then fail this workflow
should_fail = true;
}
// Add repro cmd to message
try {
body += "\nRepro cmd:\n";
body += `
\`\`\`
${env_vars.CTI_REPRO_CMD}
\`\`\`
`
} catch (err) {
if (err.code === 'ReferenceError') {
console.error("One of the following env vars is not set");
} else {
body += "[GHA DEBUG]\nFound error in actions/github-script\n";
body += err;
}
}
// Post test result on original pull request
try {
if (!should_fail) {
body += "\n :tada: Land-blocking cluster test passed! :ok_hand:"
}
await github.issues.createComment(
{
issue_number: pr_num,
owner: context.repo.owner,
repo: context.repo.repo,
body: body,
}
);
} catch (err) {
if (err.status === 401) {
// Fail silently for auth but log to console.
console.warn("GH token has expired when trying to POST\n", err);
} else {
console.error("HttpError other than 401 is not bypassed");
throw err;
}
}
// Fail the workflow if test fails or perf regresses
if (should_fail) {
throw "Land-blocking test failed";
}
- name: Early terminate workflow
if: ${{ failure() }}
uses: ./.github/actions/early-terminator
with:
github-token: ${{secrets.GITHUB_TOKEN}}

non-rust-lint:
runs-on: ubuntu-20.04-xl
timeout-minutes: 10
Expand Down Expand Up @@ -1242,67 +1057,8 @@ jobs:
if (should_fail) {
throw "Land-blocking test failed";
}
- name: Post test results on PR
id: post-test-result
if: always()
# set result to do comparison instead of fail workflow
# once we turn off CT and turn on forge, will post forge testing result instead
run: |
if [ $FGI_EXIT_CODE -eq 1 ]; then
echo "::set-output name=forge-result::failed";
else
echo "::set-output name=forge-result::success";
fi
- name: Early terminate workflow
if: ${{ failure() }}
uses: ./.github/actions/early-terminator
with:
github-token: ${{secrets.GITHUB_TOKEN}}

#this is a temp job here to compare lbt signals between Forge and Cluster Test
#it will be removed once Forge LBT is turned on
compare_lbt_signals:
name: compare-lbt_signals
runs-on: ubuntu-20.04
timeout-minutes: 2
#always run this job even if needed jobs failed or are skipped.
if: ${{ always() && needs.land-blocking-test.result!='skipped' && needs.land-blocking-test-forge.result!='skipped'}}
needs:
- prepare
- land-blocking-test
- land-blocking-test-forge
steps:
- run: |
success="${{
needs.land-blocking-test.result==needs.land-blocking-test-forge.outputs.forge-result
}}"
if [[ "$success" != "true" ]]; then
jq -n \
--arg msg "Alert: Two LBT tests results are mismatch. CT ${{needs.land-blocking-test.result}} != Forge ${{needs.land-blocking-test-forge.outputs.forge-result}}" \
--arg url "https://github.com/${{ github.repository }}/actions/runs/${{github.run_id}}" \
--arg pr_url "https://github.com/${{ github.repository }}/pull/${{ needs.prepare.outputs.changes-pull-request-number }}" \
'{
"attachments": [
{
"text": $msg,
"actions": [
{
"type": "button",
"text": "Visit Job",
"url": $url
},
{
"type": "button",
"text": "Visit PR",
"url": $pr_url
}
]
}
]
}' > /tmp/payload
curl -X POST -H 'Content-type: application/json' -d @/tmp/payload ${{ secrets.WEBHOOK_FORGE }}
fi
- name: Clean up
# always step here to avoid break workflow
if: ${{ always() }}
run: echo "===== forge tests done ===="

0 comments on commit bb9c7f7

Please sign in to comment.