ci: dont fail on certification warnings

vodlogic · Apr 6, 2023 · 7d58121 · 7d58121
1 parent c62e023
commit 7d58121
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 25 deletions.
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -256,7 +256,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           path: export-*.zip
-          name: failed certification html results
+          name: certification html results
           if-no-files-found: ignore
         if: ${{ always() }}
       - name: Stop Conformance Suite

diff --git a/certification/runner/api.js b/certification/runner/api.js
@@ -12,7 +12,7 @@ import debug from './debug.js';
 const pipeline = promisify(stream.pipeline);
 
 const FINISHED = new Set(['FINISHED']);
-const RESULTS = new Set(['REVIEW', 'PASSED', 'SKIPPED']);
+const RESULTS = new Set(['REVIEW', 'PASSED', 'WARNING', 'SKIPPED']);
 
 class API {
   constructor({ baseUrl, bearerToken } = {}) {

diff --git a/certification/runner/index.js b/certification/runner/index.js
@@ -99,29 +99,12 @@ if (VARIANT.client_registration === 'dynamic_client') {
   delete configuration.alias;
 }
 
-try {
-  const plan = await runner.createTestPlan({
-    configuration,
-    planName: PLAN_NAME,
-    variant: JSON.stringify(VARIANT),
-  });
+function summary(prefix) {
+  const backticks = '```';
+  fs.writeFileSync(process.env.GITHUB_STEP_SUMMARY, `
+${prefix} Plan Name: \`${PLAN_NAME}\`
 
-  const { id: PLAN_ID, modules: MODULES } = plan;
-
-  debug('Created test plan, new id %s', PLAN_ID);
-  debug('%s/plan-detail.html?plan=%s', SUITE_BASE_URL, PLAN_ID);
-  debug('modules to test %O', MODULES);
-
-  let failed = false;
-  describe(PLAN_NAME, () => {
-    after(() => {
-      if (failed) {
-        if (process.env.GITHUB_STEP_SUMMARY) {
-          const backticks = '```';
-          fs.writeFileSync(process.env.GITHUB_STEP_SUMMARY, `
-Failed Plan Name: \`${PLAN_NAME}\`
-
-Failed Variant:
+${prefix} Variant:
 
 ${backticks}json
 ${JSON.stringify(VARIANT, null, 4)}
@@ -137,7 +120,34 @@ ${backticks}
 </details>
 
 `, { flag: 'a' });
+}
+
+try {
+  const plan = await runner.createTestPlan({
+    configuration,
+    planName: PLAN_NAME,
+    variant: JSON.stringify(VARIANT),
+  });
+
+  const { id: PLAN_ID, modules: MODULES } = plan;
+
+  debug('Created test plan, new id %s', PLAN_ID);
+  debug('%s/plan-detail.html?plan=%s', SUITE_BASE_URL, PLAN_ID);
+  debug('modules to test %O', MODULES);
+
+  let failed = false;
+  let warned = false;
+  describe(PLAN_NAME, () => {
+    after(() => {
+      if (process.env.GITHUB_STEP_SUMMARY) {
+        if (failed) {
+          summary('Failed');
+        } else if (warned) {
+          summary('Warned');
         }
+      }
+
+      if (failed || warned) {
         runner.downloadArtifact({ planId: PLAN_ID });
       }
     });
@@ -149,14 +159,18 @@ ${backticks}
     const skips = SKIP ? SKIP.split(',') : [];
     for (const { testModule, variant } of MODULES) {
       const test = skips.includes(testModule) ? it.skip : it;
+      // eslint-disable-next-line no-loop-func
       test(`${testModule}, ${JSON.stringify(variant)}`, async () => {
         debug('\n\nRunning test module: %s', testModule);
         const { id: moduleId } = await runner.createTestFromPlan({
           plan: PLAN_ID, test: testModule, variant,
         });
         debug('Created test module, new id: %s', moduleId);
         debug('%s/log-detail.html?log=%s', SUITE_BASE_URL, moduleId);
-        await runner.waitForState({ moduleId });
+        const [, result] = await runner.waitForState({ moduleId });
+        if (result === 'WARNING') {
+          warned ||= true;
+        }
       });
     }
   });