ci: build conformance suite only once

santiq · May 26, 2021 · 153ac52 · 153ac52
1 parent c6de392
commit 153ac52
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 24 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -86,10 +86,37 @@ jobs:
           name: coverage-data
           path: coverage/
 
-  conformance-suite:
+  build-conformance-suite:
     runs-on: ubuntu-latest
     env:
       VERSION: release-v4.1.11
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+      - name: Load Cached Conformance Suite Build
+        uses: actions/cache@v2
+        id: cache
+        with:
+          path: ./conformance-suite
+          key: suite-${{ hashFiles('**/test.yml') }}
+      - name: Conformance Suite Checkout
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        run: git clone --depth 1 --single-branch --branch $VERSION https://gitlab.com/openid/conformance-suite.git
+      - name: Conformance Suite Build
+        working-directory: ./conformance-suite
+        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+        env:
+          MAVEN_CACHE: ./m2
+        run: |
+          sed -i -e 's/localhost/localhost.emobix.co.uk/g' src/main/resources/application.properties
+          sed -i -e 's/-B clean/-B -DskipTests=true/g' builder-compose.yml
+          docker-compose -f builder-compose.yml run builder
+
+  conformance-suite:
+    runs-on: ubuntu-latest
+    needs:
+      - build-conformance-suite
+    env:
       SUITE_BASE_URL: https://localhost.emobix.co.uk:8443
       SETUP: ${{ toJSON(matrix.setup) }}
     strategy:
@@ -421,25 +448,15 @@ jobs:
           PORT: 3000
           NODE_OPTIONS: --tls-cipher-list="ECDHE-ECDSA-AES128-GCM-SHA256 ECDHE-ECDSA-AES256-GCM-SHA384 ECDHE-RSA-AES128-GCM-SHA256 ECDHE-RSA-AES256-GCM-SHA384"
           NODE_TLS_REJECT_UNAUTHORIZED: 0
-      - name: Cache Conformance Suite Build
+      - name: Load Cached Conformance Suite Build
         uses: actions/cache@v2
         id: cache
         with:
           path: ./conformance-suite
-          key: suite-${{ env.VERSION }}
-      - name: Conformance Suite Build
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        env:
-          MAVEN_CACHE: ./m2
-        run: |
-          git clone --depth 1 --single-branch --branch $VERSION https://gitlab.com/openid/conformance-suite.git
-          cd conformance-suite
-          sed -i -e 's/localhost/localhost.emobix.co.uk/g' src/main/resources/application.properties
-          sed -i -e 's/-B clean/-B -DskipTests=true/g' builder-compose.yml
-          docker-compose -f builder-compose.yml run builder
+          key: suite-${{ hashFiles('**/test.yml') }}
       - name: Run Conformance Suite
+        working-directory: ./conformance-suite
         run: |
-          cd conformance-suite
           docker-compose -f docker-compose-dev.yml up -d
           while ! curl -skfail https://localhost.emobix.co.uk:8443/api/runner/available >/dev/null; do sleep 2; done
       - name: Adjust configuration files for CI
@@ -461,9 +478,9 @@ jobs:
           if-no-files-found: ignore
         if: ${{ always() }}
       - name: Stop Conformance Suite
+        working-directory: ./conformance-suite
         run: |
           killall -SIGINT node
-          cd conformance-suite
           docker-compose -f docker-compose-dev.yml down
           sudo rm -rf mongo
       - uses: actions/upload-artifact@v2

diff --git a/certification/runner/index.js b/certification/runner/index.js
@@ -59,25 +59,23 @@ runner.createTestPlan({
   planName: PLAN_NAME,
   variant: VARIANT,
 }).then((plan) => {
-  let { modules: MODULES } = plan;
-  const { id: PLAN_ID } = plan;
-
-  MODULES = MODULES.sort(() => Math.random() - 0.5);
+  const { id: PLAN_ID, modules: MODULES } = plan;
 
   debug('Created test plan, new id %s', PLAN_ID);
   debug('%s/plan-detail.html?plan=%s', SUITE_BASE_URL, PLAN_ID);
   debug('modules to test %O', MODULES);
 
   SKIP = SKIP || ('SKIP' in process.env ? process.env.SKIP.split(',') : []);
 
-  if (fs.existsSync('.download')) {
-    fs.unlinkSync('.download');
+  if (fs.existsSync('.failed')) {
+    fs.unlinkSync('.failed');
   }
 
   describe(PLAN_NAME, () => {
     after(() => {
-      if (fs.existsSync('.download')) {
-        fs.unlinkSync('.download');
+      if (fs.existsSync('.failed')) {
+        fs.unlinkSync('.failed');
+        process.exitCode |= 1;
         return runner.downloadArtifact({ planId: PLAN_ID });
       }
       return undefined;
@@ -96,7 +94,7 @@ runner.createTestPlan({
           try {
             await runner.waitForState({ moduleId });
           } catch (err) {
-            fs.writeFileSync('.download', 'foo');
+            fs.writeFileSync('.failed', Buffer.alloc(0));
             throw err;
           }
         });