Improve Workflows (viamrobotics#1028)

yeison · Jul 20, 2022 · f589a24 · f589a24
1 parent 8a8393d
commit f589a24
Show file tree

Hide file tree

Showing 16 changed files with 258 additions and 113 deletions.
diff --git a/.github/workflows/appimage.yml b/.github/workflows/appimage.yml
@@ -8,17 +8,28 @@ on:
         required: true
       REPO_READ_TOKEN:
         required: true
+      GIT_ACCESS_TOKEN:
+        required: true
 
 env:
   GOPRIVATE: "github.com/viamrobotics/*,go.viam.com/*"
 
 jobs:
-  appimages_amd64:
-    name: (x64) Viam Server AppImage
-    runs-on: [x64, qemu-host]
+  appimage:
+    name: AppImage Build
+    strategy:
+      matrix:
+        include:
+          - arch: [x64, qemu-host]
+            image: ghcr.io/viamrobotics/canon:amd64-cache
+            platform: linux/amd64
+          - arch: [arm64, qemu-host]
+            image: ghcr.io/viamrobotics/canon:arm64-cache
+            platform: linux/arm64
+    runs-on: ${{ matrix.arch }}
     container:
-      image: ghcr.io/viamrobotics/canon:amd64-cache
-      options: --platform linux/amd64
+      image: ${{ matrix.image }}
+      options: --platform ${{ matrix.platform }}
     timeout-minutes: 15
 
     steps:
@@ -27,17 +38,23 @@ jobs:
       with: 
         fetch-depth: 2
 
-    - name: Build and Package
+    - name: Clean and configure git for private repos
       run: |
         echo "machine github.com login viambot password ${{ secrets.REPO_READ_TOKEN }}" > ~/.netrc
-        make clean-all appimage
+        make clean-all
 
     - name: Authorize GCP Upload
       uses: google-github-actions/[email protected]
       with:
         credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
 
-    - name: Upload Files
+    - name: Build and Package (PR)
+      if: contains(github.event.pull_request.labels.*.name, 'appimage')
+      run: |
+        make BUILD_CHANNEL="pr-${{ github.event.pull_request.number }}" appimage
+
+    - name: Upload Files (PR)
+      if: contains(github.event.pull_request.labels.*.name, 'appimage')
       uses: google-github-actions/[email protected]
       with:
         headers: "Cache-Control: no-cache"
@@ -47,36 +64,88 @@ jobs:
         parent: false
         gzip: false
 
-  appimages_arm64:
-    name: (arm64) Viam Server AppImage
-    runs-on: [arm64, qemu-host]
+    - name: Build and Package (Latest)
+      if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
+      run: |
+        make BUILD_CHANNEL="latest" appimage
+
+    - name: Upload Files (Testing)
+      if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
+      uses: google-github-actions/[email protected]
+      with:
+        headers: "Cache-Control: no-cache"
+        path: 'etc/packaging/appimages/deploy/'
+        destination: 'packages.viam.com/apps/viam-server/testing/${{ github.sha }}/'
+        glob: '*'
+        parent: false
+        gzip: false
+
+  appimage_comment:
+    name: Add AppImage Links
+    if: contains(github.event.pull_request.labels.*.name, 'appimage')
+    needs: appimage
+    runs-on: [x64, qemu-host]
     container:
-      image: ghcr.io/viamrobotics/canon:arm64-cache
-      options: --platform linux/arm64
+      image: ghcr.io/viamrobotics/canon:amd64-cache
+      options: --platform linux/amd64
+    timeout-minutes: 5
+
+    steps:
+    - name: Add links
+      uses: marocchino/[email protected]
+      with:
+        recreate: true
+        GITHUB_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
+        message: |
+          AppImages ready!
+          <http://packages.viam.com/apps/viam-server/viam-server-pr-${{ github.event.pull_request.number }}-x86_64.AppImage>
+          <http://packages.viam.com/apps/viam-server/viam-server-pr-${{ github.event.pull_request.number }}-aarch64.AppImage>
+
+  appimage_test:
+    name: AppImage Test & Deploy
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'push'
+    strategy:
+      matrix:
+        arch: [[x64, qemu-host], [arm64, qemu-host]]
+    needs: appimage
+    runs-on: ${{ matrix.arch }}
     timeout-minutes: 15
 
     steps:
-    - name: Checkout Code
-      uses: actions/checkout@v2
-      with: 
-        fetch-depth: 2
+    - name: Clean Workspace
+      run: |
+        shopt -s dotglob
+        sudo chown -R `whoami` ./
+        rm -rf ./*
 
-    - name: Build and Package
+    - name: Test AppImage
       run: |
-        echo "machine github.com login viambot password ${{ secrets.REPO_READ_TOKEN }}" > ~/.netrc
-        make clean-all appimage
+        export TEST_DIR=`mktemp -d -t test-viam-server-XXXXXX`
+        cd $TEST_DIR
 
-    - name: Authorize GCP Upload
+        curl -o viam-server http://packages.viam.com/apps/viam-server/testing/${{ github.sha }}/viam-server-latest-`uname -m`.AppImage
+        chmod 755 viam-server
+
+        export RAND_PORT=$((30000 + $RANDOM))
+        echo "{\"network\": {\"bind_address\":\"localhost:${RAND_PORT}\"}}" > test.json
+
+        ./viam-server test.json &
+        curl --retry 5 --retry-delay 5 --retry-connrefused localhost:$RAND_PORT
+        export RET1=$?
+        kill %%
+        wait $!
+        export RET2=$?
+        cd - && rm -rf $TEST_DIR
+        [ $RET1 == 0 ] && [ $RET1 == 0 ]
+
+    - name: Authorize GCP
       uses: google-github-actions/[email protected]
       with:
         credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
 
-    - name: Upload Files
-      uses: google-github-actions/[email protected]
-      with:
-        headers: "Cache-Control: no-cache"
-        path: 'etc/packaging/appimages/deploy/'
-        destination: 'packages.viam.com/apps/viam-server/'
-        glob: '*'
-        parent: false
-        gzip: false
+    - name: Set up Cloud SDK
+      uses: 'google-github-actions/setup-gcloud@v0'
+
+    - name: Publish AppImage
+      run: |
+        gsutil mv "gs://packages.viam.com/apps/viam-server/testing/${{ github.sha }}/*`uname -m`*" "gs://packages.viam.com/apps/viam-server/"
diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml
@@ -1,5 +1,9 @@
 name: Buf Push
 
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   # Triggers the workflow on pushes to the main branch
   push:

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -1,5 +1,9 @@
 name: Docker
 
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   schedule:
     - cron: '0 6 * * *' # once a day at 6AM UTC (1AM/EST, 2AM/EDT)
@@ -35,7 +39,5 @@ jobs:
     needs: canon-cache
     uses: viamrobotics/rdk/.github/workflows/test.yml@main
     secrets:
-      ARTIFACT_GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.ARTIFACT_GOOGLE_APPLICATION_CREDENTIALS }}
-      TEST_MONGODB_URI: ${{ secrets.TEST_MONGODB_URI }}
       REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }}
       GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,9 +1,13 @@
 name: Main Branch Update
 
+concurrency: 
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   push:
-    branches: [ main ]
+    branches: [ 'main' ]
     paths-ignore:
       - 'README.md'
     tags:
@@ -17,8 +21,6 @@ jobs:
   test:
     uses: viamrobotics/rdk/.github/workflows/test.yml@main
     secrets:
-      ARTIFACT_GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.ARTIFACT_GOOGLE_APPLICATION_CREDENTIALS }}
-      TEST_MONGODB_URI: ${{ secrets.TEST_MONGODB_URI }}
       REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }}
       GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
 
@@ -28,3 +30,4 @@ jobs:
     secrets:
       GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
       REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }}
+      GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
diff --git a/.github/workflows/pullrequest.yml b/.github/workflows/pullrequest.yml
@@ -1,11 +1,14 @@
 name: Pull Request Update
 
+concurrency: 
+  group: pullrequest-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
   workflow_dispatch:
   pull_request:
     branches: [ 'main' ]
-  push:
-    branches: [ '!main' ]
+    types: [ 'labeled', 'opened', 'reopened', 'synchronize' ]
 
 # To test workflow updates you need to work in a branch directly on viamrobotics/rdk
 # and tag your working branch instead of @main in any viamrobotics/rdk "uses" below.
@@ -15,7 +18,15 @@ jobs:
   test:
     uses: viamrobotics/rdk/.github/workflows/test.yml@main
     secrets:
-      ARTIFACT_GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.ARTIFACT_GOOGLE_APPLICATION_CREDENTIALS }}
-      TEST_MONGODB_URI: ${{ secrets.TEST_MONGODB_URI }}
+      REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }}
+      GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
+
+  # This lets people add an "appimage" tag to have appimages built for the PR
+  appimage:
+    needs: test
+    if: contains(github.event.pull_request.labels.*.name, 'appimage')
+    uses: viamrobotics/rdk/.github/workflows/appimage.yml@main
+    secrets:
+      GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
       REPO_READ_TOKEN: ${{ secrets.REPO_READ_TOKEN }}
       GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
diff --git a/.github/workflows/pullrequestclose.yml b/.github/workflows/pullrequestclose.yml
@@ -0,0 +1,32 @@
+name: Pull Request Close
+
+concurrency: 
+  group: pullrequest-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    branches: [ 'main' ]
+    types: [ 'closed' ]
+
+jobs:
+  appimage_clean:
+    name: Remove PR AppImages
+    runs-on: [x64, qemu-host]
+    container:
+      image: ghcr.io/viamrobotics/canon:amd64-cache
+      options: --platform linux/amd64
+    timeout-minutes: 15
+
+    steps:
+    - name: Authorize GCP
+      uses: google-github-actions/[email protected]
+      with:
+        credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
+
+    - name: Set up Cloud SDK
+      uses: 'google-github-actions/setup-gcloud@v0'
+
+    - name: Delete Files
+      run: |
+        gsutil rm "gs://packages.viam.com/apps/viam-server/viam-server-pr-${{ github.event.pull_request.number }}-*" || true