Merge branch 'main' into system-table-docs

.github/actions/benchmark_cloud/action.yml

@@ -38,8 +38,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-
     - name: Install script dependencies
       shell: bash
       run: |
@@ -53,10 +51,12 @@ runs:
         BENCHMARK_DATASET: ${{ inputs.dataset }}
         BENCHMARK_SIZE: ${{ inputs.size }}
         BENCHMARK_IMAGE_TAG: ${{ inputs.image_tag }}
+        BENCHMARK_DATABASE: clickbench
         CLOUD_EMAIL: ${{ inputs.cloud_email }}
         CLOUD_PASSWORD: ${{ inputs.cloud_password }}
         CLOUD_ORG: ${{ inputs.cloud_org }}
         CLOUD_ENDPOINT: ${{ inputs.cloud_endpoint }}
+        CLOUD_WAREHOUSE: benchmark-${{ inputs.run_id }}
       shell: bash
       run: |
         ./benchmark_cloud.sh
@@ -93,13 +93,20 @@ runs:
             exit 1
         esac
         jq ".cluster_size = \"${BENCHMARK_CLUSTER_SIZE}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
-        jq ".machine = \"${{ inputs.size }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
-        jq ".tags = [\"${BENCHMARK_TYPE}\", \"serverless\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".machine = \"${BENCHMARK_CLUSTER_SIZE}×${{ inputs.size }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".tags = [\"${BENCHMARK_TYPE}\", \"s3\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
-        mv result.json result-${{ inputs.dataset }}-cloud.json
+        mv result.json result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
 
     - name: Upload artifact
       uses: actions/upload-artifact@v3
       with:
         name: benchmark-${{ inputs.dataset }}-cloud-${{ inputs.size }}
         path: benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
+
+    - name: Remove warehouse
+      if: always()
+      continue-on-error: true
+      shell: bash
+      run: |
+        bendsql cloud warehouse delete benchmark-${{ inputs.run_id }}

.github/actions/benchmark_local/action.yml

@@ -19,8 +19,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: ./.github/actions/setup_bendsql
-
     - name: Install script dependencies
       shell: bash
       run: |
@@ -55,7 +53,7 @@ runs:
         esac
         jq ".system = \"${BENCHMARK_SYSTEM}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".date = \"$(date -u +%Y-%m-%d)\"" <result.json >result.json.tmp && mv result.json.tmp result.json
-        jq ".machine = \"c5.4xlarge,gp3\"" <result.json >result.json.tmp && mv result.json.tmp result.json
+        jq ".machine = \"c5.4xlarge\"" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".cluster_size = 1" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".comment = \"commit:${{ inputs.sha }}\"" <result.json >result.json.tmp && mv result.json.tmp result.json
         jq ".tags = [\"${BENCHMARK_TYPE}\", \"gp3\"]" <result.json >result.json.tmp && mv result.json.tmp result.json
@@ -71,6 +69,7 @@ runs:
     - name: Clean test data
       working-directory: benchmark/clickbench
       if: always()
+      continue-on-error: true
       shell: bash
       run: |
         bendsql query <"${{ inputs.dataset }}/clear.sql"

.github/actions/create_check_status/action.yml

@@ -0,0 +1,68 @@
+name: "Create Check Status"
+description: "Create check status on github commit"
+inputs:
+  github_token:
+    description: "Github Token"
+    required: true
+  sha:
+    description: "commit sha"
+    required: true
+  name:
+    description: "check name"
+    required: true
+  status:
+    description: "check status, Default: queued, Can be one of: queued, in_progress, completed"
+    required: true
+  conclusion:
+    description: "check conclusion, Can be one of: action_required, cancelled, failure, neutral, success, skipped, stale, timed_out"
+    required: false
+  run_id:
+    description: "external id to associate with the check run"
+    required: false
+  title:
+    description: "check title"
+    required: true
+  summary:
+    description: "check summary"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    # Docs: https://docs.github.com/en/rest/checks/runs#create-a-check-run
+    # Rest: https://octokit.github.io/rest.js/v18#checks-create
+    - name: Create a check run
+      uses: actions/github-script@v6
+      env:
+        param_sha: ${{ inputs.sha }}
+        param_name: ${{ inputs.name }}
+        param_status: ${{ inputs.status }}
+        param_conclusion: ${{ inputs.conclusion }}
+        param_run_id: ${{ inputs.run_id }}
+        param_title: ${{ inputs.title }}
+        param_summary: ${{ inputs.summary }}
+        param_url: https://github.com/datafuselabs/databend/actions/runs/${{ github.run_id }}
+      with:
+        github-token: ${{ inputs.github_token }}
+        script: |
+          let summary = `${process.env.param_summary}
+          [View Details](https://github.com/datafuselabs/databend/actions/runs/${context.runId})`;
+          let data = {
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            head_sha: process.env.param_sha,
+            name: process.env.param_name,
+            status: process.env.param_status,
+            details_url: process.env.param_url,
+            output: {
+              title: process.env.param_title,
+              summary: summary,
+            },
+          };
+          if (process.env.param_status === 'completed') {
+            data.conclusion = process.env.param_conclusion;
+          }
+          if (process.env.param_run_id) {
+            data.external_id = process.env.param_run_id;
+          }
+          await github.rest.checks.create(data);

.github/actions/publish_binary/action.yml

@@ -37,7 +37,7 @@ runs:
         GH_TOKEN: ${{ inputs.github_token }}
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v1
+      uses: aws-actions/configure-aws-credentials@v2
       with:
         role-duration-seconds: 900
         role-to-assume: ${{ inputs.repo_role_arn }}
@@ -47,7 +47,7 @@ runs:
       shell: bash
       if: inputs.mode == 'normal'
       run: |
-        aws s3 cp databend-${{ inputs.version }}-${{ inputs.target }}.tar.gz s3://repo.databend.rs/databend/${{ inputs.version }}/databend-${{ inputs.version }}-${{ inputs.target }}.tar.gz
+        aws s3 cp databend-${{ inputs.version }}-${{ inputs.target }}.tar.gz s3://repo.databend.rs/databend/${{ inputs.version }}/databend-${{ inputs.version }}-${{ inputs.target }}.tar.gz --no-progress
         curl --retry 3 -fsSLo tags.json \
             --header 'content-type: application/json' \
             --header 'authorization: Bearer ${{ inputs.github_token }}' \
@@ -65,7 +65,7 @@ runs:
       shell: bash
       if: inputs.mode == 'hive'
       run: |
-        aws s3 cp databend-hive-${{ inputs.version }}-${{ inputs.target }}.tar.gz s3://repo.databend.rs/databend/${{ inputs.version }}/databend-hive-${{ inputs.version }}-${{ inputs.target }}.tar.gz
+        aws s3 cp databend-hive-${{ inputs.version }}-${{ inputs.target }}.tar.gz s3://repo.databend.rs/databend/${{ inputs.version }}/databend-hive-${{ inputs.version }}-${{ inputs.target }}.tar.gz --no-progress
         curl --retry 3 -fsSLo tags.json \
             --header 'content-type: application/json' \
             --header 'authorization: Bearer ${{ inputs.github_token }}' \

.github/actions/publish_deb/action.yml

@@ -41,7 +41,7 @@ runs:
         reprepro includedeb stable databend_${deb_version}_arm64.deb
 
     - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@v1
+      uses: aws-actions/configure-aws-credentials@v2
       with:
         role-duration-seconds: 900
         role-to-assume: ${{ inputs.repo_role_arn }}

.github/actions/setup_bendsql/action.yml

@@ -4,7 +4,7 @@ inputs:
   version:
     description: "bendsql version"
     required: false
-    default: v0.3.5
+    default: v0.3.8
 runs:
   using: "composite"
   steps:

.github/actions/setup_docker/action.yml

@@ -25,11 +25,11 @@ runs:
   using: "composite"
   steps:
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
 
     - name: Configure AWS Credentials
       if: inputs.ecr_role_arn
-      uses: aws-actions/configure-aws-credentials@v1
+      uses: aws-actions/configure-aws-credentials@v2
       with:
         role-to-assume: ${{ inputs.ecr_role_arn }}
         role-duration-seconds: 900
@@ -43,7 +43,7 @@ runs:
 
     - name: Login to DockerHub
       if: inputs.dockerhub_token
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         username: ${{ inputs.dockerhub_user }}
         password: ${{ inputs.dockerhub_token }}

.github/workflows/benchmark.yml

@@ -44,6 +44,7 @@ jobs:
           jq ".sha = \"${{ github.sha }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
           jq ".run_id = \"${{ github.run_id }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
           jq ".pr_id = \"${{ github.event.pull_request.number }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
+          jq ".head_sha = \"${{ github.event.pull_request.head.sha }}\"" <metadata.json >metadata.json.tmp && mv metadata.json.tmp metadata.json
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/dev-linux.yml

@@ -11,6 +11,9 @@ on:
       - "docker/**"
       - "scripts/setup/**"
       - ".devcontainer/**"
+  merge_group:
+    types:
+      - checks_requested
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}

.github/workflows/dev-macos.yml

@@ -65,5 +65,5 @@ jobs:
   #   runs-on: macos-11
   #   needs: build_macos
   #   steps:
-  #     - uses: actions/checkout@v2
+  #     - uses: actions/checkout@v3
   #     - uses: ./.github/actions/test_sqllogic_standalone_macos

.github/workflows/production.yml

@@ -153,7 +153,7 @@ jobs:
           - "base"
           - "ydb"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/test_sqllogic_standalone_macos
         with:
           profile: release
@@ -177,7 +177,7 @@ jobs:
           - "http"
           - "clickhouse"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/test_sqllogic_standalone_linux
         with:
           profile: release

.github/workflows/release.yml

@@ -16,6 +16,8 @@ on:
 
 permissions:
   id-token: write
+  pull-requests: write
+  checks: write
   contents: write
 
 jobs:
@@ -484,7 +486,7 @@ jobs:
     name: upload-sha256sums
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Get Pre-release
         uses: cardinalby/git-get-release-action@v1
         id: latest_pre_release