feat(api-gateway): Async native query results transformations #19323
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| push: | |
| paths: | |
| - '.github/actions/smoke.sh' | |
| - '.github/workflows/push.yml' | |
| - '.github/workflows/master.yml' | |
| - 'packages/**' | |
| - 'rust/cubestore/**' | |
| - 'rust/cubesql/**' | |
| - 'rust/cubenativeutils/**' | |
| - 'rust/cubeorchestrator/**' | |
| - 'rust/cubeshared/**' | |
| - 'rust/cubesqlplanner/**' | |
| - '.eslintrc.js' | |
| - '.prettierrc' | |
| - 'package.json' | |
| - 'lerna.json' | |
| - 'rollup.config.js' | |
| - 'yarn.lock' | |
| branches: | |
| - 'master' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/push.yml' | |
| - '.github/workflows/master.yml' | |
| - 'packages/**' | |
| - 'rust/cubestore/**' | |
| - 'rust/cubesql/**' | |
| - 'rust/cubenativeutils/**' | |
| - 'rust/cubeorchestrator/**' | |
| - 'rust/cubeshared/**' | |
| - 'rust/cubesqlplanner/**' | |
| - '.eslintrc.js' | |
| - '.prettierrc' | |
| - 'package.json' | |
| - 'lerna.json' | |
| - 'rollup.config.js' | |
| - 'yarn.lock' | |
| jobs: | |
| unit: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| needs: latest-tag-sha | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| strategy: | |
| matrix: | |
| # Current docker version + next LTS | |
| node-version: [20.x, 22.x] | |
| python-version: [3.11] | |
| fail-fast: false | |
| steps: | |
| - id: get-tag-out | |
| run: echo "$OUT" | |
| env: | |
| OUT: ${{ needs['latest-tag-sha'].outputs.sha }} | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # pulls all commits (needed for codecov) | |
| fetch-depth: 2 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (no python) | |
| run: cd packages/cubejs-backend-native && npm run native:build-release | |
| - name: Build client | |
| run: yarn build | |
| - name: Build cubejs-backend-native (with Python) | |
| run: yarn run native:build-release-python | |
| working-directory: ./packages/cubejs-backend-native | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| - name: Lerna test | |
| run: yarn lerna run --concurrency 1 --stream --no-prefix unit | |
| # - uses: codecov/codecov-action@v1 | |
| # if: (matrix.node-version == '20.x') | |
| # with: | |
| # files: ./packages/*/coverage/clover.xml | |
| # flags: cube-backend | |
| # verbose: true # optional (default = false) | |
| - name: Cargo fmt cubeorchestrator | |
| run: | | |
| cargo fmt --manifest-path rust/cubeorchestrator/Cargo.toml -- --check | |
| - name: Cargo build cubeorchestrator | |
| run: | | |
| cargo build --manifest-path rust/cubeorchestrator/Cargo.toml -j 4 | |
| - name: Cargo test cubeorchestrator | |
| run: | | |
| cargo test --manifest-path rust/cubeorchestrator/Cargo.toml -j 1 | |
| - name: Cargo fmt cubenativeutils | |
| run: | | |
| cargo fmt --manifest-path rust/cubenativeutils/Cargo.toml -- --check | |
| - name: Cargo build cubenativeutils | |
| run: | | |
| cargo build --manifest-path rust/cubenativeutils/Cargo.toml -j 4 | |
| - name: Cargo test cubenativeutils | |
| run: | | |
| cargo test --manifest-path rust/cubenativeutils/Cargo.toml -j 1 | |
| - name: Cargo fmt cubeshared | |
| run: | | |
| cargo fmt --manifest-path rust/cubeshared/Cargo.toml -- --check | |
| - name: Cargo build cubeshared | |
| run: | | |
| cargo build --manifest-path rust/cubeshared/Cargo.toml -j 4 | |
| - name: Cargo test cubeshared | |
| run: | | |
| cargo test --manifest-path rust/cubeshared/Cargo.toml -j 1 | |
| # - name: Cargo fmt cubesql | |
| # run: | | |
| # cargo fmt --manifest-path rust/cubesql/Cargo.toml -- --check | |
| # - name: Cargo build cubesql | |
| # run: | | |
| # cargo build --manifest-path rust/cubesql/Cargo.toml -j 4 | |
| # - name: Cargo test cubesql | |
| # run: | | |
| # cargo test --manifest-path rust/cubesql/Cargo.toml -j 1 | |
| # - name: Cargo fmt cubesqlplanner | |
| # run: | | |
| # cargo fmt --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -- --check | |
| # - name: Cargo build cubesqlplanner | |
| # run: | | |
| # cargo build --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 4 | |
| # - name: Cargo test cubesqlplanner | |
| # run: | | |
| # cargo test --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 1 | |
| lint: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| needs: latest-tag-sha | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| - name: Install Node.js 20.x | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20.x | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: NPM lint | |
| run: yarn lint:npm | |
| - name: Lerna lint | |
| run: yarn lerna run --concurrency 1 lint | |
| build: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| needs: latest-tag-sha | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| - name: Install Node.js 20.x | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20.x | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Check Yarn lock wasn't modified | |
| run: if [ "$(git status | grep nothing)x" = "x" ]; then echo "Non empty changeset after lerna bootstrap"; git status; exit 1; else echo "Nothing to commit. Proceeding"; fi; | |
| - name: Build Core Client libraries | |
| run: yarn build | |
| - name: Build other packages | |
| run: yarn lerna run --concurrency 1 build | |
| env: | |
| NODE_OPTIONS: --max_old_space_size=4096 | |
| build-cubestore: | |
| needs: [latest-tag-sha] | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| container: | |
| image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-01-29 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./rust/cubestore -> target | |
| # Separate path for release key to protect cache bloating | |
| shared-key: cubestore-release | |
| key: ubuntu-20.04 | |
| - name: Build Cube Store | |
| run: | | |
| cd rust/cubestore | |
| cargo build --release -j 4 -p cubestore | |
| - name: 'Upload cubestored-x86_64-unknown-linux-gnu-release artifact' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cubestored-x86_64-unknown-linux-gnu-release | |
| path: ./rust/cubestore/target/release/cubestored | |
| retention-days: 5 | |
| integration-cubestore: | |
| needs: [latest-tag-sha, build-cubestore] | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| strategy: | |
| matrix: | |
| node-version: [20.x] | |
| fail-fast: false | |
| steps: | |
| - name: Maximize build space (disk space limitations) | |
| run: | | |
| echo "Before" | |
| df -h | |
| sudo apt-get remove -y 'php.*' | |
| sudo apt-get remove -y '^mongodb-.*' | |
| sudo apt-get remove -y '^mysql-.*' | |
| sudo apt-get autoremove -y | |
| sudo apt-get clean | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| echo "After" | |
| df -h | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: ./rust/cubestore/target/release/ | |
| name: cubestored-x86_64-unknown-linux-gnu-release | |
| - name: Run Cube Store in background | |
| run: | | |
| chmod +x ./rust/cubestore/target/release/cubestored | |
| ./rust/cubestore/target/release/cubestored & | |
| - name: Run Cubestore Integration | |
| timeout-minutes: 10 | |
| run: | | |
| yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore | |
| integration: | |
| needs: [unit, lint, latest-tag-sha] | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| env: | |
| CLOUD_DATABASES: > | |
| firebolt | |
| dremio | |
| # Athena (just to check for secrets availability) | |
| DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY: ${{ secrets.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY }} | |
| strategy: | |
| matrix: | |
| node-version: [20.x] | |
| db: [ | |
| 'clickhouse', 'druid', 'elasticsearch', 'mssql', 'mysql', 'postgres', 'prestodb', | |
| 'mysql-aurora-serverless', 'crate', 'mongobi', 'firebolt', 'dremio' | |
| ] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Run Integration tests for ${{ matrix.db }} matrix | |
| uses: nick-fields/retry@v3 | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 30 | |
| command: ./.github/actions/integration/${{ matrix.db }}.sh | |
| # It's enough to test for any one secret because they are set all at once or not set all | |
| if: | | |
| (contains(env.CLOUD_DATABASES, matrix.db) && env.DRIVERS_TESTS_ATHENA_CUBEJS_AWS_KEY != '') || | |
| (!contains(env.CLOUD_DATABASES, matrix.db)) | |
| env: | |
| # Firebolt Integration | |
| DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ENGINE_NAME }} | |
| DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_NAME }} | |
| DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_FIREBOLT_ACCOUNT }} | |
| DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_USER }} | |
| DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS: ${{ secrets.DRIVERS_TESTS_FIREBOLT_CUBEJS_DB_PASS }} | |
| # Dremio Integration | |
| DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_URL }} | |
| DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_NAME }} | |
| DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }} | |
| integration-smoke: | |
| needs: [ latest-tag-sha, build-cubestore ] | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 90 | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| strategy: | |
| matrix: | |
| node-version: [ 20.x ] | |
| python-version: [ 3.11 ] | |
| fail-fast: false | |
| steps: | |
| - name: Maximize build space (disk space limitations) | |
| run: | | |
| echo "Before" | |
| df -h | |
| sudo apt-get remove -y 'php.*' | |
| sudo apt-get remove -y '^mongodb-.*' | |
| sudo apt-get remove -y '^mysql-.*' | |
| sudo apt-get autoremove -y | |
| sudo apt-get clean | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| echo "After" | |
| df -h | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Install instant client for Oracle | |
| uses: GoodManWEN/oracle-client-action@main | |
| - name: Build client | |
| run: yarn build | |
| - name: Build cubejs-backend-native (with Python) | |
| run: yarn run native:build-release-python | |
| working-directory: ./packages/cubejs-backend-native | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Download cubestored-x86_64-unknown-linux-gnu-release artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: rust/cubestore/downloaded/latest/bin/ | |
| name: cubestored-x86_64-unknown-linux-gnu-release | |
| - name: Chmod +x for cubestored | |
| run: | | |
| chmod +x ./rust/cubestore/downloaded/latest/bin/cubestored | |
| - name: Run Integration smoke tests | |
| timeout-minutes: 30 | |
| run: ./.github/actions/smoke.sh | |
| docker-image-latest-set-tag: | |
| # At least git should be completed pushed up until this moment | |
| needs: [lint, latest-tag-sha] | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| tag: ${{ steps.get-tag.outputs.tag }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - id: get-tag | |
| run: echo "tag=$(git tag --contains "$GITHUB_SHA")" >> "$GITHUB_OUTPUT" | |
| env: | |
| GITHUB_SHA: ${{ github.sha }} | |
| latest-tag-sha: | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| sha: ${{ steps.get-tag.outputs.sha }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - id: git-log | |
| run: git log HEAD~30..HEAD | |
| - id: get-tag-test | |
| run: echo "$SHA $(git rev-list -n 1 "$(git tag --contains "$SHA")")" | |
| env: | |
| SHA: ${{ github.sha }} | |
| - id: get-tag | |
| run: echo "sha=$(git rev-list -n 1 "$(git tag --contains "$SHA")")" >> "$GITHUB_OUTPUT" | |
| env: | |
| SHA: ${{ github.sha }} | |
| - id: get-tag-out | |
| run: echo "$OUT" | |
| env: | |
| OUT: ${{ steps.get-tag.outputs.sha }} | |
| docker-dev: | |
| needs: [latest-tag-sha] | |
| if: (needs['latest-tag-sha'].outputs.sha != github.sha) | |
| name: Build & Test :dev for ${{ matrix.name }} without pushing | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| services: | |
| registry: | |
| image: registry:2 | |
| ports: | |
| - 5000:5000 | |
| strategy: | |
| matrix: | |
| node-version: [ 20 ] | |
| target: [ "x86_64-unknown-linux-gnu" ] | |
| dockerfile: | |
| - dev.Dockerfile | |
| include: | |
| - dockerfile: dev.Dockerfile | |
| name: Debian | |
| tag: tmp-dev | |
| fail-fast: false | |
| steps: | |
| - name: Maximize build space (disk space limitations) | |
| run: | | |
| echo "Before" | |
| df -h | |
| sudo apt-get remove -y 'php.*' | |
| sudo apt-get remove -y '^mongodb-.*' | |
| sudo apt-get remove -y '^mysql-.*' | |
| sudo apt-get autoremove -y | |
| sudo apt-get clean | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| echo "After" | |
| df -h | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| target: ${{ matrix.target }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Install Yarn | |
| run: npm install -g yarn | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Install cargo-cp-artifact | |
| run: npm install -g [email protected] | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./packages/cubejs-backend-native | |
| key: native-${{ runner.OS }}-${{ matrix.target }} | |
| shared-key: native-${{ runner.OS }}-${{ matrix.target }} | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Build client | |
| run: yarn build | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (no python) | |
| run: cd packages/cubejs-backend-native && npm run native:build-release | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| # current .dockerignore prevents use of native build | |
| - name: Unignore native from .dockerignore | |
| run: | | |
| grep -v -E "packages/cubejs-backend-native/((native)|(index.node))" .dockerignore > .dockerignore.tmp | |
| mv .dockerignore.tmp .dockerignore | |
| - name: Build image | |
| uses: docker/build-push-action@v6 | |
| timeout-minutes: 30 | |
| with: | |
| context: . | |
| file: ./packages/cubejs-docker/${{ matrix.dockerfile }} | |
| platforms: linux/amd64 | |
| push: true | |
| tags: localhost:5000/cubejs/cube:${{ matrix.tag }} | |
| - name: Testing CubeJS (container mode) via BirdBox | |
| run: | | |
| cd packages/cubejs-testing/ | |
| export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }} | |
| export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/ | |
| export DEBUG=testcontainers | |
| yarn run dataset:minimal | |
| yarn run birdbox:postgresql | |
| yarn run birdbox:postgresql-pre-aggregations | |
| # - name: Testing Athena driver (container mode) via BirdBox | |
| # env: | |
| # CUBEJS_AWS_KEY: ${{ secrets.CUBEJS_AWS_KEY }} | |
| # CUBEJS_AWS_SECRET: ${{ secrets.CUBEJS_AWS_SECRET }} | |
| # CUBEJS_AWS_REGION: us-east-1 | |
| # CUBEJS_AWS_S3_OUTPUT_LOCATION: s3://cubejs-opensource/testing/output | |
| # CUBEJS_DB_EXPORT_BUCKET: s3://cubejs-opensource/testing/export | |
| # run: | | |
| # cd packages/cubejs-testing/ | |
| # export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }} | |
| # export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/ | |
| # export DEBUG=testcontainers | |
| # yarn run driver:athena --log=ignore --mode=docker | |
| # - name: Testing BigQuery driver (container mode) via BirdBox | |
| # env: | |
| # CUBEJS_DB_BQ_CREDENTIALS: ${{ secrets.CUBEJS_DB_BQ_CREDENTIALS }} | |
| # CUBEJS_DB_BQ_PROJECT_ID: cube-open-source | |
| # CUBEJS_DB_EXPORT_BUCKET: cube-open-source-export-bucket | |
| # run: | | |
| # cd packages/cubejs-testing/ | |
| # export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }} | |
| # export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/ | |
| # export DEBUG=testcontainers | |
| # yarn run driver:bigquery --log=ignore --mode=docker | |
| - name: Testing PostgreSQL driver (container mode) via BirdBox | |
| env: | |
| CUBEJS_DB_TYPE: postgres | |
| CUBEJS_DB_USER: postgres | |
| CUBEJS_DB_PASS: postgres | |
| run: | | |
| cd packages/cubejs-testing/ | |
| export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }} | |
| export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/ | |
| export DEBUG=testcontainers | |
| yarn run driver:postgres --log=ignore --mode=docker | |
| - name: Testing Docker image via Cypress (Chrome) | |
| env: | |
| CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} | |
| BIRDBOX_CYPRESS_UPDATE_SCREENSHOTS: ${{ contains(github.event.head_commit.message, '[update screenshots]') }} | |
| run: | | |
| cd packages/cubejs-testing/ | |
| export BIRDBOX_CUBEJS_VERSION=${{ matrix.tag }} | |
| export BIRDBOX_CUBEJS_REGISTRY_PATH=localhost:5000/ | |
| export BIRDBOX_CYPRESS_BROWSER=chrome | |
| export BIRDBOX_CYPRESS_TARGET=postgresql | |
| export DEBUG=testcontainers | |
| yarn run cypress:install | |
| yarn run cypress:birdbox | |
| - name: Upload screenshots on failure | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: cypress-screenshots-docker-dev-${{ matrix.name }} | |
| path: packages/cubejs-testing/cypress/screenshots |