feat(api-gateway): Async native query results transformations #4478
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build native | |
| on: | |
| push: | |
| paths: | |
| - '.github/workflows/rust-cubesql.yml' | |
| - 'packages/cubejs-backend-native/**' | |
| - 'rust/cubenativeutils/**' | |
| - 'rust/cubesqlplanner/**' | |
| - 'rust/cubesql/**' | |
| branches: | |
| - 'master' | |
| pull_request: | |
| paths: | |
| - '.github/workflows/rust-cubesql.yml' | |
| - 'packages/cubejs-backend-native/**' | |
| - 'rust/cubenativeutils/**' | |
| - 'rust/cubesqlplanner/**' | |
| - 'rust/cubesql/**' | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| name: Check fmt/clippy | |
| container: | |
| image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt, clippy | |
| cache: false | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./rust/cubesql -> target | |
| # default key | |
| key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu | |
| shared-key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu | |
| - name: Lint CubeSQL | |
| run: cd rust/cubesql && cargo fmt --all -- --check | |
| - name: Lint Native | |
| run: cd packages/cubejs-backend-native && cargo fmt --all -- --check | |
| - name: Lint cubenativeutils | |
| run: cd rust/cubenativeutils && cargo fmt --all -- --check | |
| - name: Lint cubesqlplanner | |
| run: cd rust/cubesqlplanner && cargo fmt --all -- --check | |
| - name: Clippy CubeSQL | |
| run: cd rust/cubesql && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings | |
| - name: Clippy Native | |
| run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going -- -D warnings | |
| - name: Clippy Native (with Python) | |
| run: cd packages/cubejs-backend-native && cargo clippy --locked --workspace --all-targets --keep-going --features python -- -D warnings | |
| unit: | |
| # We use host instead of cross container, because it's much faster | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| name: Unit (Rewrite Engine) (CUBESQL_TOP_DOWN_EXTRACTOR=${{ matrix.top-down-extractor }}) | |
| strategy: | |
| matrix: | |
| top-down-extractor: ['true', 'false'] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| # pulls all commits (needed for codecov) | |
| fetch-depth: 2 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./rust/cubesql -> target | |
| # default key | |
| key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu | |
| shared-key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu | |
| - name: Install lld | |
| run: | | |
| # ubuntu 24.04 has lld-18 | |
| sudo apt-get update && sudo apt install -y lld | |
| - name: Install [email protected] | |
| uses: baptiste0928/cargo-install@v3 | |
| with: | |
| crate: cargo-llvm-cov | |
| version: "0.6.10" | |
| - name: Unit tests (Rewrite Engine) | |
| env: | |
| CUBESQL_TESTING_CUBE_TOKEN: ${{ secrets.CUBESQL_TESTING_CUBE_TOKEN }} | |
| CUBESQL_TESTING_CUBE_URL: ${{ secrets.CUBESQL_TESTING_CUBE_URL }} | |
| CUBESQL_SQL_PUSH_DOWN: true | |
| CUBESQL_TOP_DOWN_EXTRACTOR: ${{ matrix.top-down-extractor }} | |
| CUBESQL_REWRITE_CACHE: true | |
| CUBESQL_REWRITE_TIMEOUT: 60 | |
| run: | | |
| cd rust/cubesql | |
| cargo llvm-cov --all-features --workspace --lcov --output-path lcov.info | |
| - name: Upload code coverage | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./rust/cubesql/lcov.info | |
| verbose: true | |
| flags: cubesql | |
| fail_ci_if_error: false | |
| native_linux: | |
| needs: [lint] | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| name: Build Linux GNU ${{ matrix.node-version }}.x ${{ matrix.target }} with Python ${{ matrix.python-version }} | |
| strategy: | |
| matrix: | |
| # Current used version + 1 LTS | |
| node-version: [20, 22] | |
| python-version: ["3.9", "3.10", "3.11", "3.12", "fallback"] | |
| target: ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] | |
| # minimize number of jobs | |
| exclude: | |
| - node-version: 20 | |
| target: "aarch64-unknown-linux-gnu" | |
| - python-version: 3.10 | |
| target: "aarch64-unknown-linux-gnu" | |
| - python-version: 3.11 | |
| target: "aarch64-unknown-linux-gnu" | |
| fail-fast: false | |
| container: | |
| image: cubejs/rust-cross:${{ matrix.target }}-15082024${{ matrix.python-version != 'fallback' && format('-python-{0}', matrix.python-version) || '' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| target: ${{ matrix.target }} | |
| cache: false | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ./rust/cubesql -> target | |
| key: cubesql-${{ matrix.target }} | |
| shared-key: cubesql-${{ matrix.target }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Install Yarn | |
| run: npm install -g yarn | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (fallback) | |
| if: (matrix.python-version == 'fallback') | |
| env: | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| working-directory: ./packages/cubejs-backend-native | |
| run: yarn run native:build-debug | |
| - name: Setup cross compilation | |
| if: (matrix.target == 'aarch64-unknown-linux-gnu') | |
| uses: allenevans/[email protected] | |
| with: | |
| PYO3_CROSS_PYTHON_VERSION: ${{ matrix.python-version }} | |
| - name: Build native (with Python) | |
| if: (matrix.python-version != 'fallback') | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| working-directory: ./packages/cubejs-backend-native | |
| run: yarn run native:build-debug-python | |
| - name: Test native (GNU only) | |
| if: (matrix.target == 'x86_64-unknown-linux-gnu') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| working-directory: ./packages/cubejs-backend-native | |
| run: yarn run test:unit | |
| - name: Run E2E Smoke testing over whole Cube (GNU only) | |
| if: (matrix.target == 'x86_64-unknown-linux-gnu') | |
| env: | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| working-directory: ./packages/cubejs-testing | |
| run: yarn smoke:cubesql | |
| native_macos: | |
| needs: [lint] | |
| runs-on: ${{ matrix.os-version }} | |
| timeout-minutes: 60 | |
| name: Build ${{ matrix.os-version }} ${{ matrix.target }} ${{ matrix.node-version }} with Python ${{ matrix.python-version }} | |
| strategy: | |
| matrix: | |
| # We do not need to test under all versions, we do it under linux | |
| node-version: [20.x] | |
| os-version: ["macos-13"] | |
| target: ["x86_64-apple-darwin", "aarch64-apple-darwin"] | |
| include: | |
| - target: x86_64-apple-darwin | |
| python-version: "3.9" | |
| - target: x86_64-apple-darwin | |
| python-version: "3.10" | |
| - target: x86_64-apple-darwin | |
| python-version: "3.11" | |
| - target: x86_64-apple-darwin | |
| python-version: "3.12" | |
| - target: x86_64-apple-darwin | |
| python-version: "fallback" | |
| - target: aarch64-apple-darwin | |
| python-version: "fallback" | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| target: ${{ matrix.target }} | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| if: (matrix.python-version != 'fallback') | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (fallback) | |
| if: (matrix.python-version == 'fallback') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| run: cd packages/cubejs-backend-native && yarn run native:build | |
| - name: Build native (with Python) | |
| if: (matrix.python-version != 'fallback') | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| CARGO_BUILD_TARGET: ${{ matrix.target }} | |
| run: cd packages/cubejs-backend-native && yarn run native:build-debug-python | |
| - name: Tests | |
| # We cannot test arm64 on x64 | |
| if: (matrix.target == 'x86_64-apple-darwin') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| run: cd packages/cubejs-backend-native && yarn run test:unit | |
| native_windows: | |
| needs: [lint] | |
| runs-on: ${{ matrix.os-version }} | |
| timeout-minutes: 60 | |
| name: Build ${{ matrix.os-version }} ${{ matrix.node-version }} with Python ${{ matrix.python-version }} | |
| strategy: | |
| matrix: | |
| # We do not need to test under all versions, we do it under linux | |
| node-version: [20.x] | |
| os-version: [windows-2019] | |
| python-version: ["fallback"] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Disable rustup update (issue workaround for Windows) | |
| run: rustup set auto-self-update disable | |
| shell: bash | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2024-07-15 | |
| # override: true # this is by default on | |
| rustflags: "" | |
| components: rustfmt | |
| - name: Install Python | |
| uses: actions/setup-python@v5 | |
| if: (matrix.python-version != 'fallback') | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| - name: Set Yarn version | |
| run: yarn policies set-version v1.22.22 | |
| - name: Get yarn cache directory path | |
| id: yarn-cache-dir-path | |
| run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" | |
| shell: bash | |
| - name: Restore yarn cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.yarn-cache-dir-path.outputs.dir }} | |
| key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} | |
| restore-keys: | | |
| ${{ runner.os }}-yarn- | |
| - name: Yarn install | |
| uses: nick-fields/retry@v3 | |
| env: | |
| CUBESTORE_SKIP_POST_INSTALL: true | |
| with: | |
| max_attempts: 3 | |
| retry_on: error | |
| retry_wait_seconds: 15 | |
| timeout_minutes: 20 | |
| command: yarn install --frozen-lockfile | |
| - name: Lerna tsc | |
| run: yarn tsc | |
| - name: Build native (fallback) | |
| if: (matrix.python-version == 'fallback') | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| run: cd packages/cubejs-backend-native && yarn run native:build | |
| - name: Build native (with Python) | |
| if: (matrix.python-version != 'fallback') | |
| env: | |
| PYO3_PYTHON: python${{ matrix.python-version }} | |
| run: cd packages/cubejs-backend-native && yarn run native:build-debug-python | |
| - name: Tests | |
| env: | |
| CUBESQL_STREAM_MODE: true | |
| CUBEJS_NATIVE_INTERNAL_DEBUG: true | |
| run: cd packages/cubejs-backend-native && yarn run test:unit |