diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c010ba8c1..a4042b8bc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @github/semantic-code +* @github/semantic-code @github/blackbird diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c561bd429..e8bb697e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ name: Continuous integration + on: push: branches: [main] @@ -6,6 +7,9 @@ on: schedule: - cron: "0 0 1,15 * *" +permissions: + contents: read + # In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -17,18 +21,23 @@ jobs: strategy: matrix: rust: [stable] + env: + # cargo hack does not use the default-members in Cargo.toml, so we restrict to those explicitly + CARGO_HACK: cargo hack -p lsp-positions -p stack-graphs -p tree-sitter-stack-graphs --feature-powerset --exclude-features copious-debugging steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: ${{ matrix.rust }} + - name: Install cargo-hack + run: cargo install cargo-hack - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check formatting run: cargo fmt --all -- --check - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -36,43 +45,56 @@ jobs: key: ${{ runner.OS }}-cargo-${{ hashFiles('**/Cargo.lock') }} restore-keys: | ${{ runner.OS }}-cargo- - - name: Install cargo-valgrind + - name: Build library (all feature combinations) + run: ${{ env.CARGO_HACK }} --no-dev-deps build + - name: Run test suite (all feature combinations) + run: ${{ env.CARGO_HACK }} test + - name: Run test suite with all optimizations (default features) + run: cargo test --release + - name: Install valgrind run: | sudo apt-get update sudo apt-get install -y valgrind - cargo install cargo-valgrind - - name: Build library - run: cargo build + - name: Run test suite under valgrind (default features) + id: valgrind + # We only need to use valgrind to test the crates that have C bindings. + run: script/ci-test-valgrind -p stack-graphs + - name: Upload valgrind log + if: ${{ failure() && steps.valgrind.outcome == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: valgrind logs + path: ${{ runner.temp }}/valgrind.log - name: Ensure C headers are up to date run: | script/cbindgen test -z "$(git status --porcelain)" - - name: Run test suite - run: cargo test - - name: Run test suite under valgrind - # We only need to use valgrind to test the crates that have C bindings. - run: cargo valgrind test -p stack-graphs - - name: Run lsp-positions tests without tree-sitter - run: cargo test -p lsp-positions --no-default-features - - name: Run test suite with all features enabled - run: cargo test --all-features - - name: Run test suite with all optimizations - run: cargo test --release - # Do the new project test last, because it adds the crate in the current source - # folder, and that shouldn't influence other tests. + + test-init: + needs: [test-rust] + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable] + + steps: + - name: Install Rust environment + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 + with: + rust-version: ${{ matrix.rust }} + - name: Checkout code + uses: actions/checkout@v4 + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.cargo + target + key: ${{ runner.OS }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.OS }}-cargo- - name: Generate, build, and run new language project - run: | - cargo run --bin tree-sitter-stack-graphs --features cli -- init \ - --language-name InitTest \ - --language-id init_test \ - --language-file-extension it \ - --grammar-crate-name tree-sitter-python \ - --grammar-crate-version 0.20.0 \ - --internal \ - --non-interactive - cargo check -p tree-sitter-stack-graphs-init_test --all-features - cargo test -p tree-sitter-stack-graphs-init_test - cargo run -p tree-sitter-stack-graphs-init_test --features cli -- help + run: script/ci-test-init list-languages: runs-on: ubuntu-latest @@ -81,7 +103,7 @@ jobs: working-directory: languages steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: List languages id: language-list run: echo "languages=$(find -mindepth 1 -maxdepth 1 -type d -printf '%P\n' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT @@ -98,11 +120,13 @@ jobs: steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: ${{ matrix.rust }} + - name: Install cargo-hack + run: cargo install cargo-hack - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -111,11 +135,11 @@ jobs: restore-keys: | ${{ runner.OS }}-cargo- - name: Checkout code - uses: actions/checkout@v3 - - name: Build - run: cargo build -p ${{ matrix.language }} - - name: Test - run: cargo test -p ${{ matrix.language }} + uses: actions/checkout@v4 + - name: Build (all feature combinations) + run: cargo hack -p ${{ matrix.language }} --feature-powerset build + - name: Test (all features) + run: cargo test -p ${{ matrix.language }} --all-features test-cli: runs-on: ubuntu-latest @@ -128,11 +152,11 @@ jobs: steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: ${{ matrix.rust }} - name: Cache dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -141,7 +165,7 @@ jobs: restore-keys: | ${{ runner.OS }}-cargo- - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: lfs: true - name: Build diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 5c4288c84..d68ffa75d 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -1,9 +1,14 @@ name: Performance testing + on: pull_request: paths: - 'stack-graphs/**' +permissions: + contents: read + pull-requests: write + # In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources. concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -37,7 +42,7 @@ jobs: done: ${{ steps.done.outputs.cache-hit }} steps: - name: "Checkout base code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.BASE_REPO }} ref: ${{ env.BASE_SHA }} @@ -49,7 +54,7 @@ jobs: printf 'BASE_SHA=%s\n' "$(git rev-list -1 ${{ env.BASE_SHA }} -- stack-graphs)" >> $GITHUB_ENV working-directory: ${{ env.BASE_DIR }} - name: "Checkout head code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.HEAD_REPO }} ref: ${{ env.HEAD_SHA }} @@ -62,7 +67,7 @@ jobs: working-directory: ${{ env.HEAD_DIR }} - name: "Check cached status" id: done - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: path: done key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} @@ -79,11 +84,11 @@ jobs: BASE_SHA: ${{ needs.changes.outputs.base-sha }} steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: stable - name: Cache Rust dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -94,7 +99,7 @@ jobs: sudo apt-get install -y valgrind - name: "Cache base result" id: cache-base-result - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ${{ env.MASSIF_OUT }} @@ -102,7 +107,7 @@ jobs: key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_NAME }} - name: "Checkout base code" if: steps.cache-base-result.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.BASE_REPO }} ref: ${{ env.BASE_SHA }} @@ -130,7 +135,7 @@ jobs: ${{ env.BASE_DIR }}/data/${{ env.TEST_NAME }} ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} - name: Upload results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.BASE_ARTIFACT }} path: | @@ -148,11 +153,11 @@ jobs: HEAD_SHA: ${{ needs.changes.outputs.head-sha }} steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 with: rust-version: stable - name: Cache Rust dependencies - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ~/.cargo @@ -163,7 +168,7 @@ jobs: sudo apt-get install -y valgrind - name: "Cache head result" id: cache-head-result - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: | ${{ env.MASSIF_OUT }} @@ -171,7 +176,7 @@ jobs: key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} - name: "Checkout head code" if: steps.cache-head-result.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ env.HEAD_REPO }} ref: ${{ env.HEAD_SHA }} @@ -199,7 +204,7 @@ jobs: ${{ env.HEAD_DIR }}/data/${{ env.TEST_NAME }} ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} - name: Upload results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.HEAD_ARTIFACT }} path: | @@ -233,12 +238,12 @@ jobs: # Download results # - name: Download base results - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ env.BASE_ARTIFACT }} path: ${{ env.BASE_ARTIFACT }} - name: Download head results - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: ${{ env.HEAD_ARTIFACT }} path: ${{ env.HEAD_ARTIFACT }} @@ -246,7 +251,7 @@ jobs: # Create report # - name: "Checkout code" - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: ${{ env.SRC_DIR }} - name: Generate summary @@ -268,7 +273,7 @@ jobs: - name: Create status marker run: touch done - name: "Cache status" - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: done key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} diff --git a/.github/workflows/publish-lsp-positions.yml b/.github/workflows/publish-lsp-positions.yml index 210662a00..2ca05a133 100644 --- a/.github/workflows/publish-lsp-positions.yml +++ b/.github/workflows/publish-lsp-positions.yml @@ -5,6 +5,9 @@ on: tags: - lsp-positions-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './lsp-positions' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/lsp-positions. diff --git a/.github/workflows/publish-stack-graphs.yml b/.github/workflows/publish-stack-graphs.yml index 641a93488..29b21020b 100644 --- a/.github/workflows/publish-stack-graphs.yml +++ b/.github/workflows/publish-stack-graphs.yml @@ -5,6 +5,9 @@ on: tags: - stack-graphs-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './stack-graphs' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/stack-graphs. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml index 6d7afb769..40878de95 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-java.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-java.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-java-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './languages/tree-sitter-stack-graphs-java' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-java. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml index ff6826770..b9fbceec3 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-javascript.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-javascript-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './languages/tree-sitter-stack-graphs-javascript' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-javascript. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-python.yml b/.github/workflows/publish-tree-sitter-stack-graphs-python.yml new file mode 100644 index 000000000..dc90a6974 --- /dev/null +++ b/.github/workflows/publish-tree-sitter-stack-graphs-python.yml @@ -0,0 +1,48 @@ +name: Publish tree-sitter-stack-graphs-python release + +on: + push: + tags: + - tree-sitter-stack-graphs-python-v* + +permissions: + contents: write + +jobs: + publish-crate: + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + CRATE_DIR: './languages/tree-sitter-stack-graphs-python' + steps: + - name: Install Rust environment + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 + - name: Checkout repository + uses: actions/checkout@v4 + # TODO Verify the crate version matches the tag + - name: Test crate + run: cargo test --all-features + working-directory: ${{ env.CRATE_DIR }} + - name: Verify publish crate + run: cargo publish --dry-run + working-directory: ${{ env.CRATE_DIR }} + - name: Publish crate + run: cargo publish + working-directory: ${{ env.CRATE_DIR }} + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + create-release: + needs: publish-crate + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Create GitHub release + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 + with: + body: | + Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-python. + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml index a3409f397..9eb696e2a 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs-typescript.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-typescript-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './languages/tree-sitter-stack-graphs-typescript' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -36,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs-typescript. diff --git a/.github/workflows/publish-tree-sitter-stack-graphs.yml b/.github/workflows/publish-tree-sitter-stack-graphs.yml index e9395ab1f..0bf56c143 100644 --- a/.github/workflows/publish-tree-sitter-stack-graphs.yml +++ b/.github/workflows/publish-tree-sitter-stack-graphs.yml @@ -5,6 +5,9 @@ on: tags: - tree-sitter-stack-graphs-v* +permissions: + contents: write + jobs: publish-crate: runs-on: ubuntu-latest @@ -14,9 +17,9 @@ jobs: CRATE_DIR: './tree-sitter-stack-graphs' steps: - name: Install Rust environment - uses: hecrj/setup-rust-action@v1 + uses: hecrj/setup-rust-action@110f36749599534ca96628b82f52ae67e5d95a3c # v2 - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # TODO Verify the crate version matches the tag - name: Test crate run: cargo test --all-features @@ -29,31 +32,6 @@ jobs: working-directory: ${{ env.CRATE_DIR }} env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} - publish-npm: - needs: publish-crate - runs-on: ubuntu-latest - env: - PACKAGE_DIR: './tree-sitter-stack-graphs/npm' - steps: - - name: Install Node environment - uses: actions/setup-node@v3 - with: - node-version: 16.x - registry-url: 'https://registry.npmjs.org' - - name: Checkout repository - uses: actions/checkout@v3 - # TODO Verify the package version matches the tag - - name: Install dependencies - run: npm install - working-directory: ${{ env.PACKAGE_DIR }} - - name: Verify package - run: npm publish --dry-run - working-directory: ${{ env.PACKAGE_DIR }} - - name: Publish package - run: npm publish - working-directory: ${{ env.PACKAGE_DIR }} - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} create-release: needs: publish-crate runs-on: ubuntu-latest @@ -61,9 +39,9 @@ jobs: contents: write steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create GitHub release - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@440c8c1cb0ed28b9f43e4d1d670870f059653174 # v1 with: body: | Find more info on all releases at https://crates.io/crates/tree-sitter-stack-graphs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1992fceca..3827e3473 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,8 +32,8 @@ Here are a few things you can do that will increase the likelihood of your pull If you are one of the maintainers of this package, bump the version numbers in [`Cargo.toml`](Cargo.toml) and [`README.md`](README.md), then follow the typical instructions to publish a new version to [crates.io][]: ``` -$ cargo package -$ cargo publish +cargo package +cargo publish ``` [crates.io]: https://crates.io/stack-graphs/ @@ -43,4 +43,3 @@ $ cargo publish - [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) - [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) - [GitHub Help](https://help.github.com) - diff --git a/languages/tree-sitter-stack-graphs-java/CHANGELOG.md b/languages/tree-sitter-stack-graphs-java/CHANGELOG.md index f7483cabc..d295d8ce0 100644 --- a/languages/tree-sitter-stack-graphs-java/CHANGELOG.md +++ b/languages/tree-sitter-stack-graphs-java/CHANGELOG.md @@ -5,9 +5,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v0.5.0 -- 2024-12-12 -## [0.2.0] - 2023-03-21 +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-java` dependency is updated to version 0.23.4. + +## v0.4.0 -- 2024-07-09 + +### Added + +- Add rules for the `condition` node that was missing. + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.3.0 -- 2024-03-06 + +The `tree-sitter-stack-graphs` is updated to `v0.8`. + +### Changed + +- The `cli` feature is now required to install the CLI. + +## v0.2.0 -- 2023-03-21 ### Added diff --git a/languages/tree-sitter-stack-graphs-java/Cargo.toml b/languages/tree-sitter-stack-graphs-java/Cargo.toml index 8bd3e5f93..0ad8cb687 100644 --- a/languages/tree-sitter-stack-graphs-java/Cargo.toml +++ b/languages/tree-sitter-stack-graphs-java/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tree-sitter-stack-graphs-java" -version = "0.2.0" +version = "0.5.0" description = "Stack graphs for the Java programming language" homepage = "https://github.com/github/stack-graphs/tree/main/languages/tree-sitter-stack-graphs-java" @@ -23,6 +23,7 @@ keywords = ["tree-sitter", "stack-graphs", "java"] [[bin]] name = "tree-sitter-stack-graphs-java" path = "rust/bin.rs" +required-features = ["cli"] [lib] path = "rust/lib.rs" @@ -33,8 +34,15 @@ name = "test" path = "rust/test.rs" harness = false # need to provide own main function to handle running tests +[features] +cli = ["anyhow", "clap", "tree-sitter-stack-graphs/cli"] + [dependencies] -anyhow = "1.0" -clap = { version = "4", features = ["derive"] } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs", features=["cli"] } -tree-sitter-java = { version = "=0.20.0" } +anyhow = { version = "1.0", optional = true } +clap = { version = "4", features = ["derive"], optional = true } +tree-sitter-java = { version = "=0.23.4" } +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate + +[dev-dependencies] +anyhow = { version = "1.0" } +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } diff --git a/languages/tree-sitter-stack-graphs-java/README.md b/languages/tree-sitter-stack-graphs-java/README.md index 821796b67..97820ed3f 100644 --- a/languages/tree-sitter-stack-graphs-java/README.md +++ b/languages/tree-sitter-stack-graphs-java/README.md @@ -1,30 +1,138 @@ # tree-sitter-stack-graphs definition for Java -This project defines tree-sitter-stack-graphs rules for Java using the [tree-sitter-java](https://www.npmjs.com/package/tree-sitter-java) grammar. +This project defines tree-sitter-stack-graphs rules for Java using the [tree-sitter-java][] grammar. -## Local Development +[tree-sitter-java]: https://crates.io/crates/tree-sitter-java + +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-java/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-java/CHANGELOG.md) + +## Using the API + +To use this library, add the following to your `Cargo.toml`: + +```toml +[dependencies] +tree-sitter-stack-graphs-java = "0.5" +``` + +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-java/*/) for more details on how to use this library. + +## Using the Command-line Program + +The command-line program for `tree-sitter-stack-graphs-java` lets you do stack graph based analysis and lookup from the command line. + +The CLI can be run as follows: + +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-java + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-java`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-java` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-java index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-java status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-java clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-java query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. + +## Development + +The project is written in Rust, and requires a recent version installed. Rust can be installed and updated using [rustup][]. + +[rustup]: https://rustup.rs/ The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. - Tests are put into the `test` directory. -The following commands are intended to be run from the repo root. +### Running Tests + +Run the tests as follows: + +```sh +cargo test +``` + +The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. + +Run the CLI from source as follows: + +```sh +cargo run --features cli -- ARGS +``` + +Sources are formatted using the standard Rust formatted, which is applied by running: + +```sh +cargo fmt +``` + +### Writing TSG + +The stack graph rules are written in [tree-sitter-graph][]. Checkout the [examples][], +which contain self-contained TSG rules for specific language features. A VSCode +[extension][] is available that provides syntax highlighting for TSG files. -Run all tests in the project by executing the following: +[tree-sitter-graph]: https://github.com/tree-sitter/tree-sitter-graph +[examples]: https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/ +[extension]: https://marketplace.visualstudio.com/items?itemName=tree-sitter.tree-sitter-graph - cargo test -p tree-sitter-stack-graphs-java +Parse and test a single file by executing the following commands: -Parse a single test file: - `cargo run -p tree-sitter-stack-graphs-java -- parse OPTIONS FILENAME` +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... +``` -Test a single file: - `cargo run -p tree-sitter-stack-graphs-java -- test OPTIONS FILENAME` +Generate a visualization to debug failing tests by passing the `-V` flag: -For debugging purposes, it is useful to run the visualization tool to generate graphs for the tests being run. +```sh +cargo run --features cli -- test -V TESTFILES... +``` -To run a test and generate the visualization: +To generate the visualization regardless of test outcome, execute: -`cargo run -p tree-sitter-stack-graphs-java -- test --output-mode=always -V=%r/%d/%n.html FILENAME` +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... +``` -Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-java/rust/lib.rs b/languages/tree-sitter-stack-graphs-java/rust/lib.rs index 942785044..9b2cb39c8 100644 --- a/languages/tree-sitter-stack-graphs-java/rust/lib.rs +++ b/languages/tree-sitter-stack-graphs-java/rust/lib.rs @@ -14,8 +14,6 @@ pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.java"; /// The stack graphs builtins source for this language. pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.java"); -/// The name of the file path global variable -pub const FILE_PATH_VAR: &str = "FILE_PATH"; /// The name of the project name global variable pub const PROJECT_NAME_VAR: &str = "PROJECT_NAME"; @@ -27,7 +25,7 @@ pub fn try_language_configuration( cancellation_flag: &dyn CancellationFlag, ) -> Result { LanguageConfiguration::from_sources( - tree_sitter_java::language(), + tree_sitter_java::LANGUAGE.into(), Some(String::from("source.java")), None, vec![String::from("java")], diff --git a/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md b/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md index 3b21ea383..ae90a2f01 100644 --- a/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md +++ b/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md @@ -4,3 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## v0.3.0 -- 2024-12-12 + +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-javascript` dependency is updated to version 0.23.1. + +## v0.2.0 -- 2024-07-09 + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.1.0 -- 2024-03-06 + +Initial release. diff --git a/languages/tree-sitter-stack-graphs-javascript/Cargo.toml b/languages/tree-sitter-stack-graphs-javascript/Cargo.toml index 624edbab2..d1206c6e5 100644 --- a/languages/tree-sitter-stack-graphs-javascript/Cargo.toml +++ b/languages/tree-sitter-stack-graphs-javascript/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tree-sitter-stack-graphs-javascript" -version = "0.1.0" +version = "0.3.0" description = "Stack graphs definition for JavaScript using tree-sitter-javascript" readme = "README.md" keywords = ["tree-sitter", "stack-graphs", "javascript"] @@ -30,10 +30,11 @@ anyhow = { version = "1.0", optional = true } clap = { version = "4", optional = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -stack-graphs = { version = "0.12", path = "../../stack-graphs" } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs" } -tree-sitter-javascript = { git = "https://github.com/tree-sitter/tree-sitter-javascript", rev = "5720b249490b3c17245ba772f6be4a43edb4e3b7" } +stack-graphs = { version = "0.14", path = "../../stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-graph = "0.12" +tree-sitter-javascript = "=0.23.1" +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate [dev-dependencies] anyhow = "1.0" -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs", features = ["cli"] } +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } diff --git a/languages/tree-sitter-stack-graphs-javascript/README.md b/languages/tree-sitter-stack-graphs-javascript/README.md index 3b53f03de..c7eefa955 100644 --- a/languages/tree-sitter-stack-graphs-javascript/README.md +++ b/languages/tree-sitter-stack-graphs-javascript/README.md @@ -4,27 +4,73 @@ This project defines tree-sitter-stack-graphs rules for JavaScript using the [tr [tree-sitter-javascript]: https://crates.io/crates/tree-sitter-javascript -## Usage +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-javascript/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-javascript/CHANGELOG.md) + +## Using the API To use this library, add the following to your `Cargo.toml`: -``` toml +```toml [dependencies] -tree-sitter-stack-graphs-javascript = "0.0.1" +tree-sitter-stack-graphs-javascript = "0.3" ``` Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-javascript/*/) for more details on how to use this library. -## Command-line Program +## Using the Command-line Program The command-line program for `tree-sitter-stack-graphs-javascript` lets you do stack graph based analysis and lookup from the command line. -Install the program using `cargo install` as follows: +The CLI can be run as follows: -``` sh -$ cargo install --features cli tree-sitter-stack-graphs-javascript -$ tree-sitter-stack-graphs-javascript --help -``` +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-javascript + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-javascript`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-javascript` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-javascript index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-javascript status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-javascript clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-javascript query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. ## Development @@ -35,35 +81,29 @@ The project is written in Rust, and requires a recent version installed. Rust c The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. -- Builtins sources and configuration are defined in `src/builtins.js` and `builtins.cfg` respectively. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. - Tests are put into the `test` directory. -### Building and Running Tests - -Build the project by running: - -``` sh -$ cargo build -``` +### Running Tests Run the tests as follows: -``` sh -$ cargo test +```sh +cargo test ``` The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. Run the CLI from source as follows: -``` sh -$ cargo run --features cli -- ARGS +```sh +cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: -``` sh -$ cargo fmt +```sh +cargo fmt ``` ### Writing TSG @@ -78,21 +118,21 @@ which contain self-contained TSG rules for specific language features. A VSCode Parse and test a single file by executing the following commands: -``` sh -$ cargo run --features cli -- parse FILES... -$ cargo run --features cli -- test TESTFILES... +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... ``` Generate a visualization to debug failing tests by passing the `-V` flag: -``` sh -$ cargo run --features cli -- test -V TESTFILES... +```sh +cargo run --features cli -- test -V TESTFILES... ``` To generate the visualization regardless of test outcome, execute: -``` sh -$ cargo run --features cli -- test -V --output-mode=always TESTFILES... +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... ``` -Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs b/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs index 61f113149..cbf80e5cd 100644 --- a/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs +++ b/languages/tree-sitter-stack-graphs-javascript/rust/lib.rs @@ -26,8 +26,7 @@ pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.js"; /// The stack graphs builtins source for this language. pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.js"); -/// The name of the file path global variable. -pub const FILE_PATH_VAR: &str = "FILE_PATH"; +/// The name of the project name global variable pub const PROJECT_NAME_VAR: &str = "PROJECT_NAME"; pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration { @@ -38,7 +37,7 @@ pub fn try_language_configuration( cancellation_flag: &dyn CancellationFlag, ) -> Result { let mut lc = LanguageConfiguration::from_sources( - tree_sitter_javascript::language(), + tree_sitter_javascript::LANGUAGE.into(), Some(String::from("source.js")), None, vec![String::from("js")], diff --git a/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg index d7faf39f2..7ca4eee88 100644 --- a/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg +++ b/languages/tree-sitter-stack-graphs-javascript/src/stack-graphs.tsg @@ -257,6 +257,7 @@ inherit .import_statement inherit .pkg_pop inherit .pkg_push inherit .return_or_yield +inherit .containing_class_value @@ -282,6 +283,9 @@ inherit .return_or_yield node @prog.after_scope node @prog.before_scope let @prog.closure_point = @prog.after_scope + + ; apparently it's perfectly cromulent to `return` from the top level scope (because control flow happens there too), so we make a top-level return node. + node @prog.return_or_yield } ;; ### Program Queries @@ -357,6 +361,10 @@ inherit .return_or_yield node @prog.builtins_Regex_prototype node @prog.builtins_arguments_prototype node @prog.builtins_empty_object + ; !!!! HACK + ; stack graphs currently make it impossible to test if an inherited variable + ; like this is defined or not + let @prog.containing_class_value = @prog.builtins_null } @@ -398,7 +406,19 @@ inherit .return_or_yield node @comment.after_scope node @comment.before_scope node @comment.value + node @comment.covalue + node @comment.new_bindings ; for object patterns edge @comment.after_scope -> @comment.before_scope + node @comment.source ; for export clauses with multiple exports +} + +(identifier) @identifier { + node @identifier.before_scope + node @identifier.after_scope + node @identifier.value + node @identifier.covalue + node @identifier.new_bindings + edge @identifier.after_scope -> @identifier.before_scope } @@ -503,7 +523,7 @@ inherit .return_or_yield (declaration)@decl)@export_stmt { node pop_guard_default - attr (pop_guard_default) symbol_definition = "GUARD:DEFAULT", source_node = @decl, definiens_node = @export_stmt.export_statement + attr (pop_guard_default) symbol_definition = "GUARD:DEFAULT", source_node = @decl, definiens_node = @export_stmt edge @export_stmt.exports -> pop_guard_default ;; edge export_stmt_pop_guard_default -> @decl.value ;; FIXME declarations have no .value @@ -540,13 +560,61 @@ inherit .return_or_yield node default_detour_pop attr (default_detour_push) push_symbol = "default" - attr (default_detour_pop) symbol_definition = "default", source_node = @decl, definiens_node = @decl + attr (default_detour_pop) symbol_definition = "default", source_node = @decl, definiens_node = @export_stmt edge pop_guard_default -> default_detour_push edge default_detour_push -> default_detour_pop ; edge default_detour_pop -> @decl.value ;; FIXME declarations have no .value } +(export_statement "default" + value:(_)@value)@export_stmt { + + node @export_stmt.pop_guard_default + attr (@export_stmt.pop_guard_default) symbol_definition = "GUARD:DEFAULT", source_node = @value, definiens_node = @export_stmt + edge @export_stmt.exports -> @export_stmt.pop_guard_default + edge @export_stmt.pop_guard_default -> @value.value + + ; !!!! HACK These detour nodes are a massive hack to allow find all refs land on defs + ; for the default values of modules that have useful names like the module name or + ; package name + + node detour_push + node @export_stmt.detour_pop + + scan FILE_PATH { + + "^(.+/)?([^/]+)/index\.js$" { + let module_name = $2 + attr (detour_push) push_symbol = module_name + attr (@export_stmt.detour_pop) symbol_definition = module_name, source_node = @value, definiens_node = @export_stmt + edge @export_stmt.pop_guard_default -> detour_push + edge detour_push -> @export_stmt.detour_pop + edge @export_stmt.detour_pop -> @value.value + } + + "^(.+/)?([^/]+)\.js$" { + let module_name = $2 + attr (detour_push) push_symbol = module_name + attr (@export_stmt.detour_pop) symbol_definition = module_name, source_node = @value, definiens_node = @export_stmt + edge @export_stmt.pop_guard_default -> detour_push + edge detour_push -> @export_stmt.detour_pop + edge @export_stmt.detour_pop -> @value.value + } + + } + + node default_detour_push + node @export_stmt.default_detour_pop + + attr (default_detour_push) push_symbol = "default" + attr (@export_stmt.default_detour_pop) symbol_definition = "default", source_node = @value, definiens_node = @export_stmt + edge @export_stmt.pop_guard_default -> default_detour_push + edge default_detour_push -> @export_stmt.default_detour_pop + edge @export_stmt.default_detour_pop -> @value.value + +} + (export_statement declaration: [ (function_declaration name:(identifier)@name) @@ -732,13 +800,8 @@ inherit .return_or_yield (export_statement value:(_)@default_expr)@export_stmt { - node pop_default_guard - - attr (pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @export_stmt, definiens_node = @export_stmt.export_statement edge @default_expr.before_scope -> @export_stmt.before_scope edge @export_stmt.after_scope -> @default_expr.after_scope - edge @export_stmt.exports -> pop_default_guard - edge pop_default_guard -> @default_expr.value } @@ -1428,6 +1491,7 @@ inherit .return_or_yield node @name.pop node @class_decl.class_value + let @class_decl.containing_class_value = @class_decl.class_value node guard_prototype node @class_decl.prototype node @class_decl.constructor @@ -1500,7 +1564,7 @@ inherit .return_or_yield node @method_def.after_scope node @method_def.before_scope node @method_def.method_value - + } ( @@ -1508,7 +1572,7 @@ inherit .return_or_yield name:(_)@name)@method_def (#eq? @name "constructor") ) { - + ; augmentation for the constructor attr (@name.pop) symbol_definition = "GUARD:CONSTRUCTOR", source_node = @name edge @method_def.class_value -> @name.pop @@ -1598,14 +1662,17 @@ inherit .return_or_yield -(field_definition - property:(property_identifier)@property)@field_def { - +(field_definition)@field_def { node @field_def.after_scope node @field_def.before_scope +} + +(field_definition + property:(_)@property)@field_def { + node @property.pop node property_pop_dot - + attr (@property.pop) node_definition = @property attr (property_pop_dot) pop_symbol = "GUARD:MEMBER" edge @field_def.after_scope -> property_pop_dot @@ -1627,6 +1694,15 @@ inherit .return_or_yield edge @property.pop -> @value.value } +(class_static_block body: (_) @body) @static_block { + node @static_block.before_scope + node @static_block.after_scope + + edge @body.before_scope -> @static_block.before_scope + + edge @static_block.after_scope -> @static_block.before_scope +} + ;; #### Statement Block @@ -1707,16 +1783,19 @@ inherit .return_or_yield edge @if_stmt.after_scope -> @alternative.after_scope } +(else_clause)@else_clause { + node @else_clause.after_scope + node @else_clause.before_scope +} + (else_clause - (_)@inner)@else_clause { + . (_)@inner)@else_clause { let @else_clause.hoist_point = @inner.before_scope } (else_clause (_)@inner)@else_clause { - node @else_clause.after_scope - node @else_clause.before_scope ; scopes flow in and right back out edge @inner.before_scope -> @else_clause.before_scope edge @else_clause.after_scope -> @inner.after_scope @@ -2212,12 +2291,14 @@ inherit .return_or_yield (member_expression) (parenthesized_expression) (undefined) - (primary_expression/identifier) (this) (super) (number) (string) (template_string) + (template_substitution) + (string_fragment) + (escape_sequence) (regex) (true) (false) @@ -2226,7 +2307,7 @@ inherit .return_or_yield (import) (object) (array) - (function) + (function_expression) (arrow_function) (generator_function) (class) @@ -2243,7 +2324,7 @@ inherit .return_or_yield (yield_expression) (spread_element) ]@expr { - + node @expr.after_scope node @expr.before_scope node @expr.value @@ -2251,7 +2332,7 @@ inherit .return_or_yield } [ - (function body:(_)@body) + (function_expression body:(_)@body) (arrow_function body:(_)@body) (generator_function body:(_)@body) ] { @@ -2289,8 +2370,8 @@ inherit .return_or_yield ;; #### Template Strings ; template_strings w/ no substitutions -(template_string (_)* @substs)@template_string { - if (is-empty @substs) { +(template_string (_)* @parts)@template_string { + if (is-empty @parts) { edge @template_string.after_scope -> @template_string.before_scope } } @@ -2298,33 +2379,45 @@ inherit .return_or_yield ; nonempty template string, value ; LATER-TODO this isn't really right, but it gets flows through the template string ; which may be useful -(template_string (template_substitution (_)@inner_expr))@template_string { +(template_string (_)@part)@template_string { ; the value of a template string is a template string value built from the values of its substitutions ; attr (@template_string.value) "template_string_value" - edge @template_string.value -> @inner_expr.value + edge @template_string.value -> @part.value } ; nonempty template string, first substitution -(template_string . (template_substitution (_)@first_inner_expr))@template_string { +(template_string . (_)@first)@template_string { ; scopes propagate into the first subtitution of the template string - edge @first_inner_expr.before_scope -> @template_string.before_scope + edge @first.before_scope -> @template_string.before_scope } ; nonempty template string, between substitutions (template_string - (template_substitution (_)@left_inner_expr) + (_) @left . - (template_substitution (_)@right_inner_expr))@_template_string { + (_) @right) { ; scopes propagate from left substitutions to right substitutions - edge @right_inner_expr.before_scope -> @left_inner_expr.after_scope + edge @right.before_scope -> @left.after_scope } ; nonempty template string, last substitution -(template_string . (template_substitution (_)@last_inner_expr))@template_string { +(template_string (_) @last .)@template_string { ; scopes propagate out of the last substitution to the template string - edge @template_string.after_scope -> @last_inner_expr.after_scope + edge @template_string.after_scope -> @last.after_scope } +[ + (string_fragment) + (escape_sequence) +]@part { + edge @part.after_scope -> @part.before_scope +} + +(template_substitution (_)@expr)@subst { + edge @expr.before_scope -> @subst.before_scope + edge @subst.after_scope -> @expr.after_scope + edge @subst.value -> @expr.value +} ;; #### Numbers @@ -2340,10 +2433,10 @@ inherit .return_or_yield ;; #### Variables -(primary_expression/identifier)@variable { - ; scopes don't change - edge @variable.after_scope -> @variable.before_scope - +[ + (primary_expression/identifier)@variable + (member_expression object:(identifier)@variable) +] { ; value is a lookup, ie a push attr (@variable.value) node_reference = @variable edge @variable.value -> @variable.before_scope @@ -2378,6 +2471,14 @@ inherit .return_or_yield ; this is a lookup, ie a push attr (@this.value) symbol_reference = "this", source_node = @this edge @this.value -> @this.before_scope + + node pop_this + attr (pop_this) symbol_definition = "this" + node guard_prototype + attr (guard_prototype) push_symbol = "GUARD:PROTOTYPE" + edge @this.value -> pop_this + edge pop_this -> guard_prototype + edge guard_prototype -> @this.containing_class_value } @@ -2408,6 +2509,9 @@ inherit .return_or_yield ; the value of undefined is the undefined primitive edge @undefined.value -> @undefined.builtins_undefined + + ; !!!! HACK: `undefined` is a perfectly cromulent name for a parameter, but the parser thinks it means this node instead. For the moment, work around the problem this causes by giving all `undefined` nodes covalues like parameters enjoy. + node @undefined.covalue } @@ -2439,6 +2543,9 @@ inherit .return_or_yield attr (@object.member_pop) pop_symbol = "GUARD:MEMBER" edge @object.value -> @object.member_pop + node @object.class_value + node @object.constructor + } ; empty objects @@ -2476,7 +2583,7 @@ inherit .return_or_yield ; shorthand property identifier (shorthand_property_identifier)@shorthand_property_identifier { - + node @shorthand_property_identifier.after_scope node @shorthand_property_identifier.before_scope @@ -2506,10 +2613,13 @@ inherit .return_or_yield ; pairs -(computed_property_name (_)@expr)@computed_property_name { - +(computed_property_name)@computed_property_name { node @computed_property_name.after_scope node @computed_property_name.before_scope +} + +(computed_property_name (_)@expr)@computed_property_name { + edge @expr.before_scope -> @computed_property_name.before_scope edge @computed_property_name.after_scope -> @expr.after_scope @@ -2524,7 +2634,7 @@ inherit .return_or_yield ; This is done differently depending on what the key is. See next rules. ; attr @key.pop "pop" = @key, "definition" - + node @pair.key_pop edge @pair.key_pop -> @value.value edge @object.member_pop -> @pair.key_pop @@ -2690,7 +2800,7 @@ inherit .return_or_yield ;; #### Function Literals ; functions with names -(function +(function_expression name:(_)@name parameters:(_)@call_sig)@fun { @@ -2705,7 +2815,7 @@ inherit .return_or_yield ; function -(function +(function_expression parameters:(_)@call_sig body:(_)@body)@fun { @@ -2764,7 +2874,7 @@ inherit .return_or_yield edge @fun.value_arg_scope -> JUMP_TO_SCOPE_NODE } -(function +(function_expression parameters: (formal_parameters (_)@param))@fun { @@ -2818,7 +2928,7 @@ inherit .return_or_yield ; function values have return nodes which need to be visible for returns attr (fun_value_return) pop_symbol = "GUARD:RETURN" edge fun_value_call -> fun_value_return - let @body.return_or_yield = fun_value_return + edge fun_value_return -> @fun.return_or_yield ; function values have this nodes which need to be visible for method calls attr (fun_value_this) push_symbol = "this" @@ -2839,16 +2949,9 @@ inherit .return_or_yield (arrow_function parameter:(_)@param)@fun { - - node @param.after_scope node param_arg_index - node @param.before_scope - node @param.covalue node param_pop - ; scope flows from the param right back out - edge @param.after_scope -> @param.before_scope - ; but augmented with a pop, b/c it's not a pattern attr (param_pop) node_definition = @param edge param_pop -> @param.covalue @@ -3086,8 +3189,7 @@ inherit .return_or_yield ;; ##### Member Expressions (member_expression - object: (_)@object - property: (_)@property)@member_expr + object:(_)@object property:(_)@property)@member_expr { node member_push @@ -3104,6 +3206,9 @@ inherit .return_or_yield edge @member_expr.value -> property_push edge member_push -> @object.value + ; (member_expression) nodes can occur in patterns + node @member_expr.covalue + node @member_expr.new_bindings } ;; ##### Subscript Expressions @@ -3128,6 +3233,9 @@ inherit .return_or_yield edge @subscript_expr.value -> @subscript_expr.index_push edge @subscript_expr.index_push -> subscript_expr_push_dot + ; subscript expressions can appear in array patterns, on the left, and thus require a covalue & bindings + node @subscript_expr.covalue + node @subscript_expr.new_bindings } (subscript_expression @@ -3392,20 +3500,27 @@ inherit .return_or_yield ;; #### Comma Operator / Sequence Expressions -(sequence_expression - left: (_)@left - right: (_)@right)@sequence_expr { +(sequence_expression (_)* @elems)@sequence_expr { + if (is-empty @elems) { + edge @sequence_expr.after_scope -> @sequence_expr.before_scope + } +} - ; scopes propagate left to right - edge @left.before_scope -> @sequence_expr.before_scope +(sequence_expression . (_)@first)@sequence_expr { + edge @first.before_scope -> @sequence_expr.before_scope +} + +(sequence_expression (_)@left . (_)@right) { edge @right.before_scope -> @left.after_scope - edge @sequence_expr.after_scope -> @right.after_scope +} - ; the value is just the value of the right - edge @sequence_expr.value -> @right.value +(sequence_expression (_)@last .)@sequence_expr { + edge @sequence_expr.after_scope -> @last.after_scope + edge @sequence_expr.value -> @last.value } + ;; #### Ternary Expression (ternary_expression @@ -3446,6 +3561,7 @@ inherit .return_or_yield body:(_)@body)@class { node @class.class_value + let @class.containing_class_value = @class.class_value node guard_prototype node @class.prototype node @class.constructor @@ -3463,7 +3579,7 @@ inherit .return_or_yield (class name:(_)@name body:(_)@body)@class { - + node @name.pop attr (@name.pop) syntax_type = "class" @@ -3512,7 +3628,6 @@ inherit .return_or_yield (jsx_text) (jsx_element) (jsx_self_closing_element) - (jsx_fragment) (jsx_expression) ]@first_child ) { @@ -3524,7 +3639,6 @@ inherit .return_or_yield (jsx_text) (jsx_element) (jsx_self_closing_element) - (jsx_fragment) (jsx_expression) ]@left_child . @@ -3532,7 +3646,6 @@ inherit .return_or_yield (jsx_text) (jsx_element) (jsx_self_closing_element) - (jsx_fragment) (jsx_expression) ]@right_child ) { @@ -3544,7 +3657,6 @@ inherit .return_or_yield (jsx_text) (jsx_element) (jsx_self_closing_element) - (jsx_fragment) (jsx_expression) ]@last_child . @@ -3556,25 +3668,37 @@ inherit .return_or_yield (jsx_text)@jsx_text { node @jsx_text.before_scope node @jsx_text.after_scope - + edge @jsx_text.after_scope -> @jsx_text.before_scope } -(jsx_opening_element - name:(_)@element_name)@jsx_opening_element { +(jsx_opening_element)@jsx_opening_element { node @jsx_opening_element.before_scope node @jsx_opening_element.after_scope +} + +(jsx_opening_element + name:(_)@element_name)@jsx_opening_element { + edge @element_name.before_scope -> @jsx_opening_element.before_scope } +(jsx_opening_element + !name)@jsx_opening_element +{ + + edge @jsx_opening_element.after_scope -> @jsx_opening_element.before_scope + +} + (jsx_opening_element name:(_)@element_name !attribute)@jsx_opening_element { - + edge @jsx_opening_element.after_scope -> @element_name.after_scope } @@ -3622,6 +3746,15 @@ inherit .return_or_yield } +(jsx_namespace_name (_) @lhs (_) @rhs)@name { + node @name.before_scope + node @name.after_scope + + edge @lhs.before_scope -> @name.before_scope + edge @rhs.before_scope -> @lhs.after_scope + edge @name.after_scope -> @name.before_scope +} + (jsx_self_closing_element name:(_)@element_name)@jsx_self_closing_element { @@ -3633,11 +3766,20 @@ inherit .return_or_yield } +(jsx_self_closing_element + !name + !attribute)@jsx_self_closing_element +{ + + edge @jsx_self_closing_element.after_scope -> @jsx_self_closing_element.before_scope + +} + (jsx_self_closing_element name:(_)@element_name !attribute)@jsx_self_closing_element { - + edge @jsx_self_closing_element.after_scope -> @element_name.after_scope } @@ -3671,11 +3813,15 @@ inherit .return_or_yield } -(jsx_expression (_)?@child)@jsx_expression { +(jsx_expression)@jsx_expression { node @jsx_expression.before_scope node @jsx_expression.after_scope +} + +(jsx_expression (_)?@child)@jsx_expression { + if none @child { edge @jsx_expression.after_scope -> @jsx_expression.before_scope } else { @@ -3685,13 +3831,26 @@ inherit .return_or_yield } -(jsx_closing_element - name:(_)@element_name)@jsx_closing_element +(jsx_closing_element)@jsx_closing_element { node @jsx_closing_element.before_scope node @jsx_closing_element.after_scope - + +} + +(jsx_closing_element + !name)@jsx_closing_element +{ + + edge @jsx_closing_element.after_scope -> @jsx_closing_element.before_scope + +} + +(jsx_closing_element + name:(_)@element_name)@jsx_closing_element +{ + edge @element_name.before_scope -> @jsx_closing_element.before_scope edge @jsx_closing_element.after_scope -> @element_name.after_scope @@ -3706,12 +3865,6 @@ inherit .return_or_yield name:(identifier)@element_name) ] { - - node @element_name.before_scope - node @element_name.after_scope - - edge @element_name.after_scope -> @element_name.before_scope - scan (source-text @element_name) { ; standard HTML elements "^(a|abbr|acronym|address|applet|area|article|aside|audio|b|base|basefont|bdi|bdo|big|blockquote|body|br|button|canvas|caption|center|cite|code|col|colgroup|data|datalist|dd|del|details|dfn|dialog|dir|div|dl|dt|em|embed|fieldset|figcaption|figure|font|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hgroup|hr|html|i|iframe|input|ins|kbd|label|legend|li|link|main|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|picture|pre|progress|q|rp|rt|ruby|s|samp|script|search|section|select|small|source|span|strike|strong|style|sub|summary|sup|svg|table|tbody|td|template|textarea|tfoot|th|thead|time|title|tr|track|tt|u|ul|var|video|wbr)$" { @@ -3728,86 +3881,6 @@ inherit .return_or_yield } -(nested_identifier)@nested_identifier { - node @nested_identifier.before_scope - node @nested_identifier.after_scope - node @nested_identifier.value - - edge @nested_identifier.after_scope -> @nested_identifier.before_scope -} - -(nested_identifier - (identifier)@first_part - (identifier)@second_part)@nested_identifier -{ - node @first_part.value - node @second_part.value - node guard_member - - attr (@first_part.value) node_reference = @first_part - attr (@second_part.value) node_reference = @second_part - attr (guard_member) symbol_reference = "GUARD:MEMBER" - - edge @first_part.value -> @nested_identifier.before_scope - edge guard_member -> @first_part.value - edge @second_part.value -> guard_member - edge @nested_identifier.value -> @second_part.value -} - -(nested_identifier - (nested_identifier)@first_part - (identifier)@second_part)@nested_identifier -{ - node @second_part.value - node guard_member - - attr (@second_part.value) node_reference = @second_part - attr (guard_member) symbol_reference = "GUARD:MEMBER" - - edge @first_part.before_scope -> @nested_identifier.before_scope - edge guard_member -> @first_part.value - edge @second_part.value -> guard_member - edge @nested_identifier.value -> @second_part.value -} - -(jsx_fragment (_)*@children)@jsx_fragment { - node @jsx_fragment.before_scope - node @jsx_fragment.after_scope - node @jsx_fragment.value - - if (is-empty @children) { - edge @jsx_fragment.after_scope -> @jsx_fragment.before_scope - } -} - -(jsx_fragment - . - (_)@first_child)@jsx_fragment -{ - edge @first_child.before_scope -> @jsx_fragment.before_scope -} - -(jsx_fragment - (_)@left_child - . - (_)@right_child) -{ - edge @right_child.before_scope -> @left_child.after_scope -} - -(jsx_fragment - (_)@last_child - .)@jsx_fragment -{ - edge @jsx_fragment.after_scope -> @last_child.after_scope -} - - - - - - - @@ -4050,13 +4123,11 @@ inherit .return_or_yield ;; ### Attributes Defined on Patterns ;; TODO [ - (assignment_expression left:(identifier)@pattern) (assignment_pattern)@pattern (object_pattern)@pattern (array_pattern)@pattern (rest_pattern)@pattern (pair_pattern)@pattern - (pattern/identifier)@pattern (pattern/property_identifier)@pattern (object_assignment_pattern)@pattern (shorthand_property_identifier_pattern)@pattern @@ -4081,7 +4152,6 @@ inherit .return_or_yield ; scope flows through, binding via a pop edge that goes to an unknown value attr (ident_pat_pop) node_definition = @ident_pat edge ident_pat_pop -> @ident_pat.covalue - edge @ident_pat.after_scope -> @ident_pat.before_scope edge @ident_pat.after_scope -> ident_pat_pop edge @ident_pat.new_bindings -> ident_pat_pop @@ -4402,7 +4472,7 @@ inherit .return_or_yield node pop_default_guard node pop_dot - node pop_name + node @assignment_expr.pop_name attr (pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @exports edge @assignment_expr.exports -> pop_default_guard @@ -4410,47 +4480,47 @@ inherit .return_or_yield attr (pop_dot) pop_symbol = "GUARD:MEMBER" edge pop_default_guard -> pop_dot - attr (pop_name) node_definition = @property - attr (pop_name) definiens_node = @assignment_expr - edge pop_dot -> pop_name - edge pop_name -> @right.value + attr (@assignment_expr.pop_name) node_definition = @property + attr (@assignment_expr.pop_name) definiens_node = @assignment_expr + edge pop_dot -> @assignment_expr.pop_name + edge @assignment_expr.pop_name -> @right.value ;; For ES6 interoperability, expose members as named exports - edge @assignment_expr.exports -> pop_name + edge @assignment_expr.exports -> @assignment_expr.pop_name node detour_push - node detour_pop + node @assignment_expr.detour_pop scan FILE_PATH { "^(.+/)?([^/]+)/index\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } "^(.+/)?([^/]+)\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } } node default_detour_push - node default_detour_pop + node @assignment_expr.default_detour_pop attr (default_detour_push) push_symbol = "default" - attr (default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr + attr (@assignment_expr.default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr edge pop_default_guard -> default_detour_push - edge default_detour_push -> default_detour_pop - edge default_detour_pop -> @right.value + edge default_detour_push -> @assignment_expr.default_detour_pop + edge @assignment_expr.default_detour_pop -> @right.value } @@ -4464,13 +4534,13 @@ inherit .return_or_yield (#eq? @exports "exports") ) { - node pop_default_guard + node @assignment_expr.pop_default_guard node pop_dot - attr (pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @exports - attr (pop_default_guard) definiens_node = @assignment_expr - edge @assignment_expr.exports -> pop_default_guard - edge pop_default_guard -> @right.value + attr (@assignment_expr.pop_default_guard) symbol_definition = "GUARD:DEFAULT", source_node = @exports + attr (@assignment_expr.pop_default_guard) definiens_node = @assignment_expr + edge @assignment_expr.exports -> @assignment_expr.pop_default_guard + edge @assignment_expr.pop_default_guard -> @right.value ;; For ES6 interoperability, expose members as named exports attr (pop_dot) pop_symbol = "GUARD:MEMBER" @@ -4478,38 +4548,38 @@ inherit .return_or_yield edge pop_dot -> @right.value node detour_push - node detour_pop + node @assignment_expr.detour_pop scan FILE_PATH { "^(.+/)?([^/]+)/index\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr - edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + edge @assignment_expr.pop_default_guard -> detour_push + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } "^(.+/)?([^/]+)\.js$" { let module_name = $2 attr (detour_push) push_symbol = module_name - attr (detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr - edge pop_default_guard -> detour_push - edge detour_push -> detour_pop - edge detour_pop -> @right.value + attr (@assignment_expr.detour_pop) symbol_definition = module_name, source_node = @assignment_expr, definiens_node = @assignment_expr + edge @assignment_expr.pop_default_guard -> detour_push + edge detour_push -> @assignment_expr.detour_pop + edge @assignment_expr.detour_pop -> @right.value } } node default_detour_push - node default_detour_pop + node @assignment_expr.default_detour_pop attr (default_detour_push) push_symbol = "default" - attr (default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr - edge pop_default_guard -> default_detour_push - edge default_detour_push -> default_detour_pop - edge default_detour_pop -> @right.value + attr (@assignment_expr.default_detour_pop) symbol_definition = "default", source_node = @assignment_expr, definiens_node = @assignment_expr + edge @assignment_expr.pop_default_guard -> default_detour_push + edge default_detour_push -> @assignment_expr.default_detour_pop + edge @assignment_expr.default_detour_pop -> @right.value } @@ -4625,11 +4695,19 @@ inherit .return_or_yield ;; These rules are all about declarations and terms that have the names ;; directly in them. -(class_declaration - name:(_)@name - body:(_)@_body)@class_decl { +( + (class_declaration + name:(_)@name + body:(class_body + member:(method_definition + name:(_)@_method_name)@constructor + ) + ) - attr (@name.pop) definiens_node = @class_decl + (#eq? @_method_name "constructor") +) { + + attr (@name.pop) definiens_node = @constructor } @@ -4660,7 +4738,7 @@ inherit .return_or_yield } -(function +(function_expression name:(_)@name parameters:(_)@_call_sig)@fun { @@ -4676,11 +4754,19 @@ inherit .return_or_yield } -(class - name:(_)@name - body:(_)@_body)@class { - - attr (@name.pop) definiens_node = @class +( + (class + name:(_)@name + body:(class_body + member:(method_definition + name:(_)@_method_name)@constructor + ) + ) + + (#eq? @_method_name "constructor") +) { + + attr (@name.pop) definiens_node = @constructor } @@ -4709,8 +4795,8 @@ inherit .return_or_yield property:(_)@left ) right: (_))@assignment_expr - (#not-eq @_object "module") - (#not-eq @left "exports") + (#not-eq? @_object "module") + (#not-eq? @left "exports") ) { node left_definiens_hook @@ -4730,8 +4816,8 @@ inherit .return_or_yield property:(_)@left ) right: (_))@assignment_expr - (#eq @_object "module") - (#eq @left "exports") + (#eq? @_object "module") + (#eq? @left "exports") ) { node left_definiens_hook @@ -4766,7 +4852,7 @@ inherit .return_or_yield (variable_declarator name:(identifier)@name value: [ - (function) + (function_expression) (generator_function) (arrow_function) ])) @@ -4774,7 +4860,7 @@ inherit .return_or_yield (variable_declarator name:(identifier)@name value: [ - (function) + (function_expression) (generator_function) (arrow_function) ])) @@ -4784,7 +4870,7 @@ inherit .return_or_yield ; (member_expression property:(_)@name) ; FIXME member expressions are references and have no .pop ] right: [ - (function) + (function_expression) (generator_function) (arrow_function) ]) @@ -4794,10 +4880,76 @@ inherit .return_or_yield } +( + (assignment_expression + left: [ + ( ; exports.foo = ... + (member_expression + object:(_)@_exports + property:(_)@_property) + (#eq? @_exports "exports") + ) + ( ; module.exports.foo = ... + (member_expression + object:(member_expression + object:(_)@_module + property:(_)@_exports) + property:(_)@_property) + (#eq? @_module "module") + (#eq? @_exports "exports") + ) + ] + right: [ + (function_expression) + (generator_function) + (arrow_function) + ])@assignment_expr + +) { + + attr (@assignment_expr.pop_name) syntax_type = "function" + attr (@assignment_expr.detour_pop) syntax_type = "function" + attr (@assignment_expr.default_detour_pop) syntax_type = "function" + +} + +( + (assignment_expression + left: (member_expression + object:(_)@_module + property:(_)@_exports) + right: [ + (function_expression) + (generator_function) + (arrow_function) + ])@assignment_expr + (#eq? @_module "module") + (#eq? @_exports "exports") +) { + + attr (@assignment_expr.pop_default_guard) syntax_type = "function" + attr (@assignment_expr.detour_pop) syntax_type = "function" + attr (@assignment_expr.default_detour_pop) syntax_type = "function" + +} + +(export_statement "default" + value:[ + (function_expression) + (generator_function) + (arrow_function) + ])@export_stmt { + + attr (@export_stmt.pop_guard_default) syntax_type = "function" + attr (@export_stmt.detour_pop) syntax_type = "function" + attr (@export_stmt.default_detour_pop) syntax_type = "function" + +} + (pair key: (_)@name value: [ - (function) + (function_expression) (generator_function) (arrow_function) ]) { @@ -4828,6 +4980,60 @@ inherit .return_or_yield } +( + (assignment_expression + left: [ + ( ; exports.foo = ... + (member_expression + object:(_)@_exports + property:(_)@_property) + (#eq? @_exports "exports") + ) + ( ; module.exports.foo = ... + (member_expression + object:(member_expression + object:(_)@_module + property:(_)@_exports) + property:(_)@_property) + (#eq? @_module "module") + (#eq? @_exports "exports") + ) + ] + right: (class))@assignment_expr + +) { + + attr (@assignment_expr.pop_name) syntax_type = "class" + attr (@assignment_expr.detour_pop) syntax_type = "class" + attr (@assignment_expr.default_detour_pop) syntax_type = "class" + +} + +( + (assignment_expression + left: (member_expression + object:(_)@_module + property:(_)@_exports) + right: (class))@assignment_expr + (#eq? @_module "module") + (#eq? @_exports "exports") +) { + + attr (@assignment_expr.pop_default_guard) syntax_type = "class" + attr (@assignment_expr.detour_pop) syntax_type = "class" + attr (@assignment_expr.default_detour_pop) syntax_type = "class" + +} + +(export_statement "default" + value:(class))@export_stmt { + + attr (@export_stmt.pop_guard_default) syntax_type = "class" + attr (@export_stmt.detour_pop) syntax_type = "class" + attr (@export_stmt.default_detour_pop) syntax_type = "class" + +} + (pair key: (_)@name value: (class)) { diff --git a/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js b/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js index e732ced2a..4cf3170d0 100644 --- a/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js +++ b/languages/tree-sitter-stack-graphs-javascript/test/base_syntax.js @@ -3,15 +3,37 @@ // foo export let x = 1; +export { + // x + A +}; import "foo"; debugger; var x; let x; +let x = { + get constructor() {} +}; function foo() { } +function foo(a) { } +function foo(undefined) { } function* foo() { } -class Foo { } +class Foo { + #x = null; + get [/**/ foo]() {} + static {} +} +@Foo class Bar { } +@Foo.Quux class Bar { } +@Foo() class Bar { } +@Foo.Quux() class Bar { } { } if (true) { } +if (true) { } else { } +if (/**/ true) /**/ { } else /**/ { } +if (true) return; +if (true) return; else return; +if (/**/ true) /**/ return; else /**/ return; switch (x) { } for (x; y; z) { } for (x in xs) { } @@ -22,6 +44,7 @@ with (x) { } break; continue; return; +return x; ; foo: x; @@ -45,8 +68,13 @@ undefined; []; [1, 2, 3]; function () { return; }; +function () { return function () { }; }; () => { }; +() => () => { }; function* () { yield 1; }; +function (/**/) { }; +function (x /**/) { }; +function (/**/ x) { }; foo(); foo(bar); foo.bar; @@ -59,6 +87,11 @@ x++; 1 + 1; -2; (x = 1); +(x[i] = 1); +([x] = 1); +({ x: y } = 1); +({/**/x: y } = 1); +({x:y.z} = 1); x += 1; (1, 2); 1 ? 2 : 3; @@ -67,4 +100,8 @@ class { }; <>doo {garply} - \ No newline at end of file + { } + {/**/x} +; +; +; \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/class_declaration_fields_and_methods_visible_in_other_methods.js b/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/class_declaration_fields_and_methods_visible_in_other_methods.js new file mode 100644 index 000000000..3bed1c415 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-javascript/test/computation_flow/class_declaration_fields_and_methods_visible_in_other_methods.js @@ -0,0 +1,23 @@ +class Foo { + bar = 1; + baz() { } + quux() { + this.bar; + // ^ defined: 2 + + this.baz(); + // ^ defined: 3 + } +} + +(class { + bar = 1; + baz() { } + quux() { + this.bar; + // ^ defined: 14 + + this.baz(); + // ^ defined: 15 + } +}); \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js b/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js index b397d747a..438cf9dd6 100644 --- a/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js +++ b/languages/tree-sitter-stack-graphs-javascript/test/expressions/member_expression.js @@ -15,4 +15,8 @@ let x = 1; // Flow around /**/ x; +// ^ defined: 1 + +// Optional chain +/**/ x?.foo // ^ defined: 1 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-python/.gitignore b/languages/tree-sitter-stack-graphs-python/.gitignore new file mode 100644 index 000000000..faf6459fb --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/.gitignore @@ -0,0 +1,3 @@ +*.html +/Cargo.lock +/target diff --git a/languages/tree-sitter-stack-graphs-python/CHANGELOG.md b/languages/tree-sitter-stack-graphs-python/CHANGELOG.md new file mode 100644 index 000000000..f21aecc9f --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/CHANGELOG.md @@ -0,0 +1,31 @@ +# Changelog for tree-sitter-stack-graphs-python + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## v0.3.0 -- 2024-12-12 + +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-python` dependency is updated to version 0.23.5. + +## v0.2.0 -- 2024-07-09 + +### Added + +- Added support for root paths. This fixes import problems when indexing using absolute directory paths. + +### Fixed + +- Fixed crash for lambdas with parameters. +- Fixed crash for nested functions definitions. + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.1.0 -- 2024-03-06 + +Initial release. diff --git a/languages/tree-sitter-stack-graphs-python/Cargo.toml b/languages/tree-sitter-stack-graphs-python/Cargo.toml new file mode 100644 index 000000000..afdfe6091 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "tree-sitter-stack-graphs-python" +version = "0.3.0" +description = "Stack graphs definition for Python using tree-sitter-python" +readme = "README.md" +keywords = ["tree-sitter", "stack-graphs", "python"] +authors = [ + "GitHub ", +] +license = "MIT OR Apache-2.0" +edition = "2018" + +[[bin]] +name = "tree-sitter-stack-graphs-python" +path = "rust/bin.rs" +required-features = ["cli"] + +[lib] +path = "rust/lib.rs" +test = false + +[[test]] +name = "test" +path = "rust/test.rs" +harness = false + +[features] +cli = ["anyhow", "clap", "tree-sitter-stack-graphs/cli"] + +[dependencies] +anyhow = { version = "1.0", optional = true } +clap = { version = "4", optional = true, features = ["derive"] } +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-python = "=0.23.5" + +[dev-dependencies] +anyhow = "1.0" +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } diff --git a/languages/tree-sitter-stack-graphs-python/README.md b/languages/tree-sitter-stack-graphs-python/README.md new file mode 100644 index 000000000..7ec9ded72 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/README.md @@ -0,0 +1,138 @@ +# tree-sitter-stack-graphs definition for Python + +This project defines tree-sitter-stack-graphs rules for Python using the [tree-sitter-python][] grammar. + +[tree-sitter-python]: https://crates.io/crates/tree-sitter-python + +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-python/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-python/CHANGELOG.md) + +## Using the API + +To use this library, add the following to your `Cargo.toml`: + +```toml +[dependencies] +tree-sitter-stack-graphs-python = "0.3" +``` + +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-python/*/) for more details on how to use this library. + +## Using the Command-line Program + +The command-line program for `tree-sitter-stack-graphs-python` lets you do stack graph based analysis and lookup from the command line. + +The CLI can be run as follows: + +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-python + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-python`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-python` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-python index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-python status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-python clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-python query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. + +## Development + +The project is written in Rust, and requires a recent version installed. Rust can be installed and updated using [rustup][]. + +[rustup]: https://rustup.rs/ + +The project is organized as follows: + +- The stack graph rules are defined in `src/stack-graphs.tsg`. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. +- Tests are put into the `test` directory. + +### Running Tests + +Run the tests as follows: + +```sh +cargo test +``` + +The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. + +Run the CLI from source as follows: + +```sh +cargo run --features cli -- ARGS +``` + +Sources are formatted using the standard Rust formatted, which is applied by running: + +```sh +cargo fmt +``` + +### Writing TSG + +The stack graph rules are written in [tree-sitter-graph][]. Checkout the [examples][], +which contain self-contained TSG rules for specific language features. A VSCode +[extension][] is available that provides syntax highlighting for TSG files. + +[tree-sitter-graph]: https://github.com/tree-sitter/tree-sitter-graph +[examples]: https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/ +[extension]: https://marketplace.visualstudio.com/items?itemName=tree-sitter.tree-sitter-graph + +Parse and test a single file by executing the following commands: + +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... +``` + +Generate a visualization to debug failing tests by passing the `-V` flag: + +```sh +cargo run --features cli -- test -V TESTFILES... +``` + +To generate the visualization regardless of test outcome, execute: + +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... +``` + +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-python/rust/bin.rs b/languages/tree-sitter-stack-graphs-python/rust/bin.rs new file mode 100644 index 000000000..dc9a6772c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/bin.rs @@ -0,0 +1,32 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use anyhow::anyhow; +use clap::Parser; +use tree_sitter_stack_graphs::cli::database::default_user_database_path_for_crate; +use tree_sitter_stack_graphs::cli::provided_languages::Subcommands; +use tree_sitter_stack_graphs::NoCancellation; + +fn main() -> anyhow::Result<()> { + let lc = match tree_sitter_stack_graphs_python::try_language_configuration(&NoCancellation) { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + let cli = Cli::parse(); + let default_db_path = default_user_database_path_for_crate(env!("CARGO_PKG_NAME"))?; + cli.subcommand.run(default_db_path, vec![lc]) +} + +#[derive(Parser)] +#[clap(about, version)] +pub struct Cli { + #[clap(subcommand)] + subcommand: Subcommands, +} diff --git a/languages/tree-sitter-stack-graphs-python/rust/lib.rs b/languages/tree-sitter-stack-graphs-python/rust/lib.rs new file mode 100644 index 000000000..332250511 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/lib.rs @@ -0,0 +1,45 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use tree_sitter_stack_graphs::loader::LanguageConfiguration; +use tree_sitter_stack_graphs::loader::LoadError; +use tree_sitter_stack_graphs::CancellationFlag; + +/// The stack graphs tsg source for this language. +pub const STACK_GRAPHS_TSG_PATH: &str = "src/stack-graphs.tsg"; +/// The stack graphs tsg source for this language. +pub const STACK_GRAPHS_TSG_SOURCE: &str = include_str!("../src/stack-graphs.tsg"); + +/// The stack graphs builtins configuration for this language. +pub const STACK_GRAPHS_BUILTINS_CONFIG: &str = include_str!("../src/builtins.cfg"); +/// The stack graphs builtins path for this language +pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.py"; +/// The stack graphs builtins source for this language. +pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.py"); + +pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration { + try_language_configuration(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) +} + +pub fn try_language_configuration( + cancellation_flag: &dyn CancellationFlag, +) -> Result { + LanguageConfiguration::from_sources( + tree_sitter_python::LANGUAGE.into(), + Some(String::from("source.py")), + None, + vec![String::from("py")], + STACK_GRAPHS_TSG_PATH.into(), + STACK_GRAPHS_TSG_SOURCE, + Some(( + STACK_GRAPHS_BUILTINS_PATH.into(), + STACK_GRAPHS_BUILTINS_SOURCE, + )), + Some(STACK_GRAPHS_BUILTINS_CONFIG), + cancellation_flag, + ) +} diff --git a/languages/tree-sitter-stack-graphs-python/rust/test.rs b/languages/tree-sitter-stack-graphs-python/rust/test.rs new file mode 100644 index 000000000..c81edf0d4 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/rust/test.rs @@ -0,0 +1,23 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2023, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use anyhow::anyhow; +use std::path::PathBuf; +use tree_sitter_stack_graphs::ci::Tester; +use tree_sitter_stack_graphs::NoCancellation; + +fn main() -> anyhow::Result<()> { + let lc = match tree_sitter_stack_graphs_python::try_language_configuration(&NoCancellation) { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + let test_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test"); + Tester::new(vec![lc], vec![test_path]).run() +} diff --git a/languages/tree-sitter-stack-graphs-python/src/builtins.cfg b/languages/tree-sitter-stack-graphs-python/src/builtins.cfg new file mode 100644 index 000000000..d685061be --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/builtins.cfg @@ -0,0 +1 @@ +[globals] diff --git a/languages/tree-sitter-stack-graphs-python/src/builtins.py b/languages/tree-sitter-stack-graphs-python/src/builtins.py new file mode 100644 index 000000000..e69de29bb diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm new file mode 100644 index 000000000..380af7d95 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.scm @@ -0,0 +1,779 @@ +;; -*- coding: utf-8 -*- +;; ------------------------------------------------------------------------------------------------ +;; Copyright © 2023, stack-graphs authors. +;; Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +;; Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +;; ------------------------------------------------------------------------------------------------ + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; LEGACY DEFINITION! INCLUDED FOR REFERENCE ONLY! ;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; Modules and Imports +;--------------------- + +(module) @mod +{ + var module_def = @mod.file_def + attr module_def "no_span" + var module_ref = @mod.file_ref + attr module_ref "no_span" + var parent_module_def = module_def + var parent_module_ref = module_ref + var grandparent_module_ref = module_ref + + scan filepath { + "([^/]+)/" + { + edge module_def -> module_def.dot + edge module_def.dot -> module_def.next_def + edge module_ref.next_ref -> module_ref.dot + edge module_ref.dot -> module_ref + + attr module_def "pop" = $1 + attr module_def.dot "pop" = "." + attr module_ref "push" = $1 + attr module_ref.dot "push" = "." + + set grandparent_module_ref = parent_module_ref + set parent_module_def = module_def + set parent_module_ref = module_ref + set module_ref = module_ref.next_ref + set module_def = module_def.next_def + attr module_def "no_span" + attr module_ref "no_span" + } + + "__init__\\.py$" + { + attr parent_module_def "definition" + } + + "([^/]+)$" + { + edge module_def -> module_def.dot + edge module_def.dot -> module_def.next_def + + attr module_def "definition", "pop" = (replace $1 "\\.py" "") + attr module_def.dot "pop" = "." + + set module_def = module_def.next_def + attr module_def "no_span" + attr module_ref "no_span" + } + } + + edge root -> @mod.file_def + edge @mod.file_ref -> root + edge module_def -> @mod.after_scope + + edge @mod.before_scope -> @mod.global_dot + edge @mod.global -> root + attr @mod.global "push" = "" + + edge @mod.global_dot -> @mod.global + attr @mod.global_dot "push" = "." + + var @mod::parent_module = parent_module_ref + var @mod::grandparent_module = grandparent_module_ref + var @mod::bottom = @mod.after_scope + var @mod::global = @mod.global + var @mod::global_dot = @mod.global_dot +} + +(import_statement + name: (dotted_name + . (identifier) @root_name)) @stmt +{ + edge @stmt.after_scope -> @root_name.def, "precedence" = 1 + edge @root_name.ref -> root + attr @root_name.ref "push", "reference" + attr @root_name.def "pop", "definition" +} + +(import_statement + name: (aliased_import + (dotted_name . (identifier) @root_name))) @stmt +{ + edge @stmt.after_scope -> @root_name.def + edge @root_name.ref -> root + attr @root_name.ref "push", "reference" +} + +(import_from_statement + module_name: (dotted_name + . (identifier) @prefix_root_name)) @stmt +{ + edge @prefix_root_name.ref -> root + attr @prefix_root_name.ref "push", "reference" +} + +(import_from_statement + name: (dotted_name + . (identifier) @import_root_name)) @stmt +{ + edge @stmt.after_scope -> @import_root_name.def, "precedence" = 1 + edge @import_root_name.ref -> @import_root_name.ref_dot + attr @import_root_name.def "pop", "definition" + attr @import_root_name.ref "push", "reference" + attr @import_root_name.ref_dot "push" = "." +} + +(import_from_statement + name: (aliased_import + (dotted_name + . (identifier) @import_root_name))) @stmt +{ + edge @import_root_name.ref -> @import_root_name.ref_dot + attr @import_root_name.ref "push", "reference" + attr @import_root_name.ref_dot "push" = "." +} + +(import_from_statement + module_name: [ + (dotted_name (identifier) @prefix_leaf_name .) + (relative_import (dotted_name (identifier) @prefix_leaf_name .)) + (relative_import (import_prefix) @prefix_leaf_name .) + ] + name: [ + (dotted_name + . (identifier) @import_root_name) + (aliased_import + (dotted_name + . (identifier) @import_root_name)) + ]) +{ + edge @import_root_name.ref_dot -> @prefix_leaf_name.ref +} + +[ + (import_from_statement + (aliased_import + name: (dotted_name (identifier) @name .) + alias: (identifier) @alias)) + (import_statement + (aliased_import + name: (dotted_name (identifier) @name .) + alias: (identifier) @alias)) +] @stmt +{ + edge @stmt.after_scope -> @alias + edge @alias -> @name.ref + attr @alias "pop", "definition" +} + +[ + (import_statement + name: (dotted_name + (identifier) @leaf_name .)) + (import_from_statement + name: (dotted_name + (identifier) @leaf_name .)) +] +{ + attr @leaf_name.def "pop", "definition" + attr @leaf_name.ref "push", "reference" + edge @leaf_name.def -> @leaf_name.ref +} + +(relative_import + (import_prefix) @prefix + (#eq? @prefix ".")) @import +{ + edge @prefix.ref -> @import::parent_module +} + +(relative_import + (import_prefix) @prefix + (#eq? @prefix "..")) @import +{ + edge @prefix.ref -> @import::grandparent_module +} + +(relative_import + (import_prefix) @prefix + (dotted_name + . (identifier) @name)) +{ + attr @name.ref "push", "reference" + attr @name.ref_dot "push" = "." + edge @name.ref -> @name.ref_dot + edge @name.ref_dot -> @prefix.ref +} + +[ + (import_from_statement + module_name: (relative_import + (dotted_name + (identifier) @parent_name + . + (identifier) @child_name))) + (import_from_statement + module_name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) +] +{ + attr @child_name.ref "push", "reference" + attr @child_name.ref_dot "push" = "." + edge @child_name.ref -> @child_name.ref_dot + edge @child_name.ref_dot -> @parent_name.ref +} + +(import_from_statement + module_name: (dotted_name + . (identifier) @root_name)) +{ + attr @root_name.ref "push", "reference" + edge @root_name.ref -> root +} + +(import_from_statement + module_name: (dotted_name + (identifier) @leaf_name .) + (wildcard_import) @star) @stmt +{ + edge @stmt.after_scope -> @star.ref_dot, "precedence" = 1 + edge @star.ref_dot -> @leaf_name.ref + attr @star.ref_dot "push" = "." +} + +[ + (import_statement + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) + (import_from_statement + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name)) + (import_from_statement + name: (aliased_import + name: (dotted_name + (identifier) @parent_name + . + (identifier) @child_name))) +] +{ + edge @child_name.ref -> @child_name.ref_dot + edge @child_name.ref_dot -> @parent_name.ref + edge @parent_name.def -> @parent_name.def_dot + edge @parent_name.def_dot -> @child_name.def + attr @child_name.def "pop", "definition" + attr @child_name.ref "push","reference" + attr @parent_name.def_dot "pop" = "." + attr @child_name.ref_dot "push" = "." +} + +;-------- +; Scopes +;-------- + +[ + (module (_) @last_stmt .) + (block (_) @last_stmt .) +] @block +{ + edge @block.after_scope -> @last_stmt.after_scope +} + +[ + (module (_) @stmt1 . (_) @stmt2) + (block (_) @stmt1 . (_) @stmt2) +] +{ + edge @stmt2.before_scope -> @stmt1.after_scope +} + +[ + (module (_) @stmt) + (block (_) @stmt) +] +{ + edge @stmt.after_scope -> @stmt.before_scope + let @stmt::local_scope = @stmt.before_scope +} + +[ + (block . (_) @stmt) + (module . (_) @stmt) +] @block +{ + edge @stmt.before_scope -> @block.before_scope +} + +(block (_) @stmt . ) @block +{ + edge @block.after_scope -> @stmt.after_scope +} + +(function_definition (block) @block) +{ + edge @block.before_scope -> @block::local_scope +} + +[ + (while_statement (block) @block) + (if_statement (block) @block) + (with_statement (block) @block) + (try_statement (block) @block) + (for_statement (block) @block) + (_ [ + (else_clause (block) @block) + (elif_clause (block) @block) + (except_clause (block) @block) + (finally_clause (block) @block) + ]) +] @stmt +{ + edge @block.before_scope -> @block::local_scope + edge @stmt.after_scope -> @block.after_scope +} + +(match_statement (case_clause) @block) @stmt +{ + let @block::local_scope = @block.before_scope + edge @block.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @block.after_scope +} + +[ + (for_statement) + (while_statement) +] @stmt +{ + edge @stmt.before_scope -> @stmt.after_scope +} + +;------------- +; Definitions +;------------- + +[ + (assignment + left: (_) @pattern + right: (_) @value) + (with_item + value: + (as_pattern + (_) @value + alias: (as_pattern_target (_) @pattern))) +] +{ + edge @pattern.input -> @value.output +} + +(function_definition + name: (identifier) @name + parameters: (parameters) @params + body: (block) @body) @func +{ + attr @name "definiens" = @func + edge @func.after_scope -> @name + edge @name -> @func.call + edge @func.call -> @func.return_value + edge @body.before_scope -> @params.after_scope + edge @body.before_scope -> @func.drop_scope + edge @func.drop_scope -> @func::bottom + attr @func.drop_scope "drop" + attr @name "pop", "definition" + attr @func.call "pop" = "()", "pop-scope" + attr @params.before_scope "jump-to" + attr @func.return_value "endpoint" + let @func::function_returns = @func.return_value + + ; Prevent functions defined inside of method bodies from being treated like methods + let @body::class_self_scope = nil + let @body::class_member_attr_scope = nil +} + +;; +;; BEGIN BIG GNARLY DISJUNCTION +;; +;; The following pair of rules is intended to capture the following behavior: +;; +;; If a function definition is used to define a method, by being inside a class +;; definition, then we make its syntax type `method`. Otherwise, we make it's +;; syntax type `function`. Unfortunately, because of the limitations on negation +;; and binding in tree sitter queries, we cannot negate `class_definition` or +;; similar things directly. Instead, we have to manually push the negation down +;; to form the finite disjunction it corresponds to. +;; + +[ + (class_definition (block (decorated_definition (function_definition name: (_)@name)))) + (class_definition (block (function_definition name: (_)@name))) +] +{ + attr @name "syntax_type" = "method" +} + +[ + (module (decorated_definition (function_definition name: (_)@name))) + (module (function_definition name: (_)@name)) + + (if_statement (block (decorated_definition (function_definition name: (_)@name)))) + (if_statement (block (function_definition name: (_)@name))) + + (elif_clause (block (decorated_definition (function_definition name: (_)@name)))) + (elif_clause (block (function_definition name: (_)@name))) + + (else_clause (block (decorated_definition (function_definition name: (_)@name)))) + (else_clause (block (function_definition name: (_)@name))) + + (case_clause (block (decorated_definition (function_definition name: (_)@name)))) + (case_clause (block (function_definition name: (_)@name))) + + (for_statement (block (decorated_definition (function_definition name: (_)@name)))) + (for_statement (block (function_definition name: (_)@name))) + + (while_statement (block (decorated_definition (function_definition name: (_)@name)))) + (while_statement (block (function_definition name: (_)@name))) + + (try_statement (block (decorated_definition (function_definition name: (_)@name)))) + (try_statement (block (function_definition name: (_)@name))) + + (except_clause (block (decorated_definition (function_definition name: (_)@name)))) + (except_clause (block (function_definition name: (_)@name))) + + (finally_clause (block (decorated_definition (function_definition name: (_)@name)))) + (finally_clause (block (function_definition name: (_)@name))) + + (with_statement (block (decorated_definition (function_definition name: (_)@name)))) + (with_statement (block (function_definition name: (_)@name))) + + (function_definition (block (decorated_definition (function_definition name: (_)@name)))) + (function_definition (block (function_definition name: (_)@name))) +] +{ + attr @name "syntax_type" = "function" +} + +;; +;; END BIG GNARLY DISJUNCTION +;; + +(function_definition + parameters: (parameters + . (identifier) @param) + body: (block) @body) +{ + edge @param.input -> @param::class_self_scope + edge @param::class_member_attr_scope -> @param.output + edge @param.output -> @body.after_scope + attr @param.output "push" +} + +(parameter/identifier) @param +{ + attr @param.input "definition", "pop" + attr @param.param_name "push" + edge @param.input -> @param.param_index + edge @param.input -> @param.param_name +} + +[ + (parameter/default_parameter + name: (identifier) @name + value: (_) @value) @param + (parameter/typed_default_parameter + name: (_) @name + value: (_) @value) @param +] +{ + attr @name "definition", "pop" + attr @param.param_name "push" = @name + edge @name -> @param.param_name + edge @name -> @param.param_index + edge @param.input -> @name + edge @name -> @value.output +} + +[ + (parameter/typed_parameter + . (_) @name) @param + (parameter/list_splat_pattern + (_) @name) @param + (parameter/dictionary_splat_pattern + (_) @name) @param +] +{ + attr @name "definition", "pop" + attr @param.param_name "push" = @name + edge @name -> @param.param_name + edge @name -> @param.param_index + edge @param.input -> @name +} + +[ + (pattern_list (_) @pattern) + (tuple_pattern (_) @pattern) +] @list +{ + let statement_scope = @list::local_scope + let @pattern::local_scope = @pattern.pattern_before_scope + edge statement_scope -> @pattern::local_scope, "precedence" = (+ 1 (child-index @pattern)) + + edge @pattern.pattern_index -> @list.input + edge @pattern.input -> @pattern.pattern_index + attr @pattern.pattern_index "push" = (child-index @pattern) +} + +(parameters + (_) @param) @params +{ + attr @param.param_index "push" = (child-index @param) + edge @param.param_index -> @params.before_scope + edge @params.after_scope -> @param.input + edge @param.param_name -> @params.before_scope +} + +(return_statement (_) @expr) @stmt +{ + edge @stmt::function_returns -> @expr.output +} + +(class_definition + name: (identifier) @name) @class +{ + attr @name "definiens" = @class + attr @name "syntax_type" = "class" + edge @class.parent_scope -> @class::class_parent_scope + edge @class.parent_scope -> @class::local_scope + edge @class.after_scope -> @name + edge @name -> @class.call + edge @name -> @class.dot + edge @class.dot -> @class.members + edge @class.call -> @class.call_drop + edge @class.call_drop -> @class.self_scope + edge @class.self_scope -> @class.super_scope + edge @class.self_scope -> @class.self_dot + edge @class.self_dot -> @class.members + edge @class.members -> @class.member_attrs + attr @class.call "pop" = "()", "pop-scope" + attr @class.call_drop "drop" + attr @class.dot "pop" = "." + attr @class.self_dot "pop" = "." + attr @name "pop", "definition" + attr @class.member_attrs "push" = "." + attr @class.self_scope "endpoint" + let @class::super_scope = @class.super_scope + let @class::class_parent_scope = @class.parent_scope + let @class::class_self_scope = @class.call_drop + let @class::class_member_attr_scope = @class.member_attrs +} + +(class_definition + body: (block + (_) @last_stmt .) @body) @class +{ + edge @class.members -> @last_stmt.after_scope +} + +(class_definition + superclasses: (argument_list + (_) @superclass)) @class +{ + edge @class.super_scope -> @superclass.output +} + +(decorated_definition + definition: (_) @def) @stmt +{ + edge @def.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @def.after_scope +} + +(case_clause + pattern: (_) @pattern + consequence: (_) @consequence) @clause +{ + edge @consequence.before_scope -> @pattern.new_bindings + edge @consequence.before_scope -> @clause.before_scope + edge @clause.after_scope -> @consequence.after_scope +} + +;------------- +; Expressions +;------------- + +(call + function: (_) @fn + arguments: (argument_list) @args) @call +{ + edge @call.output -> @call.output_args + edge @call.output_args -> @fn.output + attr @call.output_args "push" = "()", "push-scope" = @args +} + +(call + function: (attribute + object: (_) @receiver) + arguments: (argument_list + (expression) @arg) @args) +{ + edge @args -> @arg.arg_index + edge @receiver -> @receiver.arg_index + + attr @receiver.arg_index "pop" = "0" + edge @receiver.arg_index -> @receiver.output + + attr @arg.arg_index "pop" = (+ 1 (child-index @arg)) + edge @arg.arg_index -> @arg.output +} + +(call + arguments: (argument_list + (keyword_argument + name: (identifier) @name + value: (_) @val) @arg) @args) @call +{ + edge @args -> @arg.arg_name + attr @arg.arg_name "pop" = @name + edge @arg.arg_name -> @val.output +} + +(argument_list + (expression) @arg) @args +{ + edge @args -> @arg.arg_index + attr @arg.arg_index "pop" = (child-index @arg) + edge @arg.arg_index -> @arg.output +} + +( + (call + function: (identifier) @fn-name) @call + (#eq? @fn-name "super") +) +{ + edge @call.output -> @call::super_scope +} + +[ + (tuple (_) @element) + (expression_list (_) @element) +] @tuple +{ + edge @tuple.output -> @element.el_index + attr @element.el_index "pop" = (child-index @element) + edge @element.el_index -> @element.output + + edge @tuple.new_bindings -> @element.new_bindings +} + +(attribute + object: (_) @object + attribute: (identifier) @name) @expr +{ + edge @expr.output -> @name.output + edge @name.output -> @expr.output_dot + edge @expr.output_dot -> @object.output + edge @object.input -> @expr.input_dot + edge @expr.input_dot -> @name.input + edge @name.input -> @expr.input + attr @expr.output_dot "push" = "." + attr @expr.input_dot "pop" = "." + attr @name.input "pop" + attr @name.output "push" +} + +(pattern/attribute + attribute: (identifier) @name) +{ + attr @name.input "definition" +} + +(primary_expression/attribute + attribute: (identifier) @name) +{ + attr @name.output "reference" +} + +(primary_expression/identifier) @id +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input + attr @id.input "pop" + attr @id.output "push", "reference" + + attr @id.new_binding_pop "pop", "definition" + edge @id.new_bindings -> @id.new_binding_pop +} + +(pattern/identifier) @id +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input, "precedence" = 1 + attr @id.input "pop", "definition" + attr @id.output "push" + + attr @id.new_binding_pop "pop", "definition" + edge @id.new_bindings -> @id.new_binding_pop +} + +(as_pattern + (expression) @value + alias: (as_pattern_target (primary_expression/identifier) @id)) @as_pattern +{ + edge @id.output -> @id::local_scope + edge @id.output -> @id::class_parent_scope + edge @id::local_scope -> @id.input, "precedence" = 1 + attr @id.input "pop", "definition" + attr @id.output "push" + + edge @as_pattern.new_bindings -> @value.new_bindings + edge @as_pattern.new_bindings -> @id.new_bindings +} + +(list) @list +{ + edge @list.output -> @list.called + edge @list.called -> @list::global_dot + attr @list.called "push" = "list" +} + +(list (_) @el) @list +{ + edge @list.new_bindings -> @el.new_bindings +} + +(dictionary (pair) @pair) @dict +{ + edge @dict.new_bindings -> @pair.new_bindings +} + +(pair + value: (_) @value) @pair +{ + edge @pair.new_bindings -> @value.new_bindings +} + +(set (_) @el) @set +{ + edge @set.new_bindings -> @el.new_bindings +} + +(list_splat (_) @splatted) @splat +{ +attr @splat.new_bindings_pop "pop" = @splatted, "definition" +edge @splat.new_bindings -> @splat.new_bindings_pop +} + +(binary_operator + (_) @left + (_) @right) @binop +{ + edge @binop.new_bindings -> @left.new_bindings + edge @binop.new_bindings -> @right.new_bindings +} + +(case_pattern (_) @expr) @pat +{ + edge @pat.new_bindings -> @expr.new_bindings +} diff --git a/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg new file mode 100644 index 000000000..5fb407ae8 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/src/stack-graphs.tsg @@ -0,0 +1,1377 @@ +;; -*- coding: utf-8 -*- +;; ------------------------------------------------------------------------------------------------ +;; Copyright © 2023, stack-graphs authors. +;; Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +;; Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +;; ------------------------------------------------------------------------------------------------ + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Stack graphs definition for Python +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Global Variables +;; ^^^^^^^^^^^^^^^^ + +global FILE_PATH +global ROOT_PATH = "" +global ROOT_NODE +global JUMP_TO_SCOPE_NODE + +;; Attribute Shorthands +;; ^^^^^^^^^^^^^^^^^^^^ + +attribute node_definition = node => type = "pop_symbol", node_symbol = node, is_definition +attribute node_reference = node => type = "push_symbol", node_symbol = node, is_reference +attribute pop_node = node => type = "pop_symbol", node_symbol = node +attribute pop_scoped_node = node => type = "pop_scoped_symbol", node_symbol = node +attribute pop_scoped_symbol = symbol => type = "pop_scoped_symbol", symbol = symbol +attribute pop_symbol = symbol => type = "pop_symbol", symbol = symbol +attribute push_node = node => type = "push_symbol", node_symbol = node +attribute push_scoped_node = node => type = "push_scoped_symbol", node_symbol = node +attribute push_scoped_symbol = symbol => type = "push_scoped_symbol", symbol = symbol +attribute push_symbol = symbol => type = "push_symbol", symbol = symbol +attribute scoped_node_definition = node => type = "pop_scoped_symbol", node_symbol = node, is_definition +attribute scoped_node_reference = node => type = "push_scoped_symbol", node_symbol = node, is_reference +attribute symbol_definition = symbol => type = "pop_symbol", symbol = symbol, is_definition +attribute symbol_reference = symbol => type = "push_symbol", symbol = symbol, is_reference + +attribute node_symbol = node => symbol = (source-text node), source_node = node + +;; Nodes +;; ^^^^^ + +(module) @node { + node @node.after_scope + node @node.before_scope +} + +[ + ; _statement + ; _simple_statement + (future_import_statement) + (import_statement) + (import_from_statement) + (print_statement) + (assert_statement) + (expression_statement) + (return_statement) + (delete_statement) + (raise_statement) + (pass_statement) + (break_statement) + (continue_statement) + (global_statement) + (nonlocal_statement) + (exec_statement) + (type_alias_statement) + ; _compound_statement + (if_statement) + (for_statement) + (while_statement) + (try_statement) + (with_statement) + (function_definition) + (class_definition) + (decorated_definition) + (match_statement) + ; block + (block) + ; statement clauses + (if_clause) + (elif_clause) + (else_clause) + (except_group_clause) + (except_clause) + (finally_clause) + (with_clause) + (case_clause) +] @node { + node @node.after_scope + node @node.before_scope +} + +[ + (parameters) + (lambda_parameters) +] @node { + node @node.after_scope + node @node.before_scope +} + +[ + (identifier) +] @node { + node @node.def + node @node.def_dot + node @node.ref + node @node.ref_dot +} + +[ + (dotted_name) + (aliased_import) + (relative_import) + (wildcard_import) + (import_prefix) +] @node { + node @node.after_scope + node @node.before_scope + node @node.def + node @node.ref +} + +[ + ; expressions + (comparison_operator) + (not_operator) + (boolean_operator) + (lambda) + ;(primary_expression) ; unfolded below + (conditional_expression) + (named_expression) + (as_pattern) + ; primary_expression + (await) + (binary_operator) + (identifier) + ;(keyword_identifier) ; invalid query pattern? + (string) + (concatenated_string) + (integer) + (float) + (true) + (false) + (none) + (unary_operator) + (attribute) + (subscript) + (call) + (list) + (list_comprehension) + (dictionary) + (dictionary_comprehension) + (set) + (set_comprehension) + (tuple) + (pair) + (parenthesized_expression) + (generator_expression) + (ellipsis) + (list_splat) + + ; expression list + (expression_list) + + ; pattern + (pattern/identifier) + ;(keyword_identifier) ; invalid query pattern? + ;(subscript) + ;(attribute) + (list_splat_pattern) + (tuple_pattern) + (list_pattern) + ; _simple_patterns + (class_pattern) + (splat_pattern) + (union_pattern) + ;(list_pattern) ; already in pattern + ;(tuple_pattern) ; already in pattern + (dict_pattern) + ;(string) ; already in primary_expression + ;(concatenated_string) ; already in primary_expression + ;(true) ; already in primary_expression + ;(false) ; already in primary_expression + ;(none) ; already in primary_expression + ;(integer) ; already in primary_expression + ;(float) ; already in primary_expression + (complex_pattern) + (dotted_name) + ; _as_attern + (as_pattern) + ; keyword pattern + (keyword_pattern) + ; case pattern + (case_pattern) + ; with item + (with_item) + + ; pattern list + (pattern_list) + + ; parameter + ;(identifier) ; already in expressions + (typed_parameter) + (default_parameter) + (typed_default_parameter) + ;(list_splat_pattern) ; already in patterns + ;(tuple_pattern) ; already in patterns + (keyword_separator) + (positional_separator) + (dictionary_splat_pattern) + + ; parameters + (parameters) +] @node { + node @node.input + node @node.new_bindings + node @node.output +} + +(comment) @node { + node @node.after_scope + node @node.before_scope + node @node.def + node @node.def_dot + node @node.input + node @node.new_bindings + node @node.output + node @node.ref + node @node.ref_dot +} + +;; Inherited Variables +;; ^^^^^^^^^^^^^^^^^^^ + +inherit .bottom +inherit .class_member_attr_scope +inherit .class_parent_scope +inherit .class_self_scope +inherit .class_super_scope +inherit .function_returns +inherit .global +inherit .global_dot +inherit .grandparent_module +inherit .local_scope +inherit .parent_module + +;; +;; # # +;; ## ## #### ##### # # # ###### #### +;; # # # # # # # # # # # # # +;; # # # # # # # # # # ##### #### +;; # # # # # # # # # # # +;; # # # # # # # # # # # # +;; # # #### ##### #### ###### ###### #### +;; +;; Modules + +(module) @mod +{ + node mod_file_def + node mod_file_ref + + var module_def = mod_file_def + + node parent_module_def_node + var parent_module_def = parent_module_def_node + + var module_ref = mod_file_ref + + node parent_module_ref_node + var parent_module_ref = parent_module_ref_node + + node grandparent_module_ref_node + var grandparent_module_ref = grandparent_module_ref_node + + ; get the file path relative to the root path + let rel_path = (replace FILE_PATH ROOT_PATH "") + scan rel_path { + "([^/]+)/" + { + node def_dot + attr (def_dot) pop_symbol = "." + node next_def + ; + edge module_def -> def_dot + edge def_dot -> next_def + ; + attr (module_def) pop_symbol = $1 + ; + set parent_module_def = module_def + set module_def = next_def + + node ref_dot + attr (ref_dot) push_symbol = "." + node next_ref + ; + edge next_ref -> ref_dot + edge ref_dot -> module_ref + ; + attr (module_ref) push_symbol = $1 + ; + set grandparent_module_ref = parent_module_ref + set parent_module_ref = module_ref + set module_ref = next_ref + } + + "__init__\.py$" + { + attr (parent_module_def) is_definition, source_node = @mod, empty_source_span + } + + "([^/]+)\.py$" + { + node def_dot + attr (def_dot) pop_symbol = "." + node next_def + ; + edge module_def -> def_dot + edge def_dot -> next_def + ; + attr (module_def) pop_symbol = $1, is_definition, source_node = @mod, empty_source_span + ; + set module_def = next_def + } + } + + edge ROOT_NODE -> mod_file_def + edge mod_file_ref -> ROOT_NODE + edge module_def -> @mod.after_scope + + node global + node global_dot + + edge @mod.before_scope -> global_dot + edge global -> ROOT_NODE + attr (global) push_symbol = "" + + edge global_dot -> global + attr (global_dot) push_symbol = "." + + let @mod.parent_module = parent_module_ref + let @mod.grandparent_module = grandparent_module_ref + let @mod.bottom = @mod.after_scope + let @mod.global = global + let @mod.global_dot = global_dot + + ;; add a dummy nodes for inherited variables + node @mod.class_member_attr_scope + node @mod.class_parent_scope + node @mod.class_self_scope + node @mod.class_super_scope + node @mod.function_returns + node @mod.local_scope +} + +;; +;; ### +;; # # # ##### #### ##### ##### #### +;; # ## ## # # # # # # # # +;; # # ## # # # # # # # # #### +;; # # # ##### # # ##### # # +;; # # # # # # # # # # # +;; ### # # # #### # # # #### +;; +;; Imports + +;; Import References +;; ^^^^^^^^^^^^^^^^^ + +;;;; Dotted Names +;; +;; (dotted_name).ref node to connect to to use the reference +;; (dotted_name).before_scope node to connect from to ensure the reference resolves + +;; all names are references +[ + (import_statement name: (dotted_name (identifier) @name)) + (future_import_statement name: (dotted_name (identifier) @name)) + (import_from_statement name: (dotted_name (identifier) @name)) + (import_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @name))) + (import_from_statement module_name: (dotted_name (identifier) @name)) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @name))) +] { + attr (@name.ref) node_reference = @name +} + +;; references are chained +[ + (import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (future_import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @left . (identifier) @right))) + (import_from_statement module_name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @left . (identifier) @right))) +] { + node push_dot + attr (push_dot) push_symbol = "." + + edge @right.ref -> push_dot + edge push_dot -> @left.ref +} + +;; lookup first reference +[ + (import_statement name: (dotted_name . (identifier) @first) @dotted) + (future_import_statement name: (dotted_name . (identifier) @first) @dotted) + (import_from_statement name: (dotted_name . (identifier) @first) @dotted) + (import_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (future_import_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (import_from_statement name: (aliased_import name: (dotted_name . (identifier) @first) @dotted)) + (import_from_statement module_name: (dotted_name . (identifier) @first) @dotted) + (import_from_statement module_name: (relative_import (dotted_name . (identifier) @first) @dotted)) +] { + edge @first.ref -> @dotted.before_scope +} + +;; expose last reference +[ + (import_statement name: (dotted_name (identifier) @last .) @dotted) + (future_import_statement name: (dotted_name (identifier) @last .) @dotted) + (import_from_statement name: (dotted_name (identifier) @last .) @dotted) + (import_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (future_import_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (import_from_statement name: (aliased_import name: (dotted_name (identifier) @last .) @dotted)) + (import_from_statement module_name: (dotted_name (identifier) @last .) @dotted) + (import_from_statement module_name: (relative_import (dotted_name (identifier) @last .) @dotted)) +] { + edge @dotted.ref -> @last.ref +} + +;;;; Aliased Import +;; +;; An aliased import behaves like its wrapped dotted_name as a reference +;; +;; (aliased_import).ref node to connect to, to use the reference +;; (aliased_import).before_scope node to connect from to ensure the reference resolves + +(aliased_import name: (dotted_name) @dotted) @aliased { + edge @aliased.ref -> @dotted.ref + edge @dotted.before_scope -> @aliased.before_scope +} + +;;;; Relative Import +;; +;; A relative import behaves like its wrapped dotted_name as a reference +;; +;; (relative_import).ref node to connect to, to use the reference +;; (relative_import).before_scope node to connect from to ensure the reference resolves + +(relative_import (import_prefix) . (dotted_name) @dotted) @relative { + edge @relative.ref -> @dotted.ref + + node push_dot + attr (push_dot) push_symbol = "." + + edge @dotted.before_scope -> push_dot + edge push_dot -> @relative.before_scope +} + +(relative_import (import_prefix) .) @relative { + edge @relative.ref -> @relative.before_scope +} + +;;;; Wildcard Import +;; +;; A wildcard import simply passes through +;; +;; (wildcard_import).ref node to connect to, to use the reference +;; (wildcard_import).before_scope node to connect from to ensure the reference resolves + +(wildcard_import) @wildcard { + edge @wildcard.ref -> @wildcard.before_scope +} + +;;;; Import from +;; +;; The imported references are resolved in the from module + +[ + (import_from_statement module_name: (_) @left name: (_) @right) + (import_from_statement module_name: (_) @left (wildcard_import) @right) +] { + node push_dot + attr (push_dot) push_symbol = "." + + edge @right.before_scope -> push_dot + edge push_dot -> @left.ref +} + +;;;; Non-relative Imports +;; +;; Non-relative imports are resolved in the root scope + +[ + (import_statement name: (_) @name) + (import_from_statement module_name: (dotted_name) @name) +] { + edge @name.before_scope -> ROOT_NODE +} + +;;;; Relative Imports +;; +;; Relative imports are resolved in scopes related to the current module + +;; . imports resolve in parent module scope +(import_from_statement module_name: (relative_import (import_prefix) @prefix (#eq? @prefix ".")) @relative) { + edge @relative.before_scope -> @prefix.parent_module +} + +;; .. imports resolve in grandparent module scope +(import_from_statement module_name: (relative_import (import_prefix) @prefix (#eq? @prefix "..")) @relative) { + edge @relative.before_scope -> @prefix.grandparent_module +} + +;;;; Future Imports +;; +;; We don't know the future, so we cannot connect references of future imports anywhere. Maybe one day? + +;; Import Definitions +;; ^^^^^^^^^^^^^^^^^^ + +;;;; Dotted Names & Aliased Imports +;; +;; (dotted_name).after_scope node to connect to, to expose the definition +;; (dotted_name).def node to connect from, to give definition content + +;; unaliased names and aliases are definitions +[ + (import_statement name: (dotted_name (identifier) @name)) + (future_import_statement name: (dotted_name (identifier) @name)) + (import_from_statement name: (dotted_name (identifier) @name)) + (import_statement name: (aliased_import alias: (identifier) @name)) + (future_import_statement name: (aliased_import alias: (identifier) @name)) + (import_from_statement name: (aliased_import alias: (identifier) @name)) +] { + attr (@name.def) node_definition = @name +} + +;; definitions are chained +[ + (import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (future_import_statement name: (dotted_name (identifier) @left . (identifier) @right)) + (import_from_statement name: (dotted_name (identifier) @left . (identifier) @right)) +] { + node pop_dot + attr (pop_dot) pop_symbol = "." + + edge @left.def -> pop_dot + edge pop_dot -> @right.def +} + +;; connect last definition +[ + (import_statement name: (dotted_name (identifier) @last .) @outer) + (future_import_statement name: (dotted_name (identifier) @last .) @outer) + (import_from_statement name: (dotted_name (identifier) @last .) @outer) + (import_statement name: (aliased_import alias: (identifier) @last ) @outer) + (future_import_statement name: (aliased_import alias: (identifier) @last ) @outer) + (import_from_statement name: (aliased_import alias: (identifier) @last ) @outer) +] { + edge @last.def -> @outer.def +} + +;; expose first definition +[ + (import_statement name: (dotted_name . (identifier) @first) @outer) + (future_import_statement name: (dotted_name . (identifier) @first) @outer) + (import_from_statement name: (dotted_name . (identifier) @first) @outer) + (import_statement name: (aliased_import alias: (identifier) @first ) @outer) + (future_import_statement name: (aliased_import alias: (identifier) @first ) @outer) + (import_from_statement name: (aliased_import alias: (identifier) @first ) @outer) +] { + edge @outer.after_scope -> @first.def +} + +;;;; Wildcard Import +;; +;; Wildcard imports simply pass through + +(wildcard_import) @wildcard { + edge @wildcard.after_scope -> @wildcard.def +} + +;;;; Import Definitions -> References +;; +;; The definitions introduced by imports are connected to the corresponding references + +[ + (import_statement name: (_) @name) + (future_import_statement name: (_) @name) + (import_from_statement name: (_) @name) + (import_from_statement (wildcard_import) @name) +] { + edge @name.def -> @name.ref +} + +;;;; Imports +;; +;; The definitions introduced by imports are visible after the import statement + +[ + (import_statement name: (_) @name) + (future_import_statement name: (_) @name) + (import_from_statement name: (_) @name) + (import_from_statement (wildcard_import) @name) +] @stmt { + edge @stmt.after_scope -> @name.after_scope + attr (@stmt.after_scope -> @name.after_scope) precedence = 1 +} + +;; +;; ###### +;; # # # #### #### # # #### +;; # # # # # # # # # # +;; ###### # # # # #### #### +;; # # # # # # # # # +;; # # # # # # # # # # # +;; ###### ###### #### #### # # #### +;; +;; Blocks + +[ + (module (_) @last_stmt .) + (block (_) @last_stmt .) +] @block +{ + edge @block.after_scope -> @last_stmt.after_scope +} + +[ + (module (_) @stmt1 . (_) @stmt2) + (block (_) @stmt1 . (_) @stmt2) +] +{ + edge @stmt2.before_scope -> @stmt1.after_scope +} + +[ + (module (_) @stmt) + (block (_) @stmt) +] +{ + edge @stmt.after_scope -> @stmt.before_scope + let @stmt.local_scope = @stmt.before_scope +} + +[ + (block . (_) @stmt) + (module . (_) @stmt) +] @block +{ + edge @stmt.before_scope -> @block.before_scope +} + +(function_definition (block) @block) +{ + edge @block.before_scope -> @block.local_scope +} + +[ + (while_statement (block) @block) + (if_statement (block) @block) + (with_statement (block) @block) + (try_statement (block) @block) + (for_statement (block) @block) + (_ [ + (else_clause (block) @block) + (elif_clause (block) @block) + (except_clause (block) @block) + (finally_clause (block) @block) + ]) +] @stmt +{ + edge @block.before_scope -> @block.local_scope + edge @stmt.after_scope -> @block.after_scope +} + +(match_statement body: (_) @block) @stmt +{ + let @block.local_scope = @block.before_scope + edge @block.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @block.after_scope +} + +[ + (for_statement) + (while_statement) +] @stmt +{ + edge @stmt.before_scope -> @stmt.after_scope +} + +;; +;; ##### +;; # # ##### ## ##### ###### # # ###### # # ##### #### +;; # # # # # # ## ## # ## # # # +;; ##### # # # # ##### # ## # ##### # # # # #### +;; # # ###### # # # # # # # # # # +;; # # # # # # # # # # # ## # # # +;; ##### # # # # ###### # # ###### # # # #### +;; +;; Statements + +;;;; Simple Statements + +(print_statement) {} + +(assert_statement) {} + +(expression_statement) {} + +(return_statement (_) @expr) @stmt +{ + edge @stmt.function_returns -> @expr.output +} + +(delete_statement) {} + +(raise_statement) {} + +(pass_statement) {} + +(break_statement) {} + +(continue_statement) {} + +(global_statement) {} + +(nonlocal_statement) {} + +(exec_statement) {} + +(type_alias_statement) {} + +;;;; Compound Statements + +(if_statement) {} + +(if_clause) {} + +(elif_clause) {} + +(else_clause) {} + +(for_statement) {} + +(while_statement) {} + +(try_statement) {} + +(except_group_clause) {} + +(except_clause) {} + +(finally_clause) {} + +(with_statement) {} + +(with_clause) {} + +[ + (function_definition + parameters: (_) @params + body: (_) @body + ) @func + (lambda + parameters: (_) @params + body: (_) @body + )@func +] { + node @func.call + node return_value + node drop_scope + + edge @func.call -> return_value + edge @body.before_scope -> @params.after_scope + edge @body.before_scope -> drop_scope + edge drop_scope -> @func.bottom + attr (drop_scope) type = "drop_scopes" + attr (@func.call) pop_scoped_symbol = "()" + edge @params.before_scope -> JUMP_TO_SCOPE_NODE + attr (return_value) is_exported + let @func.function_returns = return_value +} + +(function_definition + name: (identifier) @name + body: (_) @body +) @func { + attr (@name.def) node_definition = @name + attr (@name.def) definiens_node = @func + edge @func.after_scope -> @name.def + edge @name.def -> @func.call + + ; Prevent functions defined inside of method bodies from being treated like methods + let @body.class_self_scope = #null + let @body.class_member_attr_scope = #null +} + +; method definition +(class_definition + body: (block + (function_definition + parameters: + (parameters . (identifier) @first_param) + body: (block) @method_body + ) + ) +) +{ + edge @first_param.def -> @first_param.class_self_scope + edge @first_param.class_member_attr_scope -> @first_param.output + edge @first_param.output -> @method_body.after_scope + attr (@first_param.output) push_node = @first_param +} + +[ + (parameters (_) @param) @params + (lambda_parameters (_) @param) @params +] +{ + node @param.param_index + node @param.param_name + + attr (@param.param_index) push_symbol = (named-child-index @param) + edge @param.param_index -> @params.before_scope + edge @params.after_scope -> @param.input + edge @param.param_name -> @params.before_scope +} + + +(parameter/identifier) @param @name +{ + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @param + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def +} + +[ + (parameter/default_parameter + name: (_) @name + value: (_) @value) @param + (parameter/typed_default_parameter + name: (_) @name + value: (_) @value) @param +] { + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @name + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def + edge @name.def -> @value.output +} + +[ + (parameter/typed_parameter + . (_) @name) @param + (parameter/list_splat_pattern + (_) @name) @param + (parameter/dictionary_splat_pattern + (_) @name) @param +] { + attr (@name.def) node_definition = @name + attr (@param.param_name) push_node = @name + edge @name.def -> @param.param_name + edge @name.def -> @param.param_index + edge @param.input -> @name.def +} + +(class_definition) @class { + node @class.call_drop + node @class.member_attrs + node @class.members + node @class.super_scope +} + +(class_definition + name: (identifier) @name) { + attr (@name.def) node_definition = @name +} + +(class_definition + name: (identifier) @name) @class +{ + node call + node self_dot + node self_scope + node members_dot + + attr (@name.def) definiens_node = @class + attr (@name.def) syntax_type = "class" + edge @class.after_scope -> @name.def + edge @name.def -> call + edge @name.def -> members_dot + edge members_dot -> @class.members + edge call -> @class.call_drop + edge @class.call_drop -> self_scope + edge self_scope -> @class.super_scope + edge self_scope -> self_dot + edge self_dot -> @class.members + edge @class.members -> @class.member_attrs + attr (call) pop_scoped_symbol = "()" + attr (@class.call_drop) type = "drop_scopes" + attr (members_dot) pop_symbol = "." + attr (self_dot) pop_symbol = "." + attr (@class.member_attrs) push_symbol = "." + attr (self_scope) is_exported +} + +(class_definition + body: (_) @body) @class +{ + let @body.class_member_attr_scope = @class.member_attrs + node @body.class_parent_scope + edge @body.class_parent_scope -> @class.class_parent_scope + edge @body.class_parent_scope -> @class.local_scope + let @body.class_self_scope = @class.call_drop + let @body.class_super_scope = @class.super_scope +} + +(class_definition + body: (block + (_) @last_stmt .)) @class +{ + edge @class.members -> @last_stmt.after_scope +} + +(class_definition + superclasses: (argument_list + (_) @superclass)) @class +{ + edge @class.super_scope -> @superclass.output +} + +(decorated_definition + definition: (_) @def) @stmt +{ + edge @def.before_scope -> @stmt.before_scope + edge @stmt.after_scope -> @def.after_scope +} + +(match_statement) {} + +(case_clause + (case_pattern) @pattern + consequence: (_) @consequence) +{ + edge @consequence.before_scope -> @pattern.new_bindings +} + +(case_clause + consequence: (_) @consequence) @clause +{ + edge @consequence.before_scope -> @clause.before_scope + edge @clause.after_scope -> @consequence.after_scope +} + +;; +;; ####### +;; # # # ##### ##### ###### #### #### # #### # # #### +;; # # # # # # # # # # # # # ## # # +;; ##### ## # # # # ##### #### #### # # # # # # #### +;; # ## ##### ##### # # # # # # # # # # +;; # # # # # # # # # # # # # # # ## # # +;; ####### # # # # # ###### #### #### # #### # # #### +;; +;; Expressions + +(lambda + body: (_) @body +)@lam { + ;; TODO Unify .before_scope, .local_scope, and .input to simplify + ;; uniform treatment of lambdas and function definitions. + node @body.before_scope + let @body.local_scope = @body.before_scope + edge @body.input -> @body.before_scope + + edge @lam.output -> @lam.call +} + +(conditional_expression) {} + +(named_expression) {} + +(as_pattern) {} + +(await) {} + +(binary_operator + (_) @left + (_) @right) @binop +{ + edge @binop.new_bindings -> @left.new_bindings + edge @binop.new_bindings -> @right.new_bindings +} + +(primary_expression/identifier) @name { + attr (@name.ref) node_reference = @name +} + +(primary_expression/identifier) @name +{ + edge @name.output -> @name.local_scope + edge @name.output -> @name.class_parent_scope + edge @name.local_scope -> @name.input + attr (@name.input) pop_node = @name + attr (@name.output) node_reference = @name + + edge @name.new_bindings -> @name.def +} + +(string) {} + +(concatenated_string) {} + +(integer) {} + +(float) {} + +(true) {} + +(false) {} + +(none) {} + +(unary_operator) {} + +(attribute + object: (_) @object + attribute: (identifier) @name) @expr +{ + node input_dot + node output_dot + + edge @expr.output -> @name.output + edge @name.output -> output_dot + edge output_dot -> @object.output + edge @object.input -> input_dot + edge input_dot -> @name.input + edge @name.input -> @expr.input + attr (output_dot) push_symbol = "." + attr (input_dot) pop_symbol = "." + attr (@name.input) pop_node = @name + attr (@name.output) push_node = @name +} + +(primary_expression/attribute + attribute: (identifier) @name) +{ + attr (@name.output) is_reference +} + +(subscript) {} + +(call + function: (_) @fn + arguments: (argument_list) @args) @call +{ + node output_args + + edge @call.output -> output_args + edge output_args -> @fn.output + attr (output_args) push_scoped_symbol = "()", scope = @args.args +} + +(call + function: (attribute + object: (_) @receiver) + arguments: (argument_list + (expression) @arg) @args) +{ + node receiver_arg_index + attr (receiver_arg_index) pop_symbol = "0" + edge @args.args -> receiver_arg_index + edge receiver_arg_index -> @receiver.output + + ;; FIXME the arguments will also exist with their unshifted indices because of the general + ;; rule below! + node arg_index + attr (arg_index) pop_symbol = (plus 1 (named-child-index @arg)) + edge arg_index -> @arg.output +} + +(call + arguments: (argument_list + (keyword_argument + name: (identifier) @name + value: (_) @val)) @args) +{ + node arg_name + edge @args.args -> arg_name + attr (arg_name) pop_node = @name + edge arg_name -> @val.output +} + +(argument_list) @args { + node @args.args + attr (@args.args) is_exported +} + +(argument_list + (expression) @arg) @args +{ + node arg_index + edge @args.args -> arg_index + attr (arg_index) pop_symbol = (named-child-index @arg) + edge arg_index -> @arg.output +} + +( + (call + function: (identifier) @_fn_name) @call + (#eq? @_fn_name "super") +) +{ + edge @call.output -> @call.class_super_scope +} + +(list) @list +{ + node called + + edge @list.output -> called + edge called -> @list.global_dot + attr (called) push_symbol = "list" +} + +(list (_) @el) @list +{ + edge @list.new_bindings -> @el.new_bindings +} + +(list_comprehension) {} + +(dictionary (pair) @pair) @dict +{ + edge @dict.new_bindings -> @pair.new_bindings +} + +(pair + value: (_) @value) @pair +{ + edge @pair.new_bindings -> @value.new_bindings +} + +(dictionary_comprehension) {} + +(set (_) @el) @set +{ + edge @set.new_bindings -> @el.new_bindings +} + +(set_comprehension) {} + +[ + (tuple (_) @element) + (expression_list (_) @element) +] @tuple +{ + node el_index + + edge @tuple.output -> el_index + attr (el_index) pop_symbol = (named-child-index @element) + edge el_index -> @element.output + + edge @tuple.new_bindings -> @element.new_bindings +} + +(parenthesized_expression) {} + +(generator_expression) {} + +(ellipsis) {} + +(list_splat (_) @splatted) @splat +{ + edge @splat.new_bindings -> @splatted.new_bindings +} + +;; +;; ###### +;; # # ## ##### ##### ###### ##### # # #### +;; # # # # # # # # # ## # # +;; ###### # # # # ##### # # # # # #### +;; # ###### # # # ##### # # # # +;; # # # # # # # # # ## # # +;; # # # # # ###### # # # # #### +;; +;; Patterns + + +[ + (pattern/identifier) @name @pattern + ; identifiers in patterns + (as_pattern (identifier) @name .) @pattern + (keyword_pattern (identifier) @name) @pattern + (splat_pattern (identifier) @name) @pattern + ; every context where _simple_pattern is used, because matching + ; pattern/dotted_name does not work + (case_pattern (dotted_name . (_) @name .) @pattern) + (keyword_pattern (dotted_name . (_) @name .) @pattern) + (union_pattern (dotted_name . (_) @name .) @pattern) +] { + attr (@name.def) node_definition = @name + edge @pattern.output -> @pattern.local_scope + edge @pattern.output -> @pattern.class_parent_scope + edge @pattern.local_scope -> @pattern.input + attr (@pattern.local_scope -> @pattern.input) precedence = 1 + attr (@pattern.input) node_definition = @name + attr (@pattern.output) push_node = @name + + edge @pattern.new_bindings -> @name.def +} + +(pattern/subscript) {} + +(pattern/attribute + attribute: (identifier) @name) +{ + attr (@name.input) is_definition +} + + +(list_splat_pattern (_) @pattern) @list { + edge @list.new_bindings -> @pattern.new_bindings +} + +[ + (pattern_list (_) @pattern) + (tuple_pattern (_) @pattern) + (list_pattern (_) @pattern) +] @list +{ + node pattern_index + + let statement_scope = @list.local_scope + node @pattern.local_scope + edge statement_scope -> @pattern.local_scope + attr (statement_scope -> @pattern.local_scope) precedence = (plus 1 (named-child-index @pattern)) + + edge pattern_index -> @list.input + edge @pattern.input -> pattern_index + attr (pattern_index) push_symbol = (named-child-index @pattern) +} + +(class_pattern (case_pattern) @pattern) @class { + edge @class.new_bindings -> @pattern.new_bindings +} + +(splat_pattern (_) @pattern) @splat { + edge @splat.new_bindings -> @pattern.new_bindings +} + +(union_pattern (_) @pattern) @union { + edge @union.new_bindings -> @pattern.new_bindings +} + +(dict_pattern (_) @pattern) @dict { + edge @dict.new_bindings -> @pattern.new_bindings +} + +(complex_pattern) {} + +(as_pattern + (expression) @value + alias: (as_pattern_target (identifier) @name)) @as_pattern +{ + attr (@name.local_scope -> @name.input) precedence = 1 + attr (@name.input) is_definition + + edge @as_pattern.new_bindings -> @value.new_bindings + edge @as_pattern.new_bindings -> @name.new_bindings +} + +(keyword_pattern) {} + +(case_pattern (_) @pattern) @case +{ + edge @case.new_bindings -> @pattern.new_bindings +} + +[ + (assignment + left: (_) @pattern + right: (_) @value) + (with_item + value: + (as_pattern + (_) @value + alias: (as_pattern_target (_) @pattern))) +] +{ + edge @pattern.input -> @value.output +} + +;; +;; ##### ####### +;; # # # # # # ##### ## # # # # # ##### ###### #### +;; # # # ## # # # # # # # # # # # # # +;; ##### # # # # # # # ## # # # # ##### #### +;; # # # # # # ###### ## # # ##### # # +;; # # # # ## # # # # # # # # # # # +;; ##### # # # # # # # # # # # ###### #### +;; +;; Syntax Types + +;; +;; BEGIN BIG GNARLY DISJUNCTION +;; +;; The following pair of rules is intended to capture the following behavior: +;; +;; If a function definition is used to define a method, by being inside a class +;; definition, then we make its syntax type `method`. Otherwise, we make it's +;; syntax type `function`. Unfortunately, because of the limitations on negation +;; and binding in tree sitter queries, we cannot negate `class_definition` or +;; similar things directly. Instead, we have to manually push the negation down +;; to form the finite disjunction it corresponds to. +;; + +[ + (class_definition (block (decorated_definition (function_definition name: (_)@name)))) + (class_definition (block (function_definition name: (_)@name))) +] +{ + attr (@name.def) syntax_type = "method" +} + +[ + (module (decorated_definition (function_definition name: (_)@name))) + (module (function_definition name: (_)@name)) + + (if_statement (block (decorated_definition (function_definition name: (_)@name)))) + (if_statement (block (function_definition name: (_)@name))) + + (elif_clause (block (decorated_definition (function_definition name: (_)@name)))) + (elif_clause (block (function_definition name: (_)@name))) + + (else_clause (block (decorated_definition (function_definition name: (_)@name)))) + (else_clause (block (function_definition name: (_)@name))) + + (case_clause (block (decorated_definition (function_definition name: (_)@name)))) + (case_clause (block (function_definition name: (_)@name))) + + (for_statement (block (decorated_definition (function_definition name: (_)@name)))) + (for_statement (block (function_definition name: (_)@name))) + + (while_statement (block (decorated_definition (function_definition name: (_)@name)))) + (while_statement (block (function_definition name: (_)@name))) + + (try_statement (block (decorated_definition (function_definition name: (_)@name)))) + (try_statement (block (function_definition name: (_)@name))) + + (except_clause (block (decorated_definition (function_definition name: (_)@name)))) + (except_clause (block (function_definition name: (_)@name))) + + (finally_clause (block (decorated_definition (function_definition name: (_)@name)))) + (finally_clause (block (function_definition name: (_)@name))) + + (with_statement (block (decorated_definition (function_definition name: (_)@name)))) + (with_statement (block (function_definition name: (_)@name))) + + (function_definition (block (decorated_definition (function_definition name: (_)@name)))) + (function_definition (block (function_definition name: (_)@name))) +] +{ + attr (@name.def) syntax_type = "function" +} + +;; +;; END BIG GNARLY DISJUNCTION +;; diff --git a/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py b/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py new file mode 100644 index 000000000..0373a33be --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/aliased_imports.py @@ -0,0 +1,31 @@ +#------ path: foo.py ------# + +# module +class A: + a = 1 + +class B: + class C: + class D: + d = 2 + +#------ path: main.py ---# + +from foo import A as X, B.C.D as Y +import foo as f + +print X.a, Y.d +# ^ defined: 5 +# ^ defined: 10 + +print A, B.C +# ^ defined: +# ^ defined: +# ^ defined: + +print f.B +# ^ defined: 3, 15 +# ^ defined: 7 + +print foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/attributes.py b/languages/tree-sitter-stack-graphs-python/test/attributes.py new file mode 100644 index 000000000..6dd3d0e48 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/attributes.py @@ -0,0 +1,14 @@ +a = 1 + +a.b = 2 + +a.c.d = 5 + +print a.b +# ^ defined: 1 +# ^ defined: 3 + +print a.c, a.c.d +# ^ defined: 1 +# ^ defined: +# ^ defined: 5 diff --git a/languages/tree-sitter-stack-graphs-python/test/blocks.py b/languages/tree-sitter-stack-graphs-python/test/blocks.py new file mode 100644 index 000000000..462890baf --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/blocks.py @@ -0,0 +1,21 @@ +def f(): + if a: + b = 1 + else: + c = 2 + + print b, c + # ^ defined: 3 + # ^ defined: 5 + +class G: + if d: + e = 1 + + print e + # ^ defined: 13 + +print b, c, e +# ^ defined: +# ^ defined: +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/chained_functions.py b/languages/tree-sitter-stack-graphs-python/test/chained_functions.py new file mode 100644 index 000000000..ef1dac21d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/chained_functions.py @@ -0,0 +1,23 @@ +class A: + w = 1 + x = 2 + y = 3 + z = 4 + +def get_a(): + return A +def get_b(): + return get_a() +def get_c(): + return get_b() +def get_d(): + return get_c() +def get_e(): + return get_d() +def get_f(): + return get_e() + +g = get_f(A) +print g.x, g.y +# ^ defined: 3 +# ^ defined: 4 diff --git a/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip b/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip new file mode 100644 index 000000000..9c4f53115 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/chained_methods.py.skip @@ -0,0 +1,27 @@ +class Builder: + def set_a(self, a): + self.a = a + return self + + def set_b(self, b): + self.b = b + return self + + def set_c(self, c): + self.c = c + return self + + def set_d(self, d): + self.d = d + return self + + def set_e(self, e): + self.d = d + return self + +Builder().set_a('a1').set_b('b2').set_c('c3').set_d('d4').set_e('e4') +# ^ defined: 2 +# ^ defined: 6 +# ^ defined: 10 +# ^ defined: 14 +# ^ defined: 18 diff --git a/languages/tree-sitter-stack-graphs-python/test/class_members.py b/languages/tree-sitter-stack-graphs-python/test/class_members.py new file mode 100644 index 000000000..c67ace992 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/class_members.py @@ -0,0 +1,23 @@ +a = 1 + +class B: + c = a + # ^ defined: 1 + + def d(self): + return self.c + + class E: + f = a + # ^ defined: 1 + +print B.c +# ^ defined: 3 +# ^ defined: 1, 4 + +print B.d(1) +# ^ defined: 7 + +print B.a, E.a +# ^ defined: +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/decorators.py b/languages/tree-sitter-stack-graphs-python/test/decorators.py new file mode 100644 index 000000000..b55f44b2f --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/decorators.py @@ -0,0 +1,10 @@ +from a import deprecated +from b import ignore_warnings + +class A: + @deprecated + # ^ defined: 1 + @ignore_warnings.all + # ^ defined: 2 + def b(self): + pass diff --git a/languages/tree-sitter-stack-graphs-python/test/exceptions.py b/languages/tree-sitter-stack-graphs-python/test/exceptions.py new file mode 100644 index 000000000..b6b2e57e1 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/exceptions.py @@ -0,0 +1,5 @@ +try: + print() +except Exception as e: + x = e + # ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/functions.py b/languages/tree-sitter-stack-graphs-python/test/functions.py new file mode 100644 index 000000000..31a1d6782 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/functions.py @@ -0,0 +1,31 @@ +import a.x.y +import b.x.y + +def get_x(value): + return value.x + # ^ defined: 4 + +print get_x(a).y +# ^ defined: 1 + +print get_x(b).y +# ^ defined: 2 + +def get_a(): + return a + +print get_a(b).x +# ^ defined: 1 + +print get_x(foo=1, value=a).y +# ^ defined: 1 + +def foo(w: int, x, y=1, z: int=4, *args, **dict): + local = x +# ^ defined: 23 + print(args, w, z) +# ^ defined: 23 +# ^ defined: 23 +# ^ defined: 23 + return y +# ^ defined: 23 diff --git a/languages/tree-sitter-stack-graphs-python/test/imported_functions.py b/languages/tree-sitter-stack-graphs-python/test/imported_functions.py new file mode 100644 index 000000000..495697740 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imported_functions.py @@ -0,0 +1,17 @@ +#------ path: a.py ------# + +def foo(x): + return x + +#------ path: b.py ------# + +class A: + bar = 1 + +#------ path: main.py ---------# + +from a import * +from b import * + +foo(A).bar +# ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports.py b/languages/tree-sitter-stack-graphs-python/test/imports.py new file mode 100644 index 000000000..96d561813 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports.py @@ -0,0 +1,31 @@ +#------ path: one/two.py -----------# + +# module +import a.b.c + +d = 1 + +e = a.b + +#------ path: three/__init__.py ---# + +# module +f = 3 + +#------ path: main.py -------------# + +from one.two import d, e.c +# ^ defined: 3 +# ^ defined: 6 +# ^ defined: 4, 8 + +import three +# ^ defined: 12 + +print(d, e.c) +# ^ defined: 6, 17 +# ^ defined: 4, 17 + +print three.f +# ^ defined: 12, 22 +# ^ defined: 13 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py new file mode 100644 index 000000000..ea51f9450 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_submodule.py @@ -0,0 +1,14 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +from foo import bar + +bar.BAR +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py new file mode 100644 index 000000000..1eb97ffe2 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import FOO + +FOO +# ^ defined: 7, 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py new file mode 100644 index 000000000..088ca9544 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_a_value_aliased.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import FOO as QUX + +QUX +# ^ defined: 7, 3 + +FOO +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py new file mode 100644 index 000000000..3a1de4ffe --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_module_wildcard.py @@ -0,0 +1,13 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +from foo import * + +FOO +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py new file mode 100644 index 000000000..8d89d8538 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_submodule_a_value.py @@ -0,0 +1,16 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +from foo.bar import BAR + +BAR +# ^ defined: 7, 3 + +foo +# ^ defined: + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py new file mode 100644 index 000000000..961cd1622 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/bar/test.py --- + +from .. import FOO + +FOO +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py new file mode 100644 index 000000000..75652f7cd --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_super_package_submodule_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: foo/baz/test.py --- + +from ..bar import BAR + +BAR +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py new file mode 100644 index 000000000..f8de8042c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/test.py --- + +from . import FOO + +FOO +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py new file mode 100644 index 000000000..dc64ce387 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_from_this_package_submodule_a_value.py @@ -0,0 +1,10 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: foo/test.py --- + +from .bar import BAR + +BAR +# ^ defined: 7, 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py new file mode 100644 index 000000000..897acdf63 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_module.py @@ -0,0 +1,11 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +import foo + +foo.FOO +# ^ defined: 7, 3 +# ^ defined: 3 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py new file mode 100644 index 000000000..220a6b72d --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_module_aliased.py @@ -0,0 +1,14 @@ +# --- path: foo.py --- + +FOO = 42 + +# --- path: test.py --- + +import foo as qux + +qux.FOO +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py new file mode 100644 index 000000000..312cd66a1 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule.py @@ -0,0 +1,15 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo.bar + +foo.bar.BAR +# ^ defined: 7 +# ^ defined: 7, 3 +# ^ defined: 3 + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py new file mode 100644 index 000000000..b6bfdf7b7 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/import_submodule_aliased.py @@ -0,0 +1,17 @@ +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo.bar as qux + +qux.BAR +# ^ defined: 7, 3 +# ^ defined: 3 + +foo +# ^ defined: + +bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip b/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip new file mode 100644 index 000000000..2cdb2cf66 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/relative_import_resolves_to_itself.py.skip @@ -0,0 +1,14 @@ +# --- path: foo/__init__.py --- +from . import bar +# ^ defined: 6 + +# --- path: foo/bar/__init__.py --- +BAR = 'b' + +# --- path: main.py --- +from foo import bar +# ^ defined: 6 + +bar.BAR +# ^ defined: 9, 6 +# ^ defined: 6 diff --git a/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip b/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip new file mode 100644 index 000000000..a9d12848c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/imports/require_explicit_submodule_import.py.skip @@ -0,0 +1,18 @@ +# --- path: foo/__init__.py --- + +FOO = 42 + +# --- path: foo/bar.py --- + +BAR = 42 + +# --- path: test.py --- + +import foo + +foo.FOO +# ^ defined: 11, 3 +# ^ defined: 3 + +foo.bar +# ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/instance_members.py b/languages/tree-sitter-stack-graphs-python/test/instance_members.py new file mode 100644 index 000000000..38a6b6d75 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/instance_members.py @@ -0,0 +1,38 @@ +#---- path: a.py ------- + +class A: + def __init__(self, b, c): + self.b = b + self.c = A(c, 1) + + def get_b(self): + return self.b + # ^ defined: 4, 5 + + def get_c(self): + return self.c + # ^ defined: 6 + + def get_all(self): + return [self.get_b(), self.get_c()] + # ^ defined: 8 + # ^ defined: 12 + +a = A(1, 2) +a.get_all() +# ^ defined: 16 + +a.b +# ^ defined: 4, 5 + +a.c.b +# ^ defined: 4, 5 +# ^ defined: 6 + +#----- path: main.py --------- + +import a + +print a.A, a.a +# ^ defined: 3 +# ^ defined: 21 diff --git a/languages/tree-sitter-stack-graphs-python/test/lambdas.py b/languages/tree-sitter-stack-graphs-python/test/lambdas.py new file mode 100644 index 000000000..9812df4de --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/lambdas.py @@ -0,0 +1,2 @@ +sorted([1, 2, 3], key=lambda x: x) +# ^ defined: 1 diff --git a/languages/tree-sitter-stack-graphs-python/test/loops.py b/languages/tree-sitter-stack-graphs-python/test/loops.py new file mode 100644 index 000000000..2f0663295 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/loops.py @@ -0,0 +1,19 @@ +class Node3: + value = 3 +class Node2: + value = 2 + next = Node3 +class Node1: + value = 1 + next = Node2 + +def linked_list_search(l, item): + node = l + while node: + if node.value == item: + return node + # ^ defined: 10, 11, 16 + node = node.next + +linked_list_search(Node1, 5).value +# ^ defined: 6 diff --git a/languages/tree-sitter-stack-graphs-python/test/many_definitions.py b/languages/tree-sitter-stack-graphs-python/test/many_definitions.py new file mode 100644 index 000000000..f85f3a8ec --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/many_definitions.py @@ -0,0 +1,76 @@ +#--- path: a.py ---# + +def f0(a): return X.a + X.b +def f1(a): return f0(1) +def f2(a): return f1(2) +def f3(a): return f2(3) +def f4(a): return f3(4) +def f5(a): return f4(5) +def f6(a): return f5(6) +def f7(a): return f6(7) +def f8(a): return f7(8) +def f9(a): return f8(9) + +class C1: + def m0(self, b): return f9(0) + def m1(self, b): return self.m0(1) + def m2(self, b): return self.m1(2) + def m3(self, b): return self.m2(3) + def m4(self, b): return self.m3(4) + def m5(self, b): return self.m4(5) + def m6(self, b): return self.m5(6) + def m7(self, b): return self.m6(7) + def m8(self, b): return self.m7(8) + +def f10(): return C1.m8(0) +def f11(): return f10(1) +def f12(): return X.c(2) +def f13(): return X.c(3) +def f14(): return x(4) +def f15(): return x(5) +def f16(): return x(6) +def f17(): return x(7) +def f18(): return x(8) + +class C2: + def m0(self): return X.d(0) + def m1(self): return X.d(1) + def m2(self): return X.d(2) + def m3(self): return X.d(3) + def m4(self): return X.d(4) + def m5(self): return X.d(5) + def m6(self): return X.d(6) + def m7(self): return X.d(7) + def m8(self): return X.d(8) + +#--- path: main.py ---# + +from a import * + +print f0(), f4(), f8() +# ^ defined: 3 +# ^ defined: 7 +# ^ defined: 11 + +print C1.m0, C1().m0(), C1.m4, C1().m4, C1.m8, C1.m8 +# ^ defined: 14 +# ^ defined: 15 +# ^ defined: 15 +# ^ defined: 19 +# ^ defined: 19 +# ^ defined: 23 +# ^ defined: 23 + +print f10(), f14(), f18() +# ^ defined: 25 +# ^ defined: 29 +# ^ defined: 33 + +print C2.m0, C2().m0(), C2.m4, C2().m4, C2.m8, C2.m8 +# ^ defined: 35 +# ^ defined: 36 +# ^ defined: 36 +# ^ defined: 40 +# ^ defined: 40 +# ^ defined: 44 +# ^ defined: 44 diff --git a/languages/tree-sitter-stack-graphs-python/test/nested_functions.py b/languages/tree-sitter-stack-graphs-python/test/nested_functions.py new file mode 100644 index 000000000..db655e3a3 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/nested_functions.py @@ -0,0 +1,6 @@ +def outer(a): + def inner(b): + pass + + inner(1) + # ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-python/test/not_a_method.py b/languages/tree-sitter-stack-graphs-python/test/not_a_method.py new file mode 100644 index 000000000..f0a207eb0 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/not_a_method.py @@ -0,0 +1,20 @@ +class Foo: + a = 1 + + def first_method(self): + self.a + # ^ defined: 2 + + def second_method(self): + self.a + # ^ defined: 2 + + # First argument here is not self + def not_a_method(not_self): + return not_self.a + # ^ defined: + + +def function(self): + self.a + # ^ defined: diff --git a/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py b/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py new file mode 100644 index 000000000..d6b8980a3 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/pattern_matching.py @@ -0,0 +1,42 @@ +command = 1 +current_room = 2 +character = 3 + +match command.split(): + # ^ defined: 1 + case ["quit"]: + print("Goodbye!") + quit_game() + case ["look"]: + current_room.describe() + # ^ defined: 2 + case ["get", obj]: + character.get(obj, current_room) + # ^ defined: 3 + # ^ defined: 13 + # ^ defined: 2 + case ["go", direction]: + current_room = current_room.neighbor(direction) + # ^ defined: 18 + case { "foo": foo }: + print(foo) + # ^ defined: 21 + case {"bar": bar, "quux": quux}: + print(bar,quux) + # ^ defined: 24 + # ^ defined: 24 + case ["grab", { "key": {"garply": garply}}]: + print(garply) + # ^ defined: 28 + case ["drop", *objs]: + print(objs) + # ^ defined: 31 + case ["get", obj] | ["pick", "up", obj] | ["pick", obj, "up"]: + print(obj) + # ^ defined: 34, 34, 34 + case ["go", ("north" | "south" | "east" | "west") as direction2]: + current_room = current_room.neighbor(direction2) + # ^ defined: 37 + case (foo, "bar"): + print(foo) + # ^ defined: 40 diff --git a/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py b/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py new file mode 100644 index 000000000..53afca29e --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/redundant_reexport.py @@ -0,0 +1,15 @@ +#--- path: a/__init__.py ---# + +from . import child + +#--- path: a/child.py ---- + +def f(): + pass + +#--- path: main.py ---# + +import a + +print a.child.f() +# ^ defined: 7 diff --git a/languages/tree-sitter-stack-graphs-python/test/relative_imports.py b/languages/tree-sitter-stack-graphs-python/test/relative_imports.py new file mode 100644 index 000000000..f14571575 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/relative_imports.py @@ -0,0 +1,60 @@ +#------ path: foo/bar/main.py ------# + +from . import a +from .b import B +from ..c import see +from ..c.d import D + +print a.A +# ^ defined: 3, 25 +# ^ defined: 26 + +print B.bee +# ^ defined: 4, 31 +# ^ defined: 32 + +print see() +# ^ defined: 5, 37 + + +print D.d +# ^ defined: 44 + +#------ path: foo/bar/a.py --------# + +# module +A = "a" + +#------ path: foo/bar/b.py --------# + +# module +class B: + bee = 1 + +#------ path: foo/c.py ------------# + +# module +def see(): + pass + +#------ path: foo/c/d.py ---------# + +# module +class D: + d = "d" + +#------ path: foo/e/g.py ---# + +# module +G = 1 + +#------ path: foo/e/__init__.py ---# + +# module +from .g import G +# ^ defined: 49 + +from ..c import see +# ^ defined: 37 + +E = 1 diff --git a/languages/tree-sitter-stack-graphs-python/test/root_path.py b/languages/tree-sitter-stack-graphs-python/test/root_path.py new file mode 100644 index 000000000..23937db7c --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/root_path.py @@ -0,0 +1,21 @@ +# ------ path: foo/bar/module.py -----------# +# ------ global: ROOT_PATH=foo/bar -----------# + +foo = 42 + +# ------ path: foo/bar/baz/module.py -----------# +# ------ global: ROOT_PATH=foo/bar -----------# + +bar = "hello" + +# ------ path: foo/bar/main.py -------------# +# ------ global: ROOT_PATH=foo/bar -----------# + +from module import foo +from baz.module import bar + +print(foo) +# ^ defined: 4, 14 + +print(bar) +# ^ defined: 9, 15 diff --git a/languages/tree-sitter-stack-graphs-python/test/self.py b/languages/tree-sitter-stack-graphs-python/test/self.py new file mode 100644 index 000000000..763a52eb6 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/self.py @@ -0,0 +1,11 @@ +class Foo: + a = 1 + + def mathod_1(self): + return self.a + # ^ defined: 2 + + # Self can be named anything + def method_2(actually_self): + return actually_self.a + # ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py b/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py new file mode 100644 index 000000000..6e3a13abb --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/statement_bindings.py @@ -0,0 +1,9 @@ +import a.b +import c.d + +with a as x, c as y: + print x.b, y.d + # ^ defined: 1, 4 + # ^ defined: 1 + # ^ defined: 2, 4 + # ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-python/test/superclasses.py b/languages/tree-sitter-stack-graphs-python/test/superclasses.py new file mode 100644 index 000000000..849a192a6 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/superclasses.py @@ -0,0 +1,19 @@ +class A: + def __init__(self): + self.some_attr = 2 + + def some_method(self): + print self + +class B(A): + def method2(self): + print self.some_attr, self.some_method() + # ^ defined: 3 + # ^ defined: 5, 14 + + def some_method(self): + pass + + def other(self): + super().some_method() + # ^ defined: 5 diff --git a/languages/tree-sitter-stack-graphs-python/test/test.py b/languages/tree-sitter-stack-graphs-python/test/test.py new file mode 100644 index 000000000..e69de29bb diff --git a/languages/tree-sitter-stack-graphs-python/test/tuples.py b/languages/tree-sitter-stack-graphs-python/test/tuples.py new file mode 100644 index 000000000..1ea6ee3ef --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/tuples.py @@ -0,0 +1,38 @@ +class a: + x = 1 +class b: + x = 1 +class c: + x = 1 + +(a1, b1) = (a, b) +a2, b2 = (a, b) +(a3, b3) = (a, b) +a4, b4 = a, b + +print a1.x, b1.x +# ^ defined: 2 +# ^ defined: 4 +print a2.x, b2.x +# ^ defined: 2 +# ^ defined: 4 +print a3.x, b3.x +# ^ defined: 2 +# ^ defined: 4 +print a4.x, b4.x +# ^ defined: 2 +# ^ defined: 4 + +t = (a, b), c +(a5, b5), c5 = t + +print a5.x, b5.x, c5.x +# ^ defined: 2 +# ^ defined: 4 +# ^ defined: 6 + +(a6, (b6, c6)) = (a, (b, c)) + +print a6.x, b6.x +# ^ defined: 2 +# ^ defined: 4 diff --git a/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py b/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py new file mode 100644 index 000000000..f2ac040f2 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-python/test/wildcard_import.py @@ -0,0 +1,21 @@ +#------ path: one/two.py ------# + +a = 1 +b = 2 + +#------ path: one/three.py ------# + +b = 3 +c = 4 + +#------ path: main.py ---------# + +from one.two import * +from one.three import * + +print a +# ^ defined: 3 +print b +# ^ defined: 8 +print c +# ^ defined: 9 diff --git a/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md b/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md index e5b478f09..5a3bfa3d4 100644 --- a/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md +++ b/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md @@ -5,7 +5,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v0.4.0 -- 2024-12-13 + +- The `tree-sitter-stack-graphs` dependency is updated to version 0.10. + +- The `tree-sitter-typescript` dependency is updated to version 0.23.2. + +## v0.3.0 -- 2024-07-09 + +### Added + +- Support for TSX. A new language configuration for TSX is available with `{try_,}language_configuration_tsx`. TSX is enabled in the CLI next to TypeScript. + +### Fixed + +- Imports are more robust to the presence of file extensions in the import name. + +### Changed + +- The functions `{try_,}language_configuration` have been renamed to `{try_,}language_configuration_typescript`. + +### Removed + +- The `FILE_PATH_VAR` constant has been replaced in favor of `tree_sitter_stack_graphs::FILE_PATH_VAR`. + +## v0.2.0 -- 2024-03-06 + +The `tree-sitter-stack-graphs` is updated to `v0.8`. + +### Added + +- An experimental VSCode LSP plugin that supports code navigation based on the stack graph rules. _Purely an experiment, not ready for serious use!_ Requires the `lsp` feature to be enabled. + +### Changed + +- Various improvements to the rules for imports and packages. + +## v0.1.0 -- 2023-01-27 ### Added diff --git a/languages/tree-sitter-stack-graphs-typescript/Cargo.toml b/languages/tree-sitter-stack-graphs-typescript/Cargo.toml index da5df7ef9..ed14d9112 100644 --- a/languages/tree-sitter-stack-graphs-typescript/Cargo.toml +++ b/languages/tree-sitter-stack-graphs-typescript/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "tree-sitter-stack-graphs-typescript" -version = "0.1.0" -description = "Stack graphs definition for TypeScript using tree-sitter-typescript" +version = "0.4.0" +description = "Stack graphs definition for TypeScript & TSX using tree-sitter-typescript" readme = "README.md" -keywords = ["tree-sitter", "stack-graphs", "typescript"] +keywords = ["tree-sitter", "stack-graphs", "typescript", "tsx"] authors = ["Hendrik van Antwerpen "] license = "MIT OR Apache-2.0" edition = "2018" @@ -32,11 +32,15 @@ clap = { version = "4", optional = true } glob = "0.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -stack-graphs = { version = ">=0.11, <=0.12", path = "../../stack-graphs" } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs" } -tree-sitter-typescript = "=0.20.2" +stack-graphs = { version = "0.14", path = "../../stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-stack-graphs = { version = "0.10", path = "../../tree-sitter-stack-graphs" } # explicit version is required to be able to publish crate +tree-sitter-typescript = "=0.23.2" tsconfig = "0.1.0" [dev-dependencies] anyhow = { version = "1.0" } -tree-sitter-stack-graphs = { version = "0.7", path = "../../tree-sitter-stack-graphs", features = ["cli"] } +tree-sitter-stack-graphs = { path = "../../tree-sitter-stack-graphs", features = ["cli"] } + +[build-dependencies] +anyhow = { version = "1.0" } +regex = "1.10.3" diff --git a/languages/tree-sitter-stack-graphs-typescript/README.md b/languages/tree-sitter-stack-graphs-typescript/README.md index 0c1f9d5ee..c83d5849d 100644 --- a/languages/tree-sitter-stack-graphs-typescript/README.md +++ b/languages/tree-sitter-stack-graphs-typescript/README.md @@ -4,72 +4,106 @@ This project defines tree-sitter-stack-graphs rules for TypeScript using the [tr [tree-sitter-typescript]: https://crates.io/crates/tree-sitter-typescript -## Usage +- [API documentation](https://docs.rs/tree-sitter-stack-graphs-typescript/) +- [Release notes](https://github.com/github/stack-graphs/blob/main/languages/tree-sitter-stack-graphs-typescript/CHANGELOG.md) + +## Using the API To use this library, add the following to your `Cargo.toml`: -``` toml +```toml [dependencies] -tree-sitter-stack-graphs-typescript = "0.1" +tree-sitter-stack-graphs-typescript = "0.4" ``` -Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-typescript/*/) for -more details on how to use this library. +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs-typescript/*/) for more details on how to use this library. -## Command-line Program +## Using the Command-line Program -The command-line program for `tree-sitter-stack-graphs-typescript` lets you do -stack graph based analysis and lookup from the command line. +The command-line program for `tree-sitter-stack-graphs-typescript` lets you do stack graph based analysis and lookup from the command line. -Install the program using `cargo install` as follows: +The CLI can be run as follows: -``` sh -$ cargo install --features cli tree-sitter-stack-graphs-typescript -$ tree-sitter-stack-graphs-typescript --help -``` +1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli tree-sitter-stack-graphs-typescript + ``` + + After this, the CLI should be available as `tree-sitter-stack-graphs-typescript`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs-typescript` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs-typescript index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs-typescript status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + tree-sitter-stack-graphs-typescript clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs-typescript query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. ## Development -The project is written in Rust, and requires a recent version installed. -Rust can be installed and updated using [rustup][]. +The project is written in Rust, and requires a recent version installed. Rust can be installed and updated using [rustup][]. [rustup]: https://rustup.rs/ - The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. -- Builtins sources and configuration are defined in `src/builtins.ts` and `builtins.cfg` respectively. +- Builtins sources and configuration are defined in `src/builtins.it` and `builtins.cfg` respectively. - Tests are put into the `test` directory. -### Building and Running Tests - -Build the project by running: - -``` sh -$ cargo build -``` +### Running Tests Run the tests as follows: -``` sh -$ cargo test +```sh +cargo test ``` -The project consists of a library and a CLI. -By default, running `cargo` only applies to the library. -To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. +The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. Run the CLI from source as follows: -``` sh -$ cargo run --features cli -- ARGS +```sh +cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: -``` sh -$ cargo fmt +```sh +cargo fmt ``` ### Writing TSG @@ -84,21 +118,21 @@ which contain self-contained TSG rules for specific language features. A VSCode Parse and test a single file by executing the following commands: -``` sh -$ cargo run --features cli -- parse FILES... -$ cargo run --features cli -- test TESTFILES... +```sh +cargo run --features cli -- parse FILES... +cargo run --features cli -- test TESTFILES... ``` Generate a visualization to debug failing tests by passing the `-V` flag: -``` sh -$ cargo run --features cli -- test -V TESTFILES... +```sh +cargo run --features cli -- test -V TESTFILES... ``` To generate the visualization regardless of test outcome, execute: -``` sh -$ cargo run --features cli -- test -V --output-mode=always TESTFILES... +```sh +cargo run --features cli -- test -V --output-mode=always TESTFILES... ``` -Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. +Go to for links to examples and documentation. diff --git a/languages/tree-sitter-stack-graphs-typescript/build.rs b/languages/tree-sitter-stack-graphs-typescript/build.rs new file mode 100644 index 000000000..31e3dd9de --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/build.rs @@ -0,0 +1,92 @@ +// -*- coding: utf-8 -*- +// ------------------------------------------------------------------------------------------------ +// Copyright © 2024, stack-graphs authors. +// Licensed under either of Apache License, Version 2.0, or MIT license, at your option. +// Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. +// ------------------------------------------------------------------------------------------------ + +use std::path::Path; + +use anyhow::{bail, Result}; +use regex::Regex; + +const TSG_SOURCE: &str = "src/stack-graphs.tsg"; +const DIALECTS: [&str; 2] = ["typescript", "tsx"]; + +/// preprocess the input file, removing lines that are not for the selected dialect +fn preprocess( + input: impl std::io::Read, + mut output: impl std::io::Write, + dialect: &str, +) -> Result<()> { + // Matches: ; #dialect typescript + let directive_start = Regex::new(r";\s*#dialect\s+(\w+)").unwrap(); + + // Matches: ; #end + let directive_end = Regex::new(r";\s*#end").unwrap(); + + let no_code = Regex::new(r"^[\s;]+").unwrap(); + + let input = std::io::read_to_string(input)?; + + // If the filter is None or Some(true), the lines are written to the output + let mut filter: Option = None; + + for (mut line_no, line) in input.lines().enumerate() { + // Line numbers are one based + line_no += 1; + + if let Some(captures) = directive_start.captures(line) { + if !no_code.is_match(line) { + bail!("Line {line_no}: unexpected code before directive"); + } + + if filter.is_some() { + bail!("Line {line_no}: dialect directive cannot be nested"); + } + + let directive = captures.get(1).unwrap().as_str(); + if !DIALECTS.contains(&directive) { + bail!("Line {line_no}: unknown dialect: {directive}"); + } + + filter = Some(dialect == directive); + } else if directive_end.is_match(line) { + if !no_code.is_match(line) { + bail!("Line {line_no}: unexpected code before directive end"); + } + + if filter.is_none() { + bail!("Line {line_no}: unmatched directive end"); + } + + filter = None; + } else if filter.unwrap_or(true) { + output.write_all(line.as_bytes())?; + } + // a new line is always written so that removed lines are padded to preserve line numbers + output.write(b"\n")?; + } + + if filter.is_some() { + bail!("Unmatched directive end at the end of the file"); + } + + Ok(()) +} + +fn main() { + let out_dir = std::env::var_os("OUT_DIR").expect("OUT_DIR is not set"); + + for dialect in DIALECTS { + let input = std::fs::File::open(TSG_SOURCE).expect("Failed to open stack-graphs.tsg"); + + let out_filename = Path::new(&out_dir).join(format!("stack-graphs-{dialect}.tsg")); + let output = std::fs::File::create(out_filename).expect("Failed to create output file"); + + preprocess(input, output, dialect).expect("Failed to preprocess stack-graphs.tsg"); + } + + println!("cargo:rerun-if-changed={TSG_SOURCE}"); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs b/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs index b1ee4f568..ea7713595 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/bin.rs @@ -12,17 +12,23 @@ use tree_sitter_stack_graphs::cli::provided_languages::Subcommands; use tree_sitter_stack_graphs::NoCancellation; fn main() -> anyhow::Result<()> { - let lc = match tree_sitter_stack_graphs_typescript::try_language_configuration(&NoCancellation) - { - Ok(lc) => lc, - Err(err) => { - eprintln!("{}", err.display_pretty()); - return Err(anyhow!("Language configuration error")); - } - }; let cli = Cli::parse(); + let mut lcs = Vec::new(); + for r in [ + tree_sitter_stack_graphs_typescript::try_language_configuration_typescript(&NoCancellation), + tree_sitter_stack_graphs_typescript::try_language_configuration_tsx(&NoCancellation), + ] { + let lc = match r { + Ok(lc) => lc, + Err(err) => { + eprintln!("{}", err.display_pretty()); + return Err(anyhow!("Language configuration error")); + } + }; + lcs.push(lc); + } let default_db_path = default_user_database_path_for_crate(env!("CARGO_PKG_NAME"))?; - cli.subcommand.run(default_db_path, vec![lc]) + cli.subcommand.run(default_db_path, lcs) } #[derive(Parser)] diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs b/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs index 3590649b2..8fd63881d 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/lib.rs @@ -19,7 +19,10 @@ pub mod util; /// The stacks graphs tsg path for this language. pub const STACK_GRAPHS_TSG_PATH: &str = "src/stack-graphs.tsg"; /// The stack graphs tsg source for this language -pub const STACK_GRAPHS_TSG_SOURCE: &str = include_str!("../src/stack-graphs.tsg"); +pub const STACK_GRAPHS_TSG_TS_SOURCE: &str = + include_str!(concat!(env!("OUT_DIR"), "/stack-graphs-typescript.tsg")); +pub const STACK_GRAPHS_TSG_TSX_SOURCE: &str = + include_str!(concat!(env!("OUT_DIR"), "/stack-graphs-tsx.tsg")); /// The stack graphs builtins configuration for this language pub const STACK_GRAPHS_BUILTINS_CONFIG: &str = include_str!("../src/builtins.cfg"); @@ -28,25 +31,55 @@ pub const STACK_GRAPHS_BUILTINS_PATH: &str = "src/builtins.ts"; /// The stack graphs builtins source for this language pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.ts"); -/// The name of the file path global variable -pub const FILE_PATH_VAR: &str = "FILE_PATH"; /// The name of the project name global variable pub const PROJECT_NAME_VAR: &str = "PROJECT_NAME"; -pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration { - try_language_configuration(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) +pub fn language_configuration_typescript( + cancellation_flag: &dyn CancellationFlag, +) -> LanguageConfiguration { + try_language_configuration_typescript(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) } -pub fn try_language_configuration( +pub fn try_language_configuration_typescript( cancellation_flag: &dyn CancellationFlag, ) -> Result { let mut lc = LanguageConfiguration::from_sources( - tree_sitter_typescript::language_typescript(), + tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), Some(String::from("source.ts")), None, vec![String::from("ts")], STACK_GRAPHS_TSG_PATH.into(), - STACK_GRAPHS_TSG_SOURCE, + STACK_GRAPHS_TSG_TS_SOURCE, + Some(( + STACK_GRAPHS_BUILTINS_PATH.into(), + STACK_GRAPHS_BUILTINS_SOURCE, + )), + Some(STACK_GRAPHS_BUILTINS_CONFIG), + cancellation_flag, + )?; + lc.special_files + .add("tsconfig.json".to_string(), TsConfigAnalyzer {}) + .add("package.json".to_string(), NpmPackageAnalyzer {}); + lc.no_similar_paths_in_file = true; + Ok(lc) +} + +pub fn language_configuration_tsx( + cancellation_flag: &dyn CancellationFlag, +) -> LanguageConfiguration { + try_language_configuration_tsx(cancellation_flag).unwrap_or_else(|err| panic!("{}", err)) +} + +pub fn try_language_configuration_tsx( + cancellation_flag: &dyn CancellationFlag, +) -> Result { + let mut lc = LanguageConfiguration::from_sources( + tree_sitter_typescript::LANGUAGE_TSX.into(), + Some(String::from("source.tsx")), + None, + vec![String::from("tsx")], + STACK_GRAPHS_TSG_PATH.into(), + STACK_GRAPHS_TSG_TSX_SOURCE, Some(( STACK_GRAPHS_BUILTINS_PATH.into(), STACK_GRAPHS_BUILTINS_SOURCE, diff --git a/languages/tree-sitter-stack-graphs-typescript/rust/test.rs b/languages/tree-sitter-stack-graphs-typescript/rust/test.rs index ca5b8494c..51ef9cdca 100644 --- a/languages/tree-sitter-stack-graphs-typescript/rust/test.rs +++ b/languages/tree-sitter-stack-graphs-typescript/rust/test.rs @@ -11,14 +11,18 @@ use tree_sitter_stack_graphs::ci::Tester; use tree_sitter_stack_graphs::NoCancellation; fn main() -> anyhow::Result<()> { - let lc = match tree_sitter_stack_graphs_typescript::try_language_configuration(&NoCancellation) - { - Ok(lc) => lc, - Err(err) => { - eprintln!("{}", err.display_pretty()); - return Err(anyhow!("Language configuration error")); - } - }; + let lc_factories = [ + tree_sitter_stack_graphs_typescript::try_language_configuration_typescript, + tree_sitter_stack_graphs_typescript::try_language_configuration_tsx, + ]; + + let lcs = lc_factories + .iter() + .map(|lc_factory| lc_factory(&NoCancellation)) + .collect::, _>>() + .inspect_err(|err| eprintln!("{}", err.display_pretty())) + .map_err(|_| anyhow!("Language configuration error"))?; + let test_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("test"); - Tester::new(vec![lc], vec![test_path]).run() + Tester::new(lcs, vec![test_path]).run() } diff --git a/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg b/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg index 66c432b46..773abde7c 100644 --- a/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg +++ b/languages/tree-sitter-stack-graphs-typescript/src/stack-graphs.tsg @@ -540,7 +540,8 @@ attribute node_symbol = node => symbol = (source-text node), source_n ; module reference var mod_scope = mod_ref__ns - scan (path-normalize mod_path) { + ; normalize path and remove the extension as we want to match 'foo', 'foo.js', 'foo.ts', etc. + scan (path-normalize (replace mod_path "\.(js|ts|jsx|tsx)$" "")) { "([^/]+)/?" { node mod_ref attr (mod_ref) push_symbol = $1 @@ -626,16 +627,6 @@ attribute node_symbol = node => symbol = (source-text node), source_n (import_statement "type"?@is_type (import_clause (named_imports (import_specifier name:(_)@name))@clause)) ] { if none @is_type { - node @name.expr_def - node @name.expr_def__ns - node @name.expr_ref - node @name.expr_ref__ns - - ; expr reference - attr (@name.expr_ref) node_reference = @name - edge @name.expr_ref -> @name.expr_ref__ns - ; - attr (@name.expr_ref__ns) push_symbol = "%E" edge @name.expr_ref__ns -> @clause.lexical_scope } } @@ -645,6 +636,9 @@ if none @is_type { (import_statement "type"?@is_type (import_clause (named_imports (import_specifier name:(_)@name !alias))@clause)) ] { if none @is_type { + node @name.expr_def + node @name.expr_def__ns + ; expr definition edge @clause.defs -> @name.expr_def__ns ; FIXME defs, lexical_defs? ; @@ -663,7 +657,6 @@ if none @is_type { if none @is_type { node @alias.expr_def node @alias.expr_def__ns - node @alias.expr_ref ; expr definition edge @clause.defs -> @alias.expr_def__ns @@ -1468,6 +1461,8 @@ if none @is_async { edge @dec.lexical_scope -> @class_decl.lexical_scope } + + ; definitions [ (abstract_class_declaration name:(_)@name) @@ -1625,6 +1620,7 @@ if none @is_async { [ (abstract_class_declaration body:(_)@body) (class_declaration body:(_)@body) + (interface_declaration body:(_)@body) ]@class_decl { ; propagate lexical scope ; FIXME the static members have access to type variables like this @@ -1638,16 +1634,6 @@ if none @is_async { edge @class_decl.static_type -> @body.static_members attr (@class_decl.static_type -> @body.static_members) precedence = 2 } -[ - (interface_declaration body:(_)@body) -]@class_decl { - ; propagate lexical scope - edge @body.lexical_scope -> @class_decl.generic_inner_lexical_scope - - ; interface type equals body type - edge @class_decl.generic_inner_type -> @body.type -} - ;; Class Body @@ -1661,23 +1647,35 @@ if none @is_async { ; (public_field_definition)] ; ) -(class_body)@class_body { - node @class_body.lexical_scope - node @class_body.type_members - node @class_body.static_members +[ + (class_body) + (interface_body) +]@body { + node @body.lexical_scope + node @body.type_members + node @body.static_members +} + +; decorators +[ + (class_body decorator:(_)@dec) +]@class_body { + ; connect lexical scope + edge @dec.lexical_scope -> @class_body.lexical_scope } -(class_body - (_)@mem -)@class_body { +[ + (class_body (_)@mem) + (interface_body (_)@mem) +]@body { ; propagate lexical scope - edge @mem.lexical_scope -> @class_body.lexical_scope + edge @mem.lexical_scope -> @body.lexical_scope ; body members are member definitions - edge @class_body.type_members -> @mem.type_members + edge @body.type_members -> @mem.type_members ; body static members are static member definitions - edge @class_body.static_members -> @mem.static_members + edge @body.static_members -> @mem.static_members } @@ -2333,7 +2331,13 @@ if none @is_async { (import_statement (import_clause (namespace_import (identifier)@name))) (ambient_declaration (module name:(string) @name)) ; X._ - (nested_identifier (identifier)@name) + (nested_identifier object:(identifier)@name) + ; _.X._ + (member_expression object:(identifier)@name) + ; _.X + (nested_identifier property:(_)@name) + ; _._.X + (member_expression property:(_)@name) ] { node @name.expr_def node expr_def_typeof @@ -2356,7 +2360,10 @@ if none @is_async { edge @name.type_def -> @name.type_def_member } -(nested_identifier . (_) @mod) @nested { +[ + (nested_identifier object:(_)@mod) + (member_expression object:[(member_expression) (identifier)]@mod) +]@nested { node @nested.expr_def node @nested.type_def @@ -2364,12 +2371,18 @@ if none @is_async { edge @nested.type_def -> @mod.type_def } -(nested_identifier . (_) @mod . (_) @name .) { +[ + (nested_identifier object:(_)@mod property:(_)@name) + (member_expression object:[(member_expression) (identifier)]@mod property:(_)@name) +] { edge @mod.expr_def_member -> @name.expr_def edge @mod.type_def_member -> @name.type_def } -(nested_identifier (_) @name .) @nested { +[ + (nested_identifier property:(_)@name) + (member_expression property:(_)@name) +]@nested { node @nested.expr_def_member node @nested.type_def_member @@ -2643,7 +2656,7 @@ if none @is_async { (call_expression) (class) (false) - (function) + (function_expression) (generator_function) (import) (member_expression) @@ -2665,7 +2678,16 @@ if none @is_async { (ternary_expression) (this) (true) +; #dialect typescript (type_assertion) +; #end +; #dialect tsx + (jsx_element) + (jsx_self_closing_element) + (jsx_opening_element) + (jsx_closing_element) + (jsx_expression) +; #end (unary_expression) (undefined) (update_expression) @@ -2767,91 +2789,13 @@ if none @is_async { ; x; -[ - (primary_expression/identifier)@name - ; FIXME expansion of _lhs_expression/identifier and _augmented_assignment_lhs - (for_in_statement ["var" "let" "const"]?@is_def left:(identifier)@name) - (assignment_expression left:(identifier)@name) - (augmented_assignment_expression left:(identifier)@name) - (asserts (identifier)@name) - (type_predicate name:(identifier)@name) - ; FIXME type_query has its own restricted expression production - ; we need to do this for every (identifier) inside a type query - ; this cannot be expressed, so we manually unroll three levels here - (type_query (identifier)@name) - (type_query (member_expression object:(identifier)@name)) - (type_query (member_expression object:(member_expression object:(identifier)@name))) - (type_query (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (subscript_expression object:(identifier)@name)) - (type_query (subscript_expression object:(member_expression object:(identifier)@name))) - (type_query (subscript_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(identifier)@name)) - (type_query (call_expression function:(member_expression object:(identifier)@name))) - (type_query (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - (type_query (call_expression function:(subscript_expression object:(identifier)@name))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - ; FIXME decorator has its own restricted expression production - ; we need to do this for every (identifier) inside a decorator - ; this cannot be expressed, so we manually unroll three levels here - (decorator (identifier)@name) - (decorator (member_expression object:(identifier)@name)) - (decorator (member_expression object:(member_expression object:(identifier)@name))) - (decorator (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(identifier)@name)) - (decorator (call_expression function:(member_expression object:(identifier)@name))) - (decorator (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) -] { -if none @is_def { +(identifier)@name { node @name.cotype node @name.lexical_defs node @name.lexical_scope node @name.type node @name.var_defs -} -} -[ - (primary_expression/identifier)@name - (decorator (identifier)@name) - ; FIXME expansion of _lhs_expression/identifier and _augmented_assignment_lhs - ; we need to do this for every (identifier) inside a type query - ; this cannot be expressed, so we manually unroll three levels here - (for_in_statement ["var" "let" "const"]?@is_def left:(identifier)@name) - (assignment_expression left:(identifier)@name) - (augmented_assignment_expression left:(identifier)@name) - (asserts (identifier)@name) - (type_predicate name:(identifier)@name) - ; FIXME type_query has its own restricted expression production - (type_query (identifier)@name) - (type_query (member_expression object:(identifier)@name)) - (type_query (member_expression object:(member_expression object:(identifier)@name))) - (type_query (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (subscript_expression object:(identifier)@name)) - (type_query (subscript_expression object:(member_expression object:(identifier)@name))) - (type_query (subscript_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(identifier)@name)) - (type_query (call_expression function:(member_expression object:(identifier)@name))) - (type_query (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - (type_query (call_expression function:(subscript_expression object:(identifier)@name))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(identifier)@name)))) - (type_query (call_expression function:(subscript_expression object:(member_expression object:(member_expression object:(identifier)@name))))) - ; FIXME decorator has its own restricted expression production - ; we need to do this for every (identifier) inside a decorator - ; this cannot be expressed, so we manually unroll three levels here - (decorator (identifier)@name) - (decorator (member_expression object:(identifier)@name)) - (decorator (member_expression object:(member_expression object:(identifier)@name))) - (decorator (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(identifier)@name)) - (decorator (call_expression function:(member_expression object:(identifier)@name))) - (decorator (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) - (decorator (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) -] { -if none @is_def { node @name.expr_ref node @name.expr_ref__ns node @name.expr_ref__typeof @@ -2869,7 +2813,67 @@ if none @is_def { attr (@name.expr_ref__typeof) push_symbol = ":" edge @name.expr_ref__typeof -> @name.expr_ref } -} + +; [ +; (primary_expression/identifier)@name +; ; FIXME expansion of _lhs_expression/identifier and _augmented_assignment_lhs +; ; we need to do this for every (identifier) inside a type query +; ; this cannot be expressed, so we manually unroll three levels here +; (for_in_statement ["var" "let" "const"]?@is_def left:(identifier)@name) +; (assignment_expression left:(identifier)@name) +; (augmented_assignment_expression left:(identifier)@name) +; (asserts (identifier)@name) +; (type_predicate name:(identifier)@name) +; ; FIXME type_query has its own restricted expression production +; ; we need to do this for every (identifier) inside a type query +; ; this cannot be expressed, so we manually unroll three levels here +; (type_query (identifier)@name) +; (type_query (member_expression object:(identifier)@name)) +; (type_query (member_expression object:(member_expression object:(identifier)@name))) +; (type_query (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; (type_query (subscript_expression object:(identifier)@name)) +; (type_query (subscript_expression object:(member_expression object:(identifier)@name))) +; (type_query (subscript_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; (type_query (call_expression function:(identifier)@name)) +; (type_query (call_expression function:(member_expression object:(identifier)@name))) +; (type_query (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) +; (type_query (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) +; (type_query (call_expression function:(subscript_expression object:(identifier)@name))) +; (type_query (call_expression function:(subscript_expression object:(member_expression object:(identifier)@name)))) +; (type_query (call_expression function:(subscript_expression object:(member_expression object:(member_expression object:(identifier)@name))))) +; ; FIXME decorator has its own restricted expression production +; ; we need to do this for every (identifier) inside a decorator +; ; this cannot be expressed, so we manually unroll three levels here +; (decorator (identifier)@name) +; (decorator (member_expression object:(identifier)@name)) +; (decorator (member_expression object:(member_expression object:(identifier)@name))) +; (decorator (member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; (decorator (call_expression function:(identifier)@name)) +; (decorator (call_expression function:(member_expression object:(identifier)@name))) +; (decorator (call_expression function:(member_expression object:(member_expression object:(identifier)@name)))) +; (decorator (call_expression function:(member_expression object:(member_expression object:(member_expression object:(identifier)@name))))) +; ; FIXME nested_identifier has its own restricted expression production +; ; we need to do this for every (identifier) inside a decorator +; ; this cannot be expressed, so we manually unroll three levels here +; (nested_identifier object:(identifier)@name) +; (nested_identifier object:(member_expression object:(identifier)@name)) +; (nested_identifier object:(member_expression object:(member_expression object:(identifier)@name))) +; (nested_identifier object:(member_expression object:(member_expression object:(member_expression object:(identifier)@name)))) +; ; #dialect tsx +; (jsx_opening_element name: (identifier)@name) +; (jsx_closing_element name: (identifier)@name) +; (jsx_self_closing_element name: (identifier)@name) +; ; #end +; ] { +; if none @is_def { +; ; node @name.cotype +; ; node @name.lexical_defs +; ; node @name.lexical_scope +; ; node @name.type +; ; node @name.var_defs + +; } +; } @@ -3234,20 +3238,20 @@ if none @is_def { ; function (x) {}; -; (function +; (function_expression ; (formal_parameters (identifier)) ; (statement_block)) ; this captures the parameters -; (function +; (function_expression ; parameters: (_)@params) ; this captures the body -; (function +; (function_expression ; body:(_)@body)@function ; functions with names -; (function +; (function_expression ; name:(_)@name ; parameters:(_)@call_sig)@fun { ; } @@ -3282,7 +3286,7 @@ if none @is_def { ; } [ - (function) + (function_expression) (arrow_function) (generator_function) ]@fun { @@ -3309,9 +3313,9 @@ if none @is_def { } [ - (function parameters:(_)@call_sig) - (arrow_function parameters:(_)@call_sig) - (generator_function parameters:(_)@call_sig) + (function_expression parameters:(_)@call_sig) + (arrow_function parameters:(_)@call_sig) + (generator_function parameters:(_)@call_sig) ]@fun { ; propagate lexical scope edge @call_sig.lexical_scope -> @fun.lexical_scope @@ -3360,9 +3364,9 @@ if none @is_def { } [ - (function body:(_)@body) - (arrow_function body:(_)@body) - (generator_function body:(_)@body) + (function_expression body:(_)@body) + (arrow_function body:(_)@body) + (generator_function body:(_)@body) ]@fun { ; propagate lexical scope edge @body.lexical_scope -> @fun.lexical_scope @@ -3371,8 +3375,8 @@ if none @is_def { ;;;; specified return type [ - (function return_type:(_)@return_type) - (arrow_function return_type:(_)@return_type) + (function_expression return_type:(_)@return_type) + (arrow_function return_type:(_)@return_type) ]@fun { ; propagate lexical scope edge @return_type.lexical_scope -> @fun.lexical_scope @@ -3384,8 +3388,8 @@ if none @is_def { ;;;; inferred return type [ - (function "async"?@is_async !return_type body:(_)@body) - (arrow_function "async"?@is_async !return_type body:(statement_block)@body) + (function_expression "async"?@is_async !return_type body:(_)@body) + (arrow_function "async"?@is_async !return_type body:(statement_block)@body) ]@fun { if none @is_async { ; callable is type of return statement @@ -3403,8 +3407,8 @@ if none @is_async { ;;;; inferred async return type [ - (function "async" !return_type body:(_)@body) - (arrow_function "async" body:(statement_block)@body) + (function_expression "async" !return_type body:(_)@body) + (arrow_function "async" body:(statement_block)@body) ]@fun_decl { ; function returns body return type edge @fun_decl.callable__return -> @fun_decl.async_type @@ -3523,8 +3527,8 @@ if none @is_async { ; (subscript_expression (identifier) (string)) (member_expression - object: (_)@object - property: (_)@prop + object:(_)@object + property:(_)@prop )@member_expr { node @member_expr.member node @prop.expr_ref @@ -3547,7 +3551,6 @@ if none @is_async { edge @prop.expr_ref__typeof -> @prop.expr_ref } - (subscript_expression object: (_)@object index: (_)@index @@ -3853,15 +3856,13 @@ if none @is_async { ; (sequence_expression (number) (number)) -(sequence_expression - left: (_)@left - right: (_)@right -)@sequence_expr { +(sequence_expression (_)@expr)@sequence_expr { ; propagate lexical scope - edge @left.lexical_scope -> @sequence_expr.lexical_scope - edge @right.lexical_scope -> @sequence_expr.lexical_scope + edge @expr.lexical_scope -> @sequence_expr.lexical_scope +} - ; FIXME @sequence_expr.type is type of last, but cannot express because of nesting +(sequence_expression (_)@last .)@sequence_expr { + edge @sequence_expr.type -> @last.type } @@ -4003,6 +4004,12 @@ if none @is_async { edge @class_expr.this__type_def -> @class_expr.generic_inner_type } +; decorators +(class decorator:(_)@dec)@class_expr { + ; connect lexical scope + edge @dec.lexical_scope -> @class_expr.lexical_scope +} + ; default constructor ; FIXME only if no other constructor is defined (class)@class_expr { @@ -4040,7 +4047,7 @@ if none @is_async { } - +; #dialect typescript ;; Type Assertion ; (type_assertion @@ -4055,7 +4062,7 @@ if none @is_async { ; propagate lexical scope edge @expr.lexical_scope -> @type_assert.lexical_scope } - +; #end ;; As Expression @@ -4123,16 +4130,6 @@ if none @is_async { node @pat.defs } -[ ; NOTE these are the ones not also variables - (for_in_statement ["var" "let" "const"] left:(identifier)@name) - (variable_declarator name:(identifier)@name) - (pattern/identifier)@name - (rest_pattern (identifier)@name) -] { - node @name.cotype - node @name.lexical_scope -} - [ (for_in_statement ["var" "let" "const"] left:(identifier)@name) (variable_declarator name:(identifier)@name) @@ -4534,6 +4531,12 @@ if none @is_async { ; value:(_)@value ; opt ; )@def +; decorators +(public_field_definition decorator:(_)@dec)@def { + ; connect lexical scope + edge @dec.lexical_scope -> @def.lexical_scope +} + (public_field_definition name:(_)@name )@def { @@ -4919,6 +4922,7 @@ if none @is_acc { [ (array_type) (asserts) + (asserts_annotation) (conditional_type) (constraint) (constructor_type) @@ -5034,6 +5038,9 @@ if none @is_acc { edge @type.lexical_scope -> @asserts.lexical_scope } +(asserts_annotation (_)@asserts)@asserts_annotation { + edge @asserts.lexical_scope -> @asserts_annotation.lexical_scope +} ;; Optional Type @@ -5558,8 +5565,14 @@ if none @is_acc { [ ; X (nested_type_identifier module:(identifier)@name) - ; X._, _.X._ - (nested_identifier (identifier)@name) + ; X._ + (nested_identifier object:(identifier)@name) + ; _.X._ + (member_expression object:(identifier)@name) + ; _.X + (nested_identifier property:(_)@name) + ; _._.X + (member_expression property:(_)@name) ] { node @name.type_ref attr (@name.type_ref) node_reference = @name @@ -5570,16 +5583,25 @@ if none @is_acc { edge @name.type_ref_member -> @name.type_ref } -(nested_identifier . (_)@mod) @nested { +[ + (nested_identifier object:(_)@mod) + (member_expression object:[(member_expression) (identifier)]@mod) +]@nested { node @nested.type_ref edge @mod.type_ref -> @nested.type_ref } -(nested_identifier . (_)@mod . (_)@name .) { +[ + (nested_identifier object:(_)@mod property:(_)@name) + (member_expression object:[(member_expression) (identifier)]@mod property:(_)@name) +] { edge @name.type_ref -> @mod.type_ref_member } -(nested_identifier (_)@name .) @nested { +[ + (nested_identifier property:(_)@name) + (member_expression property:(_)@name) +]@nested { node @nested.type_ref_member edge @nested.type_ref_member -> @name.type_ref_member } @@ -6122,7 +6144,7 @@ if none @is_acc { [ (arrow_function "async") - (function "async") + (function_expression "async") (function_declaration "async") (generator_function "async") (generator_function_declaration "async") @@ -6158,3 +6180,118 @@ if none @is_acc { attr (@async.async_type__promise_ref__ns) push_symbol = "%T" edge @async.async_type__promise_ref__ns -> @async.lexical_scope } + +; +; # ##### # # +; # # # # # +; # # # # +; # ##### # +; # # # # # +; # # # # # # +; ##### ##### # # +; +; ;;;;;;;;;;;;;;;;;;;; +; #dialect tsx +; ;;;;;;;;;;;;;;;;;;;; + +(jsx_element + open_tag:(_)@open_tag + close_tag:(_)@close_tag)@jsx_element { + + edge @open_tag.lexical_scope -> @jsx_element.lexical_scope + edge @close_tag.lexical_scope -> @jsx_element.lexical_scope +} + +(jsx_element + [ + (jsx_text) + (jsx_element) + (jsx_self_closing_element) + (jsx_expression) + ]@child +)@parent { + edge @child.lexical_scope -> @parent.lexical_scope +} + +(jsx_text)@jsx_text { + node @jsx_text.lexical_scope +} + +[ + (jsx_opening_element name: (_)@name)@element + (jsx_closing_element name: (_)@name)@element + (jsx_self_closing_element name: (_)@name)@element +] { + edge @name.lexical_scope -> @element.lexical_scope +} + +(jsx_opening_element + name:(_)@element_name + attribute:(_)@attr +) { + edge @attr.lexical_scope -> @element_name.lexical_scope +} + +(jsx_attribute (_) . (_)?@attr_value)@jsx_attribute { + node @jsx_attribute.lexical_scope + + if some @attr_value { + edge @attr_value.lexical_scope -> @jsx_attribute.lexical_scope + } +} + +(jsx_namespace_name (_) @lhs (_) @rhs)@name { + node @name.lexical_scope + + edge @lhs.lexical_scope -> @name.lexical_scope + edge @rhs.lexical_scope -> @name.lexical_scope +} + +(jsx_self_closing_element + name:(_)@element_name)@element { + edge @element_name.lexical_scope -> @element.lexical_scope +} + +(jsx_self_closing_element + name:(_)@element_name + attribute:(_)@attr +) { + edge @attr.lexical_scope -> @element_name.lexical_scope +} + +(jsx_expression (_)@child)@expr { + edge @child.lexical_scope -> @expr.lexical_scope +} + +(jsx_closing_element + name:(_)@element_name)@element +{ + edge @element_name.lexical_scope -> @element.lexical_scope +} + +[ + (jsx_opening_element + name:(identifier)@element_name) + (jsx_self_closing_element + name:(identifier)@element_name) + (jsx_closing_element + name:(identifier)@element_name) +] { + scan (source-text @element_name) { + ; standard HTML elements + "^(a|abbr|acronym|address|applet|area|article|aside|audio|b|base|basefont|bdi|bdo|big|blockquote|body|br|button|canvas|caption|center|cite|code|col|colgroup|data|datalist|dd|del|details|dfn|dialog|dir|div|dl|dt|em|embed|fieldset|figcaption|figure|font|footer|form|frame|frameset|h1|h2|h3|h4|h5|h6|head|header|hgroup|hr|html|i|iframe|input|ins|kbd|label|legend|li|link|main|map|mark|menu|meta|meter|nav|noframes|noscript|object|ol|optgroup|option|output|p|param|picture|pre|progress|q|rp|rt|ruby|s|samp|script|search|section|select|small|source|span|strike|strong|style|sub|summary|sup|svg|table|tbody|td|template|textarea|tfoot|th|thead|time|title|tr|track|tt|u|ul|var|video|wbr)$" { + ; do nothing! + } + + ; everything else + "^.+$" { + node element_name_pop + attr (element_name_pop) node_reference = @element_name + edge element_name_pop -> @element_name.lexical_scope + } + } +} + +; ;;;;;;;;;;;;;;;;;;;; +; #end +; ;;;;;;;;;;;;;;;;;;;; diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_core.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_core.tsx new file mode 100644 index 000000000..93285bc93 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_core.tsx @@ -0,0 +1,39 @@ +// The core of JSX tests here verify the behavior of the following node types: +// jsx_element +// jsx_identifier +// jsx_attribute +// jsx_expression +// jsx_opening_element +// jsx_closing_element +// There is no real way to avoid testing all of these at once, +// and so we don't even try to. + +let x = 1; + +// Flow In + +const el = {x}; +// ^ defined: 11 +// ^ defined: 11 + +const el2 = +// ^ defined: 11 +// ^ defined: 11 + +let y = 0; +let z = 2; + +const el = +// ^ defined: 23 + {z = 3} +// ^ defined: 24 +; + +/**/ y; +// ^ defined: 23 + +/**/ z; +// ^ defined: 24 + +/**/ x; +// ^ defined: 11 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_fragment.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_fragment.tsx new file mode 100644 index 000000000..1713694f7 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_fragment.tsx @@ -0,0 +1,12 @@ +let x = 1; + +// Flow Around + +const el = <>; + +/**/ x; +// ^ defined: 1 + +// Children +({x}); +// ^ defined: 1 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_namespace_name.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_namespace_name.tsx new file mode 100644 index 000000000..74fe0ff9b --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_namespace_name.tsx @@ -0,0 +1,30 @@ +let x = 1; + +// Flow Around +namespace noo.bar { + export let baz = 1; +} + +const el = ; +// ^ defined: 4 +// ^ defined: 4 +// ^ defined: 5 +// ^ defined: 4 +// ^ defined: 4 +// ^ defined: 5 + +/**/ x; +// ^ defined: 1 + +// Flow In + +let foo = { + bar: { + baz: 1 + } +}; + +const el2 = ; +// ^ defined: 21 +// ^ defined: 22 +// ^ defined: 23 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_self_closing_element.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_self_closing_element.tsx new file mode 100644 index 000000000..d26746137 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_self_closing_element.tsx @@ -0,0 +1,37 @@ +// The core of JSX tests here verify the behavior of the following node types: +// jsx_element +// jsx_identifier +// jsx_attribute +// jsx_expression +// jsx_opening_element +// jsx_closing_element +// There is no real way to avoid testing all of these at once, +// and so we don't even try to. + +let x = 1; + +// Flow In + +const el = ; +// ^ defined: 11 + +const el2 = +// ^ defined: 11 + +// Flow Out + +let y = 2; + +const el = ; +// ^ defined: 23 + +// Flow Across + +const el = ; +// ^ defined: 23 + +// Flow Around + +/**/ x; +// ^ defined: 11 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_text.tsx b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_text.tsx new file mode 100644 index 000000000..0cc359765 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/jsx/jsx_text.tsx @@ -0,0 +1,8 @@ +let x = 1; + +// Flow Around + +const el = bar; + +/**/ x; +// ^ defined: 1 \ No newline at end of file diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-js.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-js.ts new file mode 100644 index 000000000..92817fbaf --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-js.ts @@ -0,0 +1,6 @@ +/* --- path: src/foo.ts --- */ +export const bar = 42; + +/* --- path: src/index.ts --- */ +import { bar } from "./foo.js"; +// ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-none.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-none.ts new file mode 100644 index 000000000..f221684b4 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-none.ts @@ -0,0 +1,6 @@ +/* --- path: src/foo.ts --- */ +export const bar = 42; + +/* --- path: src/index.ts --- */ +import { bar } from "./foo"; +// ^ defined: 2 diff --git a/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-ts.ts b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-ts.ts new file mode 100644 index 000000000..1e46349c9 --- /dev/null +++ b/languages/tree-sitter-stack-graphs-typescript/test/modules/import-extension-ts.ts @@ -0,0 +1,6 @@ +/* --- path: src/foo.ts --- */ +export const bar = 42; + +/* --- path: src/index.ts --- */ +import { bar } from "./foo.ts"; +// ^ defined: 2 diff --git a/lsp-positions/CHANGELOG.md b/lsp-positions/CHANGELOG.md index 36a0d4ebd..f80bc5c5d 100644 --- a/lsp-positions/CHANGELOG.md +++ b/lsp-positions/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.3.4 -- 2024-12-12 + +Upgraded the `tree-sitter` dependency to version 0.24. + +## v0.3.3 -- 2024-03-05 + +The `tree-sitter` dependency version was updated to fix install problems. + ## v0.3.2 -- 2023-06-08 ### Added diff --git a/lsp-positions/Cargo.toml b/lsp-positions/Cargo.toml index 242a894ff..a9f7acb20 100644 --- a/lsp-positions/Cargo.toml +++ b/lsp-positions/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lsp-positions" -version = "0.3.2" +version = "0.3.4" description = "LSP-compatible character positions" homepage = "https://github.com/github/stack-graphs/tree/main/lsp-positions" repository = "https://github.com/github/stack-graphs/" @@ -22,7 +22,9 @@ tree-sitter = ["dep:tree-sitter"] [dependencies] memchr = "2.4" -tree-sitter = { version=">= 0.19", optional=true } -unicode-segmentation = { version="1.8" } -serde = { version="1", optional=true, features=["derive"] } -bincode = { version="2.0.0-rc.3", optional=true } +tree-sitter = { version = "0.24", optional = true } # keep the same minor version as the tree-sitter + # dependency of tree-sitter-stack-graphs to prevent + # install problems +unicode-segmentation = { version = "1.8" } +serde = { version = "1", features = ["derive"], optional = true } +bincode = { version = "2.0.0-rc.3", optional = true } diff --git a/script/ci-test-init b/script/ci-test-init new file mode 100755 index 000000000..0ecc60788 --- /dev/null +++ b/script/ci-test-init @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -eu + +cargo run --bin tree-sitter-stack-graphs --features cli -- init \ + --language-name InitTest \ + --language-id init_test \ + --language-file-extension it \ + --grammar-crate-name tree-sitter-python \ + --grammar-crate-version 0.23.2 \ + --internal \ + --non-interactive + +cargo check -p tree-sitter-stack-graphs-init_test --all-features + +cargo test -p tree-sitter-stack-graphs-init_test + +cargo run -p tree-sitter-stack-graphs-init_test --features cli -- -V diff --git a/script/ci-test-valgrind b/script/ci-test-valgrind new file mode 100755 index 000000000..3f5e3dae6 --- /dev/null +++ b/script/ci-test-valgrind @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -eu + +# Start by building the test binary, before we try to get its filename +cargo test "$@" --no-run + +# Cargo does not have a clean way to get the test binary, so we do some text processing here to get it +test="$(cargo test "$@" --no-run 2>&1 | grep 'Executable' | head -1 | sed -e 's/[^(]*(\(.*\)).*$/\1/')" +log="${RUNNER_TEMP-.}/valgrind.log" + +# Run the test binary under valgrind +if ! valgrind \ + --leak-check=full \ + --show-leak-kinds=all \ + --error-exitcode=1 \ + --suppressions=valgrind.supp \ + --gen-suppressions=all \ + --log-file="$log" \ + "$test"; then + echo "Valgrind detected errors! See logs: $log" + exit 1 +fi diff --git a/stack-graphs/CHANGELOG.md b/stack-graphs/CHANGELOG.md index c61221e8e..802275fc2 100644 --- a/stack-graphs/CHANGELOG.md +++ b/stack-graphs/CHANGELOG.md @@ -5,6 +5,44 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## v0.14.1 -- 2024-12-12 + +### Fixed + +- A panic when using `StackGraph::incoming_edge_degree` on some nodes without incoming edges. + +## v0.14.0 -- 2024-07-09 + +### Changed + +- The method `StackGraph::add_from_graph` returns the handles of the newly added files. + +## v0.13.0 -- 2024-03-06 + +### Added + +- New type `StitcherConfig` to specify configuration flags that control stitching. +- Path stiching statistics can now be collected during stitching for debugging purposes. Disabled by default and can be enabled with `StitcherConfig`. +- A method `StackGraph::set_edge_precedence` that allows changing the precendence of an existing edge in a stack graph. +- A method `StackGraph::incoming_edge_degree` that returns the number of edges ending in a given node. +- A method `Database::get_incoming_path_degree` that returns the number of partial paths in the database ending in a given node. +- Cycle detection improved by using incoming path or edge degrees to reduce the amount of data to keep in memory. +- Visualization uses different colors for nodes of different files, which makes understanding multi-file graphs easier. + +### Changed + +- Methods and types that do stitching in their implementation now require a `StitcherConfig` value. These include `Assertion::run`, `ForwardPartialPathStitcher::find_*`, and `Test::run`. +- The SQLite storage data encoding and queries changed to improve performance. + +### Removed + +- Method `StackGraph::remove_edge` has been removed. + +### Fixed + +- A panic when using an arena after `Arena::clear` or `SupplementalArena::clear` was called. +- Missing candidates when looking up root paths in a `Database`. + ## v0.12.0 -- 2023-07-27 ### Added diff --git a/stack-graphs/Cargo.toml b/stack-graphs/Cargo.toml index 2e55e1768..e542f4163 100644 --- a/stack-graphs/Cargo.toml +++ b/stack-graphs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stack-graphs" -version = "0.12.0" +version = "0.14.1" description = "Name binding for arbitrary programming languages" homepage = "https://github.com/github/stack-graphs/tree/main/stack-graphs" repository = "https://github.com/github/stack-graphs/" @@ -25,14 +25,14 @@ test = false [dependencies] bincode = { version = "2.0.0-rc.3", optional = true } -bitvec = "1.0" -controlled-option = "0.4" +bitvec = "1.0.1" +controlled-option = "0.4.1" either = "1.6" enumset = "1.1" fxhash = "0.2" -itertools = "0.10" +itertools = "0.10.2" libc = "0.2" -lsp-positions = { version = "0.3", path = "../lsp-positions" } +lsp-positions = { version = "0.3", path = "../lsp-positions" } # explicit version is required to be able to publish crate rusqlite = { version = "0.28", optional = true, features = ["bundled", "functions"] } serde = { version = "1.0", optional = true, features = ["derive"] } serde_json = { version = "1.0", optional = true } @@ -42,7 +42,6 @@ thiserror = { version = "1.0" } [dev-dependencies] assert-json-diff = "2" -itertools = "0.10" maplit = "1.0" pretty_assertions = "0.7" serde_json = { version = "1.0" } diff --git a/stack-graphs/README.md b/stack-graphs/README.md index 7805d19ef..69efa5bd8 100644 --- a/stack-graphs/README.md +++ b/stack-graphs/README.md @@ -9,7 +9,7 @@ To use this library, add the following to your `Cargo.toml`: ``` toml [dependencies] -stack-graphs = "0.12" +stack-graphs = "0.14" ``` Check out our [documentation](https://docs.rs/stack-graphs/) for more details on diff --git a/stack-graphs/src/graph.rs b/stack-graphs/src/graph.rs index fe1321387..d55dfc48a 100644 --- a/stack-graphs/src/graph.rs +++ b/stack-graphs/src/graph.rs @@ -1307,7 +1307,7 @@ impl StackGraph { } } - /// Removes an edge from the stack graph. + /// Sets edge precedence of the given edge. pub fn set_edge_precedence( &mut self, source: Handle, @@ -1334,7 +1334,10 @@ impl StackGraph { /// Returns the number of edges that end at a particular sink node. pub fn incoming_edge_degree(&self, sink: Handle) -> Degree { - self.incoming_edges[sink] + self.incoming_edges + .get(sink) + .cloned() + .unwrap_or(Degree::Zero) } } @@ -1497,12 +1500,16 @@ impl StackGraph { /// Copies the given stack graph into this stack graph. Panics if any of the files /// in the other stack graph are already defined in the current one. - pub fn add_from_graph(&mut self, other: &StackGraph) -> Result<(), Handle> { + pub fn add_from_graph( + &mut self, + other: &StackGraph, + ) -> Result>, Handle> { let mut files = HashMap::new(); for other_file in other.iter_files() { let file = self.add_file(other[other_file].name())?; files.insert(other_file, file); } + let files = files; let node_id = |other_node_id: NodeID| { if other_node_id.is_root() { NodeID::root() @@ -1645,7 +1652,7 @@ impl StackGraph { } } } - Ok(()) + Ok(files.into_values().collect()) } } diff --git a/stack-graphs/src/visualization/visualization.js b/stack-graphs/src/visualization/visualization.js index 116850791..6fdab1ef6 100644 --- a/stack-graphs/src/visualization/visualization.js +++ b/stack-graphs/src/visualization/visualization.js @@ -24,6 +24,7 @@ class StackGraph { this.graph = graph; this.paths = paths; + this.cleanup_data(); this.compute_data(); this.current_node = null; @@ -33,6 +34,19 @@ class StackGraph { this.render(); } + cleanup_data() { + let idx = 0; + while (idx < this.graph.edges.length) { + let edge = this.graph.edges[idx]; + if (edge.source.file === edge.sink.file && edge.source.local_id === edge.sink.local_id) { + console.log("ignoring self loop", edge); + this.graph.edges.splice(idx, 1); + } else { + idx += 1; + } + } + } + compute_data() { this.F = {}; this.ID = {}; @@ -47,7 +61,6 @@ class StackGraph { const file = graph.files[i]; this.F[file] = i; } - console.log(this.F); } compute_node_data() { diff --git a/stack-graphs/tests/it/graph.rs b/stack-graphs/tests/it/graph.rs index f19bb2c30..620d066c9 100644 --- a/stack-graphs/tests/it/graph.rs +++ b/stack-graphs/tests/it/graph.rs @@ -8,7 +8,7 @@ use std::collections::HashSet; use maplit::hashset; -use stack_graphs::graph::StackGraph; +use stack_graphs::graph::{Degree, StackGraph}; use crate::test_graphs; use crate::test_graphs::CreateStackGraph; @@ -196,3 +196,22 @@ fn can_add_graph_to_empty_graph() { ); } } + +#[test] +fn can_get_incoming_edges() { + let mut graph = StackGraph::new(); + let file = graph.get_or_create_file("test.py"); + let h1 = graph.internal_scope(file, 0); + let h2 = graph.internal_scope(file, 1); + let h3 = graph.internal_scope(file, 2); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h1)); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h2)); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h3)); + graph.add_edge(h1, h2, 0); + graph.add_edge(h3, h2, 0); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h1)); + assert_eq!(Degree::Multiple, graph.incoming_edge_degree(h2)); + assert_eq!(Degree::Zero, graph.incoming_edge_degree(h3)); + graph.add_edge(h3, h1, 0); + assert_eq!(Degree::One, graph.incoming_edge_degree(h1)); +} diff --git a/tree-sitter-stack-graphs/CHANGELOG.md b/tree-sitter-stack-graphs/CHANGELOG.md index 8d8ef7ca1..e1df552e4 100644 --- a/tree-sitter-stack-graphs/CHANGELOG.md +++ b/tree-sitter-stack-graphs/CHANGELOG.md @@ -5,7 +5,39 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## v0.10.0 -- 2024-12-12 + +Upgraded `tree-sitter` dependency to version 0.24. + +## v0.9.0 -- 2024-07-09 + +### Library + +- New crate-level constants `FILE_PATH_VAR` and `ROOT_PATH_VAR` standardize the TSG global variable names to use for the file and root path. +- The file path variable will only use the filename set in the stack graph if no value was explicitly set. + +### CLI + +#### Added + +- Tests run faster for languages with builtins sources by caching the partial paths for the builtins. +- Indexing will set a value for the root path variable that is passed to TSG. The value is based on the directory that was provided on the command line. + +#### Changed + +- Failure to index a file will not abort indexing anymore, but simply mark the file as failed, as we already do for files with parse errors. + +#### Removed + +- The NPM distribution has been removed. + +## v0.8.1 -- 2024-03-06 + +The `stack-graphs` dependency was updated to `v0.13` to fix the build problems of the `v0.8.0` release. + +## v0.8.0 -- 2024-03-05 + +The `tree-sitter` dependency version was updated to fix install problems. ### Library diff --git a/tree-sitter-stack-graphs/Cargo.toml b/tree-sitter-stack-graphs/Cargo.toml index 2de26a86f..3c1c01a80 100644 --- a/tree-sitter-stack-graphs/Cargo.toml +++ b/tree-sitter-stack-graphs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tree-sitter-stack-graphs" -version = "0.7.1" +version = "0.10.0" description = "Create stack graphs using tree-sitter parsers" homepage = "https://github.com/github/stack-graphs/tree/main/tree-sitter-stack-graphs" repository = "https://github.com/github/stack-graphs/" @@ -49,12 +49,12 @@ lsp = [ ] [dependencies] -anyhow = "1.0" +anyhow = "1.0.4" base64 = { version = "0.21", optional = true } capture-it = { version = "0.3", optional = true } clap = { version = "4", optional = true, features = ["derive"] } colored = { version = "2.0", optional = true } -controlled-option = ">=0.4" +controlled-option = "0.4.1" crossbeam-channel = { version = "0.5", optional = true } dialoguer = { version = "0.10", optional = true } dirs = { version = "5", optional = true } @@ -62,24 +62,25 @@ env_logger = { version = "0.9", optional = true } indoc = { version = "1.0", optional = true } itertools = "0.10" log = "0.4" -lsp-positions = { version="0.3", path="../lsp-positions", features=["tree-sitter"] } +lsp-positions = { version="0.3.4", path="../lsp-positions", features=["tree-sitter"] } # explicit version is required to be able to publish crate once_cell = "1" pathdiff = { version = "0.2.1", optional = true } regex = "1" rust-ini = "0.18" serde_json = { version="1.0", optional=true } sha1 = { version="0.10", optional=true } -stack-graphs = { version=">=0.11, <=0.12", path="../stack-graphs" } +stack-graphs = { version = "0.14", path="../stack-graphs" } # explicit version is required to be able to publish crate thiserror = "1.0" time = { version = "0.3", optional = true } tokio = { version = "1.26", optional = true, features = ["io-std", "rt", "rt-multi-thread"] } tower-lsp = { version = "0.19", optional = true } -tree-sitter = ">= 0.19" -tree-sitter-config = { version = "0.19", optional = true } -tree-sitter-graph = "0.11" -tree-sitter-loader = "0.20" +tree-sitter = "0.24" # keep the same minor version as the tree-sitter dependency + # of tree-sitter-graph to prevent install problems +tree-sitter-config = { version = "0.24", optional = true } +tree-sitter-graph = "0.12" +tree-sitter-loader = "0.24" walkdir = { version = "2.3", optional = true } [dev-dependencies] pretty_assertions = "0.7" -tree-sitter-python = "0.19.1" +tree-sitter-python = "=0.23.5" diff --git a/tree-sitter-stack-graphs/README.md b/tree-sitter-stack-graphs/README.md index 01ce41971..56f61e131 100644 --- a/tree-sitter-stack-graphs/README.md +++ b/tree-sitter-stack-graphs/README.md @@ -1,7 +1,6 @@ # tree-sitter-stack-graphs -The `tree-sitter-stack-graphs` crate lets you create stack graphs using the -[tree-sitter][] grammar for a language. +The `tree-sitter-stack-graphs` crate lets you create stack graphs using the [tree-sitter][] grammar for a language. [tree-sitter]: https://tree-sitter.github.io/ @@ -9,50 +8,82 @@ The `tree-sitter-stack-graphs` crate lets you create stack graphs using the - [Examples](https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/) - [Release notes](https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/CHANGELOG.md) -## Usage +## Using the API To use this library, add the following to your `Cargo.toml`: -``` toml +```toml [dependencies] -tree-sitter-stack-graphs = "0.7" +tree-sitter-stack-graphs = "0.10" ``` -Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs/*/) for -more details on how to use this library. +Check out our [documentation](https://docs.rs/tree-sitter-stack-graphs/*/) for more details on how to use this library. -## Command-line Program +## Using the Command-line Program -The command-line program for `tree-sitter-stack-graphs` lets you do stack -graph based analysis and lookup from the command line. +The command-line program for `tree-sitter-stack-graphs` lets you do stack graph based analysis and lookup from the command line. -Install the program using `cargo install` as follows: +The CLI can be run as follows: -``` sh -$ cargo install --features cli tree-sitter-stack-graphs -$ tree-sitter-stack-graphs --help -``` +1. _(Installed)_ Install the CLI using Cargo as follows: -Alternatively, the program can be invoked via NPM as follows: + ```sh + cargo install --features cli tree-sitter-stack-graphs + ``` -``` sh -$ npx tree-sitter-stack-graphs -``` + After this, the CLI should be available as `tree-sitter-stack-graphs`. + +2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `tree-sitter-stack-graphs` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + +The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + +1. Index a source folder as follows: + + ```sh + tree-sitter-stack-graphs index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + tree-sitter-stack-graphs status SOURCE_DIR + ``` -## Getting Started + To clean the database and start with a clean slate, run: -Starting a new project to develop stack graph definitions for your favourite language -is as easy as running the `init` command: + ```sh + tree-sitter-stack-graphs clean + ``` -``` sh -$ tree-sitter-stack-graphs init PROJECT_DIR + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + +2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + tree-sitter-stack-graphs query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + +Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. + +## Getting Started on a new Language + +Starting a new project to develop stack graph definitions for your favourite language is as easy as running the `init` command: + +```sh +tree-sitter-stack-graphs init PROJECT_DIR ``` -Answer the questions to provide information about the language, the grammar dependency, and -the project and hit `Generate` to generate the new project. Check out `PROJECT_DIR/README.md` -to find out how to start developing. +Answer the questions to provide information about the language, the grammar dependency, and the project and hit `Generate` to generate the new project. Check out `PROJECT_DIR/README.md` to find out how to start developing. -Also check out our [examples][] for more details on how to work with the CLI. +Check out [examples][] of stack graph rules for typical language features. [examples]: https://github.com/github/stack-graphs/blob/main/tree-sitter-stack-graphs/examples/ @@ -65,14 +96,14 @@ Rust can be installed and updated using [rustup][]. Build the project by running: -``` -$ cargo build +```sh +cargo build ``` Run the tests by running: -``` -$ cargo test +```sh +cargo test ``` The project consists of a library and a CLI. @@ -81,22 +112,22 @@ To run `cargo` commands on the CLI as well, add `--features cli` or `--all-featu Run the CLI from source as follows: -``` sh -$ cargo run --features cli -- ARGS +```sh +cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: -``` -$ cargo fmt +```sh +cargo fmt ``` ## License Licensed under either of - - [Apache License, Version 2.0][apache] ([LICENSE-APACHE](LICENSE-APACHE)) - - [MIT license][mit] ([LICENSE-MIT](LICENSE-MIT)) +- [Apache License, Version 2.0][apache] ([LICENSE-APACHE](LICENSE-APACHE)) +- [MIT license][mit] ([LICENSE-MIT](LICENSE-MIT)) at your option. diff --git a/tree-sitter-stack-graphs/examples/README.md b/tree-sitter-stack-graphs/examples/README.md index 901616422..5d5186ad5 100644 --- a/tree-sitter-stack-graphs/examples/README.md +++ b/tree-sitter-stack-graphs/examples/README.md @@ -7,19 +7,19 @@ Each directory contains a `stack-graphs.tsg` file that describes at the top what Running the examples requires the Python grammar to be available. This can be installed (in this directory) by executing: ```bash -$ ./bootstrap +./bootstrap ``` Run the tests for an example by executing: ```bash -$ ./run EXAMPLE_DIR +./run EXAMPLE_DIR ``` or, from within the example's directory: ```bash -$ ../run +../run ``` To render HTML visualizations of the stack graphs for the tests in an example, add the `-V` flag to run. @@ -27,7 +27,7 @@ To render HTML visualizations of the stack graphs for the tests in an example, a Print the parse tree of an example file by executing: ```bash -$ ./parse EXAMPLE_FILE +./parse EXAMPLE_FILE ``` The following examples are available: diff --git a/tree-sitter-stack-graphs/npm/.gitignore b/tree-sitter-stack-graphs/npm/.gitignore deleted file mode 100644 index 02b3615e1..000000000 --- a/tree-sitter-stack-graphs/npm/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.crates.toml -/.crates2.json -/bin/ -/package-lock.json diff --git a/tree-sitter-stack-graphs/npm/README.md b/tree-sitter-stack-graphs/npm/README.md deleted file mode 100644 index 27dd321be..000000000 --- a/tree-sitter-stack-graphs/npm/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# tree-sitter-stack-graphs - -This package provides a convenient way to install the [tree-sitter-stack-graphs](https://crates.io/crates/tree-sitter-stack-graphs) CLI in an NPM project. - -Add it as a dev dependency to an existing project using: - - npm i -D tree-sitter-stack-graphs - -It is also possible to invoke it directly using: - - npx tree-sitter-stack-graphs - -See the tree-sitter-stack-graphs [documentation](https://crates.io/crates/tree-sitter-stack-graphs) for details on usage. diff --git a/tree-sitter-stack-graphs/npm/cli.js b/tree-sitter-stack-graphs/npm/cli.js deleted file mode 100755 index f4d654e15..000000000 --- a/tree-sitter-stack-graphs/npm/cli.js +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env node - -const spawn = require("child_process").spawn; -const path = require("path"); - -const tssg = process.platform === "win32" - ? "tree-sitter-stack-graphs.exe" - : "tree-sitter-stack-graphs"; - -spawn( - path.join(__dirname, "bin", tssg), process.argv.slice(2), - { - "stdio": "inherit" - }, -).on('close', process.exit); diff --git a/tree-sitter-stack-graphs/npm/install.js b/tree-sitter-stack-graphs/npm/install.js deleted file mode 100644 index 6ac20756b..000000000 --- a/tree-sitter-stack-graphs/npm/install.js +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env node - -const child_process = require("child_process"); -const packageJSON = require("./package.json"); - -const cargo = process.platform === "win32" - ? "cargo.exe" - : "cargo"; - -try { - child_process.execSync(cargo); -} catch (error) { - console.error(error.message); - console.error("Failed to execute Cargo. Cargo needs to be available to install this package!"); - process.exit(1); -} - -child_process.spawn( - cargo, [ - "install", - "--quiet", - "--root", ".", - "--version", "^"+packageJSON.version, - "--features", "cli", - packageJSON.name, - ], - { - "stdio": "inherit" - }, -).on('close', process.exit); diff --git a/tree-sitter-stack-graphs/npm/package.json b/tree-sitter-stack-graphs/npm/package.json deleted file mode 100644 index a49ec5fad..000000000 --- a/tree-sitter-stack-graphs/npm/package.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "name": "tree-sitter-stack-graphs", - "version": "0.7.0", - "description": "Create stack graphs using tree-sitter parsers", - "homepage": "https://github.com/github/stack-graphs/tree/main/tree-sitter-stack-graphs", - "repository": { - "type": "git", - "url": "https://github.com/github/stack-graphs.git" - }, - "keywords": [ - "tree-sitter", - "stack-graphs" - ], - "license": "MIT OR Apache-2.0", - "author": "GitHub ", - "contributors": [ - "Douglas Creager ", - "Hendrik van Antwerpen " - ], - "bin": { - "tree-sitter-stack-graphs": "./cli.js" - }, - "scripts": { - "install": "node install.js" - } -} diff --git a/tree-sitter-stack-graphs/src/cli/index.rs b/tree-sitter-stack-graphs/src/cli/index.rs index 6d192e382..6425d8ca7 100644 --- a/tree-sitter-stack-graphs/src/cli/index.rs +++ b/tree-sitter-stack-graphs/src/cli/index.rs @@ -42,6 +42,7 @@ use crate::BuildError; use crate::CancelAfterDuration; use crate::CancellationFlag; use crate::NoCancellation; +use crate::{FILE_PATH_VAR, ROOT_PATH_VAR}; #[derive(Args)] pub struct IndexArgs { @@ -355,23 +356,15 @@ impl<'a> Indexer<'a> { if let Err(err) = result { match err.inner { BuildError::Cancelled(_) => { - file_status.warning("parsing timed out", None); + file_status.warning("timed out", None); self.db - .store_error_for_file(source_path, &tag, "parsing timed out")?; - return Ok(()); - } - BuildError::ParseErrors { .. } => { - file_status.failure("parsing failed", Some(&err.display_pretty())); - self.db.store_error_for_file( - source_path, - &tag, - &format!("parsing failed: {}", err.inner), - )?; + .store_error_for_file(source_path, &tag, "timed out")?; return Ok(()); } _ => { - file_status.failure("failed to build stack graph", Some(&err.display_pretty())); - return Err(IndexError::StackGraph); + file_status.failure("failed", Some(&err.display_pretty())); + self.db.store_error_for_file(source_path, &tag, "failed")?; + return Ok(()); } } }; @@ -437,7 +430,16 @@ impl<'a> Indexer<'a> { ) -> std::result::Result<(), BuildErrorWithSource<'b>> { let relative_source_path = source_path.strip_prefix(source_root).unwrap(); if let Some(lc) = lcs.primary { - let globals = Variables::new(); + let mut globals = Variables::new(); + + globals + .add(FILE_PATH_VAR.into(), source_path.to_str().unwrap().into()) + .expect("failed to add file path variable"); + + globals + .add(ROOT_PATH_VAR.into(), source_root.to_str().unwrap().into()) + .expect("failed to add root path variable"); + lc.sgl .build_stack_graph_into(graph, file, source, &globals, cancellation_flag) .map_err(|inner| BuildErrorWithSource { diff --git a/tree-sitter-stack-graphs/src/cli/init.rs b/tree-sitter-stack-graphs/src/cli/init.rs index e381786b9..e513e5acd 100644 --- a/tree-sitter-stack-graphs/src/cli/init.rs +++ b/tree-sitter-stack-graphs/src/cli/init.rs @@ -455,33 +455,79 @@ impl ProjectSettings<'_> { fn generate_readme(&self, project_path: &Path) -> anyhow::Result<()> { let mut file = File::create(project_path.join("README.md"))?; writedoc! {file, r####" - # tree-sitter-stack-graphs definition for {} + # tree-sitter-stack-graphs definition for {language_name} - This project defines tree-sitter-stack-graphs rules for {} using the [{}][] grammar. + This project defines tree-sitter-stack-graphs rules for {language_name} using the [{grammar_crate_name}][] grammar. - [{}]: https://crates.io/crates/{} + [{grammar_crate_name}]: https://crates.io/crates/{grammar_crate_name} - ## Usage + - [API documentation](https://docs.rs/{crate_name}/) + - [Release notes](https://github.com/github/stack-graphs/blob/main/languages/{crate_name}/CHANGELOG.md) + + ## Using the API To use this library, add the following to your `Cargo.toml`: - ``` toml + ```toml [dependencies] - {} = "{}" + {crate_name} = "{crate_version}" ``` - Check out our [documentation](https://docs.rs/{}/*/) for more details on how to use this library. + Check out our [documentation](https://docs.rs/{crate_name}/*/) for more details on how to use this library. - ## Command-line Program + ## Using the Command-line Program - The command-line program for `{}` lets you do stack graph based analysis and lookup from the command line. + The command-line program for `{crate_name}` lets you do stack graph based analysis and lookup from the command line. - Install the program using `cargo install` as follows: + The CLI can be run as follows: - ``` sh - $ cargo install --features cli {} - $ {} --help - ``` + 1. _(Installed)_ Install the CLI using Cargo as follows: + + ```sh + cargo install --features cli {crate_name} + ``` + + After this, the CLI should be available as `{crate_name}`. + + 2. _(From source)_ Instead of installing the CLI, it can also be run directly from the crate directory, as a replacement for a `{crate_name}` invocation, as follows: + + ```sh + cargo run --features cli -- + ``` + + The basic CLI workflow for the command-line program is to index source code and issue queries against the resulting database: + + 1. Index a source folder as follows: + + ```sh + {crate_name} index SOURCE_DIR + ``` + + _Indexing will skip any files that have already be indexed. To force a re-index, add the `-f` flag._ + + To check the status if a source folder, run: + + ```sh + {crate_name} status SOURCE_DIR + ``` + + To clean the database and start with a clean slate, run: + + ```sh + {crate_name} clean + ``` + + _Pass the `--delete` flag to not just empty the database, but also delete it. This is useful to resolve `unsupported database version` errors that may occur after a version update._ + + 2. Run a query to find the definition(s) for a reference on a given line and column, run: + + ```sh + {crate_name} query definition SOURCE_PATH:LINE:COLUMN + ``` + + Resulting definitions are printed, including a source line if the source file is available. + + Discover all available commands and flags by passing the `-h` flag to the CLI directly, or to any of the subcommands. ## Development @@ -492,35 +538,29 @@ impl ProjectSettings<'_> { The project is organized as follows: - The stack graph rules are defined in `src/stack-graphs.tsg`. - - Builtins sources and configuration are defined in `src/builtins.{}` and `builtins.cfg` respectively. + - Builtins sources and configuration are defined in `src/builtins.{language_file_extension}` and `builtins.cfg` respectively. - Tests are put into the `test` directory. - ### Building and Running Tests - - Build the project by running: - - ``` sh - $ cargo build - ``` + ### Running Tests Run the tests as follows: - ``` sh - $ cargo test + ```sh + cargo test ``` The project consists of a library and a CLI. By default, running `cargo` only applies to the library. To run `cargo` commands on the CLI as well, add `--features cli` or `--all-features`. Run the CLI from source as follows: - ``` sh - $ cargo run --features cli -- ARGS + ```sh + cargo run --features cli -- ARGS ``` Sources are formatted using the standard Rust formatted, which is applied by running: - ``` sh - $ cargo fmt + ```sh + cargo fmt ``` ### Writing TSG @@ -535,34 +575,30 @@ impl ProjectSettings<'_> { Parse and test a single file by executing the following commands: - ``` sh - $ cargo run --features cli -- parse FILES... - $ cargo run --features cli -- test TESTFILES... + ```sh + cargo run --features cli -- parse FILES... + cargo run --features cli -- test TESTFILES... ``` Generate a visualization to debug failing tests by passing the `-V` flag: - ``` sh - $ cargo run --features cli -- test -V TESTFILES... + ```sh + cargo run --features cli -- test -V TESTFILES... ``` To generate the visualization regardless of test outcome, execute: - ``` sh - $ cargo run --features cli -- test -V --output-mode=always TESTFILES... + ```sh + cargo run --features cli -- test -V --output-mode=always TESTFILES... ``` - Go to https://crates.io/crates/tree-sitter-stack-graphs for links to examples and documentation. + Go to for links to examples and documentation. "####, - self.language_name, - self.language_name, self.grammar_crate_name(), - self.grammar_crate_name(), self.grammar_crate_name(), - self.crate_name(), self.crate_version(), - self.crate_name(), - self.crate_name(), - self.crate_name(), - self.crate_name(), - self.language_file_extension, + language_name=self.language_name, + grammar_crate_name=self.grammar_crate_name(), + crate_name=self.crate_name(), + crate_version=self.crate_version(), + language_file_extension=self.language_file_extension, }?; Ok(()) } @@ -734,9 +770,6 @@ impl ProjectSettings<'_> { /// The stack graphs builtins source for this language. pub const STACK_GRAPHS_BUILTINS_SOURCE: &str = include_str!("../src/builtins.{}"); - /// The name of the file path global variable. - pub const FILE_PATH_VAR: &str = "FILE_PATH"; - pub fn language_configuration(cancellation_flag: &dyn CancellationFlag) -> LanguageConfiguration {{ try_language_configuration(cancellation_flag).unwrap_or_else(|err| panic!("{{}}", err)) }} @@ -745,7 +778,7 @@ impl ProjectSettings<'_> { cancellation_flag: &dyn CancellationFlag, ) -> Result {{ LanguageConfiguration::from_sources( - {}::language(), + {}::LANGUAGE.into(), Some(String::from("source.{}")), None, vec![String::from("{}")], diff --git a/tree-sitter-stack-graphs/src/cli/load.rs b/tree-sitter-stack-graphs/src/cli/load.rs index b32332f9d..1fd22fa2b 100644 --- a/tree-sitter-stack-graphs/src/cli/load.rs +++ b/tree-sitter-stack-graphs/src/cli/load.rs @@ -67,7 +67,7 @@ impl PathLoaderArgs { builtins_paths, )? } else { - let loader_config = TsConfig::load() + let loader_config = TsConfig::load(None) .and_then(|v| v.get()) .map_err(LoadError::TreeSitter)?; Loader::from_tree_sitter_configuration( diff --git a/tree-sitter-stack-graphs/src/cli/lsp.rs b/tree-sitter-stack-graphs/src/cli/lsp.rs index 03f1c8060..57ed770bb 100644 --- a/tree-sitter-stack-graphs/src/cli/lsp.rs +++ b/tree-sitter-stack-graphs/src/cli/lsp.rs @@ -560,25 +560,6 @@ impl FromStdError for std::result::Result { } } -trait FromAnyhowError { - #[must_use] - fn from_error(self) -> Result; -} - -impl FromAnyhowError for std::result::Result { - #[must_use] - fn from_error(self) -> Result { - match self { - Ok(value) => Ok(value), - Err(err) => Err(Error { - code: ErrorCode::ServerError(-1), - message: err.to_string(), - data: None, - }), - } - } -} - #[derive(Debug)] pub enum Job { IndexPath(PathBuf), diff --git a/tree-sitter-stack-graphs/src/cli/match.rs b/tree-sitter-stack-graphs/src/cli/match.rs index 108c8450e..889d2374a 100644 --- a/tree-sitter-stack-graphs/src/cli/match.rs +++ b/tree-sitter-stack-graphs/src/cli/match.rs @@ -49,7 +49,7 @@ impl MatchArgs { None => return Err(anyhow!("No stack graph language found")), }; let source = file_reader.get(&self.source_path)?; - let tree = parse(lc.language, &self.source_path, source)?; + let tree = parse(&lc.language, &self.source_path, source)?; if self.stanza.is_empty() { lc.sgl.tsg.try_visit_matches(&tree, source, true, |mat| { print_matches(lc.sgl.tsg_path(), &self.source_path, source, mat) diff --git a/tree-sitter-stack-graphs/src/cli/parse.rs b/tree-sitter-stack-graphs/src/cli/parse.rs index aff7681b1..64275b915 100644 --- a/tree-sitter-stack-graphs/src/cli/parse.rs +++ b/tree-sitter-stack-graphs/src/cli/parse.rs @@ -40,14 +40,14 @@ impl ParseArgs { None => return Err(anyhow!("No stack graph language found")), }; let source = file_reader.get(&self.source_path)?; - let tree = parse(lang, &self.source_path, source)?; + let tree = parse(&lang, &self.source_path, source)?; print_tree(tree); Ok(()) } } pub(super) fn parse( - language: tree_sitter::Language, + language: &tree_sitter::Language, path: &Path, source: &str, ) -> anyhow::Result { diff --git a/tree-sitter-stack-graphs/src/cli/test.rs b/tree-sitter-stack-graphs/src/cli/test.rs index 780d478f6..e3034cbbf 100644 --- a/tree-sitter-stack-graphs/src/cli/test.rs +++ b/tree-sitter-stack-graphs/src/cli/test.rs @@ -19,9 +19,13 @@ use stack_graphs::stitching::Database; use stack_graphs::stitching::DatabaseCandidates; use stack_graphs::stitching::ForwardPartialPathStitcher; use stack_graphs::stitching::StitcherConfig; +use std::collections::hash_map::Entry; +use std::collections::HashMap; +use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; use std::time::Duration; +use tree_sitter::Language; use tree_sitter_graph::Variables; use crate::cli::util::duration_from_seconds_str; @@ -39,6 +43,7 @@ use crate::test::Test; use crate::test::TestResult; use crate::CancelAfterDuration; use crate::CancellationFlag; +use crate::FILE_PATH_VAR; #[derive(Args)] #[clap(after_help = r#"PATH SPECIFICATIONS: @@ -176,10 +181,16 @@ impl TestArgs { pub fn run(self, mut loader: Loader) -> anyhow::Result<()> { let reporter = self.get_reporter(); let mut total_result = TestResult::new(); + let mut cache = HashMap::new(); for (test_root, test_path, _) in iter_files_and_directories(self.test_paths.clone()) { let mut file_status = CLIFileReporter::new(&reporter, &test_path); - let test_result = - self.run_test(&test_root, &test_path, &mut loader, &mut file_status)?; + let test_result = self.run_test( + &test_root, + &test_path, + &mut loader, + &mut file_status, + &mut cache, + )?; file_status.assert_reported(); total_result.absorb(test_result); } @@ -211,14 +222,15 @@ impl TestArgs { } /// Run test file. Takes care of the output when an error is returned. - fn run_test( + fn run_test<'a>( &self, test_root: &Path, test_path: &Path, - loader: &mut Loader, + loader: &'a mut Loader, file_status: &mut CLIFileReporter, + cache: &mut HashMap, ) -> anyhow::Result { - match self.run_test_inner(test_root, test_path, loader, file_status) { + match self.run_test_inner(test_root, test_path, loader, file_status, cache) { ok @ Ok(_) => ok, err @ Err(_) => { file_status.failure_if_processing("error", None); @@ -227,12 +239,13 @@ impl TestArgs { } } - fn run_test_inner( + fn run_test_inner<'a>( &self, test_root: &Path, test_path: &Path, - loader: &mut Loader, + loader: &'a mut Loader, file_status: &mut CLIFileReporter, + cache: &mut HashMap, ) -> anyhow::Result { let cancellation_flag = CancelAfterDuration::from_option(self.max_test_time); @@ -263,11 +276,24 @@ impl TestArgs { file_status.processing(); + let stitcher_config = + StitcherConfig::default().with_detect_similar_paths(!lc.no_similar_paths_in_file); + let mut partials = PartialPaths::new(); + let mut db = Database::new(); + let source = file_reader.get(test_path)?; let default_fragment_path = test_path.strip_prefix(test_root).unwrap(); let mut test = Test::from_source(test_path, source, default_fragment_path)?; if !self.no_builtins { - self.load_builtins_into(&lc, &mut test.graph)?; + self.load_builtins_into( + &lc, + &mut test.graph, + &mut partials, + &mut db, + stitcher_config, + cancellation_flag.as_ref(), + cache, + )?; } let mut globals = Variables::new(); for test_fragment in &test.fragments { @@ -291,7 +317,16 @@ impl TestArgs { &mut Some(test_fragment.source.as_ref()), )? { globals.clear(); + test_fragment.add_globals_to(&mut globals); + + globals + .add( + FILE_PATH_VAR.into(), + test_fragment.path.to_str().unwrap().into(), + ) + .unwrap_or_default(); + lc.sgl.build_stack_graph_into( &mut test.graph, test_fragment.file, @@ -325,15 +360,11 @@ impl TestArgs { Ok(_) => {} } } - let stitcher_config = - StitcherConfig::default().with_detect_similar_paths(!lc.no_similar_paths_in_file); - let mut partials = PartialPaths::new(); - let mut db = Database::new(); - for file in test.graph.iter_files() { + for fragment in &test.fragments { ForwardPartialPathStitcher::find_minimal_partial_path_set_in_file( &test.graph, &mut partials, - file, + fragment.file, stitcher_config, &cancellation_flag.as_ref(), |g, ps, p| { @@ -387,14 +418,45 @@ impl TestArgs { Ok(result) } - fn load_builtins_into( + fn load_builtins_into<'a>( &self, - lc: &LanguageConfiguration, + lc: &'a LanguageConfiguration, graph: &mut StackGraph, + partials: &mut PartialPaths, + db: &mut Database, + stitcher_config: StitcherConfig, + cancellation_flag: &dyn CancellationFlag, + cache: &mut HashMap, ) -> anyhow::Result<()> { - if let Err(h) = graph.add_from_graph(&lc.builtins) { - return Err(anyhow!("Duplicate builtin file {}", &graph[h])); - } + let files = graph + .add_from_graph(&lc.builtins) + .map_err(|h| anyhow!("Duplicate builtin file {}", &graph[h]))?; + let files = files.into_iter().collect::>(); + match cache.entry(lc.language.clone()) { + Entry::Occupied(o) => { + o.get().load_into(graph, partials, db)?; + } + Entry::Vacant(v) => { + for file in &files { + ForwardPartialPathStitcher::find_minimal_partial_path_set_in_file( + graph, + partials, + *file, + stitcher_config, + &cancellation_flag, + |g, ps, p| { + db.add_partial_path(g, ps, p.clone()); + }, + )?; + } + v.insert(db.to_serializable_filter( + graph, + partials, + &|_: &StackGraph, f: &Handle| files.contains(f), + )); + } + }; + Ok(()) } diff --git a/tree-sitter-stack-graphs/src/lib.rs b/tree-sitter-stack-graphs/src/lib.rs index 86651b6be..6319d1cb9 100644 --- a/tree-sitter-stack-graphs/src/lib.rs +++ b/tree-sitter-stack-graphs/src/lib.rs @@ -324,7 +324,7 @@ //! import sys //! print(sys.path) //! "#; -//! let grammar = tree_sitter_python::language(); +//! let grammar = tree_sitter_python::LANGUAGE.into(); //! let tsg_source = STACK_GRAPH_RULES; //! let mut language = StackGraphLanguage::from_str(grammar, tsg_source)?; //! let mut stack_graph = StackGraph::new(); @@ -435,9 +435,16 @@ static SCOPE_ATTRS: Lazy> = static PRECEDENCE_ATTR: &'static str = "precedence"; // Global variables -static ROOT_NODE_VAR: &'static str = "ROOT_NODE"; -static JUMP_TO_SCOPE_NODE_VAR: &'static str = "JUMP_TO_SCOPE_NODE"; -static FILE_PATH_VAR: &'static str = "FILE_PATH"; +/// Name of the variable used to pass the root node. +pub const ROOT_NODE_VAR: &'static str = "ROOT_NODE"; +/// Name of the variable used to pass the jump-to-scope node. +pub const JUMP_TO_SCOPE_NODE_VAR: &'static str = "JUMP_TO_SCOPE_NODE"; +/// Name of the variable used to pass the file path. +/// If a root path is given, it should be a descendant of the root path. +pub const FILE_PATH_VAR: &'static str = "FILE_PATH"; +/// Name of the variable used to pass the root path. +/// If given, should be an ancestor of the file path. +pub const ROOT_PATH_VAR: &'static str = "ROOT_PATH"; /// Holds information about how to construct stack graphs for a particular language. pub struct StackGraphLanguage { @@ -472,7 +479,7 @@ impl StackGraphLanguage { language: tree_sitter::Language, tsg_source: &str, ) -> Result { - let tsg = tree_sitter_graph::ast::File::from_str(language, tsg_source)?; + let tsg = tree_sitter_graph::ast::File::from_str(language.clone(), tsg_source)?; Ok(StackGraphLanguage { language, tsg, @@ -511,8 +518,8 @@ impl StackGraphLanguage { &mut self.functions } - pub fn language(&self) -> tree_sitter::Language { - self.language + pub fn language(&self) -> &tree_sitter::Language { + &self.language } /// Returns the original TSG path, if it was provided at construction or set with @@ -617,7 +624,7 @@ impl<'a> Builder<'a> { ) -> Result<(), BuildError> { let tree = { let mut parser = Parser::new(); - parser.set_language(self.sgl.language)?; + parser.set_language(&self.sgl.language)?; let ts_cancellation_flag = TreeSitterCancellationFlag::from(cancellation_flag); // The parser.set_cancellation_flag` is unsafe, because it does not tie the // lifetime of the parser to the lifetime of the cancellation flag in any way. @@ -635,16 +642,17 @@ impl<'a> Builder<'a> { let tree = parse_errors.into_tree(); let mut globals = Variables::nested(globals); - if globals.get(&ROOT_NODE_VAR.into()).is_none() { - let root_node = self.inject_node(NodeID::root()); - globals - .add(ROOT_NODE_VAR.into(), root_node.into()) - .expect("Failed to set ROOT_NODE"); - } + + let root_node = self.inject_node(NodeID::root()); + globals + .add(ROOT_NODE_VAR.into(), root_node.into()) + .unwrap_or_default(); + let jump_to_scope_node = self.inject_node(NodeID::jump_to()); globals .add(JUMP_TO_SCOPE_NODE_VAR.into(), jump_to_scope_node.into()) .expect("Failed to set JUMP_TO_SCOPE_NODE"); + if globals.get(&FILE_PATH_VAR.into()).is_none() { let file_name = self.stack_graph[self.file].to_string(); globals diff --git a/tree-sitter-stack-graphs/src/loader.rs b/tree-sitter-stack-graphs/src/loader.rs index 9ffc1d674..861e4ff95 100644 --- a/tree-sitter-stack-graphs/src/loader.rs +++ b/tree-sitter-stack-graphs/src/loader.rs @@ -29,6 +29,9 @@ use tree_sitter_loader::Loader as TsLoader; use crate::CancellationFlag; use crate::FileAnalyzer; use crate::StackGraphLanguage; +use crate::FILE_PATH_VAR; + +const BUILTINS_FILENAME: &str = ""; pub static DEFAULT_TSG_PATHS: Lazy> = Lazy::new(|| vec![LoadPath::Grammar("queries/stack-graphs".into())]); @@ -65,20 +68,25 @@ impl LanguageConfiguration { builtins_config: Option<&str>, cancellation_flag: &dyn CancellationFlag, ) -> Result> { - let sgl = StackGraphLanguage::from_source(language, tsg_path.clone(), tsg_source).map_err( - |err| LoadError::SglParse { - inner: err, - tsg_path, - tsg: Cow::from(tsg_source), - }, - )?; + let sgl = StackGraphLanguage::from_source(language.clone(), tsg_path.clone(), tsg_source) + .map_err(|err| LoadError::SglParse { + inner: err, + tsg_path, + tsg: Cow::from(tsg_source), + })?; let mut builtins = StackGraph::new(); if let Some((builtins_path, builtins_source)) = builtins_source { let mut builtins_globals = Variables::new(); + if let Some(builtins_config) = builtins_config { Loader::load_globals_from_config_str(builtins_config, &mut builtins_globals)?; } - let file = builtins.add_file("").unwrap(); + + builtins_globals + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) + .unwrap_or_default(); + + let file = builtins.add_file(BUILTINS_FILENAME).unwrap(); sgl.build_stack_graph_into( &mut builtins, file, @@ -264,7 +272,7 @@ impl Loader { &mut self, path: &Path, content: &mut dyn ContentProvider, - ) -> Result, LoadError<'static>> { + ) -> Result, LoadError<'static>> { match &mut self.0 { LoaderImpl::Paths(loader) => loader.load_tree_sitter_language_for_file(path, content), LoaderImpl::Provided(loader) => { @@ -325,9 +333,16 @@ impl Loader { graph: &mut StackGraph, cancellation_flag: &dyn CancellationFlag, ) -> Result<(), LoadError<'a>> { - let file = graph.add_file(&path.to_string_lossy()).unwrap(); + let file_name = path.to_string_lossy(); + let file = graph.add_file(&file_name).unwrap(); let mut globals = Variables::new(); + Self::load_globals_from_config_str(&config, &mut globals)?; + + globals + .add(FILE_PATH_VAR.into(), BUILTINS_FILENAME.into()) + .unwrap_or_default(); + sgl.build_stack_graph_into(graph, file, &source, &globals, cancellation_flag) .map_err(|err| LoadError::Builtins { inner: err, @@ -480,10 +495,10 @@ impl LanguageConfigurationsLoader { &mut self, path: &Path, content: &mut dyn ContentProvider, - ) -> Result, LoadError<'static>> { + ) -> Result, LoadError<'static>> { for configuration in self.configurations.iter() { if configuration.matches_file(path, content)? { - return Ok(Some(configuration.language)); + return Ok(Some(&configuration.language)); } } Ok(None) @@ -551,9 +566,9 @@ impl PathLoader { &mut self, path: &Path, content: &mut dyn ContentProvider, - ) -> Result, LoadError<'static>> { + ) -> Result, LoadError<'static>> { if let Some(selected_language) = self.select_language_for_file(path, content)? { - return Ok(Some(selected_language.language)); + return Ok(Some(&selected_language.language)); } Ok(None) } @@ -575,7 +590,7 @@ impl PathLoader { Some(index) => index, None => { let tsg = self.load_tsg_from_paths(&language)?; - let sgl = StackGraphLanguage::new(language.language, tsg); + let sgl = StackGraphLanguage::new(language.language.clone(), tsg); let mut builtins = StackGraph::new(); self.load_builtins_from_paths_into( @@ -586,7 +601,7 @@ impl PathLoader { )?; let lc = LanguageConfiguration { - language: language.language, + language: language.language.clone(), scope: language.scope, content_regex: language.content_regex, file_types: language.file_types, @@ -680,7 +695,7 @@ impl PathLoader { } if tsg_path.exists() { let tsg_source = std::fs::read_to_string(tsg_path)?; - return Loader::load_tsg(language.language, Cow::from(tsg_source)); + return Loader::load_tsg(language.language.clone(), Cow::from(tsg_source)); } } return Err(LoadError::NoTsgFound); @@ -770,10 +785,11 @@ impl SupplementedTsLoader { .map_err(LoadError::TreeSitter)?; let configurations = self .0 - .find_language_configurations_at_path(&path) + .find_language_configurations_at_path(&path, true) .map_err(LoadError::TreeSitter)?; let languages = languages .into_iter() + .map(|(l, _)| l) .zip(configurations.into_iter()) .map(SupplementedLanguage::from) .filter(|language| scope.map_or(true, |scope| language.matches_scope(scope))) diff --git a/tree-sitter-stack-graphs/tests/it/builder.rs b/tree-sitter-stack-graphs/tests/it/builder.rs index 661e8b973..719e25d71 100644 --- a/tree-sitter-stack-graphs/tests/it/builder.rs +++ b/tree-sitter-stack-graphs/tests/it/builder.rs @@ -9,6 +9,7 @@ use stack_graphs::graph::StackGraph; use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::NoCancellation; use tree_sitter_stack_graphs::StackGraphLanguage; +use tree_sitter_stack_graphs::FILE_PATH_VAR; use crate::edges::check_stack_graph_edges; use crate::nodes::check_stack_graph_nodes; @@ -22,13 +23,19 @@ fn can_support_preexisting_nodes() { "#; let python = "pass"; + let file_name = "test.py"; + let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); + let file = graph.get_or_create_file(file_name); let node_id = graph.new_node_id(file); let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); - let globals = Variables::new(); - let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); + let mut globals = Variables::new(); + globals + .add(FILE_PATH_VAR.into(), file_name.into()) + .expect("failed to add file path variable"); + + let language = StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg).unwrap(); language .build_stack_graph_into(&mut graph, file, python, &globals, &NoCancellation) .expect("Failed to build graph"); @@ -45,15 +52,21 @@ fn can_support_injected_nodes() { "#; let python = "pass"; + let file_name = "test.py"; + let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); + let file = graph.get_or_create_file(file_name); let node_id = graph.new_node_id(file); let _preexisting_node = graph.add_scope_node(node_id, true).unwrap(); - let language = StackGraphLanguage::from_str(tree_sitter_python::language(), tsg).unwrap(); + let language = StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg).unwrap(); let mut builder = language.builder_into_stack_graph(&mut graph, file, python); let mut globals = Variables::new(); + globals + .add(FILE_PATH_VAR.into(), file_name.into()) + .expect("failed to add file path variable"); + globals .add("EXT_NODE".into(), builder.inject_node(node_id).into()) .expect("Failed to add EXT_NODE variable"); diff --git a/tree-sitter-stack-graphs/tests/it/loader.rs b/tree-sitter-stack-graphs/tests/it/loader.rs index 257326952..401af46e5 100644 --- a/tree-sitter-stack-graphs/tests/it/loader.rs +++ b/tree-sitter-stack-graphs/tests/it/loader.rs @@ -9,6 +9,7 @@ use once_cell::sync::Lazy; use pretty_assertions::assert_eq; use stack_graphs::graph::StackGraph; use std::path::PathBuf; +use tree_sitter::Language; use tree_sitter_stack_graphs::loader::FileAnalyzers; use tree_sitter_stack_graphs::loader::LanguageConfiguration; use tree_sitter_stack_graphs::loader::Loader; @@ -25,10 +26,10 @@ static TSG: Lazy = Lazy::new(|| { #[test] fn can_load_from_provided_language_configuration() { - let language = tree_sitter_python::language(); - let sgl = StackGraphLanguage::from_str(language, &TSG).unwrap(); + let language: Language = tree_sitter_python::LANGUAGE.into(); + let sgl = StackGraphLanguage::from_str(language.clone(), &TSG).unwrap(); let lc = LanguageConfiguration { - language: language, + language: language.clone(), scope: Some("source.py".into()), content_regex: None, file_types: vec!["py".into()], @@ -43,10 +44,10 @@ fn can_load_from_provided_language_configuration() { let tsl = loader .load_tree_sitter_language_for_file(&PATH, &mut None) .expect("Expected loading tree-sitter language to succeed"); - assert_eq!(tsl, Some(language)); + assert_eq!(tsl, Some(&language)); let lc = loader .load_for_file(&PATH, &mut None, &NoCancellation) .expect("Expected loading stack graph language to succeed"); - assert_eq!(lc.primary.map(|lc| lc.language), Some(language)); + assert_eq!(lc.primary.map(|lc| &lc.language), Some(&language)); } diff --git a/tree-sitter-stack-graphs/tests/it/main.rs b/tree-sitter-stack-graphs/tests/it/main.rs index c1e37a40e..01cfd9754 100644 --- a/tree-sitter-stack-graphs/tests/it/main.rs +++ b/tree-sitter-stack-graphs/tests/it/main.rs @@ -5,6 +5,8 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use std::path::Path; + use stack_graphs::arena::Handle; use stack_graphs::graph::File; use stack_graphs::graph::StackGraph; @@ -12,6 +14,7 @@ use tree_sitter_graph::Variables; use tree_sitter_stack_graphs::BuildError; use tree_sitter_stack_graphs::NoCancellation; use tree_sitter_stack_graphs::StackGraphLanguage; +use tree_sitter_stack_graphs::FILE_PATH_VAR; mod builder; mod edges; @@ -23,11 +26,18 @@ pub(self) fn build_stack_graph( python_source: &str, tsg_source: &str, ) -> Result<(StackGraph, Handle), BuildError> { + let file_name = "test.py"; let language = - StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); + StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg_source).unwrap(); let mut graph = StackGraph::new(); - let file = graph.get_or_create_file("test.py"); - let globals = Variables::new(); + let file = graph.get_or_create_file(file_name); + let mut globals = Variables::new(); + let source_path = Path::new(file_name); + + globals + .add(FILE_PATH_VAR.into(), source_path.to_str().unwrap().into()) + .expect("failed to add file path variable"); + language.build_stack_graph_into(&mut graph, file, python_source, &globals, &NoCancellation)?; Ok((graph, file)) } diff --git a/tree-sitter-stack-graphs/tests/it/test.rs b/tree-sitter-stack-graphs/tests/it/test.rs index 98ec3ab96..fae14fe76 100644 --- a/tree-sitter-stack-graphs/tests/it/test.rs +++ b/tree-sitter-stack-graphs/tests/it/test.rs @@ -5,6 +5,7 @@ // Please see the LICENSE-APACHE or LICENSE-MIT files in this distribution for license details. // ------------------------------------------------------------------------------------------------ +use crate::FILE_PATH_VAR; use once_cell::sync::Lazy; use pretty_assertions::assert_eq; use stack_graphs::arena::Handle; @@ -72,7 +73,7 @@ fn build_stack_graph_into( globals: &Variables, ) -> Result<(), BuildError> { let language = - StackGraphLanguage::from_str(tree_sitter_python::language(), tsg_source).unwrap(); + StackGraphLanguage::from_str(tree_sitter_python::LANGUAGE.into(), tsg_source).unwrap(); language.build_stack_graph_into(graph, file, python_source, globals, &NoCancellation)?; Ok(()) } @@ -94,10 +95,20 @@ fn check_test( expected_successes + expected_failures, assertion_count, ); + let mut globals = Variables::new(); for fragments in &test.fragments { globals.clear(); + fragments.add_globals_to(&mut globals); + + globals + .add( + FILE_PATH_VAR.into(), + fragments.path.to_str().unwrap().into(), + ) + .unwrap_or_default(); + build_stack_graph_into( &mut test.graph, fragments.file, diff --git a/valgrind.supp b/valgrind.supp new file mode 100644 index 000000000..25c3cfbda --- /dev/null +++ b/valgrind.supp @@ -0,0 +1,8 @@ +{ + rust-1.83-false-positive-1 + Memcheck:Leak + match-leak-kinds: possible + fun:malloc + ... + fun:main +}