Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: tree-sitter-grammars/tree-sitter-markdown
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: main
Choose a base ref
...
head repository: jmbuhr/tree-sitter-quarto
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: main
Choose a head ref
Can’t automatically merge. Don’t worry, you can still create the pull request.
Loading
Showing with 162,853 additions and 183,877 deletions.
  1. +20 −0 .github/ISSUE_TEMPLATE/bug_report.md
  2. +11 −0 .github/ISSUE_TEMPLATE/feature_request.md
  3. +43 −0 .github/workflows/node.js.yml
  4. +2 −0 .gitignore
  5. +66 −0 CONTRIBUTING.md
  6. +16 −7 Cargo.toml
  7. +44 −0 Package.swift
  8. +8 −5 README.md
  9. +21 −0 benchmark/main.rs
  10. +10 −6 binding.gyp
  11. +13 −6 bindings/node/binding.cc
  12. +5 −3 bindings/node/index.js
  13. +33 −14 bindings/rust/build.rs
  14. +502 −25 bindings/rust/lib.rs
  15. +6 −0 bindings/swift/notes.txt
  16. +128 −0 common/grammar.js
  17. 0 { → common}/html_entities.json
  18. BIN contrib/screenshot.png
  19. +0 −69 corpus/issues.txt
  20. +0 −9,678 corpus/spec.txt
  21. +0 −1,069 grammar.js
  22. +1,906 −6 package-lock.json
  23. +30 −12 package.json
  24. +0 −34 queries/highlights.scm
  25. +0 −7 queries/injections.scm
  26. +0 −11,555 src/grammar.json
  27. +0 −1,636 src/node-types.json
  28. +0 −158,594 src/parser.c
  29. +0 −1,012 src/scanner.cc
  30. +26 −0 tree-sitter-quarto-inline/Cargo.toml
  31. +19 −0 tree-sitter-quarto-inline/binding.gyp
  32. +28 −0 tree-sitter-quarto-inline/bindings/node/binding.cc
  33. +19 −0 tree-sitter-quarto-inline/bindings/node/index.js
  34. +40 −0 tree-sitter-quarto-inline/bindings/rust/build.rs
  35. +52 −0 tree-sitter-quarto-inline/bindings/rust/lib.rs
  36. +16 −0 tree-sitter-quarto-inline/bindings/swift/TreeSitterQuartoInline/quarto_inline.h
  37. +67 −139 corpus/failing.txt → tree-sitter-quarto-inline/corpus-failing/spec.txt
  38. +65 −0 tree-sitter-quarto-inline/corpus/extension_latex.txt
  39. +32 −0 tree-sitter-quarto-inline/corpus/extension_strikethrough.txt
  40. +106 −0 tree-sitter-quarto-inline/corpus/issues.txt
  41. +4,031 −0 tree-sitter-quarto-inline/corpus/spec.txt
  42. +21 −0 tree-sitter-quarto-inline/corpus/tags.txt
  43. +455 −0 tree-sitter-quarto-inline/grammar.js
  44. +19 −0 tree-sitter-quarto-inline/package.json
  45. +37 −0 tree-sitter-quarto-inline/queries/highlights.scm
  46. +2 −0 tree-sitter-quarto-inline/queries/injections.scm
  47. +5,876 −0 tree-sitter-quarto-inline/src/grammar.json
  48. +865 −0 tree-sitter-quarto-inline/src/node-types.json
  49. +74,064 −0 tree-sitter-quarto-inline/src/parser.c
  50. +384 −0 tree-sitter-quarto-inline/src/scanner.c
  51. +1 −0 { → tree-sitter-quarto-inline}/src/tree_sitter/parser.h
  52. +26 −0 tree-sitter-quarto/Cargo.toml
  53. +19 −0 tree-sitter-quarto/binding.gyp
  54. +28 −0 tree-sitter-quarto/bindings/node/binding.cc
  55. +19 −0 tree-sitter-quarto/bindings/node/index.js
  56. +40 −0 tree-sitter-quarto/bindings/rust/build.rs
  57. +52 −0 tree-sitter-quarto/bindings/rust/lib.rs
  58. +16 −0 tree-sitter-quarto/bindings/swift/TreeSitterQuarto/quarto.h
  59. +17 −0 tree-sitter-quarto/corpus-failing/spec.txt
  60. +19 −0 tree-sitter-quarto/corpus/extension_minus_metadata.txt
  61. +182 −0 tree-sitter-quarto/corpus/extension_pipe_table.txt
  62. +19 −0 tree-sitter-quarto/corpus/extension_plus_metadata.txt
  63. +58 −0 tree-sitter-quarto/corpus/extension_task_list.txt
  64. +92 −0 tree-sitter-quarto/corpus/issues.txt
  65. +5,842 −0 tree-sitter-quarto/corpus/spec.txt
  66. +604 −0 tree-sitter-quarto/grammar.js
  67. +19 −0 tree-sitter-quarto/package.json
  68. +52 −0 tree-sitter-quarto/queries/highlights.scm
  69. +12 −0 tree-sitter-quarto/queries/injections.scm
  70. +5,390 −0 tree-sitter-quarto/src/grammar.json
  71. +960 −0 tree-sitter-quarto/src/node-types.json
  72. +58,583 −0 tree-sitter-quarto/src/parser.c
  73. +1,491 −0 tree-sitter-quarto/src/scanner.c
  74. +224 −0 tree-sitter-quarto/src/tree_sitter/parser.h
20 changes: 20 additions & 0 deletions .github/ISSUE_TEMPLATE/bug_report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug
assignees: ''

---

**Describe the bug**

**Code example**
```quarto
Your example here
```

**Expected behavior**
<!-- consider linking to a relevant section in a spec like https://github.github.com/gfm/ -->

**Actual behavior**
11 changes: 11 additions & 0 deletions .github/ISSUE_TEMPLATE/feature_request.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: enhancement
assignees: ''

---

<!-- consider linking a relevant section from a spec -->
<!-- extensions from any spec (rquarto, pandoc, ...) are welcome -->
43 changes: 43 additions & 0 deletions .github/workflows/node.js.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# This workflow will do a clean installation of node dependencies, cache/restore them, build the source code and run tests across different versions of node
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions

name: Node.js CI

on:
push:
branches: [ "split_parser" ]
pull_request:
branches: [ "split_parser" ]

jobs:
build:

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- name: Use Node.js 14
uses: actions/setup-node@v3
with:
node-version: 14
cache: 'npm'
- run: npm ci
- run: npm run build --if-present
- name: Check grammar is compiled correctly
run: git diff --exit-code
test:

runs-on: ubuntu-latest
env:
ALL_EXTENSIONS: 1

steps:
- uses: actions/checkout@v3
- name: Use Node.js 14
uses: actions/setup-node@v3
with:
node-version: 14
cache: 'npm'
- run: npm ci
- run: npm run build --if-present
- run: npm test
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
node_modules
build
target
/.build
Cargo.lock
66 changes: 66 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Contributing

All contributions are welcome. Specifically, if you found a bug or have a
suggestion for a new feature or quarto extension, you can always open an
[issue] or [pull request].

## Issues

If you open an issue please give a short description of your bug or feature
along with a code example. If there is a relevant spec like the [CommonMark
Spec][commonmark] or the [GitHub Flavored markdown Spec][gfm] please link it in
the issue.

Some bug reports belong in other repositories if they only concern the
implementation of the grammar in a specific context like `nvim-treesitter`, but
you can always open an issue here and I will point you in the right direction.

## Code Overview

Please refer to the [tree-sitter spec] for more details on how to write a tree-
sitter grammar.

This parse is split into two grammars. One for block structure, which can be
found in the `tree-sitter-quarto` folder, and one for inline structure, which
can be found in the `tree-sitter-quarto-inline` folder. Components that are
parts of either grammar can be found in the `common` folder.

For either of the grammar the most important files are the `grammar.js` which
defines most nodes and the `src/scanner.c` which defines nodes that cannot
be parsed with normal tree-sitter rules. All other files in the `src` subfolder
are auto-generated by running `tree-sitter generate --no-bindings`. (You need to
install the [tree-sitter cli tool][tree-sitter-cli] first.)

Some syntactical components can be enabled or disabled by environment variables
at compile time. The logic for this can be found in the `common/grammar.js`
file.

Tests are located in the `corpus` subfolder:
* `spec.txt` is taken from the examples in the [GFM spec][gfm].
* `failing.txt` are those examples from the spec that do not pass yet.
* `issues.txt` are test cases covering solved issues.
* `extension_<>.txt` are covering specific extensions. Some of these are also
taken from the GFM spec.

## Pull Requests

I will happily accept any pull requests.

Before submitting any code please check the following:

* You ran `tree-sitter generate --no-bindings` in the `tree-sitter-quarto` or
`tree-sitter-quarto-inline` directories respecively after modifying any
`grammar.js` file.
* When running `tree-sitter test` only the cases defined in `failing.txt` or
`extension_<>.txt` for not activated extensions fail for **both** gramars.
* If you implemented new behavior please add tests. (In most cases these belong
in a `extension_<>.txt`.)
* You deleted any auto-generated bindings and files for debugging purposes
like `log.html`

[issue]: https://github.com/jmbuhr/tree-sitter-quarto/issues/new
[pull request]: https://github.com/jmbuhr/tree-sitter-quarto/compare
[gfm]: https://github.github.com/gfm/
[commonmark]: https://spec.commonmark.org/
[tree-sitter spec]: https://tree-sitter.github.io/tree-sitter/
[tree-sitter-cli]: https://github.com/tree-sitter/tree-sitter/blob/master/cli/README.md
23 changes: 16 additions & 7 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,19 +1,21 @@
[package]
name = "tree-sitter-md"
description = "markdown grammar for the tree-sitter parsing library"
version = "0.0.1"
keywords = ["incremental", "parsing", "markdown"]
description = "quarto grammar for the tree-sitter parsing library"
version = "0.1.5"
keywords = ["incremental", "parsing", "quarto"]
categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter-markdown"
repository = "https://github.com/MDeiml/tree-sitter-quarto"
edition = "2018"
license = "MIT"

build = "bindings/rust/build.rs"
include = [
"bindings/rust/*",
"grammar.js",
"queries/*",
"src/*",
"tree-sitter-quarto/src/*",
"tree-sitter-quarto-inline/src/*",
"tree-sitter-quarto/queries/*",
"tree-sitter-quarto-inline/queries/*",
"benchmark/main.rs",
]

[lib]
@@ -24,3 +26,10 @@ tree-sitter = "~0.20"

[build-dependencies]
cc = "1.0"

[[bin]]
name = "benchmark"
path = "benchmark/main.rs"

[profile.release]
debug = true
44 changes: 44 additions & 0 deletions Package.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// swift-tools-version:5.3

import PackageDescription

let package = Package(
name: "TreeSitterquarto",
platforms: [.macOS(.v10_13), .iOS(.v11)],
products: [
.library(name: "TreeSitterquarto", targets: ["TreeSitterquarto", "TreeSitterquartoInline"]),
],
dependencies: [],
targets: [
.target(name: "TreeSitterquarto",
path: "tree-sitter-quarto",
exclude: [
"corpus",
"grammar.js",
],
sources: [
"src/parser.c",
"src/scanner.c",
],
resources: [
.copy("queries")
],
publicHeadersPath: "bindings/swift",
cSettings: [.headerSearchPath("src")]),
.target(name: "TreeSitterquartoInline",
path: "tree-sitter-quarto-inline",
exclude: [
"corpus",
"grammar.js",
],
sources: [
"src/parser.c",
"src/scanner.c",
],
resources: [
.copy("queries")
],
publicHeadersPath: "bindings/swift",
cSettings: [.headerSearchPath("src")])
]
)
13 changes: 8 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
# tree-sitter-markdown
A markdown parser for tree-sitter
# tree-sitter-quarto

For now this implements the [CommonMark Spec](https://spec.commonmark.org/). Maybe it will be extended to support [Github flavored markdown](https://github.github.com/gfm/)
A quarto parser for [tree-sitter](https://github.com/tree-sitter/).

## Disclaimer
This is a fork of <https://github.com/MDeiml/tree-sitter-markdown>
(until I figure out how extending instead of replacing a grammar works in `treesitter`).

This parser is still quite experimental.
## Contributing

All contributions are welcome. For details refer to [CONTRIBUTING.md].

[CONTRIBUTING.md]: https://github.com/jmbuhr/tree-sitter-quarto/blob/split_parser/CONTRIBUTING.md
21 changes: 21 additions & 0 deletions benchmark/main.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
use tree_sitter_md::*;

fn main() {
let mut parser = quartoParser::default();
let filename = std::env::args().nth(1).unwrap();
let source = std::fs::read(filename).unwrap();
let mut tree = parser.parse(&source, None).unwrap();
tree.edit(&tree_sitter::InputEdit {
start_byte: 0,
old_end_byte: 1,
new_end_byte: 0,
start_position: tree_sitter::Point::new(0, 0),
old_end_position: tree_sitter::Point::new(0, 1),
new_end_position: tree_sitter::Point::new(0, 0),
});
reparse(&mut parser, &source[1..], tree);
}

fn reparse(parser: &mut quartoParser, source: &[u8], old_tree: quartoTree) {
parser.parse(source, Some(&old_tree));
}
16 changes: 10 additions & 6 deletions binding.gyp
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
{
"targets": [
{
"target_name": "tree_sitter_markdown_binding",
"target_name": "tree_sitter_quarto_binding",
"include_dirs": [
"<!(node -e \"require('nan')\")",
"src"
"tree-sitter-quarto/src",
"tree-sitter-quarto-inline/src",
],
"sources": [
"bindings/node/binding.cc",
"src/parser.c",
"src/scanner.cc"
"tree-sitter-quarto/src/parser.c",
"tree-sitter-quarto/src/scanner.c",
"tree-sitter-quarto-inline/src/parser.c",
"tree-sitter-quarto-inline/src/scanner.c",
"bindings/node/binding.cc"
],
"cflags_c": [
"-std=c99",
"-std=c99"
]
}
]
}

19 changes: 13 additions & 6 deletions bindings/node/binding.cc
Original file line number Diff line number Diff line change
@@ -4,7 +4,8 @@

using namespace v8;

extern "C" TSLanguage * tree_sitter_markdown();
extern "C" TSLanguage * tree_sitter_quarto();
extern "C" TSLanguage * tree_sitter_quarto_inline();

namespace {

@@ -16,13 +17,19 @@ void Init(Local<Object> exports, Local<Object> module) {
tpl->InstanceTemplate()->SetInternalFieldCount(1);

Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance, 0, tree_sitter_markdown());

Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("markdown").ToLocalChecked());
Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
Local<Object> instance_block = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance_block, 0, tree_sitter_quarto());
Nan::Set(instance_block, Nan::New("name").ToLocalChecked(), Nan::New("quarto").ToLocalChecked());
Nan::Set(exports, Nan::New("quarto").ToLocalChecked(), instance_block);

Local<Object> instance_inline = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance_inline, 0, tree_sitter_quarto_inline());
Nan::Set(instance_inline, Nan::New("name").ToLocalChecked(), Nan::New("quarto_inline").ToLocalChecked());
Nan::Set(exports, Nan::New("quarto_inline").ToLocalChecked(), instance_inline);
}

NODE_MODULE(tree_sitter_markdown_binding, Init)
NODE_MODULE(tree_sitter_quarto_binding, Init)

} // namespace

8 changes: 5 additions & 3 deletions bindings/node/index.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
try {
module.exports = require("../../build/Release/tree_sitter_markdown_binding");
module.exports = require("../../build/Release/tree_sitter_quarto_binding");
} catch (error1) {
if (error1.code !== 'MODULE_NOT_FOUND') {
throw error1;
}
try {
module.exports = require("../../build/Debug/tree_sitter_markdown_binding");
module.exports = require("../../build/Debug/tree_sitter_quarto_binding");
} catch (error2) {
if (error2.code !== 'MODULE_NOT_FOUND') {
throw error2;
@@ -15,5 +15,7 @@ try {
}

try {
module.exports.nodeTypeInfo = require("../../src/node-types.json");
module.exports.nodeTypeInfo = require("../../tree-sitter-quarto/src/node-types.json");
module.exports.nodeTypeInfoInline = require("../../tree-sitter-quarto-inline/src/node-types.json");
} catch (_) {}

Loading