From 68845ede846592be0136679671ca6eb450a2f34a Mon Sep 17 00:00:00 2001 From: Antonin Delpeuch Date: Thu, 11 Sep 2025 18:15:55 +0200 Subject: [PATCH] fix: Reject parse trees with missing nodes Fixes #554. I had to adjust a bunch of test cases which cannot parse anymore. The fact that our JSON parser doesn't accept trailing commas is a concern, I think we should migrate to a more accepting one, such as https://github.com/Joakker/tree-sitter-json5. --- .../failing/field_visibility_conflict/Base.cc | 2 +- .../ExpectedCurrently.cc | 2 +- .../ExpectedIdeally.cc | 2 +- .../failing/field_visibility_conflict/Left.cc | 2 +- .../field_visibility_conflict/Right.cc | 2 +- examples/cpp/working/field_conflict/Base.c | 2 +- .../cpp/working/field_conflict/Expected.c | 2 +- examples/cpp/working/field_conflict/Left.c | 2 +- examples/cpp/working/field_conflict/Right.c | 2 +- examples/cpp/working/method_conflict/Base.cpp | 2 +- .../cpp/working/method_conflict/Expected.cpp | 2 +- examples/cpp/working/method_conflict/Left.cpp | 2 +- .../cpp/working/method_conflict/Right.cpp | 2 +- .../method_conflict_same_arguments/Base.cpp | 2 +- .../Expected.cpp | 2 +- .../method_conflict_same_arguments/Left.cpp | 2 +- .../method_conflict_same_arguments/Right.cpp | 2 +- src/ast.rs | 39 ++++++++++++++----- src/ast/bundle_comments.rs | 4 +- 19 files changed, 48 insertions(+), 29 deletions(-) diff --git a/examples/cpp/failing/field_visibility_conflict/Base.cc b/examples/cpp/failing/field_visibility_conflict/Base.cc index 89f86f4..9f93b5b 100644 --- a/examples/cpp/failing/field_visibility_conflict/Base.cc +++ b/examples/cpp/failing/field_visibility_conflict/Base.cc @@ -1,4 +1,4 @@ class MyClass { public: int field_1; -} +}; diff --git a/examples/cpp/failing/field_visibility_conflict/ExpectedCurrently.cc b/examples/cpp/failing/field_visibility_conflict/ExpectedCurrently.cc index cda2fec..e01051a 100644 --- a/examples/cpp/failing/field_visibility_conflict/ExpectedCurrently.cc +++ b/examples/cpp/failing/field_visibility_conflict/ExpectedCurrently.cc @@ -8,4 +8,4 @@ class MyClass { ======= char field_3; >>>>>>> RIGHT -} +}; diff --git a/examples/cpp/failing/field_visibility_conflict/ExpectedIdeally.cc b/examples/cpp/failing/field_visibility_conflict/ExpectedIdeally.cc index f576808..968cfb1 100644 --- a/examples/cpp/failing/field_visibility_conflict/ExpectedIdeally.cc +++ b/examples/cpp/failing/field_visibility_conflict/ExpectedIdeally.cc @@ -8,4 +8,4 @@ class MyClass { ======= char field_3; >>>>>>> RIGHT -} +}; diff --git a/examples/cpp/failing/field_visibility_conflict/Left.cc b/examples/cpp/failing/field_visibility_conflict/Left.cc index eeeeacc..3e19503 100644 --- a/examples/cpp/failing/field_visibility_conflict/Left.cc +++ b/examples/cpp/failing/field_visibility_conflict/Left.cc @@ -3,4 +3,4 @@ class MyClass { int field_1; private: size_t field_2; -} +}; diff --git a/examples/cpp/failing/field_visibility_conflict/Right.cc b/examples/cpp/failing/field_visibility_conflict/Right.cc index f7c98f0..e228cc1 100644 --- a/examples/cpp/failing/field_visibility_conflict/Right.cc +++ b/examples/cpp/failing/field_visibility_conflict/Right.cc @@ -2,4 +2,4 @@ class MyClass { public: int field_1; char field_3; -} +}; diff --git a/examples/cpp/working/field_conflict/Base.c b/examples/cpp/working/field_conflict/Base.c index ffd754d..49511bf 100644 --- a/examples/cpp/working/field_conflict/Base.c +++ b/examples/cpp/working/field_conflict/Base.c @@ -1,3 +1,3 @@ struct my_struct { int field_1; -} +}; diff --git a/examples/cpp/working/field_conflict/Expected.c b/examples/cpp/working/field_conflict/Expected.c index 6e52f92..6330b8c 100644 --- a/examples/cpp/working/field_conflict/Expected.c +++ b/examples/cpp/working/field_conflict/Expected.c @@ -3,4 +3,4 @@ struct my_struct { char field_2; size_t field_3; void* pointer; -} +}; diff --git a/examples/cpp/working/field_conflict/Left.c b/examples/cpp/working/field_conflict/Left.c index cb05c6a..60065ec 100644 --- a/examples/cpp/working/field_conflict/Left.c +++ b/examples/cpp/working/field_conflict/Left.c @@ -2,4 +2,4 @@ struct my_struct { int field_1; char field_2; size_t field_3; -} +}; diff --git a/examples/cpp/working/field_conflict/Right.c b/examples/cpp/working/field_conflict/Right.c index 1c77349..14f46bb 100644 --- a/examples/cpp/working/field_conflict/Right.c +++ b/examples/cpp/working/field_conflict/Right.c @@ -1,4 +1,4 @@ struct my_struct { int field_1; void* pointer; -} +}; diff --git a/examples/cpp/working/method_conflict/Base.cpp b/examples/cpp/working/method_conflict/Base.cpp index 65f6455..8e9b392 100644 --- a/examples/cpp/working/method_conflict/Base.cpp +++ b/examples/cpp/working/method_conflict/Base.cpp @@ -5,4 +5,4 @@ class MyClass { printf("too few arguments\n"); exit(1); } -} +}; diff --git a/examples/cpp/working/method_conflict/Expected.cpp b/examples/cpp/working/method_conflict/Expected.cpp index eb8aedc..e639a7a 100644 --- a/examples/cpp/working/method_conflict/Expected.cpp +++ b/examples/cpp/working/method_conflict/Expected.cpp @@ -13,4 +13,4 @@ class MyClass { void run(bool reallyFast) { printf("world\n"); } -} +}; diff --git a/examples/cpp/working/method_conflict/Left.cpp b/examples/cpp/working/method_conflict/Left.cpp index 0335dfb..71ed2a1 100644 --- a/examples/cpp/working/method_conflict/Left.cpp +++ b/examples/cpp/working/method_conflict/Left.cpp @@ -9,4 +9,4 @@ class MyClass { void run() { printf("hello\n"); } -} +}; diff --git a/examples/cpp/working/method_conflict/Right.cpp b/examples/cpp/working/method_conflict/Right.cpp index b29a9db..bf28f78 100644 --- a/examples/cpp/working/method_conflict/Right.cpp +++ b/examples/cpp/working/method_conflict/Right.cpp @@ -9,4 +9,4 @@ class MyClass { void run(bool reallyFast) { printf("world\n"); } -} +}; diff --git a/examples/cpp/working/method_conflict_same_arguments/Base.cpp b/examples/cpp/working/method_conflict_same_arguments/Base.cpp index 65f6455..8e9b392 100644 --- a/examples/cpp/working/method_conflict_same_arguments/Base.cpp +++ b/examples/cpp/working/method_conflict_same_arguments/Base.cpp @@ -5,4 +5,4 @@ class MyClass { printf("too few arguments\n"); exit(1); } -} +}; diff --git a/examples/cpp/working/method_conflict_same_arguments/Expected.cpp b/examples/cpp/working/method_conflict_same_arguments/Expected.cpp index 025b887..7e7c53c 100644 --- a/examples/cpp/working/method_conflict_same_arguments/Expected.cpp +++ b/examples/cpp/working/method_conflict_same_arguments/Expected.cpp @@ -13,4 +13,4 @@ class MyClass { void runFast() { printf("world\n"); } -} +}; diff --git a/examples/cpp/working/method_conflict_same_arguments/Left.cpp b/examples/cpp/working/method_conflict_same_arguments/Left.cpp index 0335dfb..71ed2a1 100644 --- a/examples/cpp/working/method_conflict_same_arguments/Left.cpp +++ b/examples/cpp/working/method_conflict_same_arguments/Left.cpp @@ -9,4 +9,4 @@ class MyClass { void run() { printf("hello\n"); } -} +}; diff --git a/examples/cpp/working/method_conflict_same_arguments/Right.cpp b/examples/cpp/working/method_conflict_same_arguments/Right.cpp index 610907e..e491220 100644 --- a/examples/cpp/working/method_conflict_same_arguments/Right.cpp +++ b/examples/cpp/working/method_conflict_same_arguments/Right.cpp @@ -9,4 +9,4 @@ class MyClass { void runFast() { printf("world\n"); } -} +}; diff --git a/src/ast.rs b/src/ast.rs index 3b575b9..5b5c4d0 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -319,13 +319,22 @@ impl<'a> AstNode<'a> { let idx = local_source.ceil_char_boundary(32); return Err(format!( - "parse error at {}:{}..{}:{}, starting with: {}", + "parse error at {}:{}..{}:{}, starting with: `{}`", full_range.start_point.row, full_range.start_point.column, full_range.end_point.row, full_range.end_point.column, &local_source[..idx] )); + } else if node.is_missing() { + let full_range = node.range(); + + return Err(format!( + "parse error at {}:{}, expected `{}`", + full_range.start_point.row, + full_range.start_point.column, + node.kind(), + )); } // if this is a leaf that spans multiple lines, create one child per line, @@ -1222,7 +1231,7 @@ mod tests { assert_eq!( parse, - Err("parse error at 1:1..1:3, starting with: {,".to_string()) + Err("parse error at 1:1..1:3, starting with: `{,`".to_string()) ); let parse = AstNode::parse( @@ -1234,10 +1243,20 @@ mod tests { assert_eq!( parse, - Err("parse error at 0:0..0:39, starting with: 属于个人的非赢利性开源".to_string()) + Err("parse error at 0:0..0:39, starting with: `属于个人的非赢利性开源`".to_string()) ); } + #[test] + fn missing_token() { + let ctx = ctx(); + let lang_profile = LangProfile::detect_from_filename("test.java") + .expect("Could not load the Java lang profile"); + let parse = AstNode::parse("class Test {", lang_profile, &ctx.arena, &ctx.ref_arena); + + assert_eq!(parse, Err("parse error at 0:12, expected `}`".to_string())); + } + #[test] fn heights() { let ctx = ctx(); @@ -1625,7 +1644,7 @@ mod tests { { "a": [ 1, - 2, + 2 ], "b": { "c": "foo" @@ -1643,7 +1662,7 @@ mod tests { "\ \"a\": [ 1, - 2, + 2 ]" ); assert_eq!(entry_a.indentation_shift(), Some(" ")); @@ -1653,7 +1672,7 @@ mod tests { "\ \"a\": [ 1, - 2, + 2 ]" ); assert_eq!( @@ -1661,7 +1680,7 @@ mod tests { "\ \"a\": [ 1, - 2, + 2 ]" ); @@ -1670,7 +1689,7 @@ mod tests { "\ [ 1, - 2, + 2 ]" ); assert_eq!(array.indentation_shift(), None); @@ -1680,7 +1699,7 @@ mod tests { "\ [ 1, - 2, + 2 ]" ); assert_eq!( @@ -1688,7 +1707,7 @@ mod tests { "\ [ 1, - 2, + 2 ]" ); } diff --git a/src/ast/bundle_comments.rs b/src/ast/bundle_comments.rs index 0439d16..524b31d 100644 --- a/src/ast/bundle_comments.rs +++ b/src/ast/bundle_comments.rs @@ -569,10 +569,10 @@ fn foo() { #[test] fn dont_bundle_into_delims() { let ctx = ctx(); - let source = "(/* this is a comment */)"; + let source = "fn test(/* this is a comment */) {}"; let rs = ctx.parse("a.rs", source); - let tup = rs[0][0]; + let tup = rs[0][2]; assert_n_children(tup, 3); let comment = tup[1]; assert_eq!(comment.kind, "block_comment"); -- 2.47.3