Update tree-sitter and its grammars.

We now create our modified C and C++ grammars using the
grammar.js files in third_party/grammars/weggli-{c,cpp}.
This makes it easier to add support for widely used annotations
and macros when needed. Thanks to calixteman@ for the idea.

The new tree-sitter-cpp grammar isn't fully backwards compatible,
but should give better results for most codebases.

Cargo.toml

@@ -17,7 +17,7 @@ crate-type = ["rlib", "dylib"]
 name = "weggli"
 
 [dependencies]
-tree-sitter = "0.19.2"
+tree-sitter = "0.20.2"
 log = "0.4.11"
 clap = "2.33.3"
 walkdir = "2.3.1"

build.rs

@@ -19,21 +19,21 @@ extern crate cc;
 fn main() {
     cc::Build::new()
         .include("third_party/grammars/")
-        .file("third_party/grammars/c/parser.c")
+        .file("third_party/grammars/weggli-c/src/parser.c")
         .flag("-std=c99")
         .flag("-w")
         .compile("tree-sitter-c");
 
     cc::Build::new()
         .cpp(true)
-        .include("third_party/grammars")
-        .file("third_party/grammars/cpp/scanner.cc")
+        .include("third_party/grammars/")
+        .file("third_party/grammars/weggli-cpp/src/scanner.cc")
         .flag("-w")
         .compile("tree-sitter-cpp-scanner");
 
     cc::Build::new()
-        .include("third_party/grammars")
-        .file("third_party/grammars/cpp/parser.c")
+        .include("third_party/grammars/")
+        .file("third_party/grammars/weggli-cpp/src/parser.c")
         .flag("-w")
         .compile("tree-sitter-cpp-parser");
 }

src/builder.rs

@@ -480,7 +480,7 @@ impl QueryBuilder {
             "[ (type_identifier) (sized_type_specifier) (primitive_type)]".to_string()
         } else if kind == "identifier" && pattern.starts_with('$') {
             if self.cpp {
-                "[(identifier) (field_expression) (field_identifier) (scoped_identifier) (this)]"
+                "[(identifier) (field_expression) (field_identifier) (qualified_identifier) (this)]"
                     .to_string()
             } else {
                 "[(identifier) (field_expression) (field_identifier)]".to_string()
@@ -556,7 +556,12 @@ impl QueryBuilder {
                 } else {
                     if self.cpp {
                         format! {"[(field_expression field: (field_identifier){0})
-                        (scoped_identifier name: (identifier){0}) (identifier) {0}]",capture_str}
+                        (qualified_identifier name: (identifier){0}) 
+                        (qualified_identifier name: (qualified_identifier (identifier){0})) 
+                        (qualified_identifier name: (qualified_identifier (qualified_identifier (identifier){0}))) 
+                        (qualified_identifier name: (qualified_identifier (qualified_identifier 
+                            (qualified_identifier (identifier){0})))) 
+                        (identifier) {0}]",capture_str}
                     } else {
                         format! {"[(field_expression field: (field_identifier){0})
                         (identifier) {0}]",capture_str}

src/query.rs

@@ -136,15 +136,13 @@ impl<'a> QueryTree {
     fn match_internal(&self, root: Node, source: &str, cache: &mut Cache) -> Vec<QueryResult> {
         let mut qc = tree_sitter::QueryCursor::new();
 
-        let text_callback = |n: Node| &source[n.byte_range()];
-
         let num_patterns = self.query.pattern_count();
         let mut pattern_results = Vec::with_capacity(num_patterns + 1);
         for _ in 0..num_patterns {
             pattern_results.push(Vec::new());
         }
 
-        for m in qc.matches(&self.query, root, text_callback) {
+        for m in qc.matches(&self.query, root, source.as_bytes()) {
             // Process the query match, run subqueries and store the final QueryResults in pattern_results
             pattern_results[m.pattern_index].extend(self.process_match(cache, source, &m));
         }

tests/query.rs

@@ -552,7 +552,7 @@ fn func_calls() {
     void foo() {
         std::memcpy(a,b,c);
         memcpy(a,b,c);
-        a::b::c::d::e::memcpy(a,b,c);
+        b::c::d::e::memcpy(a,b,c);
         a->func(a,b,c);
       }
     "#;
@@ -884,6 +884,19 @@ fn test_this() {
         this->x = 10;
     }"#;
 
+    let matches = parse_and_match_cpp(needle, source);
+    assert_eq!(matches, 1);
+}
+
+#[test]
+fn test_qualified_identifier() {
+    let needle = r#"_::var = 10"#;
+
+    let source = r#"
+    void foo::bar(foo *this){
+        foo::var = 10;
+    }"#;
+
     let matches = parse_and_match_cpp(needle, source);
     assert_eq!(matches, 1);
 }