Merge pull request #184 from tree-sitter/mj-no-string-blank

Get nodes for string fragments

Author: aryx

grammar.js

@@ -866,21 +866,32 @@ module.exports = grammar({
       seq(
         '"',
         repeat(choice(
-          token.immediate(prec(1, /[^"\\]+/)),
+          alias($.unescaped_double_string_fragment, $.string_fragment),
           $.escape_sequence
         )),
         '"'
       ),
       seq(
         "'",
         repeat(choice(
-          token.immediate(prec(1, /[^'\\]+/)),
+          alias($.unescaped_single_string_fragment, $.string_fragment),
           $.escape_sequence
         )),
         "'"
       )
     ),
 
+    // Workaround to https://github.com/tree-sitter/tree-sitter/issues/1156
+    // We give names to the token() constructs containing a regexp
+    // so as to obtain a node in the CST.
+    //
+    unescaped_double_string_fragment: $ =>
+      token.immediate(prec(1, /[^"\\]+/)),
+
+    // same here
+    unescaped_single_string_fragment: $ =>
+      token.immediate(prec(1, /[^'\\]+/)),
+
     escape_sequence: $ => token.immediate(seq(
       '\\',
       choice(

src/grammar.json

@@ -4721,15 +4721,13 @@
                 "type": "CHOICE",
                 "members": [
                   {
-                    "type": "IMMEDIATE_TOKEN",
+                    "type": "ALIAS",
                     "content": {
-                      "type": "PREC",
-                      "value": 1,
-                      "content": {
-                        "type": "PATTERN",
-                        "value": "[^\"\\\\]+"
-                      }
-                    }
+                      "type": "SYMBOL",
+                      "name": "unescaped_double_string_fragment"
+                    },
+                    "named": true,
+                    "value": "string_fragment"
                   },
                   {
                     "type": "SYMBOL",
@@ -4757,15 +4755,13 @@
                 "type": "CHOICE",
                 "members": [
                   {
-                    "type": "IMMEDIATE_TOKEN",
+                    "type": "ALIAS",
                     "content": {
-                      "type": "PREC",
-                      "value": 1,
-                      "content": {
-                        "type": "PATTERN",
-                        "value": "[^'\\\\]+"
-                      }
-                    }
+                      "type": "SYMBOL",
+                      "name": "unescaped_single_string_fragment"
+                    },
+                    "named": true,
+                    "value": "string_fragment"
                   },
                   {
                     "type": "SYMBOL",
@@ -4782,6 +4778,28 @@
         }
       ]
     },
+    "unescaped_double_string_fragment": {
+      "type": "IMMEDIATE_TOKEN",
+      "content": {
+        "type": "PREC",
+        "value": 1,
+        "content": {
+          "type": "PATTERN",
+          "value": "[^\"\\\\]+"
+        }
+      }
+    },
+    "unescaped_single_string_fragment": {
+      "type": "IMMEDIATE_TOKEN",
+      "content": {
+        "type": "PREC",
+        "value": 1,
+        "content": {
+          "type": "PATTERN",
+          "value": "[^'\\\\]+"
+        }
+      }
+    },
     "escape_sequence": {
       "type": "IMMEDIATE_TOKEN",
       "content": {

src/node-types.json

@@ -2378,6 +2378,10 @@
         {
           "type": "escape_sequence",
           "named": true
+        },
+        {
+          "type": "string_fragment",
+          "named": true
         }
       ]
     }
@@ -3231,6 +3235,10 @@
     "type": "static",
     "named": false
   },
+  {
+    "type": "string_fragment",
+    "named": true
+  },
   {
     "type": "super",
     "named": true