Fixing macro parsing with a specific token

Author: SuperFola

arkdoc/parser/parser.py

@@ -4,7 +4,7 @@
 from typing import List
 
 from . import Documentation, Source
-from .tokenizer import cpp_tokenize, tree_from_tokens, cpp_tokenize, tokenize, Token
+from .tokenizer import cpp_tokenize, tree_from_tokens, tokenize, Token
 from .. import logger
 
 

arkdoc/parser/tokenizer.py

@@ -22,6 +22,7 @@ def __str__(self):
 TokenSpecification = [
     ("NUMBER", r"\d+(\.\d*)?"),
     ("STRING", r"\"[^\"]*\""),
+    ("BEGIN_MACRO", r"!{"),        # TODO: this will have to change when v4 will be delivered
     ("ID", r"[\w:?=!@&<>+\-%*/.]+"),
     ("LPAREN", r"[(\[{]"),
     ("RPAREN", r"[)\]}]"),
@@ -100,7 +101,7 @@ def tree_from_tokens(tokens: List[Token]) -> List:
         L.append(token)
         token = tokens.pop(0)
 
-    if token.type == "LPAREN":
+    if token.type == "LPAREN" or token.type == "BEGIN_MACRO":
         L2 = []
         while tokens[0].type != "RPAREN":
             L2.append(tree_from_tokens(tokens))