Fix issue with parsing floats (fixes #21)

kinglozzer · Sep 8, 2021 · 9711e93 · 9711e93 · jchenevey · Sep 8, 2021
1 parent 4c63d52
commit 9711e93
Show file tree

Hide file tree

Showing 6 changed files with 388 additions and 262 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@
 ### Removed
 
 ### Fixed
+- Fixed issue with parsing of float arguments in lookups (#21)
 
 ## [1.0.1]
 ### Changed

diff --git a/src/main/gen/com/kinglozzer/silverstripe/parser/_SilverstripeLexer.java b/src/main/gen/com/kinglozzer/silverstripe/parser/_SilverstripeLexer.java
diff --git a/src/main/java/com/kinglozzer/silverstripe/parser/Silverstripe.flex b/src/main/java/com/kinglozzer/silverstripe/parser/Silverstripe.flex
@@ -44,7 +44,13 @@ import com.intellij.util.containers.Stack;
 
 CRLF= (\n|\r|\r\n)
 WHITE_SPACE= [\ \t]
-NUMBER=[0-9]+
+SIGN= [+-]
+FLOAT= [0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?
+HEXADECIMAL= 0[xX][0-9a-fA-F]+
+OCTAL= 0[0-7]+
+BINARY= 0[bB][01]+
+DECIMAL= 0 | [1-9][0-9]*
+INTEGER_OR_FLOAT= ( {SIGN} )? ( {HEXADECIMAL} | {BINARY} | {FLOAT} | {OCTAL} | {DECIMAL} )
 SS_LOOKUP= \$[a-zA-Z_]+([a-zA-Z0-9_])*
 SS_IDENTIFIER= [a-zA-Z_]+([a-zA-Z0-9_])*
 SS_BLOCK_START= <%
@@ -173,7 +179,7 @@ SS_TEXT= (([^<${\\]+) | (\\.) | (<[^%]) | (\$[^A-Za-z_]) | (\{[^\$]) | (\{\$[^A-
 
 <SS_NAMED_VAR> {
     "="                                 { return SilverstripeTokenTypes.SS_EQUALS; }
-    {NUMBER}                            { return SilverstripeTokenTypes.SS_NUMBER; }
+    {INTEGER_OR_FLOAT}                  { return SilverstripeTokenTypes.SS_NUMBER; }
     {SS_PRIMITIVE}                      { return SilverstripeTokenTypes.SS_PRIMITIVE; }
     {SS_STRING}                         { return SilverstripeTokenTypes.SS_STRING; }
     {SS_LOOKUP}                         { yypushstate(SS_LOOKUP); return SilverstripeTokenTypes.SS_LOOKUP; }
@@ -203,7 +209,7 @@ SS_TEXT= (([^<${\\]+) | (\\.) | (<[^%]) | (\$[^A-Za-z_]) | (\{[^\$]) | (\{\$[^A-
 
 <SS_IF_STATEMENT> {
     {SS_COMPARISON_OPERATOR}            { return SilverstripeTokenTypes.SS_COMPARISON_OPERATOR; }
-    {NUMBER}                            { return SilverstripeTokenTypes.SS_NUMBER; }
+    {INTEGER_OR_FLOAT}                  { return SilverstripeTokenTypes.SS_NUMBER; }
     {SS_AND_OR_OPERATOR}                { return SilverstripeTokenTypes.SS_AND_OR_OPERATOR; }
     {SS_STRING}                         { return SilverstripeTokenTypes.SS_STRING; }
     {SS_IDENTIFIER}                     { yypushstate(SS_LOOKUP); return SilverstripeTokenTypes.SS_LOOKUP; }
@@ -302,7 +308,7 @@ SS_TEXT= (([^<${\\]+) | (\\.) | (<[^%]) | (\$[^A-Za-z_]) | (\{[^\$]) | (\{\$[^A-
     {SS_IDENTIFIER}                     { yypushstate(SS_LOOKUP); return SilverstripeTokenTypes.SS_LOOKUP; }
     {SS_LOOKUP}                         { yypushstate(SS_LOOKUP); return SilverstripeTokenTypes.SS_LOOKUP; }
     {SS_STRING}                         { return SilverstripeTokenTypes.SS_STRING; }
-    {NUMBER}                            { return SilverstripeTokenTypes.SS_NUMBER; }
+    {INTEGER_OR_FLOAT}                  { return SilverstripeTokenTypes.SS_NUMBER; }
     {WHITE_SPACE}+                      { return TokenType.WHITE_SPACE; }
     {CRLF}+                             { yypushback(yylength()); yypopstate(); }
     .                                   { yypushback(yylength()); yypopstate(); }

diff --git a/src/test/java/parser/SilverstripeParserTest.java b/src/test/java/parser/SilverstripeParserTest.java
@@ -92,4 +92,8 @@ public void testIssue11() {
     public void testIssue12() {
         doTest(true);
     }
+
+    public void testIssue21() {
+        doTest(true);
+    }
 }
diff --git a/src/test/testData/parser/Issue21.ss b/src/test/testData/parser/Issue21.ss
@@ -0,0 +1,7 @@
+{$Test(0)}
+{$Test(-0123)}
+{$Test(0x1A)}
+{$Test(0b11111111)}
+{$Test(15.25)}
+{$Test(7E-2)}
+{$Test(-1.2e3)}
diff --git a/src/test/testData/parser/Issue21.txt b/src/test/testData/parser/Issue21.txt
@@ -0,0 +1,71 @@
+FILE
+  SilverstripePsiElementImpl([Ss] SS_STATEMENTS)
+    PsiElement(SS_LEFT_BRACE)('{')
+    SilverstripePsiElementImpl([Ss] SS_LOOKUP)
+      SilverstripeLookupStepImpl([Ss] SS_LOOKUP_STEP)
+        PsiElement([Ss] SS_LOOKUP)('$Test')
+        PsiElement(SS_LEFT_PARENTHESIS)('(')
+        SilverstripePsiElementImpl([Ss] SS_LOOKUP_STEP_ARGS)
+          PsiElement(SS_NUMBER)('0')
+        PsiElement(SS_RIGHT_PARENTHESIS)(')')
+    PsiElement(SS_RIGHT_BRACE)('}')
+    PsiWhiteSpace('\n')
+    PsiElement(SS_LEFT_BRACE)('{')
+    SilverstripePsiElementImpl([Ss] SS_LOOKUP)
+      SilverstripeLookupStepImpl([Ss] SS_LOOKUP_STEP)
+        PsiElement([Ss] SS_LOOKUP)('$Test')
+        PsiElement(SS_LEFT_PARENTHESIS)('(')
+        SilverstripePsiElementImpl([Ss] SS_LOOKUP_STEP_ARGS)
+          PsiElement(SS_NUMBER)('-0123')
+        PsiElement(SS_RIGHT_PARENTHESIS)(')')
+    PsiElement(SS_RIGHT_BRACE)('}')
+    PsiWhiteSpace('\n')
+    PsiElement(SS_LEFT_BRACE)('{')
+    SilverstripePsiElementImpl([Ss] SS_LOOKUP)
+      SilverstripeLookupStepImpl([Ss] SS_LOOKUP_STEP)
+        PsiElement([Ss] SS_LOOKUP)('$Test')
+        PsiElement(SS_LEFT_PARENTHESIS)('(')
+        SilverstripePsiElementImpl([Ss] SS_LOOKUP_STEP_ARGS)
+          PsiElement(SS_NUMBER)('0x1A')
+        PsiElement(SS_RIGHT_PARENTHESIS)(')')
+    PsiElement(SS_RIGHT_BRACE)('}')
+    PsiWhiteSpace('\n')
+    PsiElement(SS_LEFT_BRACE)('{')
+    SilverstripePsiElementImpl([Ss] SS_LOOKUP)
+      SilverstripeLookupStepImpl([Ss] SS_LOOKUP_STEP)
+        PsiElement([Ss] SS_LOOKUP)('$Test')
+        PsiElement(SS_LEFT_PARENTHESIS)('(')
+        SilverstripePsiElementImpl([Ss] SS_LOOKUP_STEP_ARGS)
+          PsiElement(SS_NUMBER)('0b11111111')
+        PsiElement(SS_RIGHT_PARENTHESIS)(')')
+    PsiElement(SS_RIGHT_BRACE)('}')
+    PsiWhiteSpace('\n')
+    PsiElement(SS_LEFT_BRACE)('{')
+    SilverstripePsiElementImpl([Ss] SS_LOOKUP)
+      SilverstripeLookupStepImpl([Ss] SS_LOOKUP_STEP)
+        PsiElement([Ss] SS_LOOKUP)('$Test')
+        PsiElement(SS_LEFT_PARENTHESIS)('(')
+        SilverstripePsiElementImpl([Ss] SS_LOOKUP_STEP_ARGS)
+          PsiElement(SS_NUMBER)('15.25')
+        PsiElement(SS_RIGHT_PARENTHESIS)(')')
+    PsiElement(SS_RIGHT_BRACE)('}')
+    PsiWhiteSpace('\n')
+    PsiElement(SS_LEFT_BRACE)('{')
+    SilverstripePsiElementImpl([Ss] SS_LOOKUP)
+      SilverstripeLookupStepImpl([Ss] SS_LOOKUP_STEP)
+        PsiElement([Ss] SS_LOOKUP)('$Test')
+        PsiElement(SS_LEFT_PARENTHESIS)('(')
+        SilverstripePsiElementImpl([Ss] SS_LOOKUP_STEP_ARGS)
+          PsiElement(SS_NUMBER)('7E-2')
+        PsiElement(SS_RIGHT_PARENTHESIS)(')')
+    PsiElement(SS_RIGHT_BRACE)('}')
+    PsiWhiteSpace('\n')
+    PsiElement(SS_LEFT_BRACE)('{')
+    SilverstripePsiElementImpl([Ss] SS_LOOKUP)
+      SilverstripeLookupStepImpl([Ss] SS_LOOKUP_STEP)
+        PsiElement([Ss] SS_LOOKUP)('$Test')
+        PsiElement(SS_LEFT_PARENTHESIS)('(')
+        SilverstripePsiElementImpl([Ss] SS_LOOKUP_STEP_ARGS)
+          PsiElement(SS_NUMBER)('-1.2e3')
+        PsiElement(SS_RIGHT_PARENTHESIS)(')')
+    PsiElement(SS_RIGHT_BRACE)('}')