prepare for case insensitive scanning

Author: HSorensen

tool/src/org/antlr/v4/Tool.java

@@ -137,6 +137,7 @@ public Option(String fieldName, String name, OptionArgType argType, String descr
 	public Map<String, String> grammarOptions = null;
 	public boolean warnings_are_errors = false;
 	public boolean longMessages = false;
+	public boolean ignoreCase = false;
 
     public static Option[] optionDefs = {
         new Option("outputDirectory",	"-o", OptionArgType.STRING, "specify output directory where all output is generated"),
@@ -157,6 +158,8 @@ public Option(String fieldName, String name, OptionArgType argType, String descr
 		new Option("ST_inspector_wait_for_close", "-XdbgSTWait", "wait for STViz to close before continuing"),
         new Option("force_atn",			"-Xforce-atn", "use the ATN simulator for all predictions"),
 		new Option("log",   			"-Xlog", "dump lots of logging info to antlr-timestamp.log"),
+		new Option("ignoreCase",		"-ignorecase", "the generated lexer will be case insensitive"),
+		new Option("",					"-help", "show tool help"),
 	};
 
 	// helper vars for option management
@@ -226,6 +229,10 @@ protected void handleArgs() {
 				handleOptionSetArg(arg);
 				continue;
 			}
+			if ( arg.startsWith("-help") ) {
+				help();
+				continue;
+			}
 			if ( arg.charAt(0)!='-' ) { // file name
 				if ( !grammarFiles.contains(arg) ) grammarFiles.add(arg);
 				continue;
@@ -404,6 +411,12 @@ public void processNonCombinedGrammar(Grammar g, boolean gencode) {
 		ATNFactory factory;
 		if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g);
 		else factory = new ParserATNFactory(g);
+		
+		if (ignoreCase)
+			if (g.isLexer()) {//TODO: factory.setLexerIgnoreCase(true);
+			}
+			else errMgr.toolError(ErrorType.LEXER_OPTION_IGNORECASE);
+		
 		g.atn = factory.createATN();
 
 		if ( generate_ATN_dot ) generateATNs(g);
@@ -786,7 +799,7 @@ protected void writeDOTFile(Grammar g, String name, String dot) throws IOExcepti
 	}
 
 	public void help() {
-		info("ANTLR Parser Generator (HS Styled) Version " + Tool.VERSION);
+		version();
 		for (Option o : optionDefs) {
 			String name = o.name + (o.argType!=OptionArgType.NONE? " ___" : "");
 			String s = String.format(" %-19s %s", name, o.description);

tool/src/org/antlr/v4/automata/ATNFactory.java

@@ -64,6 +64,8 @@ public String toString() {
 	void setCurrentRuleName(String name);
 
 	void setCurrentOuterAlt(int alt);
+	
+	//void setLexerIgnoreCase(boolean ignoreCase);
 
 	Handle rule(GrammarAST ruleAST, String name, Handle blk);
 

tool/src/org/antlr/v4/automata/LexerATNFactory.java

@@ -61,6 +61,14 @@
 
 public class LexerATNFactory extends ParserATNFactory {
 	public STGroup codegenTemplates;
+/*
+	public boolean lexerIgnoreCase=false;
+	
+	@Override
+	public void setLexerIgnoreCase(boolean ignoreCase) {
+		lexerIgnoreCase=ignoreCase;
+	}
+*/	
 	public LexerATNFactory(LexerGrammar g) {
 		super(g);
 		// use codegen to get correct language templates for lexer commands
@@ -243,6 +251,12 @@ else if ( t.getType()==ANTLRParser.TOKEN_REF ) {
 	@Override
 	public Handle stringLiteral(TerminalAST stringLiteralAST) {
 		String chars = stringLiteralAST.getText();
+		// TODO:ignorecase
+		//   To implement case insensitive scanning, capture the 'STRING' 
+		//   and convert to [Ss][Tt][Rr][Ii][Nn][Gg] 
+		//   and use the charSetLiteral to construct the Handle
+		// System.out.println("LexerATNFactor.java stringLiteral>"+chars+"<");
+
 		chars = CharSupport.getStringFromGrammarStringLiteral(chars);
 		int n = chars.length();
 		ATNState left = newState(stringLiteralAST);
@@ -260,6 +274,8 @@ public Handle stringLiteral(TerminalAST stringLiteralAST) {
 	/** [Aa\t \u1234a-z\]\-] char sets */
 	@Override
 	public Handle charSetLiteral(GrammarAST charSetAST) {
+		// TODO:ignorecase
+		// System.out.println("LexerATNFactor.java charSetLiteral>"+charSetAST.getText()+"<");
 		ATNState left = newState(charSetAST);
 		ATNState right = newState(charSetAST);
 		IntervalSet set = getSetFromCharSetLiteral(charSetAST);

tool/src/org/antlr/v4/automata/ParserATNFactory.java

@@ -193,6 +193,12 @@ public void setCurrentOuterAlt(int alt) {
 		currentOuterAlt = alt;
 	}
 
+/*	
+	@Override
+	public void setLexerIgnoreCase(boolean ignoreCase) {
+	}
+*/
+	
 	/* start->ruleblock->end */
 	@Override
 	public Handle rule(GrammarAST ruleAST, String name, Handle blk) {

tool/src/org/antlr/v4/tool/ErrorType.java

@@ -246,6 +246,7 @@ public enum ErrorType {
 	 * </pre>
 	 */
 	EPSILON_OPTIONAL(154, "rule '<arg>' contains an optional block with at least one alternative that can match an empty string", ErrorSeverity.WARNING),
+	LEXER_OPTION_IGNORECASE(155, "option -ignorecase is only valid for Lexer grammars", ErrorSeverity.ERROR),
 
 	// Backward incompatibility errors
 	V3_TREE_GRAMMAR(200, "tree grammars are not supported in ANTLR 4", ErrorSeverity.ERROR),

tool/test/org/antlr/v4/test/AllTests.java

@@ -20,9 +20,9 @@
 		TestPerformance.class, TestScopeParsing.class,
 		TestSemPredEvalLexer.class, TestSemPredEvalParser.class,
 		TestSets.class, TestSymbolIssues.class, TestTokenStreamRewriter.class,
-		TestTokenTypeAssignment.class, TestToolSyntaxErrors.class,
-		TestTopologicalSort.class, TestUnbufferedCharStream.class,
-		TestUnbufferedTokenStream.class })
+		TestTokenTypeAssignment.class, TestToolOptions.class,
+		TestToolSyntaxErrors.class, TestTopologicalSort.class,
+		TestUnbufferedCharStream.class, TestUnbufferedTokenStream.class })
 public class AllTests {
 
 }

tool/test/org/antlr/v4/test/BaseTest.java

@@ -697,6 +697,29 @@ public void testErrors(String[] pairs, boolean printTree) {
         }
     }
 
+
+	public void testErrors(String[] pairs, String... extraOptions ) {
+        for (int i = 0; i < pairs.length; i+=2) {
+            String input = pairs[i];
+            String expect = pairs[i+1];
+
+			String[] lines = input.split("\n");
+			String fileName = getFilenameFromFirstLineOfGrammar(lines[0]);
+			ErrorQueue equeue = antlr(fileName, fileName, input, false,extraOptions);
+
+			String actual = equeue.toString(true);
+			actual = actual.replace(tmpdir + File.separator, "");
+			System.err.println(actual);
+			String msg = input;
+			msg = msg.replace("\n","\\n");
+			msg = msg.replace("\r","\\r");
+			msg = msg.replace("\t","\\t");
+
+            assertEquals("error in: "+msg,expect,actual);
+        }
+    }
+
+	
 	public String getFilenameFromFirstLineOfGrammar(String line) {
 		String fileName = "A" + Tool.GRAMMAR_EXTENSION;
 		int grIndex = line.lastIndexOf("grammar");

tool/test/org/antlr/v4/test/TestToolOptions.java

@@ -0,0 +1,88 @@
+/*
+ * [The "BSD license"]
+ *  Copyright (c) 2012 Terence Parr
+ *  Copyright (c) 2012 Sam Harwell
+ *  All rights reserved.
+ *
+ *  Redistribution and use in source and binary forms, with or without
+ *  modification, are permitted provided that the following conditions
+ *  are met:
+ *
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in the
+ *     documentation and/or other materials provided with the distribution.
+ *  3. The name of the author may not be used to endorse or promote products
+ *     derived from this software without specific prior written permission.
+ *
+ *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.antlr.v4.test;
+
+import org.antlr.v4.tool.ErrorType;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestToolOptions extends BaseTest {
+	@Test public void testOptionIgnoreCaseParser() {
+		String[] pair = new String[] {
+			"grammar A;\n" +
+			"a : 'A' ;\n",
+			"error(" + ErrorType.LEXER_OPTION_IGNORECASE.code + "):  option -ignorecase is only valid for Lexer grammars\n",
+		};
+		super.testErrors(pair, "-ignorecase");
+	}
+	@Test public void testOptionIgnoreCaseLexer() {
+		String[] pair = new String[] {
+			"lexer grammar A;\n" +
+			"A : 'A' ;\n",
+			"", /* no errors */
+		};
+		super.testErrors(pair, "-ignorecase");
+	}
+	 
+	/*
+	 * testOptionHelp disabled for now. The testError expect output to be writtent o System.err
+	 */
+	public void testOptionHelp() {
+		String[] pair = new String[] {
+			"lexer grammar A;\n" +
+			"A : 'A' ;\n",
+			"ANTLR Parser Generator Version 4.x\n"+
+			" -o ___              specify output directory where all output is generated\n"+
+			" -lib ___            specify location of grammars, tokens files\n"+
+			" -atn                generate rule augmented transition network diagrams\n"+
+			" -encoding ___       specify grammar file encoding; e.g., euc-jp\n"+
+			" -message-format ___ specify output style for messages in antlr, gnu, vs2005\n"+
+			" -long-messages      show exception details when available for errors and warnings\n"+
+			" -listener           generate parse tree listener (default)\n"+
+			" -no-listener        don't generate parse tree listener\n"+
+			" -visitor            generate parse tree visitor\n"+
+			" -no-visitor         don't generate parse tree visitor (default)\n"+
+			" -package ___        specify a package/namespace for the generated code\n"+
+			" -depend             generate file dependencies\n"+
+			" -D<option>=value    set/override a grammar-level option\n"+
+			" -Werror             treat warnings as errors\n"+
+			" -XdbgST             launch StringTemplate visualizer on generated code\n"+
+			" -XdbgSTWait         wait for STViz to close before continuing\n"+
+			" -Xforce-atn         use the ATN simulator for all predictions\n"+
+			" -Xlog               dump lots of logging info to antlr-timestamp.log\n"+
+			" -ignorecase         the generated lexer will be case insensitive\n"+
+			" -help               show tool help\n",
+		};
+		super.testErrors(pair, "-help");
+	}
+		
+}