diff --git a/dwarf-cs/dwarf.core/lang/Lexer.cs b/dwarf-cs/dwarf.core/lang/Lexer.cs index de5d1d4..21b96ed 100644 --- a/dwarf-cs/dwarf.core/lang/Lexer.cs +++ b/dwarf-cs/dwarf.core/lang/Lexer.cs @@ -181,6 +181,7 @@ public IEnumerable Tokenize(string source) var state = 0; var pos = 0; + var tok = ""; // HACK source += (char) 0xffff; @@ -194,6 +195,7 @@ public IEnumerable Tokenize(string source) matcher.Reset(); } + tok = ""; state = 1; } @@ -214,6 +216,12 @@ public IEnumerable Tokenize(string source) break; } } + + if (token == null && source[pos] != 0xffff) + { + tok += source[pos]; + throw new Exception(String.Format("Unknown token: {0}", tok)); + } } if (token != null) @@ -223,6 +231,7 @@ public IEnumerable Tokenize(string source) } else { + tok += source[pos]; pos ++; } }