|
1 |
| -type pos = int |
2 |
| -type svalue = Tokens.svalue |
3 |
| -type pos = int |
4 |
| -type ('a, 'b) token = ('a, 'b) Tokens.token |
5 |
| -type lexresult = (svalue, pos) token |
6 |
| - |
7 |
| -val lineNum = ErrorMsg.lineNum |
8 |
| -val linePos = ErrorMsg.linePos |
9 |
| -val commentCount = ref 0 |
10 |
| -val inString = ref 0 |
11 |
| -val stringAccum = ref "" |
12 |
| -val stringStartPos = ref 0 |
13 |
| -fun err(p1,p2) = ErrorMsg.error p1 |
14 |
| - |
15 |
| -fun eof() = let val pos = hd(!linePos) in |
16 |
| - if (!commentCount <> 0) then (ErrorMsg.error pos "Unclosed Comment"; commentCount := 0) |
17 |
| - else if !inString <> 0 then (ErrorMsg.error pos "Unclosed String"; inString := 0; stringAccum := "";stringStartPos := 0) |
18 |
| - else (); Tokens.EOF(pos,pos) |
19 |
| -end |
20 |
| - |
21 |
| - |
22 |
| -%% |
23 |
| -%header (functor TigerLexFun(structure Tokens: Tiger_TOKENS)); |
24 |
| -%s COMMENT STRING; |
25 |
| -dgts = [0-9]; |
26 |
| -ws = [\ \t]; |
27 |
| -identf = [a-zA-Z][a-zA-Z0-9_]*; |
28 |
| - |
29 |
| -%% |
30 |
| -<INITIAL,COMMENT,STRING>\n => (lineNum := !lineNum+1; linePos := yypos :: !linePos; continue()); |
31 |
| -<INITIAL,COMMENT>{ws}+ => (continue()); |
32 |
| -<INITIAL>{dgts}+ => (Tokens.INT(Option.valOf(Int.fromString(yytext)),yypos, if Option.valOf(Int.fromString(yytext)) = 0 then yypos+1 else yypos+1+floor(Math.log10(real(Option.valOf(Int.fromString(yytext))))))); |
33 |
| - |
34 |
| -<INITIAL>of => (Tokens.OF(yypos,yypos+2)); |
35 |
| -<INITIAL>var => (Tokens.VAR(yypos,yypos+3)); |
36 |
| -<INITIAL>while => (Tokens.WHILE(yypos, yypos+5)); |
37 |
| -<INITIAL>for => (Tokens.FOR(yypos, yypos+3)); |
38 |
| -<INITIAL>to => (Tokens.TO(yypos, yypos+2)); |
39 |
| -<INITIAL>break => (Tokens.BREAK(yypos, yypos+5)); |
40 |
| -<INITIAL>let => (Tokens.LET(yypos, yypos+3)); |
41 |
| -<INITIAL>in => (Tokens.IN(yypos, yypos+2)); |
42 |
| -<INITIAL>end => (Tokens.END(yypos, yypos+3)); |
43 |
| -<INITIAL>function => (Tokens.FUNCTION(yypos, yypos+8)); |
44 |
| -<INITIAL>type => (Tokens.TYPE(yypos, yypos+4)); |
45 |
| -<INITIAL>array => (Tokens.ARRAY(yypos, yypos+5)); |
46 |
| -<INITIAL>if => (Tokens.IF(yypos, yypos+2)); |
47 |
| -<INITIAL>then => (Tokens.THEN(yypos, yypos+4)); |
48 |
| -<INITIAL>else => (Tokens.ELSE(yypos, yypos+4)); |
49 |
| -<INITIAL>do => (Tokens.DO(yypos, yypos+2)); |
50 |
| -<INITIAL>nil => (Tokens.NIL(yypos, yypos+3)); |
51 |
| - |
52 |
| -<INITIAL>"," => (Tokens.COMMA(yypos,yypos+1)); |
53 |
| -<INITIAL>":" => (Tokens.COLON(yypos, yypos+1)); |
54 |
| -<INITIAL>";" => (Tokens.SEMICOLON(yypos, yypos+1)); |
55 |
| -<INITIAL>"." => (Tokens.DOT(yypos, yypos+1)); |
56 |
| -<INITIAL>"(" => (Tokens.LPAREN(yypos, yypos+1)); |
57 |
| -<INITIAL>")" => (Tokens.RPAREN(yypos, yypos+1)); |
58 |
| -<INITIAL>"[" => (Tokens.LBRACK(yypos, yypos+1)); |
59 |
| -<INITIAL>"]" => (Tokens.RBRACK(yypos, yypos+1)); |
60 |
| -<INITIAL>"{" => (Tokens.LBRACE(yypos, yypos+1)); |
61 |
| -<INITIAL>"}" => (Tokens.RBRACE(yypos, yypos+1)); |
62 |
| -<INITIAL>"+" => (Tokens.PLUS(yypos, yypos+1)); |
63 |
| -<INITIAL>"-" => (Tokens.MINUS(yypos, yypos+1)); |
64 |
| -<INITIAL>"*" => (Tokens.TIMES(yypos, yypos+1)); |
65 |
| -<INITIAL>"/" => (Tokens.DIVIDE(yypos, yypos+1)); |
66 |
| -<INITIAL>"=" => (Tokens.EQ(yypos, yypos+1)); |
67 |
| -<INITIAL>":=" => (Tokens.ASSIGN(yypos, yypos+2)); |
68 |
| -<INITIAL>"<>" => (Tokens.NEQ(yypos, yypos+2)); |
69 |
| -<INITIAL>"<" => (Tokens.LT(yypos, yypos+1)); |
70 |
| -<INITIAL>">" => (Tokens.GT(yypos, yypos+1)); |
71 |
| -<INITIAL>"<=" => (Tokens.LE(yypos, yypos+2)); |
72 |
| -<INITIAL>">=" => (Tokens.GE(yypos, yypos+2)); |
73 |
| -<INITIAL>"&" => (Tokens.AND(yypos, yypos+1)); |
74 |
| -<INITIAL>"|" => (Tokens.OR(yypos, yypos+1)); |
75 |
| - |
76 |
| -<INITIAL>"/*" => (YYBEGIN COMMENT; commentCount := 1; continue()); |
77 |
| -<COMMENT>"/*" => (commentCount := !commentCount+1; continue()); |
78 |
| -<COMMENT>. => (continue()); |
79 |
| -<COMMENT>"*/" => (commentCount:= !commentCount -1; if !commentCount = 0 then YYBEGIN INITIAL else (); continue()); |
80 |
| - |
81 |
| -<INITIAL>\" => (YYBEGIN STRING;inString := 1; stringAccum := ""; stringStartPos := yypos; continue()); |
82 |
| -<STRING>\" => (YYBEGIN INITIAL;inString := 0; Tokens.STRING(!stringAccum, !stringStartPos, yypos)); |
83 |
| -<STRING>\\\" => (stringAccum := !stringAccum ^ "\""; continue()); |
84 |
| -<STRING>\\n => (stringAccum := !stringAccum ^ "\n"; continue()); |
85 |
| -<STRING>\\t => (stringAccum := !stringAccum ^ "\t"; continue()); |
86 |
| -<STRING>\\\\ => (stringAccum := !stringAccum ^ "\\"; continue()); |
87 |
| - |
88 |
| -<STRING>\\{dgts}{dgts}{dgts} => (stringAccum := !stringAccum ^ Char.toString(chr (valOf(Int.fromString (String.substring(yytext, 1, size(yytext) -1))))); continue()); |
89 |
| - |
90 |
| -<STRING>\\[\ \t\n\f]+\\ => (continue()); |
91 |
| -<STRING>\\\^(@|[A-Z]|\[|\\|\]|\^|_) => (stringAccum := !stringAccum ^ valOf (String.fromString yytext); continue()); |
92 |
| -<STRING>\\. => (ErrorMsg.error yypos ("illegal escape sequence " ^ yytext); continue()); |
93 |
| -<STRING>. => (stringAccum := !stringAccum ^ yytext; continue()); |
94 |
| - |
95 |
| - |
96 |
| -<INITIAL>{identf} => (Tokens.ID(yytext, yypos, yypos+String.size(yytext))); |
97 |
| -<INITIAL>. => (ErrorMsg.error yypos ("illegal character " ^ yytext); continue()); |
98 |
| - |
99 |
| - |
100 |
| - |
101 |
| - |
102 |
| - |
103 |
| - |
| 1 | +type pos = int |
| 2 | +type svalue = Tokens.svalue |
| 3 | +type pos = int |
| 4 | +type ('a, 'b) token = ('a, 'b) Tokens.token |
| 5 | +type lexresult = (svalue, pos) token |
| 6 | + |
| 7 | +val lineNum = ErrorMsg.lineNum |
| 8 | +val linePos = ErrorMsg.linePos |
| 9 | +val commentCount = ref 0 |
| 10 | +val inString = ref 0 |
| 11 | +val stringAccum = ref "" |
| 12 | +val stringStartPos = ref 0 |
| 13 | +fun err(p1,p2) = ErrorMsg.error p1 |
| 14 | + |
| 15 | +fun eof() = let val pos = hd(!linePos) in |
| 16 | + if (!commentCount <> 0) then (ErrorMsg.error pos "Unclosed Comment"; commentCount := 0) |
| 17 | + else if !inString <> 0 then (ErrorMsg.error pos "Unclosed String"; inString := 0; stringAccum := "";stringStartPos := 0) |
| 18 | + else (); Tokens.EOF(pos,pos) |
| 19 | +end |
| 20 | + |
| 21 | + |
| 22 | +%% |
| 23 | +%header (functor TigerLexFun(structure Tokens: Tiger_TOKENS)); |
| 24 | +%s COMMENT STRING; |
| 25 | +dgts = [0-9]; |
| 26 | +ws = [\ \t]; |
| 27 | +identf = [a-zA-Z][a-zA-Z0-9_]*; |
| 28 | + |
| 29 | +%% |
| 30 | +<INITIAL,COMMENT,STRING>\n => (lineNum := !lineNum+1; linePos := yypos :: !linePos; continue()); |
| 31 | +<INITIAL,COMMENT>{ws}+ => (continue()); |
| 32 | +<INITIAL>{dgts}+ => (Tokens.INT(Option.valOf(Int.fromString(yytext)),yypos, if Option.valOf(Int.fromString(yytext)) = 0 then yypos+1 else yypos+1+floor(Math.log10(real(Option.valOf(Int.fromString(yytext))))))); |
| 33 | + |
| 34 | +<INITIAL>of => (Tokens.OF(yypos,yypos+2)); |
| 35 | +<INITIAL>var => (Tokens.VAR(yypos,yypos+3)); |
| 36 | +<INITIAL>while => (Tokens.WHILE(yypos, yypos+5)); |
| 37 | +<INITIAL>for => (Tokens.FOR(yypos, yypos+3)); |
| 38 | +<INITIAL>to => (Tokens.TO(yypos, yypos+2)); |
| 39 | +<INITIAL>break => (Tokens.BREAK(yypos, yypos+5)); |
| 40 | +<INITIAL>let => (Tokens.LET(yypos, yypos+3)); |
| 41 | +<INITIAL>in => (Tokens.IN(yypos, yypos+2)); |
| 42 | +<INITIAL>end => (Tokens.END(yypos, yypos+3)); |
| 43 | +<INITIAL>function => (Tokens.FUNCTION(yypos, yypos+8)); |
| 44 | +<INITIAL>type => (Tokens.TYPE(yypos, yypos+4)); |
| 45 | +<INITIAL>array => (Tokens.ARRAY(yypos, yypos+5)); |
| 46 | +<INITIAL>if => (Tokens.IF(yypos, yypos+2)); |
| 47 | +<INITIAL>then => (Tokens.THEN(yypos, yypos+4)); |
| 48 | +<INITIAL>else => (Tokens.ELSE(yypos, yypos+4)); |
| 49 | +<INITIAL>do => (Tokens.DO(yypos, yypos+2)); |
| 50 | +<INITIAL>nil => (Tokens.NIL(yypos, yypos+3)); |
| 51 | + |
| 52 | +<INITIAL>"," => (Tokens.COMMA(yypos,yypos+1)); |
| 53 | +<INITIAL>":" => (Tokens.COLON(yypos, yypos+1)); |
| 54 | +<INITIAL>";" => (Tokens.SEMICOLON(yypos, yypos+1)); |
| 55 | +<INITIAL>"." => (Tokens.DOT(yypos, yypos+1)); |
| 56 | +<INITIAL>"(" => (Tokens.LPAREN(yypos, yypos+1)); |
| 57 | +<INITIAL>")" => (Tokens.RPAREN(yypos, yypos+1)); |
| 58 | +<INITIAL>"[" => (Tokens.LBRACK(yypos, yypos+1)); |
| 59 | +<INITIAL>"]" => (Tokens.RBRACK(yypos, yypos+1)); |
| 60 | +<INITIAL>"{" => (Tokens.LBRACE(yypos, yypos+1)); |
| 61 | +<INITIAL>"}" => (Tokens.RBRACE(yypos, yypos+1)); |
| 62 | +<INITIAL>"+" => (Tokens.PLUS(yypos, yypos+1)); |
| 63 | +<INITIAL>"-" => (Tokens.MINUS(yypos, yypos+1)); |
| 64 | +<INITIAL>"*" => (Tokens.TIMES(yypos, yypos+1)); |
| 65 | +<INITIAL>"/" => (Tokens.DIVIDE(yypos, yypos+1)); |
| 66 | +<INITIAL>"=" => (Tokens.EQ(yypos, yypos+1)); |
| 67 | +<INITIAL>":=" => (Tokens.ASSIGN(yypos, yypos+2)); |
| 68 | +<INITIAL>"<>" => (Tokens.NEQ(yypos, yypos+2)); |
| 69 | +<INITIAL>"<" => (Tokens.LT(yypos, yypos+1)); |
| 70 | +<INITIAL>">" => (Tokens.GT(yypos, yypos+1)); |
| 71 | +<INITIAL>"<=" => (Tokens.LE(yypos, yypos+2)); |
| 72 | +<INITIAL>">=" => (Tokens.GE(yypos, yypos+2)); |
| 73 | +<INITIAL>"&" => (Tokens.AND(yypos, yypos+1)); |
| 74 | +<INITIAL>"|" => (Tokens.OR(yypos, yypos+1)); |
| 75 | + |
| 76 | +<INITIAL>"/*" => (YYBEGIN COMMENT; commentCount := 1; continue()); |
| 77 | +<COMMENT>"/*" => (commentCount := !commentCount+1; continue()); |
| 78 | +<COMMENT>. => (continue()); |
| 79 | +<COMMENT>"*/" => (commentCount:= !commentCount -1; if !commentCount = 0 then YYBEGIN INITIAL else (); continue()); |
| 80 | + |
| 81 | +<INITIAL>\" => (YYBEGIN STRING;inString := 1; stringAccum := ""; stringStartPos := yypos; continue()); |
| 82 | +<STRING>\" => (YYBEGIN INITIAL;inString := 0; Tokens.STRING(!stringAccum, !stringStartPos, yypos)); |
| 83 | +<STRING>\\\" => (stringAccum := !stringAccum ^ "\""; continue()); |
| 84 | +<STRING>\\n => (stringAccum := !stringAccum ^ "\n"; continue()); |
| 85 | +<STRING>\\t => (stringAccum := !stringAccum ^ "\t"; continue()); |
| 86 | +<STRING>\\\\ => (stringAccum := !stringAccum ^ "\\"; continue()); |
| 87 | + |
| 88 | +<STRING>\\{dgts}{dgts}{dgts} => (stringAccum := !stringAccum ^ Char.toString(chr (valOf(Int.fromString (String.substring(yytext, 1, size(yytext) -1))))); continue()); |
| 89 | + |
| 90 | +<STRING>\\[\ \t\n\f]+\\ => (continue()); |
| 91 | +<STRING>\\\^(@|[A-Z]|\[|\\|\]|\^|_) => (stringAccum := !stringAccum ^ valOf (String.fromString yytext); continue()); |
| 92 | +<STRING>\\. => (ErrorMsg.error yypos ("illegal escape sequence " ^ yytext); continue()); |
| 93 | +<STRING>. => (stringAccum := !stringAccum ^ yytext; continue()); |
| 94 | + |
| 95 | + |
| 96 | +<INITIAL>{identf} => (Tokens.ID(yytext, yypos, yypos+String.size(yytext))); |
| 97 | +<INITIAL>. => (ErrorMsg.error yypos ("illegal character " ^ yytext); continue()); |
| 98 | + |
| 99 | + |
| 100 | + |
| 101 | + |
| 102 | + |
| 103 | + |
0 commit comments