std.zig.tokenizer: support hex escape in char literals

Author: andrewrk

std/zig/parser_test.zig

@@ -1,3 +1,16 @@
+test "zig fmt: float literal with exponent" {
+    try testCanonical(
+        \\test "aoeu" {
+        \\    switch (state) {
+        \\        TermState.Start => switch (c) {
+        \\            '\x1b' => state = TermState.Escape,
+        \\            else => try out.writeByte(c),
+        \\        },
+        \\    }
+        \\}
+        \\
+    );
+}
 test "zig fmt: float literal with exponent" {
     try testCanonical(
         \\pub const f64_true_min = 4.94065645841246544177e-324;

std/zig/tokenizer.zig

@@ -220,6 +220,8 @@ pub const Tokenizer = struct {
         MultilineStringLiteralLineBackslash,
         CharLiteral,
         CharLiteralBackslash,
+        CharLiteralEscape1,
+        CharLiteralEscape2,
         CharLiteralEnd,
         Backslash,
         Equal,
@@ -612,9 +614,32 @@ pub const Tokenizer = struct {
                         result.id = Token.Id.Invalid;
                         break;
                     },
+                    'x' => {
+                        state = State.CharLiteralEscape1;
+                    },
+                    else => {
+                        state = State.CharLiteralEnd;
+                    },
+                },
+
+                State.CharLiteralEscape1 => switch (c) {
+                    '0'...'9', 'a'...'z', 'A'...'F' => {
+                        state = State.CharLiteralEscape2;
+                    },
                     else => {
+                        result.id = Token.Id.Invalid;
+                        break;
+                    },
+                },
+
+                State.CharLiteralEscape2 => switch (c) {
+                    '0'...'9', 'a'...'z', 'A'...'F' => {
                         state = State.CharLiteralEnd;
                     },
+                    else => {
+                        result.id = Token.Id.Invalid;
+                        break;
+                    },
                 },
 
                 State.CharLiteralEnd => switch (c) {
@@ -988,6 +1013,8 @@ pub const Tokenizer = struct {
                 State.MultilineStringLiteralLineBackslash,
                 State.CharLiteral,
                 State.CharLiteralBackslash,
+                State.CharLiteralEscape1,
+                State.CharLiteralEscape2,
                 State.CharLiteralEnd,
                 State.StringLiteralBackslash => {
                     result.id = Token.Id.Invalid;
@@ -1127,6 +1154,13 @@ test "tokenizer" {
     });
 }
 
+test "tokenizer - char literal with hex escape" {
+    testTokenize( \\'\x1b'
+    , []Token.Id {
+        Token.Id.CharLiteral,
+    });
+}
+
 test "tokenizer - float literal e exponent" {
     testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {
         Token.Id.Identifier,