Implement T-SQL parser for simple SELECT statements

Add a lexer and parser that can parse the OptimizerHintsTests110 test case:
- SELECT * FROM t1 OPTION (IGNORE_NONCLUSTERED_COLUMNSTORE_INDEX)

This implements:
- Lexer with support for keywords, identifiers, operators, and comments
- Parser for SELECT statements with FROM clause and OPTION hints
- OptimizerHint AST type and JSON marshaling
- Enable the OptimizerHintsTests110 test case

Author: claude

ast/ast.go

@@ -28,12 +28,20 @@ type Statement interface {
 
 // SelectStatement represents a SELECT statement.
 type SelectStatement struct {
-	QueryExpression QueryExpression `json:"QueryExpression,omitempty"`
+	QueryExpression QueryExpression  `json:"QueryExpression,omitempty"`
+	OptimizerHints  []*OptimizerHint `json:"OptimizerHints,omitempty"`
 }
 
 func (*SelectStatement) node()      {}
 func (*SelectStatement) statement() {}
 
+// OptimizerHint represents an optimizer hint in an OPTION clause.
+type OptimizerHint struct {
+	HintKind string `json:"HintKind,omitempty"`
+}
+
+func (*OptimizerHint) node() {}
+
 // QueryExpression is the interface for query expressions.
 type QueryExpression interface {
 	Node

go.mod

@@ -1,3 +1,3 @@
 module github.com/kyleconroy/teesql
 
-go 1.25
+go 1.21

parser/lexer.go

@@ -0,0 +1,291 @@
+package parser
+
+import (
+	"strings"
+	"unicode"
+)
+
+// TokenType represents the type of a token.
+type TokenType int
+
+const (
+	TokenEOF TokenType = iota
+	TokenError
+	TokenIdent
+	TokenNumber
+	TokenString
+	TokenStar
+	TokenComma
+	TokenDot
+	TokenLParen
+	TokenRParen
+	TokenLBracket
+	TokenRBracket
+	TokenSemicolon
+	TokenEquals
+	TokenLessThan
+	TokenGreaterThan
+	TokenPlus
+	TokenMinus
+
+	// Keywords
+	TokenSelect
+	TokenFrom
+	TokenWhere
+	TokenAnd
+	TokenOr
+	TokenAs
+	TokenOption
+	TokenAll
+	TokenDistinct
+)
+
+// Token represents a lexical token.
+type Token struct {
+	Type    TokenType
+	Literal string
+	Pos     int
+}
+
+// Lexer tokenizes T-SQL input.
+type Lexer struct {
+	input   string
+	pos     int
+	readPos int
+	ch      byte
+}
+
+// NewLexer creates a new Lexer for the given input.
+func NewLexer(input string) *Lexer {
+	l := &Lexer{input: input}
+	l.readChar()
+	return l
+}
+
+func (l *Lexer) readChar() {
+	if l.readPos >= len(l.input) {
+		l.ch = 0
+	} else {
+		l.ch = l.input[l.readPos]
+	}
+	l.pos = l.readPos
+	l.readPos++
+}
+
+func (l *Lexer) peekChar() byte {
+	if l.readPos >= len(l.input) {
+		return 0
+	}
+	return l.input[l.readPos]
+}
+
+// NextToken returns the next token from the input.
+func (l *Lexer) NextToken() Token {
+	l.skipWhitespaceAndComments()
+
+	tok := Token{Pos: l.pos}
+
+	switch l.ch {
+	case 0:
+		tok.Type = TokenEOF
+		tok.Literal = ""
+	case '*':
+		tok.Type = TokenStar
+		tok.Literal = "*"
+		l.readChar()
+	case ',':
+		tok.Type = TokenComma
+		tok.Literal = ","
+		l.readChar()
+	case '.':
+		tok.Type = TokenDot
+		tok.Literal = "."
+		l.readChar()
+	case '(':
+		tok.Type = TokenLParen
+		tok.Literal = "("
+		l.readChar()
+	case ')':
+		tok.Type = TokenRParen
+		tok.Literal = ")"
+		l.readChar()
+	case '[':
+		tok = l.readBracketedIdentifier()
+	case ']':
+		tok.Type = TokenRBracket
+		tok.Literal = "]"
+		l.readChar()
+	case ';':
+		tok.Type = TokenSemicolon
+		tok.Literal = ";"
+		l.readChar()
+	case '=':
+		tok.Type = TokenEquals
+		tok.Literal = "="
+		l.readChar()
+	case '<':
+		tok.Type = TokenLessThan
+		tok.Literal = "<"
+		l.readChar()
+	case '>':
+		tok.Type = TokenGreaterThan
+		tok.Literal = ">"
+		l.readChar()
+	case '+':
+		tok.Type = TokenPlus
+		tok.Literal = "+"
+		l.readChar()
+	case '-':
+		tok.Type = TokenMinus
+		tok.Literal = "-"
+		l.readChar()
+	case '\'':
+		tok = l.readString()
+	default:
+		if isLetter(l.ch) || l.ch == '_' || l.ch == '@' || l.ch == '#' {
+			tok = l.readIdentifier()
+		} else if isDigit(l.ch) {
+			tok = l.readNumber()
+		} else {
+			tok.Type = TokenError
+			tok.Literal = string(l.ch)
+			l.readChar()
+		}
+	}
+
+	return tok
+}
+
+func (l *Lexer) skipWhitespaceAndComments() {
+	for {
+		// Skip whitespace
+		for l.ch != 0 && (l.ch == ' ' || l.ch == '\t' || l.ch == '\n' || l.ch == '\r') {
+			l.readChar()
+		}
+
+		// Skip line comments (-- ...)
+		if l.ch == '-' && l.peekChar() == '-' {
+			for l.ch != 0 && l.ch != '\n' {
+				l.readChar()
+			}
+			continue
+		}
+
+		// Skip block comments (/* ... */)
+		if l.ch == '/' && l.peekChar() == '*' {
+			l.readChar() // skip /
+			l.readChar() // skip *
+			for l.ch != 0 {
+				if l.ch == '*' && l.peekChar() == '/' {
+					l.readChar() // skip *
+					l.readChar() // skip /
+					break
+				}
+				l.readChar()
+			}
+			continue
+		}
+
+		break
+	}
+}
+
+func (l *Lexer) readIdentifier() Token {
+	startPos := l.pos
+	for isLetter(l.ch) || isDigit(l.ch) || l.ch == '_' || l.ch == '@' || l.ch == '#' {
+		l.readChar()
+	}
+	literal := l.input[startPos:l.pos]
+	return Token{
+		Type:    lookupKeyword(literal),
+		Literal: literal,
+		Pos:     startPos,
+	}
+}
+
+func (l *Lexer) readBracketedIdentifier() Token {
+	startPos := l.pos
+	l.readChar() // skip opening [
+	for l.ch != 0 && l.ch != ']' {
+		l.readChar()
+	}
+	if l.ch == ']' {
+		l.readChar() // skip closing ]
+	}
+	return Token{
+		Type:    TokenIdent,
+		Literal: l.input[startPos:l.pos],
+		Pos:     startPos,
+	}
+}
+
+func (l *Lexer) readString() Token {
+	startPos := l.pos
+	l.readChar() // skip opening quote
+	for l.ch != 0 {
+		if l.ch == '\'' {
+			if l.peekChar() == '\'' {
+				// Escaped quote
+				l.readChar()
+				l.readChar()
+				continue
+			}
+			break
+		}
+		l.readChar()
+	}
+	if l.ch == '\'' {
+		l.readChar() // skip closing quote
+	}
+	return Token{
+		Type:    TokenString,
+		Literal: l.input[startPos:l.pos],
+		Pos:     startPos,
+	}
+}
+
+func (l *Lexer) readNumber() Token {
+	startPos := l.pos
+	for isDigit(l.ch) {
+		l.readChar()
+	}
+	// Handle decimal point
+	if l.ch == '.' && isDigit(l.peekChar()) {
+		l.readChar()
+		for isDigit(l.ch) {
+			l.readChar()
+		}
+	}
+	return Token{
+		Type:    TokenNumber,
+		Literal: l.input[startPos:l.pos],
+		Pos:     startPos,
+	}
+}
+
+func isLetter(ch byte) bool {
+	return unicode.IsLetter(rune(ch))
+}
+
+func isDigit(ch byte) bool {
+	return ch >= '0' && ch <= '9'
+}
+
+var keywords = map[string]TokenType{
+	"SELECT":   TokenSelect,
+	"FROM":     TokenFrom,
+	"WHERE":    TokenWhere,
+	"AND":      TokenAnd,
+	"OR":       TokenOr,
+	"AS":       TokenAs,
+	"OPTION":   TokenOption,
+	"ALL":      TokenAll,
+	"DISTINCT": TokenDistinct,
+}
+
+func lookupKeyword(ident string) TokenType {
+	if tok, ok := keywords[strings.ToUpper(ident)]; ok {
+		return tok
+	}
+	return TokenIdent
+}