Skip to content

Commit

Permalink
v0.2.0 #2
Browse files Browse the repository at this point in the history
  • Loading branch information
@achun
achun committed May 2, 2014
1 parent 863eb72 commit 06af2ff
Show file tree
Hide file tree
Showing 4 changed files with 351 additions and 189 deletions.
47 changes: 24 additions & 23 deletions parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,19 @@ type Status int
const (
SNot Status = iota
SInvalid
SUnexpected
SMaybe
SYes
SYesKeep // SYes 并且多读了一个字符, 保持当前的字符给后续解析
)

var statusName = [...]string{
"SNot",
"SInvalid",
"SMaybe",
"SYes",
"SYesKeep",
"Not",
"Invalid",
"Unexpected",
"Maybe",
"Yes",
"YesKeep",
}

func (t Status) String() string {
Expand Down Expand Up @@ -54,8 +56,7 @@ const (
tokenComma
tokenError
tokenRuneError
lameValue // for lame
lameArray
tokenNothing
)

func (t Token) String() string {
Expand Down Expand Up @@ -85,8 +86,7 @@ var tokensName = [...]string{
"Comma",
"Error",
"EncodingError",
"Value",
"Array",
"Nothing",
}

type TokenHandler func(Token, string) error
Expand All @@ -99,6 +99,7 @@ type parser interface {
Keep()
IsTestMode() bool

Err(msg string)
Token(token Token) error
Invalid(token Token)
NotMatch(token ...Token)
Expand All @@ -123,22 +124,23 @@ func (p *parse) IsTestMode() bool {
return p.testMode
}

func (p *parse) Rune() rune {
func (p *parse) Next() rune {
if p.next {
return p.Scanner.Next()
}
p.next = true
return p.Scanner.Rune()
}

func (p *parse) Next() rune {
return p.Rune()
}

func (p *parse) Keep() {
p.next = false
}

func (p *parse) Err(msg string) {
p.err = errors.New(msg)
p.Token(tokenError)
}

func (p *parse) NotMatch(token ...Token) {
var msg string
if len(token) == 1 {
Expand Down Expand Up @@ -205,15 +207,10 @@ func itsWhitespace(r rune, flag int, maybe bool) (Status, Token) {
if maybe && flag == 0 {
return SNot, tokenWhitespace
}
switch flag {
case 0:
if isWhitespace(r) {
return SMaybe, 1
}
case 1:
if isWhitespace(r) {
return SMaybe, 1
}
if isWhitespace(r) {
return SMaybe, 1
}
if flag == 1 {
return SYesKeep, tokenWhitespace
}
return SNot, tokenWhitespace
Expand Down Expand Up @@ -491,6 +488,10 @@ func itsEOF(r rune, flag int, maybe bool) (Status, Token) {
return SNot, tokenEOF
}

func itsSNot(r rune, flag int, maybe bool) (Status, Token) {
return SNot, tokenEOF
}

func isEOF(r rune) bool {
return r == EOF
}
Expand Down
114 changes: 62 additions & 52 deletions parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,25 @@ func assertBadParse(wt want.Want, src string, msg string) {
if msg == "" {
nwt.Error(p.err, "TOML want an error: ", src)
} else {
nwt.Equal(p.err.Error(), msg)
nwt.Equal(p.err.Error(), msg, func() string {
l, c, s := p.LastLine()
return want.String("Line ", l, ", Column ", c, "\n"+s, "\n"+arrowCol(s, c))
})
}
}
func arrowCol(line string, col int) string {
ret := ""
for i, r := range line {
if i+1 == col {
break
}
if r < 256 {
ret += " "
} else {
ret += " "
}
}
return ret + "^"
}

// outs =["token","value"]
Expand All @@ -31,19 +48,22 @@ func assertParse(wt want.Want, str string, outs ...string) {
if l > 0 {
last = outs[l-1]
}

p.Handler(func(token Token, str string) (err error) {
if tokenWhitespace == token || token == tokenNewLine || token == tokenEOF {
return
}

if i == l {
wt.False(true, "outputs more: ", i, " > ", l-1, "\nlast: ", last, "\ngot token: ", token.String()+"\nFetch: "+str)
wt.False(true, "outputs more: ", i, " > ", l-1, "\nlast: ", last, "\ngot token: ", token.String()+"\nFetch: "+str, func() string {
l, c, s := p.LastLine()
return want.String("Line ", l, ", Column ", c, "\n"+s, "\n"+arrowCol(s, c))
})

}

wt.Equal(token.String()+" "+str, outs[i], func() string {
wt.Equal(token.String()+" "+str, outs[i], i, func() string {
l, c, s := p.LastLine()
return want.String("Line ", l, ", Column ", c, "\n", s)
return want.String("Line ", l, ", Column ", c, "\n"+s, "\n"+arrowCol(s, c))
})

i++
Expand All @@ -52,7 +72,11 @@ func assertParse(wt want.Want, str string, outs ...string) {
stagePlay(p, openStage())
wt = wt
wt.Skip = 3
wt.Nil(p.err)
wt.Nil(p.err, func() string {
l, c, s := p.LastLine()
return want.String("Line ", l, ", Column ", c, "\n"+s, "\n"+arrowCol(s, c))
})

if l > 0 {
j := i
if j >= l {
Expand All @@ -61,44 +85,11 @@ func assertParse(wt want.Want, str string, outs ...string) {
wt.Equal(i, l, "loss: ", outs[j])
}
}
func assertParseBug(wt want.Want, str string, outs ...string) {
scan := NewScanner([]byte(str))
p := &parse{Scanner: scan, testMode: true}
i := 0
l := len(outs)
last := ""
if l > 0 {
last = outs[l-1]
}

p.Handler(func(token Token, str string) (err error) {
if tokenWhitespace == token || token == tokenNewLine || token == tokenEOF {
return
}
// print for debug
println(token.String() + " " + str)

if i == l {
wt.False(true, "outputs more: ", i, " > ", l-1, "\nlast: ", last, "\ngot token: ", token.String()+"\nFetch: "+str)
}

wt.Equal(token.String()+" "+str, outs[i])
i++
func TestScanner(t *testing.T) {
if skipTest {
return
})
stagePlay(p, openStage())
wt = wt
wt.Skip = 3
wt.Nil(p.err)
if l > 0 {
j := i
if j >= l {
j = l - 1
}
wt.Equal(i, l, "loss: ", outs[j])
}
}
func TestScanner(t *testing.T) {
wt := want.T(t)
s := NewScanner([]byte("0123456789"))
wt.Equal(s.Fetch(true), "0")
Expand All @@ -119,9 +110,12 @@ func TestScanner(t *testing.T) {

func TestEmpty(tt *testing.T) {
t := want.Want{tt, 7}
if skipTest {
return
}
assertParse(t, ``)
assertParse(t, ` `)
assertParse(t, ` `)
assertParse(t, ` `)
assertParse(t, `
Expand All @@ -136,15 +130,11 @@ func TestToken(tt *testing.T) {
i = `Integer`
ca = `Comma ,`
)
t := want.Want{tt, 7}

if skipTest {
return
}
t := want.Want{tt, 7}
assertParse(t, `ia = [[1],[2,3],["A","B"]]`, `Key ia`, eq, al,
al, `Integer 1`, ar, ca,
al, `Integer 2`, ca, `Integer 3`, ar, ca,
al, `String "A"`, ca, `String "B"`, ar,
ar)
assertParse(t, `string = "is string \n newline"`, `Key string`, eq, `String "is string \n newline"`)
assertParse(t, `#`, `Comment #`)
assertParse(t, `#
Expand All @@ -158,23 +148,43 @@ func TestToken(tt *testing.T) {
assertParse(t, `key = []`, `Key key`, eq, al, ar)
assertParse(t, `key = [1]`, `Key key`, eq, al, `Integer 1`, ar)
assertParse(t, `ia = [1 , 2]`, `Key ia`, eq, al, `Integer 1`, ca, `Integer 2`, ar)
assertParse(t, `ia = [[1],[2,3],["A","B"]]`, `Key ia`, eq,
al,
al, `Integer 1`, ar, ca,
al, `Integer 2`, ca, `Integer 3`, ar, ca,
al, `String "A"`, ca, `String "B"`, ar,
ar)
assertParse(t, `ia = [[[ 0,1 ],["A","B"],[["D"],["E"]]],[ 2,3]]`, `Key ia`, eq,
al,

al,
al, `Integer 0`, ca, `Integer 1`, ar, ca,
al, `String "A"`, ca, `String "B"`, ar, ca,
al,
al, `String "D"`, ar, ca,
al, `String "E"`, ar, ar,
ar, ca,

al, `Integer 2`, ca, `Integer 3`, ar,
ar)

assertParse(t, `str = ""`, `Key str`, eq, `String ""`)

const noEqual = `no one can match for Whitespace Equal`
const noEqual = `roles does not match one of stageEqual`
assertBadParse(t, `key`, "invalid Key")
assertBadParse(t, `key 1`, noEqual)
assertBadParse(t, `ke y = name`, noEqual)
assertBadParse(t, `key # comment`, noEqual)

const noValues = `no one can match for Whitespace ArrayLeftBrack String Boolean Integer Float Datetime`
const noValues = `roles does not match one of stageValues`
assertBadParse(t, `key = name`, noValues)
assertBadParse(t, `key = # comment`, noValues)

const noInteger = `no one can match for Whitespace NewLine Comment ArrayRightBrack Integer` // Comma
const noInteger = `roles does not match one of stageIntegerArray`
assertBadParse(t, `key = [1,ent]`, noInteger)
assertBadParse(t, `key = [1,"ent"]`, noInteger)

const noArrayVlaues = `no one can match for Whitespace Comment NewLine ArrayLeftBrack ArrayRightBrack String Boolean Integer Float Datetime`
const noArrayVlaues = `roles does not match one of stageArray`
assertBadParse(t, `key = [`, noArrayVlaues)

assertBadParse(t, `[]`, "invalid TableName")
Expand Down
Loading

0 comments on commit 06af2ff

Please sign in to comment.