Skip to content

Commit c792161

Browse files
natemoo-renatemoo-re
authored
Properly fix nested expressions (#456)
* chore: add nested expression tests * chore: add token test * wip: fix element stack * fix: handle nested expressions with token stack * test: fix typo * chore: add changeset Co-authored-by: Nate Moore <[email protected]>
1 parent 0fb1620 commit c792161

File tree

4 files changed

+104
-24
lines changed

4 files changed

+104
-24
lines changed

‎.changeset/thick-files-taste.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@astrojs/compiler': patch
3+
---
4+
5+
Fix nested expression handling with a proper expression tokenizer stack

‎internal/printer/printer_test.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,34 @@ const groups = [[0, 1, 2], [3, 4, 5]];
648648
code: `${$$maybeRenderHead($$result)}<article>${(previous || next) && $$render` + BACKTICK + `<aside>${previous && $$render` + BACKTICK + `<div>Previous Article: <a rel="prev"${$$addAttribute(new URL(previous.link, Astro.site).pathname, "href")}>${previous.text}</a></div>` + BACKTICK + `}${next && $$render` + BACKTICK + `<div>Next Article: <a rel="next"${$$addAttribute(new URL(next.link, Astro.site).pathname, "href")}>${next.text}</a></div>` + BACKTICK + `}</aside>` + BACKTICK + `}</article>`,
649649
},
650650
},
651+
{
652+
name: "nested expressions II",
653+
source: `<article>{(previous || next) && <aside>{previous && <div>Previous Article: <a rel="prev" href={new URL(previous.link, Astro.site).pathname}>{previous.text}</a></div>} {next && <div>Next Article: <a rel="next" href={new URL(next.link, Astro.site).pathname}>{next.text}</a></div>}</aside>}</article>`,
654+
want: want{
655+
code: `${$$maybeRenderHead($$result)}<article>${(previous || next) && $$render` + BACKTICK + `<aside>${previous && $$render` + BACKTICK + `<div>Previous Article: <a rel="prev"${$$addAttribute(new URL(previous.link, Astro.site).pathname, "href")}>${previous.text}</a></div>` + BACKTICK + `} ${next && $$render` + BACKTICK + `<div>Next Article: <a rel="next"${$$addAttribute(new URL(next.link, Astro.site).pathname, "href")}>${next.text}</a></div>` + BACKTICK + `}</aside>` + BACKTICK + `}</article>`,
656+
},
657+
},
658+
{
659+
name: "nested expressions III",
660+
source: `<div>{x.map((x) => x ? <div>{true ? <span>{x}</span> : null}</div> : <div>{false ? null : <span>{x}</span>}</div>)}</div>`,
661+
want: want{
662+
code: "${$$maybeRenderHead($$result)}<div>${x.map((x) => x ? $$render`<div>${true ? $$render`<span>${x}</span>` : null}</div>` : $$render`<div>${false ? null : $$render`<span>${x}</span>`}</div>`)}</div>",
663+
},
664+
},
665+
{
666+
name: "nested expressions IV",
667+
source: `<div>{() => { if (value > 0.25) { return <span>Default</span> } else if (value > 0.5) { return <span>Another</span> } else if (value > 0.75) { return <span>Other</span> } return <span>Yet Other</span> }}</div>`,
668+
want: want{
669+
code: "${$$maybeRenderHead($$result)}<div>${() => { if (value > 0.25) { return $$render`<span>Default</span>`} else if (value > 0.5) { return $$render`<span>Another</span>`} else if (value > 0.75) { return $$render`<span>Other</span>`} return $$render`<span>Yet Other</span>`}}</div>",
670+
},
671+
},
672+
{
673+
name: "nested expressions V",
674+
source: `<div><h1>title</h1>{list.map(group => <Fragment><h2>{group.label}</h2>{group.items.map(item => <span>{item}</span>)}</Fragment>)}</div>`,
675+
want: want{
676+
code: "${$$maybeRenderHead($$result)}<div><h1>title</h1>${list.map(group => $$render`${$$renderComponent($$result,'Fragment',Fragment,{},{\"default\": () => $$render`<h2>${group.label}</h2>${group.items.map(item => $$render`<span>${item}</span>`)}`,})}`)}</div>",
677+
},
678+
},
651679
{
652680
name: "expressions with JS comments",
653681
source: `---

‎internal/token.go‎

Lines changed: 35 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -248,10 +248,10 @@ type Tokenizer struct {
248248
// r is the source of the HTML text.
249249
r io.Reader
250250
// tt is the TokenType of the current token.
251-
tt TokenType
252-
prevTokenType TokenType
253-
fm FrontmatterState
254-
m MarkdownState
251+
tt TokenType
252+
prevToken Token
253+
fm FrontmatterState
254+
m MarkdownState
255255
// err is the first error encountered during tokenization. It is possible
256256
// for tt != Error && err != nil to hold: this means that Next returned a
257257
// valid token but the subsequent Next call will return an error token.
@@ -282,6 +282,7 @@ type Tokenizer struct {
282282
// expressionStack is an array of counters tracking opening and closing
283283
// braces in nested expressions
284284
expressionStack []int
285+
expressionElementStack [][]string
285286
openBraceIsExpressionStart bool
286287
// rawTag is the "script" in "</script>" that closes the next token. If
287288
// non-empty, the subsequent call to Next will return a raw or RCDATA text
@@ -1319,35 +1320,46 @@ func (z *Tokenizer) Loc() loc.Loc {
13191320
// An expression boundary means the next tokens should be treated as a JS expression
13201321
// (_do_ handle strings, comments, regexp, etc) rather than as plain text
13211322
func (z *Tokenizer) isAtExpressionBoundary() bool {
1322-
prev := z.prevTokenType
13231323
if len(z.expressionStack) == 0 {
13241324
return false
13251325
}
1326-
switch prev {
1327-
// Inside of expressions, these tokens flag that the following tokens are plain text (not JS)
1328-
case StartTagToken, EndTagToken, SelfClosingTagToken, EndExpressionToken:
1329-
return false
1326+
return len(z.expressionElementStack[len(z.expressionElementStack)-1]) == 0
1327+
}
1328+
1329+
func (z *Tokenizer) trackExpressionElementStack() {
1330+
if len(z.expressionStack) == 0 {
1331+
return
1332+
}
1333+
i := len(z.expressionElementStack) - 1
1334+
if z.tt == StartTagToken {
1335+
z.expressionElementStack[i] = append(z.expressionElementStack[i], string(z.buf[z.data.Start:z.data.End]))
1336+
} else if z.tt == EndTagToken {
1337+
stack := z.expressionElementStack[i]
1338+
if len(stack) > 0 {
1339+
for j := 1; j < len(stack)+1; j++ {
1340+
tok := stack[len(stack)-j]
1341+
if tok == string(z.buf[z.data.Start:z.data.End]) {
1342+
// When stack is balanced, reset `openBraceIsExpressionStart`
1343+
if len(stack) == 1 {
1344+
z.expressionElementStack[i] = make([]string, 0)
1345+
z.openBraceIsExpressionStart = false
1346+
} else {
1347+
z.expressionElementStack[i] = stack[:len(stack)-1]
1348+
}
1349+
}
1350+
}
1351+
}
13301352
}
1331-
return true
13321353
}
13331354

13341355
// Next scans the next token and returns its type.
13351356
func (z *Tokenizer) Next() TokenType {
1357+
z.prevToken = z.Token()
13361358
z.raw.Start = z.raw.End
13371359
z.data.Start = z.raw.End
13381360
z.data.End = z.raw.End
1339-
z.prevTokenType = z.tt
1361+
defer z.trackExpressionElementStack()
13401362

1341-
// This handles expressions nested inside of Frontmatter elements
1342-
// but preserves `{}` as text outside of elements
1343-
if z.fm == FrontmatterOpen {
1344-
tt := z.Token().Type
1345-
switch tt {
1346-
case StartTagToken, EndTagToken:
1347-
default:
1348-
z.openBraceIsExpressionStart = false
1349-
}
1350-
}
13511363
if z.rawTag != "" {
13521364
if z.rawTag == "plaintext" {
13531365
// Read everything up to EOF.
@@ -1426,7 +1438,6 @@ loop:
14261438
break loop
14271439
}
14281440

1429-
// We're in an element again, so open braces should open an expression
14301441
z.openBraceIsExpressionStart = z.noExpressionTag == ""
14311442

14321443
// Empty <> Fragment start tag
@@ -1731,6 +1742,7 @@ expression_loop:
17311742
if z.openBraceIsExpressionStart {
17321743
z.openBraceIsExpressionStart = false
17331744
z.expressionStack = append(z.expressionStack, 0)
1745+
z.expressionElementStack = append(z.expressionElementStack, make([]string, 0))
17341746
z.data.End = z.raw.End - 1
17351747
z.tt = StartExpressionToken
17361748
return z.tt
@@ -1752,6 +1764,7 @@ expression_loop:
17521764
if z.expressionStack[len(z.expressionStack)-1] == -1 {
17531765
z.openBraceIsExpressionStart = z.noExpressionTag == ""
17541766
z.expressionStack = z.expressionStack[0 : len(z.expressionStack)-1]
1767+
z.expressionElementStack = z.expressionElementStack[0 : len(z.expressionElementStack)-1]
17551768
z.data.End = z.raw.End
17561769
z.tt = EndExpressionToken
17571770
return z.tt

‎internal/token_test.go‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,20 @@ func TestBasic(t *testing.T) {
198198
}}</div>`,
199199
[]TokenType{StartTagToken, StartExpressionToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, EndExpressionToken, EndTagToken},
200200
},
201+
{
202+
"expression with multiple elements",
203+
`<div>{() => {
204+
if (value > 0.25) {
205+
return <span>Default</span>
206+
} else if (value > 0.5) {
207+
return <span>Another</span>
208+
} else if (value > 0.75) {
209+
return <span>Other</span>
210+
}
211+
return <span>Yet Other</span>
212+
}}</div>`,
213+
[]TokenType{StartTagToken, StartExpressionToken, TextToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, StartTagToken, TextToken, EndTagToken, TextToken, TextToken, EndExpressionToken, EndTagToken},
214+
},
201215
{
202216
"attribute expression with quoted braces",
203217
`<div value={"{"} />`,
@@ -338,8 +352,8 @@ func TestBasic(t *testing.T) {
338352
},
339353
{
340354
"fragment shorthand",
341-
`<h1>A{cond && <>item <span>B{text}</span></>}</h1>`,
342-
[]TokenType{StartTagToken, TextToken, StartExpressionToken, TextToken, StartTagToken, TextToken, StartTagToken, TextToken, StartExpressionToken, TextToken, EndExpressionToken, EndTagToken, EndTagToken, EndExpressionToken, EndTagToken},
355+
`<h1>A{cond && <>item <span>{text}</span></>}</h1>`,
356+
[]TokenType{StartTagToken, TextToken, StartExpressionToken, TextToken, StartTagToken, TextToken, StartTagToken, StartExpressionToken, TextToken, EndExpressionToken, EndTagToken, EndTagToken, EndExpressionToken, EndTagToken},
343357
},
344358
{
345359
"fragment",
@@ -375,6 +389,26 @@ func TestBasic(t *testing.T) {
375389
"({})",
376390
[]TokenType{TextToken, StartExpressionToken, EndExpressionToken, TextToken},
377391
},
392+
{
393+
"expression after text",
394+
`<h1>A{cond && <span>Test {text}</span>}</h1>`,
395+
[]TokenType{StartTagToken, TextToken, StartExpressionToken, TextToken, StartTagToken, TextToken, StartExpressionToken, TextToken, EndExpressionToken, EndTagToken, EndExpressionToken, EndTagToken},
396+
},
397+
{
398+
"expression surrounded by text",
399+
`<h1>A{cond && <span>Test {text} Cool</span>}</h1>`,
400+
[]TokenType{StartTagToken, TextToken, StartExpressionToken, TextToken, StartTagToken, TextToken, StartExpressionToken, TextToken, EndExpressionToken, TextToken, EndTagToken, EndExpressionToken, EndTagToken},
401+
},
402+
{
403+
"switch statement",
404+
`<div>{() => { switch(value) { case 'a': return <A></A>; case 'b': return <B />; case 'c': return <C></C> }}}</div>`,
405+
[]TokenType{StartTagToken, StartExpressionToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, EndTagToken, TextToken, TextToken, SelfClosingTagToken, TextToken, TextToken, StartTagToken, EndTagToken, TextToken, TextToken, TextToken, EndExpressionToken, EndTagToken},
406+
},
407+
{
408+
"switch statement with expression",
409+
`<div>{() => { switch(value) { case 'a': return <A>{value}</A>; case 'b': return <B />; case 'c': return <C>{value.map(i => <span>{i}</span>)}</C> }}}</div>`,
410+
[]TokenType{StartTagToken, StartExpressionToken, TextToken, TextToken, TextToken, TextToken, TextToken, TextToken, StartTagToken, StartExpressionToken, TextToken, EndExpressionToken, EndTagToken, TextToken, TextToken, SelfClosingTagToken, TextToken, TextToken, StartTagToken, StartExpressionToken, TextToken, StartTagToken, StartExpressionToken, TextToken, EndExpressionToken, EndTagToken, TextToken, EndExpressionToken, EndTagToken, TextToken, TextToken, TextToken, EndExpressionToken, EndTagToken},
411+
},
378412
}
379413

380414
runTokenTypeTest(t, Basic)

0 commit comments

Comments
 (0)