Skip to content

Commit 7b724e4

Browse files
committed
Convert FSharpLineTokenizer.ScanToken to allocation-free voption struct return
Change ScanToken's signature from 'FSharpTokenInfo option * FSharpTokenizerLexState' to 'struct (FSharpTokenInfo voption * FSharpTokenizerLexState)', eliminating a per-token option allocation on the hot tokenization path. Update all in-tree callers: FSharpChecker.TokenizeLine, the editor Tokenizer, and the deprecated FSharp.LanguageService colorizer/scanner (Colorize.fs, Intellisense.fs). Also clean up incidental trailing whitespace in Intellisense.fs picked up by formatting.
1 parent 84f9fe8 commit 7b724e4

8 files changed

Lines changed: 183 additions & 175 deletions

File tree

‎docs/release-notes/.FSharp.Compiler.Service/11.0.100.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@
169169
* IL: use empty tables for members when possible ([PR #20249](https://github.com/dotnet/fsharp/pull/20249))
170170

171171
### Improved
172+
* `FSharpLineTokenizer.ScanToken` no longer allocates an `option` box per token; it returns a `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)` instead, removing per-token heap allocations on the hot tokenization path used by classification, brace matching, and the deprecated `FSharp.LanguageService` colorizer. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113))
172173

173174
* Nullness warning FS3261 on dotted method or property access (e.g. `x.Member`) now underlines the receiver expression and includes the member name and (when known) the binding name in the message. ([Issue #19658](https://github.com/dotnet/fsharp/issues/19658), [PR #19814](https://github.com/dotnet/fsharp/pull/19814))
174175
* Direct delegate construction ([PR ##19993](https://github.com/dotnet/fsharp/pull/19993))
@@ -188,5 +189,6 @@
188189
### Breaking Changes
189190

190191
* `FSharp.Compiler.Syntax.SynInterpolatedStringPart.FillExpr` now carries a `SynInterpolationFormatting` value (separating .NET alignment/format from printf specifiers) instead of an `Ident option`. ([PR #19971](https://github.com/dotnet/fsharp/pull/19971))
192+
* `FSharpLineTokenizer.ScanToken: lexState -> FSharpTokenInfo option * FSharpTokenizerLexState` now returns `struct (FSharpTokenInfo voption * FSharpTokenizerLexState)`. All in-tree callers (`FSharpChecker.TokenizeLine`, the editor `Tokenizer`, and the deprecated `FSharp.LanguageService` colorizer) have been updated accordingly. ([PR #20113](https://github.com/dotnet/fsharp/pull/20113))
191193
* Optimizer: don't inline named functions in debug builds ([PR #19548](https://github.com/dotnet/fsharp/pull/19548)
192194
* LexFilter: drop non-strict mode ([PR #20106](https://github.com/dotnet/fsharp/pull/20106))

‎src/Compiler/Service/ServiceLexing.fs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,7 +1011,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
10111011
false, (EOF LexerStateEncoding.revertToDefaultLexCont, 0, 0)
10121012

10131013
// Scan a token starting with the given lexer state
1014-
member x.ScanToken(lexState: FSharpTokenizerLexState) : FSharpTokenInfo option * FSharpTokenizerLexState =
1014+
member x.ScanToken(lexState: FSharpTokenizerLexState) : struct (FSharpTokenInfo voption * FSharpTokenizerLexState) =
10151015

10161016
use _ = UseBuildPhase BuildPhase.Parse
10171017
use _ = UseDiagnosticsLogger DiscardErrorsLogger
@@ -1022,12 +1022,12 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
10221022
let isCached, (token, leftc, rightc) = getTokenWithPosition lexcont
10231023

10241024
// Check for end-of-string and failure
1025-
let tokenDataOption, lexcontFinal, tokenTag =
1025+
let struct (tokenDataOption, lexcontFinal, tokenTag) =
10261026
match token with
10271027
| EOF lexcont ->
10281028
// End of text! No more tokens.
1029-
None, lexcont, 0
1030-
| LEX_FAILURE _ -> None, LexerStateEncoding.revertToDefaultLexCont, 0
1029+
struct (ValueNone, lexcont, 0)
1030+
| LEX_FAILURE _ -> struct (ValueNone, LexerStateEncoding.revertToDefaultLexCont, 0)
10311031
| _ ->
10321032
// Get the information about the token
10331033
let colorClass, charClass, triggerClass = TokenClassifications.tokenInfo token
@@ -1058,14 +1058,14 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
10581058
FullMatchedLength = fullMatchedLength
10591059
}
10601060

1061-
Some tokenData, lexcontFinal, tokenTag
1061+
struct (ValueSome tokenData, lexcontFinal, tokenTag)
10621062

10631063
// Check for patterns like #-IDENT and see if they look like meta commands for .fsx files. If they do then merge them into a single token.
1064-
let tokenDataOption, lexintFinal =
1064+
let struct (tokenDataOption, lexintFinal) =
10651065
let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal
10661066

10671067
match tokenDataOption, singleLineTokenState, tokenTagToTokenId tokenTag with
1068-
| Some tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH ->
1068+
| ValueSome tokenData, SingleLineTokenState.BeforeHash, TOKEN_HASH ->
10691069
// Don't allow further matches.
10701070
singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible
10711071
// Peek at the next token
@@ -1110,17 +1110,17 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, fi
11101110

11111111
let lexintFinal = LexerStateEncoding.encodeLexInt lexcontFinal
11121112

1113-
Some tokenData, lexintFinal
1114-
| _ -> tokenDataOption, lexintFinal
1115-
| _ -> tokenDataOption, lexintFinal
1113+
struct (ValueSome tokenData, lexintFinal)
1114+
| _ -> struct (tokenDataOption, lexintFinal)
1115+
| _ -> struct (tokenDataOption, lexintFinal)
11161116
| _, SingleLineTokenState.BeforeHash, TOKEN_WHITESPACE ->
11171117
// Allow leading whitespace.
1118-
tokenDataOption, lexintFinal
1118+
struct (tokenDataOption, lexintFinal)
11191119
| _ ->
11201120
singleLineTokenState <- SingleLineTokenState.NoFurtherMatchPossible
1121-
tokenDataOption, lexintFinal
1121+
struct (tokenDataOption, lexintFinal)
11221122

1123-
tokenDataOption, lexintFinal
1123+
struct (tokenDataOption, lexintFinal)
11241124

11251125
static member ColorStateOfLexState(lexState: FSharpTokenizerLexState) =
11261126
LexerStateEncoding.colorStateOfLexState lexState

‎src/Compiler/Service/ServiceLexing.fsi‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ type FSharpTokenInfo =
310310
[<Sealed>]
311311
type FSharpLineTokenizer =
312312
/// Scan one token from the line
313-
member ScanToken: lexState: FSharpTokenizerLexState -> FSharpTokenInfo option * FSharpTokenizerLexState
313+
member ScanToken: lexState: FSharpTokenizerLexState -> struct (FSharpTokenInfo voption * FSharpTokenizerLexState)
314314

315315
/// Get the color state from the lexer state
316316
static member ColorStateOfLexState: FSharpTokenizerLexState -> FSharpTokenizerColorState

‎src/Compiler/Service/service.fs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -629,16 +629,22 @@ type FSharpChecker
629629
member _.TokenizeLine(line: string, state: FSharpTokenizerLexState) =
630630
let tokenizer = FSharpSourceTokenizer([], None, None)
631631
let lineTokenizer = tokenizer.CreateLineTokenizer line
632-
let mutable state = (None, state)
632+
let mutable lexState = state
633+
let mutable token = ValueNone
634+
635+
let scanNext () =
636+
let struct (t, s) = lineTokenizer.ScanToken(lexState)
637+
token <- t
638+
lexState <- s
639+
token.IsSome
633640

634641
let tokens =
635642
[|
636-
while (state <- lineTokenizer.ScanToken(snd state)
637-
(fst state).IsSome) do
638-
yield (fst state).Value
643+
while scanNext () do
644+
yield token.Value
639645
|]
640646

641-
tokens, snd state
647+
tokens, lexState
642648

643649
/// Tokenize an entire file, line by line
644650
member x.TokenizeFile(source: string) : FSharpTokenInfo[][] =

‎tests/FSharp.Compiler.Service.Tests/TokenizerTests.fs‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@ open Xunit
77

88
let rec parseLine(line: string, state: FSharpTokenizerLexState ref, tokenizer: FSharpLineTokenizer) = seq {
99
match tokenizer.ScanToken(state.Value) with
10-
| Some(tok), nstate ->
10+
| ValueSome(tok), nstate ->
1111
let str = line.Substring(tok.LeftColumn, tok.RightColumn - tok.LeftColumn + 1)
1212
yield str, tok
1313
state.Value <- nstate
1414
yield! parseLine(line, state, tokenizer)
15-
| None, nstate ->
15+
| ValueNone, nstate ->
1616
state.Value <- nstate }
1717

1818
let tokenizeLines (lines:string[]) =
@@ -30,21 +30,21 @@ let scanTokens (defines: string list) (source: string) =
3030
let tokenizer = sourceTok.CreateLineTokenizer(source)
3131
let rec loop (state: FSharpTokenizerLexState) acc =
3232
match tokenizer.ScanToken(state) with
33-
| Some tok, nstate -> loop nstate (tok :: acc)
34-
| None, _ -> List.rev acc
33+
| ValueSome tok, nstate -> loop nstate (tok :: acc)
34+
| ValueNone, _ -> List.rev acc
3535
loop FSharpTokenizerLexState.Initial []
3636

3737
[<Fact>]
3838
let ``Tokenizer test - simple let with string``() =
39-
let tokenizedLines =
39+
let tokenizedLines =
4040
tokenizeLines
4141
[| "// Sets the hello world variable"
4242
"let hello = \"Hello world\" " |]
4343

44-
let actual =
44+
let actual =
4545
[ for lineNo, lineToks in tokenizedLines do
4646
yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
47-
let expected =
47+
let expected =
4848
[(0,
4949
[("LINE_COMMENT", "//"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "Sets");
5050
("LINE_COMMENT", " "); ("LINE_COMMENT", "the"); ("LINE_COMMENT", " ");
@@ -57,14 +57,14 @@ let ``Tokenizer test - simple let with string``() =
5757
("STRING_TEXT", "\""); ("STRING_TEXT", "Hello"); ("STRING_TEXT", " ");
5858
("STRING_TEXT", "world"); ("STRING", "\""); ("WHITESPACE", " ")])]
5959

60-
if actual <> expected then
60+
if actual <> expected then
6161
printfn "actual = %A" actual
6262
printfn "expected = %A" expected
6363
actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)
6464

6565
[<Fact>]
6666
let ``Tokenizer test 2 - single line non-nested string interpolation``() =
67-
let tokenizedLines =
67+
let tokenizedLines =
6868
tokenizeLines
6969
[| "// Tests tokenizing string interpolation"
7070
"let hello0 = $\"\""
@@ -74,10 +74,10 @@ let ``Tokenizer test 2 - single line non-nested string interpolation``() =
7474
"let hello1v = @$\"Hello world\" "
7575
"let hello2v = @$\"Hello world {1+1} = {2}\" " |]
7676

77-
let actual =
77+
let actual =
7878
[ for lineNo, lineToks in tokenizedLines do
7979
yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
80-
let expected =
80+
let expected =
8181
[(0,
8282
[("LINE_COMMENT", "//"); ("LINE_COMMENT", " "); ("LINE_COMMENT", "Tests");
8383
("LINE_COMMENT", " "); ("LINE_COMMENT", "tokenizing"); ("LINE_COMMENT", " ");
@@ -121,23 +121,23 @@ let ``Tokenizer test 2 - single line non-nested string interpolation``() =
121121
("STRING_TEXT", "="); ("STRING_TEXT", " "); ("INTERP_STRING_PART", "{");
122122
("INT32", "2"); ("STRING_TEXT", "}"); ("INTERP_STRING_END", "\"");
123123
("WHITESPACE", " ")]);]
124-
125-
if actual <> expected then
124+
125+
if actual <> expected then
126126
printfn "actual = %A" actual
127127
printfn "expected = %A" expected
128128
actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)
129129

130130
[<Fact>]
131131
let ``Tokenizer test - multiline non-nested string interpolation``() =
132-
let tokenizedLines =
132+
let tokenizedLines =
133133
tokenizeLines
134134
[| "let hello1t = $\"\"\"abc {1+"
135135
" 1} def\"\"\"" |]
136136

137-
let actual =
137+
let actual =
138138
[ for lineNo, lineToks in tokenizedLines do
139139
yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
140-
let expected =
140+
let expected =
141141
[(0,
142142
[("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1t");
143143
("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
@@ -146,27 +146,27 @@ let ``Tokenizer test - multiline non-nested string interpolation``() =
146146
(1,
147147
[("WHITESPACE", " "); ("INT32", "1"); ("STRING_TEXT", "}");
148148
("STRING_TEXT", " "); ("STRING_TEXT", "def"); ("INTERP_STRING_END", "\"\"\"")])]
149-
150-
if actual <> expected then
149+
150+
if actual <> expected then
151151
printfn "actual = %A" actual
152152
printfn "expected = %A" expected
153153
actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)
154154

155155
[<Fact>]
156156
// checks nested '{' and nested single-quote strings
157157
let ``Tokenizer test - multi-line nested string interpolation``() =
158-
let tokenizedLines =
158+
let tokenizedLines =
159159
tokenizeLines
160160
[| "let hello1t = $\"\"\"abc {\"a\" + "
161161
" { "
162162
" contents = \"b\" "
163163
" }.contents "
164164
" } def\"\"\"" |]
165165

166-
let actual =
166+
let actual =
167167
[ for lineNo, lineToks in tokenizedLines do
168168
yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
169-
let expected =
169+
let expected =
170170
[(0,
171171
[("LET", "let"); ("WHITESPACE", " "); ("IDENT", "hello1t");
172172
("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
@@ -188,31 +188,31 @@ let ``Tokenizer test - multi-line nested string interpolation``() =
188188
(4,
189189
[("WHITESPACE", " "); ("STRING_TEXT", "}");
190190
("STRING_TEXT", " "); ("STRING_TEXT", "def"); ("INTERP_STRING_END", "\"\"\"")])]
191-
192-
if actual <> expected then
191+
192+
if actual <> expected then
193193
printfn "actual = %A" actual
194194
printfn "expected = %A" expected
195195
actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)
196196

197197
[<Fact>]
198198
let ``Tokenizer test - single-line nested string interpolation``() =
199-
let tokenizedLines =
199+
let tokenizedLines =
200200
tokenizeLines
201201
[| " $\"abc { { contents = 1 } }\" " |]
202202

203-
let actual =
203+
let actual =
204204
[ for lineNo, lineToks in tokenizedLines do
205205
yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
206-
let expected =
206+
let expected =
207207
[(0,
208208
[("WHITESPACE", " "); ("STRING_TEXT", "$\""); ("STRING_TEXT", "abc");
209209
("STRING_TEXT", " "); ("INTERP_STRING_BEGIN_PART", "{"); ("WHITESPACE", " ");
210210
("LBRACE", "{"); ("WHITESPACE", " "); ("IDENT", "contents");
211211
("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " "); ("INT32", "1");
212212
("WHITESPACE", " "); ("RBRACE", "}"); ("WHITESPACE", " ");
213213
("STRING_TEXT", "}"); ("INTERP_STRING_END", "\""); ("WHITESPACE", " ")])]
214-
215-
if actual <> expected then
214+
215+
if actual <> expected then
216216
printfn "actual = %A" actual
217217
printfn "expected = %A" expected
218218
actual |> Assert.shouldBeEqualWith expected (sprintf "actual and expected did not match,actual =\n%A\nexpected=\n%A\n" actual expected)
@@ -291,15 +291,15 @@ let ``Tokenizer test - optional parameters with question mark``() =
291291
let actual =
292292
[ for lineNo, lineToks in tokenizedLines do
293293
yield lineNo, [ for str, info in lineToks do yield info.TokenName, str ] ]
294-
294+
295295
let expected =
296296
[(0,
297297
[("MEMBER", "member"); ("WHITESPACE", " "); ("UNDERSCORE", "_"); ("DOT", ".");
298298
("IDENT", "memb"); ("LPAREN", "("); ("QMARK", "?");
299299
("IDENT", "optional"); ("COLON", ":"); ("IDENT", "string");
300300
("RPAREN", ")"); ("WHITESPACE", " "); ("EQUALS", "="); ("WHITESPACE", " ");
301301
("IDENT", "optional")])]
302-
302+
303303
if actual <> expected then
304304
printfn "actual = %A" actual
305305
printfn "expected = %A" expected

‎vsintegration/src/FSharp.Editor/LanguageService/Tokenizer.fs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -528,12 +528,12 @@ module internal Tokenizer =
528528
tokens.Add(SavedTokenInfo.Create token)
529529

530530
let scanAndColorNextToken () =
531-
let info, nextLexState = lineTokenizer.ScanToken(previousLexState)
531+
let struct (info, nextLexState) = lineTokenizer.ScanToken(previousLexState)
532532
previousLexState <- nextLexState
533533

534534
// Apply some hacks to clean up the token stream (we apply more later)
535535
match info with
536-
| Some info when info.Tag = FSharpTokenTag.INT32_DOT_DOT ->
536+
| ValueSome info when info.Tag = FSharpTokenTag.INT32_DOT_DOT ->
537537
processToken
538538
{
539539
LeftColumn = info.LeftColumn
@@ -558,7 +558,7 @@ module internal Tokenizer =
558558
FullMatchedLength = 2
559559
}
560560

561-
| Some info -> processToken info
561+
| ValueSome info -> processToken info
562562
| _ -> ()
563563

564564
info.IsSome

0 commit comments

Comments
 (0)