Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add semantic tokenization tests #5381

Merged
merged 4 commits into from
Jun 4, 2024
Merged

Add semantic tokenization tests #5381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5406ee6
Semantic tokenization fixes
OceanOak Jun 4, 2024
71aa652
Add semantic tokenization tests
OceanOak Jun 4, 2024
ab1a7b4
Small misc fixes in WT, WT2PT, etc for better semantic tokenization
OceanOak Jun 4, 2024
3325bcc
Small fix in package.json
OceanOak Jun 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions backend/testfiles/execution/stdlib/parser.dark
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,12 +298,8 @@ module TextToTextRoundtripping =
("type MyType<'a, 'b> = (List<'a> * List<'b>)" |> roundtripCliScript) = "type MyType<'a, 'b> =\n (List<'a> * List<'b>)"
("type Generic<'a> = { x: 'a }" |> roundtripCliScript) = "type Generic<'a> =\n { x: 'a }"

// // Enum type TODO
// ("type Thing = | A | B of Int64 | C of String * Bool" |> roundtripCliScript) = "type Thing =\n | A\n | B of Int64\n | C of String * Bool"

// record type
("type Person = {name: String}" |> roundtripCliScript) = "type Person =\n { name: String }"

("type Person = {name: String; age: Int64}" |> roundtripCliScript) = "type Person =\n { name: String\n age: Int64 }"
("type Person = {name: String; age: Int64; hasPet: Bool}" |> roundtripCliScript) = "type Person =\n { name: String\n age: Int64\n hasPet: Bool }"

Expand All @@ -328,6 +324,8 @@ module TextToTextRoundtripping =
("type MyEnum =\n | A of x: Int64 * y: Int64\n | B of z: String"
|> roundtripCliScript) = "type MyEnum =\n | A of x: Int64 * y: Int64\n | B of z: String"

// // Enum type TODO
// ("type Thing = | A | B of Int64 | C of String * Bool" |> roundtripCliScript) = "type Thing =\n | A\n | B of Int64\n | C of String * Bool"

module ConstantDeclaration =
("const unitConst = ()" |> roundtripCliScript) = "const unitConst = ()"
Expand Down
1,519 changes: 1,519 additions & 0 deletions backend/testfiles/execution/stdlib/semanticTokenization.dark
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions packages/darklang/languageTools/parser/core.dark
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ module Darklang =
| [ dictPairNode; _separator ] ->
let keyNode =
findAndParseRequired dictPairNode "key" (fun node ->
(Parser.getText node) |> Stdlib.Result.Result.Ok)
(node.range, Parser.getText node) |> Stdlib.Result.Result.Ok)

let symbolEqualsNode = findField dictPairNode "symbol_equals"

Expand All @@ -242,7 +242,7 @@ module Darklang =
| [ dictPairNode ] ->
let keyNode =
findAndParseRequired dictPairNode "key" (fun node ->
(Parser.getText node) |> Stdlib.Result.Result.Ok)
(node.range, Parser.getText node) |> Stdlib.Result.Result.Ok)

let symbolEqualsNode = findField dictPairNode "symbol_equals"

Expand Down
18 changes: 10 additions & 8 deletions packages/darklang/languageTools/parser/typeDeclaration.dark
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ module Darklang =
caseNode.typ != "indent" && caseNode.typ != "dedent")

|> Stdlib.List.map (fun caseNode ->
let symbolPipeNode = findField caseNode "symbol_pipe"
let caseNameNode = findField caseNode "case_name"

let keywordOf =
Expand All @@ -129,14 +130,15 @@ module Darklang =
| None -> createUnparseableError field)
|> Stdlib.Result.collect

match caseNameNode, fields with
| Ok caseNameNode, Ok fields ->
(WrittenTypes.TypeDeclaration.EnumCase
{ range = caseNode.range
name = (caseNameNode.range, caseNameNode.text)
fields = fields
description = ""
keywordOf = keywordOf })
match symbolPipeNode, caseNameNode, fields with
| Ok symbolPipe, Ok caseNameNode, Ok fields ->
(symbolPipe.range,
(WrittenTypes.TypeDeclaration.EnumCase
{ range = caseNode.range
name = (caseNameNode.range, caseNameNode.text)
fields = fields
description = ""
keywordOf = keywordOf }))
|> Stdlib.Result.Result.Ok
| Error _ -> createUnparseableError caseNode)

Expand Down
Loading