Skip to content

Commit eaa14f7

Browse files
OceanOakOceanOak
committed
wip
1 parent 6930ff4 commit eaa14f7

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

83 files changed

+9790
-8930
lines changed

‎backend/src/BuiltinExecution/Libs/Parser.fs‎

Lines changed: 96 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ module BuiltinExecution.Libs.Parser
33
open FSharp.Control.Tasks
44
open System.Threading.Tasks
55
open System.Text
6+
open System.Globalization
7+
open System
68

79
open Prelude
810
open LibExecution.RuntimeTypes
@@ -20,6 +22,99 @@ let pointTypeName = FQTypeName.fqPackage IDs.point
2022
let rangeTypeName = FQTypeName.fqPackage IDs.range
2123
let parsedNodeTypeName = FQTypeName.fqPackage IDs.parsedNode
2224

25+
let parse (sourceCode : string) : Dval =
26+
// This was added to handle EGCs correctly
27+
let byteIndexToCharIndex (byteIndex : int) (text : string) : int =
28+
let bytes = Encoding.UTF8.GetBytes(text)
29+
let subText = Encoding.UTF8.GetString(bytes, 0, byteIndex)
30+
StringInfo.ParseCombiningCharacters(subText).Length
31+
32+
let processLine (line : string) (startIndex : int) (endIndex : int) =
33+
let textElements = StringInfo.GetTextElementEnumerator(line)
34+
let mutable result = ""
35+
let mutable currentIndex = 0
36+
while textElements.MoveNext() do
37+
if currentIndex >= startIndex && currentIndex < endIndex then
38+
result <- result + (textElements.GetTextElement())
39+
currentIndex <- currentIndex + 1
40+
result
41+
42+
let rec mapNodeAtCursor (cursor : TreeCursor) : Dval =
43+
let mutable children = []
44+
45+
if cursor.GotoFirstChild() then
46+
children <- children @ [ mapNodeAtCursor cursor ]
47+
48+
while cursor.GotoNextSibling() do
49+
children <- children @ [ mapNodeAtCursor cursor ]
50+
51+
cursor.GotoParent() |> ignore<bool>
52+
53+
let fields =
54+
let mapPoint (point : Point) =
55+
let pointRow = point.row + 1
56+
let fields = [ "row", DInt64 pointRow; "column", DInt64 point.column ]
57+
DRecord(pointTypeName, pointTypeName, [], Map fields)
58+
59+
let startPos = cursor.Current.StartPosition
60+
let endPos = cursor.Current.EndPosition
61+
62+
let range =
63+
let fields = [ "start", mapPoint startPos; "end_", mapPoint endPos ]
64+
DRecord(rangeTypeName, rangeTypeName, [], Map fields)
65+
66+
let sourceText =
67+
let lines = String.splitOnNewline sourceCode
68+
if lines.Length = 0 then
69+
""
70+
else
71+
let startLine = lines[startPos.row]
72+
let endLine = lines[endPos.row]
73+
let startCharIndex = byteIndexToCharIndex startPos.column startLine
74+
let endCharIndex = byteIndexToCharIndex endPos.column endLine
75+
76+
match startPos.row with
77+
| row when row = endPos.row ->
78+
processLine startLine startCharIndex endCharIndex
79+
| _ ->
80+
let firstLine = processLine startLine startCharIndex startLine.Length
81+
let middleLines =
82+
if startPos.row + 1 <= endPos.row - 1 then
83+
lines[startPos.row + 1 .. endPos.row - 1]
84+
|> List.map (fun line -> processLine line 0 line.Length)
85+
else
86+
[]
87+
let lastLine = processLine endLine 0 endCharIndex
88+
String.concat "\n" (firstLine :: middleLines @ [ lastLine ])
89+
90+
let stringToHex (input: string) =
91+
let bytes = Encoding.UTF8.GetBytes(input)
92+
BitConverter.ToString(bytes)
93+
94+
debuG "sourceTextHex" (stringToHex sourceText)
95+
96+
97+
let fieldName =
98+
if cursor.FieldName = null then
99+
Dval.optionNone KTString
100+
else
101+
Dval.optionSome KTString (DString cursor.FieldName)
102+
103+
[ ("fieldName", fieldName)
104+
("typ", DString cursor.Current.Kind)
105+
("text", DString sourceText)
106+
("range", range)
107+
("children", DList(VT.customType parsedNodeTypeName [], children)) ]
108+
109+
DRecord(parsedNodeTypeName, parsedNodeTypeName, [], Map fields)
110+
111+
112+
let parser = new Parser(Language = DarklangLanguage.create ())
113+
114+
let tree =
115+
parser.Parse(Encoding.UTF8.GetBytes sourceCode, InputEncoding.Utf8, None)
116+
tree.Root.Walk() |> mapNodeAtCursor
117+
23118
let fns : List<BuiltInFn> =
24119
[ { name = fn "parserParseToSimplifiedTree" 0
25120
typeParams = []
@@ -28,80 +123,7 @@ let fns : List<BuiltInFn> =
28123
description = "Parses some Darklang code"
29124
fn =
30125
(function
31-
| _, _, [ DString sourceCode ] ->
32-
// This was added to handle EGCs correctly
33-
let byteIndexToCharIndex (byteIndex : int) (text : string) : int =
34-
let bytes = Encoding.UTF8.GetBytes(text)
35-
let subText = Encoding.UTF8.GetString(bytes, 0, byteIndex)
36-
subText.Length
37-
38-
let rec mapNodeAtCursor (cursor : TreeCursor) : Dval =
39-
let mutable children = []
40-
41-
if cursor.GotoFirstChild() then
42-
children <- children @ [ mapNodeAtCursor cursor ]
43-
44-
while cursor.GotoNextSibling() do
45-
children <- children @ [ mapNodeAtCursor cursor ]
46-
47-
cursor.GotoParent() |> ignore<bool>
48-
49-
let fields =
50-
let mapPoint (point : Point) =
51-
let fields =
52-
[ "row", DInt64 point.row; "column", DInt64 point.column ]
53-
DRecord(pointTypeName, pointTypeName, [], Map fields)
54-
55-
let startPos = cursor.Current.StartPosition
56-
let endPos = cursor.Current.EndPosition
57-
58-
let range =
59-
let fields = [ "start", mapPoint startPos; "end_", mapPoint endPos ]
60-
DRecord(rangeTypeName, rangeTypeName, [], Map fields)
61-
62-
let startCharIndex = byteIndexToCharIndex startPos.column sourceCode
63-
let endCharIndex = byteIndexToCharIndex endPos.column sourceCode
64-
65-
let sourceText =
66-
let lines = String.splitOnNewline sourceCode
67-
if lines.Length = 0 then
68-
""
69-
else
70-
match startPos.row with
71-
| row when row = endPos.row ->
72-
lines[row][startCharIndex .. (endCharIndex - 1)]
73-
| _ ->
74-
let firstLine = lines[startPos.row][startCharIndex..]
75-
let middleLines =
76-
if startPos.row + 1 <= endPos.row - 1 then
77-
lines[startPos.row + 1 .. endPos.row - 1]
78-
else
79-
[]
80-
let lastLine = lines[endPos.row][.. (endCharIndex - 1)]
81-
82-
String.concat "\n" (firstLine :: middleLines @ [ lastLine ])
83-
84-
let fieldName =
85-
if cursor.FieldName = null then
86-
Dval.optionNone KTString
87-
else
88-
Dval.optionSome KTString (DString cursor.FieldName)
89-
90-
[ ("fieldName", fieldName)
91-
("typ", DString cursor.Current.Kind)
92-
("text", DString sourceText)
93-
("range", range)
94-
("children", DList(VT.customType parsedNodeTypeName [], children)) ]
95-
96-
DRecord(parsedNodeTypeName, parsedNodeTypeName, [], Map fields)
97-
98-
99-
let parser = new Parser(Language = DarklangLanguage.create ())
100-
101-
let tree =
102-
parser.Parse(Encoding.UTF8.GetBytes sourceCode, InputEncoding.Utf8, None)
103-
104-
tree.Root.Walk() |> mapNodeAtCursor |> Ply
126+
| _, _, [ DString sourceCode ] -> (parse sourceCode) |> Ply
105127
| _ -> incorrectArgs ())
106128
sqlSpec = NotQueryable
107129
previewable = Impure

‎backend/src/LibExecution/PackageIDs.fs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,9 @@ module Fn =
383383
let parseSingleTestFromFile =
384384
p [] "parseSingleTestFromFile" "53f3fbc6-25fd-427a-ab0d-ba0559543c99"
385385

386+
let parseTestFile =
387+
p [] "parseTestFile" "95dc8d95-dd38-4df2-aaac-9e78187a17be"
388+
386389
// what we expose to the outside world
387390
let idForName
388391
(owner : string)

‎backend/src/LibExecution/ProgramTypesToDarkTypes.fs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -921,6 +921,9 @@ module Expr =
921921
Exception.raiseInternal "Invalid record update" [ "update", update ])
922922
PT.ERecordUpdate(uint64 id, fromDT record, updates)
923923

924+
| DEnum(_, _, [], "EConstant", [ DInt64 id; name ]) ->
925+
PT.EConstant(uint64 id, NameResolution.fromDT FQConstantName.fromDT name)
926+
924927
| e -> Exception.raiseInternal "Invalid Expr" [ "e", e ]
925928

926929

@@ -987,6 +990,7 @@ module Const =
987990
| DEnum(_, _, [], "CBool", [ DBool b ]) -> PT.Const.CBool b
988991
| DEnum(_, _, [], "CString", [ DString s ]) -> PT.Const.CString s
989992
| DEnum(_, _, [], "CChar", [ DChar c ]) -> PT.Const.CChar c
993+
| DEnum(_, _, [], "CChar", [ DString c ]) -> PT.Const.CChar c
990994
| DEnum(_, _, [], "CFloat", [ sign; DString w; DString f ]) ->
991995
PT.Const.CFloat(Sign.fromDT sign, w, f)
992996
| DEnum(_, _, [], "CUnit", []) -> PT.Const.CUnit

‎backend/src/LibExecution/ProgramTypesToRuntimeTypes.fs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,20 @@ module MatchPattern =
165165
module Expr =
166166
let rec toRT (e : PT.Expr) : RT.Expr =
167167
match e with
168-
| PT.EChar(id, char) -> RT.EChar(id, char)
168+
| PT.EChar(id, char) ->
169+
let char =
170+
char
171+
|> fun s -> s.Replace(@"\t", "\t")
172+
|> fun s -> s.Replace(@"\n", "\n")
173+
|> fun s -> s.Replace(@"\r", "\r")
174+
|> fun s -> s.Replace(@"\b", "\b")
175+
|> fun s -> s.Replace(@"\f", "\f")
176+
|> fun s -> s.Replace(@"\v", "\v")
177+
|> fun s -> s.Replace(@"\""", "\"")
178+
|> fun s -> s.Replace(@"\'", "'")
179+
|> fun s -> s.Replace(@"\\", "\\")
180+
181+
RT.EChar(id, char)
169182
| PT.EInt64(id, num) -> RT.EInt64(id, num)
170183
| PT.EUInt64(id, num) -> RT.EUInt64(id, num)
171184
| PT.EInt8(id, num) -> RT.EInt8(id, num)
@@ -350,7 +363,28 @@ module Expr =
350363

351364
and stringSegmentToRT (segment : PT.StringSegment) : RT.StringSegment =
352365
match segment with
353-
| PT.StringText text -> RT.StringText text
366+
| PT.StringText text ->
367+
text
368+
|> fun s ->
369+
System.Text.RegularExpressions.Regex.Replace(s, @"\\x([0-9A-Fa-f]{2})",
370+
fun m ->
371+
let hexValue = System.Convert.ToByte(m.Groups[1].Value, 16)
372+
string (char hexValue))
373+
|> fun s ->
374+
System.Text.RegularExpressions.Regex.Replace(s, @"\\u([0-9A-Fa-f]{4})",
375+
fun m ->
376+
let unicodeValue = System.Convert.ToInt32(m.Groups[1].Value, 16)
377+
string (char unicodeValue))
378+
|> fun s -> s.Replace(@"\t", "\t")
379+
|> fun s -> s.Replace(@"\n", "\n")
380+
|> fun s -> s.Replace(@"\r", "\r")
381+
|> fun s -> s.Replace(@"\b", "\b")
382+
|> fun s -> s.Replace(@"\f", "\f")
383+
|> fun s -> s.Replace(@"\v", "\v")
384+
|> fun s -> s.Replace(@"\""", "\"")
385+
|> fun s -> s.Replace(@"\'", "'")
386+
|> fun s -> s.Replace(@"\\", "\\")
387+
|> RT.StringText
354388
| PT.StringInterpolation expr -> RT.StringInterpolation(toRT expr)
355389

356390

‎backend/src/LibTreeSitter/TreeSitter.fs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,7 @@ type Node(handle : Native.TsNode) =
194194
override _.ToString() =
195195
let cPtr = Native.ts_node_string.Invoke(handle)
196196
try
197+
// check on this
197198
Marshal.PtrToStringAnsi(cPtr)
198199
finally
199200
Marshal.FreeHGlobal(cPtr)

‎backend/src/Prelude/Prelude.fs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ let readFloat (f : float) : (Sign * string * string) =
290290
let makeFloat (sign : Sign) (whole : string) (fraction : string) : float =
291291
try
292292
if whole <> "" then assert_ "non-zero string" [] (whole[0] <> '-')
293-
if whole <> "0" then assertRe $"makefloat" "[1-9][0-9]*" whole
293+
if whole <> "0" then assertRe $"makefloat" "0*[0-9]+" whole
294294
let sign =
295295
match sign with
296296
| Positive -> ""

‎backend/src/Prelude/String.fs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ let lengthInEgcs (s : string) : int =
3030
System.Globalization.StringInfo(s).LengthInTextElements
3131

3232
let normalize (s : string) : string = s.Normalize()
33+
// let normalize (s : string) : string =
34+
// s.Normalize(System.Text.NormalizationForm.FormC)
3335

3436
let equalsCaseInsensitive (s1 : string) (s2 : string) : bool =
3537
System.String.Equals(s1, s2, System.StringComparison.InvariantCultureIgnoreCase)

‎backend/testfiles/execution/cloud/_events.dark‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@
44
type FruitRecord = { fruits: List<String> }
55

66
// getQueue works
7-
Builtin.testGetQueue_v0 "TestWorker" = []
7+
Builtin.testGetQueue "TestWorker" = []
88

99
// emit works
1010
(let _ = Builtin.emit "value" "TestWorker"
11-
let queue = Builtin.testGetQueue_v0 "TestWorker"
11+
let queue = Builtin.testGetQueue "TestWorker"
1212
queue) = [ "\"value\"" ]
1313

1414
// emit works with mixed values
1515
(let _ = Builtin.emit "value" "TestWorker"
1616
let _ = Builtin.emit 1 "TestWorker"
1717
let _ = Builtin.emit (FruitRecord { fruits = [ "apple"; "banana" ] }) "TestWorker"
18-
let queue = Builtin.testGetQueue_v0 "TestWorker"
19-
Stdlib.List.sort queue) = [ "\"value\""
20-
"1"
21-
"FruitRecord {\n fruits: [\n \"apple\", \"banana\"\n ]\n}" ]
18+
let queue = Builtin.testGetQueue "TestWorker"
19+
Stdlib.List.sort queue) = [ "\"value\""
20+
"1"
21+
"FruitRecord {\n fruits: [\n \"apple\", \"banana\"\n ]\n}" ]

0 commit comments

Comments
 (0)