Skip to content

Commit 62d3371

Browse files
[REPL] Fix ends_with_semicolon on adjoint operator with comment (#58364)
Removes yet another ad-hoc lexer from the REPL and replaces it with JuliaSyntax. Fixes #46189, with tests taken from (and superseding) #57974. Co-authored-by: Páll Haraldsson <[email protected]>
1 parent 9986d97 commit 62d3371

File tree

2 files changed

+16
-41
lines changed

2 files changed

+16
-41
lines changed

stdlib/REPL/src/REPL.jl

Lines changed: 9 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ using Base.Meta, Sockets, StyledStrings
4242
using JuliaSyntaxHighlighting
4343
import InteractiveUtils
4444
import FileWatching
45+
import Base.JuliaSyntax: kind, @K_str, @KSet_str, Tokenize.tokenize
4546

4647
export
4748
AbstractREPL,
@@ -1700,49 +1701,16 @@ answer_color(r::StreamREPL) = r.answer_color
17001701
input_color(r::LineEditREPL) = r.envcolors ? Base.input_color() : r.input_color
17011702
input_color(r::StreamREPL) = r.input_color
17021703

1703-
let matchend = Dict("\"" => r"\"", "\"\"\"" => r"\"\"\"", "'" => r"'",
1704-
"`" => r"`", "```" => r"```", "#" => r"$"m, "#=" => r"=#|#=")
1705-
global _rm_strings_and_comments
1706-
function _rm_strings_and_comments(code::Union{String,SubString{String}})
1707-
buf = IOBuffer(sizehint = sizeof(code))
1708-
pos = 1
1709-
while true
1710-
i = findnext(r"\"(?!\"\")|\"\"\"|'|`(?!``)|```|#(?!=)|#=", code, pos)
1711-
isnothing(i) && break
1712-
match = SubString(code, i)
1713-
j = findnext(matchend[match]::Regex, code, nextind(code, last(i)))
1714-
if match == "#=" # possibly nested
1715-
nested = 1
1716-
while j !== nothing
1717-
nested += SubString(code, j) == "#=" ? +1 : -1
1718-
iszero(nested) && break
1719-
j = findnext(r"=#|#=", code, nextind(code, last(j)))
1720-
end
1721-
elseif match[1] != '#' # quote match: check non-escaped
1722-
while j !== nothing
1723-
notbackslash = findprev(!=('\\'), code, prevind(code, first(j)))::Int
1724-
isodd(first(j) - notbackslash) && break # not escaped
1725-
j = findnext(matchend[match]::Regex, code, nextind(code, first(j)))
1726-
end
1727-
end
1728-
isnothing(j) && break
1729-
if match[1] == '#'
1730-
print(buf, SubString(code, pos, prevind(code, first(i))))
1731-
else
1732-
print(buf, SubString(code, pos, last(i)), ' ', SubString(code, j))
1733-
end
1734-
pos = nextind(code, last(j))
1735-
end
1736-
print(buf, SubString(code, pos, lastindex(code)))
1737-
return String(take!(buf))
1738-
end
1739-
end
1740-
17411704
# heuristic function to decide if the presence of a semicolon
17421705
# at the end of the expression was intended for suppressing output
1743-
ends_with_semicolon(code::AbstractString) = ends_with_semicolon(String(code))
1744-
ends_with_semicolon(code::Union{String,SubString{String}}) =
1745-
contains(_rm_strings_and_comments(code), r";\s*$")
1706+
function ends_with_semicolon(code)
1707+
semi = false
1708+
for tok in tokenize(code)
1709+
kind(tok) in KSet"Whitespace NewlineWs Comment EndMarker" && continue
1710+
semi = kind(tok) == K";"
1711+
end
1712+
return semi
1713+
end
17461714

17471715
function banner(io::IO = stdout; short = false)
17481716
if Base.GIT_VERSION_INFO.tagged_commit

stdlib/REPL/test/repl.jl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -983,6 +983,13 @@ let ends_with_semicolon = REPL.ends_with_semicolon
983983
@test ends_with_semicolon("f()= 1;")
984984
# the next result does not matter because this is not legal syntax
985985
@test_nowarn ends_with_semicolon("1; #=# 2")
986+
987+
# #46189 - adjoint operator with comment
988+
@test ends_with_semicolon("W';") == true
989+
@test ends_with_semicolon("W'; # comment")
990+
@test !ends_with_semicolon("W'")
991+
@test !ends_with_semicolon("x'")
992+
@test !ends_with_semicolon("'a'")
986993
end
987994

988995
# PR #20794, TTYTerminal with other kinds of streams

0 commit comments

Comments
 (0)