diff --git a/Project.toml b/Project.toml index 19211ca..cd2b4d3 100644 --- a/Project.toml +++ b/Project.toml @@ -6,11 +6,9 @@ version = "0.1.5" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" SIMD = "fdea26ae-647d-5447-a871-4b548cad5224" -ScanByte = "7b38b023-a4d7-4c5e-8d43-3f3097f304eb" [compat] SIMD = "3" -ScanByte = "0.3" julia = "1.6" [extras] diff --git a/src/NewlineLexers.jl b/src/NewlineLexers.jl index 5381d73..a99c12c 100644 --- a/src/NewlineLexers.jl +++ b/src/NewlineLexers.jl @@ -1,6 +1,6 @@ module NewlineLexers -using SIMD, ScanByte, Libdl +using SIMD, Libdl export Lexer, find_newlines!, possibly_not_in_string # const compat @@ -77,17 +77,9 @@ let # Feature detection -- copied from ScanByte.jl end end -@static if _AVOID_PLATFORM_SPECIFIC_LLVM_CODE - # The first argument is used to dispatch on a detected CPU feature set, - # in this case we want to use the generic fallback, so we provide "nothing". - @inline _internal_memchr(ptr::Ptr{UInt8}, len::UInt, valbs::Val) = ScanByte._memchr(nothing, ScanByte.SizedMemory(Ptr{UInt8}(ptr), len), valbs) -end -@static if !_AVOID_PLATFORM_SPECIFIC_LLVM_CODE - @inline function _internal_memchr(ptr::Ptr{UInt8}, len::UInt, valbs::Val) - ScanByte.memchr(ScanByte.SizedMemory(Ptr{UInt8}(ptr), len), valbs) - end -end -@inline _internal_memchr(ptr::Ptr{UInt8}, len::UInt, byte::UInt8) = ScanByte.memchr(ScanByte.SizedMemory(Ptr{UInt8}(ptr), len), byte) + +@inline _internal_memchr(ptr::Ptr{UInt8}, len::UInt, byte::UInt8) = @inbounds findfirst(==(byte), unsafe_wrap(Vector{UInt8}, ptr, len)) +@inline _internal_memchr(ptr::Ptr{UInt8}, len::UInt, byte::Val{B}) where B = @inbounds findfirst(in(B), unsafe_wrap(Vector{UInt8}, ptr, len)) const _DOUBLEQUOTE64 = Vec(ntuple(_->VecElement(UInt8('"')), 64)) const _SINGLEQUOTE64 = Vec(ntuple(_->VecElement(UInt8('\'')), 64)) @@ -210,7 +202,7 @@ function Base.show(io::IO, l::Lexer{E,OQ,CQ,NL}) where {E,OQ,CQ,NL} end # Returns a valid `bytes` for `ScanByte.memchr(..., bytes)` -@generated _scanbyte_bytes(::Lexer{E,OQ,CQ,NL}) where {E,OQ,CQ,NL} = Val(ScanByte.ByteSet((E,OQ,CQ,NL))) +@generated _scanbyte_bytes(::Lexer{E,OQ,CQ,NL}) where {E,OQ,CQ,NL} = OQ == CQ ? E == CQ ? Val((E,NL)) : Val((E,CQ,NL)) : Val((E,OQ,CQ,NL)) @generated _scanbyte_bytes(::Lexer{Nothing,Nothing,Nothing,NL}) where {NL} = NL # Take a 64-byte input and produce a 64-bit integer where the bits are set diff --git a/test/runtests.jl b/test/runtests.jl index 7f17e19..7f41147 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,6 @@ using Test using NewlineLexers using SIMD: Vec, vload -import ScanByte @info "_AVOID_PLATFORM_SPECIFIC_LLVM_CODE=$(NewlineLexers._AVOID_PLATFORM_SPECIFIC_LLVM_CODE)" @@ -103,13 +102,13 @@ end l = NewlineLexers.Lexer(IOBuffer(), nothing) @test NewlineLexers._scanbyte_bytes(l) == UInt8('\n') l = NewlineLexers.Lexer(IOBuffer(), UInt8('"'), UInt8('"'), UInt8('"')) - @test NewlineLexers._scanbyte_bytes(l) == Val(ScanByte.ByteSet((UInt8('"'), UInt8('\n')))) + @test NewlineLexers._scanbyte_bytes(l) == Val((UInt8('"'), UInt8('\n'))) l = NewlineLexers.Lexer(IOBuffer(), UInt8('\\'), UInt8('"'), UInt8('"')) - @test NewlineLexers._scanbyte_bytes(l) == Val(ScanByte.ByteSet((UInt8('\\'), UInt8('"'), UInt8('\n')))) + @test NewlineLexers._scanbyte_bytes(l) == Val((UInt8('\\'), UInt8('"'), UInt8('\n'))) l = NewlineLexers.Lexer(IOBuffer(), UInt8('\\'), UInt8('['), UInt8(']')) - @test NewlineLexers._scanbyte_bytes(l) == Val(ScanByte.ByteSet((UInt8('\\'), UInt8('['), UInt8(']'), UInt8('\n')))) + @test NewlineLexers._scanbyte_bytes(l) == Val((UInt8('\\'), UInt8('['), UInt8(']'), UInt8('\n'))) l = NewlineLexers.Lexer(IOBuffer(), UInt8('\\'), UInt8('['), UInt8(']'), UInt8('\r')) - @test NewlineLexers._scanbyte_bytes(l) == Val(ScanByte.ByteSet((UInt8('\\'), UInt8('['), UInt8(']'), UInt8('\r')))) + @test NewlineLexers._scanbyte_bytes(l) == Val((UInt8('\\'), UInt8('['), UInt8(']'), UInt8('\r'))) end @testset "_find_newlines_kernel!(l::Lexer{E,Q,Q}, ...)" begin