JSON.parse a Vector of {"key": key, "val": value} objects into a Dict

[Affie]

Hi, I need help parsing a JSON payload of the form:

[
    { "key": "a", "val": "1" },
    { "key": "b", "val": "2" }
]

into a Dict{Symbol, String} using typed materialization.

I tried:

• Wrapping the Dict in a struct (Metadata) and using StructUtils.make
  • I tried various simmilar things but can't quite make out how to interpret the errors like: ERROR: TypeError: in typeassert, expected Metadata, got a value of type Pair{Symbol, String}
• metadata::Dict{Symbol, String} &(json=(lower=lowerMetadata, lift=liftMetadata))
  • Got this error: MethodError: Cannot convert an object of type JSON.Object{String, Any} to an object of type String

What is the correct way to use StructUtils.jl/JSON.jl in this case?

[ReubenJ]

You've got a vector of objects as input, so you'll have to parse to a vector of your metadata objects.

Assuming you have a struct like

struct Metadata
	key::Symbol
	val::String
end

Then you can parse your JSON input like so

julia> using JSON

julia> struct Metadata
           key::Symbol
           val::String
       end

julia> input = """[
           { "key": "a", "val": "1" },
           { "key": "b", "val": "2" }
       ]"""
"[\n    { \"key\": \"a\", \"val\": \"1\" },\n    { \"key\": \"b\", \"val\": \"2\" }\n]"

julia> JSON.parse(input, Vector{Metadata})
2-element Vector{Metadata}:
 Metadata(:a, "1")
 Metadata(:b, "2")

[Affie]

Thanks for the feedback, is that the only way?

I would prefer to parse into metadata::Dict{Symbol, String} where metadata is one of the fields in another struct.

struct Foo
    metadata::Dict{Symbol, String}
end

[ReubenJ]

Ah, what do you want the constructed Foo object to look like then?

Something like

Foo(Dict(:a => "1", :b => "2"))

?

[ReubenJ]

For that, you can use the non-struct approach mentioned in the docs here.

julia> @nonstruct struct Foo
           metadata::Dict{Symbol, String}
       end

julia> input = """[
                  { "key": "a", "val": "1" },
                  { "key": "b", "val": "2" }
              ]""";

julia> function StructUtils.lift(::Type{Foo}, json_vector::Vector)
               Foo(Dict(Symbol(x["key"]) => x["val"] for x in json_vector))
       end

julia> JSON.parse(input, Foo)
Foo(Dict(:a => "1", :b => "2"))

[Affie]

Thanks that can work and I did see that option in the docs. I was hoping to avoid @nonstruct.

Before I close this as answered, I just want to make extra sure that @nonstruct on the top level struct was the only option. The metadata::Dict{Symbol, String} field is not the only field and it wil be reused in a few other structs.

I tried something similar to this before and adapted your suggestion to show what kind of solution I was hoping for:

julia> @nonstruct struct Metadata <: AbstractDict{Symbol, String}
           x::Dict{Symbol, String}
       end

julia> Metadata() = Metadata(Dict{Symbol, String}())

julia> Base.getindex(m::Metadata, k) = m.x[k]

julia> Base.setindex!(m::Metadata, v, k) = (m.x[k] = v; m)

julia> Base.length(m::Metadata) = length(m.x)

julia> Base.iterate(m::Metadata) = iterate(m.x)

julia> Base.iterate(m::Metadata, s) = iterate(m.x, s)

julia> function StructUtils.lift(::Type{Metadata}, json_vector::Vector)
           return Metadata(Dict(Symbol(x["key"]) => x["val"] for x in json_vector))
       end

julia> function StructUtils.lower(m::Metadata)
           return map(collect(pairs(m.x))) do (key, val)
               (key=key, val=val)
           end
       end

julia> struct Foo
           metadata::Metadata
           other_fields::Int
       end

julia> jstring = JSON.json(Foo(Metadata(Dict(:a=>"1", :b=>"2")), 43))
"{\"metadata\":[{\"key\":\"a\",\"val\":\"1\"},{\"key\":\"b\",\"val\":\"2\"}],\"other_fields\":43}"

julia> JSON.parse(jstring, Foo)
ERROR: MethodError: Cannot `convert` an object of type JSON.Object{String, Any} to an object of type String
The function `convert` exists, but no method is defined for this combination of argument types.

Closest candidates are:
  convert(::Type{String}, ::JSON.PtrString)
   @ JSON ~/.julia/dev/JSON/src/lazy.jl:426
  convert(::Type{String}, ::String)
   @ Base essentials.jl:461
  convert(::Type{T}, ::T) where T<:AbstractString
   @ Base strings/basic.jl:232
  ...

Stacktrace:
  [1] lift(::Type{String}, x::JSON.Object{String, Any})
    @ StructUtils ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:345
  [2] lift(st::JSON.JSONReadStyle{JSON.Object{String, Any}, Nothing}, ::Type{String}, x::JSON.Object{String, Any})
    @ StructUtils ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:364
  [3] lift(style::JSON.JSONReadStyle{JSON.Object{…}, Nothing}, ::Type{String}, x::JSON.LazyValue{String}, tags::@NamedTuple{})
    @ JSON ~/.julia/dev/JSON/src/parse.jl:393
  [4] lift
    @ ~/.julia/dev/JSON/src/parse.jl:348 [inlined]
  [5] make
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:785 [inlined]
  [6] DictClosure
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:850 [inlined]
  [7] applyarray(keyvalfunc::StructUtils.DictClosure{Metadata, JSON.JSONReadStyle{…}}, x::JSON.LazyValue{String})
    @ JSON ~/.julia/dev/JSON/src/lazy.jl:385
  [8] applyeach
    @ ~/.julia/dev/JSON/src/lazy.jl:184 [inlined]
  [9] makedict
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:858 [inlined]
 [10] makedict(style::JSON.JSONReadStyle{JSON.Object{String, Any}, Nothing}, ::Type{Metadata}, source::JSON.LazyValue{String})
    @ StructUtils ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:855
 [11] make
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:777 [inlined]
 [12] make
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:751 [inlined]
 [13] #findfield##0
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:936 [inlined]
 [14] macro expansion
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:595 [inlined]
 [15] _foreach
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:598 [inlined]
 [16] findfield(::Type{…}, k::JSON.PtrString, v::JSON.LazyValue{…}, f::StructUtils.StructClosure{…})
    @ StructUtils ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:913
 [17] StructClosure
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:945 [inlined]
 [18] applyobject(keyvalfunc::StructUtils.StructClosure{…}, x::JSON.LazyValue{…})
    @ JSON ~/.julia/dev/JSON/src/lazy.jl:284
 [19] applyeach
    @ ~/.julia/dev/JSON/src/lazy.jl:182 [inlined]
 [20] makestruct
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:972 [inlined]
 [21] make
    @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:783 [inlined]
 [22] _parse
    @ ~/.julia/dev/JSON/src/parse.jl:193 [inlined]
 [23] parse(x::JSON.LazyValue{…}, ::Type{…}; dicttype::Type{…}, null::Nothing, style::JSON.JSONReadStyle{…})
    @ JSON ~/.julia/dev/JSON/src/parse.jl:189
 [24] parse
    @ ~/.julia/dev/JSON/src/parse.jl:189 [inlined]
 [25] parse(buf::String, ::Type{Foo})
    @ JSON ~/.julia/dev/JSON/src/parse.jl:181
 [26] top-level scope
    @ REPL[37]:1
Some type information was truncated. Use `show(err)` to see complete types

It gives the same error as when I tried the simpler @tags

julia> function liftMetadata(::Type{Metadata}, json_vector::Vector)
           return Metadata(Dict(Symbol(x["key"]) => x["val"] for x in json_vector))
       end
liftMetadata (generic function with 1 method)

julia> function lowerMetadata(m::Metadata)
           return map(collect(pairs(m.x))) do (key, val)
               (key=key, val=val)
           end
       end
lowerMetadata (generic function with 1 method)

julia> @tags struct Foo
           metadata::Metadata &(json=(lower=lowerMetadata, lift=liftMetadata,))
           other_fields::Int
       end

julia> jstring = JSON.json(Foo(Metadata(Dict(:a=>"1", :b=>"2")), 43))
"{\"metadata\":[{\"key\":\"a\",\"val\":\"1\"},{\"key\":\"b\",\"val\":\"2\"}],\"other_fields\":43}"

julia> JSON.parse(jstring, Foo)
ERROR: MethodError: Cannot `convert` an object of type JSON.Object{String, Any} to an object of type String

[ReubenJ]

What you have almost works, but subtyping AbstractDict is making things a little tricky for StructUtils, because anything <:AbstractDict has a special case (see StructUtils.dictlike). If you remove this, everything works.

@nonstruct struct Metadata
	x::Dict{Symbol, String}
end

struct Foo
    metadata::Metadata
	other_fields::Int
end

function StructUtils.lift(::Type{Metadata}, metadata_vector::Vector)
	Metadata(Dict(Symbol(x["key"]) => x["val"] for x in metadata_vector))
end

input = """
{
  "metadata": [
    {
      "key": "a",
      "val": "1"
    },
    {
      "key": "b",
      "val": "2"
    }
  ],
  "other_fields": 43
}
"""

JSON.parse(input, Foo)

I'm not familiar enough with the package to say whether your use case with the AbstractDict subtyping should be supported or not.

[Affie]

Thanks, that works for me.

I'm not familiar enough with the package to say whether your use case with the AbstractDict subtyping should be supported or not.

I was expecting lift in metadata::Dict{Symbol, String} &(json=(lower=lowerMetadata, lift=liftMetadata,)) to overwrite the Dict behaviour.

I also experimented a bit with StructStyle, but can't seem to quite get the hang of lift, this is the closest I got:

struct Mystyle <: JSON.JSONStyle end

StructUtils.dictlike(::Mystyle, ::Type{Dict{Symbol, String}}) = false
StructUtils.arraylike(::Mystyle, ::Type{Dict{Symbol, String}}) = false
StructUtils.noarg(::Mystyle, ::Type{Dict{Symbol, String}}) = false
StructUtils.structlike(::Mystyle, ::Type{Dict{Symbol, String}}) = false

function StructUtils.lift(::Mystyle, ::Type{<:Dict}, json_vector)
    return Dict(Symbol(x["key"]) => x["val"] for x in json_vector)
end

function StructUtils.lower(::Mystyle, m::Dict{Symbol, String})
    return map(collect(pairs(m))) do (key, val)
        (key=key, val=val)
    end
end

struct Foo
    metadata::Dict{Symbol, String}
    other_fields::Int
end

julia> jstring = JSON.json(Foo(Dict(:a=>"1", :b=>"2"), 43); style=Mystyle())
"{\"metadata\":[{\"key\":\"a\",\"val\":\"1\"},{\"key\":\"b\",\"val\":\"2\"}],\"other_fields\":43}"

julia> JSON.parse(jstring, Foo; style=Mystyle())
ERROR: TypeError: in typeassert, expected Dict{Symbol, String}, got a value of type Pair{Symbol, String}
Stacktrace:
 [1] macro expansion
   @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:-1 [inlined]
 [2] _construct
   @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:960 [inlined]
 [3] makestruct
   @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:976 [inlined]
 [4] make
   @ ~/.julia/packages/StructUtils/CvpPP/src/StructUtils.jl:783 [inlined]
 [5] _parse
   @ ~/.julia/dev/JSON/src/parse.jl:193 [inlined]
 [6] parse(x::JSON.LazyValue{String}, ::Type{Foo}; dicttype::Type{JSON.Object{String, Any}}, null::Nothing, style::Mystyle)
   @ JSON ~/.julia/dev/JSON/src/parse.jl:189
 [7] parse
   @ ~/.julia/dev/JSON/src/parse.jl:189 [inlined]
 [8] #parse#46
   @ ~/.julia/dev/JSON/src/parse.jl:181 [inlined]
 [9] top-level scope
   @ REPL[22]:1