Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update to new protobuf and removed workaround for hparams encoding #135

Open
wants to merge 10 commits into
base: master
Choose a base branch
from
4 changes: 2 additions & 2 deletions Project.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "TensorBoardLogger"
uuid = "899adc3e-224a-11e9-021f-63837185c80f"
authors = ["Filippo Vicentini <[email protected]>"]
version = "0.1.22"
version = "0.1.23"

[deps]
CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
Expand All @@ -14,7 +14,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
[compat]
FileIO = "1.2.3"
ImageCore = "0.8.1, 0.9"
ProtoBuf = "1.0.11"
ProtoBuf = "1.0.12"
Requires = "0.5, 1"
StatsBase = "0.27, 0.28, 0.29, 0.30, 0.31, 0.32, 0.33, 0.34"
julia = "1.6"
Expand Down
3 changes: 0 additions & 3 deletions gen/Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,3 @@ FilePaths = "8fc22ac5-c921-52a6-82fd-178b2807b824"
FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429"

[compat]
ProtoBuf = "0.9.1"
45 changes: 44 additions & 1 deletion gen/compile_proto.jl
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,47 @@ function process_module(cur_module::AbstractString; base_module::AbstractString=
files_to_include = [string(module_out_dir/basename(file)) for file in infiles]
return files_to_include
end
function clean_file(filepath)
lines = readlines(filepath)
short_filepath = relpath(filepath, @__DIR__)
should_overwrite = false

if length(lines) > 1 && startswith(lines[2], "# original file:")
proto_syntax = "(" * split(lines[2], " (")[end]
proto_short_filepath = replace(short_filepath, "_pb.jl"=>".proto", "protojl"=>"proto")
lines[2] = "# original file: $proto_short_filepath $proto_syntax"
should_overwrite = true
elseif length(lines) > 0 && startswith(lines[1], "module tensorboard")
path_segments = splitpath(short_filepath)
plugin_index = findfirst(path_segments .== "plugins")
if !isnothing(plugin_index)
should_overwrite = true
plugin_name = path_segments[plugin_index+1]
lines[1] = "module tensorboard_plugin_$plugin_name"
end
end

if should_overwrite
open(filepath, "w") do io
for line in @views lines[begin:end-1]
println(io, line)
end
print(io, lines[end])
end
end
nothing
end
function clean_files(out_dir)
for (root, dirs, files) in walkdir(out_dir)
for dir in dirs
clean_file(joinpath(root, dir))
end
for file in files
clean_file(joinpath(root, file))
end
end
nothing
end

#process_module("tensorflow", input_path="tensorflow/core/protobuf")

Expand All @@ -61,5 +102,7 @@ append!(files_to_include, (process_module("tensorboard/plugins/$plugin", base_mo
# files_to_include contains all the proto files, can be used for printing and inspection
println("generated code for \n$files_to_include")

clean_files(out_dir)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's leave this for a separate PR. This censoring is redundant and messes up file paths.

Suggested change
clean_files(out_dir)
# clean_files(out_dir)

This censoring can be done with a shell script:
>>> cat file_list | xargs -I f sed -i 's|/home/lior||g' f

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Likewise the _pb.jl files can be restored to their previous version with the full path


# Finally move the output directory to the src folder
mv(out_dir, TBL_root/"src"/"protojl")
mv(out_dir, TBL_root/"src"/"protojl"; force=true)
41 changes: 0 additions & 41 deletions src/hparams.jl
Original file line number Diff line number Diff line change
Expand Up @@ -73,47 +73,6 @@ function metric_info(c::MetricConfig)
return HP.MetricInfo(mname, c.displayname, c.description, HDatasetType.DATASET_UNKNOWN)
end

# Dictionary serialisation in ProtoBuf does not work for this specific map type
# and must be overloaded so that it can be parsed. The format was derived by
# looking at the binary output of a log file created by tensorboardX.
# These protobuf overloads should be removed once https://github.com/JuliaIO/ProtoBuf.jl/pull/234 is merged.
function PB.encode(e::ProtoEncoder, i::Int, x::Dict{String,HValue})
for (k, v) in x
PB.Codecs.encode_tag(e, 1, PB.Codecs.LENGTH_DELIMITED)
total_size = PB.Codecs._encoded_size(k, 1) + PB.Codecs._encoded_size(v, 2)
PB.Codecs.vbyte_encode(e.io, UInt32(total_size)) # Add two for the wire type and length
PB.Codecs.encode(e, 1, k)
PB.Codecs.encode(e, 2, v)
end
return nothing
end

# Similarly, we must overload the size calculation to take into account the new
# format.
function PB.Codecs._encoded_size(d::Dict{String,HValue}, i::Int)
mapreduce(x->begin
total_size = PB.Codecs._encoded_size(x.first, 1) + PB.Codecs._encoded_size(x.second, 2)
return 1 + PB.Codecs._varint_size(total_size) + total_size
end, +, d, init=0)
end

function PB.Codecs.decode!(d::ProtoDecoder, buffer::Dict{String,HValue})
len = PB.Codecs.vbyte_decode(d.io, UInt32)
endpos = position(d.io) + len
while position(d.io) < endpos
pair_field_number, pair_wire_type = PB.Codecs.decode_tag(d)
pair_len = PB.Codecs.vbyte_decode(d.io, UInt32)
pair_end_pos = position(d.io) + pair_len
field_number, wire_type = PB.Codecs.decode_tag(d)
key = PB.Codecs.decode(d, K)
field_number, wire_type = PB.Codecs.decode_tag(d)
val = PB.Codecs.decode(d, Ref{V})
@assert position(d.io) == pair_end_pos
buffer[key] = val
end
@assert position(d.io) == endpos
nothing
end

"""
write_hparams!(logger::TBLogger, hparams::Dict{String, Any}, metrics::AbstractArray{String})
Expand Down
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/google/protobuf/any_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.779
# original file: /home/lior/.julia/dev/ProtoBuf/src/google/protobuf/any.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.634
# original file: proto/tensorboard/google/protobuf/any.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -41,4 +41,4 @@ function PB._encoded_size(x::var"#Any")
!isempty(x.type_url) && (encoded_size += PB._encoded_size(x.type_url, 1))
!isempty(x.value) && (encoded_size += PB._encoded_size(x.value, 2))
return encoded_size
end
end
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/google/protobuf/wrappers_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.780
# original file: /home/lior/.julia/dev/ProtoBuf/src/google/protobuf/wrappers.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.634
# original file: proto/tensorboard/google/protobuf/wrappers.proto (proto3 syntax)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This censored path doesn't look quite right


import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -276,4 +276,4 @@ function PB._encoded_size(x::StringValue)
encoded_size = 0
!isempty(x.value) && (encoded_size += PB._encoded_size(x.value, 1))
return encoded_size
end
end
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.964
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/plugins/custom_scalar/layout.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.779
# original file: proto/tensorboard/plugins/custom_scalar/tensorboard/layout.proto (proto3 syntax)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here it's unclear that this file came from the gen folder, and the path seems inconsistent?


import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -237,4 +237,4 @@ function PB._encoded_size(x::Layout)
x.version != zero(Int32) && (encoded_size += PB._encoded_size(x.version, 1))
!isempty(x.category) && (encoded_size += PB._encoded_size(x.category, 2))
return encoded_size
end
end
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ module tensorboard_plugin_custom_scalar

include("layout_pb.jl")

end # module tensorboard
end # module tensorboard
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:25.011
# original file: /home/lior/.julia/dev/ProtoBuf/src/google/protobuf/struct.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.823
# original file: proto/tensorboard/plugins/hparams/google/protobuf/struct.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -142,4 +142,4 @@ function PB._encoded_size(x::ListValue)
encoded_size = 0
!isempty(x.values) && (encoded_size += PB._encoded_size(x.values, 1))
return encoded_size
end
end
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:25.010
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/plugins/hparams/api.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.822
# original file: proto/tensorboard/plugins/hparams/tensorboard/hparams/api.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -697,4 +697,4 @@ function PB._encoded_size(x::ListSessionGroupsResponse)
!isempty(x.session_groups) && (encoded_size += PB._encoded_size(x.session_groups, 1))
x.total_size != zero(Int32) && (encoded_size += PB._encoded_size(x.total_size, 3))
return encoded_size
end
end
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:25.011
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/plugins/hparams/hparams_util.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.822
# original file: proto/tensorboard/plugins/hparams/tensorboard/hparams/hparams_util.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -95,4 +95,4 @@ function PB._encoded_size(x::HParamInfosList)
encoded_size = 0
!isempty(x.hparam_infos) && (encoded_size += PB._encoded_size(x.hparam_infos, 1))
return encoded_size
end
end
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:25.011
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/plugins/hparams/plugin_data.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.822
# original file: proto/tensorboard/plugins/hparams/tensorboard/hparams/plugin_data.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -152,4 +152,4 @@ function PB._encoded_size(x::HParamsPluginData)
encoded_size += PB._encoded_size(x.data[]::SessionEndInfo, 4)
end
return encoded_size
end
end
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ include("../google/google.jl")

include("hparams/hparams.jl")

end # module tensorboard
end # module tensorboard
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:25.012
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/plugins/text/plugin_data.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.823
# original file: proto/tensorboard/plugins/text/tensorboard/plugin_data.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -35,4 +35,4 @@ function PB._encoded_size(x::TextPluginData)
encoded_size = 0
x.version != zero(Int32) && (encoded_size += PB._encoded_size(x.version, 1))
return encoded_size
end
end
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ module tensorboard_plugin_text

include("plugin_data_pb.jl")

end # module tensorboard
end # module tensorboard
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.163
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/allocation_description.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.205
# original file: proto/tensorboard/tensorboard/allocation_description.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -65,4 +65,4 @@ function PB._encoded_size(x::AllocationDescription)
x.has_single_reference != false && (encoded_size += PB._encoded_size(x.has_single_reference, 5))
x.ptr != zero(UInt64) && (encoded_size += PB._encoded_size(x.ptr, 6))
return encoded_size
end
end
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/tensorboard/api_def_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.776
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/api_def.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.631
# original file: proto/tensorboard/tensorboard/api_def.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -272,4 +272,4 @@ function PB._encoded_size(x::ApiDefs)
encoded_size = 0
!isempty(x.op) && (encoded_size += PB._encoded_size(x.op, 1))
return encoded_size
end
end
7 changes: 3 additions & 4 deletions src/protojl/tensorboard/tensorboard/attr_value_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.718
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/attr_value.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.588
# original file: proto/tensorboard/tensorboard/attr_value.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -59,7 +59,6 @@ struct var"AttrValue.ListValue" <: var"##AbstractAttrValue.ListValue"
tensor::Vector{TensorProto}
func::Vector{<:NameAttrList}
end

PB.default_values(::Type{var"AttrValue.ListValue"}) = (;s = Vector{Vector{UInt8}}(), i = Vector{Int64}(), f = Vector{Float32}(), b = Vector{Bool}(), var"#type" = Vector{var"#DataType".T}(), shape = Vector{TensorShapeProto}(), tensor = Vector{TensorProto}(), func = Vector{NameAttrList}())
PB.field_numbers(::Type{var"AttrValue.ListValue"}) = (;s = 2, i = 3, f = 4, b = 5, var"#type" = 6, shape = 7, tensor = 8, func = 9)

Expand Down Expand Up @@ -213,4 +212,4 @@ function PB._encoded_size(x::AttrValue)
encoded_size += PB._encoded_size(x.value[]::String, 9)
end
return encoded_size
end
end
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/tensorboard/cluster_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.424
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/cluster.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.381
# original file: proto/tensorboard/tensorboard/cluster.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -71,4 +71,4 @@ function PB._encoded_size(x::ClusterDef)
encoded_size = 0
!isempty(x.job) && (encoded_size += PB._encoded_size(x.job, 1))
return encoded_size
end
end
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/tensorboard/config_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.779
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/config.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.633
# original file: proto/tensorboard/tensorboard/config.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -1080,4 +1080,4 @@ function PB._encoded_size(x::CallableOptions)
!isempty(x.fetch_devices) && (encoded_size += PB._encoded_size(x.fetch_devices, 7))
x.fetch_skip_sync != false && (encoded_size += PB._encoded_size(x.fetch_skip_sync, 8))
return encoded_size
end
end
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/tensorboard/coordination_config_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.223
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/coordination_config.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.246
# original file: proto/tensorboard/tensorboard/coordination_config.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -126,4 +126,4 @@ function PB._encoded_size(x::CoordinationServiceConfig)
!isempty(x.recoverable_jobs) && (encoded_size += PB._encoded_size(x.recoverable_jobs, 9))
x.allow_new_incarnation_to_reconnect != false && (encoded_size += PB._encoded_size(x.allow_new_incarnation_to_reconnect, 11))
return encoded_size
end
end
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/tensorboard/cost_graph_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.424
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/cost_graph.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.381
# original file: proto/tensorboard/tensorboard/cost_graph.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -282,4 +282,4 @@ function PB._encoded_size(x::CostGraphDef)
!isempty(x.node) && (encoded_size += PB._encoded_size(x.node, 1))
!isempty(x.cost) && (encoded_size += PB._encoded_size(x.cost, 2))
return encoded_size
end
end
6 changes: 3 additions & 3 deletions src/protojl/tensorboard/tensorboard/cpp_shape_inference_pb.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-06-19T18:18:24.424
# original file: /home/lior/TensorBoardLogger.jl/gen/proto/tensorboard/compat/proto/cpp_shape_inference.proto (proto3 syntax)
# Autogenerated using ProtoBuf.jl v1.0.11 on 2023-08-09T10:18:18.381
# original file: proto/tensorboard/tensorboard/cpp_shape_inference.proto (proto3 syntax)

import ProtoBuf as PB
using ProtoBuf: OneOf
Expand Down Expand Up @@ -158,4 +158,4 @@ function PB._encoded_size(x::CppShapeInferenceResult)
!isnothing(x.shape) && (encoded_size += PB._encoded_size(x.shape, 1))
!isnothing(x.handle_data) && (encoded_size += PB._encoded_size(x.handle_data, 4))
return encoded_size
end
end
Loading