Skip to content

Commit d354c76

Browse files
wcharginwchargin
authored
rust: add data_compat module (#4340)
Summary: This module performs conversions from legacy on-disk data formats. It will correspond to both the `data_compat` and `dataclass_compat` modules from Python TensorBoard. For now, we have just one function, to determine the initial summary metadata of a time series. And, for now, we only implement this for scalars, to keep things simple. Test Plan: Unit tests included. wchargin-branch: rust-data-compat
1 parent 338db03 commit d354c76

File tree

3 files changed

+202
-0
lines changed

3 files changed

+202
-0
lines changed

‎tensorboard/data/server/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ rust_library(
2727
name = "rustboard_core",
2828
srcs = [
2929
"lib.rs",
30+
"data_compat.rs",
3031
"event_file.rs",
3132
"masked_crc.rs",
3233
"reservoir.rs",

‎tensorboard/data/server/data_compat.rs

+200
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
2+
3+
Licensed under the Apache License, Version 2.0 (the "License");
4+
you may not use this file except in compliance with the License.
5+
You may obtain a copy of the License at
6+
7+
http://www.apache.org/licenses/LICENSE-2.0
8+
9+
Unless required by applicable law or agreed to in writing, software
10+
distributed under the License is distributed on an "AS IS" BASIS,
11+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
See the License for the specific language governing permissions and
13+
limitations under the License.
14+
==============================================================================*/
15+
16+
//! Conversions from legacy formats.
17+
18+
use crate::proto::tensorboard as pb;
19+
use pb::{summary::value::Value, summary_metadata::PluginData};
20+
21+
const SCALARS_PLUGIN_NAME: &str = "scalars";
22+
23+
/// Determines the metadata for a time series given its first event.
24+
///
25+
/// This fills in the plugin name and/or data class for legacy summaries for which those values are
26+
/// implied but not explicitly set. You should only need to call this function on the first event
27+
/// from each time series; doing so for subsequent events would be wasteful.
28+
///
29+
/// Rules, in order of decreasing precedence:
30+
///
31+
/// - If the initial metadata has a data class, it is taken as authoritative and returned
32+
/// verbatim.
33+
/// - If the summary value is of primitive type, an appropriate plugin metadata value is
34+
/// synthesized: e.g. a `simple_value` becomes metadata for the scalars plugin. Any existing
35+
/// metadata is ignored.
36+
/// - If the metadata has a known plugin name, the appropriate data class is added: e.g., a
37+
/// `"scalars"` metadata gets `DataClass::Scalar`.
38+
/// - Otherwise, the metadata is returned as is (or an empty metadata value synthesized if the
39+
/// given option was empty).
40+
pub fn initial_metadata(
41+
md: Option<pb::SummaryMetadata>,
42+
value: &Value,
43+
) -> Box<pb::SummaryMetadata> {
44+
fn blank(plugin_name: &str, data_class: pb::DataClass) -> Box<pb::SummaryMetadata> {
45+
Box::new(pb::SummaryMetadata {
46+
plugin_data: Some(PluginData {
47+
plugin_name: plugin_name.to_string(),
48+
..Default::default()
49+
}),
50+
data_class: data_class.into(),
51+
..Default::default()
52+
})
53+
}
54+
55+
match (md, value) {
56+
// Any summary metadata that sets its own data class is expected to already be in the right
57+
// form.
58+
(Some(md), _) if md.data_class != i32::from(pb::DataClass::Unknown) => Box::new(md),
59+
(_, Value::SimpleValue(_)) => blank(SCALARS_PLUGIN_NAME, pb::DataClass::Scalar),
60+
(Some(mut md), _) => {
61+
// Use given metadata, but first set data class based on plugin name, if known.
62+
#[allow(clippy::single_match)] // will have more patterns later
63+
match md.plugin_data.as_ref().map(|pd| pd.plugin_name.as_str()) {
64+
Some(SCALARS_PLUGIN_NAME) => {
65+
md.data_class = pb::DataClass::Scalar.into();
66+
}
67+
_ => {}
68+
};
69+
Box::new(md)
70+
}
71+
(None, _) => Box::new(pb::SummaryMetadata::default()),
72+
}
73+
}
74+
75+
#[cfg(test)]
76+
mod tests {
77+
use super::*;
78+
79+
fn tensor_shape(dims: &[i64]) -> pb::TensorShapeProto {
80+
pb::TensorShapeProto {
81+
dim: dims
82+
.iter()
83+
.map(|n| pb::tensor_shape_proto::Dim {
84+
size: *n,
85+
..Default::default()
86+
})
87+
.collect(),
88+
..Default::default()
89+
}
90+
}
91+
92+
mod scalars {
93+
use super::*;
94+
95+
#[test]
96+
fn test_tf1x_simple_value() {
97+
let md = pb::SummaryMetadata {
98+
plugin_data: Some(PluginData {
99+
plugin_name: "ignored_plugin".to_string(),
100+
content: b"ignored_content".to_vec(),
101+
..Default::default()
102+
}),
103+
..Default::default()
104+
};
105+
let v = Value::SimpleValue(0.125);
106+
let result = initial_metadata(Some(md), &v);
107+
108+
assert_eq!(
109+
*result,
110+
pb::SummaryMetadata {
111+
plugin_data: Some(PluginData {
112+
plugin_name: SCALARS_PLUGIN_NAME.to_string(),
113+
..Default::default()
114+
}),
115+
data_class: pb::DataClass::Scalar.into(),
116+
..Default::default()
117+
}
118+
);
119+
}
120+
121+
#[test]
122+
fn test_tf2x_scalar_tensor_without_dataclass() {
123+
let md = pb::SummaryMetadata {
124+
plugin_data: Some(PluginData {
125+
plugin_name: SCALARS_PLUGIN_NAME.to_string(),
126+
content: b"preserved!".to_vec(),
127+
..Default::default()
128+
}),
129+
..Default::default()
130+
};
131+
let v = Value::Tensor(pb::TensorProto {
132+
dtype: pb::DataType::DtFloat.into(),
133+
tensor_shape: Some(tensor_shape(&[])),
134+
float_val: vec![0.125],
135+
..Default::default()
136+
});
137+
let result = initial_metadata(Some(md), &v);
138+
139+
assert_eq!(
140+
*result,
141+
pb::SummaryMetadata {
142+
plugin_data: Some(PluginData {
143+
plugin_name: SCALARS_PLUGIN_NAME.to_string(),
144+
content: b"preserved!".to_vec(),
145+
..Default::default()
146+
}),
147+
data_class: pb::DataClass::Scalar.into(),
148+
..Default::default()
149+
}
150+
);
151+
}
152+
}
153+
154+
mod unknown {
155+
use super::*;
156+
157+
#[test]
158+
fn test_custom_plugin_with_dataclass() {
159+
let md = pb::SummaryMetadata {
160+
plugin_data: Some(PluginData {
161+
plugin_name: "myplugin".to_string(),
162+
content: b"mycontent".to_vec(),
163+
..Default::default()
164+
}),
165+
data_class: pb::DataClass::Tensor.into(),
166+
..Default::default()
167+
};
168+
// Even with a `SimpleValue`, dataclass-annotated metadata passes through.
169+
let v = Value::SimpleValue(0.125);
170+
let expected = md.clone();
171+
let result = initial_metadata(Some(md), &v);
172+
173+
assert_eq!(*result, expected);
174+
}
175+
176+
#[test]
177+
fn test_unknown_plugin_no_dataclass() {
178+
let md = pb::SummaryMetadata {
179+
plugin_data: Some(PluginData {
180+
plugin_name: "myplugin".to_string(),
181+
content: b"mycontent".to_vec(),
182+
..Default::default()
183+
}),
184+
..Default::default()
185+
};
186+
let v = Value::Tensor(pb::TensorProto::default());
187+
let expected = md.clone();
188+
let result = initial_metadata(Some(md), &v);
189+
190+
assert_eq!(*result, expected);
191+
}
192+
193+
#[test]
194+
fn test_empty() {
195+
let v = Value::Tensor(pb::TensorProto::default());
196+
let result = initial_metadata(None, &v);
197+
assert_eq!(*result, pb::SummaryMetadata::default());
198+
}
199+
}
200+
}

‎tensorboard/data/server/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ limitations under the License.
1515

1616
//! Core functionality for TensorBoard data loading.
1717
18+
pub mod data_compat;
1819
pub mod event_file;
1920
pub mod masked_crc;
2021
pub mod reservoir;

0 commit comments

Comments
 (0)