Merge pull request #63 from jonhoo/serialization-compat

Fix compression format: needs to have the zlib header around deflate.

Author: jonhoo

Cargo.toml

@@ -36,6 +36,7 @@ flate2 = { version = "0.2.17", optional = true }
 rand = "0.3.15"
 rug = "0.6.0"
 ieee754 = "0.2.2"
+clap = "2.26.2"
 
 [lib]
 path = "src/lib.rs"
@@ -45,3 +46,7 @@ debug=true
 
 [profile.bench]
 debug=true
+
+[[example]]
+name = "cli"
+required-features = ["serialization"]

examples/cli.rs

@@ -0,0 +1,79 @@
+/// Reads numbers from stdin, one per line, and writes them to a serialized histogram on stdout.
+
+extern crate hdrsample;
+extern crate clap;
+
+use std::io::BufRead;
+
+use clap::{App, Arg, SubCommand};
+
+use hdrsample::Histogram;
+use hdrsample::serialization::{V2Serializer, V2DeflateSerializer};
+
+fn main() {
+    let default_max = format!("{}", u64::max_value());
+    let matches = App::new("hdrsample cli")
+            .subcommand(SubCommand::with_name("serialize")
+                    .arg(Arg::with_name("min")
+                            .long("min")
+                            .help("Minimum discernible value")
+                            .takes_value(true)
+                            .default_value("1"))
+                    .arg(Arg::with_name("max")
+                            .long("max")
+                            .help("Maximum trackable value")
+                            .takes_value(true)
+                            .default_value(default_max.as_str()))
+                    .arg(Arg::with_name("sigfig")
+                            .long("sigfig")
+                            .help("Number of significant digits")
+                            .takes_value(true)
+                            .default_value("3"))
+                    .arg(Arg::with_name("compression")
+                            .short("c")
+                            .long("compression")
+                            .help("Enable compression"))
+                    .arg(Arg::with_name("resize")
+                            .short("r")
+                            .long("resize")
+                            .help("Enable auto resize")))
+            .get_matches();
+
+    match matches.subcommand_name() {
+        Some("serialize") => {
+            let sub_matches = matches.subcommand_matches("serialize").unwrap();
+            let min = sub_matches.value_of("min").unwrap().parse().unwrap();
+            let max = sub_matches.value_of("max").unwrap().parse().unwrap();
+            let sigfig = sub_matches.value_of("sigfig").unwrap().parse().unwrap();
+
+            let mut h: Histogram<u64> = Histogram::new_with_bounds(min, max, sigfig).unwrap();
+
+            if sub_matches.is_present("resize") {
+                h.auto(true);
+            }
+
+            serialize(h, sub_matches.is_present("compression"));
+        },
+        _ => unreachable!()
+    }
+}
+
+fn serialize(mut h: Histogram<u64>, compression: bool) {
+    let stdin = std::io::stdin();
+    let stdin_handle = stdin.lock();
+
+    for num in stdin_handle.lines()
+            .map(|l| l.expect("Should be able to read stdin"))
+            .map(|s| s.parse().expect("Each line must be a u64")) {
+        h.record(num).unwrap();
+    }
+
+    let stdout = std::io::stdout();
+    let mut stdout_handle = stdout.lock();
+
+    if compression {
+        V2DeflateSerializer::new().serialize(&h, &mut stdout_handle).unwrap();
+    } else {
+        V2Serializer::new().serialize(&h, &mut stdout_handle).unwrap();
+    }
+}

src/serialization/deserializer.rs

@@ -5,7 +5,7 @@ use std::io::{self, Cursor, ErrorKind, Read};
 use std::marker::PhantomData;
 use std;
 use super::byteorder::{BigEndian, ReadBytesExt};
-use super::flate2::read::DeflateDecoder;
+use super::flate2::read::ZlibDecoder;
 
 /// Errors that can happen during deserialization.
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
@@ -70,7 +70,7 @@ impl Deserializer {
             .ok_or(DeserializeError::UsizeTypeTooSmall)?;
 
         // TODO reuse deflate buf, or switch to lower-level flate2::Decompress
-        let mut deflate_reader = DeflateDecoder::new(reader.take(payload_len as u64));
+        let mut deflate_reader = ZlibDecoder::new(reader.take(payload_len as u64));
         let inner_cookie = deflate_reader.read_u32::<BigEndian>()?;
         if inner_cookie != V2_COOKIE {
             return Err(DeserializeError::InvalidCookie);

src/serialization/v2_deflate_serializer.rs

@@ -5,7 +5,7 @@ use super::byteorder::{BigEndian, WriteBytesExt};
 use super::flate2::Compression;
 use std;
 use std::io::{ErrorKind, Write};
-use super::flate2::write::DeflateEncoder;
+use super::flate2::write::ZlibEncoder;
 
 /// Errors that occur during serialization.
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
@@ -23,6 +23,9 @@ impl std::convert::From<std::io::Error> for V2DeflateSerializeError {
 }
 
 /// Serializer for the V2 + DEFLATE binary format.
+///
+/// It's called "deflate" to stay consistent with the naming used in the Java implementation, but
+/// it actually uses zlib's wrapper format around plain DEFLATE.
 pub struct V2DeflateSerializer {
     uncompressed_buf: Vec<u8>,
     compressed_buf: Vec<u8>,
@@ -76,7 +79,7 @@ impl V2DeflateSerializer {
 
         {
             // TODO reuse deflate buf, or switch to lower-level flate2::Compress
-            let mut compressor = DeflateEncoder::new(&mut self.compressed_buf, Compression::Default);
+            let mut compressor = ZlibEncoder::new(&mut self.compressed_buf, Compression::Default);
             compressor.write_all(&self.uncompressed_buf[0..uncompressed_len])?;
             let _ = compressor.finish()?;
         }