From 1193c7f96360243a7a3384a805a120618bf3f86a Mon Sep 17 00:00:00 2001 From: Kent Ross Date: Wed, 15 Jan 2025 07:29:41 -0800 Subject: [PATCH] Testing 'n' fixes (#94) * fix arithmetic overflow panic * use vec instead of a boxed array to avoid blowing the stack in dev mode * make it possible to disable compression to speed up `cargo test --benches` * disable wiring: https://github.com/louaykamel/wiring/issues/3 * add test step to build.yml * also run build step in the master branch (& give it a workflow_dispatch button) * organize & alphabetize deps; organize & alphabetize features; make compression a positive feature --- .github/workflows/build.yml | 18 ++++++++++++++++- Cargo.toml | 40 ++++++++++++++++++++++++------------- src/bench_bincode.rs | 2 +- src/datasets/mk48/mod.rs | 2 +- src/lib.rs | 22 ++++++++++++-------- 5 files changed, 59 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 122f52b..a2b979e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,16 @@ name: build on: + workflow_dispatch: pull_request: branches: [ master ] + push: + branches: + - master + paths: + - 'Cargo.toml' + - 'benches/**' + - 'src/**' env: CI: true @@ -71,9 +79,17 @@ jobs: - name: build shell: bash run: | - cargo build --benches --features regenerate-capnp,regenerate-flatbuffers,regenerate-prost + cargo build --benches --features regenerate - name: check generated code shell: bash run: | git diff --exit-code && echo 'ok!' || (echo 'committed generated code is not up to date!'; exit 1) + + # build and test in dev mode with debug assertions enabled (and slow, uninteresting code + # disabled; only the encoders, and not the compression). this should quickly sanity check + # broken benchmarks, failing debug assertions, etc. + - name: test benches + shell: bash + run: | + cargo test --benches --no-default-features --features default-encoding-set diff --git a/Cargo.toml b/Cargo.toml index 1be4ef4..89f94c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,11 +55,9 @@ cbor4ii = { version = "=0.3.3", features = [ "serde1", ], optional = true } ciborium = { version = "=0.2.2", optional = true } -criterion = "=0.5.1" databuf = { version = "=0.5.0", optional = true } dlhn = { version = "=0.1.7", optional = true } flatbuffers = { version = "=24.12.23", optional = true } -libflate = "=2.1.0" msgpacker = { version = "=0.4.5", optional = true } nachricht-serde = { version = "=0.4.0", optional = true } nanoserde = { version = "=0.1.37", optional = true } @@ -69,31 +67,38 @@ parity-scale-codec = { version = "=3.6.12", features = [ parity-scale-codec-derive = { version = "=3.6.12", optional = true } postcard = { version = "=1.1.1", features = ["alloc"], optional = true } pot = { version = "=3.0.1", optional = true } -pprof = { version = "=0.14.0", features = ["flamegraph"], optional = true } prost = { version = "=0.13.4", optional = true } -rand = "=0.8.5" rkyv = { version = "=0.8.9", optional = true } rmp-serde = { version = "=1.3.0", optional = true } ron = { version = "=0.8.1", optional = true } -serde = { version = "=1.0.216", features = ["derive"] } +savefile = { version = "=0.18.5", optional = true } +savefile-derive = { version = "=0.18.5", optional = true } serde_bare = { version = "=0.5.0", optional = true } serde-brief = { version = "=0.1.0", features = [ "std", ], optional = true } serde_cbor = { version = "=0.11.2", optional = true } -serde_json = { version = "=1.0.128", features = [ +serde_json = { version = "=1.0.134", features = [ "float_roundtrip", ], optional = true } simd-json = { version = "=0.14.3", optional = true } simd-json-derive = { version = "=0.15.0", optional = true } speedy = { version = "=0.8.7", optional = true } -savefile = { version = "=0.18.5", optional = true } -savefile-derive = { version = "=0.18.5", optional = true } wiring = { version = "=0.2.2", optional = true } + +criterion = "=0.5.1" +libflate = "=2.1.0" +pprof = { version = "=0.14.0", features = ["flamegraph"], optional = true } +rand = "=0.8.5" +serde = { version = "=1.0.216", features = ["derive"] } zstd = "=0.13.2" [features] default = [ + "default-encoding-set", + "measure-compression", +] +default-encoding-set = [ "bilrost", "bincode1", "bincode", @@ -108,29 +113,36 @@ default = [ "msgpacker", "nachricht-serde", "nanoserde", - "scale", "postcard", "pot", "prost", "rkyv", "rmp-serde", "ron", + "savefile", + "scale", "serde_bare", "serde-brief", "serde_cbor", "serde_json", "simd-json", "speedy", - "savefile", - "wiring", + # "wiring", ] + +measure-compression = [] capnp = ["dep:capnp"] prost = ["dep:capnp", "dep:prost"] -simd-json = ["dep:simd-json", "simd-json-derive"] -savefile = ["dep:savefile", "savefile-derive"] -scale = ["parity-scale-codec", "parity-scale-codec-derive"] +simd-json = ["dep:simd-json", "dep:simd-json-derive"] +savefile = ["dep:savefile", "dep:savefile-derive"] +scale = ["dep:parity-scale-codec", "dep:parity-scale-codec-derive"] # Enable these features to regenerate generated files rather than using the committed versions. +regenerate = [ + "regenerate-capnp", + "regenerate-flatbuffers", + "regenerate-prost", +] regenerate-capnp = ["dep:capnpc"] regenerate-flatbuffers = ["dep:flatc-rust"] regenerate-prost = ["dep:prost-build"] diff --git a/src/bench_bincode.rs b/src/bench_bincode.rs index 5b26e0e..a7e45fc 100644 --- a/src/bench_bincode.rs +++ b/src/bench_bincode.rs @@ -7,7 +7,7 @@ where const BUFFER_LEN: usize = 10_000_000; let mut group = c.benchmark_group(format!("{}/bincode", name)); - let mut buffer = Box::new([0u8; BUFFER_LEN]); + let mut buffer = vec![0u8; BUFFER_LEN]; let conf = bincode::config::standard(); group.bench_function("serialize", |b| { b.iter(|| { diff --git a/src/datasets/mk48/mod.rs b/src/datasets/mk48/mod.rs index 0a8a0fa..5e15698 100644 --- a/src/datasets/mk48/mod.rs +++ b/src/datasets/mk48/mod.rs @@ -535,7 +535,7 @@ impl Contact { (0..entity_type.turret_count()) .map(|_| { if rng.gen_bool(0.75) { - (base_angle as i16 + rng.gen_range(-200..200) as i16) as u16 + (base_angle as i16).wrapping_add(rng.gen_range(-200..200)) as u16 } else { rng.gen() } diff --git a/src/lib.rs b/src/lib.rs index bd76d67..dca32e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -153,16 +153,20 @@ pub fn generate_vec(rng: &mut R, range: ops::Range) pub fn bench_size(name: &str, lib: &str, bytes: &[u8]) { println!("{}/{}/size {}", name, lib, bytes.len()); - println!("{}/{}/zlib {}", name, lib, zlib_size(bytes)); - println!("{}/{}/zstd {}", name, lib, zstd_size(bytes)); - println!( - "{}/{}/zstd_time {}", - name, - lib, - bench_compression(|| zstd_size(bytes)) - ); + #[cfg(feature = "measure-compression")] + { + println!("{}/{}/zlib {}", name, lib, zlib_size(bytes)); + println!("{}/{}/zstd {}", name, lib, zstd_size(bytes)); + println!( + "{}/{}/zstd_time {}", + name, + lib, + bench_compression(|| zstd_size(bytes)) + ); + } } +#[cfg(feature = "measure-compression")] fn bench_compression(compress: impl Fn() -> usize) -> String { let start = std::time::Instant::now(); let size = compress(); @@ -176,12 +180,14 @@ fn bench_compression(compress: impl Fn() -> usize) -> String { format!("time: [{s} {s} {s}] {mb_per_second} MB/s") } +#[cfg(feature = "measure-compression")] fn zlib_size(mut bytes: &[u8]) -> usize { let mut encoder = libflate::zlib::Encoder::new(Vec::new()).unwrap(); std::io::copy(&mut bytes, &mut encoder).unwrap(); encoder.finish().into_result().unwrap().len() } +#[cfg(feature = "measure-compression")] fn zstd_size(bytes: &[u8]) -> usize { zstd::stream::encode_all(bytes, 0).unwrap().len() }