Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,7 @@ tikv-jemallocator = { version = "0.6.0", features = ["profiling", "stats"] }
tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] }
jemalloc_pprof = { version = "0.8", features = ["symbolize", "flamegraph"] }
zstd-framed = { version = "0.1.1", features = ["tokio"] }
zstd = "0.13.3"

# Vendor the openssl we rely on, rather than depend on a
# potentially very old system version.
Expand Down
29 changes: 29 additions & 0 deletions crates/client-api-messages/src/websocket/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ pub enum Compression {
Brotli,
/// Compress using gzip if a certain size threshold was met.
Gzip,
/// Opt in to Zstandard compression. Requires a server and client with Zstd support.
Zstd,
}

/// The tag recognized by the host and SDKs to mean no compression of a `ServerMessage`.
Expand All @@ -53,6 +55,33 @@ pub const SERVER_MSG_COMPRESSION_TAG_BROTLI: u8 = 1;
/// The tag recognized by the host and SDKs to mean gzip compression of a `ServerMessage`.
pub const SERVER_MSG_COMPRESSION_TAG_GZIP: u8 = 2;

/// Standard Zstandard frame, negotiated through `compression=Zstd`.
/// Existing clients retain their selected codec and are never sent this tag.
pub const SERVER_MSG_COMPRESSION_TAG_ZSTD: u8 = 3;

#[cfg(test)]
mod compression_tests {
use super::*;

#[test]
fn zstd_opt_in_preserves_defaults_and_existing_tags() {
assert_eq!(Compression::default(), Compression::Brotli);
assert_eq!(
serde_json::from_str::<Compression>("\"Zstd\"").unwrap(),
Compression::Zstd
);
assert_eq!(
[
SERVER_MSG_COMPRESSION_TAG_NONE,
SERVER_MSG_COMPRESSION_TAG_BROTLI,
SERVER_MSG_COMPRESSION_TAG_GZIP,
SERVER_MSG_COMPRESSION_TAG_ZSTD
],
[0, 1, 2, 3]
);
}
}

/// Websocket close code sent when a connection supplies a `session_id`
/// which another live connection of the same identity still holds.
///
Expand Down
1 change: 1 addition & 0 deletions crates/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ async-trait.workspace = true
base64.workspace = true
blake3.workspace = true
brotli.workspace = true
zstd = { workspace = true, features = ["zstdmt"] }
bytemuck.workspace = true
bytes.workspace = true
bytesize.workspace = true
Expand Down
24 changes: 17 additions & 7 deletions crates/core/src/client/messages.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use crate::client::consume_each_list::ConsumeEachBuffer;
use crate::host::module_host::{EventStatus, ModuleEvent, ProcedureCallError};
use crate::host::{ArgsTuple, ProcedureCallResult};
use crate::subscription::row_list_builder_pool::BsatnRowListBuilderPool;
use crate::subscription::websocket_building::{brotli_compress, decide_compression, gzip_compress};
use crate::subscription::websocket_building::{decide_compression, gzip_compress};
use bytes::{BufMut, Bytes, BytesMut};
use bytestring::ByteString;
use derive_more::From;
Expand Down Expand Up @@ -83,13 +83,13 @@ impl SerializeBuffer {

/// Compress the data from a `write_with_tag` call, and change the tag.
fn compress_with_tag(
self,
mut self,
tag: u8,
write: impl FnOnce(&[u8], &mut bytes::buf::Writer<BytesMut>),
write: impl FnOnce(&mut BytesMut, &mut bytes::buf::Writer<BytesMut>),
) -> (InUseSerializeBuffer, Bytes) {
let mut writer = self.compressed.writer();
writer.get_mut().put_u8(tag);
write(&self.uncompressed[1..], &mut writer);
write(&mut self.uncompressed, &mut writer);
let compressed = writer.into_inner().freeze();
let in_use = InUseSerializeBuffer::Compressed {
uncompressed: self.uncompressed,
Expand All @@ -111,10 +111,20 @@ fn finalize_binary_serialize_buffer(
) -> (InUseSerializeBuffer, Bytes) {
match decide_compression(uncompressed_len, compression) {
ws_v1::Compression::None => buffer.uncompressed(),
ws_v1::Compression::Brotli => {
buffer.compress_with_tag(ws_common::SERVER_MSG_COMPRESSION_TAG_BROTLI, brotli_compress)
ws_v1::Compression::Brotli => buffer.compress_with_tag(
ws_common::SERVER_MSG_COMPRESSION_TAG_BROTLI,
crate::compression::brotli_compress_frame,
),
ws_v1::Compression::Gzip => {
buffer.compress_with_tag(ws_common::SERVER_MSG_COMPRESSION_TAG_GZIP, |frame, out| {
gzip_compress(&frame[1..], out);
})
}
ws_v1::Compression::Zstd => {
buffer.compress_with_tag(ws_common::SERVER_MSG_COMPRESSION_TAG_ZSTD, |frame, out| {
crate::compression::zstd_compress(&frame[1..], out);
})
}
ws_v1::Compression::Gzip => buffer.compress_with_tag(ws_common::SERVER_MSG_COMPRESSION_TAG_GZIP, gzip_compress),
}
}

Expand Down
Loading