Skip to content

Commit

Permalink
Merge pull request #2 from AronParker/estimate_max_mem_usage
Browse files Browse the repository at this point in the history
Add compress_estimate_max_mem_usage
  • Loading branch information
AronParker authored Jan 8, 2023
2 parents 7574aa0 + a3e5b53 commit d25f7ce
Show file tree
Hide file tree
Showing 6 changed files with 42 additions and 4 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "brotlic"
version = "0.8.0"
version = "0.8.1"
edition = "2021"
authors = ["Aron Parker <[email protected]>", "The Brotli Authors"]
description = """
Expand Down
2 changes: 1 addition & 1 deletion brotlic-sys/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "brotlic-sys"
version = "0.2.0"
version = "0.2.1"
edition = "2021"
authors = ["Aron Parker <[email protected]>"]
description = """
Expand Down
2 changes: 1 addition & 1 deletion brotlic-sys/brotli
Submodule brotli updated 91 files
+8 −8 .github/workflows/build_test.yml
+7 −0 .github/workflows/fuzz.yml
+117 −0 .github/workflows/release.yaml
+1 −0 .gitignore
+23 −46 CMakeLists.txt
+5 −0 README.md
+6 −0 SECURITY.md
+14 −16 bootstrap
+2 −1 c/common/constants.h
+1 −1 c/common/dictionary.c
+2 −1 c/common/platform.c
+36 −87 c/common/platform.h
+3 −2 c/common/shared_dictionary_internal.h
+3 −2 c/dec/bit_reader.c
+10 −7 c/dec/bit_reader.h
+15 −10 c/dec/decode.c
+6 −3 c/dec/huffman.c
+2 −1 c/dec/huffman.h
+2 −1 c/dec/prefix.h
+15 −1 c/dec/state.c
+6 −2 c/dec/state.h
+2 −1 c/enc/backward_references.c
+2 −1 c/enc/backward_references.h
+2 −1 c/enc/backward_references_hq.c
+2 −1 c/enc/backward_references_hq.h
+2 −1 c/enc/bit_cost.c
+2 −1 c/enc/bit_cost.h
+2 −1 c/enc/block_splitter.h
+2 −1 c/enc/brotli_bit_stream.c
+2 −1 c/enc/brotli_bit_stream.h
+2 −1 c/enc/cluster.c
+2 −1 c/enc/cluster.h
+2 −1 c/enc/command.h
+16 −9 c/enc/compound_dictionary.c
+18 −4 c/enc/compound_dictionary.h
+2 −1 c/enc/compress_fragment.c
+2 −1 c/enc/compress_fragment.h
+2 −1 c/enc/compress_fragment_two_pass.c
+2 −1 c/enc/compress_fragment_two_pass.h
+80 −340 c/enc/encode.c
+3 −5 c/enc/encoder_dict.h
+2 −1 c/enc/entropy_encode.c
+2 −1 c/enc/entropy_encode.h
+2 −1 c/enc/entropy_encode_static.h
+2 −1 c/enc/fast_log.h
+2 −1 c/enc/find_match_length.h
+23 −5 c/enc/hash.h
+2 −1 c/enc/histogram.h
+2 −1 c/enc/literal_cost.c
+2 −1 c/enc/literal_cost.h
+2 −1 c/enc/memory.c
+2 −1 c/enc/memory.h
+2 −1 c/enc/metablock.c
+2 −1 c/enc/metablock.h
+1 −0 c/enc/params.h
+2 −1 c/enc/prefix.h
+2 −1 c/enc/quality.h
+2 −1 c/enc/ringbuffer.h
+104 −0 c/enc/state.h
+2 −1 c/enc/static_dict.h
+2 −1 c/enc/utf8_util.h
+2 −1 c/enc/write_bits.h
+3 −3 c/fuzz/test_fuzzer.sh
+0 −8 c/include/brotli/port.h
+38 −12 c/tools/brotli.c
+11 −13 c/tools/brotli.md
+123 −130 docs/brotli.1
+17 −4 go/WORKSPACE
+1 −6 java/BUILD
+1 −1 java/org/brotli/dec/BUILD
+13 −5 java/org/brotli/dec/BrotliInputStream.java
+1 −0 java/org/brotli/dec/Decode.java
+11 −2 java/org/brotli/dec/build_defs.bzl
+1 −1 java/org/brotli/wrapper/common/BUILD
+1 −1 java/org/brotli/wrapper/dec/BUILD
+1 −1 java/org/brotli/wrapper/dec/BrotliInputStream.java
+13 −10 java/org/brotli/wrapper/dec/Decoder.java
+3 −3 java/org/brotli/wrapper/dec/decoder_jni.cc
+2 −2 java/org/brotli/wrapper/dec/decoder_jni_onload.cc
+1 −1 java/org/brotli/wrapper/enc/BUILD
+19 −11 java/org/brotli/wrapper/enc/Encoder.java
+5 −2 java/org/brotli/wrapper/enc/EncoderJNI.java
+1 −2 java/org/brotli/wrapper/enc/encoder_jni.cc
+0 −78 premake5.lua
+3 −0 pyproject.toml
+375 −143 python/_brotli.c
+2 −2 research/brotli_decoder.c
+8 −3 research/brotlidump.py
+1 −0 research/dictionary.bin
+1 −0 scripts/sources.lst
+24 −32 setup.py
1 change: 1 addition & 0 deletions brotlic-sys/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ fn main() {
])
.include("brotli/c/include")
.define("BROTLI_BUILD_ENC_EXTRA_API", None)
.define("BROTLI_HAVE_LOG2", "1")
.warnings(false)
.compile("brotli");

Expand Down
16 changes: 16 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -920,6 +920,22 @@ pub fn compress_bound(input_size: usize, quality: Quality) -> Option<usize> {
}
}

/// Returns peak memory usage for a given quality and window size
///
/// Given an input of `input_size` bytes in size, a `quality` and a
/// `window_size`, estimate the peak memory usage in bytes, not counting the
/// memory needed for the input and output.
#[doc(alias = "BrotliEncoderEstimatePeakMemoryUsage")]
pub fn compress_estimate_max_mem_usage(
input_size: usize,
quality: Quality,
window_size: impl Into<LargeWindowSize>,
) -> usize {
unsafe {
BrotliEncoderEstimatePeakMemoryUsage(quality.0 as _, window_size.into().0 as _, input_size)
}
}

/// Read all bytes from `input` and decompress them into `output`, returning how
/// many bytes were written.
///
Expand Down
23 changes: 22 additions & 1 deletion tests/memory.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use brotlic::{CompressionMode, Quality, WindowSize};
use brotlic::{CompressionMode, LargeWindowSize, Quality, WindowSize};

mod common;

Expand Down Expand Up @@ -72,3 +72,24 @@ fn test_medium_entropy_large() {
fn test_max_entropy_large() {
verify(common::gen_max_entropy(8192).as_slice());
}

#[test]
fn test_encoder_estimate_peak_memory_usage() {
let usage100 =
brotlic::compress_estimate_max_mem_usage(100, Quality::best(), WindowSize::best());

assert!(usage100 > 0);
}

#[test]
fn test_google_brotli_issue_1001() {
let window_size =
brotlic::compress_estimate_max_mem_usage(1024 * 1024, Quality::best(), WindowSize::best());
let large_window_size = brotlic::compress_estimate_max_mem_usage(
1024 * 1024,
Quality::best(),
LargeWindowSize::best(),
);

assert!(large_window_size > window_size);
}

0 comments on commit d25f7ce

Please sign in to comment.