From d529cb03ff46f32d9a38e8918ac539901fad54a4 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Wed, 12 Oct 2022 11:52:43 -0500 Subject: [PATCH 01/22] Wrote a test to determine the speeds of Snappy and LZ4 compression, for compressing, decompressing, or both. --- .../CompressionAlgorithmBenchmarkTest.java | 196 ++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java new file mode 100644 index 00000000..0f58b9a9 --- /dev/null +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -0,0 +1,196 @@ +package us.ihmc.tools.compression; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import us.ihmc.commons.time.Stopwatch; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Random; + +import static org.jcodec.common.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.*; + +public class CompressionAlgorithmBenchmarkTest +{ + private static final Random rand = new Random(98753244356L); + + private final boolean COMPRESS_ONLY = false; + private final boolean DECOMPRESS_ONLY = false; + private final int ELEMENTS = 128000000; + + @Test + // This test is meant to return the run times of the Snappy and LZ4 compression algorithms + public void benchmarkTimeCompression() throws IOException + { + ArrayList snappy; + ArrayList LZ4; + + snappy = benchmarkTimeSnappy(); + LZ4 = benchmarkTimeLZ4(); + + String snappyType = COMPRESS_ONLY ? "Compressed " : (DECOMPRESS_ONLY ? "Decompressed " : "Compress and Decompress "); + String LZ4Type = COMPRESS_ONLY ? "Compressed " : (DECOMPRESS_ONLY ? "Decompressed " : "Compress and Decompress "); + + assertTrue(snappy.get(0) > LZ4.get(0)); + + System.out.println(snappyType + "Snappy Time: " + snappy.get(0) + ", for element size: " + snappy.get(1)); + System.out.println(LZ4Type + "LZ4 Time: " + LZ4.get(0) + ", for element size: " + LZ4.get(1)); + } + + + public ArrayList benchmarkTimeSnappy() throws IOException + { + // Setup buffers and variable for Snappy compression + ArrayList results = new ArrayList<>(); + Stopwatch stopwatch = new Stopwatch(); + int inOffset = rand.nextInt(ELEMENTS); + int outOffset = rand.nextInt(ELEMENTS); + int decompressOffset = rand.nextInt(ELEMENTS); + double time; + + ByteBuffer in = ByteBuffer.allocate(ELEMENTS * 4 + inOffset); + ByteBuffer out = ByteBuffer.allocate(SnappyUtils.maxCompressedLength(in.remaining()) + outOffset); + ByteBuffer decompress = ByteBuffer.allocateDirect(ELEMENTS * 4 + decompressOffset); + + in.position(inOffset); + out.position(outOffset); + decompress.position(decompressOffset); + + for (int i = 0; i < ELEMENTS; i++) + { + in.putInt(rand.nextInt()); + } + + in.flip(); + in.position(inOffset); + + if (COMPRESS_ONLY) + { + // Return the time strictly for compressing the data + stopwatch.start(); + SnappyUtils.compress(in, out); + time = stopwatch.totalElapsed(); + + results.add(time); + results.add((double) ELEMENTS); + return results; + } + else if (DECOMPRESS_ONLY) + { + SnappyUtils.compress(in, out); + + out.flip(); + out.position(outOffset); + + // Return the time strictly for decompressing the data + stopwatch.start(); + SnappyUtils.uncompress(out, decompress); + time = stopwatch.totalElapsed(); + + results.add(time); + results.add((double) ELEMENTS); + return results; + } + else + { + // Return time for compressing and decompressing the data + stopwatch.start(); + SnappyUtils.compress(in, out); + assertEquals(0, in.remaining()); + + out.flip(); + out.position(outOffset); + + SnappyUtils.uncompress(out, decompress); + time = stopwatch.totalElapsed(); + + in.position(inOffset); + decompress.flip(); + decompress.position(decompressOffset); + assertEquals(ELEMENTS * 4, decompress.remaining()); + + for (int i = 0; i < ELEMENTS; i++) + { + Assertions.assertEquals(in.getInt(), decompress.getInt()); + } + + results.add(time); + results.add((double) ELEMENTS); + return results; + } + } + + + public ArrayList benchmarkTimeLZ4() + { + // Setup buffers and variable for Lz4 compression + LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); + ArrayList results = new ArrayList<>(); + + Stopwatch stopwatch = new Stopwatch(); + double time; + + ByteBuffer in = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer out = ByteBuffer.allocateDirect(impl.maxCompressedLength(in.capacity())); + ByteBuffer decompress = ByteBuffer.allocateDirect(impl.minimumDecompressedLength(out.capacity())); + + in.position(0); + out.position(0); + + for (int i = 0; i < ELEMENTS; i++) + { + in.putInt(rand.nextInt()); + } + + in.flip(); + in.position(0); + + if (COMPRESS_ONLY) + { + // Return the time strictly for compressing the data + stopwatch.start(); + impl.compress(in, out); + time = stopwatch.totalElapsed(); + + results.add(time); + results.add((double) ELEMENTS); + return results; + } + else if (DECOMPRESS_ONLY) + { + // Return the time strictly for decompressing the data + impl.compress(in, out); + + out.flip(); + out.position(0); + + stopwatch.start(); + impl.decompress(out, decompress, decompress.limit()); + time = stopwatch.totalElapsed(); + + results.add(time); + results.add((double) ELEMENTS); + return results; + } + else + { + // Return time for compressing and decompressing the data + stopwatch.start(); + impl.compress(in, out); + + out.flip(); + out.position(0); + + impl.decompress(out, decompress, decompress.limit()); + time = stopwatch.totalElapsed(); + + assertEquals(0, in.remaining()); + Assertions.assertEquals(in, decompress); + + results.add(time); + results.add((double) ELEMENTS); + return results; + } + } +} From d9c87dfd496ed4aa515a90c1c3bc9f7064b71fa2 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Wed, 12 Oct 2022 13:56:32 -0500 Subject: [PATCH 02/22] Added a test to check the compression ratios for Snappy and LZ4 in some of the best and worst case situations. --- .../CompressionAlgorithmBenchmarkTest.java | 88 ++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 0f58b9a9..eca245d2 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -19,9 +19,95 @@ public class CompressionAlgorithmBenchmarkTest private final boolean DECOMPRESS_ONLY = false; private final int ELEMENTS = 128000000; + @Test + // This test is meant to analyze the compression ratio of Snappy and LZ4 compression algorithms + public void benchmarkTestCompression() throws IOException + { + ArrayList snappy; + ArrayList LZ4; + snappy = benchmarkTestCompressionRatioSnappy(); + LZ4 = benchmarkTestCompressionRatioLZ4(); + + System.out.println("Snappy compression ratio for random input: " + snappy.get(0) * 100); + System.out.println("Snappy compression ratio for repeating input: " + snappy.get(1) * 100); + System.out.println("LZ4 compression ratio for random input: " + LZ4.get(0) * 100); + System.out.println("LZ4 compression ratio for repeating input: " + LZ4.get(1) * 100); + } + + public ArrayList benchmarkTestCompressionRatioSnappy() throws IOException + { + // Setup buffers for Snappy compression + ArrayList results = new ArrayList<>(); + ByteBuffer in = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); + + ByteBuffer out = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(ELEMENTS * 4)); + ByteBuffer repeatOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(ELEMENTS * 4)); + + + for (int i = 0; i < ELEMENTS; i++) + { + in.putInt(rand.nextInt()); + repeat.putInt(10); + } + + in.flip(); + repeat.flip(); + + SnappyUtils.compress(in, out); + SnappyUtils.compress(repeat, repeatOut); + + out.flip(); + repeatOut.flip(); + + results.add((double) out.limit() / in.limit()); + results.add((double) repeatOut.limit() / repeat.limit()); + + + return results; + } + + public ArrayList benchmarkTestCompressionRatioLZ4() + { + // Setup buffers and variable for Lz4 compression + ArrayList results = new ArrayList<>(); + LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); + + ByteBuffer in = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer out = ByteBuffer.allocateDirect(impl.maxCompressedLength(in.capacity())); + ByteBuffer repeatOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(in.capacity())); + + in.position(0); + out.position(0); + + for (int i = 0; i < ELEMENTS; i++) + { + in.putInt(rand.nextInt()); + repeat.putInt(10); + } + + in.flip(); + in.position(0); + repeat.flip(); + repeat.position(0); + + impl.compress(in, out); + impl.compress(repeat, repeatOut); + + + out.flip(); + repeatOut.flip(); + + results.add((double) out.limit() / in.limit()); + results.add((double) repeatOut.limit() / repeat.limit()); + + return results; + } + @Test // This test is meant to return the run times of the Snappy and LZ4 compression algorithms - public void benchmarkTimeCompression() throws IOException + public void benchmarkTestForTime() throws IOException { ArrayList snappy; ArrayList LZ4; From 044749304cbb637c969260f2c72c3f03fe11d09b Mon Sep 17 00:00:00 2001 From: nkitchel Date: Thu, 13 Oct 2022 13:48:58 -0500 Subject: [PATCH 03/22] Removed the use of Double, changed return types to return class object. Added hybrid buffers that have repeating and random data. --- .../CompressionAlgorithmBenchmarkTest.java | 157 +++++++++++------- 1 file changed, 100 insertions(+), 57 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index eca245d2..6121e5a4 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -5,7 +5,6 @@ import us.ihmc.commons.time.Stopwatch; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.Random; import static org.jcodec.common.Assert.assertEquals; @@ -19,88 +18,132 @@ public class CompressionAlgorithmBenchmarkTest private final boolean DECOMPRESS_ONLY = false; private final int ELEMENTS = 128000000; + private static class BenchmarkResult + { + private double compressionRatioRandom; + private double compressionRatioHybrid; + private double compressionRatioRepeat; + + private double compressionTime; + } + @Test // This test is meant to analyze the compression ratio of Snappy and LZ4 compression algorithms public void benchmarkTestCompression() throws IOException { - ArrayList snappy; - ArrayList LZ4; + BenchmarkResult snappy; + BenchmarkResult LZ4; snappy = benchmarkTestCompressionRatioSnappy(); LZ4 = benchmarkTestCompressionRatioLZ4(); - System.out.println("Snappy compression ratio for random input: " + snappy.get(0) * 100); - System.out.println("Snappy compression ratio for repeating input: " + snappy.get(1) * 100); - System.out.println("LZ4 compression ratio for random input: " + LZ4.get(0) * 100); - System.out.println("LZ4 compression ratio for repeating input: " + LZ4.get(1) * 100); + System.out.println("Snappy compression ratio for random input: " + snappy.compressionRatioRandom * 100); + System.out.println("Snappy compression ratio for hybrid input: " + snappy.compressionRatioHybrid * 100); + System.out.println("Snappy compression ratio for repeating input: " + snappy.compressionRatioRepeat * 100); + System.out.println("LZ4 compression ratio for random input: " + LZ4.compressionRatioRandom * 100); + System.out.println("LZ4 compression ratio for random input: " + LZ4.compressionRatioHybrid * 100); + System.out.println("LZ4 compression ratio for repeating input: " + LZ4.compressionRatioRandom * 100); } - - public ArrayList benchmarkTestCompressionRatioSnappy() throws IOException + public BenchmarkResult benchmarkTestCompressionRatioSnappy() throws IOException { // Setup buffers for Snappy compression - ArrayList results = new ArrayList<>(); - ByteBuffer in = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); + BenchmarkResult results = new BenchmarkResult(); - ByteBuffer out = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(ELEMENTS * 4)); - ByteBuffer repeatOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(ELEMENTS * 4)); + ByteBuffer random = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer hybrid = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer randomOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(random.capacity())); + ByteBuffer hybridOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(hybrid.capacity())); + ByteBuffer repeatOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(repeat.capacity())); for (int i = 0; i < ELEMENTS; i++) { - in.putInt(rand.nextInt()); + random.putInt(rand.nextInt()); + + if (i % 20 < 10) + { + hybrid.putInt(12); + } + else + { + hybrid.putInt(rand.nextInt()); + } + repeat.putInt(10); } - in.flip(); + random.flip(); + hybrid.flip(); repeat.flip(); - SnappyUtils.compress(in, out); + SnappyUtils.compress(random, randomOut); + SnappyUtils.compress(hybrid, hybridOut); SnappyUtils.compress(repeat, repeatOut); - out.flip(); + randomOut.flip(); + hybridOut.flip(); repeatOut.flip(); - results.add((double) out.limit() / in.limit()); - results.add((double) repeatOut.limit() / repeat.limit()); - + results.compressionRatioRandom = (double) randomOut.limit() / random.limit(); + results.compressionRatioHybrid = (double) hybridOut.limit() / hybrid.limit(); + results.compressionRatioRepeat = (double) repeatOut.limit() / repeat.limit(); return results; } - public ArrayList benchmarkTestCompressionRatioLZ4() + public BenchmarkResult benchmarkTestCompressionRatioLZ4() { // Setup buffers and variable for Lz4 compression - ArrayList results = new ArrayList<>(); + BenchmarkResult results = new BenchmarkResult(); LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); - ByteBuffer in = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer random = ByteBuffer.allocateDirect(ELEMENTS * 4); + ByteBuffer hybrid = ByteBuffer.allocateDirect(ELEMENTS * 4); ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer out = ByteBuffer.allocateDirect(impl.maxCompressedLength(in.capacity())); - ByteBuffer repeatOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(in.capacity())); - in.position(0); - out.position(0); + ByteBuffer randomOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(random.capacity())); + ByteBuffer hybridOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(hybrid.capacity())); + ByteBuffer repeatOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(repeat.capacity())); + + random.position(0); + hybrid.position(0); + repeat.position(0); for (int i = 0; i < ELEMENTS; i++) { - in.putInt(rand.nextInt()); + random.putInt(rand.nextInt()); + + if ( i % 20 < 10) + { + hybrid.putInt(12); + } + else + { + hybrid.putInt(rand.nextInt()); + } + repeat.putInt(10); } - in.flip(); - in.position(0); + random.flip(); + hybrid.flip(); repeat.flip(); + + random.position(0); + hybrid.position(0); repeat.position(0); - impl.compress(in, out); + impl.compress(random, randomOut); + impl.compress(hybrid, hybridOut); impl.compress(repeat, repeatOut); - - out.flip(); + randomOut.flip(); + hybridOut.flip(); repeatOut.flip(); - results.add((double) out.limit() / in.limit()); - results.add((double) repeatOut.limit() / repeat.limit()); + results.compressionRatioRandom = (double) randomOut.limit() / random.limit(); + results.compressionRatioHybrid = (double) hybridOut.limit() / hybrid.limit(); + results.compressionRatioRepeat = (double) repeatOut.limit() / repeat.limit(); return results; } @@ -109,8 +152,8 @@ public ArrayList benchmarkTestCompressionRatioLZ4() // This test is meant to return the run times of the Snappy and LZ4 compression algorithms public void benchmarkTestForTime() throws IOException { - ArrayList snappy; - ArrayList LZ4; + BenchmarkResult snappy; + BenchmarkResult LZ4; snappy = benchmarkTimeSnappy(); LZ4 = benchmarkTimeLZ4(); @@ -118,17 +161,17 @@ public void benchmarkTestForTime() throws IOException String snappyType = COMPRESS_ONLY ? "Compressed " : (DECOMPRESS_ONLY ? "Decompressed " : "Compress and Decompress "); String LZ4Type = COMPRESS_ONLY ? "Compressed " : (DECOMPRESS_ONLY ? "Decompressed " : "Compress and Decompress "); - assertTrue(snappy.get(0) > LZ4.get(0)); + assertTrue(snappy.compressionTime > LZ4.compressionTime); - System.out.println(snappyType + "Snappy Time: " + snappy.get(0) + ", for element size: " + snappy.get(1)); - System.out.println(LZ4Type + "LZ4 Time: " + LZ4.get(0) + ", for element size: " + LZ4.get(1)); + System.out.println(snappyType + "Snappy Time: " + snappy.compressionTime + ", for element size: " + ELEMENTS); + System.out.println(LZ4Type + "LZ4 Time: " + LZ4.compressionTime + ", for element size: " + ELEMENTS); } - public ArrayList benchmarkTimeSnappy() throws IOException + public BenchmarkResult benchmarkTimeSnappy() throws IOException { // Setup buffers and variable for Snappy compression - ArrayList results = new ArrayList<>(); + BenchmarkResult results = new BenchmarkResult(); Stopwatch stopwatch = new Stopwatch(); int inOffset = rand.nextInt(ELEMENTS); int outOffset = rand.nextInt(ELEMENTS); @@ -158,8 +201,8 @@ public ArrayList benchmarkTimeSnappy() throws IOException SnappyUtils.compress(in, out); time = stopwatch.totalElapsed(); - results.add(time); - results.add((double) ELEMENTS); + results.compressionTime = time; + return results; } else if (DECOMPRESS_ONLY) @@ -174,8 +217,8 @@ else if (DECOMPRESS_ONLY) SnappyUtils.uncompress(out, decompress); time = stopwatch.totalElapsed(); - results.add(time); - results.add((double) ELEMENTS); + results.compressionTime = time; + return results; } else @@ -201,18 +244,18 @@ else if (DECOMPRESS_ONLY) Assertions.assertEquals(in.getInt(), decompress.getInt()); } - results.add(time); - results.add((double) ELEMENTS); + results.compressionTime = time; + return results; } } - public ArrayList benchmarkTimeLZ4() + public BenchmarkResult benchmarkTimeLZ4() { // Setup buffers and variable for Lz4 compression LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); - ArrayList results = new ArrayList<>(); + BenchmarkResult results = new BenchmarkResult(); Stopwatch stopwatch = new Stopwatch(); double time; @@ -239,8 +282,8 @@ public ArrayList benchmarkTimeLZ4() impl.compress(in, out); time = stopwatch.totalElapsed(); - results.add(time); - results.add((double) ELEMENTS); + results.compressionTime = time; + return results; } else if (DECOMPRESS_ONLY) @@ -255,8 +298,8 @@ else if (DECOMPRESS_ONLY) impl.decompress(out, decompress, decompress.limit()); time = stopwatch.totalElapsed(); - results.add(time); - results.add((double) ELEMENTS); + results.compressionTime = time; + return results; } else @@ -274,8 +317,8 @@ else if (DECOMPRESS_ONLY) assertEquals(0, in.remaining()); Assertions.assertEquals(in, decompress); - results.add(time); - results.add((double) ELEMENTS); + results.compressionTime = time; + return results; } } From 84a2fdf66ae9f66d24e10c8c955d0ee336989cce Mon Sep 17 00:00:00 2001 From: nkitchel Date: Thu, 13 Oct 2022 14:25:26 -0500 Subject: [PATCH 04/22] Added lambda to method parameter so there is only one function and the type of compression algorithm used will be passed in. Rather than two functions, one for each algorithm --- .../CompressionAlgorithmBenchmarkTest.java | 119 ++++++++---------- 1 file changed, 50 insertions(+), 69 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 6121e5a4..cc89dc29 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -5,7 +5,9 @@ import us.ihmc.commons.time.Stopwatch; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Objects; import java.util.Random; +import java.util.function.BiConsumer; import static org.jcodec.common.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.*; @@ -20,21 +22,37 @@ public class CompressionAlgorithmBenchmarkTest private static class BenchmarkResult { - private double compressionRatioRandom; - private double compressionRatioHybrid; - private double compressionRatioRepeat; + private double compressionRatioRandom = -1.0; + private double compressionRatioHybrid = -1.0; + private double compressionRatioRepeat = -1.0; private double compressionTime; } @Test // This test is meant to analyze the compression ratio of Snappy and LZ4 compression algorithms - public void benchmarkTestCompression() throws IOException + public void benchmarkTestCompression() { BenchmarkResult snappy; BenchmarkResult LZ4; - snappy = benchmarkTestCompressionRatioSnappy(); - LZ4 = benchmarkTestCompressionRatioLZ4(); + + snappy = benchmarkTestCompressionRatio("Snappy", (rawIn, compressedOut) -> + { + try + { + SnappyUtils.compress(rawIn, compressedOut); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + }); + + LZ4 = benchmarkTestCompressionRatio("LZ4", (rawIn, compressedOut) -> + { + LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); + impl.compress(rawIn, compressedOut); + }); System.out.println("Snappy compression ratio for random input: " + snappy.compressionRatioRandom * 100); System.out.println("Snappy compression ratio for hybrid input: " + snappy.compressionRatioHybrid * 100); @@ -43,77 +61,42 @@ public void benchmarkTestCompression() throws IOException System.out.println("LZ4 compression ratio for random input: " + LZ4.compressionRatioHybrid * 100); System.out.println("LZ4 compression ratio for repeating input: " + LZ4.compressionRatioRandom * 100); } - public BenchmarkResult benchmarkTestCompressionRatioSnappy() throws IOException + public BenchmarkResult benchmarkTestCompressionRatio(String type, BiConsumer algorithm) { // Setup buffers for Snappy compression BenchmarkResult results = new BenchmarkResult(); + LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); ByteBuffer random = ByteBuffer.allocateDirect(ELEMENTS * 4); ByteBuffer hybrid = ByteBuffer.allocateDirect(ELEMENTS * 4); ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer randomOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(random.capacity())); - ByteBuffer hybridOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(hybrid.capacity())); - ByteBuffer repeatOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(repeat.capacity())); + ByteBuffer randomOut; + ByteBuffer hybridOut; + ByteBuffer repeatOut; - for (int i = 0; i < ELEMENTS; i++) + if (Objects.equals(type, "Snappy")) { - random.putInt(rand.nextInt()); - - if (i % 20 < 10) - { - hybrid.putInt(12); - } - else - { - hybrid.putInt(rand.nextInt()); - } - - repeat.putInt(10); + randomOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(random.capacity())); + hybridOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(hybrid.capacity())); + repeatOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(repeat.capacity())); + } + else if (Objects.equals(type, "LZ4")) + { + randomOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(random.capacity())); + hybridOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(hybrid.capacity())); + repeatOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(repeat.capacity())); + } + else + { + return results; } - - random.flip(); - hybrid.flip(); - repeat.flip(); - - SnappyUtils.compress(random, randomOut); - SnappyUtils.compress(hybrid, hybridOut); - SnappyUtils.compress(repeat, repeatOut); - - randomOut.flip(); - hybridOut.flip(); - repeatOut.flip(); - - results.compressionRatioRandom = (double) randomOut.limit() / random.limit(); - results.compressionRatioHybrid = (double) hybridOut.limit() / hybrid.limit(); - results.compressionRatioRepeat = (double) repeatOut.limit() / repeat.limit(); - - return results; - } - - public BenchmarkResult benchmarkTestCompressionRatioLZ4() - { - // Setup buffers and variable for Lz4 compression - BenchmarkResult results = new BenchmarkResult(); - LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); - - ByteBuffer random = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer hybrid = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); - - ByteBuffer randomOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(random.capacity())); - ByteBuffer hybridOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(hybrid.capacity())); - ByteBuffer repeatOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(repeat.capacity())); - - random.position(0); - hybrid.position(0); - repeat.position(0); for (int i = 0; i < ELEMENTS; i++) { random.putInt(rand.nextInt()); - if ( i % 20 < 10) + if (i % 20 < 10) { hybrid.putInt(12); } @@ -129,13 +112,9 @@ public BenchmarkResult benchmarkTestCompressionRatioLZ4() hybrid.flip(); repeat.flip(); - random.position(0); - hybrid.position(0); - repeat.position(0); - - impl.compress(random, randomOut); - impl.compress(hybrid, hybridOut); - impl.compress(repeat, repeatOut); + algorithm.accept(random, randomOut); + algorithm.accept(hybrid, hybridOut); + algorithm.accept(repeat, repeatOut); randomOut.flip(); hybridOut.flip(); @@ -148,6 +127,7 @@ public BenchmarkResult benchmarkTestCompressionRatioLZ4() return results; } + @Test // This test is meant to return the run times of the Snappy and LZ4 compression algorithms public void benchmarkTestForTime() throws IOException @@ -173,6 +153,7 @@ public BenchmarkResult benchmarkTimeSnappy() throws IOException // Setup buffers and variable for Snappy compression BenchmarkResult results = new BenchmarkResult(); Stopwatch stopwatch = new Stopwatch(); + int inOffset = rand.nextInt(ELEMENTS); int outOffset = rand.nextInt(ELEMENTS); int decompressOffset = rand.nextInt(ELEMENTS); @@ -256,8 +237,8 @@ public BenchmarkResult benchmarkTimeLZ4() // Setup buffers and variable for Lz4 compression LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); BenchmarkResult results = new BenchmarkResult(); - Stopwatch stopwatch = new Stopwatch(); + double time; ByteBuffer in = ByteBuffer.allocateDirect(ELEMENTS * 4); From 3b80f183e0d75edab4f1302d014bba83ac0d7228 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Fri, 14 Oct 2022 09:12:01 -0500 Subject: [PATCH 05/22] Revised and added interface, there is now one method that takes in an algorithm type, and a random generator and computes the compression ratio, and the compression time for the given algorithm. --- .../CompressionAlgorithmBenchmarkTest.java | 378 +++++++----------- 1 file changed, 149 insertions(+), 229 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index cc89dc29..932a79ac 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -1,306 +1,226 @@ package us.ihmc.tools.compression; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import us.ihmc.commons.time.Stopwatch; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Objects; import java.util.Random; -import java.util.function.BiConsumer; +import java.util.function.Supplier; -import static org.jcodec.common.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.*; public class CompressionAlgorithmBenchmarkTest { - private static final Random rand = new Random(98753244356L); - - private final boolean COMPRESS_ONLY = false; - private final boolean DECOMPRESS_ONLY = false; private final int ELEMENTS = 128000000; - private static class BenchmarkResult + static class BenchmarkTest { - private double compressionRatioRandom = -1.0; - private double compressionRatioHybrid = -1.0; - private double compressionRatioRepeat = -1.0; - - private double compressionTime; + double time; + double ratio; } @Test - // This test is meant to analyze the compression ratio of Snappy and LZ4 compression algorithms - public void benchmarkTestCompression() + public void benchmarkTestCompression() throws IOException { - BenchmarkResult snappy; - BenchmarkResult LZ4; - - snappy = benchmarkTestCompressionRatio("Snappy", (rawIn, compressedOut) -> - { - try - { - SnappyUtils.compress(rawIn, compressedOut); - } - catch (IOException e) - { - throw new RuntimeException(e); - } - }); - - LZ4 = benchmarkTestCompressionRatio("LZ4", (rawIn, compressedOut) -> - { - LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); - impl.compress(rawIn, compressedOut); - }); - - System.out.println("Snappy compression ratio for random input: " + snappy.compressionRatioRandom * 100); - System.out.println("Snappy compression ratio for hybrid input: " + snappy.compressionRatioHybrid * 100); - System.out.println("Snappy compression ratio for repeating input: " + snappy.compressionRatioRepeat * 100); - System.out.println("LZ4 compression ratio for random input: " + LZ4.compressionRatioRandom * 100); - System.out.println("LZ4 compression ratio for random input: " + LZ4.compressionRatioHybrid * 100); - System.out.println("LZ4 compression ratio for repeating input: " + LZ4.compressionRatioRandom * 100); - } - public BenchmarkResult benchmarkTestCompressionRatio(String type, BiConsumer algorithm) - { - // Setup buffers for Snappy compression - BenchmarkResult results = new BenchmarkResult(); - LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); - - ByteBuffer random = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer hybrid = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer repeat = ByteBuffer.allocateDirect(ELEMENTS * 4); - - ByteBuffer randomOut; - ByteBuffer hybridOut; - ByteBuffer repeatOut; - - if (Objects.equals(type, "Snappy")) - { - randomOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(random.capacity())); - hybridOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(hybrid.capacity())); - repeatOut = ByteBuffer.allocateDirect(SnappyUtils.maxCompressedLength(repeat.capacity())); - } - else if (Objects.equals(type, "LZ4")) - { - randomOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(random.capacity())); - hybridOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(hybrid.capacity())); - repeatOut = ByteBuffer.allocateDirect(impl.maxCompressedLength(repeat.capacity())); - } - else - { - return results; - } - - for (int i = 0; i < ELEMENTS; i++) + CompressionAlgorithm snappyCompression = new CompressionAlgorithm() { - random.putInt(rand.nextInt()); - - if (i % 20 < 10) + @Override + public void compress(ByteBuffer in, ByteBuffer out) throws IOException { - hybrid.putInt(12); + SnappyUtils.compress(in, out); } - else + + @Override + public void decompress(ByteBuffer in, ByteBuffer out) throws IOException { - hybrid.putInt(rand.nextInt()); + SnappyUtils.uncompress(in, out); } - repeat.putInt(10); - } + @Override + public int maxCompressedLength(int rawDataLength) + { + return SnappyUtils.maxCompressedLength(rawDataLength); + } - random.flip(); - hybrid.flip(); - repeat.flip(); + @Override + public int minCompressedLength(int rawDataLength) + { + return ELEMENTS * 4; + } + }; - algorithm.accept(random, randomOut); - algorithm.accept(hybrid, hybridOut); - algorithm.accept(repeat, repeatOut); + CompressionAlgorithm lz4Compression = new CompressionAlgorithm() + { + final LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); + @Override + public void compress(ByteBuffer in, ByteBuffer out) + { + impl.compress(in, out); + } - randomOut.flip(); - hybridOut.flip(); - repeatOut.flip(); + @Override + public void decompress(ByteBuffer in, ByteBuffer out) + { + impl.decompress(in, out, out.limit()); + } - results.compressionRatioRandom = (double) randomOut.limit() / random.limit(); - results.compressionRatioHybrid = (double) hybridOut.limit() / hybrid.limit(); - results.compressionRatioRepeat = (double) repeatOut.limit() / repeat.limit(); + @Override + public int maxCompressedLength(int rawDataLength) + { + return impl.maxCompressedLength(rawDataLength); + } - return results; - } + public int minCompressedLength(int rawDataLength) + { + return impl.minimumDecompressedLength(rawDataLength); + } + }; + BenchmarkTest snappyRatioFullRandom = benchmarkTestCompression(snappyCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest snappyRatioHybridRandom = benchmarkTestCompression(snappyCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest snappyRatioRepeat = benchmarkTestCompression(snappyCompression, repeatRandomByteBufferGenerator(ELEMENTS)); - @Test - // This test is meant to return the run times of the Snappy and LZ4 compression algorithms - public void benchmarkTestForTime() throws IOException - { - BenchmarkResult snappy; - BenchmarkResult LZ4; + BenchmarkTest lz4RatioFullRandom = benchmarkTestCompression(lz4Compression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest lz4RatioHybridRandom = benchmarkTestCompression(lz4Compression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest lz4RatioRepeat = benchmarkTestCompression(lz4Compression, repeatRandomByteBufferGenerator(ELEMENTS)); - snappy = benchmarkTimeSnappy(); - LZ4 = benchmarkTimeLZ4(); + System.out.println("Snappy compression ratio for random input: " + snappyRatioFullRandom.ratio * 100); + System.out.println("Snappy compression ratio for hybrid input: " + snappyRatioHybridRandom.ratio * 100); + System.out.println("Snappy compression ratio for repeat input: " + snappyRatioRepeat.ratio * 100); - String snappyType = COMPRESS_ONLY ? "Compressed " : (DECOMPRESS_ONLY ? "Decompressed " : "Compress and Decompress "); - String LZ4Type = COMPRESS_ONLY ? "Compressed " : (DECOMPRESS_ONLY ? "Decompressed " : "Compress and Decompress "); + System.out.println("Snappy compression speed for random input: " + snappyRatioFullRandom.time); + System.out.println("Snappy compression speed for hybrid input: " + snappyRatioHybridRandom.time); + System.out.println("Snappy compression speed for repeat input: " + snappyRatioRepeat.time); - assertTrue(snappy.compressionTime > LZ4.compressionTime); + System.out.println("LZ4 compression ratio for random input: " + lz4RatioFullRandom.ratio * 100); + System.out.println("LZ4 compression ratio for hybrid input: " + lz4RatioHybridRandom.ratio * 100); + System.out.println("LZ4 compression ratio for repeat input: " + lz4RatioRepeat.ratio * 100); - System.out.println(snappyType + "Snappy Time: " + snappy.compressionTime + ", for element size: " + ELEMENTS); - System.out.println(LZ4Type + "LZ4 Time: " + LZ4.compressionTime + ", for element size: " + ELEMENTS); + System.out.println("LZ4 compression speed for random input: " + lz4RatioFullRandom.time); + System.out.println("LZ4 compression speed for hybrid input: " + lz4RatioHybridRandom.time); + System.out.println("LZ4 compression speed for repeat input: " + lz4RatioRepeat.time); } - - public BenchmarkResult benchmarkTimeSnappy() throws IOException + public static Supplier fullRandomByteBufferGenerator(Random random, int elements) { - // Setup buffers and variable for Snappy compression - BenchmarkResult results = new BenchmarkResult(); - Stopwatch stopwatch = new Stopwatch(); - - int inOffset = rand.nextInt(ELEMENTS); - int outOffset = rand.nextInt(ELEMENTS); - int decompressOffset = rand.nextInt(ELEMENTS); - double time; - - ByteBuffer in = ByteBuffer.allocate(ELEMENTS * 4 + inOffset); - ByteBuffer out = ByteBuffer.allocate(SnappyUtils.maxCompressedLength(in.remaining()) + outOffset); - ByteBuffer decompress = ByteBuffer.allocateDirect(ELEMENTS * 4 + decompressOffset); - - in.position(inOffset); - out.position(outOffset); - decompress.position(decompressOffset); - - for (int i = 0; i < ELEMENTS; i++) + return () -> { - in.putInt(rand.nextInt()); - } + ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); - in.flip(); - in.position(inOffset); - - if (COMPRESS_ONLY) - { - // Return the time strictly for compressing the data - stopwatch.start(); - SnappyUtils.compress(in, out); - time = stopwatch.totalElapsed(); + for (int i = 0; i < elements; i++) + { + hybrid.putInt(random.nextInt()); + } - results.compressionTime = time; + return hybrid; + }; + } - return results; - } - else if (DECOMPRESS_ONLY) + public static Supplier hybridRandomByteBufferGenerator(Random random, int elements) + { + return () -> { - SnappyUtils.compress(in, out); - - out.flip(); - out.position(outOffset); + ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); - // Return the time strictly for decompressing the data - stopwatch.start(); - SnappyUtils.uncompress(out, decompress); - time = stopwatch.totalElapsed(); + for (int i = 0; i < elements; i++) + { + if (i % 20 < 10) + { + hybrid.putInt(12); + } + else + { + hybrid.putInt(random.nextInt()); + } + } - results.compressionTime = time; + return hybrid; + }; + } - return results; - } - else + public static Supplier repeatRandomByteBufferGenerator(int elements) + { + return () -> { - // Return time for compressing and decompressing the data - stopwatch.start(); - SnappyUtils.compress(in, out); - assertEquals(0, in.remaining()); + ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); - out.flip(); - out.position(outOffset); - - SnappyUtils.uncompress(out, decompress); - time = stopwatch.totalElapsed(); - - in.position(inOffset); - decompress.flip(); - decompress.position(decompressOffset); - assertEquals(ELEMENTS * 4, decompress.remaining()); - - for (int i = 0; i < ELEMENTS; i++) + for (int i = 0; i < elements; i++) { - Assertions.assertEquals(in.getInt(), decompress.getInt()); + hybrid.putInt(10); } - results.compressionTime = time; - - return results; - } + return hybrid; + }; } - - - public BenchmarkResult benchmarkTimeLZ4() + public BenchmarkTest benchmarkTestCompression(CompressionAlgorithm algorithm, Supplier randomGenerator) throws IOException { - // Setup buffers and variable for Lz4 compression - LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); - BenchmarkResult results = new BenchmarkResult(); Stopwatch stopwatch = new Stopwatch(); + BenchmarkTest results = new BenchmarkTest(); + ByteBuffer buffer = randomGenerator.get(); + ByteBuffer bufferRatioOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); + ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); + ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minCompressedLength(bufferOut.capacity())); - double time; - - ByteBuffer in = ByteBuffer.allocateDirect(ELEMENTS * 4); - ByteBuffer out = ByteBuffer.allocateDirect(impl.maxCompressedLength(in.capacity())); - ByteBuffer decompress = ByteBuffer.allocateDirect(impl.minimumDecompressedLength(out.capacity())); + buffer.flip(); + algorithm.compress(buffer, bufferRatioOut); + bufferRatioOut.flip(); - in.position(0); - out.position(0); + results.ratio = (double) bufferRatioOut.limit() / buffer.limit(); - for (int i = 0; i < ELEMENTS; i++) + boolean decompressOnly = false; + boolean compressOnly = false; + if (compressOnly) { - in.putInt(rand.nextInt()); - } - - in.flip(); - in.position(0); - - if (COMPRESS_ONLY) - { - // Return the time strictly for compressing the data stopwatch.start(); - impl.compress(in, out); - time = stopwatch.totalElapsed(); - - results.compressionTime = time; - - return results; + algorithm.compress(buffer, bufferOut); + results.time = stopwatch.totalElapsed(); + assertEquals(0, buffer.remaining()); } - else if (DECOMPRESS_ONLY) + else if (decompressOnly) { - // Return the time strictly for decompressing the data - impl.compress(in, out); + algorithm.compress(buffer, bufferOut); + assertEquals(0, buffer.remaining()); - out.flip(); - out.position(0); + bufferOut.flip(); + bufferOut.position(0); stopwatch.start(); - impl.decompress(out, decompress, decompress.limit()); - time = stopwatch.totalElapsed(); - - results.compressionTime = time; - - return results; + algorithm.decompress(buffer, bufferDecompress); + results.time = stopwatch.totalElapsed(); } else { - // Return time for compressing and decompressing the data + buffer.flip(); stopwatch.start(); - impl.compress(in, out); + algorithm.compress(buffer, bufferOut); + assertEquals(0, buffer.remaining()); - out.flip(); - out.position(0); + bufferOut.flip(); + bufferOut.position(0); - impl.decompress(out, decompress, decompress.limit()); - time = stopwatch.totalElapsed(); + algorithm.decompress(bufferOut, bufferDecompress); + results.time = stopwatch.totalElapsed(); - assertEquals(0, in.remaining()); - Assertions.assertEquals(in, decompress); - results.compressionTime = time; + bufferDecompress.flip(); + buffer.flip(); + assertEquals(ELEMENTS * 4, bufferDecompress.remaining()); - return results; + assertEquals(buffer, bufferDecompress); } + + return results; + } + + + private interface CompressionAlgorithm + { + void compress(ByteBuffer in, ByteBuffer out) throws IOException; + + void decompress(ByteBuffer in, ByteBuffer out) throws IOException; + + int maxCompressedLength(int rawDataLength); + + int minCompressedLength(int rawDataLength); } } From 685ee57f486947d8b9b56708a4558e12ebe60078 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Fri, 14 Oct 2022 12:26:18 -0500 Subject: [PATCH 06/22] The algorithm method that computes compression and decompression runs for loops to warmup the JIT compiler and then takes an average for the benchmark. --- .../CompressionAlgorithmBenchmarkTest.java | 210 +++++++++--------- 1 file changed, 104 insertions(+), 106 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 932a79ac..733ff181 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -7,20 +7,73 @@ import java.util.Random; import java.util.function.Supplier; -import static org.junit.jupiter.api.Assertions.*; - public class CompressionAlgorithmBenchmarkTest { private final int ELEMENTS = 128000000; static class BenchmarkTest { - double time; + double compressTime = 0; + double decompressTime = 0; + double totalTime = 0; double ratio; } + public static Supplier fullRandomByteBufferGenerator(Random random, int elements) + { + return () -> + { + ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); + + for (int i = 0; i < elements; i++) + { + hybrid.putInt(random.nextInt()); + } + + return hybrid; + }; + } + + public static Supplier hybridRandomByteBufferGenerator(Random random, int elements) + { + return () -> + { + ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); + + for (int i = 0; i < elements; i++) + { + if (i % 20 < 10) + { + hybrid.putInt(12); + } + else + { + hybrid.putInt(random.nextInt()); + } + } + + return hybrid; + }; + } + + public static Supplier repeatRandomByteBufferGenerator(int elements) + { + return () -> + { + ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); + + for (int i = 0; i < elements; i++) + { + hybrid.putInt(10); + } + + return hybrid; + }; + } + + @Test - public void benchmarkTestCompression() throws IOException + public void benchmarkTestCompressionAlgorithm() throws IOException { CompressionAlgorithm snappyCompression = new CompressionAlgorithm() { @@ -76,139 +129,84 @@ public int minCompressedLength(int rawDataLength) } }; - BenchmarkTest snappyRatioFullRandom = benchmarkTestCompression(snappyCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest snappyRatioHybridRandom = benchmarkTestCompression(snappyCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest snappyRatioRepeat = benchmarkTestCompression(snappyCompression, repeatRandomByteBufferGenerator(ELEMENTS)); - - BenchmarkTest lz4RatioFullRandom = benchmarkTestCompression(lz4Compression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest lz4RatioHybridRandom = benchmarkTestCompression(lz4Compression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest lz4RatioRepeat = benchmarkTestCompression(lz4Compression, repeatRandomByteBufferGenerator(ELEMENTS)); + BenchmarkTest snappyFullRandom = benchmarkTestCompressionAlgorithm(snappyCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest snappyHybridRandom = benchmarkTestCompressionAlgorithm(snappyCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest snappyRepeat = benchmarkTestCompressionAlgorithm(snappyCompression, repeatRandomByteBufferGenerator(ELEMENTS)); - System.out.println("Snappy compression ratio for random input: " + snappyRatioFullRandom.ratio * 100); - System.out.println("Snappy compression ratio for hybrid input: " + snappyRatioHybridRandom.ratio * 100); - System.out.println("Snappy compression ratio for repeat input: " + snappyRatioRepeat.ratio * 100); + BenchmarkTest lz4FullRandom = benchmarkTestCompressionAlgorithm(lz4Compression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest lz4HybridRandom = benchmarkTestCompressionAlgorithm(lz4Compression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest lz4Repeat = benchmarkTestCompressionAlgorithm(lz4Compression, repeatRandomByteBufferGenerator(ELEMENTS)); - System.out.println("Snappy compression speed for random input: " + snappyRatioFullRandom.time); - System.out.println("Snappy compression speed for hybrid input: " + snappyRatioHybridRandom.time); - System.out.println("Snappy compression speed for repeat input: " + snappyRatioRepeat.time); + System.out.println("Snappy Random: " + snappyFullRandom.ratio * 100 + " time: " + snappyFullRandom.totalTime); + System.out.println("Snappy Hybrid: " + snappyHybridRandom.ratio * 100 + " time: " + snappyHybridRandom.totalTime); + System.out.println("Snappy Repeat: " + snappyRepeat.ratio * 100 + " time: " + snappyRepeat.totalTime); - System.out.println("LZ4 compression ratio for random input: " + lz4RatioFullRandom.ratio * 100); - System.out.println("LZ4 compression ratio for hybrid input: " + lz4RatioHybridRandom.ratio * 100); - System.out.println("LZ4 compression ratio for repeat input: " + lz4RatioRepeat.ratio * 100); - - System.out.println("LZ4 compression speed for random input: " + lz4RatioFullRandom.time); - System.out.println("LZ4 compression speed for hybrid input: " + lz4RatioHybridRandom.time); - System.out.println("LZ4 compression speed for repeat input: " + lz4RatioRepeat.time); + System.out.println("LZ4 random: " + lz4FullRandom.ratio * 100 + " time: " + lz4FullRandom.totalTime); + System.out.println("LZ4 hybrid: " + lz4HybridRandom.ratio * 100 + " time: " + lz4HybridRandom.totalTime); + System.out.println("LZ4 repeat: " + lz4Repeat.ratio * 100 + " time: " + lz4Repeat.totalTime); } - public static Supplier fullRandomByteBufferGenerator(Random random, int elements) - { - return () -> - { - ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); - for (int i = 0; i < elements; i++) - { - hybrid.putInt(random.nextInt()); - } - - return hybrid; - }; - } - - public static Supplier hybridRandomByteBufferGenerator(Random random, int elements) - { - return () -> - { - ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); - - for (int i = 0; i < elements; i++) - { - if (i % 20 < 10) - { - hybrid.putInt(12); - } - else - { - hybrid.putInt(random.nextInt()); - } - } - - return hybrid; - }; - } - - public static Supplier repeatRandomByteBufferGenerator(int elements) + public BenchmarkTest benchmarkTestCompressionAlgorithm(CompressionAlgorithm algorithm, Supplier randomGenerator) throws IOException { - return () -> - { - ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); - - for (int i = 0; i < elements; i++) - { - hybrid.putInt(10); - } - - return hybrid; - }; - } - public BenchmarkTest benchmarkTestCompression(CompressionAlgorithm algorithm, Supplier randomGenerator) throws IOException - { - Stopwatch stopwatch = new Stopwatch(); + // Initial setup of variables + Stopwatch stopwatchCompress = new Stopwatch(); + Stopwatch stopwatchDecompress = new Stopwatch(); + Stopwatch stopwatchTotal = new Stopwatch(); BenchmarkTest results = new BenchmarkTest(); ByteBuffer buffer = randomGenerator.get(); - ByteBuffer bufferRatioOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minCompressedLength(bufferOut.capacity())); - buffer.flip(); - algorithm.compress(buffer, bufferRatioOut); - bufferRatioOut.flip(); - - results.ratio = (double) bufferRatioOut.limit() / buffer.limit(); - - boolean decompressOnly = false; - boolean compressOnly = false; - if (compressOnly) - { - stopwatch.start(); - algorithm.compress(buffer, bufferOut); - results.time = stopwatch.totalElapsed(); - assertEquals(0, buffer.remaining()); - } - else if (decompressOnly) + // Warmup for algorithm methods, helps to optimize the JIT compiler + for (int i = 0; i < 500; i++) { + buffer.flip(); + bufferOut.clear(); + bufferDecompress.clear(); + algorithm.compress(buffer, bufferOut); - assertEquals(0, buffer.remaining()); bufferOut.flip(); bufferOut.position(0); - stopwatch.start(); - algorithm.decompress(buffer, bufferDecompress); - results.time = stopwatch.totalElapsed(); + algorithm.decompress(bufferOut, bufferDecompress); } - else + + int iterations = 100; + + // Run benchmark on algorithm that takes an average for the ratio and time computed + for (int i = 0; i < iterations; i++) { buffer.flip(); - stopwatch.start(); + bufferOut.clear(); + bufferDecompress.clear(); + + stopwatchTotal.start(); + stopwatchCompress.start(); + algorithm.compress(buffer, bufferOut); - assertEquals(0, buffer.remaining()); + + results.compressTime += stopwatchCompress.totalElapsed(); bufferOut.flip(); bufferOut.position(0); - algorithm.decompress(bufferOut, bufferDecompress); - results.time = stopwatch.totalElapsed(); + results.ratio += (double) bufferOut.limit() / buffer.limit(); + stopwatchDecompress.start(); - bufferDecompress.flip(); - buffer.flip(); - assertEquals(ELEMENTS * 4, bufferDecompress.remaining()); + algorithm.decompress(bufferOut, bufferDecompress); - assertEquals(buffer, bufferDecompress); + results.decompressTime += stopwatchDecompress.totalElapsed(); + results.totalTime += stopwatchTotal.totalElapsed(); } + results.ratio /= iterations; + results.compressTime /= iterations; + results.decompressTime /= iterations; + results.totalTime /= iterations; + return results; } From 380684e9ba363096d236e6cd13d148ef04fec098 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Fri, 14 Oct 2022 14:10:39 -0500 Subject: [PATCH 07/22] Added assertions to ensure the compression and decompression happens correctly. --- .../compression/CompressionAlgorithmBenchmarkTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 733ff181..14578001 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -7,6 +7,8 @@ import java.util.Random; import java.util.function.Supplier; +import static org.junit.jupiter.api.Assertions.*; + public class CompressionAlgorithmBenchmarkTest { private final int ELEMENTS = 128000000; @@ -167,10 +169,14 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(CompressionAlgorithm algo algorithm.compress(buffer, bufferOut); + assertEquals(0, buffer.remaining()); + bufferOut.flip(); bufferOut.position(0); algorithm.decompress(bufferOut, bufferDecompress); + + assertEquals(buffer, bufferDecompress); } int iterations = 100; From dfd763dfa83cf3a9965312cce672c30ed8ef9248 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Mon, 17 Oct 2022 08:57:03 -0500 Subject: [PATCH 08/22] Comments added --- .../tools/compression/CompressionAlgorithmBenchmarkTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 14578001..a591250d 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -160,8 +160,8 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(CompressionAlgorithm algo ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minCompressedLength(bufferOut.capacity())); - // Warmup for algorithm methods, helps to optimize the JIT compiler - for (int i = 0; i < 500; i++) + // Warmup for algorithm methods, helps to optimize the JIT compiler, at 400 cycles the for loop takes 15 minutes + for (int i = 0; i < 400; i++) { buffer.flip(); bufferOut.clear(); From 61f5eea1af6a45b9334ca64ff8a3e113752170d1 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Mon, 24 Oct 2022 11:46:59 -0500 Subject: [PATCH 09/22] Added LZ4 1.9 to compression benchmark to see if it is any faster than the current version Lz4 1.8. --- .../CompressionAlgorithmBenchmarkTest.java | 147 ++++++++++++++---- 1 file changed, 119 insertions(+), 28 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index a591250d..d0387065 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -1,7 +1,11 @@ package us.ihmc.tools.compression; +import org.bytedeco.javacpp.Pointer; +import org.bytedeco.javacpp.SizeTPointer; +import org.bytedeco.lz4.LZ4FDecompressionContext; import org.junit.jupiter.api.Test; import us.ihmc.commons.time.Stopwatch; + import java.io.IOException; import java.nio.ByteBuffer; import java.util.Random; @@ -11,7 +15,7 @@ public class CompressionAlgorithmBenchmarkTest { - private final int ELEMENTS = 128000000; + private final int ELEMENTS = 10200; static class BenchmarkTest { @@ -25,7 +29,7 @@ public static Supplier fullRandomByteBufferGenerator(Random random, { return () -> { - ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); + ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); for (int i = 0; i < elements; i++) { @@ -75,7 +79,7 @@ public static Supplier repeatRandomByteBufferGenerator(int elements) @Test - public void benchmarkTestCompressionAlgorithm() throws IOException + public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4ByteDecoCompressionImplementation.LZ4Exception { CompressionAlgorithm snappyCompression = new CompressionAlgorithm() { @@ -131,25 +135,78 @@ public int minCompressedLength(int rawDataLength) } }; - BenchmarkTest snappyFullRandom = benchmarkTestCompressionAlgorithm(snappyCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest snappyHybridRandom = benchmarkTestCompressionAlgorithm(snappyCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest snappyRepeat = benchmarkTestCompressionAlgorithm(snappyCompression, repeatRandomByteBufferGenerator(ELEMENTS)); + CompressionAlgorithm lz4ByteDeco = new CompressionAlgorithm() + { + final LZ4ByteDecoCompressionImplementation impl = new LZ4ByteDecoCompressionImplementation(); + + @Override + public void compress(ByteBuffer in, ByteBuffer out) + { + Pointer inPointer = new Pointer(in); + Pointer outPointer = new Pointer(out); + + LZ4ByteDecoCompressionImplementation.compress(in, inPointer, out, outPointer); + } - BenchmarkTest lz4FullRandom = benchmarkTestCompressionAlgorithm(lz4Compression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest lz4HybridRandom = benchmarkTestCompressionAlgorithm(lz4Compression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - BenchmarkTest lz4Repeat = benchmarkTestCompressionAlgorithm(lz4Compression, repeatRandomByteBufferGenerator(ELEMENTS)); + @Override + public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4ByteDecoCompressionImplementation.LZ4Exception + { + LZ4FDecompressionContext decompressionContext; + decompressionContext = LZ4ByteDecoCompressionImplementation.ByteDecoLZ4CompressionImplementation(); + Pointer inPointer = new Pointer(in); + Pointer outPointer = new Pointer(out); + SizeTPointer inSize = new SizeTPointer(in.limit()); + SizeTPointer outSize = new SizeTPointer(out.remaining()); + + LZ4ByteDecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, in, ELEMENTS); + } + @Override + public int maxCompressedLength(int rawDataLength) + { + return impl.maxCompressedLength(rawDataLength); + } + + @Override + public int minCompressedLength(int rawDataLength) + { + return impl.minimumDecompressedLength(rawDataLength); + } + }; + + // Snappy Compression + BenchmarkTest snappyFullRandom = benchmarkTestCompressionAlgorithm(true, snappyCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); System.out.println("Snappy Random: " + snappyFullRandom.ratio * 100 + " time: " + snappyFullRandom.totalTime); + + BenchmarkTest snappyHybridRandom = benchmarkTestCompressionAlgorithm(false, snappyCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); System.out.println("Snappy Hybrid: " + snappyHybridRandom.ratio * 100 + " time: " + snappyHybridRandom.totalTime); + + BenchmarkTest snappyRepeat = benchmarkTestCompressionAlgorithm(false, snappyCompression, repeatRandomByteBufferGenerator(ELEMENTS)); System.out.println("Snappy Repeat: " + snappyRepeat.ratio * 100 + " time: " + snappyRepeat.totalTime); - System.out.println("LZ4 random: " + lz4FullRandom.ratio * 100 + " time: " + lz4FullRandom.totalTime); - System.out.println("LZ4 hybrid: " + lz4HybridRandom.ratio * 100 + " time: " + lz4HybridRandom.totalTime); - System.out.println("LZ4 repeat: " + lz4Repeat.ratio * 100 + " time: " + lz4Repeat.totalTime); - } + // LZ4 1.8 Compression + BenchmarkTest lz4FullRandom = benchmarkTestCompressionAlgorithm(true, lz4Compression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + System.out.println("LZ4 1.8 random: " + lz4FullRandom.ratio * 100 + " time: " + lz4FullRandom.totalTime); + + BenchmarkTest lz4HybridRandom = benchmarkTestCompressionAlgorithm(false, lz4Compression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + System.out.println("LZ4 1.8 hybrid: " + lz4HybridRandom.ratio * 100 + " time: " + lz4HybridRandom.totalTime); + + BenchmarkTest lz4Repeat = benchmarkTestCompressionAlgorithm(false, lz4Compression, repeatRandomByteBufferGenerator(ELEMENTS)); + System.out.println("LZ4 1.8 repeat: " + lz4Repeat.ratio * 100 + " time: " + lz4Repeat.totalTime); + + // LZ4 1.9 Compression + BenchmarkTest lz4ByteDecoFullRandom = benchmarkTestCompressionAlgorithm(true, lz4ByteDeco, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + System.out.println("lz4 1.9 ByteDeco Random: " + lz4ByteDecoFullRandom.ratio * 100 + " time: " + lz4ByteDecoFullRandom.totalTime); + BenchmarkTest lz4ByteDecoHybridRandom = benchmarkTestCompressionAlgorithm(false, lz4ByteDeco, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + System.out.println("lz4 1.9 ByteDeco Hybrid: " + lz4ByteDecoHybridRandom.ratio * 100 + " time: " + lz4ByteDecoHybridRandom.totalTime); - public BenchmarkTest benchmarkTestCompressionAlgorithm(CompressionAlgorithm algorithm, Supplier randomGenerator) throws IOException + BenchmarkTest lz4ByteDecoRepeat = benchmarkTestCompressionAlgorithm(false, lz4ByteDeco, repeatRandomByteBufferGenerator(ELEMENTS)); + System.out.println("lz4 1.9 ByteDeco Repeat: " + lz4ByteDecoRepeat.ratio * 100 + " time: " + lz4ByteDecoRepeat.totalTime); + } + + public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, CompressionAlgorithm algorithm, Supplier randomGenerator) + throws IOException, LZ4ByteDecoCompressionImplementation.LZ4Exception { // Initial setup of variables Stopwatch stopwatchCompress = new Stopwatch(); @@ -160,26 +217,46 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(CompressionAlgorithm algo ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minCompressedLength(bufferOut.capacity())); - // Warmup for algorithm methods, helps to optimize the JIT compiler, at 400 cycles the for loop takes 15 minutes - for (int i = 0; i < 400; i++) + // Warmup for algorithm methods, helps to optimize the JIT compiler, optimized at about 38 iterations + if (warmup) { - buffer.flip(); - bufferOut.clear(); - bufferDecompress.clear(); + for (int i = 0; i < 5000; i++) + { + buffer.flip(); + bufferOut.clear(); + bufferDecompress.clear(); - algorithm.compress(buffer, bufferOut); + algorithm.compress(buffer, bufferOut); - assertEquals(0, buffer.remaining()); + if (bufferOut.position() == 0) + { + bufferOut.position(bufferOut.limit()); + } - bufferOut.flip(); - bufferOut.position(0); + bufferOut.flip(); - algorithm.decompress(bufferOut, bufferDecompress); + algorithm.decompress(bufferOut, bufferDecompress); + + if ( buffer.position() == 0) + { + buffer.position(buffer.limit()); + } + + if (bufferDecompress.position() == 0) + { + bufferDecompress.position(bufferDecompress.limit()); + } - assertEquals(buffer, bufferDecompress); + assertEquals(buffer, bufferDecompress); + + for (int j = 0; j < ELEMENTS; j++) + { + assertEquals(buffer.get(j), bufferDecompress.get(j)); + } + } } - int iterations = 100; + int iterations = 800; // Run benchmark on algorithm that takes an average for the ratio and time computed for (int i = 0; i < iterations; i++) @@ -195,8 +272,12 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(CompressionAlgorithm algo results.compressTime += stopwatchCompress.totalElapsed(); + if (bufferOut.position() == 0) + { + bufferOut.position(bufferOut.limit()); + } + bufferOut.flip(); - bufferOut.position(0); results.ratio += (double) bufferOut.limit() / buffer.limit(); @@ -206,6 +287,16 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(CompressionAlgorithm algo results.decompressTime += stopwatchDecompress.totalElapsed(); results.totalTime += stopwatchTotal.totalElapsed(); + + if ( buffer.position() == 0) + { + buffer.position(buffer.limit()); + } + + if (bufferDecompress.position() == 0) + { + bufferDecompress.position(bufferDecompress.limit()); + } } results.ratio /= iterations; @@ -221,7 +312,7 @@ private interface CompressionAlgorithm { void compress(ByteBuffer in, ByteBuffer out) throws IOException; - void decompress(ByteBuffer in, ByteBuffer out) throws IOException; + void decompress(ByteBuffer in, ByteBuffer out) throws IOException, LZ4ByteDecoCompressionImplementation.LZ4Exception; int maxCompressedLength(int rawDataLength); From 6bc1a49fb12d9528ab69f29cdbf53d562a698a99 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Mon, 24 Oct 2022 11:47:49 -0500 Subject: [PATCH 10/22] Added LZ4 1.9 compression implementation for compress and decompress methods. --- .../LZ4ByteDecoCompressionImplementation.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java diff --git a/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java b/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java new file mode 100644 index 00000000..47a0bcc9 --- /dev/null +++ b/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java @@ -0,0 +1,63 @@ +package us.ihmc.tools.compression; + +import java.nio.BufferOverflowException; +import java.nio.ByteBuffer; +import org.bytedeco.javacpp.*; +import org.bytedeco.lz4.*; +import org.bytedeco.lz4.global.lz4; + +public class LZ4ByteDecoCompressionImplementation +{ + public static LZ4FDecompressionContext ByteDecoLZ4CompressionImplementation() throws LZ4Exception + { + final LZ4FDecompressionContext dctx = new LZ4FDecompressionContext(); + final long ctxError = lz4.LZ4F_createDecompressionContext(dctx, lz4.LZ4F_VERSION); + checkForError(ctxError); + return dctx; + } + + public static void compress(ByteBuffer src, Pointer srcPointer, ByteBuffer dst, Pointer dstPointer) + { + lz4.LZ4F_compressFrame(dstPointer, dst.limit(), srcPointer, src.limit(), null); + } + + public static void decompress(LZ4FDecompressionContext dctx, + Pointer compressedPointer, + Pointer dstPointer, + SizeTPointer srcSize, + SizeTPointer dstSize, + ByteBuffer compressed, + int uncompressedSize) + { + if (compressed.position() + uncompressedSize > compressed.limit()) + { + throw new BufferOverflowException(); + } + + lz4.LZ4F_decompress(dctx, dstPointer, dstSize, compressedPointer, srcSize, null); + } + + public int maxCompressedLength(int uncompressedLength) + { + final int maxCompressedSize = (int) lz4.LZ4F_compressFrameBound(uncompressedLength, null); + return lz4.LZ4_compressBound(maxCompressedSize); + } + + public int minimumDecompressedLength(int compressedLength) + { + double y = (long) (compressedLength - 16) * 255; + return (int) Math.round(y / 256); + } + + private static void checkForError(long errorCode) throws LZ4Exception { + if (lz4.LZ4F_isError(errorCode) != 0) { + throw new LZ4Exception(lz4.LZ4F_getErrorName(errorCode).getString()); + } + } + + public static final class LZ4Exception extends Exception { + public LZ4Exception(final String message) { + super(message); + } + } +} \ No newline at end of file From df9de2d79a8c439968ba0c15ebae845d1a89fcf0 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Mon, 24 Oct 2022 14:38:47 -0500 Subject: [PATCH 11/22] Fixed compression ratio that wasn't being calculated correctly. --- .../CompressionAlgorithmBenchmarkTest.java | 27 ++++++++++--------- .../LZ4ByteDecoCompressionImplementation.java | 8 +++--- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index d0387065..cabd5e16 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -84,9 +84,10 @@ public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4ByteDecoC CompressionAlgorithm snappyCompression = new CompressionAlgorithm() { @Override - public void compress(ByteBuffer in, ByteBuffer out) throws IOException + public double compress(ByteBuffer in, ByteBuffer out) throws IOException { SnappyUtils.compress(in, out); + return 0; } @Override @@ -112,9 +113,9 @@ public int minCompressedLength(int rawDataLength) { final LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); @Override - public void compress(ByteBuffer in, ByteBuffer out) + public double compress(ByteBuffer in, ByteBuffer out) { - impl.compress(in, out); + return impl.compress(in, out); } @Override @@ -140,12 +141,12 @@ public int minCompressedLength(int rawDataLength) final LZ4ByteDecoCompressionImplementation impl = new LZ4ByteDecoCompressionImplementation(); @Override - public void compress(ByteBuffer in, ByteBuffer out) + public double compress(ByteBuffer in, ByteBuffer out) { Pointer inPointer = new Pointer(in); Pointer outPointer = new Pointer(out); - LZ4ByteDecoCompressionImplementation.compress(in, inPointer, out, outPointer); + return LZ4ByteDecoCompressionImplementation.compress(in, inPointer, out, outPointer); } @Override @@ -158,7 +159,7 @@ public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4ByteDecoCompress SizeTPointer inSize = new SizeTPointer(in.limit()); SizeTPointer outSize = new SizeTPointer(out.remaining()); - LZ4ByteDecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, in, ELEMENTS); + LZ4ByteDecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); } @Override @@ -213,6 +214,8 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi Stopwatch stopwatchDecompress = new Stopwatch(); Stopwatch stopwatchTotal = new Stopwatch(); BenchmarkTest results = new BenchmarkTest(); + int bytesCompressed = 0; + ByteBuffer buffer = randomGenerator.get(); ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minCompressedLength(bufferOut.capacity())); @@ -226,11 +229,11 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi bufferOut.clear(); bufferDecompress.clear(); - algorithm.compress(buffer, bufferOut); + bytesCompressed = (int) algorithm.compress(buffer, bufferOut); if (bufferOut.position() == 0) { - bufferOut.position(bufferOut.limit()); + bufferOut.position(bytesCompressed); } bufferOut.flip(); @@ -268,13 +271,13 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi stopwatchTotal.start(); stopwatchCompress.start(); - algorithm.compress(buffer, bufferOut); + bytesCompressed = (int) algorithm.compress(buffer, bufferOut); results.compressTime += stopwatchCompress.totalElapsed(); if (bufferOut.position() == 0) { - bufferOut.position(bufferOut.limit()); + bufferOut.position(bytesCompressed); } bufferOut.flip(); @@ -288,7 +291,7 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi results.decompressTime += stopwatchDecompress.totalElapsed(); results.totalTime += stopwatchTotal.totalElapsed(); - if ( buffer.position() == 0) + if (buffer.position() == 0) { buffer.position(buffer.limit()); } @@ -310,7 +313,7 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi private interface CompressionAlgorithm { - void compress(ByteBuffer in, ByteBuffer out) throws IOException; + double compress(ByteBuffer in, ByteBuffer out) throws IOException; void decompress(ByteBuffer in, ByteBuffer out) throws IOException, LZ4ByteDecoCompressionImplementation.LZ4Exception; diff --git a/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java b/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java index 47a0bcc9..3a339e25 100644 --- a/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java +++ b/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java @@ -16,9 +16,9 @@ public static LZ4FDecompressionContext ByteDecoLZ4CompressionImplementation() th return dctx; } - public static void compress(ByteBuffer src, Pointer srcPointer, ByteBuffer dst, Pointer dstPointer) + public static double compress(ByteBuffer src, Pointer srcPointer, ByteBuffer dst, Pointer dstPointer) { - lz4.LZ4F_compressFrame(dstPointer, dst.limit(), srcPointer, src.limit(), null); + return lz4.LZ4F_compressFrame(dstPointer, dst.limit(), srcPointer, src.limit(), null); } public static void decompress(LZ4FDecompressionContext dctx, @@ -26,10 +26,10 @@ public static void decompress(LZ4FDecompressionContext dctx, Pointer dstPointer, SizeTPointer srcSize, SizeTPointer dstSize, - ByteBuffer compressed, + ByteBuffer uncompressed, int uncompressedSize) { - if (compressed.position() + uncompressedSize > compressed.limit()) + if (uncompressed.position() + uncompressedSize > uncompressed.limit()) { throw new BufferOverflowException(); } From 8b3acae054aec5a237af18460857c50c515b922c Mon Sep 17 00:00:00 2001 From: nkitchel Date: Tue, 25 Oct 2022 18:01:53 -0500 Subject: [PATCH 12/22] Fixed Bytedeco to one word. --- .../CompressionAlgorithmBenchmarkTest.java | 25 +++++++++++++------ ...LZ4BytedecoCompressionImplementation.java} | 2 +- 2 files changed, 18 insertions(+), 9 deletions(-) rename src/test/java/us/ihmc/tools/compression/{LZ4ByteDecoCompressionImplementation.java => LZ4BytedecoCompressionImplementation.java} (97%) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index cabd5e16..041724d2 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -77,9 +77,18 @@ public static Supplier repeatRandomByteBufferGenerator(int elements) }; } + @Test + public void testOverAndOver() throws LZ4BytedecoCompressionImplementation.LZ4Exception, IOException + { + for (int i = 0; i < 100; i++) + { + benchmarkTestCompressionAlgorithm(); + } + } + @Test - public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4ByteDecoCompressionImplementation.LZ4Exception + public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4BytedecoCompressionImplementation.LZ4Exception { CompressionAlgorithm snappyCompression = new CompressionAlgorithm() { @@ -138,7 +147,7 @@ public int minCompressedLength(int rawDataLength) CompressionAlgorithm lz4ByteDeco = new CompressionAlgorithm() { - final LZ4ByteDecoCompressionImplementation impl = new LZ4ByteDecoCompressionImplementation(); + final LZ4BytedecoCompressionImplementation impl = new LZ4BytedecoCompressionImplementation(); @Override public double compress(ByteBuffer in, ByteBuffer out) @@ -146,20 +155,20 @@ public double compress(ByteBuffer in, ByteBuffer out) Pointer inPointer = new Pointer(in); Pointer outPointer = new Pointer(out); - return LZ4ByteDecoCompressionImplementation.compress(in, inPointer, out, outPointer); + return LZ4BytedecoCompressionImplementation.compress(in, inPointer, out, outPointer); } @Override - public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4ByteDecoCompressionImplementation.LZ4Exception + public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4BytedecoCompressionImplementation.LZ4Exception { LZ4FDecompressionContext decompressionContext; - decompressionContext = LZ4ByteDecoCompressionImplementation.ByteDecoLZ4CompressionImplementation(); + decompressionContext = LZ4BytedecoCompressionImplementation.ByteDecoLZ4CompressionImplementation(); Pointer inPointer = new Pointer(in); Pointer outPointer = new Pointer(out); SizeTPointer inSize = new SizeTPointer(in.limit()); SizeTPointer outSize = new SizeTPointer(out.remaining()); - LZ4ByteDecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); + LZ4BytedecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); } @Override @@ -207,7 +216,7 @@ public int minCompressedLength(int rawDataLength) } public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, CompressionAlgorithm algorithm, Supplier randomGenerator) - throws IOException, LZ4ByteDecoCompressionImplementation.LZ4Exception + throws IOException, LZ4BytedecoCompressionImplementation.LZ4Exception { // Initial setup of variables Stopwatch stopwatchCompress = new Stopwatch(); @@ -315,7 +324,7 @@ private interface CompressionAlgorithm { double compress(ByteBuffer in, ByteBuffer out) throws IOException; - void decompress(ByteBuffer in, ByteBuffer out) throws IOException, LZ4ByteDecoCompressionImplementation.LZ4Exception; + void decompress(ByteBuffer in, ByteBuffer out) throws IOException, LZ4BytedecoCompressionImplementation.LZ4Exception; int maxCompressedLength(int rawDataLength); diff --git a/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java similarity index 97% rename from src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java rename to src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java index 3a339e25..83f5d287 100644 --- a/src/test/java/us/ihmc/tools/compression/LZ4ByteDecoCompressionImplementation.java +++ b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java @@ -6,7 +6,7 @@ import org.bytedeco.lz4.*; import org.bytedeco.lz4.global.lz4; -public class LZ4ByteDecoCompressionImplementation +public class LZ4BytedecoCompressionImplementation { public static LZ4FDecompressionContext ByteDecoLZ4CompressionImplementation() throws LZ4Exception { From 06a6f65f53d3769c21bb5bbe63be83c6a92aa55f Mon Sep 17 00:00:00 2001 From: Nick Kitchel Date: Tue, 25 Oct 2022 21:56:42 -0500 Subject: [PATCH 13/22] File formatting for future methods. --- .../CompressionAlgorithmBenchmarkTest.java | 178 +++++++++--------- 1 file changed, 90 insertions(+), 88 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 041724d2..410a7dac 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -3,6 +3,7 @@ import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.SizeTPointer; import org.bytedeco.lz4.LZ4FDecompressionContext; +import org.bytedeco.lz4.global.lz4; import org.junit.jupiter.api.Test; import us.ihmc.commons.time.Stopwatch; @@ -15,7 +16,7 @@ public class CompressionAlgorithmBenchmarkTest { - private final int ELEMENTS = 10200; + private final int ELEMENTS = 510; static class BenchmarkTest { @@ -77,113 +78,113 @@ public static Supplier repeatRandomByteBufferGenerator(int elements) }; } - @Test - public void testOverAndOver() throws LZ4BytedecoCompressionImplementation.LZ4Exception, IOException + CompressionAlgorithm snappyCompression = new CompressionAlgorithm() { - for (int i = 0; i < 100; i++) + @Override + public double compress(ByteBuffer in, ByteBuffer out) throws IOException { - benchmarkTestCompressionAlgorithm(); + SnappyUtils.compress(in, out); + return 0; } - } - - @Test - public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4BytedecoCompressionImplementation.LZ4Exception - { - CompressionAlgorithm snappyCompression = new CompressionAlgorithm() + @Override + public void decompress(ByteBuffer in, ByteBuffer out) throws IOException { - @Override - public double compress(ByteBuffer in, ByteBuffer out) throws IOException - { - SnappyUtils.compress(in, out); - return 0; - } + SnappyUtils.uncompress(in, out); + } - @Override - public void decompress(ByteBuffer in, ByteBuffer out) throws IOException - { - SnappyUtils.uncompress(in, out); - } + @Override + public int maxCompressedLength(int rawDataLength) + { + return SnappyUtils.maxCompressedLength(rawDataLength); + } - @Override - public int maxCompressedLength(int rawDataLength) - { - return SnappyUtils.maxCompressedLength(rawDataLength); - } + @Override + public int minCompressedLength(int rawDataLength) + { + return ELEMENTS * 4; + } + }; - @Override - public int minCompressedLength(int rawDataLength) - { - return ELEMENTS * 4; - } - }; + CompressionAlgorithm lz4Compression = new CompressionAlgorithm() + { + final LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); + @Override + public double compress(ByteBuffer in, ByteBuffer out) + { + return impl.compress(in, out); + } - CompressionAlgorithm lz4Compression = new CompressionAlgorithm() + @Override + public void decompress(ByteBuffer in, ByteBuffer out) { - final LZ4CompressionImplementation impl = new LZ4CompressionImplementation(); - @Override - public double compress(ByteBuffer in, ByteBuffer out) - { - return impl.compress(in, out); - } + impl.decompress(in, out, out.limit()); + } - @Override - public void decompress(ByteBuffer in, ByteBuffer out) - { - impl.decompress(in, out, out.limit()); - } + @Override + public int maxCompressedLength(int rawDataLength) + { + return impl.maxCompressedLength(rawDataLength); + } - @Override - public int maxCompressedLength(int rawDataLength) - { - return impl.maxCompressedLength(rawDataLength); - } + public int minCompressedLength(int rawDataLength) + { + return impl.minimumDecompressedLength(rawDataLength); + } + }; - public int minCompressedLength(int rawDataLength) - { - return impl.minimumDecompressedLength(rawDataLength); - } - }; + CompressionAlgorithm lz4ByteDeco = new CompressionAlgorithm() + { + final LZ4BytedecoCompressionImplementation impl = new LZ4BytedecoCompressionImplementation(); - CompressionAlgorithm lz4ByteDeco = new CompressionAlgorithm() + @Override + public double compress(ByteBuffer in, ByteBuffer out) { - final LZ4BytedecoCompressionImplementation impl = new LZ4BytedecoCompressionImplementation(); + Pointer inPointer = new Pointer(in); + Pointer outPointer = new Pointer(out); - @Override - public double compress(ByteBuffer in, ByteBuffer out) - { - Pointer inPointer = new Pointer(in); - Pointer outPointer = new Pointer(out); + return LZ4BytedecoCompressionImplementation.compress(in, inPointer, out, outPointer); + } - return LZ4BytedecoCompressionImplementation.compress(in, inPointer, out, outPointer); - } + @Override + public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4BytedecoCompressionImplementation.LZ4Exception + { + LZ4FDecompressionContext decompressionContext; + decompressionContext = LZ4BytedecoCompressionImplementation.ByteDecoLZ4CompressionImplementation(); + Pointer inPointer = new Pointer(in); + Pointer outPointer = new Pointer(out); + SizeTPointer inSize = new SizeTPointer(in.limit()); + SizeTPointer outSize = new SizeTPointer(out.remaining()); + + LZ4BytedecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); + lz4.LZ4F_freeDecompressionContext(decompressionContext); + } - @Override - public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4BytedecoCompressionImplementation.LZ4Exception - { - LZ4FDecompressionContext decompressionContext; - decompressionContext = LZ4BytedecoCompressionImplementation.ByteDecoLZ4CompressionImplementation(); - Pointer inPointer = new Pointer(in); - Pointer outPointer = new Pointer(out); - SizeTPointer inSize = new SizeTPointer(in.limit()); - SizeTPointer outSize = new SizeTPointer(out.remaining()); - - LZ4BytedecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); - } + @Override + public int maxCompressedLength(int rawDataLength) + { + return impl.maxCompressedLength(rawDataLength); + } - @Override - public int maxCompressedLength(int rawDataLength) - { - return impl.maxCompressedLength(rawDataLength); - } + @Override + public int minCompressedLength(int rawDataLength) + { + return impl.minimumDecompressedLength(rawDataLength); + } + }; - @Override - public int minCompressedLength(int rawDataLength) - { - return impl.minimumDecompressedLength(rawDataLength); - } - }; + @Test + public void testOverAndOver() throws LZ4BytedecoCompressionImplementation.LZ4Exception, IOException + { + for (int i = 0; i < 10; i++) + { + benchmarkTestCompressionAlgorithm(); + } + } + @Test + public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4BytedecoCompressionImplementation.LZ4Exception + { // Snappy Compression BenchmarkTest snappyFullRandom = benchmarkTestCompressionAlgorithm(true, snappyCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); System.out.println("Snappy Random: " + snappyFullRandom.ratio * 100 + " time: " + snappyFullRandom.totalTime); @@ -223,7 +224,7 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi Stopwatch stopwatchDecompress = new Stopwatch(); Stopwatch stopwatchTotal = new Stopwatch(); BenchmarkTest results = new BenchmarkTest(); - int bytesCompressed = 0; + int bytesCompressed; ByteBuffer buffer = randomGenerator.get(); ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); @@ -265,6 +266,7 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi { assertEquals(buffer.get(j), bufferDecompress.get(j)); } + } } From d77e4142cc089a287e3bd3a49def2282423545e2 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Mon, 7 Nov 2022 16:19:06 -0600 Subject: [PATCH 14/22] Added implementation for LZ4 1.9 from Bytedeco. Added extensive comments so the user can understand easier. Moved every use of the new keyword, so it's not called every time. --- build.gradle.kts | 2 + .../CompressionAlgorithmBenchmarkTest.java | 100 +++++++++++++----- .../LZ4BytedecoCompressionImplementation.java | 9 +- 3 files changed, 82 insertions(+), 29 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 9303620b..d6ea80a5 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -57,6 +57,8 @@ mainDependencies { testDependencies { api("us.ihmc:ihmc-commons-testing:0.31.0") + api("org.bytedeco:lz4-platform:1.9.4-1.5.8") + api("org.bytedeco:lz4:1.9.4-1.5.8") } app.entrypoint("IHMCLogger", "us.ihmc.robotDataLogger.logger.YoVariableLoggerDispatcher") diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 410a7dac..42c15968 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -6,6 +6,7 @@ import org.bytedeco.lz4.global.lz4; import org.junit.jupiter.api.Test; import us.ihmc.commons.time.Stopwatch; +import us.ihmc.perception.MutableBytePointer; import java.io.IOException; import java.nio.ByteBuffer; @@ -14,10 +15,11 @@ import static org.junit.jupiter.api.Assertions.*; -public class CompressionAlgorithmBenchmarkTest +public class CompressionAlgorithmBenchmarkTest extends Pointer { - private final int ELEMENTS = 510; + private final int ELEMENTS = 1024; + // This class holds the variables that are used to measure the results of the benchmark static class BenchmarkTest { double compressTime = 0; @@ -26,10 +28,12 @@ static class BenchmarkTest double ratio; } + // Uses lambda to create a fully random ByteBuffer, the supplier is an interface that can return a result + // There are three of these, each returns a differently filled ByteBuffer that is used in the test public static Supplier fullRandomByteBufferGenerator(Random random, int elements) { return () -> - { + { // For all three of these the ByteBuffer is already passed in and just needs its space allocated ByteBuffer hybrid = ByteBuffer.allocateDirect(elements * 4); for (int i = 0; i < elements; i++) @@ -41,6 +45,7 @@ public static Supplier fullRandomByteBufferGenerator(Random random, }; } + // This ByteBuffer is half random and half repetitive, useful because lz4 compression does better with repetitive data public static Supplier hybridRandomByteBufferGenerator(Random random, int elements) { return () -> @@ -49,6 +54,7 @@ public static Supplier hybridRandomByteBufferGenerator(Random random for (int i = 0; i < elements; i++) { + // For 10 indexes, the value 12 will be uses and the next 10 will be completely random if (i % 20 < 10) { hybrid.putInt(12); @@ -63,6 +69,7 @@ public static Supplier hybridRandomByteBufferGenerator(Random random }; } + // This ByteBuffer will be filled entirely with 10's. Useful to see how the compression ratio is affected public static Supplier repeatRandomByteBufferGenerator(int elements) { return () -> @@ -100,7 +107,7 @@ public int maxCompressedLength(int rawDataLength) } @Override - public int minCompressedLength(int rawDataLength) + public int minDecompressedLength(int rawDataLength) { return ELEMENTS * 4; } @@ -127,21 +134,36 @@ public int maxCompressedLength(int rawDataLength) return impl.maxCompressedLength(rawDataLength); } - public int minCompressedLength(int rawDataLength) + public int minDecompressedLength(int rawDataLength) { return impl.minimumDecompressedLength(rawDataLength); } }; - CompressionAlgorithm lz4ByteDeco = new CompressionAlgorithm() + CompressionAlgorithm lz4ByteDecoCompression = new CompressionAlgorithm() { + // Creates the variables needed to compress and decompress, they are only created when the algorithm is used final LZ4BytedecoCompressionImplementation impl = new LZ4BytedecoCompressionImplementation(); + final MutableBytePointer inPointer = new MutableBytePointer(); + final MutableBytePointer outPointer = new MutableBytePointer(); + final SizeTPointer inSize = new SizeTPointer(1); + final SizeTPointer outSize = new SizeTPointer(1); + + // decompressionContext is used to check for any errors with the LZ4 decompression + LZ4FDecompressionContext decompressionContext; @Override public double compress(ByteBuffer in, ByteBuffer out) { - Pointer inPointer = new Pointer(in); - Pointer outPointer = new Pointer(out); + // Sets the address of the pointer to the address of the ByteBuffer, does the same thing with limit and capacity + inPointer.setAddress(getDirectBufferAddress(in)); + outPointer.setAddress(getDirectBufferAddress(out)); + + inPointer.setLimit(in.limit()); + outPointer.setLimit(out.limit()); + + inPointer.setCapacity(in.capacity()); + outPointer.setCapacity(out.capacity()); return LZ4BytedecoCompressionImplementation.compress(in, inPointer, out, outPointer); } @@ -149,12 +171,20 @@ public double compress(ByteBuffer in, ByteBuffer out) @Override public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4BytedecoCompressionImplementation.LZ4Exception { - LZ4FDecompressionContext decompressionContext; + // The decompress method requires decompressionContext to check for errors decompressionContext = LZ4BytedecoCompressionImplementation.ByteDecoLZ4CompressionImplementation(); - Pointer inPointer = new Pointer(in); - Pointer outPointer = new Pointer(out); - SizeTPointer inSize = new SizeTPointer(in.limit()); - SizeTPointer outSize = new SizeTPointer(out.remaining()); + + inSize.put(in.limit()); + outSize.put(out.remaining()); + + inPointer.setAddress(getDirectBufferAddress(in)); + outPointer.setAddress(getDirectBufferAddress(out)); + + inPointer.setLimit(in.limit()); + outPointer.setLimit(out.limit()); + + inPointer.setCapacity(in.capacity()); + outPointer.setCapacity(out.capacity()); LZ4BytedecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); lz4.LZ4F_freeDecompressionContext(decompressionContext); @@ -167,16 +197,18 @@ public int maxCompressedLength(int rawDataLength) } @Override - public int minCompressedLength(int rawDataLength) + public int minDecompressedLength(int rawDataLength) { return impl.minimumDecompressedLength(rawDataLength); } }; + // There are several odd bugs with using the compression algorithms, this test was used because often times the compression + // would work for one usage but not the next one. This confirms that the test works over and over again without failing @Test public void testOverAndOver() throws LZ4BytedecoCompressionImplementation.LZ4Exception, IOException { - for (int i = 0; i < 10; i++) + for (int i = 0; i < 100; i++) { benchmarkTestCompressionAlgorithm(); } @@ -185,7 +217,7 @@ public void testOverAndOver() throws LZ4BytedecoCompressionImplementation.LZ4Exc @Test public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4BytedecoCompressionImplementation.LZ4Exception { - // Snappy Compression + // Snappy Compression for fullRandom, hybridRandom, and repetitive BenchmarkTest snappyFullRandom = benchmarkTestCompressionAlgorithm(true, snappyCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); System.out.println("Snappy Random: " + snappyFullRandom.ratio * 100 + " time: " + snappyFullRandom.totalTime); @@ -195,7 +227,7 @@ public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4BytedecoC BenchmarkTest snappyRepeat = benchmarkTestCompressionAlgorithm(false, snappyCompression, repeatRandomByteBufferGenerator(ELEMENTS)); System.out.println("Snappy Repeat: " + snappyRepeat.ratio * 100 + " time: " + snappyRepeat.totalTime); - // LZ4 1.8 Compression + // LZ4 1.8 Compression for fullRandom, hybridRandom, and repetitive BenchmarkTest lz4FullRandom = benchmarkTestCompressionAlgorithm(true, lz4Compression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); System.out.println("LZ4 1.8 random: " + lz4FullRandom.ratio * 100 + " time: " + lz4FullRandom.totalTime); @@ -205,14 +237,14 @@ public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4BytedecoC BenchmarkTest lz4Repeat = benchmarkTestCompressionAlgorithm(false, lz4Compression, repeatRandomByteBufferGenerator(ELEMENTS)); System.out.println("LZ4 1.8 repeat: " + lz4Repeat.ratio * 100 + " time: " + lz4Repeat.totalTime); - // LZ4 1.9 Compression - BenchmarkTest lz4ByteDecoFullRandom = benchmarkTestCompressionAlgorithm(true, lz4ByteDeco, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + // LZ4 1.9 Compression for fullRandom, hybridRandom, and repetitive + BenchmarkTest lz4ByteDecoFullRandom = benchmarkTestCompressionAlgorithm(true, lz4ByteDecoCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); System.out.println("lz4 1.9 ByteDeco Random: " + lz4ByteDecoFullRandom.ratio * 100 + " time: " + lz4ByteDecoFullRandom.totalTime); - BenchmarkTest lz4ByteDecoHybridRandom = benchmarkTestCompressionAlgorithm(false, lz4ByteDeco, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + BenchmarkTest lz4ByteDecoHybridRandom = benchmarkTestCompressionAlgorithm(false, lz4ByteDecoCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); System.out.println("lz4 1.9 ByteDeco Hybrid: " + lz4ByteDecoHybridRandom.ratio * 100 + " time: " + lz4ByteDecoHybridRandom.totalTime); - BenchmarkTest lz4ByteDecoRepeat = benchmarkTestCompressionAlgorithm(false, lz4ByteDeco, repeatRandomByteBufferGenerator(ELEMENTS)); + BenchmarkTest lz4ByteDecoRepeat = benchmarkTestCompressionAlgorithm(false, lz4ByteDecoCompression, repeatRandomByteBufferGenerator(ELEMENTS)); System.out.println("lz4 1.9 ByteDeco Repeat: " + lz4ByteDecoRepeat.ratio * 100 + " time: " + lz4ByteDecoRepeat.totalTime); } @@ -226,21 +258,25 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi BenchmarkTest results = new BenchmarkTest(); int bytesCompressed; + // Each ByteBuffer is filled with a random generator that is passed in during the method call ByteBuffer buffer = randomGenerator.get(); ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); - ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minCompressedLength(bufferOut.capacity())); + ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minDecompressedLength(bufferOut.capacity())); - // Warmup for algorithm methods, helps to optimize the JIT compiler, optimized at about 38 iterations + // Warmup for algorithm methods, helps to optimize the JIT compiler and is only called if warmup is set as true in the parameters if (warmup) { - for (int i = 0; i < 5000; i++) + for (int i = 0; i < 25000; i++) { + // Resets buffers for each start of the loop because the positions change during compression and decompression buffer.flip(); bufferOut.clear(); bufferDecompress.clear(); + // Compresses data into bufferOut and returns the number of bytes that were compressed bytesCompressed = (int) algorithm.compress(buffer, bufferOut); + // LZ4 1.9 uses pointers to implement so the positions of the buffers don't actually change, this ensures that the positions get updates if (bufferOut.position() == 0) { bufferOut.position(bytesCompressed); @@ -248,9 +284,11 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi bufferOut.flip(); + // Decompress the compressed data into bufferDecompress algorithm.decompress(bufferOut, bufferDecompress); - if ( buffer.position() == 0) + // LZ4 1.9 uses pointers to implement so the positions of the buffers don't actually change, this ensures that the positions get updates + if (buffer.position() == 0) { buffer.position(buffer.limit()); } @@ -260,19 +298,20 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi bufferDecompress.position(bufferDecompress.limit()); } + // Tests to see if the initial data and the decompressed data are the same, this makes sure the test actually works assertEquals(buffer, bufferDecompress); for (int j = 0; j < ELEMENTS; j++) { assertEquals(buffer.get(j), bufferDecompress.get(j)); } - } } int iterations = 800; // Run benchmark on algorithm that takes an average for the ratio and time computed + // This loop is the same as the warmup loop but keeps track of time for the benchmark for (int i = 0; i < iterations; i++) { buffer.flip(); @@ -313,6 +352,7 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi } } + // After the benchmark has finished, the times and ratio's get divided by the number of time the for loop ran, this gives results for a single use results.ratio /= iterations; results.compressTime /= iterations; results.decompressTime /= iterations; @@ -321,15 +361,19 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi return results; } - + // This interface is used to define each of our compression algorithms private interface CompressionAlgorithm { + // Takes in two ByteBuffers, the first is full of data that will be compressed into the second buffer double compress(ByteBuffer in, ByteBuffer out) throws IOException; + // Takes in two ByteBuffers, the first contains compressed data, which will be decompressed into the second buffer void decompress(ByteBuffer in, ByteBuffer out) throws IOException, LZ4BytedecoCompressionImplementation.LZ4Exception; + // Returns the max amount of space the given data can be compressed into int maxCompressedLength(int rawDataLength); - int minCompressedLength(int rawDataLength); + // Returns the minimum amount of space the data can be decompressed out to + int minDecompressedLength(int rawDataLength); } } diff --git a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java index 83f5d287..d0a3dbcd 100644 --- a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java +++ b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java @@ -8,19 +8,23 @@ public class LZ4BytedecoCompressionImplementation { + final static LZ4FDecompressionContext dctx = new LZ4FDecompressionContext(); + + // Method checks to see if anything goes wrong with the decomopression of the data public static LZ4FDecompressionContext ByteDecoLZ4CompressionImplementation() throws LZ4Exception { - final LZ4FDecompressionContext dctx = new LZ4FDecompressionContext(); final long ctxError = lz4.LZ4F_createDecompressionContext(dctx, lz4.LZ4F_VERSION); checkForError(ctxError); return dctx; } + // Compresses the data using pointers that point to the addresses of the ByteBuffers public static double compress(ByteBuffer src, Pointer srcPointer, ByteBuffer dst, Pointer dstPointer) { return lz4.LZ4F_compressFrame(dstPointer, dst.limit(), srcPointer, src.limit(), null); } + // Decompresses the data using several pointers that point to the ByteBuffers public static void decompress(LZ4FDecompressionContext dctx, Pointer compressedPointer, Pointer dstPointer, @@ -37,18 +41,21 @@ public static void decompress(LZ4FDecompressionContext dctx, lz4.LZ4F_decompress(dctx, dstPointer, dstSize, compressedPointer, srcSize, null); } + // Computes the amount of space required for the compress Buffer based on the size of the input data public int maxCompressedLength(int uncompressedLength) { final int maxCompressedSize = (int) lz4.LZ4F_compressFrameBound(uncompressedLength, null); return lz4.LZ4_compressBound(maxCompressedSize); } + // Computes the amount of space required for the decompress Buffer bases on the size of the compressed data public int minimumDecompressedLength(int compressedLength) { double y = (long) (compressedLength - 16) * 255; return (int) Math.round(y / 256); } + // Checks for errors with the decompress method and throw's an exception from LZ4Exception if one is found private static void checkForError(long errorCode) throws LZ4Exception { if (lz4.LZ4F_isError(errorCode) != 0) { throw new LZ4Exception(lz4.LZ4F_getErrorName(errorCode).getString()); From 5b702c9d6dfae0a8de33fcfde80fd3f7d1378544 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Mon, 7 Nov 2022 16:33:50 -0600 Subject: [PATCH 15/22] Update to fix merge conflict with develop --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index d6ea80a5..1fb4dc25 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -56,7 +56,7 @@ mainDependencies { } testDependencies { - api("us.ihmc:ihmc-commons-testing:0.31.0") + api("us.ihmc:ihmc-commons-testing:0.32.0") api("org.bytedeco:lz4-platform:1.9.4-1.5.8") api("org.bytedeco:lz4:1.9.4-1.5.8") } From ae16b70d02449f2ffede6777c3ec4c36dc8dbea3 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Mon, 7 Nov 2022 16:46:53 -0600 Subject: [PATCH 16/22] ByteDeco -> Bytedeco --- .../CompressionAlgorithmBenchmarkTest.java | 18 ++++++++++-------- .../LZ4BytedecoCompressionImplementation.java | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 42c15968..8212c6f9 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -140,7 +140,7 @@ public int minDecompressedLength(int rawDataLength) } }; - CompressionAlgorithm lz4ByteDecoCompression = new CompressionAlgorithm() + CompressionAlgorithm lz4BytedecoCompression = new CompressionAlgorithm() { // Creates the variables needed to compress and decompress, they are only created when the algorithm is used final LZ4BytedecoCompressionImplementation impl = new LZ4BytedecoCompressionImplementation(); @@ -172,7 +172,7 @@ public double compress(ByteBuffer in, ByteBuffer out) public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4BytedecoCompressionImplementation.LZ4Exception { // The decompress method requires decompressionContext to check for errors - decompressionContext = LZ4BytedecoCompressionImplementation.ByteDecoLZ4CompressionImplementation(); + decompressionContext = LZ4BytedecoCompressionImplementation.BytedecoLZ4CompressionImplementation(); inSize.put(in.limit()); outSize.put(out.remaining()); @@ -238,14 +238,16 @@ public void benchmarkTestCompressionAlgorithm() throws IOException, LZ4BytedecoC System.out.println("LZ4 1.8 repeat: " + lz4Repeat.ratio * 100 + " time: " + lz4Repeat.totalTime); // LZ4 1.9 Compression for fullRandom, hybridRandom, and repetitive - BenchmarkTest lz4ByteDecoFullRandom = benchmarkTestCompressionAlgorithm(true, lz4ByteDecoCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - System.out.println("lz4 1.9 ByteDeco Random: " + lz4ByteDecoFullRandom.ratio * 100 + " time: " + lz4ByteDecoFullRandom.totalTime); + BenchmarkTest lz4BytedecoFullRandom = benchmarkTestCompressionAlgorithm(true, + lz4BytedecoCompression, fullRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + System.out.println("lz4 1.9 Bytedeco Random: " + lz4BytedecoFullRandom.ratio * 100 + " time: " + lz4BytedecoFullRandom.totalTime); - BenchmarkTest lz4ByteDecoHybridRandom = benchmarkTestCompressionAlgorithm(false, lz4ByteDecoCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); - System.out.println("lz4 1.9 ByteDeco Hybrid: " + lz4ByteDecoHybridRandom.ratio * 100 + " time: " + lz4ByteDecoHybridRandom.totalTime); + BenchmarkTest lz4BytedecoHybridRandom = benchmarkTestCompressionAlgorithm(false, + lz4BytedecoCompression, hybridRandomByteBufferGenerator(new Random(1234), ELEMENTS)); + System.out.println("lz4 1.9 Bytedeco Hybrid: " + lz4BytedecoHybridRandom.ratio * 100 + " time: " + lz4BytedecoHybridRandom.totalTime); - BenchmarkTest lz4ByteDecoRepeat = benchmarkTestCompressionAlgorithm(false, lz4ByteDecoCompression, repeatRandomByteBufferGenerator(ELEMENTS)); - System.out.println("lz4 1.9 ByteDeco Repeat: " + lz4ByteDecoRepeat.ratio * 100 + " time: " + lz4ByteDecoRepeat.totalTime); + BenchmarkTest lz4BytedecoRepeat = benchmarkTestCompressionAlgorithm(false, lz4BytedecoCompression, repeatRandomByteBufferGenerator(ELEMENTS)); + System.out.println("lz4 1.9 Bytedeco Repeat: " + lz4BytedecoRepeat.ratio * 100 + " time: " + lz4BytedecoRepeat.totalTime); } public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, CompressionAlgorithm algorithm, Supplier randomGenerator) diff --git a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java index d0a3dbcd..359a31bc 100644 --- a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java +++ b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java @@ -11,7 +11,7 @@ public class LZ4BytedecoCompressionImplementation final static LZ4FDecompressionContext dctx = new LZ4FDecompressionContext(); // Method checks to see if anything goes wrong with the decomopression of the data - public static LZ4FDecompressionContext ByteDecoLZ4CompressionImplementation() throws LZ4Exception + public static LZ4FDecompressionContext BytedecoLZ4CompressionImplementation() throws LZ4Exception { final long ctxError = lz4.LZ4F_createDecompressionContext(dctx, lz4.LZ4F_VERSION); checkForError(ctxError); From 66299fd595a105b0bbb7542a55f6d7ecd29fef39 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Tue, 8 Nov 2022 18:00:12 -0600 Subject: [PATCH 17/22] Created wrapper method to set a MutableBytePointer to a given ByteBuffer's address, limit, and capacity. Then used this method in the benchmark to clean up the implementation. --- .../CompressionAlgorithmBenchmarkTest.java | 23 +++++-------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 8212c6f9..07c367f2 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -155,15 +155,9 @@ public int minDecompressedLength(int rawDataLength) @Override public double compress(ByteBuffer in, ByteBuffer out) { - // Sets the address of the pointer to the address of the ByteBuffer, does the same thing with limit and capacity - inPointer.setAddress(getDirectBufferAddress(in)); - outPointer.setAddress(getDirectBufferAddress(out)); - - inPointer.setLimit(in.limit()); - outPointer.setLimit(out.limit()); - - inPointer.setCapacity(in.capacity()); - outPointer.setCapacity(out.capacity()); + //The wrapMutableBytePointer sets the address, limit, and capacity of a MutableBytePointer to the ByteBuffer that's passed in + inPointer.wrapMutableBytePointer(in); + outPointer.wrapMutableBytePointer(out); return LZ4BytedecoCompressionImplementation.compress(in, inPointer, out, outPointer); } @@ -177,14 +171,9 @@ public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4BytedecoCompress inSize.put(in.limit()); outSize.put(out.remaining()); - inPointer.setAddress(getDirectBufferAddress(in)); - outPointer.setAddress(getDirectBufferAddress(out)); - - inPointer.setLimit(in.limit()); - outPointer.setLimit(out.limit()); - - inPointer.setCapacity(in.capacity()); - outPointer.setCapacity(out.capacity()); + //The wrapMutableBytePointer sets the address, limit, and capacity of a MutableBytePointer to the ByteBuffer that's passed in + inPointer.wrapMutableBytePointer(in); + outPointer.wrapMutableBytePointer(out); LZ4BytedecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); lz4.LZ4F_freeDecompressionContext(decompressionContext); From f0f799e94dc2fda4fc90e93cdd9191abcaa7d089 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Wed, 9 Nov 2022 09:31:34 -0600 Subject: [PATCH 18/22] Moved the buffer fill call into the for loop to make use of the supplier so each time the for loop is called new data gets filled, rather than testing the same data over and over again. --- .../CompressionAlgorithmBenchmarkTest.java | 46 ++++++------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 07c367f2..3288cdb0 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -249,20 +249,19 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi BenchmarkTest results = new BenchmarkTest(); int bytesCompressed; - // Each ByteBuffer is filled with a random generator that is passed in during the method call - ByteBuffer buffer = randomGenerator.get(); - ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); - ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minDecompressedLength(bufferOut.capacity())); - // Warmup for algorithm methods, helps to optimize the JIT compiler and is only called if warmup is set as true in the parameters if (warmup) { for (int i = 0; i < 25000; i++) { - // Resets buffers for each start of the loop because the positions change during compression and decompression + // Each ByteBuffer is filled with a random generator that is passed in during the method call + // This is in the for loop so the supplier can get new values at each iteration of the for loop using get and a set seed + ByteBuffer buffer = randomGenerator.get(); + ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); + ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minDecompressedLength(bufferOut.capacity())); + + // When using the supplier the position of this buffer gets moved and needs to be reset before compress is called buffer.flip(); - bufferOut.clear(); - bufferDecompress.clear(); // Compresses data into bufferOut and returns the number of bytes that were compressed bytesCompressed = (int) algorithm.compress(buffer, bufferOut); @@ -278,20 +277,7 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi // Decompress the compressed data into bufferDecompress algorithm.decompress(bufferOut, bufferDecompress); - // LZ4 1.9 uses pointers to implement so the positions of the buffers don't actually change, this ensures that the positions get updates - if (buffer.position() == 0) - { - buffer.position(buffer.limit()); - } - - if (bufferDecompress.position() == 0) - { - bufferDecompress.position(bufferDecompress.limit()); - } - // Tests to see if the initial data and the decompressed data are the same, this makes sure the test actually works - assertEquals(buffer, bufferDecompress); - for (int j = 0; j < ELEMENTS; j++) { assertEquals(buffer.get(j), bufferDecompress.get(j)); @@ -305,9 +291,13 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi // This loop is the same as the warmup loop but keeps track of time for the benchmark for (int i = 0; i < iterations; i++) { + // Each ByteBuffer is filled with a random generator that is passed in during the method call + // This is in the for loop so the supplier can get new values at each iteration of the for loop using get and a set seed + ByteBuffer buffer = randomGenerator.get(); + ByteBuffer bufferOut = ByteBuffer.allocateDirect(algorithm.maxCompressedLength(buffer.capacity())); + ByteBuffer bufferDecompress = ByteBuffer.allocateDirect(algorithm.minDecompressedLength(bufferOut.capacity())); + buffer.flip(); - bufferOut.clear(); - bufferDecompress.clear(); stopwatchTotal.start(); stopwatchCompress.start(); @@ -331,16 +321,6 @@ public BenchmarkTest benchmarkTestCompressionAlgorithm(boolean warmup, Compressi results.decompressTime += stopwatchDecompress.totalElapsed(); results.totalTime += stopwatchTotal.totalElapsed(); - - if (buffer.position() == 0) - { - buffer.position(buffer.limit()); - } - - if (bufferDecompress.position() == 0) - { - bufferDecompress.position(bufferDecompress.limit()); - } } // After the benchmark has finished, the times and ratio's get divided by the number of time the for loop ran, this gives results for a single use From d1d05b70c8f6ca301698041812c3f00f16cf7e2e Mon Sep 17 00:00:00 2001 From: nkitchel Date: Wed, 9 Nov 2022 10:00:35 -0600 Subject: [PATCH 19/22] Reverted version --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 1fb4dc25..d6b0f46b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -43,7 +43,7 @@ mainDependencies { api("us.ihmc:ihmc-java-decklink-capture:0.4.0") api("us.ihmc:ihmc-pub-sub:0.18.1") api("us.ihmc:ihmc-pub-sub-serializers-extra:0.18.1") - api("us.ihmc:ihmc-commons:0.31.0") + api("us.ihmc:ihmc-commons:0.32.0") api("us.ihmc:ihmc-graphics-description:0.19.8") api("us.ihmc:mecano:17-0.11.5") api("com.hierynomus:sshj:0.31.0") From 3fd2f73678c0b02bdce8f9637f01dacf2613d007 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Wed, 9 Nov 2022 16:12:35 -0600 Subject: [PATCH 20/22] Updated MutableBytePointer to use new wrapper method --- .../CompressionAlgorithmBenchmarkTest.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java index 3288cdb0..1c4c252d 100644 --- a/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java +++ b/src/test/java/us/ihmc/tools/compression/CompressionAlgorithmBenchmarkTest.java @@ -1,6 +1,5 @@ package us.ihmc.tools.compression; -import org.bytedeco.javacpp.Pointer; import org.bytedeco.javacpp.SizeTPointer; import org.bytedeco.lz4.LZ4FDecompressionContext; import org.bytedeco.lz4.global.lz4; @@ -15,7 +14,7 @@ import static org.junit.jupiter.api.Assertions.*; -public class CompressionAlgorithmBenchmarkTest extends Pointer +public class CompressionAlgorithmBenchmarkTest { private final int ELEMENTS = 1024; @@ -156,8 +155,8 @@ public int minDecompressedLength(int rawDataLength) public double compress(ByteBuffer in, ByteBuffer out) { //The wrapMutableBytePointer sets the address, limit, and capacity of a MutableBytePointer to the ByteBuffer that's passed in - inPointer.wrapMutableBytePointer(in); - outPointer.wrapMutableBytePointer(out); + inPointer.wrapByteBuffer(in); + outPointer.wrapByteBuffer(out); return LZ4BytedecoCompressionImplementation.compress(in, inPointer, out, outPointer); } @@ -172,8 +171,8 @@ public void decompress(ByteBuffer in, ByteBuffer out) throws LZ4BytedecoCompress outSize.put(out.remaining()); //The wrapMutableBytePointer sets the address, limit, and capacity of a MutableBytePointer to the ByteBuffer that's passed in - inPointer.wrapMutableBytePointer(in); - outPointer.wrapMutableBytePointer(out); + inPointer.wrapByteBuffer(in); + outPointer.wrapByteBuffer(out); LZ4BytedecoCompressionImplementation.decompress(decompressionContext, inPointer, outPointer, inSize, outSize, out, ELEMENTS); lz4.LZ4F_freeDecompressionContext(decompressionContext); From 3e48f1fbf66512cd8a5f3d648a420bafb9d1a805 Mon Sep 17 00:00:00 2001 From: nkitchel Date: Thu, 10 Nov 2022 09:57:22 -0600 Subject: [PATCH 21/22] Changed to full name variables --- .../LZ4BytedecoCompressionImplementation.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java index 359a31bc..39fe307b 100644 --- a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java +++ b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java @@ -8,28 +8,28 @@ public class LZ4BytedecoCompressionImplementation { - final static LZ4FDecompressionContext dctx = new LZ4FDecompressionContext(); + final static LZ4FDecompressionContext lz4DecompressionContext = new LZ4FDecompressionContext(); - // Method checks to see if anything goes wrong with the decomopression of the data + // Method checks to see if anything goes wrong with the decompression of the data public static LZ4FDecompressionContext BytedecoLZ4CompressionImplementation() throws LZ4Exception { - final long ctxError = lz4.LZ4F_createDecompressionContext(dctx, lz4.LZ4F_VERSION); - checkForError(ctxError); - return dctx; + final long contextError = lz4.LZ4F_createDecompressionContext(lz4DecompressionContext, lz4.LZ4F_VERSION); + checkForError(contextError); + return lz4DecompressionContext; } // Compresses the data using pointers that point to the addresses of the ByteBuffers - public static double compress(ByteBuffer src, Pointer srcPointer, ByteBuffer dst, Pointer dstPointer) + public static double compress(ByteBuffer source, Pointer sourcePointer, ByteBuffer destination, Pointer destionationPointer) { - return lz4.LZ4F_compressFrame(dstPointer, dst.limit(), srcPointer, src.limit(), null); + return lz4.LZ4F_compressFrame(destionationPointer, destination.limit(), sourcePointer, source.limit(), null); } // Decompresses the data using several pointers that point to the ByteBuffers - public static void decompress(LZ4FDecompressionContext dctx, + public static void decompress(LZ4FDecompressionContext decompressionContext, Pointer compressedPointer, - Pointer dstPointer, - SizeTPointer srcSize, - SizeTPointer dstSize, + Pointer destinationPointer, + SizeTPointer compressSize, + SizeTPointer destinationSize, ByteBuffer uncompressed, int uncompressedSize) { @@ -38,7 +38,7 @@ public static void decompress(LZ4FDecompressionContext dctx, throw new BufferOverflowException(); } - lz4.LZ4F_decompress(dctx, dstPointer, dstSize, compressedPointer, srcSize, null); + lz4.LZ4F_decompress(decompressionContext, destinationPointer, destinationSize, compressedPointer, compressSize, null); } // Computes the amount of space required for the compress Buffer based on the size of the input data From 787fee53057a0b6e5ed72fece2b50ee94ebdfcfd Mon Sep 17 00:00:00 2001 From: nkitchel Date: Thu, 10 Nov 2022 10:47:07 -0600 Subject: [PATCH 22/22] Fixed formatting mistakes, added javadoc comments for readability --- .../LZ4BytedecoCompressionImplementation.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java index 39fe307b..6d315773 100644 --- a/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java +++ b/src/test/java/us/ihmc/tools/compression/LZ4BytedecoCompressionImplementation.java @@ -10,7 +10,7 @@ public class LZ4BytedecoCompressionImplementation { final static LZ4FDecompressionContext lz4DecompressionContext = new LZ4FDecompressionContext(); - // Method checks to see if anything goes wrong with the decompression of the data + /** Method checks to see if anything goes wrong with the decompression of the data */ public static LZ4FDecompressionContext BytedecoLZ4CompressionImplementation() throws LZ4Exception { final long contextError = lz4.LZ4F_createDecompressionContext(lz4DecompressionContext, lz4.LZ4F_VERSION); @@ -18,13 +18,13 @@ public static LZ4FDecompressionContext BytedecoLZ4CompressionImplementation() th return lz4DecompressionContext; } - // Compresses the data using pointers that point to the addresses of the ByteBuffers + /** Compresses the data using pointers that point to the addresses of the ByteBuffers */ public static double compress(ByteBuffer source, Pointer sourcePointer, ByteBuffer destination, Pointer destionationPointer) { return lz4.LZ4F_compressFrame(destionationPointer, destination.limit(), sourcePointer, source.limit(), null); } - // Decompresses the data using several pointers that point to the ByteBuffers + /** Decompresses the data using several pointers that point to the ByteBuffers */ public static void decompress(LZ4FDecompressionContext decompressionContext, Pointer compressedPointer, Pointer destinationPointer, @@ -41,28 +41,30 @@ public static void decompress(LZ4FDecompressionContext decompressionContext, lz4.LZ4F_decompress(decompressionContext, destinationPointer, destinationSize, compressedPointer, compressSize, null); } - // Computes the amount of space required for the compress Buffer based on the size of the input data + /** Computes the amount of space required for the compress Buffer based on the size of the input data */ public int maxCompressedLength(int uncompressedLength) { final int maxCompressedSize = (int) lz4.LZ4F_compressFrameBound(uncompressedLength, null); return lz4.LZ4_compressBound(maxCompressedSize); } - // Computes the amount of space required for the decompress Buffer bases on the size of the compressed data + /** Computes the amount of space required for the decompress Buffer bases on the size of the compressed data */ public int minimumDecompressedLength(int compressedLength) { double y = (long) (compressedLength - 16) * 255; return (int) Math.round(y / 256); } - // Checks for errors with the decompress method and throw's an exception from LZ4Exception if one is found - private static void checkForError(long errorCode) throws LZ4Exception { + /** Checks for errors with the decompress method and throw's an exception from LZ4Exception if one is found */ + private static void checkForError(long errorCode) throws LZ4Exception + { if (lz4.LZ4F_isError(errorCode) != 0) { throw new LZ4Exception(lz4.LZ4F_getErrorName(errorCode).getString()); } } - public static final class LZ4Exception extends Exception { + public static final class LZ4Exception extends Exception + { public LZ4Exception(final String message) { super(message); }