diff --git a/README.adoc b/README.adoc index 72da903..5510a37 100644 --- a/README.adoc +++ b/README.adoc @@ -37,18 +37,20 @@ You can find latest version from the maven logo link above. === Compressing To compress in the older 1.x format, use class ``Compressor``. For 2.x, use ``GorillaCompressor`` (recommended). -``ByteBufferLongOutput`` is also recommended compared to ``ByteBufferBitOutput`` because of performance. +``LongArrayOutput`` is also recommended compared to ``ByteBufferBitOutput`` because of performance. One can supply +alternative predictor to the ``GorillaCompressor`` if required. One such implementation is included, +``DifferentialFCM`` that provides better compression ratio for some data patterns. [source, java] ---- long now = LocalDateTime.now(ZoneOffset.UTC).truncatedTo(ChronoUnit.HOURS) .toInstant(ZoneOffset.UTC).toEpochMilli(); -ByteBufferLongOutput output = new ByteBufferLongOutput(); +LongArrayOutput output = new LongArrayOutput(); GorillaCompressor c = new GorillaCompressor(now, output); ---- -Compression class requires a block timestamp and an implementation of `BitOutput` interface. `ByteBufferLongOutput` is an in-memory example that uses off-heap storage. +Compression class requires a block timestamp and an implementation of `BitOutput` interface. [source, java] ---- @@ -70,17 +72,18 @@ which flushes the remaining data to the stream and writes closing information. === Decompressing To decompress from the older 1.x format, use class ``Decompressor``. For 2.x, use ``GorillaDecompressor`` (recommended). -``ByteBufferLongInput`` is also recommended compared to ``ByteBufferBitInput`` because of performance if the 2.x -format was used to compress the time series. +``LongArrayInput`` is also recommended compared to ``ByteBufferBitInput`` because of performance if the 2.x +format was used to compress the time series. If the original compressor used different predictor than +``LastValuePredictor`` it must be defined in the constructor. [source, java] ---- -ByteBufferLongInput input = new ByteBufferLongInput(byteBuffer); +LongArrayInput input = new LongArrayInput(byteBuffer); GorillaDecompressor d = new GorillaDecompressor(input); ---- To decompress a stream of bytes, supply `GorillaDecompressor` with a suitable implementation of `BitInput` interface. - The ByteBufferLongInput allows to decompress a long array or existing `ByteBuffer` presentation with 8 byte word + The LongArrayInput allows to decompress a long array or existing `ByteBuffer` presentation with 8 byte word length. [source, java] @@ -125,12 +128,10 @@ not from the small changes to the output format. There were few things I wanted to get to 2.0.0, but had to decide against due to lack of time. I will implement these later with potentially some breaking API changes: - * Support timestamp only and value only compressions (2.1.x) - * Support for Java 8 streams in decompression and compression API (2.1.x) - * Change delta-of-delta ranges based on some real-world results when using millisecond resolution (3.0.x) + * Support timestamp only compressions (2.2.x) + * Include ByteBufferLongOutput/ByteBufferLongInput in the package (2.2.x) * Move bit operations to inside the GorillaCompressor/GorillaDecompressor to allow easier usage with - Netty's ByteBuf and allocator (or Agrona etc). (3.0.x) - ** Example for Netty: https://gist.github.com/burmanm/d6ad33089aaf78104d5f0f7977f097ce + other allocators (2.2.x) == Internals @@ -157,7 +158,7 @@ File an issue and/or send a pull request. === License .... - Copyright 2016-2017 Michael Burman and/or other contributors. + Copyright 2016-2018 Michael Burman and/or other contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/main/java/fi/iki/yak/ts/compression/gorilla/GorillaDecompressor.java b/src/main/java/fi/iki/yak/ts/compression/gorilla/GorillaDecompressor.java index 58a9916..616e86e 100644 --- a/src/main/java/fi/iki/yak/ts/compression/gorilla/GorillaDecompressor.java +++ b/src/main/java/fi/iki/yak/ts/compression/gorilla/GorillaDecompressor.java @@ -1,5 +1,7 @@ package fi.iki.yak.ts.compression.gorilla; +import java.util.stream.Stream; + import fi.iki.yak.ts.compression.gorilla.predictors.LastValuePredictor; /** @@ -15,8 +17,8 @@ public class GorillaDecompressor { private long storedVal = 0; private boolean endOfStream = false; - private BitInput in; - private ValueDecompressor decompressor; + private final BitInput in; + private final ValueDecompressor decompressor; public GorillaDecompressor(BitInput input) { this(input, new LastValuePredictor()); @@ -125,5 +127,4 @@ public static int decodeZigZag32(final int n) { } // END: From protobuf - } \ No newline at end of file diff --git a/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayInput.java b/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayInput.java index 2fe9935..7f7ea36 100644 --- a/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayInput.java +++ b/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayInput.java @@ -22,7 +22,7 @@ * @author Michael Burman */ public class LongArrayInput implements BitInput { - private long[] longArray; // TODO Investigate also the ByteBuffer performance here.. or Unsafe + private final long[] longArray; // TODO Investigate also the ByteBuffer performance here.. or Unsafe private long lB; private int position = 0; private int bitsLeft = 0; diff --git a/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayOutput.java b/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayOutput.java index 5872d79..284c7ab 100644 --- a/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayOutput.java +++ b/src/main/java/fi/iki/yak/ts/compression/gorilla/LongArrayOutput.java @@ -6,7 +6,7 @@ * @author Michael Burman */ public class LongArrayOutput implements BitOutput { - public static final int DEFAULT_ALLOCATION = 4096*32; + public static final int DEFAULT_ALLOCATION = 256; private long[] longArray; private int position = 0; @@ -129,22 +129,17 @@ public void writeBits(long value, int bits) { } /** - * Causes the currently handled byte to be written to the stream + * Causes the currently handled word to be written to the stream */ @Override public void flush() { - flipWord(); // Causes write to the ByteBuffer - } - - public void reset() { - position = 0; - bitsLeft = Long.SIZE; - lB = 0; + flipWord(); } + /** + * Changed in 2.1, returns a reference to the underlying array (no copy anymore) + */ public long[] getLongArray() { - long[] copy = new long[position+1]; - System.arraycopy(longArray, 0, copy, 0, position); - return copy; + return this.longArray; } } diff --git a/src/main/java/fi/iki/yak/ts/compression/gorilla/ValueDecompressor.java b/src/main/java/fi/iki/yak/ts/compression/gorilla/ValueDecompressor.java index 43a08ad..667eb72 100644 --- a/src/main/java/fi/iki/yak/ts/compression/gorilla/ValueDecompressor.java +++ b/src/main/java/fi/iki/yak/ts/compression/gorilla/ValueDecompressor.java @@ -8,8 +8,8 @@ * @author Michael Burman */ public class ValueDecompressor { - private BitInput in; - private Predictor predictor; + private final BitInput in; + private final Predictor predictor; private int storedLeadingZeros = Integer.MAX_VALUE; private int storedTrailingZeros = 0; diff --git a/src/main/java/fi/iki/yak/ts/compression/gorilla/predictors/DifferentialFCM.java b/src/main/java/fi/iki/yak/ts/compression/gorilla/predictors/DifferentialFCM.java index bac05f9..229187d 100644 --- a/src/main/java/fi/iki/yak/ts/compression/gorilla/predictors/DifferentialFCM.java +++ b/src/main/java/fi/iki/yak/ts/compression/gorilla/predictors/DifferentialFCM.java @@ -10,7 +10,7 @@ public class DifferentialFCM implements Predictor { private long lastValue = 0L; - private long[] table; + private final long[] table; private int lastHash = 0; private final int mask; @@ -29,7 +29,7 @@ public DifferentialFCM(int size) { this.table = new long[newSize]; this.mask = newSize - 1; } else { - throw new IllegalArgumentException("Size must be positive and a power of two"); + throw new IllegalArgumentException("Size must be positive"); } }