diff --git a/evcache-core/build.gradle b/evcache-core/build.gradle index 63874384..938137e3 100644 --- a/evcache-core/build.gradle +++ b/evcache-core/build.gradle @@ -43,6 +43,7 @@ dependencies { api group:"joda-time", name:"joda-time", version:"latest.release" api group:"javax.annotation", name:"javax.annotation-api", version:"latest.release" api group:"com.github.fzakaria", name:"ascii85", version:"latest.release" + api group:"com.github.luben", name:"zstd-jni", version:"1.5.7-11" testImplementation group:"org.testng", name:"testng", version:"7.5" testImplementation group:"com.beust", name:"jcommander", version:"1.72" diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java index 67f7a1fd..234a31fc 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheImpl.java @@ -164,7 +164,7 @@ public class EVCacheImpl implements EVCache, EVCacheImplMBean { this.maxHashLength = propertyRepository.get(appName + ".max.hash.length", Integer.class).orElse(-1); this.encoderBase = propertyRepository.get(appName + ".hash.encoder", String.class).orElse("base64"); this.autoHashKeys = propertyRepository.get(_appName + ".auto.hash.keys", Boolean.class).orElseGet("evcache.auto.hash.keys").orElse(false); - this.evcacheValueTranscoder = new EVCacheTranscoder(); + this.evcacheValueTranscoder = new EVCacheTranscoder(_appName, propertyRepository); evcacheValueTranscoder.setCompressionThreshold(Integer.MAX_VALUE); // default max key length is 200, instead of using what is defined in MemcachedClientIF.MAX_KEY_LENGTH (250). This is to accommodate diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java index 95c7a86e..e2cf7afc 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheSerializingTranscoder.java @@ -22,31 +22,36 @@ package com.netflix.evcache; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdInputStream; +import com.netflix.archaius.api.Property; import com.netflix.evcache.metrics.EVCacheMetricsFactory; -import com.netflix.evcache.pool.ServerGroup; import com.netflix.spectator.api.BasicTag; import com.netflix.spectator.api.Tag; -import com.netflix.spectator.api.Timer; import net.spy.memcached.CachedData; import net.spy.memcached.transcoders.BaseSerializingTranscoder; import net.spy.memcached.transcoders.Transcoder; import net.spy.memcached.transcoders.TranscoderUtils; -import net.spy.memcached.util.StringUtils; -import java.time.Duration; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Transcoder that serializes and compresses objects. */ public class EVCacheSerializingTranscoder extends BaseSerializingTranscoder implements Transcoder { + private static final Logger logger = LoggerFactory.getLogger(EVCacheSerializingTranscoder.class); // General flags static final int SERIALIZED = 1; @@ -63,10 +68,16 @@ public class EVCacheSerializingTranscoder extends BaseSerializingTranscoder impl static final int SPECIAL_DOUBLE = (7 << 8); static final int SPECIAL_BYTEARRAY = (8 << 8); - static final String COMPRESSION = "COMPRESSION_METRIC"; + public enum CompressionAlgorithm { GZIP, ZSTD } + + public static final int DEFAULT_ZSTD_COMPRESSION_LEVEL = 3; + + private static final int ZSTD_MAGIC = 0xFD2FB528; private final TranscoderUtils tu = new TranscoderUtils(true); - private Timer timer; + private Property compressionAlgorithmProperty; + private Property zstdLevelProperty; + protected final String appName; /** * Get a serializing transcoder with the default max data size. @@ -79,7 +90,23 @@ public EVCacheSerializingTranscoder() { * Get a serializing transcoder that specifies the max data size. */ public EVCacheSerializingTranscoder(int max) { + this(null, max); + } + + /** + * Get a serializing transcoder that specifies the owning app name and the max data size. + */ + public EVCacheSerializingTranscoder(String appName, int max) { super(max); + this.appName = appName; + } + + public void setCompressionAlgorithmProperty(Property algorithmProperty) { + this.compressionAlgorithmProperty = algorithmProperty; + } + + public void setCompressionLevelProperty(Property levelProperty) { + this.zstdLevelProperty = levelProperty; } @Override @@ -179,31 +206,106 @@ public CachedData encode(Object o) { } assert b != null; if (b.length > compressionThreshold) { + int originalLength = b.length; byte[] compressed = compress(b); - if (compressed.length < b.length) { + if (compressed.length < originalLength) { getLogger().trace("Compressed %s from %d to %d", - o.getClass().getName(), b.length, compressed.length); + o.getClass().getName(), originalLength, compressed.length); b = compressed; flags |= COMPRESSED; } else { getLogger().debug("Compression increased the size of %s from %d to %d", - o.getClass().getName(), b.length, compressed.length); + o.getClass().getName(), originalLength, compressed.length); } - - long compression_ratio = Math.round((double) compressed.length / b.length * 100); - updateTimerWithCompressionRatio(compression_ratio); } return new CachedData(flags, b, getMaxSize()); } - private void updateTimerWithCompressionRatio(long ratio_percentage) { - if(timer == null) { - final List tagList = new ArrayList(1); - tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, "gzip")); - timer = EVCacheMetricsFactory.getInstance().getPercentileTimer(EVCacheMetricsFactory.COMPRESSION_RATIO, tagList, Duration.ofMillis(100)); - }; + @Override + protected byte[] compress(byte[] in) { + if (in == null) throw new NullPointerException("Can't compress null"); + + CompressionAlgorithm compressionAlgorithm = compressionAlgorithmProperty == null ? CompressionAlgorithm.GZIP + : CompressionAlgorithm.valueOf(compressionAlgorithmProperty.orElse(CompressionAlgorithm.GZIP.name()).get().toUpperCase()); + + byte[] compressed; + switch (compressionAlgorithm) { + case ZSTD: + int zstdLevel = zstdLevelProperty == null ? DEFAULT_ZSTD_COMPRESSION_LEVEL + : zstdLevelProperty.orElse(DEFAULT_ZSTD_COMPRESSION_LEVEL).get(); + logger.debug("algorithm: {}, level: {}, appName: {}", compressionAlgorithm, zstdLevel, appName); + compressed = Zstd.compress(in, zstdLevel); + break; + case GZIP: + logger.debug("algorithm: {}, appName: {}", compressionAlgorithm, appName); + compressed = super.compress(in); + break; + default: + throw new IllegalArgumentException("Unsupported compression algorithm: " + compressionAlgorithm); + } + + if (compressed != null) { + long ratioPerCent = Math.round((double) compressed.length / in.length * 100.0); + recordCompressionRatio(ratioPerCent, compressionAlgorithm); + } + + return compressed; + } + + @Override + protected byte[] decompress(byte[] in) { + if (in == null || in.length == 0) return in; + if (isZstdCompressed(in)) return decompressZstd(in); + return super.decompress(in); + } + + private boolean isZstdCompressed(byte[] data) { + if (data == null || data.length < 4) return false; + int magic = ByteBuffer.wrap(data, 0, 4).order(ByteOrder.LITTLE_ENDIAN).getInt(); + return magic == ZSTD_MAGIC; + } + + private byte[] decompressZstd(byte[] in) { + long originalSize = Zstd.getFrameContentSize(in); + if (originalSize > Integer.MAX_VALUE) { + throw new RuntimeException("Zstd decompressed size exceeds int range: " + originalSize); + } + if (originalSize > 0) { + // Fast path: frame carries a content-size header (compress() above always does). + return Zstd.decompress(in, (int) originalSize); + } + // Slow path: declared size is 0, unknown (-1), or invalid (-2) — stream-decode and let + // ZstdInputStream surface any frame errors. + ZstdInputStream zis = null; + try { + zis = new ZstdInputStream(new ByteArrayInputStream(in)); + return readAll(zis); + } catch (IOException e) { + getLogger().error("Error reading Zstd input stream", e); + return null; + } finally { + try { if (zis != null) zis.close(); } catch (IOException ignored) {} + } + } - timer.record(ratio_percentage, TimeUnit.MILLISECONDS); + private static byte[] readAll(InputStream in) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buf = new byte[8192]; + int n; + while ((n = in.read(buf)) != -1) { + out.write(buf, 0, n); + } + return out.toByteArray(); } + private void recordCompressionRatio(long ratioPerCent, CompressionAlgorithm compressionAlgorithm) { + final List tagList = new ArrayList(2); + tagList.add(new BasicTag(EVCacheMetricsFactory.COMPRESSION_TYPE, compressionAlgorithm.name().toLowerCase())); + if (appName != null && !appName.isEmpty()) { + tagList.add(new BasicTag(EVCacheMetricsFactory.CACHE, appName)); + } + EVCacheMetricsFactory.getInstance() + .getDistributionSummary(EVCacheMetricsFactory.COMPRESSION_RATIO, tagList) + .record(ratioPerCent); + } } diff --git a/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java b/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java index 97be808b..38359892 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java +++ b/evcache-core/src/main/java/com/netflix/evcache/EVCacheTranscoder.java @@ -1,5 +1,7 @@ package com.netflix.evcache; +import com.netflix.archaius.api.Property; +import com.netflix.archaius.api.PropertyRepository; import com.netflix.evcache.util.EVCacheConfig; import net.spy.memcached.CachedData; @@ -7,26 +9,61 @@ public class EVCacheTranscoder extends EVCacheSerializingTranscoder { public EVCacheTranscoder() { - this(EVCacheConfig.getInstance().getPropertyRepository().get("default.evcache.max.data.size", Integer.class).orElse(20 * 1024 * 1024).get()); + this((String) null); + } + + public EVCacheTranscoder(String appName) { + this(appName, EVCacheConfig.getInstance().getPropertyRepository()); } public EVCacheTranscoder(int max) { - this(max, EVCacheConfig.getInstance().getPropertyRepository().get("default.evcache.compression.threshold", Integer.class).orElse(120).get()); + this(null, max); + } + + public EVCacheTranscoder(String appName, int max) { + this(appName, EVCacheConfig.getInstance().getPropertyRepository(), max); } public EVCacheTranscoder(int max, int compressionThreshold) { - super(max); - setCompressionThreshold(compressionThreshold); + this(null, max, compressionThreshold); } - @Override - public boolean asyncDecode(CachedData d) { - return super.asyncDecode(d); + public EVCacheTranscoder(String appName, int max, int compressionThreshold) { + this(appName, EVCacheConfig.getInstance().getPropertyRepository(), max, compressionThreshold); } - @Override - public Object decode(CachedData d) { - return super.decode(d); + /** + * Repository-aware constructors. The compression algorithm/level are read dynamically from the + * supplied {@link PropertyRepository}, so callers must pass the repository that is wired to Fast + * Properties (e.g. {@code poolManager.getEVCacheConfig().getPropertyRepository()}) for FP overrides + * to take effect. The no-repository constructors above fall back to {@link EVCacheConfig#getInstance()}. + */ + public EVCacheTranscoder(String appName, PropertyRepository config) { + this(appName, config, config.get("default.evcache.max.data.size", Integer.class).orElse(20 * 1024 * 1024).get()); + } + + public EVCacheTranscoder(String appName, PropertyRepository config, int max) { + this(appName, config, max, config.get("default.evcache.compression.threshold", Integer.class).orElse(120).get()); + } + + public EVCacheTranscoder(String appName, PropertyRepository config, int max, int compressionThreshold) { + super(appName, max); + setCompressionThreshold(compressionThreshold); + Property algoProperty = getProperty(config, "evcacheclient.compression.algo", String.class); + setCompressionAlgorithmProperty(algoProperty); + Property zstdLevelProperty = getProperty(config, "evcacheclient.compression.zstd.level", Integer.class); + setCompressionLevelProperty(zstdLevelProperty); + } + + /** + * Resolves a property preferring the appName-prefixed key (e.g. {@code EVCACHE_VH_ARCHIVE.default.evcache.compression.algo}) + * and falling back to the global {@code default.evcache.*} key when no app-specific override exists. + */ + private Property getProperty(PropertyRepository config, String key, Class type) { + if (appName == null || appName.isEmpty()) { + return config.get(key, type); + } + return config.get(appName + "." + key, type).orElseGet(key); } @Override diff --git a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java index 1204156f..2b12c3b8 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java +++ b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseAsciiConnectionFactory.java @@ -115,7 +115,8 @@ public BlockingQueue createWriteOperationQueue() { } public Transcoder getDefaultTranscoder() { - return new EVCacheTranscoder(); + return new EVCacheTranscoder(appName, + client.getPool().getEVCacheClientPoolManager().getEVCacheConfig().getPropertyRepository()); } public FailureMode getFailureMode() { diff --git a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java index 7cd7b290..37d300f9 100644 --- a/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java +++ b/evcache-core/src/main/java/com/netflix/evcache/connection/BaseConnectionFactory.java @@ -109,7 +109,8 @@ public BlockingQueue createWriteOperationQueue() { } public Transcoder getDefaultTranscoder() { - return new EVCacheTranscoder(); + return new EVCacheTranscoder(appName, + client.getPool().getEVCacheClientPoolManager().getEVCacheConfig().getPropertyRepository()); } public FailureMode getFailureMode() { diff --git a/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java new file mode 100644 index 00000000..432d9719 --- /dev/null +++ b/evcache-core/src/test/java/com/netflix/evcache/EVCacheSerializingTranscoderTest.java @@ -0,0 +1,358 @@ +package com.netflix.evcache; + +import com.netflix.archaius.DefaultPropertyFactory; +import com.netflix.archaius.api.PropertyRepository; +import com.netflix.archaius.config.DefaultSettableConfig; +import com.netflix.evcache.metrics.EVCacheMetricsFactory; +import com.netflix.evcache.util.EVCacheConfig; +import com.netflix.spectator.api.DefaultRegistry; +import com.netflix.spectator.api.Id; +import com.netflix.spectator.api.Meter; +import com.netflix.spectator.api.Registry; +import com.netflix.spectator.api.Spectator; +import com.netflix.spectator.api.Tag; +import net.spy.memcached.CachedData; +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class EVCacheSerializingTranscoderTest { + + private EVCacheSerializingTranscoder buildTranscoder(String algo, Integer level) { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("test.algo", algo); + if (level != null) config.setProperty("test.level", level); + PropertyRepository repo = new DefaultPropertyFactory(config); + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(CachedData.MAX_SIZE); + t.setCompressionAlgorithmProperty(repo.get("test.algo", String.class)); + t.setCompressionLevelProperty(repo.get("test.level", Integer.class)); + return t; + } + + @Test + public void testEnumValues() { + assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("GZIP"), + EVCacheSerializingTranscoder.CompressionAlgorithm.GZIP); + assertEquals(EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("ZSTD"), + EVCacheSerializingTranscoder.CompressionAlgorithm.ZSTD); + } + + @Test + public void testDefaultZstdLevelConstant() { + assertEquals(EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL, 3); + } + + @Test + public void testGzipDefaultProducesGzipMagicBytes() { + EVCacheSerializingTranscoder t = buildTranscoder("GZIP", null); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "GZIP property must produce gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "GZIP property must produce gzip magic byte 1"); + } + + @Test + public void testZstdPropertyProducesZstdMagicBytes() { + EVCacheSerializingTranscoder t = buildTranscoder("ZSTD", null); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "ZSTD property must produce zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "ZSTD property must produce zstd magic byte 1"); + } + + @Test + public void testCustomZstdLevelRoundTrip() { + EVCacheSerializingTranscoder t = buildTranscoder("ZSTD", 5); + t.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = t.encode(original); + String decoded = (String) t.decode(encoded); + assertEquals(decoded, original, "Round-trip must succeed with custom zstd level 5"); + } + + @Test + public void testGzipEncodeSetsGzipMagicBytes() { + EVCacheSerializingTranscoder t = buildTranscoder("GZIP", null); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "Expected gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "Expected gzip magic byte 1"); + } + + @Test + public void testZstdEncodeSetsZstdMagicBytes() { + EVCacheSerializingTranscoder t = buildTranscoder("ZSTD", null); + t.setCompressionThreshold(0); + CachedData encoded = t.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + // Zstd magic is 0xFD2FB528 in little-endian: bytes 0x28 0xB5 0x2F 0xFD + assertEquals(data[0], (byte) 0x28, "Expected zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "Expected zstd magic byte 1"); + assertEquals(data[2], (byte) 0x2F, "Expected zstd magic byte 2"); + assertEquals(data[3], (byte) 0xFD, "Expected zstd magic byte 3"); + } + + @Test + public void testGzipRoundTrip() { + EVCacheSerializingTranscoder transcoder = buildTranscoder("GZIP", null); + transcoder.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testZstdRoundTrip() { + EVCacheSerializingTranscoder transcoder = buildTranscoder("ZSTD", null); + transcoder.setCompressionThreshold(1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testGzipTranscoderDecodesZstdData() { + // zstd transcoder writes, gzip transcoder reads → cross-decode via magic-byte detection + EVCacheSerializingTranscoder writer = buildTranscoder("ZSTD", null); + writer.setCompressionThreshold(1); + EVCacheSerializingTranscoder reader = buildTranscoder("GZIP", null); + + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = writer.encode(original); + String decoded = (String) reader.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testZstdTranscoderDecodesGzipData() { + // gzip transcoder writes, zstd transcoder reads → cross-decode via magic-byte detection + EVCacheSerializingTranscoder writer = buildTranscoder("GZIP", null); + writer.setCompressionThreshold(1); + EVCacheSerializingTranscoder reader = buildTranscoder("ZSTD", null); + + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = writer.encode(original); + String decoded = (String) reader.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testEVCacheTranscoderDefaultsToGzip() { + EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 0); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "EVCacheTranscoder must default to gzip"); + assertEquals(data[1], (byte) 0x8b, "EVCacheTranscoder must default to gzip"); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } + + @Test + public void testEVCacheTranscoderExplicitZstdAlgorithm() { + DefaultSettableConfig testConfig = new DefaultSettableConfig(); + testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); + testConfig.setProperty("evcacheclient.compression.zstd.level", + EVCacheSerializingTranscoder.DEFAULT_ZSTD_COMPRESSION_LEVEL); + PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); + EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); + try { + EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original); + } finally { + EVCacheConfig.setPropertyRepository(savedRepo); + } + } + + @Test + public void testAppNamePrefixedAlgoOverridesDefault() { + DefaultSettableConfig testConfig = new DefaultSettableConfig(); + testConfig.setProperty("evcacheclient.compression.algo", "GZIP"); + testConfig.setProperty("EVCACHE_VH_ARCHIVE.evcacheclient.compression.algo", "ZSTD"); + PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); + EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); + try { + EVCacheTranscoder transcoder = new EVCacheTranscoder("EVCACHE_VH_ARCHIVE", CachedData.MAX_SIZE, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "app-specific ZSTD override must win over default GZIP"); + assertEquals(data[1], (byte) 0xB5, "app-specific ZSTD override must win over default GZIP"); + } finally { + EVCacheConfig.setPropertyRepository(savedRepo); + } + } + + @Test + public void testAppNameFallsBackToDefaultAlgoWhenNoOverride() { + DefaultSettableConfig testConfig = new DefaultSettableConfig(); + testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); + PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); + EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); + try { + EVCacheTranscoder transcoder = new EVCacheTranscoder("EVCACHE_NO_OVERRIDE", CachedData.MAX_SIZE, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "must fall back to default ZSTD when no app-specific override exists"); + assertEquals(data[1], (byte) 0xB5, "must fall back to default ZSTD when no app-specific override exists"); + } finally { + EVCacheConfig.setPropertyRepository(savedRepo); + } + } + + @Test + public void testAppNamePrefixedZstdLevelRoundTrip() { + DefaultSettableConfig testConfig = new DefaultSettableConfig(); + testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); + testConfig.setProperty("evcacheclient.compression.zstd.level", 1); + testConfig.setProperty("EVCACHE_VH_ARCHIVE.evcacheclient.compression.zstd.level", 5); + PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); + EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); + try { + EVCacheTranscoder transcoder = new EVCacheTranscoder("EVCACHE_VH_ARCHIVE", CachedData.MAX_SIZE, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original, "app-specific zstd level override round-trip must succeed"); + } finally { + EVCacheConfig.setPropertyRepository(savedRepo); + } + } + + @Test + public void testCompressionRatioMetricTaggedWithAppName() { + final String appName = "EVCACHE_RATIO_TEST"; + Registry registry = new DefaultRegistry(); + Spectator.globalRegistry().add(registry); + try { + DefaultSettableConfig config = new DefaultSettableConfig(); + config.setProperty("test.algo", "GZIP"); + PropertyRepository repo = new DefaultPropertyFactory(config); + EVCacheSerializingTranscoder t = new EVCacheSerializingTranscoder(appName, CachedData.MAX_SIZE); + t.setCompressionAlgorithmProperty(repo.get("test.algo", String.class)); + t.setCompressionLevelProperty(repo.get("test.level", Integer.class)); + t.setCompressionThreshold(0); + t.encode("hello world hello world hello world hello world hello world"); + + assertTrue(hasCompressionRatioCacheTag(registry, appName), + "compression ratio metric must carry the " + EVCacheMetricsFactory.CACHE + " tag with the app name"); + } finally { + Spectator.globalRegistry().remove(registry); + } + } + + @Test + public void testCompressionRatioMetricNotTaggedWhenNoAppName() { + Registry registry = new DefaultRegistry(); + Spectator.globalRegistry().add(registry); + try { + EVCacheSerializingTranscoder t = buildTranscoder("GZIP", null); + t.setCompressionThreshold(0); + t.encode("hello world hello world hello world hello world hello world"); + + for (Meter meter : registry) { + Id id = meter.id(); + if (EVCacheMetricsFactory.COMPRESSION_RATIO.equals(id.name())) { + for (Tag tag : id.tags()) { + assertNotEquals(tag.key(), EVCacheMetricsFactory.CACHE, + "no app name tag must be added when app name is absent"); + } + } + } + } finally { + Spectator.globalRegistry().remove(registry); + } + } + + private boolean hasCompressionRatioCacheTag(Registry registry, String appName) { + for (Meter meter : registry) { + Id id = meter.id(); + if (EVCacheMetricsFactory.COMPRESSION_RATIO.equals(id.name())) { + for (Tag tag : id.tags()) { + if (EVCacheMetricsFactory.CACHE.equals(tag.key()) && appName.equals(tag.value())) { + return true; + } + } + } + } + return false; + } + + @Test(expectedExceptions = IllegalArgumentException.class) + public void testInvalidAlgorithmEnumThrows() { + EVCacheSerializingTranscoder.CompressionAlgorithm.valueOf("INVALID"); + } + + @Test + public void testFPAlgorithmGzip() { + DefaultSettableConfig testConfig = new DefaultSettableConfig(); + testConfig.setProperty("evcacheclient.compression.algo", "GZIP"); + PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); + EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); + try { + EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x1f, "FP GZIP must produce gzip magic byte 0"); + assertEquals(data[1], (byte) 0x8b, "FP GZIP must produce gzip magic byte 1"); + } finally { + EVCacheConfig.setPropertyRepository(savedRepo); + } + } + + @Test + public void testFPAlgorithmZstd() { + DefaultSettableConfig testConfig = new DefaultSettableConfig(); + testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); + PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); + EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); + try { + EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); + CachedData encoded = transcoder.encode("hello world hello world hello world hello world hello world"); + assertTrue((encoded.getFlags() & EVCacheSerializingTranscoder.COMPRESSED) != 0, + "COMPRESSED flag must be set"); + byte[] data = encoded.getData(); + assertEquals(data[0], (byte) 0x28, "FP ZSTD must produce zstd magic byte 0"); + assertEquals(data[1], (byte) 0xB5, "FP ZSTD must produce zstd magic byte 1"); + } finally { + EVCacheConfig.setPropertyRepository(savedRepo); + } + } + + @Test + public void testFPZstdLevel() { + DefaultSettableConfig testConfig = new DefaultSettableConfig(); + testConfig.setProperty("evcacheclient.compression.algo", "ZSTD"); + testConfig.setProperty("evcacheclient.compression.zstd.level", 1); + PropertyRepository savedRepo = EVCacheConfig.getInstance().getPropertyRepository(); + EVCacheConfig.setPropertyRepository(new DefaultPropertyFactory(testConfig)); + try { + EVCacheTranscoder transcoder = new EVCacheTranscoder((String) null, CachedData.MAX_SIZE, 1); + String original = "hello world hello world hello world hello world hello world"; + CachedData encoded = transcoder.encode(original); + String decoded = (String) transcoder.decode(encoded); + assertEquals(decoded, original, "FP zstd level 1 round-trip must succeed"); + } finally { + EVCacheConfig.setPropertyRepository(savedRepo); + } + } +} diff --git a/evcache-core/src/test/java/test-suite.xml b/evcache-core/src/test/java/test-suite.xml index f031a615..194ea07e 100644 --- a/evcache-core/src/test/java/test-suite.xml +++ b/evcache-core/src/test/java/test-suite.xml @@ -10,6 +10,11 @@ + + + + +