diff --git a/.sdkmanrc b/.sdkmanrc index 1a28648b26c..8427edd8456 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -1,4 +1,4 @@ # Enable auto-env through the sdkman_auto_env config # Add key=value pairs of SDKs to use below -java=21.0.8-tem +java=25-tem diff --git a/build.gradle b/build.gradle index a235d2235dc..d7709244c37 100644 --- a/build.gradle +++ b/build.gradle @@ -58,7 +58,7 @@ plugins { apply plugin: 'de.undercouch.download' -ext.javaLanguageVersion = 21 +ext.javaLanguageVersion = 25 // Adoptium also covers Temurin ext.javaVendor = JvmVendorSpec.ADOPTIUM diff --git a/container_build/docker_java/Dockerfile b/container_build/docker_java/Dockerfile index ba24a6be271..c199f6c3c5b 100644 --- a/container_build/docker_java/Dockerfile +++ b/container_build/docker_java/Dockerfile @@ -18,7 +18,7 @@ # It also contains plantuml for plant uml image generation # Using 'openjdk' on Alpine is not fully supported so using Eclipse Temurin JDK to ensure we have a known jdk version # See https://github.com/docker-library/docs/blob/master/openjdk/README.md#openjdkversion-alpine -FROM eclipse-temurin:21.0.7_6-jdk-alpine +FROM eclipse-temurin:25_36-jdk-alpine-3.22 # Work from the shared git repo dir WORKDIR /builder/shared diff --git a/gradle/gradle-daemon-jvm.properties b/gradle/gradle-daemon-jvm.properties index 63e5bbdf484..5a334ba9a3d 100644 --- a/gradle/gradle-daemon-jvm.properties +++ b/gradle/gradle-daemon-jvm.properties @@ -1,2 +1,2 @@ #This file is generated by updateDaemonJvm -toolchainVersion=21 +toolchainVersion=25 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index c2d6d080867..c069e5f5e0d 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -52,7 +52,7 @@ commons-io = { module = "commons-io:commons-io", version.ref = "commons-io" } commons-lang = { module = "org.apache.commons:commons-lang3" } # version controlled by dropwizard-dependencies commons-pool2 = { module = "org.apache.commons:commons-pool2", version = "2.12.1" } commons-text = { module = "org.apache.commons:commons-text" } # version controlled by dropwizard-dependencies -classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.179" } +classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.181" } data-faker = { module = "net.datafaker:datafaker", version = "2.4.2" } dropwizard-assets = { module = "io.dropwizard:dropwizard-assets" } # version controlled by dropwizard-dependencies dropwizard-auth = { module = "io.dropwizard:dropwizard-auth" } # version controlled by dropwizard-dependencies diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index a4b76b9530d..8bdaf60c75a 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e18bc253b85..2e1113280ef 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.12.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.1.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index f3b75f3b0d4..adff685a034 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. @@ -172,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -205,15 +203,14 @@ fi DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ - org.gradle.wrapper.GradleWrapperMain \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" # Stop when "xargs" is not available. diff --git a/gradlew.bat b/gradlew.bat index 9b42019c791..e509b2dd8fe 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -70,11 +70,10 @@ goto fail :execute @rem Setup the command line -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* :end @rem End local scope for the variables with windows NT shell diff --git a/stroom-app/docker/Dockerfile b/stroom-app/docker/Dockerfile index 998b61aef4a..0f6dc98beca 100644 --- a/stroom-app/docker/Dockerfile +++ b/stroom-app/docker/Dockerfile @@ -23,7 +23,7 @@ # jstat/jmap/jcmd/etc. # Using 'openjdk' on Alpine is not fully supported so using Eclipse Temurin JDK to ensure we have a known jdk version # See https://github.com/docker-library/docs/blob/master/openjdk/README.md#openjdkversion-alpine -FROM eclipse-temurin:21.0.8_9-jdk-alpine AS stroom-base-stage +FROM eclipse-temurin:25_36-jdk-alpine-3.22 AS stroom-base-stage # bash and jq are required for Kubernetes lifecycle scripts, which interact with the API # curl is required for the docker healthcheck diff --git a/stroom-aws/stroom-aws-s3-impl/build.gradle b/stroom-aws/stroom-aws-s3-impl/build.gradle index 35502d61ae6..45e3fc22bc2 100644 --- a/stroom-aws/stroom-aws-s3-impl/build.gradle +++ b/stroom-aws/stroom-aws-s3-impl/build.gradle @@ -33,3 +33,7 @@ dependencies { testImplementation libs.bundles.common.test.implementation testRuntimeOnly libs.bundles.common.test.runtime } + +tasks.withType(AbstractTestTask).configureEach { + failOnNoDiscoveredTests = false +} diff --git a/stroom-bytebuffer/build.gradle b/stroom-bytebuffer/build.gradle index 30d81bc2814..8ba99169f30 100644 --- a/stroom-bytebuffer/build.gradle +++ b/stroom-bytebuffer/build.gradle @@ -1,7 +1,6 @@ ext.moduleName = 'stroom.bytebuffer' dependencies { - implementation project(':stroom-hadoop') implementation project(':stroom-util') implementation project(':stroom-util-shared') // diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/ByteBufferUtils.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/ByteBufferUtils.java index 647667d191d..76f17ff3388 100644 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/ByteBufferUtils.java +++ b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/ByteBufferUtils.java @@ -470,6 +470,33 @@ public static void padMax(final ByteBuffer byteBuffer, final int offset, final i } } + /** + * Check for byte buffer equality over portions of two buffers. This is generally quicker than slicing as no object + * creation is required. + * + * @param a Byte buffer 1. + * @param aOff Byte buffer 1 offset. + * @param b Byte buffer 2. + * @param bOff Byte buffer 2 offset. + * @param length Length to compare. + * @return True if byte buffer portions are equal. + */ + public static boolean equals(final ByteBuffer a, + final int aOff, + final ByteBuffer b, + final int bOff, + final int length) { + if (length > 7) { + return a.slice(aOff, length).equals(b.slice(bOff, length)); + } + for (int i = 0; i < length; i++) { + if (a.get(aOff + i) != b.get(bOff + i)) { + return false; + } + } + return true; + } + public static void skip(final ByteBuffer byteBuffer, final int len) { byteBuffer.position(byteBuffer.position() + len); } diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/PooledByteBufferOutputStream.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/PooledByteBufferOutputStream.java index 7de4b54e043..680d3b23f1c 100644 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/PooledByteBufferOutputStream.java +++ b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/PooledByteBufferOutputStream.java @@ -125,6 +125,12 @@ public void write(final byte[] b, final int off, final int len) throws IOExcepti } } + public void writeLong(final long l) throws IOException { + checkWriteableState(); + checkSizeAndGrow(Long.BYTES); + getCurrentPooledBuffer().getByteBuffer().putLong(l); + } + /** * Writes byteBuffer into the outputStream. Respects the position/limit of byteBuffer. * After reading, byteBuffer is rewound to return it to its passed state. diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/ByteBufferUtils.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/ByteBufferUtils.java deleted file mode 100644 index 72688a312df..00000000000 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/ByteBufferUtils.java +++ /dev/null @@ -1,1463 +0,0 @@ -/* - * Copy of https://github.com/apache/hbase/blob/master/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java - * to avoid having to pull in all of hbase to use the util methods. - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stroom.bytebuffer.hbase; - -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.WritableUtils; - -import java.io.ByteArrayOutputStream; -import java.io.DataInput; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.util.Arrays; - -/** - * Utility functions for working with byte buffers, such as reading/writing variable-length long - * numbers. - */ -public final class ByteBufferUtils { - - // "Compressed integer" serialization helper constants. - public final static int VALUE_MASK = 0x7f; - public final static int NEXT_BIT_SHIFT = 7; - public final static int NEXT_BIT_MASK = 1 << 7; - final static boolean UNSAFE_AVAIL = HBasePlatformDependent.isUnsafeAvailable(); - public final static boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned(); - - private ByteBufferUtils() { - } - - static abstract class Comparer { - - abstract int compareTo(byte[] buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2); - - abstract int compareTo(ByteBuffer buf1, int o1, int l1, ByteBuffer buf2, int o2, int l2); - } - - static abstract class Converter { - - abstract short toShort(ByteBuffer buffer, int offset); - - abstract int toInt(ByteBuffer buffer); - - abstract int toInt(ByteBuffer buffer, int offset); - - abstract long toLong(ByteBuffer buffer, int offset); - - abstract void putInt(ByteBuffer buffer, int val); - - abstract int putInt(ByteBuffer buffer, int index, int val); - - abstract void putShort(ByteBuffer buffer, short val); - - abstract int putShort(ByteBuffer buffer, int index, short val); - - abstract void putLong(ByteBuffer buffer, long val); - - abstract int putLong(ByteBuffer buffer, int index, long val); - } - - static abstract class CommonPrefixer { - - abstract int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, byte[] right, - int rightOffset, int rightLength); - - abstract int findCommonPrefix(ByteBuffer left, int leftOffset, int leftLength, ByteBuffer right, - int rightOffset, int rightLength); - } - - static class ComparerHolder { - - static final String UNSAFE_COMPARER_NAME = ComparerHolder.class.getName() + "$UnsafeComparer"; - - static final Comparer BEST_COMPARER = getBestComparer(); - - static Comparer getBestComparer() { - try { - final Class theClass = - Class.forName(UNSAFE_COMPARER_NAME).asSubclass(Comparer.class); - - return theClass.getConstructor().newInstance(); - } catch (final Throwable t) { // ensure we really catch *everything* - return PureJavaComparer.INSTANCE; - } - } - - static final class PureJavaComparer extends Comparer { - - static final PureJavaComparer INSTANCE = new PureJavaComparer(); - - private PureJavaComparer() { - } - - @Override - public int compareTo(final byte[] buf1, final int o1, final int l1, final ByteBuffer buf2, final int o2, final int l2) { - final int end1 = o1 + l1; - final int end2 = o2 + l2; - for (int i = o1, j = o2; i < end1 && j < end2; i++, j++) { - final int a = buf1[i] & 0xFF; - final int b = buf2.get(j) & 0xFF; - if (a != b) { - return a - b; - } - } - return l1 - l2; - } - - @Override - public int compareTo(final ByteBuffer buf1, final int o1, final int l1, final ByteBuffer buf2, final int o2, final int l2) { - final int end1 = o1 + l1; - final int end2 = o2 + l2; - for (int i = o1, j = o2; i < end1 && j < end2; i++, j++) { - final int a = buf1.get(i) & 0xFF; - final int b = buf2.get(j) & 0xFF; - if (a != b) { - return a - b; - } - } - return l1 - l2; - } - } - - static final class UnsafeComparer extends Comparer { - - public UnsafeComparer() { - } - - static { - if (!UNSAFE_UNALIGNED) { - throw new Error(); - } - } - - @Override - public int compareTo(final byte[] buf1, final int o1, final int l1, final ByteBuffer buf2, final int o2, final int l2) { - final long offset2Adj; - Object refObj2 = null; - if (buf2.isDirect()) { - offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2); - } else { - offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - refObj2 = buf2.array(); - } - return compareToUnsafe(buf1, o1 + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET, l1, refObj2, - offset2Adj, l2); - } - - @Override - public int compareTo(final ByteBuffer buf1, final int o1, final int l1, final ByteBuffer buf2, final int o2, final int l2) { - final long offset1Adj; - final long offset2Adj; - Object refObj1 = null, refObj2 = null; - if (buf1.isDirect()) { - offset1Adj = o1 + UnsafeAccess.directBufferAddress(buf1); - } else { - offset1Adj = o1 + buf1.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - refObj1 = buf1.array(); - } - if (buf2.isDirect()) { - offset2Adj = o2 + UnsafeAccess.directBufferAddress(buf2); - } else { - offset2Adj = o2 + buf2.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - refObj2 = buf2.array(); - } - return compareToUnsafe(refObj1, offset1Adj, l1, refObj2, offset2Adj, l2); - } - } - } - - static class ConverterHolder { - - static final String UNSAFE_CONVERTER_NAME = - ConverterHolder.class.getName() + "$UnsafeConverter"; - static final Converter BEST_CONVERTER = getBestConverter(); - - static Converter getBestConverter() { - try { - final Class theClass = - Class.forName(UNSAFE_CONVERTER_NAME).asSubclass(Converter.class); - - // yes, UnsafeComparer does implement Comparer - return theClass.getConstructor().newInstance(); - } catch (final Throwable t) { // ensure we really catch *everything* - return PureJavaConverter.INSTANCE; - } - } - - static final class PureJavaConverter extends Converter { - - static final PureJavaConverter INSTANCE = new PureJavaConverter(); - - private PureJavaConverter() { - } - - @Override - short toShort(final ByteBuffer buffer, final int offset) { - return buffer.getShort(offset); - } - - @Override - int toInt(final ByteBuffer buffer) { - return buffer.getInt(); - } - - @Override - int toInt(final ByteBuffer buffer, final int offset) { - return buffer.getInt(offset); - } - - @Override - long toLong(final ByteBuffer buffer, final int offset) { - return buffer.getLong(offset); - } - - @Override - void putInt(final ByteBuffer buffer, final int val) { - buffer.putInt(val); - } - - @Override - int putInt(final ByteBuffer buffer, final int index, final int val) { - buffer.putInt(index, val); - return index + Bytes.SIZEOF_INT; - } - - @Override - void putShort(final ByteBuffer buffer, final short val) { - buffer.putShort(val); - } - - @Override - int putShort(final ByteBuffer buffer, final int index, final short val) { - buffer.putShort(index, val); - return index + Bytes.SIZEOF_SHORT; - } - - @Override - void putLong(final ByteBuffer buffer, final long val) { - buffer.putLong(val); - } - - @Override - int putLong(final ByteBuffer buffer, final int index, final long val) { - buffer.putLong(index, val); - return index + Bytes.SIZEOF_LONG; - } - } - - static final class UnsafeConverter extends Converter { - - public UnsafeConverter() { - } - - static { - if (!UNSAFE_UNALIGNED) { - throw new Error(); - } - } - - @Override - short toShort(final ByteBuffer buffer, final int offset) { - return UnsafeAccess.toShort(buffer, offset); - } - - @Override - int toInt(final ByteBuffer buffer) { - final int i = UnsafeAccess.toInt(buffer, buffer.position()); - buffer.position(buffer.position() + Bytes.SIZEOF_INT); - return i; - } - - @Override - int toInt(final ByteBuffer buffer, final int offset) { - return UnsafeAccess.toInt(buffer, offset); - } - - @Override - long toLong(final ByteBuffer buffer, final int offset) { - return UnsafeAccess.toLong(buffer, offset); - } - - @Override - void putInt(final ByteBuffer buffer, final int val) { - final int newPos = UnsafeAccess.putInt(buffer, buffer.position(), val); - buffer.position(newPos); - } - - @Override - int putInt(final ByteBuffer buffer, final int index, final int val) { - return UnsafeAccess.putInt(buffer, index, val); - } - - @Override - void putShort(final ByteBuffer buffer, final short val) { - final int newPos = UnsafeAccess.putShort(buffer, buffer.position(), val); - buffer.position(newPos); - } - - @Override - int putShort(final ByteBuffer buffer, final int index, final short val) { - return UnsafeAccess.putShort(buffer, index, val); - } - - @Override - void putLong(final ByteBuffer buffer, final long val) { - final int newPos = UnsafeAccess.putLong(buffer, buffer.position(), val); - buffer.position(newPos); - } - - @Override - int putLong(final ByteBuffer buffer, final int index, final long val) { - return UnsafeAccess.putLong(buffer, index, val); - } - } - } - - static class CommonPrefixerHolder { - - static final String UNSAFE_COMMON_PREFIXER_NAME = - CommonPrefixerHolder.class.getName() + "$UnsafeCommonPrefixer"; - - static final CommonPrefixer BEST_COMMON_PREFIXER = getBestCommonPrefixer(); - - static CommonPrefixer getBestCommonPrefixer() { - try { - final Class theClass = - Class.forName(UNSAFE_COMMON_PREFIXER_NAME).asSubclass(CommonPrefixer.class); - - return theClass.getConstructor().newInstance(); - } catch (final Throwable t) { // ensure we really catch *everything* - return PureJavaCommonPrefixer.INSTANCE; - } - } - - static final class PureJavaCommonPrefixer extends CommonPrefixer { - - static final PureJavaCommonPrefixer INSTANCE = new PureJavaCommonPrefixer(); - - private PureJavaCommonPrefixer() { - } - - @Override - public int findCommonPrefix(final ByteBuffer left, final int leftOffset, final int leftLength, final byte[] right, - final int rightOffset, final int rightLength) { - final int length = Math.min(leftLength, rightLength); - int result = 0; - - while ( - result < length - && ByteBufferUtils.toByte(left, leftOffset + result) == right[rightOffset + result] - ) { - result++; - } - - return result; - } - - @Override - int findCommonPrefix(final ByteBuffer left, final int leftOffset, final int leftLength, final ByteBuffer right, - final int rightOffset, final int rightLength) { - final int length = Math.min(leftLength, rightLength); - int result = 0; - - while ( - result < length && ByteBufferUtils.toByte(left, leftOffset + result) - == ByteBufferUtils.toByte(right, rightOffset + result) - ) { - result++; - } - - return result; - } - } - - static final class UnsafeCommonPrefixer extends CommonPrefixer { - - static { - if (!UNSAFE_UNALIGNED) { - throw new Error(); - } - } - - public UnsafeCommonPrefixer() { - } - - @Override - public int findCommonPrefix(final ByteBuffer left, final int leftOffset, final int leftLength, final byte[] right, - final int rightOffset, final int rightLength) { - final long offset1Adj; - Object refObj1 = null; - if (left.isDirect()) { - offset1Adj = leftOffset + UnsafeAccess.directBufferAddress(left); - } else { - offset1Adj = leftOffset + left.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - refObj1 = left.array(); - } - return findCommonPrefixUnsafe(refObj1, offset1Adj, leftLength, right, - rightOffset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET, rightLength); - } - - @Override - public int findCommonPrefix(final ByteBuffer left, final int leftOffset, final int leftLength, final ByteBuffer right, - final int rightOffset, final int rightLength) { - final long offset1Adj; - final long offset2Adj; - Object refObj1 = null, refObj2 = null; - if (left.isDirect()) { - offset1Adj = leftOffset + UnsafeAccess.directBufferAddress(left); - } else { - offset1Adj = leftOffset + left.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - refObj1 = left.array(); - } - if (right.isDirect()) { - offset2Adj = rightOffset + UnsafeAccess.directBufferAddress(right); - } else { - offset2Adj = rightOffset + right.arrayOffset() + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - refObj2 = right.array(); - } - return findCommonPrefixUnsafe(refObj1, offset1Adj, leftLength, refObj2, offset2Adj, - rightLength); - } - } - } - - /** - * Similar to {@link WritableUtils#writeVLong(java.io.DataOutput, long)}, but writes to a - * {@link ByteBuffer}. - */ - public static void writeVLong(final ByteBuffer out, long i) { - if (i >= -112 && i <= 127) { - out.put((byte) i); - return; - } - - int len = -112; - if (i < 0) { - i ^= -1L; // take one's complement - len = -120; - } - - long tmp = i; - while (tmp != 0) { - tmp = tmp >> 8; - len--; - } - - out.put((byte) len); - - len = (len < -120) - ? -(len + 120) - : -(len + 112); - - for (int idx = len; idx != 0; idx--) { - final int shiftbits = (idx - 1) * 8; - final long mask = 0xFFL << shiftbits; - out.put((byte) ((i & mask) >> shiftbits)); - } - } - -// /** -// * Similar to {@link WritableUtils#readVLong(java.io.DataInput)} but reads from a -// * {@link ByteBuff}. -// */ -// public static long readVLong(ByteBuff buf) { -// byte firstByte = buf.get(); -// int len = WritableUtils.decodeVIntSize(firstByte); -// if (len == 1) { -// return firstByte; -// } else { -// int remaining = len - 1; -// long i = 0; -// int offsetFromPos = 0; -// if (remaining >= Bytes.SIZEOF_INT) { -// // The int read has to be converted to unsigned long so the & op -// i = (buf.getIntAfterPosition(offsetFromPos) & 0x00000000ffffffffL); -// remaining -= Bytes.SIZEOF_INT; -// offsetFromPos += Bytes.SIZEOF_INT; -// } -// if (remaining >= Bytes.SIZEOF_SHORT) { -// short s = buf.getShortAfterPosition(offsetFromPos); -// i = i << 16; -// i = i | (s & 0xFFFF); -// remaining -= Bytes.SIZEOF_SHORT; -// offsetFromPos += Bytes.SIZEOF_SHORT; -// } -// for (int idx = 0; idx < remaining; idx++) { -// byte b = buf.getByteAfterPosition(offsetFromPos + idx); -// i = i << 8; -// i = i | (b & 0xFF); -// } -// buf.skip(len - 1); -// return WritableUtils.isNegativeVInt(firstByte) -// ? ~i -// : i; -// } -// } - - /** - * Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuffer}. - */ - public static long readVLong(final ByteBuffer buf) { - final byte firstByte = buf.get(); - final int len = WritableUtils.decodeVIntSize(firstByte); - if (len == 1) { - return firstByte; - } else { - int remaining = len - 1; - long i = 0; - int offsetFromPos = 0; - if (remaining >= Bytes.SIZEOF_INT) { - // The int read has to be converted to unsigned long so the & op - i = (buf.getInt(buf.position() + offsetFromPos) & 0x00000000ffffffffL); - remaining -= Bytes.SIZEOF_INT; - offsetFromPos += Bytes.SIZEOF_INT; - } - if (remaining >= Bytes.SIZEOF_SHORT) { - final short s = buf.getShort(buf.position() + offsetFromPos); - i = i << 16; - i = i | (s & 0xFFFF); - remaining -= Bytes.SIZEOF_SHORT; - offsetFromPos += Bytes.SIZEOF_SHORT; - } - for (int idx = 0; idx < remaining; idx++) { - final byte b = buf.get(buf.position() + offsetFromPos + idx); - i = i << 8; - i = i | (b & 0xFF); - } - buf.position(buf.position() + len - 1); - return WritableUtils.isNegativeVInt(firstByte) - ? ~i - : i; - } - } - - /** - * Put in buffer integer using 7 bit encoding. For each written byte: 7 bits are used to store - * value 1 bit is used to indicate whether there is next bit. - * - * @param value Int to be compressed. - * @param out Where to put compressed data - * @return Number of bytes written. - * @throws IOException on stream error - */ - public static int putCompressedInt(final OutputStream out, final int value) throws IOException { - int i = 0; - int tmpvalue = value; - do { - byte b = (byte) (tmpvalue & VALUE_MASK); - tmpvalue >>>= NEXT_BIT_SHIFT; - if (tmpvalue != 0) { - b |= (byte) NEXT_BIT_MASK; - } - out.write(b); - i++; - } while (tmpvalue != 0); - return i; - } - -// /** -// * Put in output stream 32 bit integer (Big Endian byte order). -// * -// * @param out Where to put integer. -// * @param value Value of integer. -// * @throws IOException On stream error. -// */ -// public static void putInt(OutputStream out, final int value) throws IOException { -// // We have writeInt in ByteBufferOutputStream so that it can directly write -// // int to underlying -// // ByteBuffer in one step. -// if (out instanceof ByteBufferWriter) { -// ((ByteBufferWriter) out).writeInt(value); -// } else { -// StreamUtils.writeInt(out, value); -// } -// } - - public static byte toByte(final ByteBuffer buffer, final int offset) { - if (UNSAFE_AVAIL) { - return UnsafeAccess.toByte(buffer, offset); - } else { - return buffer.get(offset); - } - } - -// /** -// * Copy the data to the output stream and update position in buffer. -// * -// * @param out the stream to write bytes to -// * @param in the buffer to read bytes from -// * @param length the number of bytes to copy -// */ -// public static void moveBufferToStream(OutputStream out, ByteBuffer in, int length) -// throws IOException { -// copyBufferToStream(out, in, in.position(), length); -// skip(in, length); -// } - -// /** -// * Copy data from a buffer to an output stream. Does not update the position in the buffer. -// * -// * @param out the stream to write bytes to -// * @param in the buffer to read bytes from -// * @param offset the offset in the buffer (from the buffer's array offset) to start copying bytes -// * from -// * @param length the number of bytes to copy -// */ -// public static void copyBufferToStream(OutputStream out, ByteBuffer in, int offset, int length) -// throws IOException { -// if (out instanceof ByteBufferWriter) { -// ((ByteBufferWriter) out).write(in, offset, length); -// } else if (in.hasArray()) { -// out.write(in.array(), in.arrayOffset() + offset, length); -// } else { -// for (int i = 0; i < length; ++i) { -// out.write(toByte(in, offset + i)); -// } -// } -// } - -// /** -// * Copy data from a buffer to an output stream. Does not update the position in the buffer. -// * -// * @param out the output stream to write bytes to -// * @param in the buffer to read bytes from -// * @param offset the offset in the buffer (from the buffer's array offset) to start copying bytes -// * from -// * @param length the number of bytes to copy -// */ -// public static void copyBufferToStream(DataOutput out, ByteBuffer in, int offset, int length) -// throws IOException { -// if (out instanceof ByteBufferWriter) { -// ((ByteBufferWriter) out).write(in, offset, length); -// } else if (in.hasArray()) { -// out.write(in.array(), in.arrayOffset() + offset, length); -// } else { -// for (int i = 0; i < length; ++i) { -// out.write(toByte(in, offset + i)); -// } -// } -// } - - public static int putLong(final OutputStream out, final long value, final int fitInBytes) - throws IOException { - long tmpValue = value; - for (int i = 0; i < fitInBytes; ++i) { - out.write((byte) (tmpValue & 0xff)); - tmpValue >>>= 8; - } - return fitInBytes; - } - - public static int putByte(final ByteBuffer buffer, final int offset, final byte b) { - if (UNSAFE_AVAIL) { - return UnsafeAccess.putByte(buffer, offset, b); - } else { - buffer.put(offset, b); - return offset + 1; - } - } - - /** - * Check how many bytes are required to store value. - * - * @param value Value which size will be tested. - * @return How many bytes are required to store value. - */ - public static int longFitsIn(final long value) { - if (value < 0) { - return 8; - } - - if (value < (1L << (4 * 8))) { - // no more than 4 bytes - if (value < (1L << (2 * 8))) { - if (value < (1L << (1 * 8))) { - return 1; - } - return 2; - } - if (value < (1L << (3 * 8))) { - return 3; - } - return 4; - } - // more than 4 bytes - if (value < (1L << (6 * 8))) { - if (value < (1L << (5 * 8))) { - return 5; - } - return 6; - } - if (value < (1L << (7 * 8))) { - return 7; - } - return 8; - } - - /** - * Check how many bytes is required to store value. - * - * @param value Value which size will be tested. - * @return How many bytes are required to store value. - */ - public static int intFitsIn(final int value) { - if (value < 0) { - return 4; - } - - if (value < (1 << (2 * 8))) { - if (value < (1 << (1 * 8))) { - return 1; - } - return 2; - } - if (value <= (1 << (3 * 8))) { - return 3; - } - return 4; - } - - /** - * Read integer from stream coded in 7 bits and increment position. - * - * @return the integer that has been read - * @throws IOException on stream error - */ - public static int readCompressedInt(final InputStream input) throws IOException { - int result = 0; - int i = 0; - byte b; - do { - b = (byte) input.read(); - result += (b & VALUE_MASK) << (NEXT_BIT_SHIFT * i); - i++; - if (i > Bytes.SIZEOF_INT + 1) { - throw new IllegalStateException( - "Corrupted compressed int (too long: " + (i + 1) + " bytes)"); - } - } while (0 != (b & NEXT_BIT_MASK)); - return result; - } - - /** - * Read integer from buffer coded in 7 bits and increment position. - * - * @return Read integer. - */ - public static int readCompressedInt(final ByteBuffer buffer) { - final byte b = buffer.get(); - if ((b & NEXT_BIT_MASK) != 0) { - return (b & VALUE_MASK) + (readCompressedInt(buffer) << NEXT_BIT_SHIFT); - } - return b & VALUE_MASK; - } - - /** - * Read long which was written to fitInBytes bytes and increment position. - * - * @param fitInBytes In how many bytes given long is stored. - * @return The value of parsed long. - * @throws IOException on stream error - */ - public static long readLong(final InputStream in, final int fitInBytes) throws IOException { - long tmpLong = 0; - for (int i = 0; i < fitInBytes; ++i) { - tmpLong |= (in.read() & 0xffL) << (8 * i); - } - return tmpLong; - } - - /** - * Read long which was written to fitInBytes bytes and increment position. - * - * @param fitInBytes In how many bytes given long is stored. - * @return The value of parsed long. - */ - public static long readLong(final ByteBuffer in, final int fitInBytes) { - long tmpLength = 0; - for (int i = 0; i < fitInBytes; ++i) { - tmpLength |= (in.get() & 0xffL) << (8L * i); - } - return tmpLength; - } - - /** - * Copy the given number of bytes from the given stream and put it at the current position of the - * given buffer, updating the position in the buffer. - * - * @param out the buffer to write data to - * @param in the stream to read data from - * @param length the number of bytes to read/write - */ - public static void copyFromStreamToBuffer(final ByteBuffer out, final DataInputStream in, final int length) - throws IOException { - if (out.hasArray()) { - in.readFully(out.array(), out.position() + out.arrayOffset(), length); - skip(out, length); - } else { - for (int i = 0; i < length; ++i) { - out.put(in.readByte()); - } - } - } - - /** - * Copy from the InputStream to a new heap ByteBuffer until the InputStream is exhausted. - */ - public static ByteBuffer drainInputStreamToBuffer(final InputStream is) throws IOException { - final ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); - IOUtils.copyBytes(is, baos, 4096, true); - final ByteBuffer buffer = ByteBuffer.wrap(baos.toByteArray()); - buffer.rewind(); - return buffer; - } - - /** - * Copy one buffer's whole data to another. Write starts at the current position of 'out' buffer. - * Note : This will advance the position marker of {@code out} and also change the position maker - * for {@code in}. - * - * @param in source buffer - * @param out destination buffer - */ - public static void copyFromBufferToBuffer(final ByteBuffer in, final ByteBuffer out) { - if (in.hasArray() && out.hasArray()) { - final int length = in.remaining(); - System.arraycopy(in.array(), in.arrayOffset(), out.array(), out.arrayOffset(), length); - out.position(out.position() + length); - in.position(in.limit()); - } else if (UNSAFE_AVAIL) { - final int length = in.remaining(); - UnsafeAccess.copy(in, in.position(), out, out.position(), length); - out.position(out.position() + length); - in.position(in.limit()); - } else { - out.put(in); - } - } - - /** - * Copy from one buffer to another from given offset. This will be absolute positional copying and - * won't affect the position of any of the buffers. - * - * @param in input bytebuffer - * @param out destination bytebuffer - * @param sourceOffset offset of source buffer - * @param destinationOffset offset of destination buffer - * @param length the number of bytes to copy - */ - public static void copyFromBufferToBuffer(final ByteBuffer in, final ByteBuffer out, final int sourceOffset, - final int destinationOffset, final int length) { - if (in.hasArray() && out.hasArray()) { - System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out.array(), - out.arrayOffset() + destinationOffset, length); - } else if (UNSAFE_AVAIL) { - UnsafeAccess.copy(in, sourceOffset, out, destinationOffset, length); - } else { - final ByteBuffer outDup = out.duplicate(); - outDup.position(destinationOffset); - final ByteBuffer inDup = in.duplicate(); - inDup.position(sourceOffset).limit(sourceOffset + length); - outDup.put(inDup); - } - // We used to return a result but disabled; return destinationOffset + length; - } - - /** - * Copy from one buffer to another from given offset. - *

- * Note : This will advance the position marker of {@code out} but not change the position maker - * for {@code in} - * - * @param in source buffer - * @param out destination buffer - * @param sourceOffset offset in the source buffer - * @param length how many bytes to copy - */ - public static void copyFromBufferToBuffer(final ByteBuffer in, final ByteBuffer out, final int sourceOffset, - final int length) { - if (in.hasArray() && out.hasArray()) { - System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out.array(), - out.position() + out.arrayOffset(), length); - skip(out, length); - } else if (UNSAFE_AVAIL) { - UnsafeAccess.copy(in, sourceOffset, out, out.position(), length); - skip(out, length); - } else { - final ByteBuffer inDup = in.duplicate(); - inDup.position(sourceOffset).limit(sourceOffset + length); - out.put(inDup); - } - } - - /** - * Find length of common prefix in two arrays. - * - * @param left Array to be compared. - * @param leftOffset Offset in left array. - * @param leftLength Length of left array. - * @param right Array to be compared. - * @param rightOffset Offset in right array. - * @param rightLength Length of right array. - */ - public static int findCommonPrefix(final byte[] left, final int leftOffset, final int leftLength, final byte[] right, - final int rightOffset, final int rightLength) { - return Bytes.findCommonPrefix(left, right, leftLength, rightLength, leftOffset, rightOffset); - } - - /** - * Find length of common prefix in two arrays. - * - * @param left ByteBuffer to be compared. - * @param leftOffset Offset in left ByteBuffer. - * @param leftLength Length of left ByteBuffer. - * @param right ByteBuffer to be compared. - * @param rightOffset Offset in right ByteBuffer. - * @param rightLength Length of right ByteBuffer. - */ - public static int findCommonPrefix(final ByteBuffer left, final int leftOffset, final int leftLength, - final ByteBuffer right, final int rightOffset, final int rightLength) { - return CommonPrefixerHolder.BEST_COMMON_PREFIXER.findCommonPrefix(left, leftOffset, leftLength, - right, rightOffset, rightLength); - } - - /** - * Find length of common prefix in two arrays. - * - * @param left ByteBuffer to be compared. - * @param leftOffset Offset in left ByteBuffer. - * @param leftLength Length of left ByteBuffer. - * @param right Array to be compared - * @param rightOffset Offset in right Array. - * @param rightLength Length of right Array. - */ - public static int findCommonPrefix(final ByteBuffer left, final int leftOffset, final int leftLength, final byte[] right, - final int rightOffset, final int rightLength) { - return CommonPrefixerHolder.BEST_COMMON_PREFIXER.findCommonPrefix(left, leftOffset, leftLength, - right, rightOffset, rightLength); - } - - /** - * Check whether two parts in the same buffer are equal. - * - * @param buffer In which buffer there are parts - * @param offsetLeft Beginning of first part. - * @param lengthLeft Length of the first part. - * @param offsetRight Beginning of the second part. - * @param lengthRight Length of the second part. - * @return True if equal - */ - public static boolean arePartsEqual(final ByteBuffer buffer, final int offsetLeft, final int lengthLeft, - final int offsetRight, final int lengthRight) { - if (lengthLeft != lengthRight) { - return false; - } - - if (buffer.hasArray()) { - return 0 == Bytes.compareTo(buffer.array(), buffer.arrayOffset() + offsetLeft, lengthLeft, - buffer.array(), buffer.arrayOffset() + offsetRight, lengthRight); - } - - for (int i = 0; i < lengthRight; ++i) { - if (buffer.get(offsetLeft + i) != buffer.get(offsetRight + i)) { - return false; - } - } - return true; - } - - /** - * Increment position in buffer. - * - * @param buffer In this buffer. - * @param length By that many bytes. - */ - public static void skip(final ByteBuffer buffer, final int length) { - buffer.position(buffer.position() + length); - } - - public static void extendLimit(final ByteBuffer buffer, final int numBytes) { - buffer.limit(buffer.limit() + numBytes); - } - - /** - * Copy the bytes from position to limit into a new byte[] of the exact length and sets the - * position and limit back to their original values (though not thread safe). - * - * @param buffer copy from here - * @param startPosition put buffer.get(startPosition) into byte[0] - * @return a new byte[] containing the bytes in the specified range - */ - public static byte[] toBytes(final ByteBuffer buffer, final int startPosition) { - final int originalPosition = buffer.position(); - final byte[] output = new byte[buffer.limit() - startPosition]; - buffer.position(startPosition); - buffer.get(output); - buffer.position(originalPosition); - return output; - } - - /** - * Copy the given number of bytes from specified offset into a new byte[] - * - * @param buffer input bytebuffer to read - * @param offset input offset where Bytes are - * @param length the number of bytes to read - * @return a new byte[] containing the bytes in the specified range - */ - public static byte[] toBytes(final ByteBuffer buffer, final int offset, final int length) { - final byte[] output = new byte[length]; - for (int i = 0; i < length; i++) { - output[i] = buffer.get(offset + i); - } - return output; - } - - public static boolean equals(final ByteBuffer buf1, final int o1, final int l1, final ByteBuffer buf2, final int o2, final int l2) { - if ((l1 == 0) || (l2 == 0)) { - // both 0 length, return true, or else false - return l1 == l2; - } - // Since we're often comparing adjacent sorted data, - // it's usual to have equal arrays except for the very last byte - // so check that first - if (toByte(buf1, o1 + l1 - 1) != toByte(buf2, o2 + l2 - 1)) { - return false; - } - return compareTo(buf1, o1, l1, buf2, o2, l2) == 0; - } - - /** - * ByteBuffer to hash offset to start from length to hash - */ - public static int hashCode(final ByteBuffer buf, final int offset, final int length) { - int hash = 1; - for (int i = offset; i < offset + length; i++) { - hash = (31 * hash) + (int) toByte(buf, i); - } - return hash; - } - - public static int compareTo(final ByteBuffer buf1, final int o1, final int l1, final ByteBuffer buf2, final int o2, final int l2) { - return ComparerHolder.BEST_COMPARER.compareTo(buf1, o1, l1, buf2, o2, l2); - } - - public static boolean equals(final ByteBuffer buf1, final int o1, final int l1, final byte[] buf2, final int o2, final int l2) { - if ((l1 == 0) || (l2 == 0)) { - // both 0 length, return true, or else false - return l1 == l2; - } - // Since we're often comparing adjacent sorted data, - // it's usual to have equal arrays except for the very last byte - // so check that first - if (toByte(buf1, o1 + l1 - 1) != buf2[o2 + l2 - 1]) { - return false; - } - return compareTo(buf1, o1, l1, buf2, o2, l2) == 0; - } - - // The below two methods show up in lots of places. Versions of them in commons util and in - // Cassandra. In guava too? They are copied from ByteBufferUtils. They are here as static - // privates. Seems to make code smaller and make Hotspot happier (comes of compares and study - // of compiled code via jitwatch). - - public static int compareTo(final byte[] buf1, final int o1, final int l1, final ByteBuffer buf2, final int o2, final int l2) { - return ComparerHolder.BEST_COMPARER.compareTo(buf1, o1, l1, buf2, o2, l2); - } - - public static int compareTo(final ByteBuffer buf1, final int o1, final int l1, final byte[] buf2, final int o2, final int l2) { - return compareTo(buf2, o2, l2, buf1, o1, l1) * -1; - } - - static int compareToUnsafe(final Object obj1, final long o1, final int l1, final Object obj2, final long o2, final int l2) { - final int stride = 8; - final int minLength = Math.min(l1, l2); - final int strideLimit = minLength & ~(stride - 1); - int i; - - /* - * Compare 8 bytes at a time. Benchmarking shows comparing 8 bytes at a time is no slower than - * comparing 4 bytes at a time even on 32-bit. On the other hand, it is substantially faster on - * 64-bit. - */ - for (i = 0; i < strideLimit; i += stride) { - final long lw = HBasePlatformDependent.getLong(obj1, o1 + (long) i); - final long rw = HBasePlatformDependent.getLong(obj2, o2 + (long) i); - if (lw != rw) { - if (!UnsafeAccess.LITTLE_ENDIAN) { - return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) - ? -1 - : 1; - } - - /* - * We want to compare only the first index where left[index] != right[index]. This - * corresponds to the least significant nonzero byte in lw ^ rw, since lw and rw are - * little-endian. Long.numberOfTrailingZeros(diff) tells us the least significant nonzero - * bit, and zeroing out the first three bits of L.nTZ gives us the shift to get that least - * significant nonzero byte. This comparison logic is based on UnsignedBytes from guava v21 - */ - final int n = Long.numberOfTrailingZeros(lw ^ rw) & ~0x7; - return ((int) ((lw >>> n) & 0xFF)) - ((int) ((rw >>> n) & 0xFF)); - } - } - - // The epilogue to cover the last (minLength % stride) elements. - for (; i < minLength; i++) { - final int il = (HBasePlatformDependent.getByte(obj1, o1 + i) & 0xFF); - final int ir = (HBasePlatformDependent.getByte(obj2, o2 + i) & 0xFF); - if (il != ir) { - return il - ir; - } - } - return l1 - l2; - } - - static int findCommonPrefixUnsafe(final Object left, final long leftOffset, final int leftLength, final Object right, - final long rightOffset, final int rightLength) { - final int stride = 8; - final int minLength = Math.min(leftLength, rightLength); - final int strideLimit = minLength & ~(stride - 1); - int result = 0; - int i; - - for (i = 0; i < strideLimit; i += stride) { - final long lw = HBasePlatformDependent.getLong(left, leftOffset + (long) i); - final long rw = HBasePlatformDependent.getLong(right, rightOffset + (long) i); - - if (lw != rw) { - if (!UnsafeAccess.LITTLE_ENDIAN) { - return result + (Long.numberOfLeadingZeros(lw ^ rw) / Bytes.SIZEOF_LONG); - } else { - return result + (Long.numberOfTrailingZeros(lw ^ rw) / Bytes.SIZEOF_LONG); - } - } else { - result += Bytes.SIZEOF_LONG; - } - } - - // The epilogue to cover the last (minLength % stride) elements. - for (; i < minLength; i++) { - final byte il = HBasePlatformDependent.getByte(left, leftOffset + i); - final byte ir = HBasePlatformDependent.getByte(right, rightOffset + i); - if (il != ir) { - return result; - } else { - result++; - } - } - - return result; - } - - /** - * Reads a short value at the given buffer's offset. - * - * @param buffer input byte buffer to read - * @param offset input offset where short is - * @return short value at offset - */ - public static short toShort(final ByteBuffer buffer, final int offset) { - return ConverterHolder.BEST_CONVERTER.toShort(buffer, offset); - } - - /** - * Reads an int value at the given buffer's current position. Also advances the buffer's position - */ - public static int toInt(final ByteBuffer buffer) { - return ConverterHolder.BEST_CONVERTER.toInt(buffer); - } - - /** - * Reads an int value at the given buffer's offset. - * - * @param buffer input byte buffer to read - * @param offset input offset where int is - * @return int value at offset - */ - public static int toInt(final ByteBuffer buffer, final int offset) { - return ConverterHolder.BEST_CONVERTER.toInt(buffer, offset); - } - - /** - * Converts a ByteBuffer to an int value - * - * @param buf The ByteBuffer - * @param offset Offset to int value - * @param length Number of bytes used to store the int value. - * @return the int value if there's not enough bytes left in the buffer after the given offset - */ - public static int readAsInt(final ByteBuffer buf, final int offset, final int length) { - if (offset + length > buf.limit()) { - throw new IllegalArgumentException("offset (" + offset + ") + length (" + length - + ") exceed the" + " limit of the buffer: " + buf.limit()); - } - int n = 0; - for (int i = offset; i < (offset + length); i++) { - n <<= 8; - n ^= toByte(buf, i) & 0xFF; - } - return n; - } - - /** - * Reads a long value at the given buffer's offset. - * - * @param buffer input byte buffer to read - * @param offset input offset where Long is - * @return long value at offset - */ - public static long toLong(final ByteBuffer buffer, final int offset) { - return ConverterHolder.BEST_CONVERTER.toLong(buffer, offset); - } - - /** - * Put an int value out to the given ByteBuffer's current position in big-endian format. This also - * advances the position in buffer by int size. - * - * @param buffer the ByteBuffer to write to - * @param val int to write out - */ - public static void putInt(final ByteBuffer buffer, final int val) { - ConverterHolder.BEST_CONVERTER.putInt(buffer, val); - } - - public static int putInt(final ByteBuffer buffer, final int index, final int val) { - return ConverterHolder.BEST_CONVERTER.putInt(buffer, index, val); - } - - /** - * Reads a double value at the given buffer's offset. - * - * @param buffer input byte buffer to read - * @param offset offset where double is - * @return double value at offset - */ - public static double toDouble(final ByteBuffer buffer, final int offset) { - return Double.longBitsToDouble(toLong(buffer, offset)); - } - - /** - * Reads a BigDecimal value at the given buffer's offset. - * - * @param buffer input bytebuffer to read - * @param offset input offset - * @return BigDecimal value at offset - */ - public static BigDecimal toBigDecimal(final ByteBuffer buffer, final int offset, final int length) { - if (buffer == null || length < Bytes.SIZEOF_INT + 1 || (offset + length > buffer.limit())) { - return null; - } - - final int scale = toInt(buffer, offset); - final byte[] tcBytes = new byte[length - Bytes.SIZEOF_INT]; - copyFromBufferToArray(tcBytes, buffer, offset + Bytes.SIZEOF_INT, 0, length - Bytes.SIZEOF_INT); - return new BigDecimal(new BigInteger(tcBytes), scale); - } - - /** - * Put a short value out to the given ByteBuffer's current position in big-endian format. This - * also advances the position in buffer by short size. - * - * @param buffer the ByteBuffer to write to - * @param val short to write out - */ - public static void putShort(final ByteBuffer buffer, final short val) { - ConverterHolder.BEST_CONVERTER.putShort(buffer, val); - } - - public static int putShort(final ByteBuffer buffer, final int index, final short val) { - return ConverterHolder.BEST_CONVERTER.putShort(buffer, index, val); - } - - public static int putAsShort(final ByteBuffer buf, final int index, int val) { - buf.put(index + 1, (byte) val); - val >>= 8; - buf.put(index, (byte) val); - return index + Bytes.SIZEOF_SHORT; - } - - /** - * Put a long value out to the given ByteBuffer's current position in big-endian format. This also - * advances the position in buffer by long size. - * - * @param buffer the ByteBuffer to write to - * @param val long to write out - */ - public static void putLong(final ByteBuffer buffer, final long val) { - ConverterHolder.BEST_CONVERTER.putLong(buffer, val); - } - - public static int putLong(final ByteBuffer buffer, final int index, final long val) { - return ConverterHolder.BEST_CONVERTER.putLong(buffer, index, val); - } - - /** - * Copies the bytes from given array's offset to length part into the given buffer. Puts the bytes - * to buffer's current position. This also advances the position in the 'out' buffer by 'length' - * - * @param out output bytebuffer to copy to - * @param in input array to copy from - * @param inOffset input offset to copy from - * @param length the number of bytes to copy - */ - public static void copyFromArrayToBuffer(final ByteBuffer out, final byte[] in, final int inOffset, final int length) { - if (out.hasArray()) { - System.arraycopy(in, inOffset, out.array(), out.arrayOffset() + out.position(), length); - // Move the position in out by length - out.position(out.position() + length); - } else if (UNSAFE_AVAIL) { - UnsafeAccess.copy(in, inOffset, out, out.position(), length); - // Move the position in out by length - out.position(out.position() + length); - } else { - out.put(in, inOffset, length); - } - } - - /** - * Copies bytes from given array's offset to length part into the given buffer. Puts the bytes to - * buffer's given position. This doesn't affect the position of buffer. - * - * @param out output bytebuffer to copy to - * @param outOffset output buffer offset - * @param in input array to copy from - * @param inOffset input offset to copy from - * @param length the number of bytes to copy - */ - public static void copyFromArrayToBuffer(final ByteBuffer out, final int outOffset, final byte[] in, final int inOffset, - final int length) { - if (out.hasArray()) { - System.arraycopy(in, inOffset, out.array(), out.arrayOffset() + outOffset, length); - } else if (UNSAFE_AVAIL) { - UnsafeAccess.copy(in, inOffset, out, outOffset, length); - } else { - final ByteBuffer outDup = out.duplicate(); - outDup.position(outOffset); - outDup.put(in, inOffset, length); - } - } - - /** - * Copies specified number of bytes from given offset of 'in' ByteBuffer to the array. This - * doesn't affect the position of buffer. - * - * @param out output array to copy input bytebuffer to - * @param in input bytebuffer to copy from - * @param sourceOffset offset of source bytebuffer - * @param destinationOffset offset of destination array - * @param length the number of bytes to copy - */ - public static void copyFromBufferToArray(final byte[] out, final ByteBuffer in, final int sourceOffset, - final int destinationOffset, final int length) { - if (in.hasArray()) { - System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out, destinationOffset, length); - } else if (UNSAFE_AVAIL) { - UnsafeAccess.copy(in, sourceOffset, out, destinationOffset, length); - } else { - final ByteBuffer inDup = in.duplicate(); - inDup.position(sourceOffset); - inDup.get(out, destinationOffset, length); - } - } - - /** - * Similar to {@link Arrays#copyOfRange(byte[], int, int)} - * - * @param original the buffer from which the copy has to happen - * @param from the starting index - * @param to the ending index - * @return a byte[] created out of the copy - */ - public static byte[] copyOfRange(final ByteBuffer original, final int from, final int to) { - final int newLength = to - from; - if (newLength < 0) { - throw new IllegalArgumentException(from + " > " + to); - } - final byte[] copy = new byte[newLength]; - ByteBufferUtils.copyFromBufferToArray(copy, original, from, 0, newLength); - return copy; - } - - // For testing purpose - public static String toStringBinary(final ByteBuffer b, final int off, int len) { - final StringBuilder result = new StringBuilder(); - // Just in case we are passed a 'len' that is > buffer length... - if (off >= b.capacity()) { - return result.toString(); - } - if (off + len > b.capacity()) { - len = b.capacity() - off; - } - for (int i = off; i < off + len; ++i) { - final int ch = b.get(i) & 0xFF; - if ( - (ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') - || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0 - ) { - result.append((char) ch); - } else { - result.append(String.format("\\x%02X", ch)); - } - } - return result.toString(); - } - - public static String toStringBinary(final ByteBuffer b) { - return toStringBinary(b, 0, b.capacity()); - } - - /** - * Find index of passed delimiter. - * - * @return Index of delimiter having started from start of b moving rightward. - */ - public static int searchDelimiterIndex(final ByteBuffer b, final int offset, final int length, - final int delimiter) { - for (int i = offset, n = offset + length; i < n; i++) { - if (b.get(i) == delimiter) { - return i; - } - } - return -1; - } - - /** - * Find index of passed delimiter walking from end of buffer backwards. - * - * @return Index of delimiter - */ - public static int searchDelimiterIndexInReverse(final ByteBuffer b, final int offset, final int length, - final int delimiter) { - for (int i = offset + length - 1; i >= offset; i--) { - if (b.get(i) == delimiter) { - return i; - } - } - return -1; - } -} diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/Bytes.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/Bytes.java deleted file mode 100644 index 0f7043595a0..00000000000 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/Bytes.java +++ /dev/null @@ -1,2743 +0,0 @@ -/* - * Copy of https://github.com/apache/hbase/blob/master/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java - * to avoid having to pull in all of hbase to use the util methods. - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stroom.bytebuffer.hbase; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.io.RawComparator; -import org.apache.hadoop.io.WritableComparator; -import org.apache.hadoop.io.WritableUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.Random; - - -/** - * Utility class that handles byte arrays, conversions to/from other types, comparisons, hash code - * generation, manufacturing keys for HashMaps or HashSets, and can be used as key in maps or trees. - */ -@SuppressWarnings("MixedMutabilityReturnType") -public class Bytes implements Comparable { - - // Using the charset canonical name for String/byte[] conversions is much - // more efficient due to use of cached encoders/decoders. - private static final String UTF8_CSN = StandardCharsets.UTF_8.name(); - - // HConstants.EMPTY_BYTE_ARRAY should be updated if this changed - private static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; - - private static final Logger LOG = LoggerFactory.getLogger(Bytes.class); - - /** - * Size of boolean in bytes - */ - public static final int SIZEOF_BOOLEAN = Byte.SIZE / Byte.SIZE; - - /** - * Size of byte in bytes - */ - public static final int SIZEOF_BYTE = SIZEOF_BOOLEAN; - - /** - * Size of char in bytes - */ - public static final int SIZEOF_CHAR = Character.SIZE / Byte.SIZE; - - /** - * Size of double in bytes - */ - public static final int SIZEOF_DOUBLE = Double.SIZE / Byte.SIZE; - - /** - * Size of float in bytes - */ - public static final int SIZEOF_FLOAT = Float.SIZE / Byte.SIZE; - - /** - * Size of int in bytes - */ - public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; - - /** - * Size of long in bytes - */ - public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; - - /** - * Size of short in bytes - */ - public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; - - /** - * Mask to apply to a long to reveal the lower int only. Use like this: int i = - * (int)(0xFFFFFFFF00000000L ^ some_long_value); - */ - public static final long MASK_FOR_LOWER_INT_IN_LONG = 0xFFFFFFFF00000000L; - - /** - * Estimate of size cost to pay beyond payload in jvm for instance of byte []. Estimate based on - * study of jhat and jprofiler numbers. - */ - // JHat says BU is 56 bytes. - // SizeOf which uses java.lang.instrument says 24 bytes. (3 longs?) - public static final int ESTIMATED_HEAP_TAX = 16; - - static final boolean UNSAFE_UNALIGNED = HBasePlatformDependent.unaligned(); - - /** - * Returns length of the byte array, returning 0 if the array is null. Useful for calculating - * sizes. - * - * @param b byte array, which can be null - * @return 0 if b is null, otherwise returns length - */ - final public static int len(final byte[] b) { - return b == null - ? 0 - : b.length; - } - - private byte[] bytes; - private int offset; - private int length; - - /** - * Create a zero-size sequence. - */ - public Bytes() { - super(); - } - - /** - * Create a Bytes using the byte array as the initial value. - * - * @param bytes This array becomes the backing storage for the object. - */ - public Bytes(final byte[] bytes) { - this(bytes, 0, bytes.length); - } - - /** - * Set the new Bytes to the contents of the passed ibw. - * - * @param ibw the value to set this Bytes to. - */ - public Bytes(final Bytes ibw) { - this(ibw.get(), ibw.getOffset(), ibw.getLength()); - } - - /** - * Set the value to a given byte range - * - * @param bytes the new byte range to set to - * @param offset the offset in newData to start at - * @param length the number of bytes in the range - */ - public Bytes(final byte[] bytes, final int offset, final int length) { - this.bytes = bytes; - this.offset = offset; - this.length = length; - } - - /** - * Get the data from the Bytes. - * - * @return The data is only valid between offset and offset+length. - */ - public byte[] get() { - if (this.bytes == null) { - throw new IllegalStateException( - "Uninitialiized. Null constructor " + "called w/o accompaying readFields invocation"); - } - return this.bytes; - } - - /** - * Use passed bytes as backing array for this instance. - */ - public void set(final byte[] b) { - set(b, 0, b.length); - } - - /** - * Use passed bytes as backing array for this instance. - */ - public void set(final byte[] b, final int offset, final int length) { - this.bytes = b; - this.offset = offset; - this.length = length; - } - - /** - * Returns the number of valid bytes in the buffer - */ - public int getLength() { - if (this.bytes == null) { - throw new IllegalStateException( - "Uninitialiized. Null constructor " + "called w/o accompaying readFields invocation"); - } - return this.length; - } - - /** - * Return the offset into the buffer. - */ - public int getOffset() { - return this.offset; - } - - @Override - public int hashCode() { - return Bytes.hashCode(bytes, offset, length); - } - - /** - * Define the sort order of the Bytes. - * - * @param that The other bytes writable - * @return Positive if left is bigger than right, 0 if they are equal, and negative if left is - * smaller than right. - */ - @Override - public int compareTo(final Bytes that) { - return BYTES_RAWCOMPARATOR.compare(this.bytes, this.offset, this.length, that.bytes, - that.offset, that.length); - } - - /** - * Compares the bytes in this object to the specified byte array - * - * @return Positive if left is bigger than right, 0 if they are equal, and negative if left is - * smaller than right. - */ - public int compareTo(final byte[] that) { - return BYTES_RAWCOMPARATOR.compare(this.bytes, this.offset, this.length, that, 0, that.length); - } - - @Override - public boolean equals(final Object right_obj) { - if (right_obj instanceof byte[]) { - return compareTo((byte[]) right_obj) == 0; - } - if (right_obj instanceof Bytes) { - return compareTo((Bytes) right_obj) == 0; - } - return false; - } - - @Override - public String toString() { - return Bytes.toString(bytes, offset, length); - } - - /** - * Convert a list of byte[] to an array - * - * @param array List of byte []. - * @return Array of byte []. - */ - public static byte[][] toArray(final List array) { - // List#toArray doesn't work on lists of byte []. - final byte[][] results = new byte[array.size()][]; - for (int i = 0; i < array.size(); i++) { - results[i] = array.get(i); - } - return results; - } - - /** - * Returns a copy of the bytes referred to by this writable - */ - public byte[] copyBytes() { - return Arrays.copyOfRange(bytes, offset, offset + length); - } - - /** - * Byte array comparator class. - */ - public static class ByteArrayComparator implements RawComparator { - - public ByteArrayComparator() { - super(); - } - - @Override - public int compare(final byte[] left, final byte[] right) { - return compareTo(left, right); - } - - @Override - public int compare(final byte[] b1, final int s1, final int l1, final byte[] b2, final int s2, final int l2) { - return LexicographicalComparerHolder.BEST_COMPARER.compareTo(b1, s1, l1, b2, s2, l2); - } - } - - /** - * A {@link ByteArrayComparator} that treats the empty array as the largest value. This is useful - * for comparing row end keys for regions. - */ - // TODO: unfortunately, HBase uses byte[0] as both start and end keys for region - // boundaries. Thus semantically, we should treat empty byte array as the smallest value - // while comparing row keys, start keys etc; but as the largest value for comparing - // region boundaries for endKeys. - public static class RowEndKeyComparator extends ByteArrayComparator { - - @Override - public int compare(final byte[] left, final byte[] right) { - return compare(left, 0, left.length, right, 0, right.length); - } - - @Override - public int compare(final byte[] b1, final int s1, final int l1, final byte[] b2, final int s2, final int l2) { - if (b1 == b2 && s1 == s2 && l1 == l2) { - return 0; - } - if (l1 == 0) { - return l2; // 0 or positive - } - if (l2 == 0) { - return -1; - } - return super.compare(b1, s1, l1, b2, s2, l2); - } - } - - /** - * Pass this to TreeMaps where byte [] are keys. - */ - public final static Comparator BYTES_COMPARATOR = new ByteArrayComparator(); - - /** - * Use comparing byte arrays, byte-by-byte - */ - public final static RawComparator BYTES_RAWCOMPARATOR = new ByteArrayComparator(); - - /** - * Read byte-array written with a WritableableUtils.vint prefix. - * - * @param in Input to read from. - * @return byte array read off in - * @throws IOException e - */ - public static byte[] readByteArray(final DataInput in) throws IOException { - final int len = WritableUtils.readVInt(in); - if (len < 0) { - throw new NegativeArraySizeException(Integer.toString(len)); - } - final byte[] result = new byte[len]; - in.readFully(result, 0, len); - return result; - } - - /** - * Read byte-array written with a WritableableUtils.vint prefix. IOException is converted to a - * RuntimeException. - * - * @param in Input to read from. - * @return byte array read off in - */ - public static byte[] readByteArrayThrowsRuntime(final DataInput in) { - try { - return readByteArray(in); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Write byte-array with a WritableableUtils.vint prefix. - * - * @param out output stream to be written to - * @param b array to write - * @throws IOException e - */ - public static void writeByteArray(final DataOutput out, final byte[] b) throws IOException { - if (b == null) { - WritableUtils.writeVInt(out, 0); - } else { - writeByteArray(out, b, 0, b.length); - } - } - - /** - * Write byte-array to out with a vint length prefix. - * - * @param out output stream - * @param b array - * @param offset offset into array - * @param length length past offset - * @throws IOException e - */ - public static void writeByteArray(final DataOutput out, final byte[] b, final int offset, - final int length) throws IOException { - WritableUtils.writeVInt(out, length); - out.write(b, offset, length); - } - - /** - * Write byte-array from src to tgt with a vint length prefix. - * - * @param tgt target array - * @param tgtOffset offset into target array - * @param src source array - * @param srcOffset source offset - * @param srcLength source length - * @return New offset in src array. - */ - public static int writeByteArray(final byte[] tgt, final int tgtOffset, final byte[] src, - final int srcOffset, final int srcLength) { - final byte[] vint = vintToBytes(srcLength); - System.arraycopy(vint, 0, tgt, tgtOffset, vint.length); - final int offset = tgtOffset + vint.length; - System.arraycopy(src, srcOffset, tgt, offset, srcLength); - return offset + srcLength; - } - - /** - * Put bytes at the specified byte array position. - * - * @param tgtBytes the byte array - * @param tgtOffset position in the array - * @param srcBytes array to write out - * @param srcOffset source offset - * @param srcLength source length - * @return incremented offset - */ - public static int putBytes(final byte[] tgtBytes, final int tgtOffset, final byte[] srcBytes, final int srcOffset, - final int srcLength) { - System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); - return tgtOffset + srcLength; - } - - /** - * Write a single byte out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param b byte to write out - * @return incremented offset - */ - public static int putByte(final byte[] bytes, final int offset, final byte b) { - bytes[offset] = b; - return offset + 1; - } - - /** - * Add the whole content of the ByteBuffer to the bytes arrays. The ByteBuffer is modified. - * - * @param bytes the byte array - * @param offset position in the array - * @param buf ByteBuffer to write out - * @return incremented offset - */ - public static int putByteBuffer(final byte[] bytes, final int offset, final ByteBuffer buf) { - final int len = buf.remaining(); - buf.get(bytes, offset, len); - return offset + len; - } - - /** - * Returns a new byte array, copied from the given {@code buf}, from the index 0 (inclusive) to - * the limit (exclusive), regardless of the current position. The position and the other index - * parameters are not changed. - * - * @param buf a byte buffer - * @return the byte array - * @see #getBytes(ByteBuffer) - */ - public static byte[] toBytes(final ByteBuffer buf) { - final ByteBuffer dup = buf.duplicate(); - dup.position(0); - return readBytes(dup); - } - - private static byte[] readBytes(final ByteBuffer buf) { - final byte[] result = new byte[buf.remaining()]; - buf.get(result); - return result; - } - - /** - * Convert a byte[] into a string. Charset is assumed to be UTF-8. - * - * @param b Presumed UTF-8 encoded byte array. - * @return String made from b - */ - public static String toString(final byte[] b) { - if (b == null) { - return null; - } - return toString(b, 0, b.length); - } - - /** - * Joins two byte arrays together using a separator. - * - * @param b1 The first byte array. - * @param sep The separator to use. - * @param b2 The second byte array. - */ - public static String toString(final byte[] b1, final String sep, final byte[] b2) { - return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); - } - - /** - * This method will convert utf8 encoded bytes into a string. If the given byte array is null, - * this method will return null. - * - * @param b Presumed UTF-8 encoded byte array. - * @param off offset into array - * @return String made from b or null - */ - public static String toString(final byte[] b, final int off) { - if (b == null) { - return null; - } - final int len = b.length - off; - if (len <= 0) { - return ""; - } - try { - return new String(b, off, len, UTF8_CSN); - } catch (final UnsupportedEncodingException e) { - // should never happen! - throw new IllegalArgumentException("UTF8 encoding is not supported", e); - } - } - - /** - * This method will convert utf8 encoded bytes into a string. If the given byte array is null, - * this method will return null. - * - * @param b Presumed UTF-8 encoded byte array. - * @param off offset into array - * @param len length of utf-8 sequence - * @return String made from b or null - */ - public static String toString(final byte[] b, final int off, final int len) { - if (b == null) { - return null; - } - if (len == 0) { - return ""; - } - try { - return new String(b, off, len, UTF8_CSN); - } catch (final UnsupportedEncodingException e) { - // should never happen! - throw new IllegalArgumentException("UTF8 encoding is not supported", e); - } - } - - /** - * Write a printable representation of a byte array. - * - * @param b byte array - * @see #toStringBinary(byte[], int, int) - */ - public static String toStringBinary(final byte[] b) { - if (b == null) { - return "null"; - } - return toStringBinary(b, 0, b.length); - } - - /** - * Converts the given byte buffer to a printable representation, from the index 0 (inclusive) to - * the limit (exclusive), regardless of the current position. The position and the other index - * parameters are not changed. - * - * @param buf a byte buffer - * @return a string representation of the buffer's binary contents - * @see #toBytes(ByteBuffer) - * @see #getBytes(ByteBuffer) - */ - public static String toStringBinary(final ByteBuffer buf) { - if (buf == null) { - return "null"; - } - if (buf.hasArray()) { - return toStringBinary(buf.array(), buf.arrayOffset(), buf.limit()); - } - return toStringBinary(toBytes(buf)); - } - - private static final char[] HEX_CHARS_UPPER = - {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; - - /** - * Write a printable representation of a byte array. Non-printable characters are hex escaped in - * the format \\x%02X, eg: \x00 \x05 etc - * - * @param b array to write out - * @param off offset to start at - * @param len length to write - * @return string output - */ - public static String toStringBinary(final byte[] b, final int off, int len) { - final StringBuilder result = new StringBuilder(); - // Just in case we are passed a 'len' that is > buffer length... - if (off >= b.length) { - return result.toString(); - } - if (off + len > b.length) { - len = b.length - off; - } - for (int i = off; i < off + len; ++i) { - final int ch = b[i] & 0xFF; - if (ch >= ' ' && ch <= '~' && ch != '\\') { - result.append((char) ch); - } else { - result.append("\\x"); - result.append(HEX_CHARS_UPPER[ch / 0x10]); - result.append(HEX_CHARS_UPPER[ch % 0x10]); - } - } - return result.toString(); - } - - private static boolean isHexDigit(final char c) { - return (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9'); - } - - /** - * Takes a ASCII digit in the range A-F0-9 and returns the corresponding integer/ordinal value. - * - * @param ch The hex digit. - * @return The converted hex value as a byte. - */ - public static byte toBinaryFromHex(final byte ch) { - if (ch >= 'A' && ch <= 'F') { - return (byte) ((byte) 10 + (byte) (ch - 'A')); - } - // else - return (byte) (ch - '0'); - } - - public static byte[] toBytesBinary(final String in) { - // this may be bigger than we need, but let's be safe. - final byte[] b = new byte[in.length()]; - int size = 0; - for (int i = 0; i < in.length(); ++i) { - final char ch = in.charAt(i); - if (ch == '\\' && in.length() > i + 1 && in.charAt(i + 1) == 'x') { - // ok, take next 2 hex digits. - final char hd1 = in.charAt(i + 2); - final char hd2 = in.charAt(i + 3); - - // they need to be A-F0-9: - if (!isHexDigit(hd1) || !isHexDigit(hd2)) { - // bogus escape code, ignore: - continue; - } - // turn hex ASCII digit -> number - final byte d = (byte) ((toBinaryFromHex((byte) hd1) << 4) + toBinaryFromHex((byte) hd2)); - - b[size++] = d; - i += 3; // skip 3 - } else { - b[size++] = (byte) ch; - } - } - // resize: - final byte[] b2 = new byte[size]; - System.arraycopy(b, 0, b2, 0, size); - return b2; - } - - /** - * Converts a string to a UTF-8 byte array. - * - * @param s string - * @return the byte array - */ - public static byte[] toBytes(final String s) { - try { - return s.getBytes(UTF8_CSN); - } catch (final UnsupportedEncodingException e) { - // should never happen! - throw new IllegalArgumentException("UTF8 decoding is not supported", e); - } - } - - /** - * Convert a boolean to a byte array. True becomes -1 and false becomes 0. - * - * @param b value - * @return b encoded in a byte array. - */ - public static byte[] toBytes(final boolean b) { - return new byte[]{ - b - ? (byte) -1 - : (byte) 0}; - } - - /** - * Reverses {@link #toBytes(boolean)} - * - * @param b array - * @return True or false. - */ - public static boolean toBoolean(final byte[] b) { - if (b.length != 1) { - throw new IllegalArgumentException("Array has wrong size: " + b.length); - } - return b[0] != (byte) 0; - } - - /** - * Convert a long value to a byte array using big-endian. - * - * @param val value to convert - * @return the byte array - */ - public static byte[] toBytes(long val) { - final byte[] b = new byte[8]; - for (int i = 7; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a long value. Reverses {@link #toBytes(long)} - * - * @param bytes array - * @return the long value - */ - public static long toLong(final byte[] bytes) { - return toLong(bytes, 0, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. Assumes there will be {@link #SIZEOF_LONG} bytes - * available. - * - * @param bytes bytes - * @param offset offset - * @return the long value - */ - public static long toLong(final byte[] bytes, final int offset) { - return toLong(bytes, offset, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. - * - * @param bytes array of bytes - * @param offset offset into array - * @param length length of data (must be {@link #SIZEOF_LONG}) - * @return the long value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or if there's not enough - * room in the array at the offset indicated. - */ - public static long toLong(final byte[] bytes, final int offset, final int length) { - if (length != SIZEOF_LONG || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); - } - return ConverterHolder.BEST_CONVERTER.toLong(bytes, offset, length); - } - - private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes, - final int offset, - final int length, - final int expectedLength) { - final String reason; - if (length != expectedLength) { - reason = "Wrong length: " + length + ", expected " + expectedLength; - } else { - reason = "offset (" + offset + ") + length (" + length + ") exceed the" - + " capacity of the array: " + bytes.length; - } - return new IllegalArgumentException(reason); - } - - /** - * Put a long value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val long to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have enough room at the offset - * specified. - */ - public static int putLong(final byte[] bytes, final int offset, final long val) { - if (bytes.length - offset < SIZEOF_LONG) { - throw new IllegalArgumentException("Not enough room to put a long at" + " offset " + offset - + " in a " + bytes.length + " byte array"); - } - return ConverterHolder.BEST_CONVERTER.putLong(bytes, offset, val); - } - - /** - * Put a float value out to the specified byte array position. Presumes float encoded as IEEE 754 - * floating-point "single format" - * - * @param bytes byte array - * @return Float made from passed byte array. - */ - public static float toFloat(final byte[] bytes) { - return toFloat(bytes, 0); - } - - /** - * Put a float value out to the specified byte array position. Presumes float encoded as IEEE 754 - * floating-point "single format" - * - * @param bytes array to convert - * @param offset offset into array - * @return Float made from passed byte array. - */ - public static float toFloat(final byte[] bytes, final int offset) { - return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); - } - - /** - * Put a float value out to the specified byte array position. - * - * @param bytes byte array - * @param offset offset to write to - * @param f float value - * @return New offset in bytes - */ - public static int putFloat(final byte[] bytes, final int offset, final float f) { - return putInt(bytes, offset, Float.floatToRawIntBits(f)); - } - - /** - * Return the float represented as byte[] - */ - public static byte[] toBytes(final float f) { - // Encode it as int - return Bytes.toBytes(Float.floatToRawIntBits(f)); - } - - /** - * Return double made from passed bytes. - */ - public static double toDouble(final byte[] bytes) { - return toDouble(bytes, 0); - } - - /** - * Return double made from passed bytes. - */ - public static double toDouble(final byte[] bytes, final int offset) { - return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); - } - - /** - * Put a double value out to the specified byte array position as the IEEE 754 double format. - * - * @param bytes byte array - * @param offset offset to write to - * @param d value - * @return New offset into array bytes - */ - public static int putDouble(final byte[] bytes, final int offset, final double d) { - return putLong(bytes, offset, Double.doubleToLongBits(d)); - } - - /** - * Serialize a double as the IEEE 754 double format output. The resultant array will be 8 bytes - * long. - * - * @param d value - * @return the double represented as byte [] - */ - public static byte[] toBytes(final double d) { - // Encode it as a long - return Bytes.toBytes(Double.doubleToRawLongBits(d)); - } - - /** - * Convert an int value to a byte array. Big-endian. Same as what DataOutputStream.writeInt does. - * - * @param val value - * @return the byte array - */ - public static byte[] toBytes(int val) { - final byte[] b = new byte[4]; - for (int i = 3; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to an int value - * - * @param bytes byte array - * @return the int value - */ - public static int toInt(final byte[] bytes) { - return toInt(bytes, 0, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * - * @param bytes byte array - * @param offset offset into array - * @return the int value - */ - public static int toInt(final byte[] bytes, final int offset) { - return toInt(bytes, offset, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * - * @param bytes byte array - * @param offset offset into array - * @param length length of int (has to be {@link #SIZEOF_INT}) - * @return the int value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or if there's not enough - * room in the array at the offset indicated. - */ - public static int toInt(final byte[] bytes, final int offset, final int length) { - if (length != SIZEOF_INT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); - } - return ConverterHolder.BEST_CONVERTER.toInt(bytes, offset, length); - } - - /** - * Converts a byte array to an int value - * - * @param bytes byte array - * @param offset offset into array - * @param length how many bytes should be considered for creating int - * @return the int value - * @throws IllegalArgumentException if there's not enough room in the array at the offset - * indicated. - */ - public static int readAsInt(final byte[] bytes, final int offset, final int length) { - if (offset + length > bytes.length) { - throw new IllegalArgumentException("offset (" + offset + ") + length (" + length - + ") exceed the" + " capacity of the array: " + bytes.length); - } - int n = 0; - for (int i = offset; i < (offset + length); i++) { - n <<= 8; - n ^= bytes[i] & 0xFF; - } - return n; - } - - /** - * Put an int value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val int to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have enough room at the offset - * specified. - */ - public static int putInt(final byte[] bytes, final int offset, final int val) { - if (bytes.length - offset < SIZEOF_INT) { - throw new IllegalArgumentException("Not enough room to put an int at" + " offset " + offset - + " in a " + bytes.length + " byte array"); - } - return ConverterHolder.BEST_CONVERTER.putInt(bytes, offset, val); - } - - /** - * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long. - * - * @param val value - * @return the byte array - */ - public static byte[] toBytes(short val) { - final byte[] b = new byte[SIZEOF_SHORT]; - b[1] = (byte) val; - val >>= 8; - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a short value - * - * @param bytes byte array - * @return the short value - */ - public static short toShort(final byte[] bytes) { - return toShort(bytes, 0, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * - * @param bytes byte array - * @param offset offset into array - * @return the short value - */ - public static short toShort(final byte[] bytes, final int offset) { - return toShort(bytes, offset, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * - * @param bytes byte array - * @param offset offset into array - * @param length length, has to be {@link #SIZEOF_SHORT} - * @return the short value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} or if there's not - * enough room in the array at the offset indicated. - */ - public static short toShort(final byte[] bytes, final int offset, final int length) { - if (length != SIZEOF_SHORT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); - } - return ConverterHolder.BEST_CONVERTER.toShort(bytes, offset, length); - } - - /** - * Returns a new byte array, copied from the given {@code buf}, from the position (inclusive) to - * the limit (exclusive). The position and the other index parameters are not changed. - * - * @param buf a byte buffer - * @return the byte array - * @see #toBytes(ByteBuffer) - */ - public static byte[] getBytes(final ByteBuffer buf) { - return readBytes(buf.duplicate()); - } - - /** - * Put a short value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val short to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have enough room at the offset - * specified. - */ - public static int putShort(final byte[] bytes, final int offset, final short val) { - if (bytes.length - offset < SIZEOF_SHORT) { - throw new IllegalArgumentException("Not enough room to put a short at" + " offset " + offset - + " in a " + bytes.length + " byte array"); - } - return ConverterHolder.BEST_CONVERTER.putShort(bytes, offset, val); - } - - /** - * Put an int value as short out to the specified byte array position. Only the lower 2 bytes of - * the short will be put into the array. The caller of the API need to make sure they will not - * loose the value by doing so. This is useful to store an unsigned short which is represented as - * int in other parts. - * - * @param bytes the byte array - * @param offset position in the array - * @param val value to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have enough room at the offset - * specified. - */ - public static int putAsShort(final byte[] bytes, final int offset, int val) { - if (bytes.length - offset < SIZEOF_SHORT) { - throw new IllegalArgumentException("Not enough room to put a short at" + " offset " + offset - + " in a " + bytes.length + " byte array"); - } - bytes[offset + 1] = (byte) val; - val >>= 8; - bytes[offset] = (byte) val; - return offset + SIZEOF_SHORT; - } - - /** - * Convert a BigDecimal value to a byte array - */ - public static byte[] toBytes(final BigDecimal val) { - final byte[] valueBytes = val.unscaledValue().toByteArray(); - final byte[] result = new byte[valueBytes.length + SIZEOF_INT]; - final int offset = putInt(result, 0, val.scale()); - putBytes(result, offset, valueBytes, 0, valueBytes.length); - return result; - } - - /** - * Converts a byte array to a BigDecimal - */ - public static BigDecimal toBigDecimal(final byte[] bytes) { - return toBigDecimal(bytes, 0, bytes.length); - } - - /** - * Converts a byte array to a BigDecimal value - */ - public static BigDecimal toBigDecimal(final byte[] bytes, final int offset, final int length) { - if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) { - return null; - } - - final int scale = toInt(bytes, offset); - final byte[] tcBytes = new byte[length - SIZEOF_INT]; - System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT); - return new BigDecimal(new BigInteger(tcBytes), scale); - } - - /** - * Put a BigDecimal value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val BigDecimal to write out - * @return incremented offset - */ - public static int putBigDecimal(final byte[] bytes, int offset, final BigDecimal val) { - if (bytes == null) { - return offset; - } - - final byte[] valueBytes = val.unscaledValue().toByteArray(); - final byte[] result = new byte[valueBytes.length + SIZEOF_INT]; - offset = putInt(result, offset, val.scale()); - return putBytes(result, offset, valueBytes, 0, valueBytes.length); - } - - /** - * Encode a long value as a variable length integer. - * - * @param vint Integer to make a vint of. - * @return Vint as bytes array. - */ - public static byte[] vintToBytes(final long vint) { - long i = vint; - final int size = WritableUtils.getVIntSize(i); - final byte[] result = new byte[size]; - int offset = 0; - if (i >= -112 && i <= 127) { - result[offset] = (byte) i; - return result; - } - - int len = -112; - if (i < 0) { - i ^= -1L; // take one's complement' - len = -120; - } - - long tmp = i; - while (tmp != 0) { - tmp = tmp >> 8; - len--; - } - - result[offset++] = (byte) len; - - len = (len < -120) - ? -(len + 120) - : -(len + 112); - - for (int idx = len; idx != 0; idx--) { - final int shiftbits = (idx - 1) * 8; - final long mask = 0xFFL << shiftbits; - result[offset++] = (byte) ((i & mask) >> shiftbits); - } - return result; - } - - /** - * Reads a zero-compressed encoded long from input buffer and returns it. - * - * @param buffer buffer to convert - * @return vint bytes as an integer. - */ - public static long bytesToVint(final byte[] buffer) { - int offset = 0; - final byte firstByte = buffer[offset++]; - final int len = WritableUtils.decodeVIntSize(firstByte); - if (len == 1) { - return firstByte; - } - long i = 0; - for (int idx = 0; idx < len - 1; idx++) { - final byte b = buffer[offset++]; - i = i << 8; - i = i | (b & 0xFF); - } - return (WritableUtils.isNegativeVInt(firstByte) - ? ~i - : i); - } - - /** - * Reads a zero-compressed encoded long from input buffer and returns it. - * - * @param buffer Binary array - * @param offset Offset into array at which vint begins. - * @return deserialized long from buffer. - */ - public static long readAsVLong(final byte[] buffer, final int offset) { - final byte firstByte = buffer[offset]; - final int len = WritableUtils.decodeVIntSize(firstByte); - if (len == 1) { - return firstByte; - } - long i = 0; - for (int idx = 0; idx < len - 1; idx++) { - final byte b = buffer[offset + 1 + idx]; - i = i << 8; - i = i | (b & 0xFF); - } - return (WritableUtils.isNegativeVInt(firstByte) - ? ~i - : i); - } - - /** - * Lexicographically compare two arrays. - * - * @param left left operand - * @param right right operand - * @return 0 if equal, < 0 if left is less than right, etc. - */ - public static int compareTo(final byte[] left, final byte[] right) { - return LexicographicalComparerHolder.BEST_COMPARER.compareTo(left, - 0, - left == null - ? 0 - : left.length, - right, - 0, - right == null - ? 0 - : right.length); - } - - /** - * Lexicographically compare two arrays. - * - * @param buffer1 left operand - * @param buffer2 right operand - * @param offset1 Where to start comparing in the left buffer - * @param offset2 Where to start comparing in the right buffer - * @param length1 How much to compare from the left buffer - * @param length2 How much to compare from the right buffer - * @return 0 if equal, < 0 if left is less than right, etc. - */ - public static int compareTo(final byte[] buffer1, final int offset1, final int length1, final byte[] buffer2, final int offset2, - final int length2) { - return LexicographicalComparerHolder.BEST_COMPARER.compareTo(buffer1, offset1, length1, buffer2, - offset2, length2); - } - - interface Comparer { - - int compareTo(T buffer1, int offset1, int length1, T buffer2, int offset2, int length2); - } - - static abstract class Converter { - - abstract long toLong(byte[] bytes, int offset, int length); - - abstract int putLong(byte[] bytes, int offset, long val); - - abstract int toInt(byte[] bytes, int offset, final int length); - - abstract int putInt(byte[] bytes, int offset, int val); - - abstract short toShort(byte[] bytes, int offset, final int length); - - abstract int putShort(byte[] bytes, int offset, short val); - - } - - static abstract class CommonPrefixer { - - abstract int findCommonPrefix(byte[] left, int leftOffset, int leftLength, byte[] right, - int rightOffset, int rightLength); - } - - static Comparer lexicographicalComparerJavaImpl() { - return LexicographicalComparerHolder.PureJavaComparer.INSTANCE; - } - - static class ConverterHolder { - - static final String UNSAFE_CONVERTER_NAME = - ConverterHolder.class.getName() + "$UnsafeConverter"; - - static final Converter BEST_CONVERTER = getBestConverter(); - - /** - * Returns the Unsafe-using Converter, or falls back to the pure-Java implementation if unable - * to do so. - */ - static Converter getBestConverter() { - try { - final Class theClass = Class.forName(UNSAFE_CONVERTER_NAME); - - // yes, UnsafeComparer does implement Comparer - @SuppressWarnings("unchecked") final Converter converter = (Converter) theClass.getConstructor().newInstance(); - return converter; - } catch (final Throwable t) { // ensure we really catch *everything* - return PureJavaConverter.INSTANCE; - } - } - - protected static final class PureJavaConverter extends Converter { - - static final PureJavaConverter INSTANCE = new PureJavaConverter(); - - private PureJavaConverter() { - } - - @Override - long toLong(final byte[] bytes, final int offset, final int length) { - long l = 0; - for (int i = offset; i < offset + length; i++) { - l <<= 8; - l ^= bytes[i] & 0xFF; - } - return l; - } - - @Override - int putLong(final byte[] bytes, final int offset, long val) { - for (int i = offset + 7; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_LONG; - } - - @Override - int toInt(final byte[] bytes, final int offset, final int length) { - int n = 0; - for (int i = offset; i < (offset + length); i++) { - n <<= 8; - n ^= bytes[i] & 0xFF; - } - return n; - } - - @Override - int putInt(final byte[] bytes, final int offset, int val) { - for (int i = offset + 3; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_INT; - } - - @Override - short toShort(final byte[] bytes, final int offset, final int length) { - short n = 0; - n = (short) ((n ^ bytes[offset]) & 0xFF); - n = (short) (n << 8); - n ^= (short) (bytes[offset + 1] & 0xFF); - return n; - } - - @Override - int putShort(final byte[] bytes, final int offset, short val) { - bytes[offset + 1] = (byte) val; - val >>= 8; - bytes[offset] = (byte) val; - return offset + SIZEOF_SHORT; - } - } - - protected static final class UnsafeConverter extends Converter { - - public UnsafeConverter() { - } - - static { - if (!UNSAFE_UNALIGNED) { - // It doesn't matter what we throw; - // it's swallowed in getBestComparer(). - throw new Error(); - } - - // sanity check - this should never fail - if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) { - throw new AssertionError(); - } - } - - @Override - long toLong(final byte[] bytes, final int offset, final int length) { - return UnsafeAccess.toLong(bytes, offset); - } - - @Override - int putLong(final byte[] bytes, final int offset, final long val) { - return UnsafeAccess.putLong(bytes, offset, val); - } - - @Override - int toInt(final byte[] bytes, final int offset, final int length) { - return UnsafeAccess.toInt(bytes, offset); - } - - @Override - int putInt(final byte[] bytes, final int offset, final int val) { - return UnsafeAccess.putInt(bytes, offset, val); - } - - @Override - short toShort(final byte[] bytes, final int offset, final int length) { - return UnsafeAccess.toShort(bytes, offset); - } - - @Override - int putShort(final byte[] bytes, final int offset, final short val) { - return UnsafeAccess.putShort(bytes, offset, val); - } - } - } - - /** - * Provides a lexicographical comparer implementation; either a Java implementation or a faster - * implementation based on {@code Unsafe}. - *

- * Uses reflection to gracefully fall back to the Java implementation if {@code Unsafe} isn't - * available. - */ - static class LexicographicalComparerHolder { - - static final String UNSAFE_COMPARER_NAME = - LexicographicalComparerHolder.class.getName() + "$UnsafeComparer"; - - static final Comparer BEST_COMPARER = getBestComparer(); - - /** - * Returns the Unsafe-using Comparer, or falls back to the pure-Java implementation if unable to - * do so. - */ - static Comparer getBestComparer() { - try { - final Class theClass = Class.forName(UNSAFE_COMPARER_NAME); - - // yes, UnsafeComparer does implement Comparer - @SuppressWarnings("unchecked") final Comparer comparer = (Comparer) theClass.getEnumConstants()[0]; - return comparer; - } catch (final Throwable t) { // ensure we really catch *everything* - return lexicographicalComparerJavaImpl(); - } - } - - enum PureJavaComparer implements Comparer { - INSTANCE; - - @Override - public int compareTo(final byte[] buffer1, final int offset1, final int length1, final byte[] buffer2, final int offset2, - final int length2) { - // Short circuit equal case - if (buffer1 == buffer2 && offset1 == offset2 && length1 == length2) { - return 0; - } - // Bring WritableComparator code local - final int end1 = offset1 + length1; - final int end2 = offset2 + length2; - for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { - final int a = (buffer1[i] & 0xff); - final int b = (buffer2[j] & 0xff); - if (a != b) { - return a - b; - } - } - return length1 - length2; - } - } - - enum UnsafeComparer implements Comparer { - INSTANCE; - - static { - if (!UNSAFE_UNALIGNED) { - // It doesn't matter what we throw; - // it's swallowed in getBestComparer(). - throw new Error(); - } - - // sanity check - this should never fail - if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) { - throw new AssertionError(); - } - } - - /** - * Lexicographically compare two arrays. - * - * @param buffer1 left operand - * @param buffer2 right operand - * @param offset1 Where to start comparing in the left buffer - * @param offset2 Where to start comparing in the right buffer - * @param length1 How much to compare from the left buffer - * @param length2 How much to compare from the right buffer - * @return 0 if equal, < 0 if left is less than right, etc. - */ - @Override - public int compareTo(final byte[] buffer1, final int offset1, final int length1, final byte[] buffer2, final int offset2, - final int length2) { - - // Short circuit equal case - if (buffer1 == buffer2 && offset1 == offset2 && length1 == length2) { - return 0; - } - final int stride = 8; - final int minLength = Math.min(length1, length2); - final int strideLimit = minLength & ~(stride - 1); - final long offset1Adj = offset1 + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - final long offset2Adj = offset2 + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - int i; - - /* - * Compare 8 bytes at a time. Benchmarking on x86 shows a stride of 8 bytes is no slower - * than 4 bytes even on 32-bit. On the other hand, it is substantially faster on 64-bit. - */ - for (i = 0; i < strideLimit; i += stride) { - final long lw = HBasePlatformDependent.getLong(buffer1, offset1Adj + i); - final long rw = HBasePlatformDependent.getLong(buffer2, offset2Adj + i); - if (lw != rw) { - if (!UnsafeAccess.LITTLE_ENDIAN) { - return ((lw + Long.MIN_VALUE) < (rw + Long.MIN_VALUE)) - ? -1 - : 1; - } - - /* - * We want to compare only the first index where left[index] != right[index]. This - * corresponds to the least significant nonzero byte in lw ^ rw, since lw and rw are - * little-endian. Long.numberOfTrailingZeros(diff) tells us the least significant - * nonzero bit, and zeroing out the first three bits of L.nTZ gives us the shift to get - * that least significant nonzero byte. This comparison logic is based on UnsignedBytes - * comparator from guava v21 - */ - final int n = Long.numberOfTrailingZeros(lw ^ rw) & ~0x7; - return ((int) ((lw >>> n) & 0xFF)) - ((int) ((rw >>> n) & 0xFF)); - } - } - - // The epilogue to cover the last (minLength % stride) elements. - for (; i < minLength; i++) { - final int a = (buffer1[offset1 + i] & 0xFF); - final int b = (buffer2[offset2 + i] & 0xFF); - if (a != b) { - return a - b; - } - } - return length1 - length2; - } - } - } - - static class CommonPrefixerHolder { - - static final String UNSAFE_COMMON_PREFIXER_NAME = - CommonPrefixerHolder.class.getName() + "$UnsafeCommonPrefixer"; - - static final CommonPrefixer BEST_COMMON_PREFIXER = getBestCommonPrefixer(); - - static CommonPrefixer getBestCommonPrefixer() { - try { - final Class theClass = - Class.forName(UNSAFE_COMMON_PREFIXER_NAME).asSubclass(CommonPrefixer.class); - - return theClass.getConstructor().newInstance(); - } catch (final Throwable t) { // ensure we really catch *everything* - return CommonPrefixerHolder.PureJavaCommonPrefixer.INSTANCE; - } - } - - static final class PureJavaCommonPrefixer extends CommonPrefixer { - - static final PureJavaCommonPrefixer INSTANCE = new PureJavaCommonPrefixer(); - - private PureJavaCommonPrefixer() { - } - - @Override - public int findCommonPrefix(final byte[] left, final int leftOffset, final int leftLength, final byte[] right, - final int rightOffset, final int rightLength) { - final int length = Math.min(leftLength, rightLength); - int result = 0; - - while (result < length && left[leftOffset + result] == right[rightOffset + result]) { - result++; - } - return result; - } - } - - static final class UnsafeCommonPrefixer extends CommonPrefixer { - - static { - if (!UNSAFE_UNALIGNED) { - throw new Error(); - } - - // sanity check - this should never fail - if (HBasePlatformDependent.arrayIndexScale(byte[].class) != 1) { - throw new AssertionError(); - } - } - - public UnsafeCommonPrefixer() { - } - - @Override - public int findCommonPrefix(final byte[] left, final int leftOffset, final int leftLength, final byte[] right, - final int rightOffset, final int rightLength) { - final int stride = 8; - final int minLength = Math.min(leftLength, rightLength); - final int strideLimit = minLength & ~(stride - 1); - final long leftOffsetAdj = leftOffset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - final long rightOffsetAdj = rightOffset + UnsafeAccess.BYTE_ARRAY_BASE_OFFSET; - int result = 0; - int i; - - for (i = 0; i < strideLimit; i += stride) { - final long lw = HBasePlatformDependent.getLong(left, leftOffsetAdj + i); - final long rw = HBasePlatformDependent.getLong(right, rightOffsetAdj + i); - if (lw != rw) { - if (!UnsafeAccess.LITTLE_ENDIAN) { - return result + (Long.numberOfLeadingZeros(lw ^ rw) / Bytes.SIZEOF_LONG); - } else { - return result + (Long.numberOfTrailingZeros(lw ^ rw) / Bytes.SIZEOF_LONG); - } - } else { - result += Bytes.SIZEOF_LONG; - } - } - - // The epilogue to cover the last (minLength % stride) elements. - for (; i < minLength; i++) { - final int il = (left[leftOffset + i]); - final int ir = (right[rightOffset + i]); - if (il != ir) { - return result; - } else { - result++; - } - } - - return result; - } - } - } - - /** - * Lexicographically determine the equality of two arrays. - * - * @param left left operand - * @param right right operand - * @return True if equal - */ - public static boolean equals(final byte[] left, final byte[] right) { - // Could use Arrays.equals? - // noinspection SimplifiableConditionalExpression - if (left == right) { - return true; - } - if (left == null || right == null) { - return false; - } - if (left.length != right.length) { - return false; - } - if (left.length == 0) { - return true; - } - - // Since we're often comparing adjacent sorted data, - // it's usual to have equal arrays except for the very last byte - // so check that first - if (left[left.length - 1] != right[right.length - 1]) { - return false; - } - - return compareTo(left, right) == 0; - } - - /** - * Lexicographically determine the equality of two arrays. - * - * @param left left operand - * @param leftOffset offset into left operand - * @param leftLen length of left operand - * @param right right operand - * @param rightOffset offset into right operand - * @param rightLen length of right operand - * @return True if equal - */ - public static boolean equals(final byte[] left, final int leftOffset, final int leftLen, final byte[] right, - final int rightOffset, final int rightLen) { - // short circuit case - if (left == right && leftOffset == rightOffset && leftLen == rightLen) { - return true; - } - // different lengths fast check - if (leftLen != rightLen) { - return false; - } - if (leftLen == 0) { - return true; - } - - // Since we're often comparing adjacent sorted data, - // it's usual to have equal arrays except for the very last byte - // so check that first - if (left[leftOffset + leftLen - 1] != right[rightOffset + rightLen - 1]) { - return false; - } - - return LexicographicalComparerHolder.BEST_COMPARER.compareTo(left, leftOffset, leftLen, right, - rightOffset, rightLen) == 0; - } - - /** - * Lexicographically determine the equality of two byte[], one as ByteBuffer. - * - * @param a left operand - * @param buf right operand - * @return True if equal - */ - public static boolean equals(final byte[] a, final ByteBuffer buf) { - if (a == null) { - return buf == null; - } - if (buf == null) { - return false; - } - if (a.length != buf.remaining()) { - return false; - } - - // Thou shalt not modify the original byte buffer in what should be read only operations. - final ByteBuffer b = buf.duplicate(); - for (final byte anA : a) { - if (anA != b.get()) { - return false; - } - } - return true; - } - - /** - * Return true if the byte array on the right is a prefix of the byte array on the left. - */ - public static boolean startsWith(final byte[] bytes, final byte[] prefix) { - return bytes != null && prefix != null && bytes.length >= prefix.length - && LexicographicalComparerHolder.BEST_COMPARER.compareTo(bytes, 0, prefix.length, prefix, 0, - prefix.length) == 0; - } - - /** - * Calculate a hash code from a given byte array. - * - * @param b bytes to hash - * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the passed in array. This - * method is what {@link org.apache.hadoop.io.Text} use calculating hash code. - */ - public static int hashCode(final byte[] b) { - return hashCode(b, b.length); - } - - /** - * Calculate a hash code from a given byte array. - * - * @param b value - * @param length length of the value - * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the passed in array. This - * method is what {@link org.apache.hadoop.io.Text} use calculating hash code. - */ - public static int hashCode(final byte[] b, final int length) { - return WritableComparator.hashBytes(b, length); - } - - /** - * Calculate a hash code from a given byte array suitable for use as a key in maps. - * - * @param b bytes to hash - * @return A hash of b as an Integer that can be used as key in Maps. - */ - public static Integer mapKey(final byte[] b) { - return hashCode(b); - } - - /** - * Calculate a hash code from a given byte array suitable for use as a key in maps. - * - * @param b bytes to hash - * @param length length to hash - * @return A hash of b as an Integer that can be used as key in Maps. - */ - public static Integer mapKey(final byte[] b, final int length) { - return hashCode(b, length); - } - - /** - * Concatenate byte arrays. - * - * @param a lower half - * @param b upper half - * @return New array that has a in lower half and b in upper half. - */ - public static byte[] add(final byte[] a, final byte[] b) { - return add(a, b, EMPTY_BYTE_ARRAY); - } - - /** - * Concatenate byte arrays. - * - * @param a first third - * @param b second third - * @param c third third - * @return New array made from a, b and c - */ - public static byte[] add(final byte[] a, final byte[] b, final byte[] c) { - final byte[] result = new byte[a.length + b.length + c.length]; - System.arraycopy(a, 0, result, 0, a.length); - System.arraycopy(b, 0, result, a.length, b.length); - System.arraycopy(c, 0, result, a.length + b.length, c.length); - return result; - } - - /** - * Concatenate byte arrays. - * - * @param arrays all the arrays to concatenate together. - * @return New array made from the concatenation of the given arrays. - */ - public static byte[] add(final byte[][] arrays) { - int length = 0; - for (int i = 0; i < arrays.length; i++) { - length += arrays[i].length; - } - final byte[] result = new byte[length]; - int index = 0; - for (int i = 0; i < arrays.length; i++) { - System.arraycopy(arrays[i], 0, result, index, arrays[i].length); - index += arrays[i].length; - } - return result; - } - - /** - * Make a new byte array from a subset of bytes at the head of another. - * - * @param a array - * @param length amount of bytes to grab - * @return First length bytes from a - */ - public static byte[] head(final byte[] a, final int length) { - if (a.length < length) { - return null; - } - final byte[] result = new byte[length]; - System.arraycopy(a, 0, result, 0, length); - return result; - } - - /** - * Make a new byte array from a subset of bytes at the tail of another. - * - * @param a array - * @param length amount of bytes to snarf - * @return Last length bytes from a - */ - public static byte[] tail(final byte[] a, final int length) { - if (a.length < length) { - return null; - } - final byte[] result = new byte[length]; - System.arraycopy(a, a.length - length, result, 0, length); - return result; - } - - /** - * Make a new byte array from a subset of bytes at the head of another, zero padded as desired. - * - * @param a array - * @param length new array size - * @return Value in a plus length prepended 0 bytes - */ - public static byte[] padHead(final byte[] a, final int length) { - final byte[] padding = new byte[length]; - for (int i = 0; i < length; i++) { - padding[i] = 0; - } - return add(padding, a); - } - - /** - * Make a new byte array from a subset of bytes at the tail of another, zero padded as desired. - * - * @param a array - * @param length new array size - * @return Value in a plus length appended 0 bytes - */ - public static byte[] padTail(final byte[] a, final int length) { - final byte[] padding = new byte[length]; - for (int i = 0; i < length; i++) { - padding[i] = 0; - } - return add(a, padding); - } - - /** - * Split passed range. Expensive operation relatively. Uses BigInteger math. Useful splitting - * ranges for MapReduce jobs. - * - * @param a Beginning of range - * @param b End of range - * @param num Number of times to split range. Pass 1 if you want to split the range in two; i.e. - * one split. - * @return Array of dividing values - */ - public static byte[][] split(final byte[] a, final byte[] b, final int num) { - return split(a, b, false, num); - } - - /** - * Split passed range. Expensive operation relatively. Uses BigInteger math. Useful splitting - * ranges for MapReduce jobs. - * - * @param a Beginning of range - * @param b End of range - * @param inclusive Whether the end of range is prefix-inclusive or is considered an exclusive - * boundary. Automatic splits are generally exclusive and manual splits with an - * explicit range utilize an inclusive end of range. - * @param num Number of times to split range. Pass 1 if you want to split the range in two; - * i.e. one split. - * @return Array of dividing values - */ - public static byte[][] split(final byte[] a, final byte[] b, final boolean inclusive, final int num) { - final byte[][] ret = new byte[num + 2][]; - int i = 0; - final Iterable iter = iterateOnSplits(a, b, inclusive, num); - if (iter == null) { - return null; - } - for (final byte[] elem : iter) { - ret[i++] = elem; - } - return ret; - } - - /** - * Iterate over keys within the passed range, splitting at an [a,b) boundary. - */ - public static Iterable iterateOnSplits(final byte[] a, final byte[] b, final int num) { - return iterateOnSplits(a, b, false, num); - } - - /** - * Iterate over keys within the passed range. - */ - public static Iterable iterateOnSplits(final byte[] a, final byte[] b, final boolean inclusive, - final int num) { - final byte[] aPadded; - final byte[] bPadded; - if (a.length < b.length) { - aPadded = padTail(a, b.length - a.length); - bPadded = b; - } else if (b.length < a.length) { - aPadded = a; - bPadded = padTail(b, a.length - b.length); - } else { - aPadded = a; - bPadded = b; - } - if (compareTo(aPadded, bPadded) >= 0) { - throw new IllegalArgumentException("b <= a"); - } - if (num <= 0) { - throw new IllegalArgumentException("num cannot be <= 0"); - } - final byte[] prependHeader = {1, 0}; - final BigInteger startBI = new BigInteger(add(prependHeader, aPadded)); - final BigInteger stopBI = new BigInteger(add(prependHeader, bPadded)); - BigInteger diffBI = stopBI.subtract(startBI); - if (inclusive) { - diffBI = diffBI.add(BigInteger.ONE); - } - final BigInteger splitsBI = BigInteger.valueOf(num + 1); - // when diffBI < splitBI, use an additional byte to increase diffBI - if (diffBI.compareTo(splitsBI) < 0) { - final byte[] aPaddedAdditional = new byte[aPadded.length + 1]; - final byte[] bPaddedAdditional = new byte[bPadded.length + 1]; - for (int i = 0; i < aPadded.length; i++) { - aPaddedAdditional[i] = aPadded[i]; - } - for (int j = 0; j < bPadded.length; j++) { - bPaddedAdditional[j] = bPadded[j]; - } - aPaddedAdditional[aPadded.length] = 0; - bPaddedAdditional[bPadded.length] = 0; - return iterateOnSplits(aPaddedAdditional, bPaddedAdditional, inclusive, num); - } - final BigInteger intervalBI; - try { - intervalBI = diffBI.divide(splitsBI); - } catch (final Exception e) { - LOG.error("Exception caught during division", e); - return null; - } - - final Iterator iterator = new Iterator() { - private int i = -1; - - @Override - public boolean hasNext() { - return i < num + 1; - } - - @Override - public byte[] next() { - i++; - if (i == 0) { - return a; - } - if (i == num + 1) { - return b; - } - - final BigInteger curBI = startBI.add(intervalBI.multiply(BigInteger.valueOf(i))); - byte[] padded = curBI.toByteArray(); - if (padded[1] == 0) { - padded = tail(padded, padded.length - 2); - } else { - padded = tail(padded, padded.length - 1); - } - return padded; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - }; - - return new Iterable() { - @Override - public Iterator iterator() { - return iterator; - } - }; - } - - /** - * Calculate the hash code for a given range of bytes. - * - * @param bytes array to hash - * @param offset offset to start from - * @param length length to hash - */ - public static int hashCode(final byte[] bytes, final int offset, final int length) { - int hash = 1; - for (int i = offset; i < offset + length; i++) { - hash = (31 * hash) + bytes[i]; - } - return hash; - } - - /** - * Create an array of byte[] given an array of String. - * - * @param t operands - * @return Array of byte arrays made from passed array of Text - */ - public static byte[][] toByteArrays(final String[] t) { - final byte[][] result = new byte[t.length][]; - for (int i = 0; i < t.length; i++) { - result[i] = Bytes.toBytes(t[i]); - } - return result; - } - - /** - * Create an array of byte[] given an array of String. - * - * @param t operands - * @return Array of binary byte arrays made from passed array of binary strings - */ - public static byte[][] toBinaryByteArrays(final String[] t) { - final byte[][] result = new byte[t.length][]; - for (int i = 0; i < t.length; i++) { - result[i] = Bytes.toBytesBinary(t[i]); - } - return result; - } - - /** - * Create a byte[][] where first and only entry is column - * - * @param column operand - * @return A byte array of a byte array where first and only entry is column - */ - public static byte[][] toByteArrays(final String column) { - return toByteArrays(toBytes(column)); - } - - /** - * Create a byte[][] where first and only entry is column - * - * @param column operand - * @return A byte array of a byte array where first and only entry is column - */ - public static byte[][] toByteArrays(final byte[] column) { - final byte[][] result = new byte[1][]; - result[0] = column; - return result; - } - - /** - * Binary search for keys in indexes using Bytes.BYTES_RAWCOMPARATOR. - * - * @param arr array of byte arrays to search for - * @param key the key you want to find - * @param offset the offset in the key you want to find - * @param length the length of the key - * @return zero-based index of the key, if the key is present in the array. Otherwise, a value -(i - * + 1) such that the key is between arr[i - 1] and arr[i] non-inclusively, where i is in - * [0, i], if we define arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above - * means that this function can return 2N + 1 different values ranging from -(N + 1) to N - * - 1. - */ - public static int binarySearch(final byte[][] arr, final byte[] key, final int offset, final int length) { - int low = 0; - int high = arr.length - 1; - - while (low <= high) { - final int mid = low + ((high - low) >> 1); - // we have to compare in this order, because the comparator order - // has special logic when the 'left side' is a special key. - final int cmp = - Bytes.BYTES_RAWCOMPARATOR.compare(key, offset, length, arr[mid], 0, arr[mid].length); - // key lives above the midpoint - if (cmp > 0) { - low = mid + 1; - } - // key lives below the midpoint - else if (cmp < 0) { - high = mid - 1; - } - // BAM. how often does this really happen? - else { - return mid; - } - } - return -(low + 1); - } - -// /** -// * Binary search for keys in indexes. -// * -// * @param arr array of byte arrays to search for -// * @param key the key you want to find -// * @param comparator a comparator to compare. -// * @return zero-based index of the key, if the key is present in the array. Otherwise, a value -(i -// * + 1) such that the key is between arr[i - 1] and arr[i] non-inclusively, where i is in -// * [0, i], if we define arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above -// * means that this function can return 2N + 1 different values ranging from -(N + 1) to N -// * - 1. -// * @return the index of the block -// */ -// public static int binarySearch(Cell[] arr, Cell key, CellComparator comparator) { -// int low = 0; -// int high = arr.length - 1; -// while (low <= high) { -// int mid = low + ((high - low) >> 1); -// // we have to compare in this order, because the comparator order -// // has special logic when the 'left side' is a special key. -// int cmp = comparator.compare(key, arr[mid]); -// // key lives above the midpoint -// if (cmp > 0) { -// low = mid + 1; -// } -// // key lives below the midpoint -// else if (cmp < 0) { -// high = mid - 1; -// } -// // BAM. how often does this really happen? -// else { -// return mid; -// } -// } -// return -(low + 1); -// } - - /** - * Bytewise binary increment/deincrement of long contained in byte array on given amount. - * - * @param value - array of bytes containing long (length <= SIZEOF_LONG) - * @param amount value will be incremented on (deincremented if negative) - * @return array of bytes containing incremented long (length == SIZEOF_LONG) - */ - public static byte[] incrementBytes(final byte[] value, final long amount) { - byte[] val = value; - if (val.length < SIZEOF_LONG) { - // Hopefully this doesn't happen too often. - final byte[] newvalue; - if (val[0] < 0) { - newvalue = new byte[]{-1, -1, -1, -1, -1, -1, -1, -1}; - } else { - newvalue = new byte[SIZEOF_LONG]; - } - System.arraycopy(val, 0, newvalue, newvalue.length - val.length, val.length); - val = newvalue; - } else if (val.length > SIZEOF_LONG) { - throw new IllegalArgumentException("Increment Bytes - value too big: " + val.length); - } - if (amount == 0) { - return val; - } - if (val[0] < 0) { - return binaryIncrementNeg(val, amount); - } - return binaryIncrementPos(val, amount); - } - - /* increment/deincrement for positive value */ - private static byte[] binaryIncrementPos(final byte[] value, final long amount) { - long amo = amount; - int sign = 1; - if (amount < 0) { - amo = -amount; - sign = -1; - } - for (int i = 0; i < value.length; i++) { - final int cur = ((int) amo % 256) * sign; - amo = (amo >> 8); - final int val = value[value.length - i - 1] & 0x0ff; - int total = val + cur; - if (total > 255) { - amo += sign; - total %= 256; - } else if (total < 0) { - amo -= sign; - } - value[value.length - i - 1] = (byte) total; - if (amo == 0) { - return value; - } - } - return value; - } - - /* increment/deincrement for negative value */ - private static byte[] binaryIncrementNeg(final byte[] value, final long amount) { - long amo = amount; - int sign = 1; - if (amount < 0) { - amo = -amount; - sign = -1; - } - for (int i = 0; i < value.length; i++) { - final int cur = ((int) amo % 256) * sign; - amo = (amo >> 8); - final int val = (~value[value.length - i - 1] & 0x0ff) + 1; - int total = cur - val; - if (total >= 0) { - amo += sign; - } else if (total < -256) { - amo -= sign; - total %= 256; - } - value[value.length - i - 1] = (byte) total; - if (amo == 0) { - return value; - } - } - return value; - } - - /** - * Writes a string as a fixed-size field, padded with zeros. - */ - public static void writeStringFixedSize(final DataOutput out, final String s, final int size) - throws IOException { - final byte[] b = toBytes(s); - if (b.length > size) { - throw new IOException("Trying to write " + b.length + " bytes (" + toStringBinary(b) - + ") into a field of length " + size); - } - - out.writeBytes(s); - for (int i = 0; i < size - s.length(); ++i) { - out.writeByte(0); - } - } - - /** - * Reads a fixed-size field and interprets it as a string padded with zeros. - */ - public static String readStringFixedSize(final DataInput in, final int size) throws IOException { - final byte[] b = new byte[size]; - in.readFully(b); - int n = b.length; - while (n > 0 && b[n - 1] == 0) { - --n; - } - - return toString(b, 0, n); - } - - /** - * Copy the byte array given in parameter and return an instance of a new byte array with the same - * length and the same content. - * - * @param bytes the byte array to duplicate - * @return a copy of the given byte array - */ - public static byte[] copy(final byte[] bytes) { - if (bytes == null) { - return null; - } - final byte[] result = new byte[bytes.length]; - System.arraycopy(bytes, 0, result, 0, bytes.length); - return result; - } - - /** - * Copy the byte array given in parameter and return an instance of a new byte array with the same - * length and the same content. - * - * @param bytes the byte array to copy from - * @return a copy of the given designated byte array - */ - public static byte[] copy(final byte[] bytes, final int offset, final int length) { - if (bytes == null) { - return null; - } - final byte[] result = new byte[length]; - System.arraycopy(bytes, offset, result, 0, length); - return result; - } - - /** - * Search sorted array "a" for byte "key". I can't remember if I wrote this or copied it from - * somewhere. (mcorgan) - * - * @param a Array to search. Entries must be sorted and unique. - * @param fromIndex First index inclusive of "a" to include in the search. - * @param toIndex Last index exclusive of "a" to include in the search. - * @param key The byte to search for. - * @return The index of key if found. If not found, return -(index + 1), where negative indicates - * "not found" and the "index + 1" handles the "-0" case. - */ - public static int unsignedBinarySearch(final byte[] a, final int fromIndex, final int toIndex, final byte key) { - final int unsignedKey = key & 0xff; - int low = fromIndex; - int high = toIndex - 1; - - while (low <= high) { - final int mid = low + ((high - low) >> 1); - final int midVal = a[mid] & 0xff; - - if (midVal < unsignedKey) { - low = mid + 1; - } else if (midVal > unsignedKey) { - high = mid - 1; - } else { - return mid; // key found - } - } - return -(low + 1); // key not found. - } - - /** - * Treat the byte[] as an unsigned series of bytes, most significant bits first. Start by adding 1 - * to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes. - * - * @param input The byte[] to increment. - * @return The incremented copy of "in". May be same length or 1 byte longer. - */ - public static byte[] unsignedCopyAndIncrement(final byte[] input) { - final byte[] copy = copy(input); - if (copy == null) { - throw new IllegalArgumentException("cannot increment null array"); - } - for (int i = copy.length - 1; i >= 0; --i) { - if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum - copy[i] = 0; - } else { - ++copy[i]; - return copy; - } - } - // we maxed out the array - final byte[] out = new byte[copy.length + 1]; - out[0] = 1; - System.arraycopy(copy, 0, out, 1, copy.length); - return out; - } - - public static boolean equals(final List a, final List b) { - if (a == null) { - if (b == null) { - return true; - } - return false; - } - if (b == null) { - return false; - } - if (a.size() != b.size()) { - return false; - } - for (int i = 0; i < a.size(); ++i) { - if (!Bytes.equals(a.get(i), b.get(i))) { - return false; - } - } - return true; - } - -// public static boolean isSorted(Collection arrays) { -// if (!CollectionUtils.isEmpty(arrays)) { -// byte[] previous = new byte[0]; -// for (byte[] array : arrays) { -// if (Bytes.compareTo(previous, array) > 0) { -// return false; -// } -// previous = array; -// } -// } -// return true; -// } - - public static List getUtf8ByteArrays(final List strings) { - if (strings.isEmpty()) { - return Collections.emptyList(); - } - final List byteArrays = new ArrayList<>(strings.size()); - strings.forEach(s -> byteArrays.add(Bytes.toBytes(s))); - return byteArrays; - } - - /** - * Returns the index of the first appearance of the value {@code target} in {@code array}. - * - * @param array an array of {@code byte} values, possibly empty - * @param target a primitive {@code byte} value - * @return the least index {@code i} for which {@code array[i] == target}, or {@code -1} if no - * such index exists. - */ - public static int indexOf(final byte[] array, final byte target) { - for (int i = 0; i < array.length; i++) { - if (array[i] == target) { - return i; - } - } - return -1; - } - - /** - * Returns the start position of the first occurrence of the specified {@code - * target} within {@code array}, or {@code -1} if there is no such occurrence. - *

- * More formally, returns the lowest index {@code i} such that {@code - * java.util.Arrays.copyOfRange(array, i, i + target.length)} contains exactly the same elements - * as {@code target}. - * - * @param array the array to search for the sequence {@code target} - * @param target the array to search for as a sub-sequence of {@code array} - */ - public static int indexOf(final byte[] array, final byte[] target) { - Preconditions.checkNotNull(array, "array"); - Preconditions.checkNotNull(target, "target"); - if (target.length == 0) { - return 0; - } - - outer: - for (int i = 0; i < array.length - target.length + 1; i++) { - for (int j = 0; j < target.length; j++) { - if (array[i + j] != target[j]) { - continue outer; - } - } - return i; - } - return -1; - } - - /** - * Return true if target is present as an element anywhere in the given array. - * - * @param array an array of {@code byte} values, possibly empty - * @param target a primitive {@code byte} value - * @return {@code true} if {@code target} is present as an element anywhere in {@code array}. - */ - public static boolean contains(final byte[] array, final byte target) { - return indexOf(array, target) > -1; - } - - /** - * Return true if target is present as an element anywhere in the given array. - * - * @param array an array of {@code byte} values, possibly empty - * @param target an array of {@code byte} - * @return {@code true} if {@code target} is present anywhere in {@code array} - */ - public static boolean contains(final byte[] array, final byte[] target) { - return indexOf(array, target) > -1; - } - - /** - * Fill given array with zeros. - * - * @param b array which needs to be filled with zeros - */ - public static void zero(final byte[] b) { - zero(b, 0, b.length); - } - - /** - * Fill given array with zeros at the specified position. - */ - public static void zero(final byte[] b, final int offset, final int length) { - Preconditions.checkPositionIndex(offset, b.length, "offset"); - Preconditions.checkArgument(length > 0, "length must be greater than 0"); - Preconditions.checkPositionIndex(offset + length, b.length, "offset + length"); - Arrays.fill(b, offset, offset + length, (byte) 0); - } - - // Pseudorandom random number generator, do not use SecureRandom here - private static final Random RNG = new Random(); - - /** - * Fill given array with random bytes. - * - * @param b array which needs to be filled with random bytes - *

- * If you want random bytes generated by a strong source of randomness use - * {@link Bytes#secureRandom(byte[])}. - * @param b array which needs to be filled with random bytes - */ - public static void random(final byte[] b) { - RNG.nextBytes(b); - } - - /** - * Fill given array with random bytes at the specified position. - *

- * If you want random bytes generated by a strong source of randomness use - * {@link Bytes#secureRandom(byte[], int, int)}. - * - * @param b array which needs to be filled with random bytes - * @param offset staring offset in array - * @param length number of bytes to fill - */ - public static void random(final byte[] b, final int offset, final int length) { - Preconditions.checkPositionIndex(offset, b.length, "offset"); - Preconditions.checkArgument(length > 0, "length must be greater than 0"); - Preconditions.checkPositionIndex(offset + length, b.length, "offset + length"); - final byte[] buf = new byte[length]; - RNG.nextBytes(buf); - System.arraycopy(buf, 0, b, offset, length); - } - - // Bytes.secureRandom may be used to create key material. - private static final SecureRandom SECURE_RNG = new SecureRandom(); - - /** - * Fill given array with random bytes using a strong random number generator. - * - * @param b array which needs to be filled with random bytes - */ - public static void secureRandom(final byte[] b) { - SECURE_RNG.nextBytes(b); - } - - /** - * Fill given array with random bytes at the specified position using a strong random number - * generator. - * - * @param b array which needs to be filled with random bytes - * @param offset staring offset in array - * @param length number of bytes to fill - */ - public static void secureRandom(final byte[] b, final int offset, final int length) { - Preconditions.checkPositionIndex(offset, b.length, "offset"); - Preconditions.checkArgument(length > 0, "length must be greater than 0"); - Preconditions.checkPositionIndex(offset + length, b.length, "offset + length"); - final byte[] buf = new byte[length]; - SECURE_RNG.nextBytes(buf); - System.arraycopy(buf, 0, b, offset, length); - } - - /** - * Create a max byte array with the specified max byte count - * - * @param maxByteCount the length of returned byte array - * @return the created max byte array - */ - public static byte[] createMaxByteArray(final int maxByteCount) { - final byte[] maxByteArray = new byte[maxByteCount]; - for (int i = 0; i < maxByteArray.length; i++) { - maxByteArray[i] = (byte) 0xff; - } - return maxByteArray; - } - - /** - * Create a byte array which is multiple given bytes - * - * @return byte array - */ - public static byte[] multiple(final byte[] srcBytes, final int multiNum) { - if (multiNum <= 0) { - return new byte[0]; - } - final byte[] result = new byte[srcBytes.length * multiNum]; - for (int i = 0; i < multiNum; i++) { - System.arraycopy(srcBytes, 0, result, i * srcBytes.length, srcBytes.length); - } - return result; - } - - private static final char[] HEX_CHARS = - {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}; - - /** - * Convert a byte range into a hex string - */ - public static String toHex(final byte[] b, final int offset, final int length) { - Preconditions.checkArgument(length <= Integer.MAX_VALUE / 2); - final int numChars = length * 2; - final char[] ch = new char[numChars]; - for (int i = 0; i < numChars; i += 2) { - final byte d = b[offset + i / 2]; - ch[i] = HEX_CHARS[(d >> 4) & 0x0F]; - ch[i + 1] = HEX_CHARS[d & 0x0F]; - } - return new String(ch); - } - - /** - * Convert a byte array into a hex string - */ - public static String toHex(final byte[] b) { - return toHex(b, 0, b.length); - } - - private static int hexCharToNibble(final char ch) { - if (ch <= '9' && ch >= '0') { - return ch - '0'; - } else if (ch >= 'a' && ch <= 'f') { - return ch - 'a' + 10; - } else if (ch >= 'A' && ch <= 'F') { - return ch - 'A' + 10; - } - throw new IllegalArgumentException("Invalid hex char: " + ch); - } - - private static byte hexCharsToByte(final char c1, final char c2) { - return (byte) ((hexCharToNibble(c1) << 4) | hexCharToNibble(c2)); - } - - /** - * Create a byte array from a string of hash digits. The length of the string must be a multiple - * of 2 - */ - public static byte[] fromHex(final String hex) { - Preconditions.checkArgument(hex.length() % 2 == 0, "length must be a multiple of 2"); - final int len = hex.length(); - final byte[] b = new byte[len / 2]; - for (int i = 0; i < len; i += 2) { - b[i / 2] = hexCharsToByte(hex.charAt(i), hex.charAt(i + 1)); - } - return b; - } - - /** - * Find index of passed delimiter. - * - * @return Index of delimiter having started from start of b moving rightward. - */ - public static int searchDelimiterIndex(final byte[] b, final int offset, final int length, - final int delimiter) { - if (b == null) { - throw new IllegalArgumentException("Passed buffer is null"); - } - int result = -1; - for (int i = offset; i < length + offset; i++) { - if (b[i] == delimiter) { - result = i; - break; - } - } - return result; - } - - /** - * Find index of passed delimiter walking from end of buffer backwards. - * - * @return Index of delimiter - */ - public static int searchDelimiterIndexInReverse(final byte[] b, final int offset, - final int length, final int delimiter) { - if (b == null) { - throw new IllegalArgumentException("Passed buffer is null"); - } - int result = -1; - for (int i = (offset + length) - 1; i >= offset; i--) { - if (b[i] == delimiter) { - result = i; - break; - } - } - return result; - } - - public static int findCommonPrefix(final byte[] left, final byte[] right, final int leftLength, final int rightLength, - final int leftOffset, final int rightOffset) { - return CommonPrefixerHolder.BEST_COMMON_PREFIXER.findCommonPrefix(left, leftOffset, leftLength, - right, rightOffset, rightLength); - } -} diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBasePlatformDependent.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBasePlatformDependent.java deleted file mode 100644 index dbbaa9ed73d..00000000000 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBasePlatformDependent.java +++ /dev/null @@ -1,577 +0,0 @@ -/* - * Copy of https://github.com/apache/hbase-thirdparty/blob/master/hbase-unsafe/src/main/java/org/apache/hadoop/hbase/unsafe/HBasePlatformDependent.java - * to avoid having to pull in all of hbase to use the util methods. - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stroom.bytebuffer.hbase; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.security.AccessController; -import java.security.PrivilegedAction; -import java.util.function.BiConsumer; - -/** - * Delegate all methods of {@link HBaseUnsafeInternal} and {@link HBaseSignalInternal} so we will - * not touch the actual {@code sun.misc.Unsafe} and {@code sun.misc.Signal} classes until we - * actually call the methods. - */ -public final class HBasePlatformDependent { - - private static final String CLASS_NAME = "sun.misc.Unsafe"; - private static final Logger LOG = LoggerFactory.getLogger(HBasePlatformDependent.class); - private static final boolean AVAIL; - private static final boolean UNALIGNED; - - static { - AVAIL = AccessController.doPrivileged(new PrivilegedAction() { - - @Override - public Boolean run() { - return checkAvailable(); - } - }); - UNALIGNED = checkUnaligned(); - } - - private static boolean checkAvailable() { - try { - final Class clazz = Class.forName(CLASS_NAME); - final Field f = clazz.getDeclaredField("theUnsafe"); - f.setAccessible(true); - final Object theUnsafe = f.get(null); - if (theUnsafe == null) { - LOG.warn("Could not get static instance from sun.misc.Unsafe"); - return false; - } - // Check for availability of all methods used by UnsafeAccess - Method m; - try { - m = clazz.getDeclaredMethod("arrayBaseOffset", Class.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing arrayBaseOffset(Class)"); - return false; - } - m = clazz.getDeclaredMethod("copyMemory", Object.class, long.class, Object.class, - long.class, long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing copyMemory(Object,long,Object,long,long)"); - return false; - } - m = clazz.getDeclaredMethod("getByte", Object.class, long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing getByte(Object,long)"); - return false; - } - m = clazz.getDeclaredMethod("getShort", long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing getShort(long)"); - return false; - } - m = clazz.getDeclaredMethod("getShort", Object.class, long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing getShort(Object,long)"); - return false; - } - m = clazz.getDeclaredMethod("getInt", long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing getInt(long)"); - return false; - } - m = clazz.getDeclaredMethod("getInt", Object.class, long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing getInt(Object,long)"); - return false; - } - m = clazz.getDeclaredMethod("getLong", long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing getLong(long)"); - return false; - } - m = clazz.getDeclaredMethod("getLong", Object.class, long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing getLong(Object,long)"); - return false; - } - m = clazz.getDeclaredMethod("putByte", long.class, byte.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putByte(long,byte)"); - return false; - } - m = clazz.getDeclaredMethod("putByte", Object.class, long.class, byte.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putByte(Object,long,byte)"); - return false; - } - m = clazz.getDeclaredMethod("putShort", long.class, short.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putShort(long,short)"); - return false; - } - m = clazz.getDeclaredMethod("putShort", Object.class, long.class, short.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putShort(Object,long,short)"); - return false; - } - m = clazz.getDeclaredMethod("putInt", long.class, int.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putInt(long,int)"); - return false; - } - m = clazz.getDeclaredMethod("putInt", Object.class, long.class, int.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putInt(Object,long,int)"); - return false; - } - m = clazz.getDeclaredMethod("putLong", long.class, long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putLong(long,long)"); - return false; - } - m = clazz.getDeclaredMethod("putLong", Object.class, long.class, long.class); - if (m == null) { - LOG.warn("sun.misc.Unsafe is missing putLong(Object,long,long)"); - return false; - } - // theUnsafe is accessible and all methods are available - return true; - } catch (final Throwable e) { - LOG.warn("sun.misc.Unsafe is missing one or more required methods", e); - } - } catch (final Throwable e) { - LOG.warn("sun.misc.Unsafe is not available/accessible", e); - } - return false; - } - - private static boolean checkUnaligned() { - // When Unsafe itself is not available/accessible consider unaligned as false. - if (!AVAIL) { - return false; - } - final String arch = System.getProperty("os.arch"); - if ("ppc64".equals(arch) || "ppc64le".equals(arch) || "aarch64".equals(arch)) { - // java.nio.Bits.unaligned() wrongly returns false on ppc (JDK-8165231), - return true; - } - try { - // Using java.nio.Bits#unaligned() to check for unaligned-access capability - final Class clazz = Class.forName("java.nio.Bits"); - final Method m = clazz.getDeclaredMethod("unaligned"); - m.setAccessible(true); - return (Boolean) m.invoke(null); - } catch (final Exception e) { - LOG.warn("java.nio.Bits#unaligned() check failed." - + "Unsafe based read/write of primitive types won't be used", e); - } - return false; - } - - /** - * @return true when running JVM is having sun's Unsafe package available in it and it is - * accessible. - */ - public static boolean isUnsafeAvailable() { - return AVAIL; - } - - /** - * @return true when running JVM is having sun's Unsafe package available in it and underlying - * system having unaligned-access capability. - */ - public static boolean unaligned() { - return UNALIGNED; - } - - private HBasePlatformDependent() { - // private constructor to avoid instantiation - } - - public static int getInt(final Object o, final long offset) { - return HBaseUnsafeInternal.getInt(o, offset); - } - - public static void putInt(final Object o, final long offset, final int x) { - HBaseUnsafeInternal.putInt(o, offset, x); - } - - public static Object getObject(final Object o, final long offset) { - return HBaseUnsafeInternal.getObject(o, offset); - } - - public static void putObject(final Object o, final long offset, final Object x) { - HBaseUnsafeInternal.putObject(o, offset, x); - } - - public static boolean getBoolean(final Object o, final long offset) { - return HBaseUnsafeInternal.getBoolean(o, offset); - } - - public static void putBoolean(final Object o, final long offset, final boolean x) { - HBaseUnsafeInternal.putBoolean(o, offset, x); - } - - public static byte getByte(final Object o, final long offset) { - return HBaseUnsafeInternal.getByte(o, offset); - } - - public static void putByte(final Object o, final long offset, final byte x) { - HBaseUnsafeInternal.putByte(o, offset, x); - } - - public static short getShort(final Object o, final long offset) { - return HBaseUnsafeInternal.getShort(o, offset); - } - - public static void putShort(final Object o, final long offset, final short x) { - HBaseUnsafeInternal.putShort(o, offset, x); - } - - public static char getChar(final Object o, final long offset) { - return HBaseUnsafeInternal.getChar(o, offset); - } - - public static void putChar(final Object o, final long offset, final char x) { - HBaseUnsafeInternal.putChar(o, offset, x); - } - - public static long getLong(final Object o, final long offset) { - return HBaseUnsafeInternal.getLong(o, offset); - } - - public static void putLong(final Object o, final long offset, final long x) { - HBaseUnsafeInternal.putLong(o, offset, x); - } - - public static float getFloat(final Object o, final long offset) { - return HBaseUnsafeInternal.getFloat(o, offset); - } - - public static void putFloat(final Object o, final long offset, final float x) { - HBaseUnsafeInternal.putFloat(o, offset, x); - } - - public static double getDouble(final Object o, final long offset) { - return HBaseUnsafeInternal.getDouble(o, offset); - } - - public static void putDouble(final Object o, final long offset, final double x) { - HBaseUnsafeInternal.putDouble(o, offset, x); - } - - public static byte getByte(final long address) { - return HBaseUnsafeInternal.getByte(address); - } - - public static void putByte(final long address, final byte x) { - HBaseUnsafeInternal.putByte(address, x); - } - - public static short getShort(final long address) { - return HBaseUnsafeInternal.getShort(address); - } - - public static void putShort(final long address, final short x) { - HBaseUnsafeInternal.putShort(address, x); - } - - public static char getChar(final long address) { - return HBaseUnsafeInternal.getChar(address); - } - - public static void putChar(final long address, final char x) { - HBaseUnsafeInternal.putChar(address, x); - } - - public static int getInt(final long address) { - return HBaseUnsafeInternal.getInt(address); - } - - public static void putInt(final long address, final int x) { - HBaseUnsafeInternal.putInt(address, x); - } - - public static long getLong(final long address) { - return HBaseUnsafeInternal.getLong(address); - } - - public static void putLong(final long address, final long x) { - HBaseUnsafeInternal.putLong(address, x); - } - - public static float getFloat(final long address) { - return HBaseUnsafeInternal.getFloat(address); - } - - public static void putFloat(final long address, final float x) { - HBaseUnsafeInternal.putFloat(address, x); - } - - public static double getDouble(final long address) { - return HBaseUnsafeInternal.getDouble(address); - } - - public static void putDouble(final long address, final double x) { - HBaseUnsafeInternal.putDouble(address, x); - } - - public static long getAddress(final long address) { - return HBaseUnsafeInternal.getAddress(address); - } - - public static void putAddress(final long address, final long x) { - HBaseUnsafeInternal.putAddress(address, x); - } - - public static long allocateMemory(final long bytes) { - return HBaseUnsafeInternal.allocateMemory(bytes); - } - - public static long reallocateMemory(final long address, final long bytes) { - return HBaseUnsafeInternal.reallocateMemory(address, bytes); - } - - public static void setMemory(final Object o, final long offset, final long bytes, final byte value) { - HBaseUnsafeInternal.setMemory(o, offset, bytes, value); - } - - public static void setMemory(final long address, final long bytes, final byte value) { - HBaseUnsafeInternal.setMemory(address, bytes, value); - } - - public static void copyMemory(final Object srcBase, final long srcOffset, final Object destBase, final long destOffset, - final long bytes) { - HBaseUnsafeInternal.copyMemory(srcBase, srcOffset, destBase, destOffset, bytes); - } - - public static void copyMemory(final long srcAddress, final long destAddress, final long bytes) { - HBaseUnsafeInternal.copyMemory(srcAddress, destAddress, bytes); - } - - public static void freeMemory(final long address) { - HBaseUnsafeInternal.freeMemory(address); - } - - public static long staticFieldOffset(final Field f) { - return HBaseUnsafeInternal.staticFieldOffset(f); - } - - public static long objectFieldOffset(final Field f) { - return HBaseUnsafeInternal.objectFieldOffset(f); - } - - public static Object staticFieldBase(final Field f) { - return HBaseUnsafeInternal.staticFieldBase(f); - } - - public static boolean shouldBeInitialized(final Class c) { - return HBaseUnsafeInternal.shouldBeInitialized(c); - } - - public static void ensureClassInitialized(final Class c) { - HBaseUnsafeInternal.ensureClassInitialized(c); - } - - public static int arrayBaseOffset(final Class arrayClass) { - return HBaseUnsafeInternal.arrayBaseOffset(arrayClass); - } - - public static int arrayIndexScale(final Class arrayClass) { - return HBaseUnsafeInternal.arrayIndexScale(arrayClass); - } - - public static int addressSize() { - return HBaseUnsafeInternal.addressSize(); - } - - public static int pageSize() { - return HBaseUnsafeInternal.pageSize(); - } - -// public static Class defineClass(String name, byte[] b, int off, int len, ClassLoader loader, -// ProtectionDomain protectionDomain) { -// return HBaseUnsafeInternal.defineClass(name, b, off, len, loader, protectionDomain); -// } - -// public static Class defineAnonymousClass(Class hostClass, byte[] data, Object[] cpPatches) { -// return HBaseUnsafeInternal.defineAnonymousClass(hostClass, data, cpPatches); -// } - - public static Object allocateInstance(final Class cls) throws InstantiationException { - return HBaseUnsafeInternal.allocateInstance(cls); - } - - public static void throwException(final Throwable ee) { - HBaseUnsafeInternal.throwException(ee); - } - - public static boolean compareAndSwapObject(final Object o, final long offset, final Object expected, final Object x) { - return HBaseUnsafeInternal.compareAndSwapObject(o, offset, expected, x); - } - - public static boolean compareAndSwapInt(final Object o, final long offset, final int expected, final int x) { - return HBaseUnsafeInternal.compareAndSwapInt(o, offset, expected, x); - } - - public static boolean compareAndSwapLong(final Object o, final long offset, final long expected, final long x) { - return HBaseUnsafeInternal.compareAndSwapLong(o, offset, expected, x); - } - - public static Object getObjectVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getObjectVolatile(o, offset); - } - - public static void putObjectVolatile(final Object o, final long offset, final Object x) { - HBaseUnsafeInternal.putObjectVolatile(o, offset, x); - } - - public static int getIntVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getIntVolatile(o, offset); - } - - public static void putIntVolatile(final Object o, final long offset, final int x) { - HBaseUnsafeInternal.putIntVolatile(o, offset, x); - } - - public static boolean getBooleanVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getBooleanVolatile(o, offset); - } - - public static void putBooleanVolatile(final Object o, final long offset, final boolean x) { - HBaseUnsafeInternal.putBooleanVolatile(o, offset, x); - } - - public static byte getByteVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getByteVolatile(o, offset); - } - - public static void putByteVolatile(final Object o, final long offset, final byte x) { - HBaseUnsafeInternal.putByteVolatile(o, offset, x); - } - - public static short getShortVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getShortVolatile(o, offset); - } - - public static void putShortVolatile(final Object o, final long offset, final short x) { - HBaseUnsafeInternal.putShortVolatile(o, offset, x); - } - - public static char getCharVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getCharVolatile(o, offset); - } - - public static void putCharVolatile(final Object o, final long offset, final char x) { - HBaseUnsafeInternal.putCharVolatile(o, offset, x); - } - - public static long getLongVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getLongVolatile(o, offset); - } - - public static void putLongVolatile(final Object o, final long offset, final long x) { - HBaseUnsafeInternal.putLongVolatile(o, offset, x); - } - - public static float getFloatVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getFloatVolatile(o, offset); - } - - public static void putFloatVolatile(final Object o, final long offset, final float x) { - HBaseUnsafeInternal.putFloatVolatile(o, offset, x); - } - - public static double getDoubleVolatile(final Object o, final long offset) { - return HBaseUnsafeInternal.getDoubleVolatile(o, offset); - } - - public static void putDoubleVolatile(final Object o, final long offset, final double x) { - HBaseUnsafeInternal.putDoubleVolatile(o, offset, x); - } - - public static void putOrderedObject(final Object o, final long offset, final Object x) { - HBaseUnsafeInternal.putOrderedObject(o, offset, x); - } - - public static void putOrderedInt(final Object o, final long offset, final int x) { - HBaseUnsafeInternal.putOrderedInt(o, offset, x); - } - - public static void putOrderedLong(final Object o, final long offset, final long x) { - HBaseUnsafeInternal.putOrderedLong(o, offset, x); - } - - public static void unpark(final Object thread) { - HBaseUnsafeInternal.unpark(thread); - } - - public static void park(final boolean isAbsolute, final long time) { - HBaseUnsafeInternal.park(isAbsolute, time); - } - - public static int getLoadAverage(final double[] loadavg, final int nelems) { - return HBaseUnsafeInternal.getLoadAverage(loadavg, nelems); - } - - public static int getAndAddInt(final Object o, final long offset, final int delta) { - return HBaseUnsafeInternal.getAndAddInt(o, offset, delta); - } - - public static long getAndAddLong(final Object o, final long offset, final long delta) { - return HBaseUnsafeInternal.getAndAddLong(o, offset, delta); - } - - public static int getAndSetInt(final Object o, final long offset, final int newValue) { - return HBaseUnsafeInternal.getAndSetInt(o, offset, newValue); - } - - public static long getAndSetLong(final Object o, final long offset, final long newValue) { - return HBaseUnsafeInternal.getAndSetLong(o, offset, newValue); - } - - public static Object getAndSetObject(final Object o, final long offset, final Object newValue) { - return HBaseUnsafeInternal.getAndSetObject(o, offset, newValue); - } - - public static void loadFence() { - HBaseUnsafeInternal.loadFence(); - } - - public static void storeFence() { - HBaseUnsafeInternal.storeFence(); - } - - public static void fullFence() { - HBaseUnsafeInternal.fullFence(); - } - - /** - * Delegate {@code sun.misc.Signal}. - * - * @param signal the name of the signal, such as 'HUP'. - * @param handler the handler of the signal, the first parameter is the number of the signal, - * while the second one is the name of the sinal. - */ - public static void handle(final String signal, final BiConsumer handler) { - HBaseSignalInternal.handle(signal, handler); - } -} diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBaseSignalInternal.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBaseSignalInternal.java deleted file mode 100644 index 7bb87dd9332..00000000000 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBaseSignalInternal.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copy of https://github.com/apache/hbase-thirdparty/blob/master/hbase-unsafe/src/main/java/org/apache/hadoop/hbase/unsafe/HBaseSignalInternal.java - * to avoid having to pull in all of hbase to use the util methods. - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stroom.bytebuffer.hbase; - -import sun.misc.Signal; - -import java.util.function.BiConsumer; - -/** - * Delegation of {@code sun.misc.Signal}. - */ -@SuppressWarnings("restriction") -public final class HBaseSignalInternal { - - private HBaseSignalInternal() { - } - - public static void handle(final String signal, final BiConsumer handler) { - Signal.handle(new Signal(signal), s -> handler.accept(s.getNumber(), s.getName())); - } -} diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBaseUnsafeInternal.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBaseUnsafeInternal.java deleted file mode 100644 index ca07d4494b3..00000000000 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/HBaseUnsafeInternal.java +++ /dev/null @@ -1,420 +0,0 @@ -/* - * Copy of https://github.com/apache/hbase-thirdparty/blob/master/hbase-unsafe/src/main/java/org/apache/hadoop/hbase/unsafe/HBaseUnsafeInternal.java - * to avoid having to pull in all of hbase to use the util methods. - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stroom.bytebuffer.hbase; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import sun.misc.Unsafe; - -import java.lang.reflect.Field; -import java.security.AccessController; -import java.security.PrivilegedAction; - -/** - * Delegate all the method in sun.misc.Unsafe. - */ -@SuppressWarnings("restriction") -final class HBaseUnsafeInternal { - - private static final Logger LOG = LoggerFactory.getLogger(HBaseUnsafeInternal.class); - - private static final Unsafe UNSAFE; - - static { - UNSAFE = (Unsafe) AccessController.doPrivileged(new PrivilegedAction() { - @Override - public Object run() { - try { - final Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - return f.get(null); - } catch (final Throwable e) { - LOG.warn("sun.misc.Unsafe is not accessible", e); - } - return null; - } - }); - } - - private HBaseUnsafeInternal() { - } - - public static int getInt(final Object o, final long offset) { - return UNSAFE.getInt(o, offset); - } - - public static void putInt(final Object o, final long offset, final int x) { - UNSAFE.putInt(o, offset, x); - } - - public static Object getObject(final Object o, final long offset) { - return UNSAFE.getObject(o, offset); - } - - public static void putObject(final Object o, final long offset, final Object x) { - UNSAFE.putObject(o, offset, x); - } - - public static boolean getBoolean(final Object o, final long offset) { - return UNSAFE.getBoolean(o, offset); - } - - public static void putBoolean(final Object o, final long offset, final boolean x) { - UNSAFE.putBoolean(o, offset, x); - } - - public static byte getByte(final Object o, final long offset) { - return UNSAFE.getByte(o, offset); - } - - public static void putByte(final Object o, final long offset, final byte x) { - UNSAFE.putByte(o, offset, x); - } - - public static short getShort(final Object o, final long offset) { - return UNSAFE.getShort(o, offset); - } - - public static void putShort(final Object o, final long offset, final short x) { - UNSAFE.putShort(o, offset, x); - } - - public static char getChar(final Object o, final long offset) { - return UNSAFE.getChar(o, offset); - } - - public static void putChar(final Object o, final long offset, final char x) { - UNSAFE.putChar(o, offset, x); - } - - public static long getLong(final Object o, final long offset) { - return UNSAFE.getLong(o, offset); - } - - public static void putLong(final Object o, final long offset, final long x) { - UNSAFE.putLong(o, offset, x); - } - - public static float getFloat(final Object o, final long offset) { - return UNSAFE.getFloat(o, offset); - } - - public static void putFloat(final Object o, final long offset, final float x) { - UNSAFE.putFloat(o, offset, x); - } - - public static double getDouble(final Object o, final long offset) { - return UNSAFE.getDouble(o, offset); - } - - public static void putDouble(final Object o, final long offset, final double x) { - UNSAFE.putDouble(o, offset, x); - } - - public static byte getByte(final long address) { - return UNSAFE.getByte(address); - } - - public static void putByte(final long address, final byte x) { - UNSAFE.putByte(address, x); - } - - public static short getShort(final long address) { - return UNSAFE.getShort(address); - } - - public static void putShort(final long address, final short x) { - UNSAFE.putShort(address, x); - } - - public static char getChar(final long address) { - return UNSAFE.getChar(address); - } - - public static void putChar(final long address, final char x) { - UNSAFE.putChar(address, x); - } - - public static int getInt(final long address) { - return UNSAFE.getInt(address); - } - - public static void putInt(final long address, final int x) { - UNSAFE.putInt(address, x); - } - - public static long getLong(final long address) { - return UNSAFE.getLong(address); - } - - public static void putLong(final long address, final long x) { - UNSAFE.putLong(address, x); - } - - public static float getFloat(final long address) { - return UNSAFE.getFloat(address); - } - - public static void putFloat(final long address, final float x) { - UNSAFE.putFloat(address, x); - } - - public static double getDouble(final long address) { - return UNSAFE.getDouble(address); - } - - public static void putDouble(final long address, final double x) { - UNSAFE.putDouble(address, x); - } - - public static long getAddress(final long address) { - return UNSAFE.getAddress(address); - } - - public static void putAddress(final long address, final long x) { - UNSAFE.putAddress(address, x); - } - - public static long allocateMemory(final long bytes) { - return UNSAFE.allocateMemory(bytes); - } - - public static long reallocateMemory(final long address, final long bytes) { - return UNSAFE.reallocateMemory(address, bytes); - } - - public static void setMemory(final Object o, final long offset, final long bytes, final byte value) { - UNSAFE.setMemory(o, offset, bytes, value); - } - - public static void setMemory(final long address, final long bytes, final byte value) { - UNSAFE.setMemory(address, bytes, value); - } - - public static void copyMemory(final Object srcBase, final long srcOffset, final Object destBase, final long destOffset, - final long bytes) { - UNSAFE.copyMemory(srcBase, srcOffset, destBase, destOffset, bytes); - } - - public static void copyMemory(final long srcAddress, final long destAddress, final long bytes) { - UNSAFE.copyMemory(srcAddress, destAddress, bytes); - } - - public static void freeMemory(final long address) { - UNSAFE.freeMemory(address); - } - - public static long staticFieldOffset(final Field f) { - return UNSAFE.staticFieldOffset(f); - } - - public static long objectFieldOffset(final Field f) { - return UNSAFE.objectFieldOffset(f); - } - - public static Object staticFieldBase(final Field f) { - return UNSAFE.staticFieldBase(f); - } - - public static boolean shouldBeInitialized(final Class c) { - return UNSAFE.shouldBeInitialized(c); - } - - public static void ensureClassInitialized(final Class c) { - UNSAFE.ensureClassInitialized(c); - } - - public static int arrayBaseOffset(final Class arrayClass) { - return UNSAFE.arrayBaseOffset(arrayClass); - } - - public static int arrayIndexScale(final Class arrayClass) { - return UNSAFE.arrayIndexScale(arrayClass); - } - - public static int addressSize() { - return UNSAFE.addressSize(); - } - - public static int pageSize() { - return UNSAFE.pageSize(); - } - -// public static Class defineClass(String name, byte[] b, int off, int len, ClassLoader loader, -// ProtectionDomain protectionDomain) { -// return UNSAFE.defineClass(name, b, off, len, loader, protectionDomain); -// MethodHandles.lookup(). -// } -// -// public static Class defineAnonymousClass(Class hostClass, byte[] data, Object[] cpPatches) { -// return UNSAFE.defineAnonymousClass(hostClass, data, cpPatches); -// } - - public static Object allocateInstance(final Class cls) throws InstantiationException { - return UNSAFE.allocateInstance(cls); - } - - public static void throwException(final Throwable ee) { - UNSAFE.throwException(ee); - } - - public static boolean compareAndSwapObject(final Object o, final long offset, final Object expected, final Object x) { - return UNSAFE.compareAndSwapObject(o, offset, expected, x); - } - - public static boolean compareAndSwapInt(final Object o, final long offset, final int expected, final int x) { - return UNSAFE.compareAndSwapInt(o, offset, expected, x); - } - - public static boolean compareAndSwapLong(final Object o, final long offset, final long expected, final long x) { - return UNSAFE.compareAndSwapLong(o, offset, expected, x); - } - - public static Object getObjectVolatile(final Object o, final long offset) { - return UNSAFE.getObjectVolatile(o, offset); - } - - public static void putObjectVolatile(final Object o, final long offset, final Object x) { - UNSAFE.putObjectVolatile(o, offset, x); - } - - public static int getIntVolatile(final Object o, final long offset) { - return UNSAFE.getIntVolatile(o, offset); - } - - public static void putIntVolatile(final Object o, final long offset, final int x) { - UNSAFE.putIntVolatile(o, offset, x); - } - - public static boolean getBooleanVolatile(final Object o, final long offset) { - return UNSAFE.getBooleanVolatile(o, offset); - } - - public static void putBooleanVolatile(final Object o, final long offset, final boolean x) { - UNSAFE.putBooleanVolatile(o, offset, x); - } - - public static byte getByteVolatile(final Object o, final long offset) { - return UNSAFE.getByteVolatile(o, offset); - } - - public static void putByteVolatile(final Object o, final long offset, final byte x) { - UNSAFE.putByteVolatile(o, offset, x); - } - - public static short getShortVolatile(final Object o, final long offset) { - return UNSAFE.getShortVolatile(o, offset); - } - - public static void putShortVolatile(final Object o, final long offset, final short x) { - UNSAFE.putShortVolatile(o, offset, x); - } - - public static char getCharVolatile(final Object o, final long offset) { - return UNSAFE.getCharVolatile(o, offset); - } - - public static void putCharVolatile(final Object o, final long offset, final char x) { - UNSAFE.putCharVolatile(o, offset, x); - } - - public static long getLongVolatile(final Object o, final long offset) { - return UNSAFE.getLongVolatile(o, offset); - } - - public static void putLongVolatile(final Object o, final long offset, final long x) { - UNSAFE.putLongVolatile(o, offset, x); - } - - public static float getFloatVolatile(final Object o, final long offset) { - return UNSAFE.getFloatVolatile(o, offset); - } - - public static void putFloatVolatile(final Object o, final long offset, final float x) { - UNSAFE.putFloatVolatile(o, offset, x); - } - - public static double getDoubleVolatile(final Object o, final long offset) { - return UNSAFE.getDoubleVolatile(o, offset); - } - - public static void putDoubleVolatile(final Object o, final long offset, final double x) { - UNSAFE.putDoubleVolatile(o, offset, x); - } - - public static void putOrderedObject(final Object o, final long offset, final Object x) { - UNSAFE.putOrderedObject(o, offset, x); - } - - public static void putOrderedInt(final Object o, final long offset, final int x) { - UNSAFE.putOrderedInt(o, offset, x); - } - - public static void putOrderedLong(final Object o, final long offset, final long x) { - UNSAFE.putOrderedLong(o, offset, x); - } - - public static void unpark(final Object thread) { - UNSAFE.unpark(thread); - } - - public static void park(final boolean isAbsolute, final long time) { - UNSAFE.park(isAbsolute, time); - } - - public static int getLoadAverage(final double[] loadavg, final int nelems) { - return UNSAFE.getLoadAverage(loadavg, nelems); - } - - public static int getAndAddInt(final Object o, final long offset, final int delta) { - return UNSAFE.getAndAddInt(o, offset, delta); - } - - public static long getAndAddLong(final Object o, final long offset, final long delta) { - return UNSAFE.getAndAddLong(o, offset, delta); - } - - public static int getAndSetInt(final Object o, final long offset, final int newValue) { - return UNSAFE.getAndSetInt(o, offset, newValue); - } - - public static long getAndSetLong(final Object o, final long offset, final long newValue) { - return UNSAFE.getAndSetLong(o, offset, newValue); - } - - public static Object getAndSetObject(final Object o, final long offset, final Object newValue) { - return UNSAFE.getAndSetObject(o, offset, newValue); - } - - public static void loadFence() { - UNSAFE.loadFence(); - } - - public static void storeFence() { - UNSAFE.storeFence(); - } - - public static void fullFence() { - UNSAFE.fullFence(); - } - -} diff --git a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/UnsafeAccess.java b/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/UnsafeAccess.java deleted file mode 100644 index 3902a9c0979..00000000000 --- a/stroom-bytebuffer/src/main/java/stroom/bytebuffer/hbase/UnsafeAccess.java +++ /dev/null @@ -1,478 +0,0 @@ -/* - * Copy of https://github.com/apache/hbase/blob/master/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java - * to avoid having to pull in all of hbase to use the util methods. - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stroom.bytebuffer.hbase; - -import org.apache.hbase.thirdparty.io.netty.util.internal.PlatformDependent; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public final class UnsafeAccess { - - /** - * The offset to the first element in a byte array. - */ - public static final long BYTE_ARRAY_BASE_OFFSET; - - public static final boolean LITTLE_ENDIAN = - ByteOrder.nativeOrder().equals(ByteOrder.LITTLE_ENDIAN); - - // This number limits the number of bytes to copy per call to Unsafe's - // copyMemory method. A limit is imposed to allow for safepoint polling - // during a large copy - static final long UNSAFE_COPY_THRESHOLD = 1024L * 1024L; - - static { - if (HBasePlatformDependent.isUnsafeAvailable()) { - BYTE_ARRAY_BASE_OFFSET = HBasePlatformDependent.arrayBaseOffset(byte[].class); - } else { - BYTE_ARRAY_BASE_OFFSET = -1; - } - } - - private UnsafeAccess() { - } - - // APIs to read primitive data from a byte[] using Unsafe way - - /** - * Converts a byte array to a short value considering it was written in big-endian format. - * - * @param bytes byte array - * @param offset offset into array - * @return the short value - */ - public static short toShort(final byte[] bytes, final int offset) { - if (LITTLE_ENDIAN) { - return Short - .reverseBytes(HBasePlatformDependent.getShort(bytes, offset + BYTE_ARRAY_BASE_OFFSET)); - } else { - return HBasePlatformDependent.getShort(bytes, offset + BYTE_ARRAY_BASE_OFFSET); - } - } - - /** - * Converts a byte array to an int value considering it was written in big-endian format. - * - * @param bytes byte array - * @param offset offset into array - * @return the int value - */ - public static int toInt(final byte[] bytes, final int offset) { - if (LITTLE_ENDIAN) { - return Integer - .reverseBytes(HBasePlatformDependent.getInt(bytes, offset + BYTE_ARRAY_BASE_OFFSET)); - } else { - return HBasePlatformDependent.getInt(bytes, offset + BYTE_ARRAY_BASE_OFFSET); - } - } - - /** - * Converts a byte array to a long value considering it was written in big-endian format. - * - * @param bytes byte array - * @param offset offset into array - * @return the long value - */ - public static long toLong(final byte[] bytes, final int offset) { - if (LITTLE_ENDIAN) { - return Long - .reverseBytes(HBasePlatformDependent.getLong(bytes, offset + BYTE_ARRAY_BASE_OFFSET)); - } else { - return HBasePlatformDependent.getLong(bytes, offset + BYTE_ARRAY_BASE_OFFSET); - } - } - - // APIs to write primitive data to a byte[] using Unsafe way - - /** - * Put a short value out to the specified byte array position in big-endian format. - * - * @param bytes the byte array - * @param offset position in the array - * @param val short to write out - * @return incremented offset - */ - public static int putShort(final byte[] bytes, final int offset, short val) { - if (LITTLE_ENDIAN) { - val = Short.reverseBytes(val); - } - HBasePlatformDependent.putShort(bytes, offset + BYTE_ARRAY_BASE_OFFSET, val); - return offset + Bytes.SIZEOF_SHORT; - } - - /** - * Put an int value out to the specified byte array position in big-endian format. - * - * @param bytes the byte array - * @param offset position in the array - * @param val int to write out - * @return incremented offset - */ - public static int putInt(final byte[] bytes, final int offset, int val) { - if (LITTLE_ENDIAN) { - val = Integer.reverseBytes(val); - } - HBasePlatformDependent.putInt(bytes, offset + BYTE_ARRAY_BASE_OFFSET, val); - return offset + Bytes.SIZEOF_INT; - } - - /** - * Put a long value out to the specified byte array position in big-endian format. - * - * @param bytes the byte array - * @param offset position in the array - * @param val long to write out - * @return incremented offset - */ - public static int putLong(final byte[] bytes, final int offset, long val) { - if (LITTLE_ENDIAN) { - val = Long.reverseBytes(val); - } - HBasePlatformDependent.putLong(bytes, offset + BYTE_ARRAY_BASE_OFFSET, val); - return offset + Bytes.SIZEOF_LONG; - } - - // APIs to read primitive data from a ByteBuffer using Unsafe way - - /** - * Reads a short value at the given buffer's offset considering it was written in big-endian - * format. - * - * @return short value at offset - */ - public static short toShort(final ByteBuffer buf, final int offset) { - if (LITTLE_ENDIAN) { - return Short.reverseBytes(getAsShort(buf, offset)); - } - return getAsShort(buf, offset); - } - - /** - * Reads a short value at the given Object's offset considering it was written in big-endian - * format. - * - * @return short value at offset - */ - public static short toShort(final Object ref, final long offset) { - if (LITTLE_ENDIAN) { - return Short.reverseBytes(HBasePlatformDependent.getShort(ref, offset)); - } - return HBasePlatformDependent.getShort(ref, offset); - } - - /** - * Reads bytes at the given offset as a short value. - * - * @return short value at offset - */ - private static short getAsShort(final ByteBuffer buf, final int offset) { - if (buf.isDirect()) { - return HBasePlatformDependent.getShort(directBufferAddress(buf) + offset); - } - return HBasePlatformDependent.getShort(buf.array(), - BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset); - } - - /** - * Reads an int value at the given buffer's offset considering it was written in big-endian - * format. - * - * @return int value at offset - */ - public static int toInt(final ByteBuffer buf, final int offset) { - if (LITTLE_ENDIAN) { - return Integer.reverseBytes(getAsInt(buf, offset)); - } - return getAsInt(buf, offset); - } - - /** - * Reads a int value at the given Object's offset considering it was written in big-endian format. - * - * @return int value at offset - */ - public static int toInt(final Object ref, final long offset) { - if (LITTLE_ENDIAN) { - return Integer.reverseBytes(HBasePlatformDependent.getInt(ref, offset)); - } - return HBasePlatformDependent.getInt(ref, offset); - } - - /** - * Reads bytes at the given offset as an int value. - * - * @return int value at offset - */ - private static int getAsInt(final ByteBuffer buf, final int offset) { - if (buf.isDirect()) { - return HBasePlatformDependent.getInt(directBufferAddress(buf) + offset); - } - return HBasePlatformDependent.getInt(buf.array(), - BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset); - } - - /** - * Reads a long value at the given buffer's offset considering it was written in big-endian - * format. - * - * @return long value at offset - */ - public static long toLong(final ByteBuffer buf, final int offset) { - if (LITTLE_ENDIAN) { - return Long.reverseBytes(getAsLong(buf, offset)); - } - return getAsLong(buf, offset); - } - - /** - * Reads a long value at the given Object's offset considering it was written in big-endian - * format. - * - * @return long value at offset - */ - public static long toLong(final Object ref, final long offset) { - if (LITTLE_ENDIAN) { - return Long.reverseBytes(HBasePlatformDependent.getLong(ref, offset)); - } - return HBasePlatformDependent.getLong(ref, offset); - } - - /** - * Reads bytes at the given offset as a long value. - * - * @return long value at offset - */ - private static long getAsLong(final ByteBuffer buf, final int offset) { - if (buf.isDirect()) { - return HBasePlatformDependent.getLong(directBufferAddress(buf) + offset); - } - return HBasePlatformDependent.getLong(buf.array(), - BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset); - } - - /** - * Returns the byte at the given offset - * - * @param buf the buffer to read - * @param offset the offset at which the byte has to be read - * @return the byte at the given offset - */ - public static byte toByte(final ByteBuffer buf, final int offset) { - if (buf.isDirect()) { - return HBasePlatformDependent.getByte(directBufferAddress(buf) + offset); - } else { - return HBasePlatformDependent.getByte(buf.array(), - BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset); - } - } - - /** - * Returns the byte at the given offset of the object - * - * @return the byte at the given offset - */ - public static byte toByte(final Object ref, final long offset) { - return HBasePlatformDependent.getByte(ref, offset); - } - - /** - * Put an int value out to the specified ByteBuffer offset in big-endian format. - * - * @param buf the ByteBuffer to write to - * @param offset offset in the ByteBuffer - * @param val int to write out - * @return incremented offset - */ - public static int putInt(final ByteBuffer buf, final int offset, int val) { - if (LITTLE_ENDIAN) { - val = Integer.reverseBytes(val); - } - if (buf.isDirect()) { - HBasePlatformDependent.putInt(directBufferAddress(buf) + offset, val); - } else { - HBasePlatformDependent.putInt(buf.array(), - offset + buf.arrayOffset() + BYTE_ARRAY_BASE_OFFSET, val); - } - return offset + Bytes.SIZEOF_INT; - } - - // APIs to copy data. This will be direct memory location copy and will be much faster - - /** - * Copies the bytes from given array's offset to length part into the given buffer. - * - * @param src source array - * @param srcOffset offset into source buffer - * @param dest destination buffer - * @param destOffset offset into destination buffer - * @param length length of data to copy - */ - public static void copy(final byte[] src, final int srcOffset, final ByteBuffer dest, final int destOffset, final int length) { - long destAddress = destOffset; - Object destBase = null; - if (dest.isDirect()) { - destAddress = destAddress + directBufferAddress(dest); - } else { - destAddress = destAddress + BYTE_ARRAY_BASE_OFFSET + dest.arrayOffset(); - destBase = dest.array(); - } - final long srcAddress = srcOffset + BYTE_ARRAY_BASE_OFFSET; - unsafeCopy(src, srcAddress, destBase, destAddress, length); - } - - private static void unsafeCopy(final Object src, long srcAddr, final Object dst, long destAddr, long len) { - while (len > 0) { - final long size = (len > UNSAFE_COPY_THRESHOLD) - ? UNSAFE_COPY_THRESHOLD - : len; - HBasePlatformDependent.copyMemory(src, srcAddr, dst, destAddr, size); - len -= size; - srcAddr += size; - destAddr += size; - } - } - - /** - * Copies specified number of bytes from given offset of {@code src} ByteBuffer to the - * {@code dest} array. - * - * @param src source buffer - * @param srcOffset offset into source buffer - * @param dest destination array - * @param destOffset offset into destination buffer - * @param length length of data to copy - */ - public static void copy(final ByteBuffer src, final int srcOffset, final byte[] dest, final int destOffset, final int length) { - long srcAddress = srcOffset; - Object srcBase = null; - if (src.isDirect()) { - srcAddress = srcAddress + directBufferAddress(src); - } else { - srcAddress = srcAddress + BYTE_ARRAY_BASE_OFFSET + src.arrayOffset(); - srcBase = src.array(); - } - final long destAddress = destOffset + BYTE_ARRAY_BASE_OFFSET; - unsafeCopy(srcBase, srcAddress, dest, destAddress, length); - } - - /** - * Copies specified number of bytes from given offset of {@code src} buffer into the {@code dest} - * buffer. - * - * @param src source buffer - * @param srcOffset offset into source buffer - * @param dest destination buffer - * @param destOffset offset into destination buffer - * @param length length of data to copy - */ - public static void copy(final ByteBuffer src, final int srcOffset, final ByteBuffer dest, final int destOffset, - final int length) { - final long srcAddress; - final long destAddress; - Object srcBase = null, destBase = null; - if (src.isDirect()) { - srcAddress = srcOffset + directBufferAddress(src); - } else { - srcAddress = (long) srcOffset + src.arrayOffset() + BYTE_ARRAY_BASE_OFFSET; - srcBase = src.array(); - } - if (dest.isDirect()) { - destAddress = destOffset + directBufferAddress(dest); - } else { - destAddress = destOffset + BYTE_ARRAY_BASE_OFFSET + dest.arrayOffset(); - destBase = dest.array(); - } - unsafeCopy(srcBase, srcAddress, destBase, destAddress, length); - } - - // APIs to add primitives to BBs - - /** - * Put a short value out to the specified BB position in big-endian format. - * - * @param buf the byte buffer - * @param offset position in the buffer - * @param val short to write out - * @return incremented offset - */ - public static int putShort(final ByteBuffer buf, final int offset, short val) { - if (LITTLE_ENDIAN) { - val = Short.reverseBytes(val); - } - if (buf.isDirect()) { - HBasePlatformDependent.putShort(directBufferAddress(buf) + offset, val); - } else { - HBasePlatformDependent.putShort(buf.array(), - BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset, val); - } - return offset + Bytes.SIZEOF_SHORT; - } - - /** - * Put a long value out to the specified BB position in big-endian format. - * - * @param buf the byte buffer - * @param offset position in the buffer - * @param val long to write out - * @return incremented offset - */ - public static int putLong(final ByteBuffer buf, final int offset, long val) { - if (LITTLE_ENDIAN) { - val = Long.reverseBytes(val); - } - if (buf.isDirect()) { - HBasePlatformDependent.putLong(directBufferAddress(buf) + offset, val); - } else { - HBasePlatformDependent.putLong(buf.array(), - BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset, val); - } - return offset + Bytes.SIZEOF_LONG; - } - - /** - * Put a byte value out to the specified BB position in big-endian format. - * - * @param buf the byte buffer - * @param offset position in the buffer - * @param b byte to write out - * @return incremented offset - */ - public static int putByte(final ByteBuffer buf, final int offset, final byte b) { - if (buf.isDirect()) { - HBasePlatformDependent.putByte(directBufferAddress(buf) + offset, b); - } else { - HBasePlatformDependent.putByte(buf.array(), - BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset, b); - } - return offset + 1; - } - - public static long directBufferAddress(final ByteBuffer buf) { - return PlatformDependent.directBufferAddress(buf); - } - - public static void freeDirectBuffer(final ByteBuffer buffer) { - // here we just use the method in netty - PlatformDependent.freeDirectBuffer(buffer); - } -} diff --git a/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestByteBufferUtils.java b/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestByteBufferUtils.java index aaebf0c88ed..e73b2c8c895 100644 --- a/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestByteBufferUtils.java +++ b/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestByteBufferUtils.java @@ -405,8 +405,7 @@ void testCopyPerformance() { src.flip(); final Map> funcMap = Map.of( - "Simple", this::doSimpleCopyTest, - "Hadoop", this::doHadoopCopyTest); + "Simple", this::doSimpleCopyTest); for (int j = 0; j < rounds; j++) { final int round = j; @@ -433,10 +432,6 @@ void testCopyPerformance() { } } - private void doHadoopCopyTest(final ByteBuffer src, final ByteBuffer dest) { - stroom.bytebuffer.hbase.ByteBufferUtils.copyFromBufferToBuffer(src, dest); - } - private void doSimpleCopyTest(final ByteBuffer src, final ByteBuffer dest) { dest.put(src); } diff --git a/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestPooledByteBufferOutputStream.java b/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestPooledByteBufferOutputStream.java index 8fff29fc41b..0b1133ca45e 100644 --- a/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestPooledByteBufferOutputStream.java +++ b/stroom-bytebuffer/src/test/java/stroom/bytebuffer/TestPooledByteBufferOutputStream.java @@ -308,10 +308,31 @@ void testWrite_byteBuffer() throws IOException { .isGreaterThan(BYTES); assertThat(pooledBuffer.limit()) .isEqualTo(BYTES); - assertThat(ByteBufferUtils.compareTo( - byteBuffer, 5, BYTES, - pooledBuffer, 0, BYTES)) - .isZero(); + assertThat(byteBuffer.slice(5, BYTES)).isEqualTo(pooledBuffer.slice(0, BYTES)); + } + } + + @Test + void testWriteLong() throws IOException { + final ByteBufferPool byteBufferPool = getByteBufferPool(); + try (final PooledByteBufferOutputStream pooledByteBufferOutputStream = new PooledByteBufferOutputStream( + byteBufferPool, + BYTES)) { + pooledByteBufferOutputStream.writeLong(234556L); + final ByteBuffer pooledBuffer = pooledByteBufferOutputStream.getByteBuffer(); + final byte[] actual = ByteBufferUtils.toBytes(pooledBuffer); + final byte[] expected = new byte[BYTES]; + oldPutLong(expected, 0, 234556L); + assertThat(actual).isEqualTo(expected); + } + } + + private int oldPutLong(final byte[] bytes, final int offset, long val) { + for (int i = offset + 7; i > offset; i--) { + bytes[i] = (byte) val; + val >>>= 8; } + bytes[offset] = (byte) val; + return offset + BYTES; } } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/databases/ValueStoreDb.java b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/databases/ValueStoreDb.java index bf2415e9f7e..56c0e74c84c 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/databases/ValueStoreDb.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/databases/ValueStoreDb.java @@ -214,7 +214,7 @@ public ByteBuffer getOrCreateKey(final Txn writeTxn, short lastKeyId = -1; while (isFound) { - if (ValueStoreKeySerde.compareValueHashCode(startKey, cursor.key()) != 0) { + if (!ValueStoreKeySerde.valueHashCodeEquals(startKey, cursor.key())) { // cursor key has a different hashcode to ours so we can stop looping break; } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/StagingRefDataValueSerde.java b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/StagingRefDataValueSerde.java index fc697b6e642..f85ec95f3f5 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/StagingRefDataValueSerde.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/StagingRefDataValueSerde.java @@ -1,7 +1,6 @@ package stroom.pipeline.refdata.store.offheapstore.serdes; import stroom.bytebuffer.PooledByteBufferOutputStream; -import stroom.bytebuffer.hbase.Bytes; import stroom.lmdb.serde.Deserializer; import stroom.lmdb.serde.Serde; import stroom.lmdb.serde.Serializer; @@ -68,7 +67,7 @@ public ByteBuffer serialize(final PooledByteBufferOutputStream pooledByteBufferO try { pooledByteBufferOutputStream.write(stagingRefDataValue.getTypeId()); - pooledByteBufferOutputStream.write(Bytes.toBytes(refDataValue.getValueHashCode(valueStoreHashAlgorithm))); + pooledByteBufferOutputStream.writeLong(refDataValue.getValueHashCode(valueStoreHashAlgorithm)); final ByteBuffer refDataValueBuffer = genericRefDataValueSerde.serialize( pooledByteBufferOutputStream, refDataValue); diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreKeySerde.java b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreKeySerde.java index 2fb6cfad63f..1e7ff4ad7b3 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreKeySerde.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreKeySerde.java @@ -23,6 +23,7 @@ import stroom.util.logging.LogUtil; import java.nio.ByteBuffer; +import java.util.Objects; public class ValueStoreKeySerde implements Serde { @@ -88,17 +89,19 @@ public static short extractId(final ByteBuffer byteBuffer) { } /** - * Compare the valueHashCode part of both byte buffers, comparing in byte form + * Check equality of valueHashCode part of both byte buffers */ - public static int compareValueHashCode(final ByteBuffer thisBuffer, final ByteBuffer thatBuffer) { + public static boolean valueHashCodeEquals(final ByteBuffer thisBuffer, final ByteBuffer thatBuffer) { try { - return ByteBufferUtils.compareTo( - thisBuffer, VALUE_HASH_CODE_OFFSET, VALUE_HASH_CODE_BYTES, - thatBuffer, VALUE_HASH_CODE_OFFSET, VALUE_HASH_CODE_BYTES); + return Objects.equals(getValueHashCodeSlice(thisBuffer), getValueHashCodeSlice(thatBuffer)); } catch (final Exception e) { - throw new RuntimeException(LogUtil.message("Error comparing [{}] & [{}]", + throw new RuntimeException(LogUtil.message("Error checking equality [{}] & [{}]", ByteBufferUtils.byteBufferInfo(thisBuffer), ByteBufferUtils.byteBufferInfo(thatBuffer)), e); } } + + private static ByteBuffer getValueHashCodeSlice(final ByteBuffer byteBuffer) { + return byteBuffer.slice(VALUE_HASH_CODE_OFFSET, VALUE_HASH_CODE_BYTES); + } } diff --git a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreMetaSerde.java b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreMetaSerde.java index 846e50186cc..a3ba1f63f33 100644 --- a/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreMetaSerde.java +++ b/stroom-pipeline/src/main/java/stroom/pipeline/refdata/store/offheapstore/serdes/ValueStoreMetaSerde.java @@ -25,14 +25,19 @@ public class ValueStoreMetaSerde implements Serde { private static final Logger LOGGER = LoggerFactory.getLogger(ValueStoreMetaSerde.class); private static final UnsignedBytes REF_COUNT_UNSIGNED_BYTES = UnsignedBytesInstances.THREE; - private static final byte[] REF_COUNT_ZERO = REF_COUNT_UNSIGNED_BYTES.toBytes(0); - private static final byte[] REF_COUNT_ONE = REF_COUNT_UNSIGNED_BYTES.toBytes(1); + private static final int REFERENCE_COUNT_BYTES = REF_COUNT_UNSIGNED_BYTES.length(); private static final int TYPE_ID_OFFSET = 0; private static final int TYPE_ID_BYTES = 1; + /** + * The offset of the first byte of the reference count + */ private static final int REFERENCE_COUNT_OFFSET = TYPE_ID_OFFSET + TYPE_ID_BYTES; + /** + * The offset of the last byte of the reference count + */ + public static final int REFERENCE_COUNT_END_OFFSET = REFERENCE_COUNT_OFFSET + REFERENCE_COUNT_BYTES - 1; - private static final int REFERENCE_COUNT_BYTES = REF_COUNT_UNSIGNED_BYTES.length(); private static final int BUFFER_CAPACITY = TYPE_ID_BYTES + REFERENCE_COUNT_BYTES; @Override @@ -53,8 +58,7 @@ public ValueStoreMeta deserialize(final ByteBuffer byteBuffer) { @Override public void serialize(final ByteBuffer byteBuffer, final ValueStoreMeta valueStoreMeta) { - - byteBuffer.put((byte) valueStoreMeta.getTypeId()); + byteBuffer.put(valueStoreMeta.getTypeId()); REF_COUNT_UNSIGNED_BYTES.put(byteBuffer, valueStoreMeta.getReferenceCount()); byteBuffer.flip(); } @@ -77,15 +81,21 @@ public int extractReferenceCount(final ByteBuffer byteBuffer) { * @return True if the reference count is one or zero. */ public boolean isLastReference(final ByteBuffer byteBuffer) { - // Ever so slightly cheaper than extracting the count and checking the long value - // TODO could maybe use ByteBufferUtils.equals - return stroom.bytebuffer.hbase.ByteBufferUtils.compareTo( - REF_COUNT_ONE, 0, REFERENCE_COUNT_BYTES, - byteBuffer, REFERENCE_COUNT_OFFSET, REFERENCE_COUNT_BYTES) == 0L - || - stroom.bytebuffer.hbase.ByteBufferUtils.compareTo( - REF_COUNT_ZERO, 0, REFERENCE_COUNT_BYTES, - byteBuffer, REFERENCE_COUNT_OFFSET, REFERENCE_COUNT_BYTES) == 0L; + // This relies on UnsignedBytes serialising 1 to 001 and 0 to 000. + + // Check the last byte first as low numbers are more likely than high numbers. + final byte lastByte = byteBuffer.get(REFERENCE_COUNT_END_OFFSET); + if (lastByte != 1 && lastByte != 0) { + return false; + } + + // Everything else should be zero + for (int j = REFERENCE_COUNT_END_OFFSET - 1; j >= REFERENCE_COUNT_OFFSET; j--) { + if (byteBuffer.get(j) != 0) { + return false; + } + } + return true; } public void cloneAndDecrementRefCount(final ByteBuffer sourceBuffer, final ByteBuffer destBuffer) { diff --git a/stroom-pipeline/src/test/java/stroom/pipeline/refdata/store/offheapstore/serdes/TestValueStoreMetaSerde.java b/stroom-pipeline/src/test/java/stroom/pipeline/refdata/store/offheapstore/serdes/TestValueStoreMetaSerde.java index 14c789bcb67..57bb44293ea 100644 --- a/stroom-pipeline/src/test/java/stroom/pipeline/refdata/store/offheapstore/serdes/TestValueStoreMetaSerde.java +++ b/stroom-pipeline/src/test/java/stroom/pipeline/refdata/store/offheapstore/serdes/TestValueStoreMetaSerde.java @@ -85,6 +85,11 @@ Stream testIsLastReference() { .addCase(1, true) .addCase(2, false) .addCase(3, false) + .addCase(8, false) + .addCase(10, false) + .addCase(100, false) + .addCase(1000, false) + .addCase(10000, false) .addCase(16_000_000, false) .build(); } @@ -174,6 +179,7 @@ void testDecrementRefCount() { @Override TypeLiteral getSerdeType() { - return new TypeLiteral(){}; + return new TypeLiteral() { + }; } } diff --git a/stroom-proxy/stroom-proxy-app/docker/Dockerfile b/stroom-proxy/stroom-proxy-app/docker/Dockerfile index c4edfeedf5d..8106abadbcc 100644 --- a/stroom-proxy/stroom-proxy-app/docker/Dockerfile +++ b/stroom-proxy/stroom-proxy-app/docker/Dockerfile @@ -23,7 +23,7 @@ # jstat/jmap/jcmd/etc. # Using 'openjdk' on Alpine is not fully supported so using Eclipse Temurin JDK to ensure we have a known jdk version # See https://github.com/docker-library/docs/blob/master/openjdk/README.md#openjdkversion-alpine -FROM eclipse-temurin:21.0.8_9-jdk-alpine AS stroom-base-stage +FROM eclipse-temurin:25_36-jdk-alpine-3.22 AS stroom-base-stage # curl is required for the docker healthcheck # su-exec required for running stroom as not-root user diff --git a/stroom-state/stroom-state-impl/build.gradle b/stroom-state/stroom-state-impl/build.gradle index 72bbf323f4b..2035b0d4727 100644 --- a/stroom-state/stroom-state-impl/build.gradle +++ b/stroom-state/stroom-state-impl/build.gradle @@ -39,3 +39,7 @@ dependencies { testImplementation libs.bundles.common.test.implementation testRuntimeOnly libs.bundles.common.test.runtime } + +tasks.withType(AbstractTestTask).configureEach { + failOnNoDiscoveredTests = false +} diff --git a/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/rollup/RollUpBitMask.java b/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/rollup/RollUpBitMask.java index 4285f40b41b..a3f42b6a76d 100644 --- a/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/rollup/RollUpBitMask.java +++ b/stroom-statistics/stroom-statistics-impl-hbase/src/main/java/stroom/statistics/impl/hbase/rollup/RollUpBitMask.java @@ -16,8 +16,6 @@ package stroom.statistics.impl.hbase.rollup; -import stroom.bytebuffer.hbase.Bytes; - import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -66,11 +64,9 @@ public class RollUpBitMask { } private final short mask; - private final byte[] maskAsBytes; private RollUpBitMask(final short mask) { this.mask = mask; - this.maskAsBytes = Bytes.toBytes(mask); } /** @@ -140,28 +136,6 @@ public static RollUpBitMask fromRolledupTagList(final String allTags, final Stri return fromTagPositions(rolledUpTagPositions); } - /** - * @param allTags String containing all tags in any order delimited by comma, - * e.g. "tag1,tag3,tag2" - * @param rolledUpTags String containing just those tags that are to be rolled up in - * any order e.g. "tag3,tag2" - * @return The byte value of the mask - */ - public static byte[] byteValueFromTagList(final String allTags, final String rolledUpTags) { - return fromRolledupTagList(allTags, rolledUpTags).asBytes(); - } - - /** - * @param allTags String containing all tags in any order delimited by comma, - * e.g. "tag1,tag3,tag2" - * @param rolledUpTags String containing just those tags that are to be rolled up in - * any order e.g. "tag3,tag2" - * @return The int value of the mask - */ - public static int intValueFromTagList(final String allTags, final String rolledUpTags) { - return fromRolledupTagList(allTags, rolledUpTags).asShort(); - } - /** * Generates a {@link RollUpBitMask} from the passed list of rolled up tag * positions. It uses a cache of tag positions to save it having to build @@ -261,16 +235,6 @@ private static short setMaskValueAtPosition(final int position, final short exis } - /** - * Constructor - * - * @param bytes The byte array to convert from - * @return A {@link RollUpBitMask} object built from the byte array - */ - public static RollUpBitMask fromBytes(final byte[] bytes) { - return new RollUpBitMask(Bytes.toShort(bytes)); - } - /** * @param tagCount The number of tags you want permutations for * @return A set of {@link RollUpBitMask} objects, one for each permutation @@ -529,17 +493,6 @@ public RollUpBitMask convert(final Map newToOldFieldPositionMa return RollUpBitMask.fromTagPositions(new TreeSet<>(rolledUpFieldPositions)); } - public short asShort() { - return this.mask; - } - - /** - * @return The mask as a byte array - */ - public byte[] asBytes() { - return this.maskAsBytes; - } - /** * Output has mask position zero on the right */ @@ -550,29 +503,16 @@ public String toString() { } @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + mask; - result = prime * result + Arrays.hashCode(maskAsBytes); - return result; + public boolean equals(final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final RollUpBitMask that = (RollUpBitMask) o; + return mask == that.mask; } @Override - public boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final RollUpBitMask other = (RollUpBitMask) obj; - if (mask != other.mask) { - return false; - } - return Arrays.equals(maskAsBytes, other.maskAsBytes); + public int hashCode() { + return Objects.hashCode(mask); } } diff --git a/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/rollup/ByteArrayUtils.java b/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/rollup/ByteArrayUtils.java deleted file mode 100644 index 2362964ca13..00000000000 --- a/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/rollup/ByteArrayUtils.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2016 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package stroom.statistics.impl.sql.rollup; - -import stroom.bytebuffer.hbase.Bytes; - -import jakarta.xml.bind.DatatypeConverter; - -public class ByteArrayUtils { - - /** - * Private constructor to prevent instantiation - */ - private ByteArrayUtils() { - // Do nothing, should never be called. - } - - /** - * Returns a string representation of a byte array - * - * @param arr The byte array - * @return A space delimited series of byte values - */ - public static String byteArrayToString(final byte[] arr) { - final StringBuilder sb = new StringBuilder(); - for (final byte b : arr) { - sb.append(b); - sb.append(" "); - } - return sb.toString().replaceAll(" $", ""); - } - - /** - * Converts a byte array into a hex representation with a space between each - * byte e.g 00 00 01 00 05 59 B3 - * - * @param arr The byte array to convert - * @return The byte array as a string of hex values separated by a spaces - */ - public static String byteArrayToHex(final byte[] arr) { - final StringBuilder sb = new StringBuilder(); - if (arr != null) { - for (final byte b : arr) { - final byte[] oneByteArr = new byte[1]; - oneByteArr[0] = b; - sb.append(DatatypeConverter.printHexBinary(oneByteArr)); - sb.append(" "); - } - } - return sb.toString().replaceAll(" $", ""); - } - - /** - * @return The array represented in hex, decimal and 'hbase' forms. The - * hbase form is mix of ascii and deciaml, so an ascii char if the - * byte value exists in the ascii table - */ - public static String byteArrayToAllForms(final byte[] arr) { - return ByteArrayUtils.byteArrayToHex(arr) + " (hex) | " + ByteArrayUtils.byteArrayToString(arr) + " (dec) | " - + Bytes.toStringBinary(arr) + " (hbase)"; - } -} diff --git a/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/rollup/RollUpBitMask.java b/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/rollup/RollUpBitMask.java index 8a1aa98003b..fdd19a4177f 100644 --- a/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/rollup/RollUpBitMask.java +++ b/stroom-statistics/stroom-statistics-impl-sql/src/main/java/stroom/statistics/impl/sql/rollup/RollUpBitMask.java @@ -16,10 +16,9 @@ package stroom.statistics.impl.sql.rollup; -import stroom.bytebuffer.hbase.Bytes; - import jakarta.xml.bind.DatatypeConverter; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -68,11 +67,17 @@ public class RollUpBitMask { } private final short mask; - private final byte[] maskAsBytes; private RollUpBitMask(final short mask) { this.mask = mask; - this.maskAsBytes = Bytes.toBytes(mask); + } + + public static byte[] shortToBytes(final short s) { + return ByteBuffer.allocate(Short.BYTES).putShort(s).flip().array(); + } + + public static short bytesToShort(final byte[] bytes) { + return ByteBuffer.wrap(bytes).getShort(); } /** @@ -270,7 +275,7 @@ private static short setMaskValueAtPosition(final int position, final short exis * @return A {@link RollUpBitMask} object built from the byte array */ public static RollUpBitMask fromBytes(final byte[] bytes) { - return new RollUpBitMask(Bytes.toShort(bytes)); + return new RollUpBitMask(bytesToShort(bytes)); } /** @@ -539,11 +544,11 @@ public short asShort() { * @return The mask as a byte array */ public byte[] asBytes() { - return this.maskAsBytes; + return shortToBytes(mask); } public String asHexString() { - return DatatypeConverter.printHexBinary(this.maskAsBytes); + return DatatypeConverter.printHexBinary(asBytes()); } /** @@ -556,29 +561,16 @@ public String toString() { } @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + mask; - result = prime * result + Arrays.hashCode(maskAsBytes); - return result; + public boolean equals(final Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + final RollUpBitMask that = (RollUpBitMask) o; + return mask == that.mask; } @Override - public boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final RollUpBitMask other = (RollUpBitMask) obj; - if (mask != other.mask) { - return false; - } - return Arrays.equals(maskAsBytes, other.maskAsBytes); + public int hashCode() { + return Objects.hashCode(mask); } } diff --git a/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/rollup/TestRollUpBitMask.java b/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/rollup/TestRollUpBitMask.java index 95d4bc3da8a..8c2ad66f3a9 100644 --- a/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/rollup/TestRollUpBitMask.java +++ b/stroom-statistics/stroom-statistics-impl-sql/src/test/java/stroom/statistics/impl/sql/rollup/TestRollUpBitMask.java @@ -17,6 +17,7 @@ package stroom.statistics.impl.sql.rollup; +import stroom.bytebuffer.ByteArrayUtils; import stroom.test.common.util.test.StroomUnitTest; import jakarta.xml.bind.DatatypeConverter; diff --git a/unreleased_changes/20250929_155237_704__0.md b/unreleased_changes/20250929_155237_704__0.md new file mode 100644 index 00000000000..b53942e84f9 --- /dev/null +++ b/unreleased_changes/20250929_155237_704__0.md @@ -0,0 +1,55 @@ +* Uplift docker image JDK to `eclipse-temurin:25_36-jdk-alpine-3.22`. + + +```sh +# ONLY the top line will be included as a change entry in the CHANGELOG. +# The entry should be in GitHub flavour markdown and should be written on a SINGLE +# line with no hard breaks. You can have multiple change files for a single GitHub issue. +# The entry should be written in the imperative mood, i.e. 'Fix nasty bug' rather than +# 'Fixed nasty bug'. +# +# Examples of acceptable entries are: +# +# +# * Issue **123** : Fix bug with an associated GitHub issue in this repository +# +# * Issue **namespace/other-repo#456** : Fix bug with an associated GitHub issue in another repository +# +# * Fix bug with no associated GitHub issue. + + +# -------------------------------------------------------------------------------- +# The following is random text to make this file unique for git's change detection +# yr776q2zhysVAjaqa7trGdJeu2B1ZhnkO4Pwcht5ORyHEBL5W85b9N7vkh4qIg4NvQIkYxWsuto3FiId +# e8q8SkwmP12NBl2fgKR80xzKywXWSO77SECSd6sjGkiCyJenhi2K0ErE6i9rFhgCNGFchCuf2baQTd59 +# So4FO7Q6bN2rqf7qAcFvn3AWAnatNky3qhG24B08TsCEnFIzkxQir3tsosRWQy2OQJHQfzSvGUQ7lejn +# BSMdnihwjUFfdb4IM3MSvpOANV0sTohrkbPEJi7DEB3Nco59JAjmfvtSTyVp2b3kcjjmRuRlPeDAcbaS +# lcLNoJm4vmvPlc7SrcHWRBaQ7lXDmju3KkxGlh4aYy5psWQOPn1Szpia1jXJaCi7Jth5u1w5ivmh51CL +# YT5culhn07GngjuHPgWZTb22LZ3aKf5CNGVpEHpFhyoVZpAbZ4n3oOvOaLdPLQb92i6NAsyGQ9DncRZZ +# uomCrZ5H4Sb7utLUJRdPZZGdhs4ycvo2YavOBizzRsDcPV9GuwywIUsBNZKowayXK50Kypk0Ng9sOLRk +# Ed08oyLRrmj0c7MS2Q26tUIGbjPT7OjV5s26IHX6wm2F83r0mA4RbQJgmi4KZQNbCIFTPsMxSte7llEH +# BU6BHJyHRN7mI8TfEElkoQbYTcWrA7oTT2J3yTcx1Q0Lf4cCGGVXtaECubHzxmzHCpVaqBziogth4mqT +# bxAspho777Qn7X4wBlOPA8eSGZIIDshajXFje36W4CVPAlyYph0FmdFEOR2EwZA0Uy0lDlywDskg0J4B +# Fzmvqi4Cih0uCPmep4mFPVwlomfpiZtN5Z1eTVDq5qG5l3UN4HWhHYVFoM9xCcyjpbLELF2G4Ie5BHoW +# pFdCtJfSCzoOrzsoIDDFf5P53ihFMb5R5sjwHLRWp5j7NATbOWfgLGeT5gYETn4n2v9atjIx431q0XHL +# mimd4DDZQ8wsg8ci9Tx7bNIfodueBwNLYy4ClFn5ucq8an6rIsCk9208fB7JrWelqj9CUTwkgFWmfkLl +# 3lkkvYwQh9We4iyNgVfWQzwsXs3vgeiddB8ZQcYVIK8EVUf4fOmqSOXLpURaGIBlgRrcWGiksZDoT37s +# PjA4P4g2E8Vu1QRtvAjFHwqMAIQoEK4OU7b1X12P221oOZuyzAdT4SGX8G5TgxVR7S3hhNGbDBEDhIQu +# 1hOCoqV08YWlzHaGeyLFLlk71l709cp9ITUZsMwA4KxGXwxj0c1Pkel3aat9hdFg5lqD4PfdyTUW1HrN +# h1zLgeC1XtegO1EmA71DBkc3O2il4r381GEKMAJRCAmKDskKS8tlxIaDYiEp7PsO7MQ9q5QW3vwVH3d0 +# WwHYswoIoqb6LESZsMmE6vMcKQrTY3maJahBJ7meAuFnjUcsVfooKR93jYDu8G4Kvijhews1WQxLU9uF +# 6IEDieeS93HjqPyYdVceFmrZXlk83jeAU7ggkjF0y9Ko8HYfqiinbjHyN4QAdmXoVtHSOOGwrK5BPNNV +# KBwZCTZFIOySTH3xwe4qvLDL2x5feebWGwQtpCZt13DS3JHFlD3ItNwOcgmEMykTzRgVCjh3pAD0Yxuh +# ldJucKYT54DBOzqpBIsHkCwo5ZE2Rr5Nqu7VQdXqQRMjLK35jEPJtdCibTd3Q4kwPWYkLONpKXra5reL +# fvvE7gvlaWc5h18yoJtjNXmXaWro2fDfJVnWXqU5qBaRIomh99A8GlX69Mo3hfRO0BLivX6fIE5SV0Rr +# BODUaOxCTKNiLJHzMYrF0SNK2U0avVQxdIkvWXLOlKSBqPcakHM3xk6ek0aiKhPPCcZfZwyomUtA2hzy +# CA0NT9VXTUu0dzBeEQvyENdu77OSVqfYUiSLasGp3Rp3jcBICq9dYgbYvBE6ZFKqLUkRlWLd4m0DGrLY +# knZFXy73WoA6zu9k1C9nySsYmqvK73FLlUNFskIqapjpnpoAehtLnqQ4KW1yAJwdJCyZNLHqhU35xMCc +# AluVt9moa3smPSO4gFMkk6E6DwtuXBOfaJFiXtsx7La3Uw3DnTcmKIxwMvjZqnEmbWo5K2UdUTt6YqVF +# M3exlMKz63FVkpzbNt1myCaTZtVKMhdolyJr0nMn5ScKr7SzHwmKXYkBRwvZKDptxExCLDBrhMz1ZOX0 +# WpMFMXKOsXYmOugVizE4G3GYr7OF68HL2TlHiR3z7A4VwZ4miV9p89kiw9mlUUXkqpwrDoP3xWNPuejj +# kXlKGmLulKZq3bxWvbfoe0ZL8RhZuAdvtPW2WODI974980Y2AFAGK18F7Cvcku6kzz9kFHzmALN0WX1E +# Bze5UASkMbe4rxghVcyYQwjstRqn8h7HhQGyDRQMi2uLW1AQ8HxigKJGxZE8sRSjGMB60k40T1aeh8aO +# -------------------------------------------------------------------------------- + +```