Backport AVX2 AES-GCM implementation from BoringSSL

This uses the AVX2 versions of the AESENC and PCLMULQDQ instructions; on
Zen 3 this provides an up to 80% performance improvement.

Original source:
https://github.com/google/boringssl/blob/13840dd094f9e9c1b00a7368aa25e656554221f1/gen/bcm/aes-gcm-avx2-x86_64-linux.S

See the original BoringSSL commit at
google/boringssl@3b6e1be.

Signed-off-by: Joel Low <joel@joelsplace.sg>

Author: lowjoel

config/toolchain-simd.m4

@@ -24,6 +24,8 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD], [
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_AES
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_PCLMULQDQ
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES
+			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVEOPT
 			ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVES
@@ -426,6 +428,46 @@ AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_MOVBE], [
 	])
 ])
 
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VAES], [
+	AC_MSG_CHECKING([whether host toolchain supports VAES])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+			__asm__ __volatile__("vaesenc %ymm0, %ymm1, %ymm0");
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_VAES], 1, [Define if host toolchain supports VAES])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
+dnl #
+dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_VPCLMULQDQ], [
+	AC_MSG_CHECKING([whether host toolchain supports VPCLMULQDQ])
+
+	AC_LINK_IFELSE([AC_LANG_SOURCE([
+	[
+		void main()
+		{
+			__asm__ __volatile__("vpclmulqdq %0, %%ymm4, %%ymm3, %%ymm5" :: "i"(0));
+		}
+	]])], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE([HAVE_VPCLMULQDQ], 1, [Define if host toolchain supports VPCLMULQDQ])
+	], [
+		AC_MSG_RESULT([no])
+	])
+])
+
 dnl #
 dnl # ZFS_AC_CONFIG_TOOLCHAIN_CAN_BUILD_XSAVE
 dnl #

include/os/linux/kernel/linux/simd_x86.h

@@ -597,6 +597,32 @@ zfs_movbe_available(void)
 #endif
 }
 
+/*
+ * Check if VAES instruction set is available
+ */
+static inline boolean_t
+zfs_vaes_available(void)
+{
+#if defined(X86_FEATURE_VAES)
+	return (!!boot_cpu_has(X86_FEATURE_VAES));
+#else
+	return (B_FALSE);
+#endif
+}
+
+/*
+ * Check if VPCLMULQDQ instruction set is available
+ */
+static inline boolean_t
+zfs_vpclmulqdq_available(void)
+{
+#if defined(X86_FEATURE_VPCLMULQDQ)
+	return (!!boot_cpu_has(X86_FEATURE_VPCLMULQDQ));
+#else
+	return (B_FALSE);
+#endif
+}
+
 /*
  * Check if SHA_NI instruction set is available
  */

lib/libicp/Makefile.am

@@ -69,6 +69,7 @@ nodist_libicp_la_SOURCES += \
 	module/icp/asm-x86_64/aes/aes_aesni.S \
 	module/icp/asm-x86_64/modes/gcm_pclmulqdq.S \
 	module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S \
+	module/icp/asm-x86_64/modes/aesni-gcm-avx2.S \
 	module/icp/asm-x86_64/modes/ghash-x86_64.S \
 	module/icp/asm-x86_64/sha2/sha256-x86_64.S \
 	module/icp/asm-x86_64/sha2/sha512-x86_64.S \

lib/libspl/include/sys/simd.h

@@ -101,7 +101,9 @@ typedef enum cpuid_inst_sets {
 	AES,
 	PCLMULQDQ,
 	MOVBE,
-	SHA_NI
+	SHA_NI,
+	VAES,
+	VPCLMULQDQ
 } cpuid_inst_sets_t;
 
 /*
@@ -126,6 +128,8 @@ typedef struct cpuid_feature_desc {
 #define	_AES_BIT		(1U << 25)
 #define	_PCLMULQDQ_BIT		(1U << 1)
 #define	_MOVBE_BIT		(1U << 22)
+#define	_VAES_BIT		(1U << 9)
+#define	_VPCLMULQDQ_BIT		(1U << 10)
 #define	_SHA_NI_BIT		(1U << 29)
 
 /*
@@ -156,6 +160,8 @@ static const cpuid_feature_desc_t cpuid_features[] = {
 	[PCLMULQDQ]	= {1U, 0U, _PCLMULQDQ_BIT,	ECX	},
 	[MOVBE]		= {1U, 0U, _MOVBE_BIT,		ECX	},
 	[SHA_NI]	= {7U, 0U, _SHA_NI_BIT,		EBX	},
+	[VAES]		= {7U, 0U, _VAES_BIT,		ECX	},
+	[VPCLMULQDQ]	= {7U, 0U, _VPCLMULQDQ_BIT,	ECX	},
 };
 
 /*
@@ -230,6 +236,8 @@ CPUID_FEATURE_CHECK(aes, AES);
 CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ);
 CPUID_FEATURE_CHECK(movbe, MOVBE);
 CPUID_FEATURE_CHECK(shani, SHA_NI);
+CPUID_FEATURE_CHECK(vaes, VAES);
+CPUID_FEATURE_CHECK(vpclmulqdq, VPCLMULQDQ);
 
 /*
  * Detect register set support
@@ -380,6 +388,24 @@ zfs_shani_available(void)
 	return (__cpuid_has_shani());
 }
 
+/*
+ * Check if VAES instruction is available
+ */
+static inline boolean_t
+zfs_vaes_available(void)
+{
+	return (__cpuid_has_vaes());
+}
+
+/*
+ * Check if VPCLMULQDQ instruction is available
+ */
+static inline boolean_t
+zfs_vpclmulqdq_available(void)
+{
+	return (__cpuid_has_vpclmulqdq());
+}
+
 /*
  * AVX-512 family of instruction sets:
  *

module/Kbuild.in

@@ -135,6 +135,7 @@ ICP_OBJS_X86_64 := \
 	asm-x86_64/sha2/sha256-x86_64.o \
 	asm-x86_64/sha2/sha512-x86_64.o \
 	asm-x86_64/modes/aesni-gcm-x86_64.o \
+	asm-x86_64/modes/aesni-gcm-avx2.o \
 	asm-x86_64/modes/gcm_pclmulqdq.o \
 	asm-x86_64/modes/ghash-x86_64.o
 
@@ -178,6 +179,7 @@ $(addprefix $(obj)/icp/,$(ICP_OBJS) $(ICP_OBJS_X86) $(ICP_OBJS_X86_64) \
 
 # Suppress objtool "return with modified stack frame" warnings.
 OBJECT_FILES_NON_STANDARD_aesni-gcm-x86_64.o := y
+OBJECT_FILES_NON_STANDARD_aesni-gcm-avx2.o := y
 
 # Suppress objtool "unsupported stack pointer realignment" warnings.
 # See #6950 for the reasoning.