aesni: Add target_feature annotations to allow intrinsic inlining

Otherwise when built with nocheck feature, the AES-NI intrinsics are
not inlined causing a severe performance drop.

Author: jack-signal

aes/aesni/src/aes128.rs

@@ -19,8 +19,9 @@ pub struct Aes128 {
 impl Aes128 {
     #[inline(always)]
     pub(crate) fn encrypt8(&self, mut blocks: [__m128i; 8]) -> [__m128i; 8] {
-        let keys = self.encrypt_keys;
-        unsafe {
+        #[inline]
+        #[target_feature(enable="aes")]
+        unsafe fn aesni128_encrypt8(keys: &[__m128i; 11], blocks: &mut [__m128i; 8]) {
             xor8!(blocks, keys[0]);
             aesenc8!(blocks, keys[1]);
             aesenc8!(blocks, keys[2]);
@@ -33,13 +34,15 @@ impl Aes128 {
             aesenc8!(blocks, keys[9]);
             aesenclast8!(blocks, keys[10]);
         }
+        unsafe { aesni128_encrypt8(&self.encrypt_keys, &mut blocks) };
         blocks
     }
 
     #[inline(always)]
-    pub(crate) fn encrypt(&self, mut block: __m128i) -> __m128i {
-        let keys = self.encrypt_keys;
-        unsafe {
+    pub(crate) fn encrypt(&self, block: __m128i) -> __m128i {
+        #[inline]
+        #[target_feature(enable="aes")]
+        unsafe fn aesni128_encrypt1(keys: &[__m128i; 11], mut block: __m128i) -> __m128i {
             block = _mm_xor_si128(block, keys[0]);
             block = _mm_aesenc_si128(block, keys[1]);
             block = _mm_aesenc_si128(block, keys[2]);
@@ -52,6 +55,7 @@ impl Aes128 {
             block = _mm_aesenc_si128(block, keys[9]);
             _mm_aesenclast_si128(block, keys[10])
         }
+        unsafe { aesni128_encrypt1(&self.encrypt_keys, block) }
     }
 }
 

aes/aesni/src/aes192.rs

@@ -19,8 +19,9 @@ pub struct Aes192 {
 impl Aes192 {
     #[inline(always)]
     pub(crate) fn encrypt8(&self, mut blocks: [__m128i; 8]) -> [__m128i; 8] {
-        let keys = self.encrypt_keys;
-        unsafe {
+        #[inline]
+        #[target_feature(enable="aes")]
+        unsafe fn aesni192_encrypt8(keys: &[__m128i; 13], blocks: &mut [__m128i; 8]) {
             xor8!(blocks, keys[0]);
             aesenc8!(blocks, keys[1]);
             aesenc8!(blocks, keys[2]);
@@ -35,13 +36,15 @@ impl Aes192 {
             aesenc8!(blocks, keys[11]);
             aesenclast8!(blocks, keys[12]);
         }
+        unsafe { aesni192_encrypt8(&self.encrypt_keys, &mut blocks) };
         blocks
     }
 
     #[inline(always)]
-    pub(crate) fn encrypt(&self, mut block: __m128i) -> __m128i {
-        let keys = self.encrypt_keys;
-        unsafe {
+    pub(crate) fn encrypt(&self, block: __m128i) -> __m128i {
+        #[inline]
+        #[target_feature(enable="aes")]
+        unsafe fn aesni192_encrypt1(keys: &[__m128i; 13], mut block: __m128i) -> __m128i {
             block = _mm_xor_si128(block, keys[0]);
             block = _mm_aesenc_si128(block, keys[1]);
             block = _mm_aesenc_si128(block, keys[2]);
@@ -56,6 +59,7 @@ impl Aes192 {
             block = _mm_aesenc_si128(block, keys[11]);
             _mm_aesenclast_si128(block, keys[12])
         }
+        unsafe { aesni192_encrypt1(&self.encrypt_keys, block) }
     }
 }
 

aes/aesni/src/aes256.rs

@@ -19,8 +19,9 @@ pub struct Aes256 {
 impl Aes256 {
     #[inline(always)]
     pub(crate) fn encrypt8(&self, mut blocks: [__m128i; 8]) -> [__m128i; 8] {
-        let keys = self.encrypt_keys;
-        unsafe {
+        #[inline]
+        #[target_feature(enable="aes")]
+        unsafe fn aesni256_encrypt8(keys: &[__m128i; 15], blocks: &mut [__m128i; 8]) {
             xor8!(blocks, keys[0]);
             aesenc8!(blocks, keys[1]);
             aesenc8!(blocks, keys[2]);
@@ -37,13 +38,15 @@ impl Aes256 {
             aesenc8!(blocks, keys[13]);
             aesenclast8!(blocks, keys[14]);
         }
+        unsafe { aesni256_encrypt8(&self.encrypt_keys, &mut blocks) };
         blocks
     }
 
     #[inline(always)]
-    pub(crate) fn encrypt(&self, mut block: __m128i) -> __m128i {
-        let keys = self.encrypt_keys;
-        unsafe {
+    pub(crate) fn encrypt(&self, block: __m128i) -> __m128i {
+        #[inline]
+        #[target_feature(enable="aes")]
+        unsafe fn aesni256_encrypt1(keys: &[__m128i; 15], mut block: __m128i) -> __m128i {
             block = _mm_xor_si128(block, keys[0]);
             block = _mm_aesenc_si128(block, keys[1]);
             block = _mm_aesenc_si128(block, keys[2]);
@@ -60,6 +63,7 @@ impl Aes256 {
             block = _mm_aesenc_si128(block, keys[13]);
             _mm_aesenclast_si128(block, keys[14])
         }
+        unsafe { aesni256_encrypt1(&self.encrypt_keys, block) }
     }
 }