Add sfence after non-temporal stores on x86

ackxolotl · ackxolotl · commit 27a847756f5b · 2024-08-20T11:28:30.000+02:00
We definitely need fences there, see: https://doc.rust-lang.org/core/arch/x86/fn._mm_sfence.html Even more interesting, the discussion on NT stores in Rust: rust-lang/rust#114582 And on broken NT stores in LLVM: llvm/llvm-project#64521 Oh boy ...
diff --git a/benches/memcpy.rs b/benches/memcpy.rs
@@ -39,6 +39,7 @@ unsafe fn memcpy_avx(mut src: *const u8, mut dst: *mut u8, count: usize) {
         _mm_prefetch::<_MM_HINT_T2>(src as *const i8);
         dst = dst.add(32);
     }
+    _mm_sfence();
 }
 
 #[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "sse", target_feature = "avx512f"))]
@@ -51,6 +52,7 @@ unsafe fn memcpy_avx512(mut src: *const u8, mut dst: *mut u8, count: usize) {
         _mm_prefetch::<_MM_HINT_T2>(src as *const i8);
         dst = dst.add(64);
     }
+    _mm_sfence();
 }
 
 #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]

Original file line number	Diff line number	Diff line change
`@@ -39,6 +39,7 @@ unsafe fn memcpy_avx(mut src: const u8, mut dst: mut u8, count: usize) {`
`39`	`39`	`_mm_prefetch::<_MM_HINT_T2>(src as *const i8);`
`40`	`40`	`dst = dst.add(32);`
`41`	`41`	`}`
	`42`	`+ _mm_sfence();`
`42`	`43`	`}`
`43`	`44`
`44`	`45`	`#[cfg(all(any(target_arch = "x86", target_arch = "x86_64"), target_feature = "sse", target_feature = "avx512f"))]`
`@@ -51,6 +52,7 @@ unsafe fn memcpy_avx512(mut src: const u8, mut dst: mut u8, count: usize) {`
`51`	`52`	`_mm_prefetch::<_MM_HINT_T2>(src as *const i8);`
`52`	`53`	`dst = dst.add(64);`
`53`	`54`	`}`
	`55`	`+ _mm_sfence();`
`54`	`56`	`}`
`55`	`57`
`56`	`58`	`#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]`