|
| 1 | +use sync::atomic::Ordering; |
| 2 | + |
| 3 | +#[cfg(test)] |
| 4 | +use stdsimd_test::assert_instr; |
| 5 | + |
| 6 | +/// Compare and exchange 16 bytes (128 bits) of data atomically. |
| 7 | +/// |
| 8 | +/// This intrinsic corresponds to the `cmpxchg16b` instruction on x86_64 |
| 9 | +/// processors. It performs an atomic compare-and-swap, updating the `ptr` |
| 10 | +/// memory location to `val` if the current value in memory equals `old`. |
| 11 | +/// |
| 12 | +/// # Return value |
| 13 | +/// |
| 14 | +/// This function returns the previous value at the memory location. If it is |
| 15 | +/// equal to `old` then the memory was updated to `new`. |
| 16 | +/// |
| 17 | +/// # Memory Orderings |
| 18 | +/// |
| 19 | +/// This atomic operations has the same semantics of memory orderings as |
| 20 | +/// `AtomicUsize::compare_exchange` does, only operating on 16 bytes of memory |
| 21 | +/// instead of just a pointer. |
| 22 | +/// |
| 23 | +/// For more information on memory orderings here see the `compare_exchange` |
| 24 | +/// documentation for other `Atomic*` types in the standard library. |
| 25 | +/// |
| 26 | +/// # Unsafety |
| 27 | +/// |
| 28 | +/// This method is unsafe because it takes a raw pointer and will attempt to |
| 29 | +/// read and possibly write the memory at the pointer. The pointer must also be |
| 30 | +/// aligned on a 16-byte boundary. |
| 31 | +/// |
| 32 | +/// This method also requires the `cmpxchg16b` CPU feature to be available at |
| 33 | +/// runtime to work correctly. If the CPU running the binary does not actually |
| 34 | +/// support `cmpxchg16b` and the program enters an execution path that |
| 35 | +/// eventually would reach this function the behavior is undefined. |
| 36 | +/// |
| 37 | +/// The `success` ordering must also be stronger or equal to `failure`, or this |
| 38 | +/// function call is undefined. See the `Atomic*` documentation's |
| 39 | +/// `compare_exchange` function for more information. When `compare_exchange` |
| 40 | +/// panics, this is undefined behavior. Currently this function aborts the |
| 41 | +/// process with an undefined instruction. |
| 42 | +#[inline] |
| 43 | +#[cfg_attr(test, assert_instr(cmpxchg16b, success = Ordering::SeqCst, failure = Ordering::SeqCst))] |
| 44 | +#[target_feature(enable = "cmpxchg16b")] |
| 45 | +pub unsafe fn cmpxchg16b( |
| 46 | + dst: *mut u128, |
| 47 | + old: u128, |
| 48 | + new: u128, |
| 49 | + success: Ordering, |
| 50 | + failure: Ordering, |
| 51 | +) -> u128 { |
| 52 | + use intrinsics; |
| 53 | + use sync::atomic::Ordering::*; |
| 54 | + |
| 55 | + debug_assert!(dst as usize % 16 == 0); |
| 56 | + |
| 57 | + let (val, _ok) = match (success, failure) { |
| 58 | + (Acquire, Acquire) => intrinsics::atomic_cxchg_acq(dst, old, new), |
| 59 | + (Release, Relaxed) => intrinsics::atomic_cxchg_rel(dst, old, new), |
| 60 | + (AcqRel, Acquire) => intrinsics::atomic_cxchg_acqrel(dst, old, new), |
| 61 | + (Relaxed, Relaxed) => intrinsics::atomic_cxchg_relaxed(dst, old, new), |
| 62 | + (SeqCst, SeqCst) => intrinsics::atomic_cxchg(dst, old, new), |
| 63 | + (Acquire, Relaxed) => intrinsics::atomic_cxchg_acq_failrelaxed(dst, old, new), |
| 64 | + (AcqRel, Relaxed) => intrinsics::atomic_cxchg_acqrel_failrelaxed(dst, old, new), |
| 65 | + (SeqCst, Relaxed) => intrinsics::atomic_cxchg_failrelaxed(dst, old, new), |
| 66 | + (SeqCst, Acquire) => intrinsics::atomic_cxchg_failacq(dst, old, new), |
| 67 | + |
| 68 | + // The above block is all copied from libcore, and this statement is |
| 69 | + // also copied from libcore except that it's a panic in libcore and we |
| 70 | + // have a little bit more of a lightweight panic here. |
| 71 | + _ => ::coresimd::x86::ud2(), |
| 72 | + }; |
| 73 | + val |
| 74 | +} |
0 commit comments