|
| 1 | +// simplify configuration |
| 2 | +#![allow(dead_code)] |
| 3 | + |
| 4 | +// Note: `usize_leading_zeros` happens to produce the correct `usize::leading_zeros(0)` |
| 5 | +// value without a explicit zero check. Zero is probably common enough that it could |
| 6 | +// warrant adding a zero check at the beginning, but `__clzsi2` has a precondition that |
| 7 | +// `x != 0`. Compilers will insert the check for zero in cases where it is needed. |
| 8 | + |
| 9 | +/// Returns the number of leading binary zeros in `x`. |
| 10 | +pub fn usize_leading_zeros_default(x: usize) -> usize { |
| 11 | + // The basic idea is to test if the higher bits of `x` are zero and bisect the number |
| 12 | + // of leading zeros. It is possible for all branches of the bisection to use the same |
| 13 | + // code path by conditionally shifting the higher parts down to let the next bisection |
| 14 | + // step work on the higher or lower parts of `x`. Instead of starting with `z == 0` |
| 15 | + // and adding to the number of zeros, it is slightly faster to start with |
| 16 | + // `z == usize::MAX.count_ones()` and subtract from the potential number of zeros, |
| 17 | + // because it simplifies the final bisection step. |
| 18 | + let mut x = x; |
| 19 | + // the number of potential leading zeros |
| 20 | + let mut z = usize::MAX.count_ones() as usize; |
| 21 | + // a temporary |
| 22 | + let mut t: usize; |
| 23 | + #[cfg(target_pointer_width = "64")] |
| 24 | + { |
| 25 | + t = x >> 32; |
| 26 | + if t != 0 { |
| 27 | + z -= 32; |
| 28 | + x = t; |
| 29 | + } |
| 30 | + } |
| 31 | + #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] |
| 32 | + { |
| 33 | + t = x >> 16; |
| 34 | + if t != 0 { |
| 35 | + z -= 16; |
| 36 | + x = t; |
| 37 | + } |
| 38 | + } |
| 39 | + t = x >> 8; |
| 40 | + if t != 0 { |
| 41 | + z -= 8; |
| 42 | + x = t; |
| 43 | + } |
| 44 | + t = x >> 4; |
| 45 | + if t != 0 { |
| 46 | + z -= 4; |
| 47 | + x = t; |
| 48 | + } |
| 49 | + t = x >> 2; |
| 50 | + if t != 0 { |
| 51 | + z -= 2; |
| 52 | + x = t; |
| 53 | + } |
| 54 | + // the last two bisections are combined into one conditional |
| 55 | + t = x >> 1; |
| 56 | + if t != 0 { |
| 57 | + z - 2 |
| 58 | + } else { |
| 59 | + z - x |
| 60 | + } |
| 61 | + |
| 62 | + // We could potentially save a few cycles by using the LUT trick from |
| 63 | + // "https://embeddedgurus.com/state-space/2014/09/ |
| 64 | + // fast-deterministic-and-portable-counting-leading-zeros/". |
| 65 | + // However, 256 bytes for a LUT is too large for embedded use cases. We could remove the |
| 66 | + // last 3 bisections and use this 16 byte LUT for the rest of the work: |
| 67 | + //const LUT: [u8; 16] = [0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]; |
| 68 | + //z -= LUT[x] as usize; |
| 69 | + //z |
| 70 | + // However, it ends up generating about the same number of instructions. When benchmarked |
| 71 | + // on x86_64, it is slightly faster to use the LUT, but this is probably because of OOO |
| 72 | + // execution effects. Changing to using a LUT and branching is risky for smaller cores. |
| 73 | +} |
| 74 | + |
| 75 | +// The above method does not compile well on RISC-V, producing code with short branches or |
| 76 | +// using an excessively long branchless solution. This method takes advantage of the |
| 77 | +// set-if-less-than instruction on RISC-V that allows `(x >= power-of-two) as usize` to be |
| 78 | +// branchless. |
| 79 | + |
| 80 | +/// Returns the number of leading binary zeros in `x`. |
| 81 | +pub fn usize_leading_zeros_riscv(x: usize) -> usize { |
| 82 | + let mut x = x; |
| 83 | + // the number of potential leading zeros |
| 84 | + let mut z = usize::MAX.count_ones() as usize; |
| 85 | + // a temporary |
| 86 | + let mut t: usize; |
| 87 | + |
| 88 | + // RISC-V does not have a set-if-greater-than-or-equal instruction and |
| 89 | + // `(x >= power-of-two) as usize` will get compiled into two instructions, but this is |
| 90 | + // still the most optimal method. A conditional set can only be turned into a single |
| 91 | + // immediate instruction if `x` is compared with an immediate `imm` (that can fit into |
| 92 | + // 12 bits) like `x < imm` but not `imm < x` (because the immediate is always on the |
| 93 | + // right). If we try to save an instruction by using `x < imm` for each bisection, we |
| 94 | + // have to shift `x` left and compare with powers of two approaching `usize::MAX + 1`, |
| 95 | + // but the immediate will never fit into 12 bits and never save an instruction. |
| 96 | + #[cfg(target_pointer_width = "64")] |
| 97 | + { |
| 98 | + // If the upper 32 bits of `x` are not all 0, `t` is set to `1 << 5`, otherwise `t` is |
| 99 | + // set to 0. |
| 100 | + t = ((x >= (1 << 32)) as usize) << 5; |
| 101 | + // If `t` was set to `1 << 5`, then the upper 32 bits are shifted down for the next step |
| 102 | + // to process. |
| 103 | + x >>= t; |
| 104 | + // If `t` was set to `1 << 5`, then we subtract 32 from the number of potential leading |
| 105 | + // zeros |
| 106 | + z -= t; |
| 107 | + } |
| 108 | + #[cfg(any(target_pointer_width = "32", target_pointer_width = "64"))] |
| 109 | + { |
| 110 | + t = ((x >= (1 << 16)) as usize) << 4; |
| 111 | + x >>= t; |
| 112 | + z -= t; |
| 113 | + } |
| 114 | + t = ((x >= (1 << 8)) as usize) << 3; |
| 115 | + x >>= t; |
| 116 | + z -= t; |
| 117 | + t = ((x >= (1 << 4)) as usize) << 2; |
| 118 | + x >>= t; |
| 119 | + z -= t; |
| 120 | + t = ((x >= (1 << 2)) as usize) << 1; |
| 121 | + x >>= t; |
| 122 | + z -= t; |
| 123 | + t = (x >= (1 << 1)) as usize; |
| 124 | + x >>= t; |
| 125 | + z -= t; |
| 126 | + // All bits except the LSB are guaranteed to be zero for this final bisection step. |
| 127 | + // If `x != 0` then `x == 1` and subtracts one potential zero from `z`. |
| 128 | + z - x |
| 129 | +} |
| 130 | + |
| 131 | +#[cfg(not(any(target_arch = "riscv32", target_arch = "riscv64")))] |
| 132 | +intrinsics! { |
| 133 | + #[maybe_use_optimized_c_shim] |
| 134 | + #[cfg(any( |
| 135 | + target_pointer_width = "16", |
| 136 | + target_pointer_width = "32", |
| 137 | + target_pointer_width = "64" |
| 138 | + ))] |
| 139 | + /// Returns the number of leading binary zeros in `x`. |
| 140 | + pub extern "C" fn __clzsi2(x: usize) -> usize { |
| 141 | + usize_leading_zeros_default(x) |
| 142 | + } |
| 143 | +} |
| 144 | + |
| 145 | +#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] |
| 146 | +intrinsics! { |
| 147 | + #[maybe_use_optimized_c_shim] |
| 148 | + #[cfg(any( |
| 149 | + target_pointer_width = "16", |
| 150 | + target_pointer_width = "32", |
| 151 | + target_pointer_width = "64" |
| 152 | + ))] |
| 153 | + /// Returns the number of leading binary zeros in `x`. |
| 154 | + pub extern "C" fn __clzsi2(x: usize) -> usize { |
| 155 | + usize_leading_zeros_riscv(x) |
| 156 | + } |
| 157 | +} |
0 commit comments