Merge: Lifetimes, UBs, and Documentation!

Author: ashvardanian

.gitignore

@@ -5,6 +5,7 @@ build_release/
 build_relwithdebinfo/
 build_go/
 build_golang/
+build_test/
 build_artifacts*
 
 # Yes, everyone loves keeping this file in the history.

README.md

@@ -1089,10 +1089,10 @@ Similar to GCC, StringZilla is 32 bytes in size, and similar to Clang it can fit
 Our layout might be preferential, if you want to avoid branches.
 If you use a different compiler, you may want to check its SSO buffer size with a [simple Gist](https://gist.github.com/ashvardanian/c197f15732d9855c4e070797adf17b21).
 
-|                       | `libstdc++` in  GCC 13 | `libc++` in Clang 17 | StringZilla |
-| :-------------------- | ---------------------: | -------------------: | ----------: |
-| `sizeof(std::string)` |                     32 |                   24 |          32 |
-| Small String Capacity |                     15 |               __22__ |      __22__ |
+|                 | `libstdc++` in  GCC 13 | `libc++` in Clang 17 | StringZilla |
+| :-------------- | ---------------------: | -------------------: | ----------: |
+| String `sizeof` |                     32 |                   24 |          32 |
+| Inner Capacity  |                     15 |               __22__ |      __22__ |
 
 This design has been since ported to many high-level programming languages.
 Swift, for example, [can store 15 bytes](https://developer.apple.com/documentation/swift/substring/withutf8(_:)#discussion) in the `String` instance itself.

include/stringzilla/find.h

@@ -905,7 +905,7 @@ SZ_PUBLIC sz_cptr_t sz_find_westmere(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
     sz_locate_needle_anomalies_(n, n_length, &offset_first, &offset_mid, &offset_last);
 
     // Broadcast those characters into XMM registers.
-    int matches;
+    sz_u32_vec_t matches_vec;
     sz_u128_vec_t h_first_vec, h_mid_vec, h_last_vec, n_first_vec, n_mid_vec, n_last_vec;
     n_first_vec.xmm = _mm_set1_epi8(n[offset_first]);
     n_mid_vec.xmm = _mm_set1_epi8(n[offset_mid]);
@@ -916,14 +916,14 @@ SZ_PUBLIC sz_cptr_t sz_find_westmere(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
         h_first_vec.xmm = _mm_lddqu_si128((__m128i const *)(h + offset_first));
         h_mid_vec.xmm = _mm_lddqu_si128((__m128i const *)(h + offset_mid));
         h_last_vec.xmm = _mm_lddqu_si128((__m128i const *)(h + offset_last));
-        matches = //
+        matches_vec.i32 = //
             _mm_movemask_epi8(_mm_cmpeq_epi8(h_first_vec.xmm, n_first_vec.xmm)) &
             _mm_movemask_epi8(_mm_cmpeq_epi8(h_mid_vec.xmm, n_mid_vec.xmm)) &
             _mm_movemask_epi8(_mm_cmpeq_epi8(h_last_vec.xmm, n_last_vec.xmm));
-        while (matches) {
-            int potential_offset = sz_u32_ctz(matches);
+        while (matches_vec.u32) {
+            int potential_offset = sz_u32_ctz(matches_vec.u32);
             if (sz_equal_westmere(h + potential_offset, n, n_length)) return h + potential_offset;
-            matches &= matches - 1;
+            matches_vec.u32 &= matches_vec.u32 - 1;
         }
     }
 
@@ -940,7 +940,7 @@ SZ_PUBLIC sz_cptr_t sz_rfind_westmere(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
     sz_locate_needle_anomalies_(n, n_length, &offset_first, &offset_mid, &offset_last);
 
     // Broadcast those characters into XMM registers.
-    int matches;
+    sz_u32_vec_t matches_vec;
     sz_u128_vec_t h_first_vec, h_mid_vec, h_last_vec, n_first_vec, n_mid_vec, n_last_vec;
     n_first_vec.xmm = _mm_set1_epi8(n[offset_first]);
     n_mid_vec.xmm = _mm_set1_epi8(n[offset_mid]);
@@ -953,15 +953,15 @@ SZ_PUBLIC sz_cptr_t sz_rfind_westmere(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
         h_first_vec.xmm = _mm_lddqu_si128((__m128i const *)(h_reversed + offset_first));
         h_mid_vec.xmm = _mm_lddqu_si128((__m128i const *)(h_reversed + offset_mid));
         h_last_vec.xmm = _mm_lddqu_si128((__m128i const *)(h_reversed + offset_last));
-        matches = //
+        matches_vec.i32 = //
             _mm_movemask_epi8(_mm_cmpeq_epi8(h_first_vec.xmm, n_first_vec.xmm)) &
             _mm_movemask_epi8(_mm_cmpeq_epi8(h_mid_vec.xmm, n_mid_vec.xmm)) &
             _mm_movemask_epi8(_mm_cmpeq_epi8(h_last_vec.xmm, n_last_vec.xmm));
-        while (matches) {
-            int potential_offset = sz_u32_clz(matches) - 16;
+        while (matches_vec.u32) {
+            int potential_offset = sz_u32_clz(matches_vec.u32) - 16;
             if (sz_equal_westmere(h + h_length - n_length - potential_offset, n, n_length))
                 return h + h_length - n_length - potential_offset;
-            matches &= ~(1 << (15 - potential_offset));
+            matches_vec.u32 &= ~(1u << (15 - potential_offset));
         }
     }
 
@@ -1029,7 +1029,7 @@ SZ_PUBLIC sz_cptr_t sz_find_haswell(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n
     sz_locate_needle_anomalies_(n, n_length, &offset_first, &offset_mid, &offset_last);
 
     // Broadcast those characters into YMM registers.
-    int matches;
+    sz_u32_vec_t matches_vec;
     sz_u256_vec_t h_first_vec, h_mid_vec, h_last_vec, n_first_vec, n_mid_vec, n_last_vec;
     n_first_vec.ymm = _mm256_set1_epi8(n[offset_first]);
     n_mid_vec.ymm = _mm256_set1_epi8(n[offset_mid]);
@@ -1040,14 +1040,14 @@ SZ_PUBLIC sz_cptr_t sz_find_haswell(sz_cptr_t h, sz_size_t h_length, sz_cptr_t n
         h_first_vec.ymm = _mm256_lddqu_si256((__m256i const *)(h + offset_first));
         h_mid_vec.ymm = _mm256_lddqu_si256((__m256i const *)(h + offset_mid));
         h_last_vec.ymm = _mm256_lddqu_si256((__m256i const *)(h + offset_last));
-        matches = //
+        matches_vec.i32 = //
             _mm256_movemask_epi8(_mm256_cmpeq_epi8(h_first_vec.ymm, n_first_vec.ymm)) &
             _mm256_movemask_epi8(_mm256_cmpeq_epi8(h_mid_vec.ymm, n_mid_vec.ymm)) &
             _mm256_movemask_epi8(_mm256_cmpeq_epi8(h_last_vec.ymm, n_last_vec.ymm));
-        while (matches) {
-            int potential_offset = sz_u32_ctz(matches);
+        while (matches_vec.u32) {
+            int potential_offset = sz_u32_ctz(matches_vec.u32);
             if (sz_equal_haswell(h + potential_offset, n, n_length)) return h + potential_offset;
-            matches &= matches - 1;
+            matches_vec.u32 &= matches_vec.u32 - 1;
         }
     }
 
@@ -1065,7 +1065,7 @@ SZ_PUBLIC sz_cptr_t sz_rfind_haswell(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
     sz_locate_needle_anomalies_(n, n_length, &offset_first, &offset_mid, &offset_last);
 
     // Broadcast those characters into YMM registers.
-    int matches;
+    sz_u32_vec_t matches_vec;
     sz_u256_vec_t h_first_vec, h_mid_vec, h_last_vec, n_first_vec, n_mid_vec, n_last_vec;
     n_first_vec.ymm = _mm256_set1_epi8(n[offset_first]);
     n_mid_vec.ymm = _mm256_set1_epi8(n[offset_mid]);
@@ -1078,15 +1078,15 @@ SZ_PUBLIC sz_cptr_t sz_rfind_haswell(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
         h_first_vec.ymm = _mm256_lddqu_si256((__m256i const *)(h_reversed + offset_first));
         h_mid_vec.ymm = _mm256_lddqu_si256((__m256i const *)(h_reversed + offset_mid));
         h_last_vec.ymm = _mm256_lddqu_si256((__m256i const *)(h_reversed + offset_last));
-        matches = //
+        matches_vec.i32 = //
             _mm256_movemask_epi8(_mm256_cmpeq_epi8(h_first_vec.ymm, n_first_vec.ymm)) &
             _mm256_movemask_epi8(_mm256_cmpeq_epi8(h_mid_vec.ymm, n_mid_vec.ymm)) &
             _mm256_movemask_epi8(_mm256_cmpeq_epi8(h_last_vec.ymm, n_last_vec.ymm));
-        while (matches) {
-            int potential_offset = sz_u32_clz(matches);
+        while (matches_vec.u32) {
+            int potential_offset = sz_u32_clz(matches_vec.u32);
             if (sz_equal_haswell(h + h_length - n_length - potential_offset, n, n_length))
                 return h + h_length - n_length - potential_offset;
-            matches &= ~(1 << (31 - potential_offset));
+            matches_vec.u32 &= ~(1u << (31 - potential_offset));
         }
     }
 
@@ -1388,9 +1388,8 @@ SZ_PUBLIC sz_cptr_t sz_rfind_skylake(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
             int potential_offset = sz_u64_clz(matches);
             if (n_length <= 3 || sz_equal_skylake(h + h_length - n_length - potential_offset, n, n_length))
                 return h + h_length - n_length - potential_offset;
-            sz_assert_((matches & ((sz_u64_t)1 << (63 - potential_offset))) != 0 &&
-                       "The bit must be set before we squash it");
-            matches &= ~((sz_u64_t)1 << (63 - potential_offset));
+            sz_assert_((matches & (1ull << (63 - potential_offset))) != 0 && "The bit must be set before we squash it");
+            matches &= ~(1ull << (63 - potential_offset));
         }
     }
 
@@ -1409,9 +1408,8 @@ SZ_PUBLIC sz_cptr_t sz_rfind_skylake(sz_cptr_t h, sz_size_t h_length, sz_cptr_t
             int potential_offset = sz_u64_clz(matches);
             if (n_length <= 3 || sz_equal_skylake(h + 64 - potential_offset - 1, n, n_length))
                 return h + 64 - potential_offset - 1;
-            sz_assert_((matches & ((sz_u64_t)1 << (63 - potential_offset))) != 0 &&
-                       "The bit must be set before we squash it");
-            matches &= ~((sz_u64_t)1 << (63 - potential_offset));
+            sz_assert_((matches & (1ull << (63 - potential_offset))) != 0 && "The bit must be set before we squash it");
+            matches &= ~(1ull << (63 - potential_offset));
         }
     }
 

include/stringzilla/intersect.h

@@ -288,7 +288,7 @@ SZ_PUBLIC sz_status_t sz_sequence_intersect_serial(
     // Allocate memory for the hash table and initialize it with 0xFF.
     // The higher is the `hash_table_slots` multiple - the more memory we will use,
     // but the less likely the collisions will be.
-    sz_size_t const hash_table_slots = sz_size_bit_ceil(small_sequence->count) * (1 << SZ_SEQUENCE_INTERSECT_BUDGET);
+    sz_size_t const hash_table_slots = sz_size_bit_ceil(small_sequence->count) * (1u << SZ_SEQUENCE_INTERSECT_BUDGET);
     sz_size_t const bytes_per_entry = sizeof(sz_size_t) + sizeof(sz_u64_t);
     sz_size_t *const table_positions = (sz_size_t *)alloc->allocate(hash_table_slots * bytes_per_entry, alloc);
     if (!table_positions) return sz_bad_alloc_k;
@@ -414,7 +414,7 @@ SZ_PUBLIC sz_status_t sz_sequence_intersect_ice(
     // Allocate memory for the hash table and initialize it with 0xFF.
     // The higher is the `hash_table_slots` multiple - the more memory we will use,
     // but the less likely the collisions will be.
-    sz_size_t const hash_table_slots = sz_size_bit_ceil(small_sequence->count) * (1 << SZ_SEQUENCE_INTERSECT_BUDGET);
+    sz_size_t const hash_table_slots = sz_size_bit_ceil(small_sequence->count) * (1u << SZ_SEQUENCE_INTERSECT_BUDGET);
     sz_size_t const bytes_per_entry = sizeof(sz_size_t) + sizeof(sz_u64_t);
     sz_size_t *table_positions = (sz_size_t *)alloc->allocate(hash_table_slots * bytes_per_entry, alloc);
     if (!table_positions) return sz_bad_alloc_k;

include/stringzilla/memory.h

@@ -1194,6 +1194,7 @@ SZ_PUBLIC void sz_lookup_neon(sz_ptr_t target, sz_size_t length, sz_cptr_t sourc
 
     sz_size_t head_length = (16 - ((sz_size_t)target % 16)) % 16; // 15 or less.
     sz_size_t tail_length = (sz_size_t)(target + length) % 16;    // 15 or less.
+    sz_size_t body_length = length - head_length - tail_length;
 
     // We need to pull the lookup table into 16x NEON registers. We have a total of 32 such registers.
     // According to the Neoverse V2 manual, the 4-table lookup has a latency of 6 cycles, and 4x throughput.
@@ -1216,9 +1217,7 @@ SZ_PUBLIC void sz_lookup_neon(sz_ptr_t target, sz_size_t length, sz_cptr_t sourc
     // to perform a 4-table lookup in a single instruction. The XORs are used to adjust the lookup position
     // within each 64-byte range of the table.
     // Details on the 4-table lookup: https://lemire.me/blog/2019/07/23/arbitrary-byte-to-byte-maps-using-arm-neon/
-    length -= head_length;
-    length -= tail_length;
-    for (; length >= 16; source += 16, target += 16, length -= 16) {
+    for (; body_length >= 16; source += 16, target += 16, body_length -= 16) {
         source_vec.u8x16 = vld1q_u8((sz_u8_t const *)source);
         lookup_0_to_63_vec.u8x16 = vqtbl4q_u8(lut_0_to_63_vec, source_vec.u8x16);
         lookup_64_to_127_vec.u8x16 = vqtbl4q_u8(lut_64_to_127_vec, veorq_u8(source_vec.u8x16, vdupq_n_u8(0x40)));