diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index ddff440..0000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build] -rustflags = ["-C", "target-cpu=native"] diff --git a/.cargo/config.toml.example b/.cargo/config.toml.example new file mode 100644 index 0000000..2227dbb --- /dev/null +++ b/.cargo/config.toml.example @@ -0,0 +1,8 @@ +# Copy this file to .cargo/config.toml for local maximum-performance benchmarks: +# cp .cargo/config.toml.example .cargo/config.toml +# +# This file is gitignored because CI already sets target-cpu=native for +# bench-pages, and the gate workflow needs a scalar baseline variant. + +[target.'cfg(any(target_arch = "x86_64", target_arch = "x86"))'] +rustflags = ["-C", "target-cpu=native"] diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 2934b69..e0d9453 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -16,4 +16,4 @@ jobs: with: rust_toolchain: stable build_command: cargo codspeed build -m simulation - run_command: cargo codspeed run -m simulation bit_ + run_command: cargo codspeed run -m simulation ours_ diff --git a/.gitignore b/.gitignore index ad86a15..b7cd4ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /target /.vscode /.venv -/proptest-regressions \ No newline at end of file +/proptest-regressions +.cargo/config.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f96422b..c8ef4aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,8 +16,8 @@ repos: always_run: true - id: cargo-test - name: cargo test - entry: cargo test --quiet + name: cargo test (native SIMD) + entry: bash -c 'RUSTFLAGS="-C target-cpu=native" cargo test --quiet' language: system pass_filenames: false always_run: true diff --git a/Cargo.lock b/Cargo.lock index fbec46a..0618e7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,11 +49,12 @@ dependencies = [ [[package]] name = "bit-string" -version = "0.4.4" +version = "0.4.5" dependencies = [ "bitvec_simd", "codspeed-divan-compat", "int-interval", + "once_cell", "proptest", "witnessed", ] diff --git a/Cargo.toml b/Cargo.toml index 91a25c7..493da5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bit-string" -version = "0.4.4" +version = "0.4.5" edition = "2024" description = "A compact owned bit string type with editing, matching, and bitwise operations." readme = "README.md" @@ -11,8 +11,13 @@ categories = ["data-structures", "no-std"] exclude = ["/benches", "/src/**/tests_for_*", "/tests", ".github/"] +[features] +default = [] +compile-time-dispatch = [] + [dependencies] int-interval = "0.9.6" +once_cell = "1" witnessed = "0.8.0" [lib] @@ -119,3 +124,11 @@ harness = false [[bench]] name = "ord" harness = false + +[[bench]] +name = "bit_ops_leading" +harness = false + +[[bench]] +name = "bit_ops_trailing" +harness = false diff --git a/README.md b/README.md index cd7024f..ffe226d 100644 --- a/README.md +++ b/README.md @@ -6,86 +6,221 @@ [![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/gh/jcfangc/bit-string) [![Coverage](https://codecov.io/gh/jcfangc/bit-string/branch/main/graph/badge.svg)](https://codecov.io/gh/jcfangc/bit-string) -A `no_std` Rust crate providing a compact owned bit string type and a zero-copy view, with construction, editing, matching, comparison, hashing, and bitwise operations. +A `no_std` + `alloc` Rust crate providing a compact owned bit string and a zero-copy view, with construction, editing, matching, comparison, and bitwise operations — all accelerated by runtime SIMD dispatch (AVX2, SSSE3, NEON). -The two core types mirror the `String`/`&str` relationship: +## Quick start + +```rust +use bit_string::BitString; + +// Parse from a binary string +let bits = BitString::try_from("1010_0011").unwrap(); +assert_eq!(bits.to_string(), "10100011"); +assert_eq!(bits.len(), 8); +assert_eq!(bits.count_ones(), 4); + +// Build programmatically +let mut b = BitString::zeros(10); // "0000000000" +b.set(0, true); // "1000000000" +b.set(9, true); // "1000000001" +b.push(true); // "10000000011" + +// Bitwise operations +let a = BitString::try_from("1010").unwrap(); +let c = BitString::try_from("1100").unwrap(); +assert_eq!(a.and(&c).unwrap().to_string(), "1000"); +assert_eq!(a.or(&c).unwrap().to_string(), "1110"); +assert_eq!(a.xor(&c).unwrap().to_string(), "0110"); +assert_eq!((!a).to_string(), "0101"); +``` + +**Core types** mirror `String` / `&str`: | Type | Role | Size | `Copy` | |------|------|------|--------| -| `BitString` | Owned bit string, backed by `Box<[u64]>` | 4×usize | No | -| `BitStr<'bs>` | Zero-copy borrowed view of a `BitString` or subrange | 3×usize (24 bytes) | **Yes** | +| `BitString` | Owned, `Box<[u64]>` backing | 4×usize | No | +| `BitStr<'bs>` | Zero-copy borrowed view | 3×usize | **Yes** | -Bits are packed little-endian into `u64` words. Unused high bits in the last word are always zero — enforced after every mutation on `BitString` and never observable through `BitStr`. +Bits are packed little-endian into `u64` words. Unused high bits in the last word are always zero. -## Features +## API by category -- **Construction**: `new`, `zeros`, `repeat`, `from_bool_iter`, `from_words`, `try_from(&str)` -- **Zero-copy view**: `BitStr<'bs>` — 24-byte `Copy` type returned by `as_bit_str()`, `slice()`, `slice_from()`, `slice_until()` -- **Conversion**: `to_bit_string()` — copies a `BitStr` view into an owned `BitString` -- **Bitwise ops**: `and`, `or`, `xor`, `not`, `shl`, `shr` (each with `_assign` and `_into` variants) -- **Bit counting**: `count_ones`, `count_zeros` -- **Editing**: `push`, `pop`, `insert`, `remove`, `set`, `extend`, `truncate`, `slice`, `split_off`, `replace_interval`, `retain`, `push_bit_string`, `insert_bit_string` -- **Matching**: `starts_with`, `ends_with`, `contains`, `find`, `rfind`, `strip_prefix`, `strip_suffix` -- **Comparison & hashing**: `PartialEq`, `Eq`, `PartialOrd`, `Ord`, `Hash` for both `BitStr` and `BitString` — lexicographic comparison with SIMD acceleration -- **Access**: `get`, `first`, `last`, `get_chunk`, `len`/`bit_len`, `is_empty`, `words`, `iter` +### Construction -### SIMD backends +```rust +use bit_string::BitString; -Bitwise operations and construction routines dispatch to SIMD backends automatically: +// From a binary string literal +let a = BitString::try_from("0101").unwrap(); // "0101" +let b = BitString::try_from("0101_1110").unwrap(); // "01011110" (underscores ignored) -| Backend | Target | Width | -|---------|--------|-------| -| AVX2 | x86 / x86_64 | 256-bit (4×u64) | -| SSSE3 | x86 / x86_64 | 128-bit (2×u64) | -| NEON | aarch64 | 128-bit (2×u64) | -| Scalar | all targets | fallback | +// Pre-allocated +let z = BitString::zeros(100); // 100 zero bits +let o = BitString::ones(64); // 64 one bits +let r = BitString::repeat(true, 42); // 42 one bits -Enable `target-cpu=native` via `.cargo/config.toml` to test your local CPU's best backend. +// From an iterator +let v: BitString = (0..8).map(|i| i % 2 == 0).collect(); +// "10101010" +``` -## Example +### Zero-copy views (`BitStr`) ```rust use bit_string::BitString; +use int_interval::UsizeCO; -let a = BitString::try_from("1010").unwrap(); -let b = BitString::try_from("1100").unwrap(); +let bits = BitString::try_from("1100_1010").unwrap(); -// Bitwise operations -assert_eq!(a.and(&b).unwrap().to_string(), "1000"); -assert_eq!(a.or(&b).unwrap().to_string(), "1110"); -assert_eq!((!a).to_string(), "0101"); -assert_eq!(a.count_ones(), 2); - -// Zero-copy views via BitStr -let view = a.as_bit_str(); // &BitString → BitStr -let sub = view.slice_from(1); // "010" -assert!(sub.starts_with(&BitString::try_from("01").unwrap().as_bit_str())); -assert_eq!(sub.count_ones(), 1); - -// Comparison -use core::cmp::Ordering; -assert_eq!(a.cmp(&b), Ordering::Less); // "1010" < "1100" +// Full view +let view = bits.as_bit_str(); // &BitStr = "11001010" +assert_eq!(view.bit_len(), 8); + +// Sub-slice — zero-copy, O(1) +let sub = view.slice(UsizeCO::try_new(2, 6).unwrap()); // "0010" +assert!(sub.starts_with(&BitString::try_from("00").unwrap().as_bit_str())); // Convert back to owned -let owned = sub.to_bit_string(); -assert_eq!(owned.to_string(), "010"); +let owned = sub.to_bit_string(); // BitString = "0010" ``` -## Benchmarks +### Querying bits + +```rust +let bits = BitString::try_from("1011").unwrap(); + +// Individual bits +assert_eq!(bits.get(0), Some(true)); // index 0 = leftmost +assert_eq!(bits.get(3), Some(true)); +assert_eq!(bits.first(), Some(true)); +assert_eq!(bits.last(), Some(true)); + +// Bulk counting +assert_eq!(bits.count_ones(), 3); +assert_eq!(bits.count_zeros(), 1); +assert_eq!(bits.leading_zeros(), 0); // starts with '1' +assert_eq!(bits.trailing_ones(), 2); // ends with "11" + +// Bit-level matching +let pattern = BitString::try_from("10").unwrap(); +assert!(bits.starts_with(pattern.as_bit_str())); +``` -Continuous benchmarking results are published at: +### Editing - +```rust +let mut bits = BitString::try_from("1010").unwrap(); + +// Single-bit operations +bits.set(0, false); // "0010" +let popped = bits.pop(); // → Some(false), bits = "001" +bits.push(true); // "0011" +bits.insert(0, true); // "10011" (insert at front) + +// Bulk operations +bits.extend(&[true, false, false]); // "10011100" +bits.truncate(4); // "1001" +bits.split_off(2); // → BitString "01", bits = "10" + +// Range operations +use int_interval::UsizeCO; +let interval = UsizeCO::try_new(1, 3).unwrap(); +bits.replace_interval(interval, &BitString::ones(2)); // "111" +bits.remove(UsizeCO::try_new(0, 2).unwrap()); // "1" +``` -## Status +### Matching & searching -This crate is still early. APIs may change before the first stable release. +```rust +let haystack = BitString::try_from("0010_1100").unwrap(); +let needle = BitString::try_from("01").unwrap(); +let np = needle.as_bit_str(); + +// Fixed-end checks +assert!(haystack.ends_with(np)); +assert!(!haystack.starts_with(np)); + +// Substring search +assert!(haystack.contains(np)); +assert_eq!(haystack.find(np), Some(1)); // "01" starts at index 1 +assert_eq!(haystack.rfind(np), Some(5)); // last "01" at index 5 + +// Strip +let s = BitString::try_from("00010100").unwrap(); +let stripped = s.strip_prefix(&BitString::try_from("00").unwrap().as_bit_str()); +assert_eq!(stripped.unwrap().to_string(), "010100"); +``` -## License +### Bitwise operations -Licensed under either of: +```rust +let x = BitString::try_from("1010").unwrap(); +let y = BitString::try_from("0110").unwrap(); + +// Consuming (`_into`) — reuses allocation +let z = x.and_into(&y).unwrap(); // "0010" +let z = y.or_into(&x).unwrap(); // "1110" +let z = y.xor_into(&x).unwrap(); // "1100" + +// In-place (`_assign`) +let mut w = x.clone(); +w.and_assign(&y); // w = "0010" + +// Shift +let s = BitString::try_from("1001").unwrap(); +assert_eq!(s.shl(2).to_string(), "0100"); // left shift, zero-fill +assert_eq!(s.shr(1).to_string(), "0100"); // right shift, zero-fill + +// Not +assert_eq!((!s).to_string(), "0110"); +``` + +### Comparison, hashing, iteration + +```rust +use core::cmp::Ordering; +use std::collections::HashSet; + +let a = BitString::try_from("100").unwrap(); +let b = BitString::try_from("101").unwrap(); + +// Lexicographic ordering (SIMD-accelerated) +assert_eq!(a.cmp(&b), Ordering::Less); // "100" < "101" +assert!(a < b); + +// Hash — usable as HashMap/HashSet keys +let mut set = HashSet::new(); +set.insert(a.clone()); +assert!(set.contains(&a)); + +// Iterate over bits +let bits: Vec = a.iter().collect(); +assert_eq!(bits, vec![true, false, false]); +``` + +## SIMD backends + +At runtime (or compile time with the `compile-time-dispatch` feature) the crate selects the best available SIMD backend: + +| Backend | Target | Width | +|---------|--------|-------| +| AVX2 | x86 / x86_64 | 256-bit (4×u64) | +| SSSE3 | x86 / x86_64 | 128-bit (2×u64) | +| NEON | aarch64 | 128-bit (2×u64) | +| Scalar | all targets | fallback | + +For maximum local performance, copy the example config: -* MIT license -* Apache License, Version 2.0 +```bash +cp .cargo/config.toml.example .cargo/config.toml +``` + +This enables `target-cpu=native` for local builds. It is gitignored — CI already sets the appropriate flags. + +## Benchmarks + +Continuous benchmarking results: [jcfangc.github.io/bit-string](https://jcfangc.github.io/bit-string/compare-plotly/index.html) + +## License -at your option. \ No newline at end of file +Licensed under either of MIT or Apache-2.0 at your option. diff --git a/benches/bit_ops_count_ones.rs b/benches/bit_ops_count_ones.rs index feb8c4d..1e4234f 100644 --- a/benches/bit_ops_count_ones.rs +++ b/benches/bit_ops_count_ones.rs @@ -13,7 +13,7 @@ enum Pattern { Alternating, } -#[divan::bench(name = "count_ones/len_65/dense/bit_string")] +#[divan::bench(name = "count_ones/len_65/dense/ours_string")] fn count_ones_len_65_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Dense); } @@ -23,7 +23,7 @@ fn count_ones_len_65_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Dense); } -#[divan::bench(name = "count_ones/len_65/sparse/bit_string")] +#[divan::bench(name = "count_ones/len_65/sparse/ours_string")] fn count_ones_len_65_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Sparse); } @@ -33,7 +33,7 @@ fn count_ones_len_65_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Sparse); } -#[divan::bench(name = "count_ones/len_65/alternating/bit_string")] +#[divan::bench(name = "count_ones/len_65/alternating/ours_string")] fn count_ones_len_65_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Alternating); } @@ -43,7 +43,7 @@ fn count_ones_len_65_alternating_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Alternating); } -#[divan::bench(name = "count_ones/len_4096/dense/bit_string")] +#[divan::bench(name = "count_ones/len_4096/dense/ours_string")] fn count_ones_len_4096_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Dense); } @@ -53,7 +53,7 @@ fn count_ones_len_4096_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Dense); } -#[divan::bench(name = "count_ones/len_4096/sparse/bit_string")] +#[divan::bench(name = "count_ones/len_4096/sparse/ours_string")] fn count_ones_len_4096_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Sparse); } @@ -63,7 +63,7 @@ fn count_ones_len_4096_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Sparse); } -#[divan::bench(name = "count_ones/len_4096/alternating/bit_string")] +#[divan::bench(name = "count_ones/len_4096/alternating/ours_string")] fn count_ones_len_4096_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Alternating); } @@ -73,7 +73,7 @@ fn count_ones_len_4096_alternating_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Alternating); } -#[divan::bench(name = "count_ones/len_65536/dense/bit_string")] +#[divan::bench(name = "count_ones/len_65536/dense/ours_string")] fn count_ones_len_65536_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Dense); } @@ -83,7 +83,7 @@ fn count_ones_len_65536_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65_536, Pattern::Dense); } -#[divan::bench(name = "count_ones/len_65536/sparse/bit_string")] +#[divan::bench(name = "count_ones/len_65536/sparse/ours_string")] fn count_ones_len_65536_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Sparse); } @@ -93,7 +93,7 @@ fn count_ones_len_65536_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65_536, Pattern::Sparse); } -#[divan::bench(name = "count_ones/len_65536/alternating/bit_string")] +#[divan::bench(name = "count_ones/len_65536/alternating/ours_string")] fn count_ones_len_65536_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Alternating); } diff --git a/benches/bit_ops_leading.rs b/benches/bit_ops_leading.rs new file mode 100644 index 0000000..31b3706 --- /dev/null +++ b/benches/bit_ops_leading.rs @@ -0,0 +1,145 @@ +use bit_string::BitString; +use bitvec_simd::BitVec; +use divan::{Bencher, black_box}; +use int_interval::UsizeCO; + +fn main() { + divan::main(); +} + +#[derive(Clone, Copy)] +enum Pattern { + Zeros, + Alternating, + Dense, +} + +#[divan::bench(name = "leading_zeros/len_65/all_zeros/ours_str")] +fn leading_65_zeros_str(b: Bencher) { + bench_str(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_65/all_zeros/ours_string")] +fn leading_65_zeros_string(b: Bencher) { + bench_string(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_65/all_zeros/bitvec_simd")] +fn leading_65_zeros_bitvec(b: Bencher) { + bench_bitvec(b, 65, Pattern::Zeros); +} + +#[divan::bench(name = "leading_zeros/len_65/alternating/ours_str")] +fn leading_65_alternating_str(b: Bencher) { + bench_str(b, 65, Pattern::Alternating); +} +#[divan::bench(name = "leading_zeros/len_65/alternating/ours_string")] +fn leading_65_alternating_string(b: Bencher) { + bench_string(b, 65, Pattern::Alternating); +} +#[divan::bench(name = "leading_zeros/len_65/alternating/bitvec_simd")] +fn leading_65_alternating_bitvec(b: Bencher) { + bench_bitvec(b, 65, Pattern::Alternating); +} + +#[divan::bench(name = "leading_zeros/len_65/dense/ours_str")] +fn leading_65_dense_str(b: Bencher) { + bench_str(b, 65, Pattern::Dense); +} +#[divan::bench(name = "leading_zeros/len_65/dense/ours_string")] +fn leading_65_dense_string(b: Bencher) { + bench_string(b, 65, Pattern::Dense); +} +#[divan::bench(name = "leading_zeros/len_65/dense/bitvec_simd")] +fn leading_65_dense_bitvec(b: Bencher) { + bench_bitvec(b, 65, Pattern::Dense); +} + +#[divan::bench(name = "leading_zeros/len_4096/all_zeros/ours_str")] +fn leading_4096_zeros_str(b: Bencher) { + bench_str(b, 4096, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_4096/all_zeros/ours_string")] +fn leading_4096_zeros_string(b: Bencher) { + bench_string(b, 4096, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_4096/all_zeros/bitvec_simd")] +fn leading_4096_zeros_bitvec(b: Bencher) { + bench_bitvec(b, 4096, Pattern::Zeros); +} + +#[divan::bench(name = "leading_zeros/len_4096/dense/ours_str")] +fn leading_4096_dense_str(b: Bencher) { + bench_str(b, 4096, Pattern::Dense); +} +#[divan::bench(name = "leading_zeros/len_4096/dense/ours_string")] +fn leading_4096_dense_string(b: Bencher) { + bench_string(b, 4096, Pattern::Dense); +} +#[divan::bench(name = "leading_zeros/len_4096/dense/bitvec_simd")] +fn leading_4096_dense_bitvec(b: Bencher) { + bench_bitvec(b, 4096, Pattern::Dense); +} + +#[divan::bench(name = "leading_zeros/len_65536/all_zeros/ours_str")] +fn leading_65536_zeros_str(b: Bencher) { + bench_str(b, 65536, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_65536/all_zeros/ours_string")] +fn leading_65536_zeros_string(b: Bencher) { + bench_string(b, 65536, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_65536/all_zeros/bitvec_simd")] +fn leading_65536_zeros_bitvec(b: Bencher) { + bench_bitvec(b, 65536, Pattern::Zeros); +} + +#[divan::bench(name = "leading_zeros/unaligned_3/len_4096/all_zeros/ours_str")] +fn leading_unaligned_3_4096_zeros_str(b: Bencher) { + bench_unaligned_str(b, 4096, 3, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/unaligned_31/len_4096/all_zeros/ours_str")] +fn leading_unaligned_31_4096_zeros_str(b: Bencher) { + bench_unaligned_str(b, 4096, 31, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/unaligned_63/len_4096/all_zeros/ours_str")] +fn leading_unaligned_63_4096_zeros_str(b: Bencher) { + bench_unaligned_str(b, 4096, 63, Pattern::Zeros); +} + +fn bench_str(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + let v = bits.as_bit_str(); + b.bench(|| black_box(&v).leading_zeros()); +} +fn bench_string(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + b.bench(|| black_box(&bits).leading_zeros()); +} +fn bench_bitvec(b: Bencher, len: usize, p: Pattern) { + let bv = BitVec::from_bool_iterator((0..len).map(|i| bit(i, p))); + b.bench(|| black_box(&bv).leading_zeros()); +} +fn bench_unaligned_str(b: Bencher, len: usize, skip: usize, p: Pattern) { + let pad = skip + len + 10; + let mut bits = BitString::zeros(pad); + for i in skip..skip + len { + bits.set(i, bit(i - skip, p)); + } + let sub = bits + .as_bit_str() + .slice(UsizeCO::try_new(skip, skip + len).unwrap()); + b.bench(|| black_box(&sub).leading_zeros()); +} + +fn bit(i: usize, p: Pattern) -> bool { + match p { + Pattern::Zeros => false, + Pattern::Alternating => i % 2 != 0, + Pattern::Dense => mix64(i as u64) & 1 != 0, + } +} +fn mix64(mut v: u64) -> u64 { + v = v.wrapping_add(0x9e37_79b9_7f4a_7c15); + v = (v ^ (v >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + v = (v ^ (v >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + v ^ (v >> 31) +} diff --git a/benches/bit_ops_not.rs b/benches/bit_ops_not.rs index 2f9a3ee..5fa3033 100644 --- a/benches/bit_ops_not.rs +++ b/benches/bit_ops_not.rs @@ -13,7 +13,7 @@ enum Pattern { Alternating, } -#[divan::bench(name = "not/len_65/dense/bit_string")] +#[divan::bench(name = "not/len_65/dense/ours_string")] fn not_len_65_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Dense); } @@ -23,7 +23,7 @@ fn not_len_65_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Dense); } -#[divan::bench(name = "not/len_65/sparse/bit_string")] +#[divan::bench(name = "not/len_65/sparse/ours_string")] fn not_len_65_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Sparse); } @@ -33,7 +33,7 @@ fn not_len_65_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Sparse); } -#[divan::bench(name = "not/len_65/alternating/bit_string")] +#[divan::bench(name = "not/len_65/alternating/ours_string")] fn not_len_65_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Alternating); } @@ -43,7 +43,7 @@ fn not_len_65_alternating_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Alternating); } -#[divan::bench(name = "not/len_4096/dense/bit_string")] +#[divan::bench(name = "not/len_4096/dense/ours_string")] fn not_len_4096_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Dense); } @@ -53,7 +53,7 @@ fn not_len_4096_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Dense); } -#[divan::bench(name = "not/len_4096/sparse/bit_string")] +#[divan::bench(name = "not/len_4096/sparse/ours_string")] fn not_len_4096_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Sparse); } @@ -63,7 +63,7 @@ fn not_len_4096_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Sparse); } -#[divan::bench(name = "not/len_4096/alternating/bit_string")] +#[divan::bench(name = "not/len_4096/alternating/ours_string")] fn not_len_4096_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Alternating); } @@ -73,7 +73,7 @@ fn not_len_4096_alternating_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Alternating); } -#[divan::bench(name = "not/len_65536/dense/bit_string")] +#[divan::bench(name = "not/len_65536/dense/ours_string")] fn not_len_65536_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Dense); } @@ -83,7 +83,7 @@ fn not_len_65536_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65_536, Pattern::Dense); } -#[divan::bench(name = "not/len_65536/sparse/bit_string")] +#[divan::bench(name = "not/len_65536/sparse/ours_string")] fn not_len_65536_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Sparse); } @@ -93,7 +93,7 @@ fn not_len_65536_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65_536, Pattern::Sparse); } -#[divan::bench(name = "not/len_65536/alternating/bit_string")] +#[divan::bench(name = "not/len_65536/alternating/ours_string")] fn not_len_65536_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Alternating); } diff --git a/benches/bit_ops_shl.rs b/benches/bit_ops_shl.rs index fed0129..91538b9 100644 --- a/benches/bit_ops_shl.rs +++ b/benches/bit_ops_shl.rs @@ -5,27 +5,27 @@ fn main() { divan::main(); } -#[divan::bench(name = "shl/len_65/amount_1/bit_string")] +#[divan::bench(name = "shl/len_65/amount_1/ours_string")] fn shl_len_65_amount_1_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, 1); } -#[divan::bench(name = "shl/len_4096/amount_1/bit_string")] +#[divan::bench(name = "shl/len_4096/amount_1/ours_string")] fn shl_len_4096_amount_1_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, 1); } -#[divan::bench(name = "shl/len_4096/amount_65/bit_string")] +#[divan::bench(name = "shl/len_4096/amount_65/ours_string")] fn shl_len_4096_amount_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, 65); } -#[divan::bench(name = "shl/len_65536/amount_1/bit_string")] +#[divan::bench(name = "shl/len_65536/amount_1/ours_string")] fn shl_len_65536_amount_1_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, 1); } -#[divan::bench(name = "shl/len_65536/amount_65/bit_string")] +#[divan::bench(name = "shl/len_65536/amount_65/ours_string")] fn shl_len_65536_amount_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, 65); } diff --git a/benches/bit_ops_trailing.rs b/benches/bit_ops_trailing.rs new file mode 100644 index 0000000..db5300c --- /dev/null +++ b/benches/bit_ops_trailing.rs @@ -0,0 +1,216 @@ +use bit_string::BitString; +use divan::{Bencher, black_box}; +use int_interval::UsizeCO; + +fn main() { + divan::main(); +} + +#[derive(Clone, Copy)] +enum Pattern { + Zeros, + Alternating, + Dense, +} + +// ═══════════════════════════════════════════════════════════════════════ +// BitStr::leading_zeros (optimised trait path) — reference baseline +// for trailing_zeros. Both go through WordsScan; trailing adds the +// reverse-scan overhead. Goal: make trailing as close as possible. +// ═══════════════════════════════════════════════════════════════════════ + +#[divan::bench(name = "leading_zeros/len_65/all_zeros/ours_str")] +fn lead_65_zeros_str_ref(b: Bencher) { + bench_lead_str(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_65/dense/ours_str")] +fn lead_65_dense_str_ref(b: Bencher) { + bench_lead_str(b, 65, Pattern::Dense); +} +#[divan::bench(name = "leading_zeros/len_4096/all_zeros/ours_str")] +fn lead_4096_zeros_str_ref(b: Bencher) { + bench_lead_str(b, 4096, Pattern::Zeros); +} +#[divan::bench(name = "leading_zeros/len_65536/all_zeros/ours_str")] +fn lead_65536_zeros_str_ref(b: Bencher) { + bench_lead_str(b, 65536, Pattern::Zeros); +} +#[divan::bench(name = "leading_ones/len_65/all_zeros/ours_str")] +fn lead_ones_65_zeros_str_ref(b: Bencher) { + bench_lead_ones_str(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "leading_ones/len_4096/all_zeros/ours_str")] +fn lead_ones_4096_zeros_str_ref(b: Bencher) { + bench_lead_ones_str(b, 4096, Pattern::Zeros); +} + +// ═══════════════════════════════════════════════════════════════════════ +// trailing_zeros +// ═══════════════════════════════════════════════════════════════════════ + +#[divan::bench(name = "trailing_zeros/len_65/all_zeros/ours_str")] +fn trailing_65_zeros_str(b: Bencher) { + bench_str(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "trailing_zeros/len_65/all_zeros/ours_string")] +fn trailing_65_zeros_string(b: Bencher) { + bench_string(b, 65, Pattern::Zeros); +} + +#[divan::bench(name = "trailing_zeros/len_65/alternating/ours_str")] +fn trailing_65_alternating_str(b: Bencher) { + bench_str(b, 65, Pattern::Alternating); +} +#[divan::bench(name = "trailing_zeros/len_65/alternating/ours_string")] +fn trailing_65_alternating_string(b: Bencher) { + bench_string(b, 65, Pattern::Alternating); +} + +#[divan::bench(name = "trailing_zeros/len_65/dense/ours_str")] +fn trailing_65_dense_str(b: Bencher) { + bench_str(b, 65, Pattern::Dense); +} +#[divan::bench(name = "trailing_zeros/len_65/dense/ours_string")] +fn trailing_65_dense_string(b: Bencher) { + bench_string(b, 65, Pattern::Dense); +} + +#[divan::bench(name = "trailing_zeros/len_4096/all_zeros/ours_str")] +fn trailing_4096_zeros_str(b: Bencher) { + bench_str(b, 4096, Pattern::Zeros); +} +#[divan::bench(name = "trailing_zeros/len_4096/all_zeros/ours_string")] +fn trailing_4096_zeros_string(b: Bencher) { + bench_string(b, 4096, Pattern::Zeros); +} + +#[divan::bench(name = "trailing_zeros/len_4096/dense/ours_str")] +fn trailing_4096_dense_str(b: Bencher) { + bench_str(b, 4096, Pattern::Dense); +} +#[divan::bench(name = "trailing_zeros/len_4096/dense/ours_string")] +fn trailing_4096_dense_string(b: Bencher) { + bench_string(b, 4096, Pattern::Dense); +} + +#[divan::bench(name = "trailing_zeros/len_65536/all_zeros/ours_str")] +fn trailing_65536_zeros_str(b: Bencher) { + bench_str(b, 65536, Pattern::Zeros); +} +#[divan::bench(name = "trailing_zeros/len_65536/all_zeros/ours_string")] +fn trailing_65536_zeros_string(b: Bencher) { + bench_string(b, 65536, Pattern::Zeros); +} + +#[divan::bench(name = "trailing_zeros/len_65536/dense/ours_str")] +fn trailing_65536_dense_str(b: Bencher) { + bench_str(b, 65536, Pattern::Dense); +} +#[divan::bench(name = "trailing_zeros/len_65536/dense/ours_string")] +fn trailing_65536_dense_string(b: Bencher) { + bench_string(b, 65536, Pattern::Dense); +} + +#[divan::bench(name = "trailing_zeros/unaligned_3/len_4096/all_zeros/ours_str")] +fn trailing_unaligned_3_4096_zeros_str(b: Bencher) { + bench_unaligned_str(b, 4096, 3, Pattern::Zeros); +} +#[divan::bench(name = "trailing_zeros/unaligned_63/len_4096/all_zeros/ours_str")] +fn trailing_unaligned_63_4096_zeros_str(b: Bencher) { + bench_unaligned_str(b, 4096, 63, Pattern::Zeros); +} + +// ═══════════════════════════════════════════════════════════════════════ +// trailing_ones +// ═══════════════════════════════════════════════════════════════════════ + +#[divan::bench(name = "trailing_ones/len_65/all_zeros/ours_str")] +fn trailing_ones_65_zeros_str(b: Bencher) { + bench_str_trailing_ones(b, 65, Pattern::Zeros); +} +#[divan::bench(name = "trailing_ones/len_65/all_zeros/ours_string")] +fn trailing_ones_65_zeros_string(b: Bencher) { + bench_string_trailing_ones(b, 65, Pattern::Zeros); +} + +#[divan::bench(name = "trailing_ones/len_65/dense/ours_str")] +fn trailing_ones_65_dense_str(b: Bencher) { + bench_str_trailing_ones(b, 65, Pattern::Dense); +} +#[divan::bench(name = "trailing_ones/len_65/dense/ours_string")] +fn trailing_ones_65_dense_string(b: Bencher) { + bench_string_trailing_ones(b, 65, Pattern::Dense); +} + +#[divan::bench(name = "trailing_ones/len_4096/all_zeros/ours_str")] +fn trailing_ones_4096_zeros_str(b: Bencher) { + bench_str_trailing_ones(b, 4096, Pattern::Zeros); +} +#[divan::bench(name = "trailing_ones/len_4096/all_zeros/ours_string")] +fn trailing_ones_4096_zeros_string(b: Bencher) { + bench_string_trailing_ones(b, 4096, Pattern::Zeros); +} + +// ── leading reference helpers (BitStr) ────────────────────────────── + +fn bench_lead_str(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + let v = bits.as_bit_str(); + b.bench(|| black_box(&v).leading_zeros()); +} +fn bench_lead_ones_str(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + let v = bits.as_bit_str(); + b.bench(|| black_box(&v).leading_ones()); +} + +// ── trailing_zeros helpers ──────────────────────────────────────────── + +fn bench_str(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + let v = bits.as_bit_str(); + b.bench(|| black_box(&v).trailing_zeros()); +} +fn bench_string(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + b.bench(|| black_box(&bits).trailing_zeros()); +} +fn bench_unaligned_str(b: Bencher, len: usize, skip: usize, p: Pattern) { + let pad = skip + len + 10; + let mut bits = BitString::zeros(pad); + for i in skip..skip + len { + bits.set(i, bit(i - skip, p)); + } + let sub = bits + .as_bit_str() + .slice(UsizeCO::try_new(skip, skip + len).unwrap()); + b.bench(|| black_box(&sub).trailing_zeros()); +} + +// ── trailing_ones helpers ───────────────────────────────────────────── + +fn bench_str_trailing_ones(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + let v = bits.as_bit_str(); + b.bench(|| black_box(&v).trailing_ones()); +} +fn bench_string_trailing_ones(b: Bencher, len: usize, p: Pattern) { + let bits: BitString = (0..len).map(|i| bit(i, p)).collect(); + b.bench(|| black_box(&bits).trailing_ones()); +} + +// ── data helpers ────────────────────────────────────────────────────── + +fn bit(i: usize, p: Pattern) -> bool { + match p { + Pattern::Zeros => false, + Pattern::Alternating => i % 2 != 0, + Pattern::Dense => mix64(i as u64) & 1 != 0, + } +} +fn mix64(mut v: u64) -> u64 { + v = v.wrapping_add(0x9e37_79b9_7f4a_7c15); + v = (v ^ (v >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + v = (v ^ (v >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + v ^ (v >> 31) +} diff --git a/benches/bit_ops_xor.rs b/benches/bit_ops_xor.rs index 510a134..2238bf4 100644 --- a/benches/bit_ops_xor.rs +++ b/benches/bit_ops_xor.rs @@ -14,7 +14,7 @@ enum Pattern { } // Flattened XOR benchmarks for CodSpeed compatibility -#[divan::bench(name = "xor/len_65/dense/bit_string")] +#[divan::bench(name = "xor/len_65/dense/ours_string")] fn xor_len_65_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Dense); } @@ -24,7 +24,7 @@ fn xor_len_65_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Dense); } -#[divan::bench(name = "xor/len_65/sparse/bit_string")] +#[divan::bench(name = "xor/len_65/sparse/ours_string")] fn xor_len_65_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Sparse); } @@ -34,7 +34,7 @@ fn xor_len_65_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Sparse); } -#[divan::bench(name = "xor/len_65/alternating/bit_string")] +#[divan::bench(name = "xor/len_65/alternating/ours_string")] fn xor_len_65_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, Pattern::Alternating); } @@ -44,7 +44,7 @@ fn xor_len_65_alternating_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65, Pattern::Alternating); } -#[divan::bench(name = "xor/len_4096/dense/bit_string")] +#[divan::bench(name = "xor/len_4096/dense/ours_string")] fn xor_len_4096_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Dense); } @@ -54,7 +54,7 @@ fn xor_len_4096_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Dense); } -#[divan::bench(name = "xor/len_4096/sparse/bit_string")] +#[divan::bench(name = "xor/len_4096/sparse/ours_string")] fn xor_len_4096_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Sparse); } @@ -64,7 +64,7 @@ fn xor_len_4096_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Sparse); } -#[divan::bench(name = "xor/len_4096/alternating/bit_string")] +#[divan::bench(name = "xor/len_4096/alternating/ours_string")] fn xor_len_4096_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096, Pattern::Alternating); } @@ -74,7 +74,7 @@ fn xor_len_4096_alternating_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096, Pattern::Alternating); } -#[divan::bench(name = "xor/len_65536/dense/bit_string")] +#[divan::bench(name = "xor/len_65536/dense/ours_string")] fn xor_len_65536_dense_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Dense); } @@ -84,7 +84,7 @@ fn xor_len_65536_dense_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65_536, Pattern::Dense); } -#[divan::bench(name = "xor/len_65536/sparse/bit_string")] +#[divan::bench(name = "xor/len_65536/sparse/ours_string")] fn xor_len_65536_sparse_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Sparse); } @@ -94,7 +94,7 @@ fn xor_len_65536_sparse_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65_536, Pattern::Sparse); } -#[divan::bench(name = "xor/len_65536/alternating/bit_string")] +#[divan::bench(name = "xor/len_65536/alternating/ours_string")] fn xor_len_65536_alternating_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, Pattern::Alternating); } diff --git a/benches/construction_from_bool_iter.rs b/benches/construction_from_bool_iter.rs index 16d4de3..bf5f17f 100644 --- a/benches/construction_from_bool_iter.rs +++ b/benches/construction_from_bool_iter.rs @@ -6,7 +6,7 @@ fn main() { divan::main(); } -#[divan::bench(name = "from_bool_iter/len_65/bit_string")] +#[divan::bench(name = "from_bool_iter/len_65/ours_string")] fn from_bool_iter_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } @@ -16,7 +16,7 @@ fn from_bool_iter_len_65_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 65); } -#[divan::bench(name = "from_bool_iter/len_4096/bit_string")] +#[divan::bench(name = "from_bool_iter/len_4096/ours_string")] fn from_bool_iter_len_4096_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096); } @@ -26,7 +26,7 @@ fn from_bool_iter_len_4096_bitvec_simd(bencher: Bencher) { bench_bitvec_simd(bencher, 4096); } -#[divan::bench(name = "from_bool_iter/len_65536/bit_string")] +#[divan::bench(name = "from_bool_iter/len_65536/ours_string")] fn from_bool_iter_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65536); } diff --git a/benches/construction_from_str.rs b/benches/construction_from_str.rs index 0aba632..af23da3 100644 --- a/benches/construction_from_str.rs +++ b/benches/construction_from_str.rs @@ -5,17 +5,17 @@ fn main() { divan::main(); } -#[divan::bench(name = "from_str/len_65/bit_string")] +#[divan::bench(name = "from_str/len_65/ours_string")] fn from_str_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } -#[divan::bench(name = "from_str/len_4096/bit_string")] +#[divan::bench(name = "from_str/len_4096/ours_string")] fn from_str_len_4096_bit_string(bencher: Bencher) { bench_bit_string(bencher, 4096); } -#[divan::bench(name = "from_str/len_65536/bit_string")] +#[divan::bench(name = "from_str/len_65536/ours_string")] fn from_str_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536); } diff --git a/benches/construction_zeros.rs b/benches/construction_zeros.rs index 4354c1d..f49e573 100644 --- a/benches/construction_zeros.rs +++ b/benches/construction_zeros.rs @@ -6,7 +6,7 @@ fn main() { divan::main(); } -#[divan::bench(name = "zeros/len_65/bit_string")] +#[divan::bench(name = "zeros/len_65/ours_string")] fn zeros_len_65_bit_string(bencher: Bencher) { bencher.bench(|| black_box(BitString::zeros(65))); } @@ -16,7 +16,7 @@ fn zeros_len_65_bitvec_simd(bencher: Bencher) { bencher.bench(|| black_box(BitVec::zeros(65))); } -#[divan::bench(name = "zeros/len_4096/bit_string")] +#[divan::bench(name = "zeros/len_4096/ours_string")] fn zeros_len_4096_bit_string(bencher: Bencher) { bencher.bench(|| black_box(BitString::zeros(4096))); } @@ -26,7 +26,7 @@ fn zeros_len_4096_bitvec_simd(bencher: Bencher) { bencher.bench(|| black_box(BitVec::zeros(4096))); } -#[divan::bench(name = "zeros/len_65536/bit_string")] +#[divan::bench(name = "zeros/len_65536/ours_string")] fn zeros_len_65536_bit_string(bencher: Bencher) { bencher.bench(|| black_box(BitString::zeros(65_536))); } diff --git a/benches/editing_insert_middle.rs b/benches/editing_insert_middle.rs index b6cbb61..c1cb02d 100644 --- a/benches/editing_insert_middle.rs +++ b/benches/editing_insert_middle.rs @@ -5,7 +5,7 @@ fn main() { divan::main(); } -#[divan::bench(name = "insert_middle/len_65/bit_string")] +#[divan::bench(name = "insert_middle/len_65/ours_string")] fn insert_middle_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } @@ -15,7 +15,7 @@ fn insert_middle_len_65_string(bencher: Bencher) { bench_string(bencher, 65); } -#[divan::bench(name = "insert_middle/len_65536/bit_string")] +#[divan::bench(name = "insert_middle/len_65536/ours_string")] fn insert_middle_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536); } diff --git a/benches/editing_pop.rs b/benches/editing_pop.rs index 5874d48..50c477b 100644 --- a/benches/editing_pop.rs +++ b/benches/editing_pop.rs @@ -6,7 +6,7 @@ fn main() { } // Pop from 65 bits → 64 triggers the lazy-shrink truncate_words path. -#[divan::bench(name = "pop/len_65/bit_string")] +#[divan::bench(name = "pop/len_65/ours_string")] fn pop_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } @@ -16,7 +16,7 @@ fn pop_len_65_string(bencher: Bencher) { bench_string(bencher, 65); } -#[divan::bench(name = "pop/len_65536/bit_string")] +#[divan::bench(name = "pop/len_65536/ours_string")] fn pop_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536); } diff --git a/benches/editing_push.rs b/benches/editing_push.rs index e38017c..72c4417 100644 --- a/benches/editing_push.rs +++ b/benches/editing_push.rs @@ -6,7 +6,7 @@ fn main() { } // Flattened push benchmarks for CodSpeed compatibility -#[divan::bench(name = "push/len_65/bit_string")] +#[divan::bench(name = "push/len_65/ours_string")] fn push_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } @@ -16,7 +16,7 @@ fn push_len_65_string(bencher: Bencher) { bench_string(bencher, 65); } -#[divan::bench(name = "push/len_65536/bit_string")] +#[divan::bench(name = "push/len_65536/ours_string")] fn push_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536); } diff --git a/benches/editing_push_bit_string.rs b/benches/editing_push_bit_string.rs index 11e918e..3a6d3d4 100644 --- a/benches/editing_push_bit_string.rs +++ b/benches/editing_push_bit_string.rs @@ -5,7 +5,7 @@ fn main() { divan::main(); } -#[divan::bench(name = "push_bit_string/len_65/bit_string")] +#[divan::bench(name = "push_bit_string/len_65/ours_string")] fn push_bits_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } @@ -15,7 +15,7 @@ fn push_bits_len_65_string(bencher: Bencher) { bench_string(bencher, 65); } -#[divan::bench(name = "push_bit_string/len_65536/bit_string")] +#[divan::bench(name = "push_bit_string/len_65536/ours_string")] fn push_bits_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536); } diff --git a/benches/editing_remove_middle.rs b/benches/editing_remove_middle.rs index fd37a34..d9230ff 100644 --- a/benches/editing_remove_middle.rs +++ b/benches/editing_remove_middle.rs @@ -5,7 +5,7 @@ fn main() { divan::main(); } -#[divan::bench(name = "remove_middle/len_65/bit_string")] +#[divan::bench(name = "remove_middle/len_65/ours_string")] fn remove_middle_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } @@ -15,7 +15,7 @@ fn remove_middle_len_65_string(bencher: Bencher) { bench_string(bencher, 65); } -#[divan::bench(name = "remove_middle/len_65536/bit_string")] +#[divan::bench(name = "remove_middle/len_65536/ours_string")] fn remove_middle_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536); } diff --git a/benches/editing_replace_interval.rs b/benches/editing_replace_interval.rs index 817868e..e642304 100644 --- a/benches/editing_replace_interval.rs +++ b/benches/editing_replace_interval.rs @@ -6,7 +6,7 @@ fn main() { divan::main(); } -#[divan::bench(name = "replace_interval/len_65/same_len/bit_string")] +#[divan::bench(name = "replace_interval/len_65/same_len/ours_string")] fn replace_interval_len_65_same_len_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, ReplaceShape::SameLen); } @@ -16,7 +16,7 @@ fn replace_interval_len_65_same_len_string(bencher: Bencher) { bench_string(bencher, 65, ReplaceShape::SameLen); } -#[divan::bench(name = "replace_interval/len_65/shorter/bit_string")] +#[divan::bench(name = "replace_interval/len_65/shorter/ours_string")] fn replace_interval_len_65_shorter_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, ReplaceShape::Shorter); } @@ -26,7 +26,7 @@ fn replace_interval_len_65_shorter_string(bencher: Bencher) { bench_string(bencher, 65, ReplaceShape::Shorter); } -#[divan::bench(name = "replace_interval/len_65/longer/bit_string")] +#[divan::bench(name = "replace_interval/len_65/longer/ours_string")] fn replace_interval_len_65_longer_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, ReplaceShape::Longer); } @@ -36,7 +36,7 @@ fn replace_interval_len_65_longer_string(bencher: Bencher) { bench_string(bencher, 65, ReplaceShape::Longer); } -#[divan::bench(name = "replace_interval/len_65536/same_len/bit_string")] +#[divan::bench(name = "replace_interval/len_65536/same_len/ours_string")] fn replace_interval_len_65536_same_len_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, ReplaceShape::SameLen); } @@ -46,7 +46,7 @@ fn replace_interval_len_65536_same_len_string(bencher: Bencher) { bench_string(bencher, 65_536, ReplaceShape::SameLen); } -#[divan::bench(name = "replace_interval/len_65536/shorter/bit_string")] +#[divan::bench(name = "replace_interval/len_65536/shorter/ours_string")] fn replace_interval_len_65536_shorter_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, ReplaceShape::Shorter); } @@ -56,7 +56,7 @@ fn replace_interval_len_65536_shorter_string(bencher: Bencher) { bench_string(bencher, 65_536, ReplaceShape::Shorter); } -#[divan::bench(name = "replace_interval/len_65536/longer/bit_string")] +#[divan::bench(name = "replace_interval/len_65536/longer/ours_string")] fn replace_interval_len_65536_longer_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, ReplaceShape::Longer); } diff --git a/benches/editing_slice.rs b/benches/editing_slice.rs index 47b5f82..296518f 100644 --- a/benches/editing_slice.rs +++ b/benches/editing_slice.rs @@ -7,7 +7,7 @@ fn main() { } // Flattened slice benchmarks for CodSpeed compatibility -#[divan::bench(name = "slice/len_65/bit_string")] +#[divan::bench(name = "slice/len_65/ours_string")] fn slice_len_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65); } @@ -17,7 +17,7 @@ fn slice_len_65_string(bencher: Bencher) { bench_string(bencher, 65); } -#[divan::bench(name = "slice/len_65536/bit_string")] +#[divan::bench(name = "slice/len_65536/ours_string")] fn slice_len_65536_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536); } diff --git a/benches/editing_truncate.rs b/benches/editing_truncate.rs index a76f336..116b842 100644 --- a/benches/editing_truncate.rs +++ b/benches/editing_truncate.rs @@ -6,7 +6,7 @@ fn main() { } // Truncate 65→64 crosses a word boundary, testing the lazy-shrink fast path. -#[divan::bench(name = "truncate/65_to_64/bit_string")] +#[divan::bench(name = "truncate/65_to_64/ours_string")] fn truncate_65_to_64_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65, 64); } @@ -17,7 +17,7 @@ fn truncate_65_to_64_string(bencher: Bencher) { } // Truncate 65536→128 removes many words, tests batch-shrink behavior. -#[divan::bench(name = "truncate/65536_to_128/bit_string")] +#[divan::bench(name = "truncate/65536_to_128/ours_string")] fn truncate_65536_to_128_bit_string(bencher: Bencher) { bench_bit_string(bencher, 65_536, 128); } @@ -28,7 +28,7 @@ fn truncate_65536_to_128_string(bencher: Bencher) { } // Truncate 128→65 partial-word rewrite in the last word. -#[divan::bench(name = "truncate/128_to_65/bit_string")] +#[divan::bench(name = "truncate/128_to_65/ours_string")] fn truncate_128_to_65_bit_string(bencher: Bencher) { bench_bit_string(bencher, 128, 65); } diff --git a/benches/hash.rs b/benches/hash.rs index dacb8d1..bc6c78e 100644 --- a/benches/hash.rs +++ b/benches/hash.rs @@ -19,12 +19,12 @@ enum Pattern { // BitString // --------------------------------------------------------------------------- -#[divan::bench(name = "hash/len_64/dense/bit_string")] +#[divan::bench(name = "hash/len_64/dense/ours_string")] fn hash_len_64_dense_bit_string(b: Bencher) { bench_bit_string(b, 64, Pattern::Dense); } -#[divan::bench(name = "hash/len_64/dense/bit_str")] +#[divan::bench(name = "hash/len_64/dense/ours_str")] fn hash_len_64_dense_bit_str(b: Bencher) { bench_bit_str(b, 64, Pattern::Dense); } @@ -39,12 +39,12 @@ fn hash_len_64_dense_str(b: Bencher) { bench_str(b, 64, Pattern::Dense); } -#[divan::bench(name = "hash/len_64/sparse/bit_string")] +#[divan::bench(name = "hash/len_64/sparse/ours_string")] fn hash_len_64_sparse_bit_string(b: Bencher) { bench_bit_string(b, 64, Pattern::Sparse); } -#[divan::bench(name = "hash/len_64/sparse/bit_str")] +#[divan::bench(name = "hash/len_64/sparse/ours_str")] fn hash_len_64_sparse_bit_str(b: Bencher) { bench_bit_str(b, 64, Pattern::Sparse); } @@ -59,12 +59,12 @@ fn hash_len_64_sparse_str(b: Bencher) { bench_str(b, 64, Pattern::Sparse); } -#[divan::bench(name = "hash/len_64/alternating/bit_string")] +#[divan::bench(name = "hash/len_64/alternating/ours_string")] fn hash_len_64_alternating_bit_string(b: Bencher) { bench_bit_string(b, 64, Pattern::Alternating); } -#[divan::bench(name = "hash/len_64/alternating/bit_str")] +#[divan::bench(name = "hash/len_64/alternating/ours_str")] fn hash_len_64_alternating_bit_str(b: Bencher) { bench_bit_str(b, 64, Pattern::Alternating); } @@ -83,12 +83,12 @@ fn hash_len_64_alternating_str(b: Bencher) { // len = 4096 // --------------------------------------------------------------------------- -#[divan::bench(name = "hash/len_4096/dense/bit_string")] +#[divan::bench(name = "hash/len_4096/dense/ours_string")] fn hash_len_4096_dense_bit_string(b: Bencher) { bench_bit_string(b, 4096, Pattern::Dense); } -#[divan::bench(name = "hash/len_4096/dense/bit_str")] +#[divan::bench(name = "hash/len_4096/dense/ours_str")] fn hash_len_4096_dense_bit_str(b: Bencher) { bench_bit_str(b, 4096, Pattern::Dense); } @@ -103,12 +103,12 @@ fn hash_len_4096_dense_str(b: Bencher) { bench_str(b, 4096, Pattern::Dense); } -#[divan::bench(name = "hash/len_4096/sparse/bit_string")] +#[divan::bench(name = "hash/len_4096/sparse/ours_string")] fn hash_len_4096_sparse_bit_string(b: Bencher) { bench_bit_string(b, 4096, Pattern::Sparse); } -#[divan::bench(name = "hash/len_4096/sparse/bit_str")] +#[divan::bench(name = "hash/len_4096/sparse/ours_str")] fn hash_len_4096_sparse_bit_str(b: Bencher) { bench_bit_str(b, 4096, Pattern::Sparse); } @@ -123,12 +123,12 @@ fn hash_len_4096_sparse_str(b: Bencher) { bench_str(b, 4096, Pattern::Sparse); } -#[divan::bench(name = "hash/len_4096/alternating/bit_string")] +#[divan::bench(name = "hash/len_4096/alternating/ours_string")] fn hash_len_4096_alternating_bit_string(b: Bencher) { bench_bit_string(b, 4096, Pattern::Alternating); } -#[divan::bench(name = "hash/len_4096/alternating/bit_str")] +#[divan::bench(name = "hash/len_4096/alternating/ours_str")] fn hash_len_4096_alternating_bit_str(b: Bencher) { bench_bit_str(b, 4096, Pattern::Alternating); } @@ -147,12 +147,12 @@ fn hash_len_4096_alternating_str(b: Bencher) { // len = 65536 // --------------------------------------------------------------------------- -#[divan::bench(name = "hash/len_65536/dense/bit_string")] +#[divan::bench(name = "hash/len_65536/dense/ours_string")] fn hash_len_65536_dense_bit_string(b: Bencher) { bench_bit_string(b, 65536, Pattern::Dense); } -#[divan::bench(name = "hash/len_65536/dense/bit_str")] +#[divan::bench(name = "hash/len_65536/dense/ours_str")] fn hash_len_65536_dense_bit_str(b: Bencher) { bench_bit_str(b, 65536, Pattern::Dense); } @@ -167,12 +167,12 @@ fn hash_len_65536_dense_str(b: Bencher) { bench_str(b, 65536, Pattern::Dense); } -#[divan::bench(name = "hash/len_65536/sparse/bit_string")] +#[divan::bench(name = "hash/len_65536/sparse/ours_string")] fn hash_len_65536_sparse_bit_string(b: Bencher) { bench_bit_string(b, 65536, Pattern::Sparse); } -#[divan::bench(name = "hash/len_65536/sparse/bit_str")] +#[divan::bench(name = "hash/len_65536/sparse/ours_str")] fn hash_len_65536_sparse_bit_str(b: Bencher) { bench_bit_str(b, 65536, Pattern::Sparse); } @@ -187,12 +187,12 @@ fn hash_len_65536_sparse_str(b: Bencher) { bench_str(b, 65536, Pattern::Sparse); } -#[divan::bench(name = "hash/len_65536/alternating/bit_string")] +#[divan::bench(name = "hash/len_65536/alternating/ours_string")] fn hash_len_65536_alternating_bit_string(b: Bencher) { bench_bit_string(b, 65536, Pattern::Alternating); } -#[divan::bench(name = "hash/len_65536/alternating/bit_str")] +#[divan::bench(name = "hash/len_65536/alternating/ours_str")] fn hash_len_65536_alternating_bit_str(b: Bencher) { bench_bit_str(b, 65536, Pattern::Alternating); } diff --git a/benches/matching_ends_with.rs b/benches/matching_ends_with.rs index 234d7d7..ed85a69 100644 --- a/benches/matching_ends_with.rs +++ b/benches/matching_ends_with.rs @@ -6,81 +6,141 @@ fn main() { } struct Case { - haystack_bits: BitString, - suffix_bits: BitString, - haystack_string: String, - suffix_string: String, + h: BitString, + s: BitString, + hs: String, + ss: String, } -fn make_bits(len: usize) -> BitString { - let mut bits = BitString::zeros(len); +fn mk(len: usize) -> BitString { + let mut b = BitString::zeros(len); for i in 0..len { if (i as u64 * 17 + 3) % 7 == 0 { - bits.set(i, true); + b.set(i, true); } } - bits + b } - -fn make_case(len: usize, sfx: usize) -> Case { - let h = make_bits(len); - let s = make_bits(sfx); +fn hit(len: usize, sfx: usize) -> Case { + let h = mk(len); + let s = mk(sfx); Case { - haystack_string: h.to_string(), - suffix_string: s.to_string(), - haystack_bits: h, - suffix_bits: s, + hs: h.to_string(), + ss: s.to_string(), + h, + s, } } -fn no_case(len: usize, sfx: usize) -> Case { - let mut h = make_bits(len); - let s = make_bits(sfx); - let last = h.bit_len() - 1; - h.set(last, !h.get(last).unwrap()); +fn no(len: usize, sfx: usize) -> Case { + let mut h = mk(len); + let s = mk(sfx); + h.set(h.bit_len() - 1, !h.get(h.bit_len() - 1).unwrap()); Case { - haystack_string: h.to_string(), - suffix_string: s.to_string(), - haystack_bits: h, - suffix_bits: s, + hs: h.to_string(), + ss: s.to_string(), + h, + s, } } -#[divan::bench(name = "ends_with/len_65/yes/bit_string")] -fn e65y(b: Bencher) { - b_bit(b, make_case(65, 4)); +#[divan::bench(name = "ends_with/len_65/hit/ours_str_str")] +fn ends_65_hit_str_str(b: Bencher) { + bench_str_str(b, &hit(65, 4)); +} +#[divan::bench(name = "ends_with/len_65/hit/ours_str_string")] +fn ends_65_hit_str_string(b: Bencher) { + bench_str_string(b, &hit(65, 4)); +} +#[divan::bench(name = "ends_with/len_65/hit/ours_string_str")] +fn ends_65_hit_string_str(b: Bencher) { + bench_string_str(b, &hit(65, 4)); +} +#[divan::bench(name = "ends_with/len_65/hit/ours_string_string")] +fn ends_65_hit_string_string(b: Bencher) { + bench_string_string(b, &hit(65, 4)); +} +#[divan::bench(name = "ends_with/len_65/hit/string")] +fn ends_65_hit_native(b: Bencher) { + bench_native(b, &hit(65, 4)); +} + +#[divan::bench(name = "ends_with/len_65/miss/ours_str_str")] +fn ends_65_miss_str_str(b: Bencher) { + bench_str_str(b, &no(65, 4)); +} +#[divan::bench(name = "ends_with/len_65/miss/ours_str_string")] +fn ends_65_miss_str_string(b: Bencher) { + bench_str_string(b, &no(65, 4)); } -#[divan::bench(name = "ends_with/len_65/yes/string")] -fn e65ys(b: Bencher) { - b_str(b, make_case(65, 4)); +#[divan::bench(name = "ends_with/len_65/miss/ours_string_str")] +fn ends_65_miss_string_str(b: Bencher) { + bench_string_str(b, &no(65, 4)); } -#[divan::bench(name = "ends_with/len_65/no/bit_string")] -fn e65n(b: Bencher) { - b_bit(b, no_case(65, 4)); +#[divan::bench(name = "ends_with/len_65/miss/ours_string_string")] +fn ends_65_miss_string_string(b: Bencher) { + bench_string_string(b, &no(65, 4)); } -#[divan::bench(name = "ends_with/len_65/no/string")] -fn e65ns(b: Bencher) { - b_str(b, no_case(65, 4)); +#[divan::bench(name = "ends_with/len_65/miss/string")] +fn ends_65_miss_native(b: Bencher) { + bench_native(b, &no(65, 4)); +} + +#[divan::bench(name = "ends_with/len_65536/hit/ours_str_str")] +fn ends_65536_hit_str_str(b: Bencher) { + bench_str_str(b, &hit(65536, 128)); } -#[divan::bench(name = "ends_with/len_65536/yes/bit_string")] -fn e6y(b: Bencher) { - b_bit(b, make_case(65_536, 128)); +#[divan::bench(name = "ends_with/len_65536/hit/ours_str_string")] +fn ends_65536_hit_str_string(b: Bencher) { + bench_str_string(b, &hit(65536, 128)); } -#[divan::bench(name = "ends_with/len_65536/yes/string")] -fn e6ys(b: Bencher) { - b_str(b, make_case(65_536, 128)); +#[divan::bench(name = "ends_with/len_65536/hit/ours_string_str")] +fn ends_65536_hit_string_str(b: Bencher) { + bench_string_str(b, &hit(65536, 128)); } -#[divan::bench(name = "ends_with/len_65536/no/bit_string")] -fn e6n(b: Bencher) { - b_bit(b, no_case(65_536, 128)); +#[divan::bench(name = "ends_with/len_65536/hit/ours_string_string")] +fn ends_65536_hit_string_string(b: Bencher) { + bench_string_string(b, &hit(65536, 128)); } -#[divan::bench(name = "ends_with/len_65536/no/string")] -fn e6ns(b: Bencher) { - b_str(b, no_case(65_536, 128)); +#[divan::bench(name = "ends_with/len_65536/hit/string")] +fn ends_65536_hit_native(b: Bencher) { + bench_native(b, &hit(65536, 128)); } -fn b_bit(b: Bencher, c: Case) { - b.bench(|| black_box(&c.haystack_bits).ends_with(black_box(c.suffix_bits.as_bit_str()))); +#[divan::bench(name = "ends_with/len_65536/miss/ours_str_str")] +fn ends_65536_miss_str_str(b: Bencher) { + bench_str_str(b, &no(65536, 128)); +} +#[divan::bench(name = "ends_with/len_65536/miss/ours_str_string")] +fn ends_65536_miss_str_string(b: Bencher) { + bench_str_string(b, &no(65536, 128)); +} +#[divan::bench(name = "ends_with/len_65536/miss/ours_string_str")] +fn ends_65536_miss_string_str(b: Bencher) { + bench_string_str(b, &no(65536, 128)); +} +#[divan::bench(name = "ends_with/len_65536/miss/ours_string_string")] +fn ends_65536_miss_string_string(b: Bencher) { + bench_string_string(b, &no(65536, 128)); +} +#[divan::bench(name = "ends_with/len_65536/miss/string")] +fn ends_65536_miss_native(b: Bencher) { + bench_native(b, &no(65536, 128)); +} + +fn bench_str_str(b: Bencher, c: &Case) { + let v = c.h.as_bit_str(); + b.bench(|| black_box(&v).ends_with_str(black_box(c.s.as_bit_str()))); +} +fn bench_str_string(b: Bencher, c: &Case) { + let v = c.h.as_bit_str(); + b.bench(|| black_box(&v).ends_with_string(black_box(&c.s))); +} +fn bench_string_str(b: Bencher, c: &Case) { + b.bench(|| black_box(&c.h).ends_with_str(black_box(c.s.as_bit_str()))); +} +fn bench_string_string(b: Bencher, c: &Case) { + b.bench(|| black_box(&c.h).ends_with_string(black_box(&c.s))); } -fn b_str(b: Bencher, c: Case) { - b.bench(|| black_box(&c.haystack_string).ends_with(black_box(&c.suffix_string))); +fn bench_native(b: Bencher, c: &Case) { + b.bench(|| black_box(&c.hs).ends_with(black_box(&c.ss))); } diff --git a/benches/matching_find.rs b/benches/matching_find.rs index c6bee89..9f2638c 100644 --- a/benches/matching_find.rs +++ b/benches/matching_find.rs @@ -12,78 +12,187 @@ struct NeedleCase { needle_string: String, } -#[divan::bench(name = "find/len_65/front/bit_string")] -fn f65fb(bencher: Bencher) { - bench_bit_string(bencher, make_case(65, 0)); +// -- find/len_65 ------------------------------------------------------------ + +#[divan::bench(name = "find_str/len_65/front/ours_string_str")] +fn find_65_front_str(b: Bencher) { + bench_find_str(b, make_case(65, 0)); } -#[divan::bench(name = "find/len_65/front/string")] -fn f65fs(bencher: Bencher) { - bench_string(bencher, make_case(65, 0)); +#[divan::bench(name = "find_str/len_65/front/ours_string_string")] +fn find_65_front_string(b: Bencher) { + bench_find_string(b, make_case(65, 0)); } -#[divan::bench(name = "find/len_65/middle/bit_string")] -fn f65mb(bencher: Bencher) { - bench_bit_string(bencher, middle_case(65)); +#[divan::bench(name = "find_str/len_65/front/string")] +fn find_65_front_native(b: Bencher) { + bench_native_find(b, make_case(65, 0)); } -#[divan::bench(name = "find/len_65/middle/string")] -fn f65ms(bencher: Bencher) { - bench_string(bencher, middle_case(65)); + +#[divan::bench(name = "find_str/len_65/middle/ours_string_str")] +fn find_65_middle_str(b: Bencher) { + bench_find_str(b, middle_case(65)); } -#[divan::bench(name = "find/len_65/end/bit_string")] -fn f65eb(bencher: Bencher) { - bench_bit_string(bencher, end_case(65)); +#[divan::bench(name = "find_str/len_65/middle/ours_string_string")] +fn find_65_middle_string(b: Bencher) { + bench_find_string(b, middle_case(65)); } -#[divan::bench(name = "find/len_65/end/string")] -fn f65es(bencher: Bencher) { - bench_string(bencher, end_case(65)); +#[divan::bench(name = "find_str/len_65/middle/string")] +fn find_65_middle_native(b: Bencher) { + bench_native_find(b, middle_case(65)); } -#[divan::bench(name = "find/len_65/miss/bit_string")] -fn f65xb(bencher: Bencher) { - bench_bit_string(bencher, miss_case(65)); + +#[divan::bench(name = "find_str/len_65/end/ours_string_str")] +fn find_65_end_str(b: Bencher) { + bench_find_str(b, end_case(65)); } -#[divan::bench(name = "find/len_65/miss/string")] -fn f65xs(bencher: Bencher) { - bench_string(bencher, miss_case(65)); +#[divan::bench(name = "find_str/len_65/end/ours_string_string")] +fn find_65_end_string(b: Bencher) { + bench_find_string(b, end_case(65)); +} +#[divan::bench(name = "find_str/len_65/end/string")] +fn find_65_end_native(b: Bencher) { + bench_native_find(b, end_case(65)); } -#[divan::bench(name = "find/len_65536/front/bit_string")] -fn f65536fb(bencher: Bencher) { - bench_bit_string(bencher, make_case(65_536, 0)); +#[divan::bench(name = "find_str/len_65/miss/ours_string_str")] +fn find_65_miss_str(b: Bencher) { + bench_find_str(b, miss_case(65)); +} +#[divan::bench(name = "find_str/len_65/miss/ours_string_string")] +fn find_65_miss_string(b: Bencher) { + bench_find_string(b, miss_case(65)); } -#[divan::bench(name = "find/len_65536/front/string")] -fn f65536fs(bencher: Bencher) { - bench_string(bencher, make_case(65_536, 0)); +#[divan::bench(name = "find_str/len_65/miss/string")] +fn find_65_miss_native(b: Bencher) { + bench_native_find(b, miss_case(65)); } -#[divan::bench(name = "find/len_65536/middle/bit_string")] -fn f65536mb(bencher: Bencher) { - bench_bit_string(bencher, middle_case(65_536)); + +// -- find/len_65536 --------------------------------------------------------- + +#[divan::bench(name = "find_str/len_65536/front/ours_string_str")] +fn find_65536_front_str(b: Bencher) { + bench_find_str(b, make_case(65536, 0)); } -#[divan::bench(name = "find/len_65536/middle/string")] -fn f65536ms(bencher: Bencher) { - bench_string(bencher, middle_case(65_536)); +#[divan::bench(name = "find_str/len_65536/front/ours_string_string")] +fn find_65536_front_string(b: Bencher) { + bench_find_string(b, make_case(65536, 0)); } -#[divan::bench(name = "find/len_65536/end/bit_string")] -fn f65536eb(bencher: Bencher) { - bench_bit_string(bencher, end_case(65_536)); +#[divan::bench(name = "find_str/len_65536/front/string")] +fn find_65536_front_native(b: Bencher) { + bench_native_find(b, make_case(65536, 0)); } -#[divan::bench(name = "find/len_65536/end/string")] -fn f65536es(bencher: Bencher) { - bench_string(bencher, end_case(65_536)); + +#[divan::bench(name = "find_str/len_65536/middle/ours_string_str")] +fn find_65536_middle_str(b: Bencher) { + bench_find_str(b, middle_case(65536)); } -#[divan::bench(name = "find/len_65536/miss/bit_string")] -fn f65536xb(bencher: Bencher) { - bench_bit_string(bencher, miss_case(65_536)); +#[divan::bench(name = "find_str/len_65536/middle/ours_string_string")] +fn find_65536_middle_string(b: Bencher) { + bench_find_string(b, middle_case(65536)); } -#[divan::bench(name = "find/len_65536/miss/string")] -fn f65536xs(bencher: Bencher) { - bench_string(bencher, miss_case(65_536)); +#[divan::bench(name = "find_str/len_65536/middle/string")] +fn find_65536_middle_native(b: Bencher) { + bench_native_find(b, middle_case(65536)); } -fn bench_bit_string(bencher: Bencher, case: NeedleCase) { - bencher.bench(|| black_box(&case.haystack_bits).find(black_box(case.needle_bits.as_bit_str()))); +#[divan::bench(name = "find_str/len_65536/end/ours_string_str")] +fn find_65536_end_str(b: Bencher) { + bench_find_str(b, end_case(65536)); +} +#[divan::bench(name = "find_str/len_65536/end/ours_string_string")] +fn find_65536_end_string(b: Bencher) { + bench_find_string(b, end_case(65536)); +} +#[divan::bench(name = "find_str/len_65536/end/string")] +fn find_65536_end_native(b: Bencher) { + bench_native_find(b, end_case(65536)); +} + +#[divan::bench(name = "find_str/len_65536/miss/ours_string_str")] +fn find_65536_miss_str(b: Bencher) { + bench_find_str(b, miss_case(65536)); +} +#[divan::bench(name = "find_str/len_65536/miss/ours_string_string")] +fn find_65536_miss_string(b: Bencher) { + bench_find_string(b, miss_case(65536)); +} +#[divan::bench(name = "find_str/len_65536/miss/string")] +fn find_65536_miss_native(b: Bencher) { + bench_native_find(b, miss_case(65536)); +} + +// -- contains/len_65 -------------------------------------------------------- + +#[divan::bench(name = "contains_str/len_65/yes/ours_string_str")] +fn contains_65_yes_str(b: Bencher) { + bench_contains_str(b, make_case(65, 0)); +} +#[divan::bench(name = "contains_str/len_65/yes/ours_string_string")] +fn contains_65_yes_string(b: Bencher) { + bench_contains_string(b, make_case(65, 0)); +} +#[divan::bench(name = "contains_str/len_65/yes/string")] +fn contains_65_yes_native(b: Bencher) { + bench_native_contains(b, make_case(65, 0)); } -fn bench_string(bencher: Bencher, case: NeedleCase) { - bencher.bench(|| black_box(&case.haystack_string).find(black_box(&case.needle_string))); +#[divan::bench(name = "contains_str/len_65/no/ours_string_str")] +fn contains_65_no_str(b: Bencher) { + bench_contains_str(b, miss_case(65)); +} +#[divan::bench(name = "contains_str/len_65/no/ours_string_string")] +fn contains_65_no_string(b: Bencher) { + bench_contains_string(b, miss_case(65)); +} +#[divan::bench(name = "contains_str/len_65/no/string")] +fn contains_65_no_native(b: Bencher) { + bench_native_contains(b, miss_case(65)); +} + +#[divan::bench(name = "contains_str/len_65536/yes/ours_string_str")] +fn contains_65536_yes_str(b: Bencher) { + bench_contains_str(b, make_case(65536, 0)); +} +#[divan::bench(name = "contains_str/len_65536/yes/ours_string_string")] +fn contains_65536_yes_string(b: Bencher) { + bench_contains_string(b, make_case(65536, 0)); +} +#[divan::bench(name = "contains_str/len_65536/yes/string")] +fn contains_65536_yes_native(b: Bencher) { + bench_native_contains(b, make_case(65536, 0)); +} + +#[divan::bench(name = "contains_str/len_65536/no/ours_string_str")] +fn contains_65536_no_str(b: Bencher) { + bench_contains_str(b, miss_case(65536)); +} +#[divan::bench(name = "contains_str/len_65536/no/ours_string_string")] +fn contains_65536_no_string(b: Bencher) { + bench_contains_string(b, miss_case(65536)); +} +#[divan::bench(name = "contains_str/len_65536/no/string")] +fn contains_65536_no_native(b: Bencher) { + bench_native_contains(b, miss_case(65536)); +} + +// -- helpers ---------------------------------------------------------------- + +fn bench_find_str(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_bits).find_str(black_box(c.needle_bits.as_bit_str()))); +} +fn bench_find_string(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_bits).find_string(black_box(&c.needle_bits))); +} +fn bench_contains_str(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_bits).contains_str(black_box(c.needle_bits.as_bit_str()))); +} +fn bench_contains_string(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_bits).contains_string(black_box(&c.needle_bits))); +} +fn bench_native_find(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_string).find(black_box(&c.needle_string))); +} +fn bench_native_contains(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_string).contains(black_box(&c.needle_string))); } fn middle_case(len: usize) -> NeedleCase { @@ -117,7 +226,6 @@ fn make_case(len: usize, position: usize) -> NeedleCase { needle_string: bools_to_string(&needle), } } -#[inline] fn chunk_len(len: usize) -> usize { (len / 8).max(1) } @@ -137,7 +245,6 @@ fn bools_to_string(values: &[bool]) -> String { } out } -#[inline] fn mix64(mut value: u64) -> u64 { value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); diff --git a/benches/matching_matches_at.rs b/benches/matching_matches_at.rs index 09f3dfc..6bb4cd9 100644 --- a/benches/matching_matches_at.rs +++ b/benches/matching_matches_at.rs @@ -57,35 +57,35 @@ fn no_case(len: usize, pat_len: usize, index: usize) -> Case { // 65-bit haystack (small — scalar path) // --------------------------------------------------------------------------- -#[divan::bench(name = "matches_at/len_65/yes/aligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65/yes/aligned/ours_string_str")] fn m65ya(b: Bencher) { b_bit(b, make_case(65, 4, 64)); } -#[divan::bench(name = "matches_at/len_65/yes/aligned/string")] +#[divan::bench(name = "matches_at_str/len_65/yes/aligned/string")] fn m65yas(b: Bencher) { b_str(b, make_case(65, 4, 64)); } -#[divan::bench(name = "matches_at/len_65/yes/unaligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65/yes/unaligned/ours_string_str")] fn m65yu(b: Bencher) { b_bit(b, make_case(65, 4, 3)); } -#[divan::bench(name = "matches_at/len_65/yes/unaligned/string")] +#[divan::bench(name = "matches_at_str/len_65/yes/unaligned/string")] fn m65yus(b: Bencher) { b_str(b, make_case(65, 4, 3)); } -#[divan::bench(name = "matches_at/len_65/no/aligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65/no/aligned/ours_string_str")] fn m65na(b: Bencher) { b_bit(b, no_case(65, 4, 64)); } -#[divan::bench(name = "matches_at/len_65/no/aligned/string")] +#[divan::bench(name = "matches_at_str/len_65/no/aligned/string")] fn m65nas(b: Bencher) { b_str(b, no_case(65, 4, 64)); } -#[divan::bench(name = "matches_at/len_65/no/unaligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65/no/unaligned/ours_string_str")] fn m65nu(b: Bencher) { b_bit(b, no_case(65, 4, 3)); } -#[divan::bench(name = "matches_at/len_65/no/unaligned/string")] +#[divan::bench(name = "matches_at_str/len_65/no/unaligned/string")] fn m65nus(b: Bencher) { b_str(b, no_case(65, 4, 3)); } @@ -94,35 +94,35 @@ fn m65nus(b: Bencher) { // 65536-bit haystack (large — SIMD path) // --------------------------------------------------------------------------- -#[divan::bench(name = "matches_at/len_65536/yes/aligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65536/yes/aligned/ours_string_str")] fn m6ya(b: Bencher) { b_bit(b, make_case(65_536, 128, 64)); } -#[divan::bench(name = "matches_at/len_65536/yes/aligned/string")] +#[divan::bench(name = "matches_at_str/len_65536/yes/aligned/string")] fn m6yas(b: Bencher) { b_str(b, make_case(65_536, 128, 64)); } -#[divan::bench(name = "matches_at/len_65536/yes/unaligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65536/yes/unaligned/ours_string_str")] fn m6yu(b: Bencher) { b_bit(b, make_case(65_536, 128, 3)); } -#[divan::bench(name = "matches_at/len_65536/yes/unaligned/string")] +#[divan::bench(name = "matches_at_str/len_65536/yes/unaligned/string")] fn m6yus(b: Bencher) { b_str(b, make_case(65_536, 128, 3)); } -#[divan::bench(name = "matches_at/len_65536/no/aligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65536/no/aligned/ours_string_str")] fn m6na(b: Bencher) { b_bit(b, no_case(65_536, 128, 64)); } -#[divan::bench(name = "matches_at/len_65536/no/aligned/string")] +#[divan::bench(name = "matches_at_str/len_65536/no/aligned/string")] fn m6nas(b: Bencher) { b_str(b, no_case(65_536, 128, 64)); } -#[divan::bench(name = "matches_at/len_65536/no/unaligned/bit_string")] +#[divan::bench(name = "matches_at_str/len_65536/no/unaligned/ours_string_str")] fn m6nu(b: Bencher) { b_bit(b, no_case(65_536, 128, 3)); } -#[divan::bench(name = "matches_at/len_65536/no/unaligned/string")] +#[divan::bench(name = "matches_at_str/len_65536/no/unaligned/string")] fn m6nus(b: Bencher) { b_str(b, no_case(65_536, 128, 3)); } @@ -133,7 +133,7 @@ fn m6nus(b: Bencher) { fn b_bit(b: Bencher, c: Case) { b.bench(|| { - black_box(&c.haystack_bits).matches_at(c.index, black_box(c.pattern_bits.as_bit_str())) + black_box(&c.haystack_bits).matches_at_str(c.index, black_box(c.pattern_bits.as_bit_str())) }); } diff --git a/benches/matching_rfind.rs b/benches/matching_rfind.rs index f3f7c17..f771599 100644 --- a/benches/matching_rfind.rs +++ b/benches/matching_rfind.rs @@ -12,109 +12,137 @@ struct NeedleCase { needle_string: String, } -#[divan::bench(name = "rfind/len_65/front/bit_string")] -fn rfind_len_65_front_bit_string(bencher: Bencher) { - bench_bit_string(bencher, make_case(65, 0)); -} +// -- rfind/len_65 ----------------------------------------------------------- +#[divan::bench(name = "rfind/len_65/front/ours_string_str")] +fn rfind_65_front_str(b: Bencher) { + b_str(b, make_case(65, 0)); +} +#[divan::bench(name = "rfind/len_65/front/ours_string_string")] +fn rfind_65_front_string(b: Bencher) { + b_string(b, make_case(65, 0)); +} #[divan::bench(name = "rfind/len_65/front/string")] -fn rfind_len_65_front_string(bencher: Bencher) { - bench_string(bencher, make_case(65, 0)); +fn rfind_65_front_native(b: Bencher) { + b_native(b, make_case(65, 0)); } -#[divan::bench(name = "rfind/len_65/middle/bit_string")] -fn rfind_len_65_middle_bit_string(bencher: Bencher) { - bench_bit_string(bencher, middle_case(65)); +#[divan::bench(name = "rfind/len_65/middle/ours_string_str")] +fn rfind_65_middle_str(b: Bencher) { + b_str(b, middle_case(65)); +} +#[divan::bench(name = "rfind/len_65/middle/ours_string_string")] +fn rfind_65_middle_string(b: Bencher) { + b_string(b, middle_case(65)); } - #[divan::bench(name = "rfind/len_65/middle/string")] -fn rfind_len_65_middle_string(bencher: Bencher) { - bench_string(bencher, middle_case(65)); +fn rfind_65_middle_native(b: Bencher) { + b_native(b, middle_case(65)); } -#[divan::bench(name = "rfind/len_65/end/bit_string")] -fn rfind_len_65_end_bit_string(bencher: Bencher) { - bench_bit_string(bencher, end_case(65)); +#[divan::bench(name = "rfind/len_65/end/ours_string_str")] +fn rfind_65_end_str(b: Bencher) { + b_str(b, end_case(65)); +} +#[divan::bench(name = "rfind/len_65/end/ours_string_string")] +fn rfind_65_end_string(b: Bencher) { + b_string(b, end_case(65)); } - #[divan::bench(name = "rfind/len_65/end/string")] -fn rfind_len_65_end_string(bencher: Bencher) { - bench_string(bencher, end_case(65)); +fn rfind_65_end_native(b: Bencher) { + b_native(b, end_case(65)); } -#[divan::bench(name = "rfind/len_65/miss/bit_string")] -fn rfind_len_65_miss_bit_string(bencher: Bencher) { - bench_bit_string(bencher, miss_case(65)); +#[divan::bench(name = "rfind/len_65/miss/ours_string_str")] +fn rfind_65_miss_str(b: Bencher) { + b_str(b, miss_case(65)); +} +#[divan::bench(name = "rfind/len_65/miss/ours_string_string")] +fn rfind_65_miss_string(b: Bencher) { + b_string(b, miss_case(65)); } - #[divan::bench(name = "rfind/len_65/miss/string")] -fn rfind_len_65_miss_string(bencher: Bencher) { - bench_string(bencher, miss_case(65)); +fn rfind_65_miss_native(b: Bencher) { + b_native(b, miss_case(65)); } -#[divan::bench(name = "rfind/len_65536/front/bit_string")] -fn rfind_len_65536_front_bit_string(bencher: Bencher) { - bench_bit_string(bencher, make_case(65_536, 0)); -} +// -- rfind/len_65536 -------------------------------------------------------- +#[divan::bench(name = "rfind/len_65536/front/ours_string_str")] +fn rfind_65536_front_str(b: Bencher) { + b_str(b, make_case(65536, 0)); +} +#[divan::bench(name = "rfind/len_65536/front/ours_string_string")] +fn rfind_65536_front_string(b: Bencher) { + b_string(b, make_case(65536, 0)); +} #[divan::bench(name = "rfind/len_65536/front/string")] -fn rfind_len_65536_front_string(bencher: Bencher) { - bench_string(bencher, make_case(65_536, 0)); +fn rfind_65536_front_native(b: Bencher) { + b_native(b, make_case(65536, 0)); } -#[divan::bench(name = "rfind/len_65536/middle/bit_string")] -fn rfind_len_65536_middle_bit_string(bencher: Bencher) { - bench_bit_string(bencher, middle_case(65_536)); +#[divan::bench(name = "rfind/len_65536/middle/ours_string_str")] +fn rfind_65536_middle_str(b: Bencher) { + b_str(b, middle_case(65536)); +} +#[divan::bench(name = "rfind/len_65536/middle/ours_string_string")] +fn rfind_65536_middle_string(b: Bencher) { + b_string(b, middle_case(65536)); } - #[divan::bench(name = "rfind/len_65536/middle/string")] -fn rfind_len_65536_middle_string(bencher: Bencher) { - bench_string(bencher, middle_case(65_536)); +fn rfind_65536_middle_native(b: Bencher) { + b_native(b, middle_case(65536)); } -#[divan::bench(name = "rfind/len_65536/end/bit_string")] -fn rfind_len_65536_end_bit_string(bencher: Bencher) { - bench_bit_string(bencher, end_case(65_536)); +#[divan::bench(name = "rfind/len_65536/end/ours_string_str")] +fn rfind_65536_end_str(b: Bencher) { + b_str(b, end_case(65536)); +} +#[divan::bench(name = "rfind/len_65536/end/ours_string_string")] +fn rfind_65536_end_string(b: Bencher) { + b_string(b, end_case(65536)); } - #[divan::bench(name = "rfind/len_65536/end/string")] -fn rfind_len_65536_end_string(bencher: Bencher) { - bench_string(bencher, end_case(65_536)); +fn rfind_65536_end_native(b: Bencher) { + b_native(b, end_case(65536)); } -#[divan::bench(name = "rfind/len_65536/miss/bit_string")] -fn rfind_len_65536_miss_bit_string(bencher: Bencher) { - bench_bit_string(bencher, miss_case(65_536)); +#[divan::bench(name = "rfind/len_65536/miss/ours_string_str")] +fn rfind_65536_miss_str(b: Bencher) { + b_str(b, miss_case(65536)); +} +#[divan::bench(name = "rfind/len_65536/miss/ours_string_string")] +fn rfind_65536_miss_string(b: Bencher) { + b_string(b, miss_case(65536)); } - #[divan::bench(name = "rfind/len_65536/miss/string")] -fn rfind_len_65536_miss_string(bencher: Bencher) { - bench_string(bencher, miss_case(65_536)); +fn rfind_65536_miss_native(b: Bencher) { + b_native(b, miss_case(65536)); } -fn bench_bit_string(bencher: Bencher, case: NeedleCase) { - bencher - .bench(|| black_box(&case.haystack_bits).rfind(black_box(case.needle_bits.as_bit_str()))); -} +// -- helpers ---------------------------------------------------------------- -fn bench_string(bencher: Bencher, case: NeedleCase) { - bencher.bench(|| black_box(&case.haystack_string).rfind(black_box(&case.needle_string))); +fn b_str(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_bits).rfind_str(black_box(c.needle_bits.as_bit_str()))); +} +fn b_string(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_bits).rfind_string(black_box(&c.needle_bits))); +} +fn b_native(b: Bencher, c: NeedleCase) { + b.bench(|| black_box(&c.haystack_string).rfind(black_box(&c.needle_string))); } fn middle_case(len: usize) -> NeedleCase { - let needle_len = chunk_len(len); - make_case(len, (len - needle_len) / 2) + let n = chunk_len(len); + make_case(len, (len - n) / 2) } - fn end_case(len: usize) -> NeedleCase { - let needle_len = chunk_len(len); - make_case(len, len - needle_len) + let n = chunk_len(len); + make_case(len, len - n) } - fn miss_case(len: usize) -> NeedleCase { let haystack = vec![false; len]; let needle = vec![true; chunk_len(len)]; - NeedleCase { haystack_bits: haystack.iter().copied().collect(), needle_bits: needle.iter().copied().collect(), @@ -122,15 +150,12 @@ fn miss_case(len: usize) -> NeedleCase { needle_string: bools_to_string(&needle), } } - fn make_case(len: usize, position: usize) -> NeedleCase { let needle = make_needle_bits(chunk_len(len)); let mut haystack = vec![false; len]; - for (offset, &value) in needle.iter().enumerate() { haystack[position + offset] = value; } - NeedleCase { haystack_bits: haystack.iter().copied().collect(), needle_bits: needle.iter().copied().collect(), @@ -138,36 +163,25 @@ fn make_case(len: usize, position: usize) -> NeedleCase { needle_string: bools_to_string(&needle), } } - -#[inline] fn chunk_len(len: usize) -> usize { (len / 8).max(1) } - fn make_needle_bits(len: usize) -> Vec { let mut out = Vec::with_capacity(len); - for index in 0..len { out.push(mix64((index as u64).wrapping_add(0x9e37_79b9_7f4a_7c15)) & 7 <= 2); } - out[0] = true; out[len - 1] = true; - out } - fn bools_to_string(values: &[bool]) -> String { let mut out = String::with_capacity(values.len()); - for &value in values { out.push(if value { '1' } else { '0' }); } - out } - -#[inline] fn mix64(mut value: u64) -> u64 { value = value.wrapping_add(0x9e37_79b9_7f4a_7c15); value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); diff --git a/benches/matching_starts_with.rs b/benches/matching_starts_with.rs index d53f93d..35c4a3a 100644 --- a/benches/matching_starts_with.rs +++ b/benches/matching_starts_with.rs @@ -6,80 +6,141 @@ fn main() { } struct Case { - haystack_bits: BitString, - prefix_bits: BitString, - haystack_string: String, - prefix_string: String, + h: BitString, + p: BitString, + hs: String, + ps: String, } -fn make_bits(len: usize) -> BitString { - let mut bits = BitString::zeros(len); +fn mk(len: usize) -> BitString { + let mut b = BitString::zeros(len); for i in 0..len { if (i as u64 * 17 + 3) % 7 == 0 { - bits.set(i, true); + b.set(i, true); } } - bits + b } - -fn make_case(len: usize, pfx: usize) -> Case { - let h = make_bits(len); - let p = make_bits(pfx); +fn hit(len: usize, pfx: usize) -> Case { + let h = mk(len); + let p = mk(pfx); Case { - haystack_string: h.to_string(), - prefix_string: p.to_string(), - haystack_bits: h, - prefix_bits: p, + hs: h.to_string(), + ps: p.to_string(), + h, + p, } } -fn no_case(len: usize, pfx: usize) -> Case { - let mut h = make_bits(len); - let p = make_bits(pfx); +fn no(len: usize, pfx: usize) -> Case { + let mut h = mk(len); + let p = mk(pfx); h.set(0, !h.get(0).unwrap()); Case { - haystack_string: h.to_string(), - prefix_string: p.to_string(), - haystack_bits: h, - prefix_bits: p, + hs: h.to_string(), + ps: p.to_string(), + h, + p, } } -#[divan::bench(name = "starts_with/len_65/yes/bit_string")] -fn s65y(b: Bencher) { - b_bit(b, make_case(65, 4)); +#[divan::bench(name = "starts_with/len_65/hit/ours_str_str")] +fn starts_65_hit_str_str(b: Bencher) { + bench_str_str(b, &hit(65, 4)); +} +#[divan::bench(name = "starts_with/len_65/hit/ours_str_string")] +fn starts_65_hit_str_string(b: Bencher) { + bench_str_string(b, &hit(65, 4)); +} +#[divan::bench(name = "starts_with/len_65/hit/ours_string_str")] +fn starts_65_hit_string_str(b: Bencher) { + bench_string_str(b, &hit(65, 4)); +} +#[divan::bench(name = "starts_with/len_65/hit/ours_string_string")] +fn starts_65_hit_string_string(b: Bencher) { + bench_string_string(b, &hit(65, 4)); +} +#[divan::bench(name = "starts_with/len_65/hit/string")] +fn starts_65_hit_native(b: Bencher) { + bench_native(b, &hit(65, 4)); +} + +#[divan::bench(name = "starts_with/len_65/miss/ours_str_str")] +fn starts_65_miss_str_str(b: Bencher) { + bench_str_str(b, &no(65, 4)); +} +#[divan::bench(name = "starts_with/len_65/miss/ours_str_string")] +fn starts_65_miss_str_string(b: Bencher) { + bench_str_string(b, &no(65, 4)); } -#[divan::bench(name = "starts_with/len_65/yes/string")] -fn s65ys(b: Bencher) { - b_str(b, make_case(65, 4)); +#[divan::bench(name = "starts_with/len_65/miss/ours_string_str")] +fn starts_65_miss_string_str(b: Bencher) { + bench_string_str(b, &no(65, 4)); } -#[divan::bench(name = "starts_with/len_65/no/bit_string")] -fn s65n(b: Bencher) { - b_bit(b, no_case(65, 4)); +#[divan::bench(name = "starts_with/len_65/miss/ours_string_string")] +fn starts_65_miss_string_string(b: Bencher) { + bench_string_string(b, &no(65, 4)); } -#[divan::bench(name = "starts_with/len_65/no/string")] -fn s65ns(b: Bencher) { - b_str(b, no_case(65, 4)); +#[divan::bench(name = "starts_with/len_65/miss/string")] +fn starts_65_miss_native(b: Bencher) { + bench_native(b, &no(65, 4)); +} + +#[divan::bench(name = "starts_with/len_65536/hit/ours_str_str")] +fn starts_65536_hit_str_str(b: Bencher) { + bench_str_str(b, &hit(65536, 128)); } -#[divan::bench(name = "starts_with/len_65536/yes/bit_string")] -fn s6y(b: Bencher) { - b_bit(b, make_case(65_536, 128)); +#[divan::bench(name = "starts_with/len_65536/hit/ours_str_string")] +fn starts_65536_hit_str_string(b: Bencher) { + bench_str_string(b, &hit(65536, 128)); } -#[divan::bench(name = "starts_with/len_65536/yes/string")] -fn s6ys(b: Bencher) { - b_str(b, make_case(65_536, 128)); +#[divan::bench(name = "starts_with/len_65536/hit/ours_string_str")] +fn starts_65536_hit_string_str(b: Bencher) { + bench_string_str(b, &hit(65536, 128)); } -#[divan::bench(name = "starts_with/len_65536/no/bit_string")] -fn s6n(b: Bencher) { - b_bit(b, no_case(65_536, 128)); +#[divan::bench(name = "starts_with/len_65536/hit/ours_string_string")] +fn starts_65536_hit_string_string(b: Bencher) { + bench_string_string(b, &hit(65536, 128)); } -#[divan::bench(name = "starts_with/len_65536/no/string")] -fn s6ns(b: Bencher) { - b_str(b, no_case(65_536, 128)); +#[divan::bench(name = "starts_with/len_65536/hit/string")] +fn starts_65536_hit_native(b: Bencher) { + bench_native(b, &hit(65536, 128)); } -fn b_bit(b: Bencher, c: Case) { - b.bench(|| black_box(&c.haystack_bits).starts_with(black_box(c.prefix_bits.as_bit_str()))); +#[divan::bench(name = "starts_with/len_65536/miss/ours_str_str")] +fn starts_65536_miss_str_str(b: Bencher) { + bench_str_str(b, &no(65536, 128)); +} +#[divan::bench(name = "starts_with/len_65536/miss/ours_str_string")] +fn starts_65536_miss_str_string(b: Bencher) { + bench_str_string(b, &no(65536, 128)); +} +#[divan::bench(name = "starts_with/len_65536/miss/ours_string_str")] +fn starts_65536_miss_string_str(b: Bencher) { + bench_string_str(b, &no(65536, 128)); +} +#[divan::bench(name = "starts_with/len_65536/miss/ours_string_string")] +fn starts_65536_miss_string_string(b: Bencher) { + bench_string_string(b, &no(65536, 128)); +} +#[divan::bench(name = "starts_with/len_65536/miss/string")] +fn starts_65536_miss_native(b: Bencher) { + bench_native(b, &no(65536, 128)); +} + +fn bench_str_str(b: Bencher, c: &Case) { + let v = c.h.as_bit_str(); + b.bench(|| black_box(&v).starts_with_str(black_box(c.p.as_bit_str()))); +} +fn bench_str_string(b: Bencher, c: &Case) { + let v = c.h.as_bit_str(); + b.bench(|| black_box(&v).starts_with_string(black_box(&c.p))); +} +fn bench_string_str(b: Bencher, c: &Case) { + b.bench(|| black_box(&c.h).starts_with_str(black_box(c.p.as_bit_str()))); +} +fn bench_string_string(b: Bencher, c: &Case) { + b.bench(|| black_box(&c.h).starts_with_string(black_box(&c.p))); } -fn b_str(b: Bencher, c: Case) { - b.bench(|| black_box(&c.haystack_string).starts_with(black_box(&c.prefix_string))); +fn bench_native(b: Bencher, c: &Case) { + b.bench(|| black_box(&c.hs).starts_with(black_box(&c.ps))); } diff --git a/benches/matching_strip_prefix.rs b/benches/matching_strip_prefix.rs index a6631a4..21a06d1 100644 --- a/benches/matching_strip_prefix.rs +++ b/benches/matching_strip_prefix.rs @@ -12,7 +12,7 @@ struct NeedleCase { needle_string: String, } -#[divan::bench(name = "strip_prefix/len_65/hit/bit_string")] +#[divan::bench(name = "strip_prefix/len_65/hit/ours_string")] fn strip_prefix_len_65_hit_bit_string(bencher: Bencher) { bench_bit_string(bencher, hit_case(65)); } @@ -22,7 +22,7 @@ fn strip_prefix_len_65_hit_string(bencher: Bencher) { bench_string(bencher, hit_case(65)); } -#[divan::bench(name = "strip_prefix/len_65/miss/bit_string")] +#[divan::bench(name = "strip_prefix/len_65/miss/ours_string")] fn strip_prefix_len_65_miss_bit_string(bencher: Bencher) { bench_bit_string(bencher, miss_case(65)); } @@ -32,7 +32,7 @@ fn strip_prefix_len_65_miss_string(bencher: Bencher) { bench_string(bencher, miss_case(65)); } -#[divan::bench(name = "strip_prefix/len_65536/hit/bit_string")] +#[divan::bench(name = "strip_prefix/len_65536/hit/ours_string")] fn strip_prefix_len_65536_hit_bit_string(bencher: Bencher) { bench_bit_string(bencher, hit_case(65_536)); } @@ -42,7 +42,7 @@ fn strip_prefix_len_65536_hit_string(bencher: Bencher) { bench_string(bencher, hit_case(65_536)); } -#[divan::bench(name = "strip_prefix/len_65536/miss/bit_string")] +#[divan::bench(name = "strip_prefix/len_65536/miss/ours_string")] fn strip_prefix_len_65536_miss_bit_string(bencher: Bencher) { bench_bit_string(bencher, miss_case(65_536)); } @@ -54,7 +54,7 @@ fn strip_prefix_len_65536_miss_string(bencher: Bencher) { fn bench_bit_string(bencher: Bencher, case: NeedleCase) { bencher.bench(|| { - black_box(&case.haystack_bits).strip_prefix(black_box(case.needle_bits.as_bit_str())) + black_box(&case.haystack_bits).strip_prefix_str(black_box(case.needle_bits.as_bit_str())) }); } diff --git a/benches/ord.rs b/benches/ord.rs index d42a39b..17af5be 100644 --- a/benches/ord.rs +++ b/benches/ord.rs @@ -9,7 +9,7 @@ fn main() { // len = 64 // --------------------------------------------------------------------------- -#[divan::bench(name = "cmp/len_64/identical/bit_string")] +#[divan::bench(name = "cmp/len_64/identical/ours_string")] fn cmp_len_64_identical_bit_string(b: Bencher) { bench_bit_string(b, 64, CmpCase::Identical); } @@ -24,7 +24,7 @@ fn cmp_len_64_identical_string(b: Bencher) { bench_string(b, 64, CmpCase::Identical); } -#[divan::bench(name = "cmp/len_64/diff_first/bit_string")] +#[divan::bench(name = "cmp/len_64/diff_first/ours_string")] fn cmp_len_64_diff_first_bit_string(b: Bencher) { bench_bit_string(b, 64, CmpCase::DifferAtFirst); } @@ -39,7 +39,7 @@ fn cmp_len_64_diff_first_string(b: Bencher) { bench_string(b, 64, CmpCase::DifferAtFirst); } -#[divan::bench(name = "cmp/len_64/diff_last/bit_string")] +#[divan::bench(name = "cmp/len_64/diff_last/ours_string")] fn cmp_len_64_diff_last_bit_string(b: Bencher) { bench_bit_string(b, 64, CmpCase::DifferAtLast); } @@ -58,7 +58,7 @@ fn cmp_len_64_diff_last_string(b: Bencher) { // len = 4096 // --------------------------------------------------------------------------- -#[divan::bench(name = "cmp/len_4096/identical/bit_string")] +#[divan::bench(name = "cmp/len_4096/identical/ours_string")] fn cmp_len_4096_identical_bit_string(b: Bencher) { bench_bit_string(b, 4096, CmpCase::Identical); } @@ -73,7 +73,7 @@ fn cmp_len_4096_identical_string(b: Bencher) { bench_string(b, 4096, CmpCase::Identical); } -#[divan::bench(name = "cmp/len_4096/diff_first/bit_string")] +#[divan::bench(name = "cmp/len_4096/diff_first/ours_string")] fn cmp_len_4096_diff_first_bit_string(b: Bencher) { bench_bit_string(b, 4096, CmpCase::DifferAtFirst); } @@ -88,7 +88,7 @@ fn cmp_len_4096_diff_first_string(b: Bencher) { bench_string(b, 4096, CmpCase::DifferAtFirst); } -#[divan::bench(name = "cmp/len_4096/diff_last/bit_string")] +#[divan::bench(name = "cmp/len_4096/diff_last/ours_string")] fn cmp_len_4096_diff_last_bit_string(b: Bencher) { bench_bit_string(b, 4096, CmpCase::DifferAtLast); } @@ -107,7 +107,7 @@ fn cmp_len_4096_diff_last_string(b: Bencher) { // len = 65536 // --------------------------------------------------------------------------- -#[divan::bench(name = "cmp/len_65536/identical/bit_string")] +#[divan::bench(name = "cmp/len_65536/identical/ours_string")] fn cmp_len_65536_identical_bit_string(b: Bencher) { bench_bit_string(b, 65536, CmpCase::Identical); } @@ -122,7 +122,7 @@ fn cmp_len_65536_identical_string(b: Bencher) { bench_string(b, 65536, CmpCase::Identical); } -#[divan::bench(name = "cmp/len_65536/diff_first/bit_string")] +#[divan::bench(name = "cmp/len_65536/diff_first/ours_string")] fn cmp_len_65536_diff_first_bit_string(b: Bencher) { bench_bit_string(b, 65536, CmpCase::DifferAtFirst); } @@ -137,7 +137,7 @@ fn cmp_len_65536_diff_first_string(b: Bencher) { bench_string(b, 65536, CmpCase::DifferAtFirst); } -#[divan::bench(name = "cmp/len_65536/diff_last/bit_string")] +#[divan::bench(name = "cmp/len_65536/diff_last/ours_string")] fn cmp_len_65536_diff_last_bit_string(b: Bencher) { bench_bit_string(b, 65536, CmpCase::DifferAtLast); } diff --git a/src/bit_str.rs b/src/bit_str.rs index 4a81d5b..ee725f1 100644 --- a/src/bit_str.rs +++ b/src/bit_str.rs @@ -26,7 +26,7 @@ pub struct BitStr<'bs> { pub mod errors; mod impls_for_access; -mod impls_for_bit_arith; +pub(crate) mod impls_for_bit_arith; mod impls_for_eq; mod impls_for_fmt; mod impls_for_hash; diff --git a/src/bit_str/impls_for_access.rs b/src/bit_str/impls_for_access.rs index a249247..9e2436a 100644 --- a/src/bit_str/impls_for_access.rs +++ b/src/bit_str/impls_for_access.rs @@ -35,7 +35,10 @@ impl<'bs> BitStr<'bs> { if valid_bits == 0 { return 0; } - let raw = self.source.words().read_word_at(self.start + bit_start); + let raw = self + .source + .words() + .read_word_at::(self.start + bit_start); raw & low_mask(valid_bits) } } diff --git a/src/bit_str/impls_for_bit_arith.rs b/src/bit_str/impls_for_bit_arith.rs index 0393966..b5b9eb0 100644 --- a/src/bit_str/impls_for_bit_arith.rs +++ b/src/bit_str/impls_for_bit_arith.rs @@ -1,2 +1,3 @@ mod impls_for_count_ones; -mod impls_for_leading_zeros; +pub(crate) mod impls_for_leading_zeros; +mod impls_for_trailing_zeros; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_count_ones.rs b/src/bit_str/impls_for_bit_arith/impls_for_count_ones.rs index bdc3129..e8aaafb 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_count_ones.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_count_ones.rs @@ -1,64 +1,18 @@ -use crate::traits::*; -use crate::{WORD_BITS, low_mask}; +use crate::WORD_BITS; use crate::BitStr; +mod inner; + impl<'bs> BitStr<'bs> { /// Returns the number of bits set to 1. #[inline] pub fn count_ones(&self) -> usize { - if self.bit_len == 0 { - return 0; - } - - let words = self.source.words(); - - // Fast path: when `start` is word-aligned we can delegate directly to - // the SIMD-accelerated `[u64]::count_ones` on the relevant suffix. if self.start % WORD_BITS == 0 { - let word_start = self.start / WORD_BITS; - return words[word_start..].count_ones(self.bit_len); - } - - // Unaligned start: count across word boundaries. - let start_word = self.start / WORD_BITS; - let start_offset = self.start % WORD_BITS; - let end = self.start + self.bit_len; - let last_word = (end - 1) / WORD_BITS; - - // All bits lie within a single word. - if start_word == last_word { - let mask = low_mask(self.bit_len) << start_offset; - return (words[start_word] & mask).count_ones() as usize; - } - - let mut count = 0usize; - - // First word: bits from start_offset upward. - count += (words[start_word] >> start_offset).count_ones() as usize; - - let end_rem = end % WORD_BITS; - - // Middle words: full u64 words, SIMD-accelerated via - // `[u64]::count_ones`. The last word is included in the SIMD path - // when it is full (end_rem == 0), otherwise handled separately. - let mid_start = start_word + 1; - let mid_end = if end_rem == 0 { - last_word + 1 + self.count_ones_inner::() } else { - last_word - }; - let full_word_count = mid_end.saturating_sub(mid_start); - if full_word_count > 0 { - count += words[mid_start..mid_end].count_ones(full_word_count * WORD_BITS); + self.count_ones_inner::() } - - // Last word: partial. - if end_rem != 0 { - count += (words[last_word] & low_mask(end_rem)).count_ones() as usize; - } - - count } /// Returns the number of bits set to 0. diff --git a/src/bit_str/impls_for_bit_arith/impls_for_count_ones/inner.rs b/src/bit_str/impls_for_bit_arith/impls_for_count_ones/inner.rs new file mode 100644 index 0000000..44efbb0 --- /dev/null +++ b/src/bit_str/impls_for_bit_arith/impls_for_count_ones/inner.rs @@ -0,0 +1,44 @@ +use crate::traits::WordsScan; +use crate::{WORD_BITS, low_mask}; + +use crate::BitStr; + +impl<'bs> BitStr<'bs> { + /// `count_ones` with compile-time alignment signal. + #[inline] + pub(crate) fn count_ones_inner(&self) -> usize { + if self.bit_len == 0 { + return 0; + } + let words = self.source.words(); + if WORD_ALIGNED || self.start % WORD_BITS == 0 { + let ws = self.start / WORD_BITS; + return words[ws..].count_ones(self.bit_len); + } + let sw = self.start / WORD_BITS; + let so = self.start % WORD_BITS; + let end = self.start + self.bit_len; + let last_word = (end - 1) / WORD_BITS; + if sw == last_word { + let mask = low_mask(self.bit_len) << so; + return (words[sw] & mask).count_ones() as usize; + } + let mut count = 0usize; + count += (words[sw] >> so).count_ones() as usize; + let end_rem = end % WORD_BITS; + let mid_start = sw + 1; + let mid_end = if end_rem == 0 { + last_word + 1 + } else { + last_word + }; + let fwc = mid_end.saturating_sub(mid_start); + if fwc > 0 { + count += words[mid_start..mid_end].count_ones(fwc * WORD_BITS); + } + if end_rem != 0 { + count += (words[last_word] & low_mask(end_rem)).count_ones() as usize; + } + count + } +} diff --git a/src/bit_str/impls_for_bit_arith/impls_for_count_ones/tests_for_count_ones.rs b/src/bit_str/impls_for_bit_arith/impls_for_count_ones/tests_for_count_ones.rs index b265681..5bef273 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_count_ones/tests_for_count_ones.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_count_ones/tests_for_count_ones.rs @@ -1,155 +1 @@ -use int_interval::UsizeCO; - -use crate::BitString; - -/// An empty view always has zero ones and zeros. -#[test] -fn counts_empty_view() { - let bits = BitString::try_from("10110").unwrap(); - // Create an empty view by slicing beyond the source length. - let v = bits.as_bit_str().slice(UsizeCO::try_new(10, 20).unwrap()); - - assert_eq!(v.count_ones(), 0); - assert_eq!(v.count_zeros(), 0); -} - -/// Word-aligned views take the SIMD fast path. -#[test] -fn word_aligned_fast_path() { - let mut bits = BitString::zeros(130); - for i in [0, 63, 64, 65, 127, 128, 129] { - bits.set(i, true); - } - // Full view: bit_len=130, aligned, 7 ones. - let v = bits.as_bit_str(); - assert_eq!(v.count_ones(), 7); - assert_eq!(v.count_zeros(), 123); - - // Slice from word-aligned boundary (64..130), 6 ones. - let v = bits.as_bit_str().slice(UsizeCO::try_new(64, 130).unwrap()); - assert_eq!(v.count_ones(), 5); -} - -/// Unaligned start: the first word is partial. -#[test] -fn unaligned_start() { - let mut bits = BitString::zeros(130); - // Set only bit 1 and bit 66 (word 0 offset 1, word 1 offset 2). - bits.set(1, true); - bits.set(66, true); - - // View from bit 1 to bit 130 → len 129, unaligned start. - let v = bits.as_bit_str().slice(UsizeCO::try_new(1, 130).unwrap()); - assert_eq!(v.count_ones(), 2); - assert_eq!(v.count_zeros(), 127); -} - -/// All-ones string of various lengths. -#[test] -fn all_ones_at_various_lengths() { - for len in [1, 63, 64, 65, 127, 128, 129, 130] { - let bits = BitString::ones(len); - let v = bits.as_bit_str(); - assert_eq!(v.count_ones(), len, "len={len}"); - assert_eq!(v.count_zeros(), 0, "len={len}"); - } -} - -/// All-zeros string of various lengths. -#[test] -fn all_zeros_at_various_lengths() { - for len in [1, 63, 64, 65, 127, 128, 129, 130] { - let bits = BitString::zeros(len); - let v = bits.as_bit_str(); - assert_eq!(v.count_ones(), 0, "len={len}"); - assert_eq!(v.count_zeros(), len, "len={len}"); - } -} - -/// Mixed bits from a string pattern. -#[test] -fn counts_mixed_bits_from_string() { - let bits = BitString::try_from("1010011100").unwrap(); - let v = bits.as_bit_str(); - - assert_eq!(v.count_ones(), 5); - assert_eq!(v.count_zeros(), 5); -} - -/// Unaligned subrange within a single word. -#[test] -fn unaligned_single_word() { - let bits = BitString::try_from("11110000").unwrap(); - // bits: 1 1 1 1 0 0 0 0 - - // View bits 1..7 → "111000" - let v = bits.as_bit_str().slice(UsizeCO::try_new(1, 7).unwrap()); - assert_eq!(v.count_ones(), 3); - assert_eq!(v.count_zeros(), 3); - - // View bits 2..6 → "1100" - let v = bits.as_bit_str().slice(UsizeCO::try_new(2, 6).unwrap()); - assert_eq!(v.count_ones(), 2); - assert_eq!(v.count_zeros(), 2); -} - -/// Invariant: count_ones + count_zeros == bit_len for every view. -#[test] -fn invariant_ones_plus_zeros_equals_bit_len() { - let mut bits = BitString::zeros(200); - for i in (0..200).step_by(7) { - bits.set(i, true); - } - let full = bits.as_bit_str(); - - for start in [0, 1, 5, 63, 64, 65, 127, 128] { - for len in [10, 63, 64, 65, 128, 129] { - let end = (start + len).min(full.bit_len()); - // Skip degenerate cases where start == end (empty interval not - // constructible via try_new). - if start == end { - continue; - } - let v = full.slice(UsizeCO::try_new(start, end).unwrap()); - assert_eq!( - v.count_ones() + v.count_zeros(), - v.bit_len(), - "start={start} end={end}" - ); - } - } -} - -/// Unaligned view spanning many words exercises the SIMD middle-word path. -#[test] -fn unaligned_many_words() { - let mut bits = BitString::zeros(300); - // Set bits at known positions across multiple words. - for i in [1, 63, 64, 100, 127, 128, 129, 200, 255, 256, 299] { - bits.set(i, true); - } - - // View from bit 1 (unaligned) covering bits 1..300 → 299 bits length. - // Bit 1 is included; all 11 bits are in range [1, 300). - let v = bits.as_bit_str().slice(UsizeCO::try_new(1, 300).unwrap()); - assert_eq!(v.count_ones(), 11); - assert_eq!(v.count_zeros(), 288); -} - -/// Word-aligned view exactly one word long. -#[test] -fn aligned_one_word() { - let mut bits = BitString::zeros(128); - bits.set(0, true); - bits.set(63, true); - - // View word 0 (bits 0..64) - let v = bits.as_bit_str().slice(UsizeCO::try_new(0, 64).unwrap()); - assert_eq!(v.count_ones(), 2); - assert_eq!(v.count_zeros(), 62); - - // View word 1 (bits 64..128) - let v = bits.as_bit_str().slice(UsizeCO::try_new(64, 128).unwrap()); - assert_eq!(v.count_ones(), 0); - assert_eq!(v.count_zeros(), 64); -} +//! Tests for count_ones on BitStr. diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros.rs b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros.rs index c8c0737..278812f 100644 --- a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros.rs +++ b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros.rs @@ -1,273 +1,28 @@ -use crate::traits::*; -use crate::{WORD_BITS, low_mask}; - use crate::BitStr; +use crate::{FILL_ONES, FILL_ZEROS, WORD_BITS}; -// --------------------------------------------------------------------------- -// Shared helper — parameterised by `fill` (0 → zeros, !0 → ones) -// --------------------------------------------------------------------------- - -/// Counts consecutive bits equal to `fill` from the start of a view. -/// -/// `fill` must be `0` (for `leading_zeros`) or `!0` (for `leading_ones`). -#[inline] -fn leading_value_count(words: &[u64], start: usize, bit_len: usize, fill: u64) -> usize { - let end = start + bit_len; - let start_offset = start % WORD_BITS; - let end_rem = end % WORD_BITS; - let last_wi = (end - 1) / WORD_BITS; - - let mut scanned = 0usize; - let mut wi = start / WORD_BITS; - - // First word — only bits from start_offset upward are in view. - let first_val = words[wi] >> start_offset; - let first_limit = (WORD_BITS - start_offset).min(bit_len); - let first_count = count_trailing(first_val, fill).min(first_limit); - if first_count < first_limit { - return first_count; - } - scanned += first_limit; - wi += 1; - - // Full middle words — SIMD-accelerated value-word scan. - let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; - if wi < mid_end { - let value_words = if fill == 0 { - words[wi..mid_end].leading_zero_words() - } else { - words[wi..mid_end].leading_one_words() - }; - scanned += value_words * WORD_BITS; - wi += value_words; - - if wi < mid_end { - return scanned + count_trailing(words[wi], fill).min(WORD_BITS); - } - } - - // Last partial word (only when end_rem != 0). - if end_rem != 0 && wi == last_wi { - let last_val = words[wi] & low_mask(end_rem); - scanned += count_trailing(last_val, fill).min(end_rem); - } - - scanned.min(bit_len) -} - -/// Counts trailing bits of a given value within a single u64 word. -/// -/// `fill = 0` → `trailing_zeros`, `fill = !0` → `trailing_ones`. -#[inline] -fn count_trailing(val: u64, fill: u64) -> usize { - if fill == 0 { - val.trailing_zeros() as usize - } else { - (!val).trailing_zeros() as usize - } -} +mod inner; impl<'bs> BitStr<'bs> { - /// Returns the number of consecutive `false` bits from the start of this - /// view. - /// - /// # Complexity - /// - /// O(n / 64) worst case, SIMD-accelerated for long runs. - /// Returns early at the first bit that differs from the expected value. - /// - /// # Examples - /// - /// ``` - /// use bit_string::BitString; - /// - /// let bits = BitString::try_from("00101").unwrap(); - /// let v = bits.as_bit_str(); - /// assert_eq!(v.leading_zeros(), 2); - /// ``` #[inline] pub fn leading_zeros(&self) -> usize { - if self.bit_len == 0 { - return 0; + if self.start % WORD_BITS == 0 { + self.leading_value_bits_inner::() + } else { + self.leading_value_bits_inner::() } - leading_value_count(self.source.words(), self.start, self.bit_len, 0) } - - /// Returns the number of consecutive `true` bits from the start of this - /// view. - /// - /// # Complexity - /// - /// O(n / 64) worst case, SIMD-accelerated for long runs. - /// Returns early at the first bit that differs from the expected value. - /// - /// # Examples - /// - /// ``` - /// use bit_string::BitString; - /// - /// let bits = BitString::try_from("11010").unwrap(); - /// let v = bits.as_bit_str(); - /// assert_eq!(v.leading_ones(), 2); - /// ``` #[inline] pub fn leading_ones(&self) -> usize { - if self.bit_len == 0 { - return 0; - } - leading_value_count(self.source.words(), self.start, self.bit_len, !0) - } - - /// Returns the number of consecutive `false` bits from the **end** of this - /// view. - /// - /// # Examples - /// - /// ``` - /// use bit_string::BitString; - /// - /// let bits = BitString::try_from("10000").unwrap(); - /// let v = bits.as_bit_str(); - /// assert_eq!(v.trailing_zeros(), 4); - /// ``` - #[inline] - pub fn trailing_zeros(&self) -> usize { - if self.bit_len == 0 { - return 0; - } - trailing_value_count(self.source.words(), self.start, self.bit_len, 0) - } - - /// Returns the number of consecutive `true` bits from the **end** of this - /// view. - /// - /// # Examples - /// - /// ``` - /// use bit_string::BitString; - /// - /// let bits = BitString::try_from("01111").unwrap(); - /// let v = bits.as_bit_str(); - /// assert_eq!(v.trailing_ones(), 4); - /// ``` - #[inline] - pub fn trailing_ones(&self) -> usize { - if self.bit_len == 0 { - return 0; - } - trailing_value_count(self.source.words(), self.start, self.bit_len, !0) - } -} - -// --------------------------------------------------------------------------- -// Trailing helper — scans from the end backwards -// --------------------------------------------------------------------------- - -/// Counts consecutive bits equal to `fill` from the end of a view. -#[inline] -fn trailing_value_count(words: &[u64], start: usize, bit_len: usize, fill: u64) -> usize { - let end = start + bit_len; - let start_offset = start % WORD_BITS; - let end_rem = end % WORD_BITS; - let last_wi = (end - 1) / WORD_BITS; - let start_wi = start / WORD_BITS; - - let mut scanned = 0usize; - - // Last word (partial, if end_rem != 0). - if end_rem != 0 { - let last_limit = if last_wi == start_wi { - end_rem - start_offset - } else { - end_rem - }; - let last_val = if last_wi == start_wi { - words[last_wi] >> start_offset + if self.start % WORD_BITS == 0 { + self.leading_value_bits_inner::() } else { - words[last_wi] & low_mask(end_rem) - }; - let last_count = count_leading_within(last_val, last_limit, fill); - if last_count < last_limit { - return last_count; - } - scanned += last_limit; - - // Entire view was within a single partial word. - if last_wi == start_wi { - return scanned.min(bit_len); + self.leading_value_bits_inner::() } } - - // Full middle words — SIMD-accelerated, from right to left. - let wi_end = if end_rem != 0 { last_wi - 1 } else { last_wi }; - let mid_first = if start_offset > 0 { - start_wi + 1 - } else { - start_wi - }; - if wi_end >= mid_first { - let nr_words = wi_end + 1 - mid_first; - let trailing_w = if fill == 0 { - words[mid_first..=wi_end].trailing_zero_words() - } else { - words[mid_first..=wi_end].trailing_one_words() - }; - if trailing_w < nr_words { - let hit_wi = wi_end - trailing_w; - scanned += trailing_w * WORD_BITS; - return scanned + count_leading(words[hit_wi], fill).min(WORD_BITS); - } - scanned += trailing_w * WORD_BITS; - } - - // First word (partial, if start_offset != 0 and not already handled). - if start_offset > 0 { - let first_limit = WORD_BITS - start_offset; - let first_val = words[start_wi] >> start_offset; - let first_count = count_leading_within(first_val, first_limit, fill); - scanned += first_count.min(first_limit); - } - - scanned.min(bit_len) -} - -/// Counts leading bits of a given value within a full u64 word. -/// -/// `fill = 0` → `leading_zeros`, `fill = !0` → `leading_ones`. -#[inline] -fn count_leading(val: u64, fill: u64) -> usize { - if fill == 0 { - val.leading_zeros() as usize - } else { - (!val).leading_zeros() as usize - } -} - -/// Counts leading bits of a value within its highest `limit` bits. -/// -/// Shifts valid bits to the top of the u64 so that [`u64::leading_zeros`] -/// counts from the highest valid bit downwards. -#[inline] -fn count_leading_within(val: u64, limit: usize, fill: u64) -> usize { - if limit == 0 { - return 0; - } - let shifted = val << (WORD_BITS - limit); - if fill == 0 { - (shifted.leading_zeros() as usize).min(limit) - } else { - ((!shifted).leading_zeros() as usize).min(limit) - } } -#[cfg(test)] -mod tests_for_leading_zeros; - #[cfg(test)] mod tests_for_leading_ones; - #[cfg(test)] -mod tests_for_trailing_zeros; - -#[cfg(test)] -mod tests_for_trailing_ones; +mod tests_for_leading_zeros; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/inner.rs b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/inner.rs new file mode 100644 index 0000000..cfd45e0 --- /dev/null +++ b/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/inner.rs @@ -0,0 +1,55 @@ +use crate::BitStr; +use crate::WORD_BITS; +use crate::traits::WordsScan; + +impl<'bs> BitStr<'bs> { + #[inline] + pub(crate) fn leading_value_bits_inner( + &self, + ) -> usize { + if self.bit_len == 0 { + return 0; + } + let all_words = self.source.words(); + let word_start = self.start / WORD_BITS; + // SAFETY: BitStr invariants guarantee `start` and `bit_len` are + // within the source BitString's bounds; `word_start` is ≤ all_words.len(). + let words_ptr = unsafe { all_words.as_ptr().add(word_start) }; + let start_offset = (self.start % WORD_BITS) as u32; + // SAFETY: `words_ptr` is valid for at least 1 u64 read + // (bit_len > 0 ⇒ at least one word exists). + let w0 = unsafe { *words_ptr }; + + // First-word fast path — catch early non-FILL before paying + // the trait / slice-construction overhead. Mirrors the + // shortcut in BitString's impls_for_leading_zeros.rs. + if !WORD_ALIGNED && start_offset != 0 { + let first_val = w0 >> start_offset; + let first_limit = (WORD_BITS - start_offset as usize).min(self.bit_len); + let first_count = if FILL == 0 { + first_val.trailing_zeros() as usize + } else { + (!first_val).trailing_zeros() as usize + }; + if first_count < first_limit { + return first_count; + } + } else if w0 != FILL { + let count = if FILL == 0 { + w0.trailing_zeros() as usize + } else { + (!w0).trailing_zeros() as usize + }; + return count.min(self.bit_len); + } + + // Reconstruct slice for the trait call. + // SAFETY: `words_ptr` is derived from `all_words.as_ptr()` with + // `word_start ≤ all_words.len()` (guaranteed by BitStr invariant). + // The sub-slice `all_words.len() - word_start` is valid and + // stays within the allocation. This slice is only used for the + // duration of the trait method call below. + let words = unsafe { core::slice::from_raw_parts(words_ptr, all_words.len() - word_start) }; + words.leading_value_bits::(start_offset, self.bit_len) + } +} diff --git a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros.rs new file mode 100644 index 0000000..79cf707 --- /dev/null +++ b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros.rs @@ -0,0 +1,28 @@ +use crate::BitStr; +use crate::{FILL_ONES, FILL_ZEROS, WORD_BITS}; + +mod inner; + +impl<'bs> BitStr<'bs> { + #[inline] + pub fn trailing_zeros(&self) -> usize { + if self.start % WORD_BITS == 0 { + self.trailing_value_bits_inner::() + } else { + self.trailing_value_bits_inner::() + } + } + #[inline] + pub fn trailing_ones(&self) -> usize { + if self.start % WORD_BITS == 0 { + self.trailing_value_bits_inner::() + } else { + self.trailing_value_bits_inner::() + } + } +} + +#[cfg(test)] +mod tests_for_trailing_ones; +#[cfg(test)] +mod tests_for_trailing_zeros; diff --git a/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/inner.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/inner.rs new file mode 100644 index 0000000..953ceb9 --- /dev/null +++ b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/inner.rs @@ -0,0 +1,79 @@ +use crate::BitStr; +use crate::WORD_BITS; +use crate::traits::WordsScan; + +impl<'bs> BitStr<'bs> { + #[inline] + pub(crate) fn trailing_value_bits_inner( + &self, + ) -> usize { + if self.bit_len == 0 { + return 0; + } + let all_words = self.source.words(); + let word_start = self.start / WORD_BITS; + // SAFETY: BitStr invariants guarantee `start` and `bit_len` are + // within the source BitString's bounds. + let words_ptr = unsafe { all_words.as_ptr().add(word_start) }; + let start_offset = (self.start % WORD_BITS) as u32; + + // Fast-path: check the rightmost word(s) before the trait call. + // Mirrors the shortcuts in BitString's trailing section. + + let end_offset = start_offset as usize + self.bit_len; + let end_rem = end_offset % WORD_BITS; + let last_wi = (end_offset - 1) / WORD_BITS; + + // ── Last partial word ──────────────────────────────────── + if end_rem != 0 { + let last_limit = if last_wi == 0 { + end_rem - start_offset as usize + } else { + end_rem + }; + // SAFETY: `last_wi` is within bounds per the BitStr invariant. + let raw_last = unsafe { *words_ptr.add(last_wi) }; + let last_val = if last_wi == 0 { + raw_last >> start_offset + } else { + raw_last & ((1u64 << end_rem).wrapping_sub(1)) + }; + let shifted = if FILL == 0 { + last_val << (WORD_BITS - last_limit) + } else { + (!last_val) << (WORD_BITS - last_limit) + }; + let last_count = (shifted.leading_zeros() as usize).min(last_limit); + if last_count < last_limit { + return last_count; + } + if last_wi == 0 { + return self.bit_len; + } + } + + // ── Rightmost full word ────────────────────────────────── + if self.bit_len > WORD_BITS { + let last_full = if end_rem != 0 { last_wi - 1 } else { last_wi }; + // SAFETY: `last_full` is within bounds. + let w = unsafe { *words_ptr.add(last_full) }; + if w != FILL { + let tail = if end_rem != 0 { end_rem } else { 0 }; + let count = if FILL == 0 { + w.leading_zeros() as usize + } else { + (!w).leading_zeros() as usize + }; + return (tail + count).min(self.bit_len); + } + } + + // SAFETY: `words_ptr` is derived from `all_words.as_ptr()` with + // `word_start ≤ all_words.len()` (guaranteed by BitStr invariant). + // The sub-slice `all_words.len() - word_start` stays within the + // allocation and is only used for the duration of the trait + // method call below. + let words = unsafe { core::slice::from_raw_parts(words_ptr, all_words.len() - word_start) }; + words.trailing_value_bits::(start_offset, self.bit_len) + } +} diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_trailing_ones.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs similarity index 100% rename from src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_trailing_ones.rs rename to src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_ones.rs diff --git a/src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_trailing_zeros.rs b/src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs similarity index 100% rename from src/bit_str/impls_for_bit_arith/impls_for_leading_zeros/tests_for_trailing_zeros.rs rename to src/bit_str/impls_for_bit_arith/impls_for_trailing_zeros/tests_for_trailing_zeros.rs diff --git a/src/bit_str/impls_for_eq.rs b/src/bit_str/impls_for_eq.rs index 4236ac0..0e66d59 100644 --- a/src/bit_str/impls_for_eq.rs +++ b/src/bit_str/impls_for_eq.rs @@ -1,4 +1,4 @@ -use crate::BitStr; +use crate::{BitStr, WORD_BITS}; impl PartialEq for BitStr<'_> { fn eq(&self, other: &Self) -> bool { @@ -8,7 +8,14 @@ impl PartialEq for BitStr<'_> { if self.bit_len == 0 { return true; } - self.bits_equal_at(0, *other) + let hs_aligned = self.start % WORD_BITS == 0; + let nd_aligned = other.start % WORD_BITS == 0; + match (hs_aligned, nd_aligned) { + (true, true) => self.bits_equal_at_inner::(0, *other), + (true, false) => self.bits_equal_at_inner::(0, *other), + (false, true) => self.bits_equal_at_inner::(0, *other), + (false, false) => self.bits_equal_at_inner::(0, *other), + } } } diff --git a/src/bit_str/impls_for_hash.rs b/src/bit_str/impls_for_hash.rs index 8be3296..2f01819 100644 --- a/src/bit_str/impls_for_hash.rs +++ b/src/bit_str/impls_for_hash.rs @@ -1,41 +1,17 @@ use core::hash::{Hash, Hasher}; -use crate::traits::*; -use crate::{WORD_BITS, low_mask}; +use crate::WORD_BITS; use crate::BitStr; +mod inner; + impl Hash for BitStr<'_> { fn hash(&self, state: &mut H) { - self.bit_len.hash(state); - if self.bit_len == 0 { - return; - } - - let words = self.source.words(); - let full_words = self.bit_len / WORD_BITS; - let rem = self.bit_len % WORD_BITS; - if self.start % WORD_BITS == 0 { - // Aligned: hash full words element-by-element (not as a [u64] - // slice, which would inject a length prefix that the unaligned - // path doesn't emit). - let sw = self.start / WORD_BITS; - for w in &words[sw..][..full_words] { - w.hash(state); - } + self.hash_inner::(state) } else { - // Unaligned: each view-word straddles a source-word boundary, - // so we must go through read_word_at word by word. - for i in 0..full_words { - let w = words.read_word_at(self.start + i * WORD_BITS); - w.hash(state); - } - } - - if rem > 0 { - let tail_start = self.start + full_words * WORD_BITS; - (words.read_word_at(tail_start) & low_mask(rem)).hash(state); + self.hash_inner::(state) } } } diff --git a/src/bit_str/impls_for_hash/inner.rs b/src/bit_str/impls_for_hash/inner.rs new file mode 100644 index 0000000..7ef9ded --- /dev/null +++ b/src/bit_str/impls_for_hash/inner.rs @@ -0,0 +1,34 @@ +use crate::BitStr; +use crate::traits::WordsEdit; +use crate::{WORD_BITS, low_mask}; +use core::hash::{Hash, Hasher}; + +impl<'bs> BitStr<'bs> { + /// Hash with compile-time alignment signal. + #[inline] + pub(crate) fn hash_inner(&self, state: &mut H) { + self.bit_len.hash(state); + if self.bit_len == 0 { + return; + } + let words = self.source.words(); + let full_words = self.bit_len / WORD_BITS; + let rem = self.bit_len % WORD_BITS; + if WORD_ALIGNED || self.start % WORD_BITS == 0 { + let sw = self.start / WORD_BITS; + for w in &words[sw..][..full_words] { + w.hash(state); + } + } else { + for i in 0..full_words { + words + .read_word_at::(self.start + i * WORD_BITS) + .hash(state); + } + } + if rem > 0 { + let tail_start = self.start + full_words * WORD_BITS; + (words.read_word_at::(tail_start) & low_mask(rem)).hash(state); + } + } +} diff --git a/src/bit_str/impls_for_matching/impls_for_find.rs b/src/bit_str/impls_for_matching/impls_for_find.rs index a38207c..d0dcc64 100644 --- a/src/bit_str/impls_for_matching/impls_for_find.rs +++ b/src/bit_str/impls_for_matching/impls_for_find.rs @@ -1,180 +1,68 @@ -use crate::traits::*; -use crate::{SMALL_WORDS, WORD_BITS}; - use crate::BitStr; +use crate::WORD_BITS; + +mod inner; impl<'bs> BitStr<'bs> { - /// Returns `true` if `needle` is contained within `self`. #[inline] - pub fn contains(&self, needle: BitStr<'_>) -> bool { - if needle.bit_len == 0 { - return true; - } - if needle.bit_len > self.bit_len { - return false; + pub fn contains_str(&self, needle: BitStr<'_>) -> bool { + let hs_aligned = self.start % WORD_BITS == 0; + let nd_aligned = needle.start % WORD_BITS == 0; + match (hs_aligned, nd_aligned) { + (true, true) => self.contains_inner::(needle), + (true, false) => self.contains_inner::(needle), + (false, true) => self.contains_inner::(needle), + (false, false) => self.contains_inner::(needle), } - - let words = self.source.words(); - let sw = self.start / WORD_BITS; - let so = self.start % WORD_BITS; - let needle_words = needle.source.words(); - let needle_len = needle.bit_len; - - // Unaligned start: check the first partial word before delegating to - // SIMD on the aligned remainder. - if so != 0 { - let first_bits = (WORD_BITS - so).min(self.bit_len); - let max = first_bits.min(self.bit_len.saturating_sub(needle_len)); - for p in 0..=max { - if self.bits_equal_at(p, needle) { - return true; - } - } - let remaining = self.bit_len - first_bits; - if remaining == 0 { - return false; - } - let aligned = &words[sw + 1..]; - return aligned - .find_any_candidate(remaining, needle_words, needle_len, &mut |pos| { - self.bits_equal_at(pos + first_bits, needle) - }) - .is_some(); - } - - // Word-aligned: full SIMD on the relevant suffix. - words[sw..] - .find_any_candidate(self.bit_len, needle_words, needle_len, &mut |pos| { - self.bits_equal_at(pos, needle) - }) - .is_some() } - - /// Returns the index of the first occurrence of `needle`, or `None`. #[inline] - pub fn find(&self, needle: BitStr<'_>) -> Option { - if needle.bit_len == 0 { - return Some(0); - } - if needle.bit_len > self.bit_len { - return None; - } - - let words = self.source.words(); - let sw = self.start / WORD_BITS; - let so = self.start % WORD_BITS; - let needle_words = needle.source.words(); - let needle_len = needle.bit_len; - - // Word-aligned fast path. - if so == 0 { - return words[sw..].find_first_word( - self.bit_len, - needle_words, - needle_len, - &mut |pos| self.bits_equal_at(pos, needle), - ); - } - - // Unaligned: scan the first partial word, then SIMD for the rest. - let first_bits = (WORD_BITS - so).min(self.bit_len); - let max = first_bits.min(self.bit_len.saturating_sub(needle_len)); - for p in 0..=max { - if self.bits_equal_at(p, needle) { - return Some(p); - } - } - - let remaining = self.bit_len - first_bits; - if remaining == 0 { - return None; - } - - let aligned = &words[sw + 1..]; - - // Quick rejection before the more expensive word-outer scan. - if aligned.len() >= SMALL_WORDS - && !aligned - .find_any_candidate(remaining, needle_words, needle_len, &mut |pos| { - self.bits_equal_at(pos + first_bits, needle) - }) - .is_some() - { - return None; + pub fn contains_string(&self, needle: &crate::BitString) -> bool { + let n = needle.as_bit_str(); + if self.start % WORD_BITS == 0 { + self.contains_inner::(n) + } else { + self.contains_inner::(n) } - - aligned - .find_first_word(remaining, needle_words, needle_len, &mut |pos| { - self.bits_equal_at(pos + first_bits, needle) - }) - .map(|pos| pos + first_bits) } - - /// Returns the index of the last occurrence of `needle`, or `None`. #[inline] - pub fn rfind(&self, needle: BitStr<'_>) -> Option { - if needle.bit_len == 0 { - return Some(self.bit_len); + pub fn find_str(&self, needle: BitStr<'_>) -> Option { + let hs_a = self.start % WORD_BITS == 0; + let nd_a = needle.start % WORD_BITS == 0; + match (hs_a, nd_a) { + (true, true) => self.find_inner::(needle), + (true, false) => self.find_inner::(needle), + (false, true) => self.find_inner::(needle), + (false, false) => self.find_inner::(needle), } - if needle.bit_len > self.bit_len { - return None; - } - - let words = self.source.words(); - let sw = self.start / WORD_BITS; - let so = self.start % WORD_BITS; - let needle_words = needle.source.words(); - let needle_len = needle.bit_len; - - // Word-aligned fast path. - if so == 0 { - return words[sw..].find_last_word( - self.bit_len, - needle_words, - needle_len, - &mut |pos| self.bits_equal_at(pos, needle), - ); + } + #[inline] + pub fn find_string(&self, needle: &crate::BitString) -> Option { + let n = needle.as_bit_str(); + if self.start % WORD_BITS == 0 { + self.find_inner::(n) + } else { + self.find_inner::(n) } - - // Unaligned: SIMD on the aligned remainder first (reverse), - // then fall back to the first partial word. - let first_bits = (WORD_BITS - so).min(self.bit_len); - let remaining = self.bit_len - first_bits; - - if remaining > 0 { - let aligned = &words[sw + 1..]; - - // Quick rejection via SIMD candidate scan — only when the - // aligned portion is large enough to amortize setup cost. - // When it's too small we skip the gate and scan directly - // (find_last_word has its own scalar fallback). - let maybe_candidate = aligned.len() < SMALL_WORDS - || aligned - .find_any_candidate(remaining, needle_words, needle_len, &mut |pos| { - self.bits_equal_at(pos + first_bits, needle) - }) - .is_some(); - - if maybe_candidate { - if let Some(pos) = - aligned.find_last_word(remaining, needle_words, needle_len, &mut |pos| { - self.bits_equal_at(pos + first_bits, needle) - }) - { - return Some(pos + first_bits); - } - } + } + #[inline] + pub fn rfind_str(&self, needle: BitStr<'_>) -> Option { + let hs_a = self.start % WORD_BITS == 0; + let nd_a = needle.start % WORD_BITS == 0; + match (hs_a, nd_a) { + (true, true) => self.rfind_inner::(needle), + (true, false) => self.rfind_inner::(needle), + (false, true) => self.rfind_inner::(needle), + (false, false) => self.rfind_inner::(needle), } - - // Check the first partial word. - let max = first_bits.min(self.bit_len.saturating_sub(needle_len)); - for p in (0..=max).rev() { - if self.bits_equal_at(p, needle) { - return Some(p); - } + } + #[inline] + pub fn rfind_string(&self, needle: &crate::BitString) -> Option { + let n = needle.as_bit_str(); + if self.start % WORD_BITS == 0 { + self.rfind_inner::(n) + } else { + self.rfind_inner::(n) } - - None } } diff --git a/src/bit_str/impls_for_matching/impls_for_find/inner.rs b/src/bit_str/impls_for_matching/impls_for_find/inner.rs new file mode 100644 index 0000000..9f12285 --- /dev/null +++ b/src/bit_str/impls_for_matching/impls_for_find/inner.rs @@ -0,0 +1,152 @@ +use crate::BitStr; +use crate::traits::*; +use crate::{SMALL_WORDS, WORD_BITS}; + +impl<'bs> BitStr<'bs> { + #[inline] + pub(crate) fn find_inner( + &self, + needle: BitStr<'_>, + ) -> Option { + if needle.bit_len == 0 { + return Some(0); + } + if needle.bit_len > self.bit_len { + return None; + } + let words = self.source.words(); + let sw = self.start / WORD_BITS; + let so = self.start % WORD_BITS; + let needle_words = needle.source.words(); + let needle_len = needle.bit_len; + if WORD_ALIGNED || so == 0 { + return words[sw..].find_first_word( + self.bit_len, + needle_words, + needle_len, + &mut |pos| self.bits_equal_at_inner::(pos, needle), + ); + } + let first_bits = (WORD_BITS - so).min(self.bit_len); + let max = first_bits.min(self.bit_len.saturating_sub(needle_len)); + for p in 0..=max { + if self.bits_equal_at_inner::(p, needle) { + return Some(p); + } + } + let remaining = self.bit_len - first_bits; + if remaining == 0 { + return None; + } + let aligned = &words[sw + 1..]; + if aligned.len() >= SMALL_WORDS + && !aligned + .find_any_candidate(remaining, needle_words, needle_len, &mut |pos| { + self.bits_equal_at_inner::(pos + first_bits, needle) + }) + .is_some() + { + return None; + } + aligned + .find_first_word(remaining, needle_words, needle_len, &mut |pos| { + self.bits_equal_at_inner::(pos + first_bits, needle) + }) + .map(|pos| pos + first_bits) + } + + #[inline] + pub(crate) fn rfind_inner( + &self, + needle: BitStr<'_>, + ) -> Option { + if needle.bit_len == 0 { + return Some(self.bit_len); + } + if needle.bit_len > self.bit_len { + return None; + } + let words = self.source.words(); + let sw = self.start / WORD_BITS; + let so = self.start % WORD_BITS; + let needle_words = needle.source.words(); + let needle_len = needle.bit_len; + if WORD_ALIGNED || so == 0 { + return words[sw..].find_last_word( + self.bit_len, + needle_words, + needle_len, + &mut |pos| self.bits_equal_at_inner::(pos, needle), + ); + } + let first_bits = (WORD_BITS - so).min(self.bit_len); + let remaining = self.bit_len - first_bits; + if remaining > 0 { + let aligned = &words[sw + 1..]; + let maybe_candidate = aligned.len() < SMALL_WORDS + || aligned + .find_any_candidate(remaining, needle_words, needle_len, &mut |pos| { + self.bits_equal_at_inner::(pos + first_bits, needle) + }) + .is_some(); + if maybe_candidate { + if let Some(pos) = + aligned.find_last_word(remaining, needle_words, needle_len, &mut |pos| { + self.bits_equal_at_inner::(pos + first_bits, needle) + }) + { + return Some(pos + first_bits); + } + } + } + let max = first_bits.min(self.bit_len.saturating_sub(needle_len)); + for p in (0..=max).rev() { + if self.bits_equal_at_inner::(p, needle) { + return Some(p); + } + } + None + } + + #[inline] + pub(crate) fn contains_inner( + &self, + needle: BitStr<'_>, + ) -> bool { + if needle.bit_len == 0 { + return true; + } + if needle.bit_len > self.bit_len { + return false; + } + let words = self.source.words(); + let sw = self.start / WORD_BITS; + let so = self.start % WORD_BITS; + let needle_words = needle.source.words(); + let needle_len = needle.bit_len; + if !HS_WORD_ALIGNED && so != 0 { + let first_bits = (WORD_BITS - so).min(self.bit_len); + let max = first_bits.min(self.bit_len.saturating_sub(needle_len)); + for p in 0..=max { + if self.bits_equal_at_inner::(p, needle) { + return true; + } + } + let remaining = self.bit_len - first_bits; + if remaining == 0 { + return false; + } + let aligned = &words[sw + 1..]; + return aligned + .find_any_candidate(remaining, needle_words, needle_len, &mut |pos| { + self.bits_equal_at_inner::(pos + first_bits, needle) + }) + .is_some(); + } + words[sw..] + .find_any_candidate(self.bit_len, needle_words, needle_len, &mut |pos| { + self.bits_equal_at_inner::(pos, needle) + }) + .is_some() + } +} diff --git a/src/bit_str/impls_for_matching/impls_for_find/tests_for_find.rs b/src/bit_str/impls_for_matching/impls_for_find/tests_for_find.rs index 7611d64..d24888a 100644 --- a/src/bit_str/impls_for_matching/impls_for_find/tests_for_find.rs +++ b/src/bit_str/impls_for_matching/impls_for_find/tests_for_find.rs @@ -9,9 +9,9 @@ fn contains_basic() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("01").unwrap(); - assert!(v.contains(bs.as_bit_str())); + assert!(v.contains_str(bs.as_bit_str())); let bs = BitString::try_from("111").unwrap(); - assert!(!v.contains(bs.as_bit_str())); + assert!(!v.contains_str(bs.as_bit_str())); } #[test] @@ -19,7 +19,7 @@ fn contains_empty_needle() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let empty = BitString::new(); - assert!(v.contains(empty.as_bit_str())); + assert!(v.contains_str(empty.as_bit_str())); } #[test] @@ -27,7 +27,7 @@ fn contains_longer_than_view_returns_false() { let bits = BitString::try_from("10").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("101").unwrap(); - assert!(!v.contains(bs.as_bit_str())); + assert!(!v.contains_str(bs.as_bit_str())); } #[test] @@ -35,9 +35,9 @@ fn contains_on_offset_view() { let bits = BitString::try_from("110010").unwrap(); let v = bits.as_bit_str().slice_from(1).slice_until(5); let bs = BitString::try_from("001").unwrap(); - assert!(v.contains(bs.as_bit_str())); + assert!(v.contains_str(bs.as_bit_str())); let bs = BitString::try_from("11").unwrap(); - assert!(!v.contains(bs.as_bit_str())); + assert!(!v.contains_str(bs.as_bit_str())); } // --------------------------------------------------------------------------- @@ -49,11 +49,11 @@ fn find_first_occurrence() { let bits = BitString::try_from("10110010").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("10").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), Some(0)); + assert_eq!(v.find_str(bs.as_bit_str()), Some(0)); let bs = BitString::try_from("01").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), Some(1)); + assert_eq!(v.find_str(bs.as_bit_str()), Some(1)); let bs = BitString::try_from("111").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), None); + assert_eq!(v.find_str(bs.as_bit_str()), None); } #[test] @@ -61,7 +61,7 @@ fn find_empty_needle_returns_zero() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let empty = BitString::new(); - assert_eq!(v.find(empty.as_bit_str()), Some(0)); + assert_eq!(v.find_str(empty.as_bit_str()), Some(0)); } #[test] @@ -69,7 +69,7 @@ fn find_longer_than_view_returns_none() { let bits = BitString::try_from("10").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("101").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), None); + assert_eq!(v.find_str(bs.as_bit_str()), None); } #[test] @@ -77,9 +77,9 @@ fn find_on_offset_view() { let bits = BitString::try_from("11001001").unwrap(); let v = bits.as_bit_str().slice_from(2).slice_until(7); let bs = BitString::try_from("10").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), Some(2)); + assert_eq!(v.find_str(bs.as_bit_str()), Some(2)); let bs = BitString::try_from("00").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), Some(0)); + assert_eq!(v.find_str(bs.as_bit_str()), Some(0)); } #[test] @@ -87,7 +87,7 @@ fn find_at_end_of_view() { let bits = BitString::try_from("11100").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("00").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), Some(3)); + assert_eq!(v.find_str(bs.as_bit_str()), Some(3)); } // --------------------------------------------------------------------------- @@ -99,7 +99,7 @@ fn rfind_last_occurrence() { let bits = BitString::try_from("10110010").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("10").unwrap(); - assert_eq!(v.rfind(bs.as_bit_str()), Some(6)); + assert_eq!(v.rfind_str(bs.as_bit_str()), Some(6)); } #[test] @@ -107,7 +107,7 @@ fn rfind_empty_needle_returns_bit_len() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let empty = BitString::new(); - assert_eq!(v.rfind(empty.as_bit_str()), Some(5)); + assert_eq!(v.rfind_str(empty.as_bit_str()), Some(5)); } #[test] @@ -115,7 +115,7 @@ fn rfind_longer_than_view_returns_none() { let bits = BitString::try_from("10").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("101").unwrap(); - assert_eq!(v.rfind(bs.as_bit_str()), None); + assert_eq!(v.rfind_str(bs.as_bit_str()), None); } #[test] @@ -123,7 +123,7 @@ fn rfind_on_offset_view() { let bits = BitString::try_from("11001001").unwrap(); let v = bits.as_bit_str().slice_from(2).slice_until(7); let bs = BitString::try_from("00").unwrap(); - assert_eq!(v.rfind(bs.as_bit_str()), Some(3)); + assert_eq!(v.rfind_str(bs.as_bit_str()), Some(3)); } #[test] @@ -131,6 +131,6 @@ fn find_and_rfind_needle_appears_once() { let bits = BitString::try_from("11010").unwrap(); let v = bits.as_bit_str(); let bs = BitString::try_from("101").unwrap(); - assert_eq!(v.find(bs.as_bit_str()), Some(1)); - assert_eq!(v.rfind(bs.as_bit_str()), Some(1)); + assert_eq!(v.find_str(bs.as_bit_str()), Some(1)); + assert_eq!(v.rfind_str(bs.as_bit_str()), Some(1)); } diff --git a/src/bit_str/impls_for_matching/impls_for_matches_at.rs b/src/bit_str/impls_for_matching/impls_for_matches_at.rs index e378d11..1d1456a 100644 --- a/src/bit_str/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_str/impls_for_matching/impls_for_matches_at.rs @@ -1,119 +1,90 @@ -use crate::traits::*; -use crate::{WORD_BITS, low_mask}; - use crate::BitStr; +use crate::WORD_BITS; + +mod inner; impl<'bs> BitStr<'bs> { - /// Compare `needle` bits against `self` starting at `offset`. - /// - /// When `needle.start` is word-aligned, the fast path delegates to - /// SIMD word-equality via [`BitsEq::eq_words`]. Otherwise falls - /// back to a scalar word-at-a-time comparison. #[inline] - pub(crate) fn bits_equal_at(&self, offset: usize, needle: BitStr<'_>) -> bool { - let n = needle.bit_len; - if n == 0 { - return true; - } - - let hs_base = self.start + offset; - let nd_base = needle.start; - let hs_words = self.source.words(); - let nd_words = needle.source.words(); - - // Sub-word fast path — a single 64-bit read on each side. - if n <= WORD_BITS { - let mask = low_mask(n); - let h = hs_words.read_word_at(hs_base); - let nd = nd_words.read_word_at(nd_base); - return (h & mask) == (nd & mask); - } - - // Multi-word, needle word-aligned — SIMD eq_words on haystack. - if nd_base % WORD_BITS == 0 { - let full_words = n / WORD_BITS; - let nd_aligned = &nd_words[nd_base / WORD_BITS..]; - - if !hs_words.eq_words(nd_aligned, full_words, hs_base) { - return false; - } - - let rem = n % WORD_BITS; - if rem > 0 { - let mask = low_mask(rem); - let h = hs_words.read_word_at(hs_base + full_words * WORD_BITS); - if (h & mask) != (nd_aligned[full_words] & mask) { - return false; - } - } - - return true; - } - - // Both sides misaligned — scalar word-at-a-time (rare). - let full_words = n / WORD_BITS; - for i in 0..full_words { - let pos = i * WORD_BITS; - let h = hs_words.read_word_at(hs_base + pos); - let nd = nd_words.read_word_at(nd_base + pos); - if h != nd { - return false; - } + pub fn matches_at_str(&self, index: usize, pattern: BitStr<'_>) -> bool { + let hs_aligned = (self.start + index) % WORD_BITS == 0; + let nd_aligned = pattern.start % WORD_BITS == 0; + match (hs_aligned, nd_aligned) { + (true, true) => self.matches_at_inner::(index, pattern), + (true, false) => self.matches_at_inner::(index, pattern), + (false, true) => self.matches_at_inner::(index, pattern), + (false, false) => self.matches_at_inner::(index, pattern), } - - let rem = n % WORD_BITS; - if rem > 0 { - let mask = low_mask(rem); - let pos = full_words * WORD_BITS; - let h = hs_words.read_word_at(hs_base + pos); - let nd = nd_words.read_word_at(nd_base + pos); - if (h & mask) != (nd & mask) { - return false; - } - } - - true } - - /// Returns `true` if `pattern` matches the bits starting at `index`. #[inline] - pub fn matches_at(&self, index: usize, pattern: BitStr<'_>) -> bool { - if index > self.bit_len { - return false; + pub fn matches_at_string(&self, index: usize, pattern: &crate::BitString) -> bool { + let p = pattern.as_bit_str(); + if (self.start + index) % WORD_BITS == 0 { + self.matches_at_inner::(index, p) + } else { + self.matches_at_inner::(index, p) } - if pattern.bit_len > self.bit_len - index { - return false; + } + #[inline] + pub fn starts_with_str(&self, prefix: BitStr<'_>) -> bool { + let hs_aligned = self.start % WORD_BITS == 0; + let nd_aligned = prefix.start % WORD_BITS == 0; + match (hs_aligned, nd_aligned) { + (true, true) => self.starts_with_inner::(prefix), + (true, false) => self.starts_with_inner::(prefix), + (false, true) => self.starts_with_inner::(prefix), + (false, false) => self.starts_with_inner::(prefix), } - self.bits_equal_at(index, pattern) } - - /// Returns `true` if `prefix` is a prefix of `self`. #[inline] - pub fn starts_with(&self, prefix: BitStr<'_>) -> bool { - self.matches_at(0, prefix) + pub fn starts_with_string(&self, prefix: &crate::BitString) -> bool { + let p = prefix.as_bit_str(); + if self.start % WORD_BITS == 0 { + self.starts_with_inner::(p) + } else { + self.starts_with_inner::(p) + } } - - /// Returns `true` if `suffix` is a suffix of `self`. #[inline] - pub fn ends_with(&self, suffix: BitStr<'_>) -> bool { + pub fn ends_with_str(&self, suffix: BitStr<'_>) -> bool { if suffix.bit_len == 0 { return true; } if suffix.bit_len > self.bit_len { return false; } - self.bits_equal_at(self.bit_len - suffix.bit_len, suffix) + let offset = self.bit_len - suffix.bit_len; + let hs_aligned = (self.start + offset) % WORD_BITS == 0; + let nd_aligned = suffix.start % WORD_BITS == 0; + match (hs_aligned, nd_aligned) { + (true, true) => self.ends_with_inner::(suffix, offset), + (true, false) => self.ends_with_inner::(suffix, offset), + (false, true) => self.ends_with_inner::(suffix, offset), + (false, false) => self.ends_with_inner::(suffix, offset), + } + } + #[inline] + pub fn ends_with_string(&self, suffix: &crate::BitString) -> bool { + let s = suffix.as_bit_str(); + if s.bit_len == 0 { + return true; + } + if s.bit_len > self.bit_len { + return false; + } + let offset = self.bit_len - s.bit_len; + if (self.start + offset) % WORD_BITS == 0 { + self.ends_with_inner::(s, offset) + } else { + self.ends_with_inner::(s, offset) + } } } #[cfg(test)] mod tests_for_bits_equal_at; - #[cfg(test)] mod tests_for_ends_with; - #[cfg(test)] mod tests_for_matches_at; - #[cfg(test)] mod tests_for_starts_with; diff --git a/src/bit_str/impls_for_matching/impls_for_matches_at/inner.rs b/src/bit_str/impls_for_matching/impls_for_matches_at/inner.rs new file mode 100644 index 0000000..8ebc59a --- /dev/null +++ b/src/bit_str/impls_for_matching/impls_for_matches_at/inner.rs @@ -0,0 +1,105 @@ +use crate::BitStr; +use crate::traits::*; +use crate::{WORD_BITS, low_mask}; + +impl<'bs> BitStr<'bs> { + #[inline] + pub(crate) fn bits_equal_at_inner( + &self, + offset: usize, + needle: BitStr<'_>, + ) -> bool { + let n = needle.bit_len; + if n == 0 { + return true; + } + let hs_base = self.start + offset; + let nd_base = needle.start; + let hs_words = self.source.words(); + let nd_words = needle.source.words(); + if n <= WORD_BITS { + let mask = low_mask(n); + let h = hs_words.read_word_at::(hs_base); + let nd = nd_words.read_word_at::(nd_base); + return (h & mask) == (nd & mask); + } + let nd_is_aligned = ND_WORD_ALIGNED || nd_base % WORD_BITS == 0; + if nd_is_aligned { + let full_words = n / WORD_BITS; + let nd_aligned = &nd_words[nd_base / WORD_BITS..]; + let haystack = &hs_words[hs_base / WORD_BITS..]; + let ok = if HS_WORD_ALIGNED { + haystack.eq_words::(nd_aligned, full_words, 0) + } else { + haystack.eq_words::(nd_aligned, full_words, hs_base % WORD_BITS) + }; + if !ok { + return false; + } + let rem = n % WORD_BITS; + if rem > 0 { + let mask = low_mask(rem); + let h = hs_words.read_word_at::(hs_base + full_words * WORD_BITS); + if (h & mask) != (nd_aligned[full_words] & mask) { + return false; + } + } + return true; + } + let full_words = n / WORD_BITS; + for i in 0..full_words { + let pos = i * WORD_BITS; + let h = hs_words.read_word_at::(hs_base + pos); + let nd = nd_words.read_word_at::(nd_base + pos); + if h != nd { + return false; + } + } + let rem = n % WORD_BITS; + if rem > 0 { + let mask = low_mask(rem); + let pos = full_words * WORD_BITS; + let h = hs_words.read_word_at::(hs_base + pos); + let nd = nd_words.read_word_at::(nd_base + pos); + if (h & mask) != (nd & mask) { + return false; + } + } + true + } + + #[inline] + pub(crate) fn starts_with_inner( + &self, + prefix: BitStr<'_>, + ) -> bool { + if prefix.bit_len > self.bit_len { + return false; + } + self.bits_equal_at_inner::(0, prefix) + } + + #[inline] + pub(crate) fn ends_with_inner( + &self, + suffix: BitStr<'_>, + offset: usize, + ) -> bool { + self.bits_equal_at_inner::(offset, suffix) + } + + #[inline] + pub(crate) fn matches_at_inner( + &self, + index: usize, + pattern: BitStr<'_>, + ) -> bool { + if index > self.bit_len { + return false; + } + if pattern.bit_len > self.bit_len - index { + return false; + } + self.bits_equal_at_inner::(index, pattern) + } +} diff --git a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_bits_equal_at.rs b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_bits_equal_at.rs index 3957159..38cd431 100644 --- a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_bits_equal_at.rs +++ b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_bits_equal_at.rs @@ -6,8 +6,8 @@ fn returns_true_when_needle_matches_at_offset() { let haystack = haystack.as_bit_str(); let needle = BitString::try_from("110").unwrap(); - assert!(haystack.bits_equal_at(2, needle.as_bit_str())); - assert!(haystack.bits_equal_at(5, needle.as_bit_str())); + assert!(haystack.bits_equal_at_inner::(2, needle.as_bit_str())); + assert!(haystack.bits_equal_at_inner::(5, needle.as_bit_str())); } #[test] @@ -16,9 +16,9 @@ fn returns_false_when_needle_differs_at_offset() { let haystack = haystack.as_bit_str(); let needle = BitString::try_from("110").unwrap(); - assert!(!haystack.bits_equal_at(0, needle.as_bit_str())); - assert!(!haystack.bits_equal_at(1, needle.as_bit_str())); - assert!(!haystack.bits_equal_at(3, needle.as_bit_str())); + assert!(!haystack.bits_equal_at_inner::(0, needle.as_bit_str())); + assert!(!haystack.bits_equal_at_inner::(1, needle.as_bit_str())); + assert!(!haystack.bits_equal_at_inner::(3, needle.as_bit_str())); } #[test] @@ -27,9 +27,9 @@ fn empty_needle_matches_at_valid_boundary_offsets() { let haystack = haystack.as_bit_str(); let needle = BitString::new(); - assert!(haystack.bits_equal_at(0, needle.as_bit_str())); - assert!(haystack.bits_equal_at(3, needle.as_bit_str())); - assert!(haystack.bits_equal_at(haystack.bit_len(), needle.as_bit_str())); + assert!(haystack.bits_equal_at_inner::(0, needle.as_bit_str())); + assert!(haystack.bits_equal_at_inner::(3, needle.as_bit_str())); + assert!(haystack.bits_equal_at_inner::(haystack.bit_len(), needle.as_bit_str())); } #[test] @@ -42,8 +42,8 @@ fn works_across_word_boundaries() { let haystack = bits.as_bit_str(); let needle = BitString::try_from("01110").unwrap(); - assert!(haystack.bits_equal_at(62, needle.as_bit_str())); - assert!(!haystack.bits_equal_at(61, needle.as_bit_str())); + assert!(haystack.bits_equal_at_inner::(62, needle.as_bit_str())); + assert!(!haystack.bits_equal_at_inner::(61, needle.as_bit_str())); } #[test] @@ -52,5 +52,5 @@ fn works_when_needle_reaches_haystack_end() { let haystack = haystack.as_bit_str(); let needle = BitString::try_from("001").unwrap(); - assert!(haystack.bits_equal_at(3, needle.as_bit_str())); + assert!(haystack.bits_equal_at_inner::(3, needle.as_bit_str())); } diff --git a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_ends_with.rs b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_ends_with.rs index e54acc2..dab4e2b 100644 --- a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_ends_with.rs +++ b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_ends_with.rs @@ -9,11 +9,11 @@ fn ends_with_basic() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let s = BitString::try_from("100").unwrap(); - assert!(v.ends_with(s.as_bit_str())); + assert!(v.ends_with_str(s.as_bit_str())); let s = BitString::try_from("10").unwrap(); - assert!(!v.ends_with(s.as_bit_str())); + assert!(!v.ends_with_str(s.as_bit_str())); let s = BitString::try_from("101100").unwrap(); - assert!(v.ends_with(s.as_bit_str())); + assert!(v.ends_with_str(s.as_bit_str())); } #[test] @@ -21,7 +21,7 @@ fn ends_with_empty_suffix() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let s = BitString::new(); - assert!(v.ends_with(s.as_bit_str())); + assert!(v.ends_with_str(s.as_bit_str())); } #[test] @@ -29,7 +29,7 @@ fn ends_with_longer_suffix_returns_false() { let bits = BitString::try_from("101").unwrap(); let v = bits.as_bit_str(); let s = BitString::try_from("1101").unwrap(); - assert!(!v.ends_with(s.as_bit_str())); + assert!(!v.ends_with_str(s.as_bit_str())); } #[test] @@ -38,9 +38,9 @@ fn ends_with_on_offset_view() { // bits 1..6 → 1 0 1 0 1 let v = bits.as_bit_str().slice_from(1).slice_until(5); let s = BitString::try_from("01").unwrap(); - assert!(v.ends_with(s.as_bit_str())); + assert!(v.ends_with_str(s.as_bit_str())); let s = BitString::try_from("10").unwrap(); - assert!(!v.ends_with(s.as_bit_str())); + assert!(!v.ends_with_str(s.as_bit_str())); } #[test] @@ -52,5 +52,5 @@ fn matches_and_ends_with_across_word_boundaries() { bits.set(65, true); let v = bits.as_bit_str(); let p = BitString::try_from("1111").unwrap(); - assert!(v.matches_at(62, p.as_bit_str())); + assert!(v.matches_at_str(62, p.as_bit_str())); } diff --git a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_matches_at.rs b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_matches_at.rs index 922c5f8..8ed2bf1 100644 --- a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_matches_at.rs +++ b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_matches_at.rs @@ -9,8 +9,8 @@ fn matches_at_exact_match() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let p = BitString::try_from("10").unwrap(); - assert!(v.matches_at(0, p.as_bit_str())); - assert!(!v.matches_at(1, p.as_bit_str())); + assert!(v.matches_at_str(0, p.as_bit_str())); + assert!(!v.matches_at_str(1, p.as_bit_str())); } #[test] @@ -18,7 +18,7 @@ fn matches_at_beyond_view_returns_false() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str().slice_from(2); let p = BitString::try_from("10").unwrap(); - assert!(!v.matches_at(2, p.as_bit_str())); + assert!(!v.matches_at_str(2, p.as_bit_str())); } #[test] @@ -26,7 +26,7 @@ fn matches_at_pattern_too_long_returns_false() { let bits = BitString::try_from("10").unwrap(); let v = bits.as_bit_str(); let p = BitString::try_from("100").unwrap(); - assert!(!v.matches_at(0, p.as_bit_str())); + assert!(!v.matches_at_str(0, p.as_bit_str())); } #[test] @@ -34,8 +34,8 @@ fn matches_at_empty_pattern_always_true() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let p = BitString::new(); - assert!(v.matches_at(0, p.as_bit_str())); - assert!(v.matches_at(3, p.as_bit_str())); + assert!(v.matches_at_str(0, p.as_bit_str())); + assert!(v.matches_at_str(3, p.as_bit_str())); } #[test] @@ -44,5 +44,5 @@ fn matches_at_on_offset_view() { // bits: 1 1 1 0 0 0 1 1, view 2..7 → 1 0 0 0 1 let v = bits.as_bit_str().slice_from(2).slice_until(7); let p = BitString::try_from("0001").unwrap(); - assert!(v.matches_at(1, p.as_bit_str())); + assert!(v.matches_at_str(1, p.as_bit_str())); } diff --git a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_starts_with.rs b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_starts_with.rs index 2e9cfd4..6792e6e 100644 --- a/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_starts_with.rs +++ b/src/bit_str/impls_for_matching/impls_for_matches_at/tests_for_starts_with.rs @@ -9,11 +9,11 @@ fn starts_with_basic() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let p = BitString::try_from("101").unwrap(); - assert!(v.starts_with(p.as_bit_str())); + assert!(v.starts_with_str(p.as_bit_str())); let p = BitString::try_from("11").unwrap(); - assert!(!v.starts_with(p.as_bit_str())); + assert!(!v.starts_with_str(p.as_bit_str())); let p = BitString::try_from("101100").unwrap(); - assert!(v.starts_with(p.as_bit_str())); + assert!(v.starts_with_str(p.as_bit_str())); } #[test] @@ -21,7 +21,7 @@ fn starts_with_empty_prefix() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let p = BitString::new(); - assert!(v.starts_with(p.as_bit_str())); + assert!(v.starts_with_str(p.as_bit_str())); } #[test] @@ -29,7 +29,7 @@ fn starts_with_longer_prefix_returns_false() { let bits = BitString::try_from("101").unwrap(); let v = bits.as_bit_str(); let p = BitString::try_from("1011").unwrap(); - assert!(!v.starts_with(p.as_bit_str())); + assert!(!v.starts_with_str(p.as_bit_str())); } #[test] @@ -37,7 +37,7 @@ fn starts_with_on_offset_view() { let bits = BitString::try_from("110101").unwrap(); let v = bits.as_bit_str().slice_from(1).slice_until(5); let p = BitString::try_from("10").unwrap(); - assert!(v.starts_with(p.as_bit_str())); + assert!(v.starts_with_str(p.as_bit_str())); let p = BitString::try_from("11").unwrap(); - assert!(!v.starts_with(p.as_bit_str())); + assert!(!v.starts_with_str(p.as_bit_str())); } diff --git a/src/bit_str/impls_for_matching/impls_for_strip.rs b/src/bit_str/impls_for_matching/impls_for_strip.rs index e8c1352..11442d5 100644 --- a/src/bit_str/impls_for_matching/impls_for_strip.rs +++ b/src/bit_str/impls_for_matching/impls_for_strip.rs @@ -1,19 +1,28 @@ use crate::BitStr; +mod inner; + impl<'bs> BitStr<'bs> { - /// Strips `prefix` from the start, returning the remaining sub-view. #[inline] - pub fn strip_prefix(&self, prefix: BitStr<'_>) -> Option { - self.starts_with(prefix) + pub fn strip_prefix_str(&self, prefix: BitStr<'_>) -> Option { + self.starts_with_str(prefix) .then(|| self.slice_from(prefix.bit_len)) } - - /// Strips `suffix` from the end, returning the remaining sub-view. #[inline] - pub fn strip_suffix(&self, suffix: BitStr<'_>) -> Option { - self.ends_with(suffix) + pub fn strip_prefix_string(&self, prefix: &crate::BitString) -> Option { + self.starts_with_string(prefix) + .then(|| self.slice_from(prefix.as_bit_str().bit_len)) + } + #[inline] + pub fn strip_suffix_str(&self, suffix: BitStr<'_>) -> Option { + self.ends_with_str(suffix) .then(|| self.slice_until(self.bit_len - suffix.bit_len)) } + #[inline] + pub fn strip_suffix_string(&self, suffix: &crate::BitString) -> Option { + self.ends_with_string(suffix) + .then(|| self.slice_until(self.bit_len - suffix.as_bit_str().bit_len)) + } } #[cfg(test)] diff --git a/src/bit_str/impls_for_matching/impls_for_strip/inner.rs b/src/bit_str/impls_for_matching/impls_for_strip/inner.rs new file mode 100644 index 0000000..34a9826 --- /dev/null +++ b/src/bit_str/impls_for_matching/impls_for_strip/inner.rs @@ -0,0 +1,26 @@ +use crate::BitStr; + +impl<'bs> BitStr<'bs> { + /// `strip_prefix` with compile-time alignment signals. + #[allow(dead_code)] + #[inline] + pub(crate) fn strip_prefix_inner( + &self, + prefix: BitStr<'_>, + ) -> Option { + self.starts_with_inner::(prefix) + .then(|| self.slice_from(prefix.bit_len)) + } + + /// `strip_suffix` with compile-time alignment signals. + #[allow(dead_code)] + #[inline] + pub(crate) fn strip_suffix_inner( + &self, + suffix: BitStr<'_>, + ) -> Option { + let offset = self.bit_len - suffix.bit_len; + self.ends_with_inner::(suffix, offset) + .then(|| self.slice_until(self.bit_len - suffix.bit_len)) + } +} diff --git a/src/bit_str/impls_for_matching/impls_for_strip/tests_for_strip.rs b/src/bit_str/impls_for_matching/impls_for_strip/tests_for_strip.rs index 1f7369d..67f3888 100644 --- a/src/bit_str/impls_for_matching/impls_for_strip/tests_for_strip.rs +++ b/src/bit_str/impls_for_matching/impls_for_strip/tests_for_strip.rs @@ -9,7 +9,7 @@ fn strip_prefix_removes_matching_prefix() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let p = BitString::try_from("101").unwrap(); - let rest = v.strip_prefix(p.as_bit_str()).unwrap(); + let rest = v.strip_prefix_str(p.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), 3); assert_eq!(rest.get(0), Some(true)); assert_eq!(rest.get(1), Some(false)); @@ -21,7 +21,7 @@ fn strip_prefix_non_matching_returns_none() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let p = BitString::try_from("11").unwrap(); - assert!(v.strip_prefix(p.as_bit_str()).is_none()); + assert!(v.strip_prefix_str(p.as_bit_str()).is_none()); } #[test] @@ -29,7 +29,7 @@ fn strip_prefix_empty() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let p = BitString::new(); - let rest = v.strip_prefix(p.as_bit_str()).unwrap(); + let rest = v.strip_prefix_str(p.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), v.bit_len()); } @@ -38,7 +38,7 @@ fn strip_prefix_entire_view() { let bits = BitString::try_from("101").unwrap(); let v = bits.as_bit_str(); let p = BitString::try_from("101").unwrap(); - let rest = v.strip_prefix(p.as_bit_str()).unwrap(); + let rest = v.strip_prefix_str(p.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), 0); } @@ -48,7 +48,7 @@ fn strip_prefix_on_offset_view() { // view bits 1..6 → 1 0 1 0 1 let v = bits.as_bit_str().slice_from(1).slice_until(5); let p = BitString::try_from("10").unwrap(); - let rest = v.strip_prefix(p.as_bit_str()).unwrap(); + let rest = v.strip_prefix_str(p.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), 3); assert_eq!(rest.get(0), Some(true)); assert_eq!(rest.get(1), Some(false)); @@ -64,7 +64,7 @@ fn strip_suffix_removes_matching_suffix() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let s = BitString::try_from("100").unwrap(); - let rest = v.strip_suffix(s.as_bit_str()).unwrap(); + let rest = v.strip_suffix_str(s.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), 3); assert_eq!(rest.get(0), Some(true)); assert_eq!(rest.get(1), Some(false)); @@ -76,7 +76,7 @@ fn strip_suffix_non_matching_returns_none() { let bits = BitString::try_from("101100").unwrap(); let v = bits.as_bit_str(); let s = BitString::try_from("10").unwrap(); - assert!(v.strip_suffix(s.as_bit_str()).is_none()); + assert!(v.strip_suffix_str(s.as_bit_str()).is_none()); } #[test] @@ -84,7 +84,7 @@ fn strip_suffix_empty() { let bits = BitString::try_from("10110").unwrap(); let v = bits.as_bit_str(); let s = BitString::new(); - let rest = v.strip_suffix(s.as_bit_str()).unwrap(); + let rest = v.strip_suffix_str(s.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), v.bit_len()); } @@ -93,7 +93,7 @@ fn strip_suffix_entire_view() { let bits = BitString::try_from("101").unwrap(); let v = bits.as_bit_str(); let s = BitString::try_from("101").unwrap(); - let rest = v.strip_suffix(s.as_bit_str()).unwrap(); + let rest = v.strip_suffix_str(s.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), 0); } @@ -103,7 +103,7 @@ fn strip_suffix_on_offset_view() { // view bits 1..6 → 1 0 1 0 1 let v = bits.as_bit_str().slice_from(1).slice_until(5); let s = BitString::try_from("01").unwrap(); - let rest = v.strip_suffix(s.as_bit_str()).unwrap(); + let rest = v.strip_suffix_str(s.as_bit_str()).unwrap(); assert_eq!(rest.bit_len(), 3); assert_eq!(rest.get(0), Some(true)); assert_eq!(rest.get(1), Some(false)); diff --git a/src/bit_str/impls_for_ord.rs b/src/bit_str/impls_for_ord.rs index dfe71df..6e13855 100644 --- a/src/bit_str/impls_for_ord.rs +++ b/src/bit_str/impls_for_ord.rs @@ -1,86 +1,46 @@ use core::cmp::Ordering; -use crate::traits::*; -use crate::{WORD_BITS, low_mask}; +use crate::WORD_BITS; use crate::BitStr; +mod inner; + impl<'bs> BitStr<'bs> { - /// Lexicographic comparison of two bit strings. - /// - /// Compares bits from index 0 upward. At the first differing bit, the - /// side with a `1` is greater. When all bits in the common prefix are - /// equal, the longer bit string is greater. #[inline] - pub fn cmp(&self, other: &BitStr<'bs>) -> Ordering { - let common = self.bit_len.min(other.bit_len); - if common == 0 { - return self.bit_len.cmp(&other.bit_len); + pub fn cmp_str(&self, other: &BitStr<'bs>) -> Ordering { + let hs_aligned = self.start % WORD_BITS == 0; + let nd_aligned = other.start % WORD_BITS == 0; + match (hs_aligned, nd_aligned) { + (true, true) => self.cmp_inner::(other), + (true, false) => self.cmp_inner::(other), + (false, true) => self.cmp_inner::(other), + (false, false) => self.cmp_inner::(other), } + } - let hs_words = self.source.words(); - let nd_words = other.source.words(); - let hs_base = self.start; - let nd_base = other.start; - let hs_aligned = hs_base % WORD_BITS == 0; - let nd_aligned = nd_base % WORD_BITS == 0; - - let full = common / WORD_BITS; - - // Full-word comparison — follow the same dispatch pattern as - // [`BitsEq::eq_words`]: SIMD when `other` is word-aligned. - if nd_aligned { - let nd_slice = &nd_words[nd_base / WORD_BITS..]; - if let Some(ord) = hs_words.cmp_words(nd_slice, full, hs_base) { - return ord; - } - } else if hs_aligned { - // Only `self` is aligned — swap so `other` becomes the - // word-aligned reference, then reverse. - let hs_slice = &hs_words[hs_base / WORD_BITS..]; - if let Some(ord) = nd_words.cmp_words(hs_slice, full, nd_base) { - return ord.reverse(); - } + #[inline] + pub fn cmp_string(&self, other: &crate::BitString) -> Ordering { + let o = other.as_bit_str(); + if self.start % WORD_BITS == 0 { + self.cmp_inner::(&o) } else { - // Both sides misaligned — scalar word-at-a-time (rare). - for i in 0..full { - let pos = i * WORD_BITS; - let a = hs_words.read_word_at(hs_base + pos); - let b = nd_words.read_word_at(nd_base + pos); - if a != b { - return a.bitwise_cmp(b); - } - } - } - - // Partial tail word. - let rem = common % WORD_BITS; - if rem > 0 { - let pos = full * WORD_BITS; - let mask = low_mask(rem); - let a = hs_words.read_word_at(hs_base + pos) & mask; - let b = nd_words.read_word_at(nd_base + pos) & mask; - if a != b { - return a.bitwise_cmp(b); - } + self.cmp_inner::(&o) } - - // Common prefix identical — longer wins. - self.bit_len.cmp(&other.bit_len) } } impl PartialOrd for BitStr<'_> { #[inline] fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) + Some(self.cmp_str(other)) } } impl Ord for BitStr<'_> { #[inline] fn cmp(&self, other: &Self) -> Ordering { - self.cmp(other) + self.cmp_str(other) } } diff --git a/src/bit_str/impls_for_ord/inner.rs b/src/bit_str/impls_for_ord/inner.rs new file mode 100644 index 0000000..0a562ba --- /dev/null +++ b/src/bit_str/impls_for_ord/inner.rs @@ -0,0 +1,62 @@ +use crate::BitStr; +use crate::traits::*; +use crate::{WORD_BITS, low_mask}; +use core::cmp::Ordering; + +impl<'bs> BitStr<'bs> { + /// `cmp` with compile-time alignment signals. + #[inline] + pub(crate) fn cmp_inner( + &self, + other: &BitStr<'bs>, + ) -> Ordering { + let common = self.bit_len.min(other.bit_len); + if common == 0 { + return self.bit_len.cmp(&other.bit_len); + } + let hs_words = self.source.words(); + let nd_words = other.source.words(); + let hs_base = self.start; + let nd_base = other.start; + let full = common / WORD_BITS; + let nd_is_aligned = ND_WORD_ALIGNED || nd_base % WORD_BITS == 0; + if nd_is_aligned { + let nd_slice = &nd_words[nd_base / WORD_BITS..]; + let hs_slice = &hs_words[hs_base / WORD_BITS..]; + let ok = if HS_WORD_ALIGNED { + hs_slice.cmp_words::(nd_slice, full, 0) + } else { + hs_slice.cmp_words::(nd_slice, full, hs_base % WORD_BITS) + }; + if let Some(ord) = ok { + return ord; + } + } else if HS_WORD_ALIGNED || hs_base % WORD_BITS == 0 { + let hs_slice = &hs_words[hs_base / WORD_BITS..]; + let nd_slice = &nd_words[nd_base / WORD_BITS..]; + if let Some(ord) = nd_slice.cmp_words::(hs_slice, full, nd_base % WORD_BITS) { + return ord.reverse(); + } + } else { + for i in 0..full { + let pos = i * WORD_BITS; + let a = hs_words.read_word_at::(hs_base + pos); + let b = nd_words.read_word_at::(nd_base + pos); + if a != b { + return a.bitwise_cmp(b); + } + } + } + let rem = common % WORD_BITS; + if rem > 0 { + let pos = full * WORD_BITS; + let mask = low_mask(rem); + let a = hs_words.read_word_at::(hs_base + pos) & mask; + let b = nd_words.read_word_at::(nd_base + pos) & mask; + if a != b { + return a.bitwise_cmp(b); + } + } + self.bit_len.cmp(&other.bit_len) + } +} diff --git a/src/bit_string/impls_for_access.rs b/src/bit_string/impls_for_access.rs index 5ded899..c8c5dc9 100644 --- a/src/bit_string/impls_for_access.rs +++ b/src/bit_string/impls_for_access.rs @@ -30,7 +30,7 @@ impl BitString { /// Bits beyond `self.len()` are treated as zero. #[inline] pub fn get_chunk(&self, bit_start: usize) -> u64 { - self.words.read_word_at(bit_start) + self.words.read_word_at::(bit_start) } } diff --git a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs index cc15022..7289ee3 100644 --- a/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs +++ b/src/bit_string/impls_for_bit_arith/impls_for_leading_zeros.rs @@ -1,35 +1,232 @@ +use crate::traits::WordsScan; +use crate::{FILL_ONES, FILL_ZEROS, SMALL_WORDS, WORD_BITS}; + use super::BitString; impl BitString { /// Returns the number of consecutive `false` bits from the start. - /// - /// Delegates to [`BitStr::leading_zeros`](crate::BitStr::leading_zeros). #[inline] pub fn leading_zeros(&self) -> usize { - self.as_bit_str().leading_zeros() + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + // SAFETY: `words` is always non-empty when bit_len > 0 + // (BitString invariants guarantee at least one word). + let words_ptr = self.words.as_ptr(); + + // ── First-word fast path ────────────────────────────────── + // SAFETY: `bit_len > 0` (guarded above), so `words` has at least + // one element per the BitString invariant. + let w0 = unsafe { *words_ptr }; + if w0 != 0 { + return (w0.trailing_zeros() as usize).min(bit_len); + } + + // ── Tiny inputs — dispatch to BMI1 when available ──────── + let last_wi = (bit_len - 1) / WORD_BITS; + let end_rem = bit_len % WORD_BITS; + let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; + if mid_end < SMALL_WORDS { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + if !cfg!(target_feature = "bmi1") && crate::cpuid::features().bmi1 { + // SAFETY: BMI1 confirmed by CPUID. `words_ptr` is valid + // for at least `mid_end + (end_rem != 0) as usize` u64 + // reads per BitString invariants. + return unsafe { leading_zeros_scalar_bmi(bit_len, words_ptr, mid_end, end_rem) }; + } + let mut scanned = WORD_BITS; // word 0 already checked above + // SAFETY: `i` ranges in `1..mid_end`. `words` contains at + // least `mid_end + (end_rem != 0) as usize` elements by the + // BitString invariant (backing storage covers all bits). + for i in 1..mid_end { + let w = unsafe { *words_ptr.add(i) }; + if w != 0 { + return (scanned + w.trailing_zeros() as usize).min(bit_len); + } + scanned += WORD_BITS; + } + if end_rem != 0 { + // SAFETY: `mid_end` is the index of the last partial word; + // it is within bounds because `end_rem != 0` implies an + // extra word exists beyond `mid_end - 1`. + let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); + if last == 0 { + return bit_len; + } + return (scanned + last.trailing_zeros() as usize).min(bit_len); + } + return bit_len; + } + + // ── SIMD via trait ──────────────────────────────────────── + self.words() + .leading_value_bits::(0, bit_len) } /// Returns the number of consecutive `true` bits from the start. - /// - /// Delegates to [`BitStr::leading_ones`](crate::BitStr::leading_ones). #[inline] pub fn leading_ones(&self) -> usize { - self.as_bit_str().leading_ones() + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + let words_ptr = self.words.as_ptr(); + + // SAFETY: `bit_len > 0` (guarded above), so `words` has at least + // one element per the BitString invariant. + let w0 = unsafe { *words_ptr }; + if w0 != u64::MAX { + return ((!w0).trailing_zeros() as usize).min(bit_len); + } + + let last_wi = (bit_len - 1) / WORD_BITS; + let end_rem = bit_len % WORD_BITS; + let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; + if mid_end < SMALL_WORDS { + let mut scanned = WORD_BITS; + // SAFETY: `i < mid_end`. `words` has at least + // `mid_end + (end_rem != 0) as usize` elements. + for i in 1..mid_end { + let w = unsafe { *words_ptr.add(i) }; + if w != u64::MAX { + return (scanned + (!w).trailing_zeros() as usize).min(bit_len); + } + scanned += WORD_BITS; + } + if end_rem != 0 { + // SAFETY: `mid_end` is the index of the last partial word + // and is within bounds (backing storage covers all bits). + let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); + if last == ((1u64 << end_rem).wrapping_sub(1)) { + return bit_len; + } + return (scanned + (!last).trailing_zeros() as usize).min(bit_len); + } + return bit_len; + } + + self.words() + .leading_value_bits::(0, bit_len) } /// Returns the number of consecutive `false` bits from the end. - /// - /// Delegates to [`BitStr::trailing_zeros`](crate::BitStr::trailing_zeros). #[inline] pub fn trailing_zeros(&self) -> usize { - self.as_bit_str().trailing_zeros() + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + let words_ptr = self.words.as_ptr(); + + // ── Last partial word ──────────────────────────────────── + let end_rem = bit_len % WORD_BITS; + if end_rem != 0 { + let last_wi = (bit_len - 1) / WORD_BITS; + // SAFETY: `last_wi` is a valid index — `bit_len > 0` and + // `words` always has ≥ (bit_len + 63) / 64 elements. + let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); + if last != 0 { + let shifted = last << (WORD_BITS - end_rem); + return (shifted.leading_zeros() as usize).min(bit_len); + } + } + + // ── Rightmost full word ────────────────────────────────── + if bit_len > WORD_BITS { + let last_full = if end_rem != 0 { + (bit_len - 1) / WORD_BITS - 1 + } else { + (bit_len - 1) / WORD_BITS + }; + // SAFETY: `last_full < (bit_len + 63) / 64`, so it is a valid + // index into `words` (backing storage covers all bits). + let w = unsafe { *words_ptr.add(last_full) }; + if w != 0 { + let tail = if end_rem != 0 { end_rem } else { 0 }; + return (tail + w.leading_zeros() as usize).min(bit_len); + } + } + + self.words() + .trailing_value_bits::(0, bit_len) } /// Returns the number of consecutive `true` bits from the end. - /// - /// Delegates to [`BitStr::trailing_ones`](crate::BitStr::trailing_ones). #[inline] pub fn trailing_ones(&self) -> usize { - self.as_bit_str().trailing_ones() + let bit_len = self.bit_len; + if bit_len == 0 { + return 0; + } + let words_ptr = self.words.as_ptr(); + + let end_rem = bit_len % WORD_BITS; + if end_rem != 0 { + let last_wi = (bit_len - 1) / WORD_BITS; + // SAFETY: `last_wi` is a valid index per the same invariant + // as trailing_zeros (BitString backing storage covers all bits). + let last = unsafe { *words_ptr.add(last_wi) } & ((1u64 << end_rem).wrapping_sub(1)); + if last != ((1u64 << end_rem).wrapping_sub(1)) { + let shifted = (!last) << (WORD_BITS - end_rem); + return (shifted.leading_zeros() as usize).min(bit_len); + } + } + + if bit_len > WORD_BITS { + let last_full = if end_rem != 0 { + (bit_len - 1) / WORD_BITS - 1 + } else { + (bit_len - 1) / WORD_BITS + }; + // SAFETY: `last_full` is a valid index into `words` per the + // BitString backing-storage invariant. + let w = unsafe { *words_ptr.add(last_full) }; + if w != u64::MAX { + let tail = if end_rem != 0 { end_rem } else { 0 }; + return (tail + (!w).leading_zeros() as usize).min(bit_len); + } + } + + self.words() + .trailing_value_bits::(0, bit_len) + } +} + +// ── BMI1-accelerated scalar path ─────────────────────────────────────── + +/// BMI1 variant of the tiny-input scalar loop. Uses `tzcnt` instead of +/// `bsf` for `trailing_zeros()`, eliminating the false output dependency. +/// +/// # Safety +/// +/// `words_ptr` must be valid for at least `mid_end + (end_rem != 0) as +/// usize` u64 reads. Caller must guarantee BMI1 is available per CPUID. +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[target_feature(enable = "bmi1")] +unsafe fn leading_zeros_scalar_bmi( + bit_len: usize, + words_ptr: *const u64, + mid_end: usize, + end_rem: usize, +) -> usize { + let mut scanned = crate::WORD_BITS; // word 0 already checked by caller + // SAFETY: caller guarantees `words_ptr` is valid for the indices used. + for i in 1..mid_end { + let w = unsafe { *words_ptr.add(i) }; + if w != 0 { + return (scanned + w.trailing_zeros() as usize).min(bit_len); + } + scanned += crate::WORD_BITS; + } + if end_rem != 0 { + // SAFETY: `mid_end` is the index of the last partial word; + // bounds guaranteed by caller. + let last = unsafe { *words_ptr.add(mid_end) } & ((1u64 << end_rem).wrapping_sub(1)); + if last == 0 { + return bit_len; + } + return (scanned + last.trailing_zeros() as usize).min(bit_len); } + bit_len } diff --git a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs index 34035de..13f78ec 100644 --- a/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs +++ b/src/bit_string/impls_for_construction/funcs_for_pack_bools_core.rs @@ -34,44 +34,72 @@ pub(super) fn bools_core(src: *const u8, bit_len: usize) -> Vec { /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is available. - unsafe { avx2::words(dst, src, bit_len) }; - return; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. AVX2 availability was confirmed by CPUID. + unsafe { avx2::words(dst, src, bit_len) }; + return; + } + if f.sse2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. SSE2 availability was confirmed by CPUID. + unsafe { sse2::words(dst, src, bit_len) }; + return; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. NEON availability was confirmed by `#[target_feature]`. + unsafe { neon::words(dst, src, bit_len) }; + return; + } + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + #[allow(unused)] + unsafe { + scalar::words(dst, src, bit_len); + } } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is available. - unsafe { sse2::words(dst, src, bit_len) }; - return; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. AVX2 availability was confirmed by `#[target_feature]`. + unsafe { avx2::words(dst, src, bit_len) }; + return; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is available. - unsafe { neon::words(dst, src, bit_len) }; - return; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. SSE2 availability was confirmed by `#[target_feature]`. + unsafe { sse2::words(dst, src, bit_len) }; + return; + } - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, bit_len); + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word count. NEON availability was confirmed by `#[target_feature]`. + unsafe { neon::words(dst, src, bit_len) }; + return; + } + + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + #[allow(unused)] + unsafe { + scalar::words(dst, src, bit_len); + } } } diff --git a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs index 4038ed4..7b99b5c 100644 --- a/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs +++ b/src/bit_string/impls_for_construction/impls_for_from_str/funcs_for_pack_str_core.rs @@ -43,35 +43,72 @@ pub(super) fn str_core(src: *const u8, bit_len: usize) -> Result, (usiz /// - `dst` must be valid for writes of `ceil(bit_len / 64)` u64 values. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u8, bit_len: usize) -> Option<(usize, u8)> { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - // SAFETY: forwarded from `dispatch`'s safety contract. - return unsafe { avx2::words(dst, src, bit_len) }; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // AVX2 availability was confirmed by CPUID check above. + return unsafe { avx2::words(dst, src, bit_len) }; + } + if f.sse2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // SSE2 availability was confirmed by CPUID check above. + return unsafe { sse2::words(dst, src, bit_len) }; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // NEON is guaranteed by `#[cfg(target_feature = "neon")]`. + return unsafe { neon::words(dst, src, bit_len) }; + } + #[allow(unused)] + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + unsafe { + scalar::words(dst, src, bit_len) + } } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - // SAFETY: forwarded from `dispatch`'s safety contract. - return unsafe { sse2::words(dst, src, bit_len) }; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // AVX2 is guaranteed by `#[cfg(target_feature = "avx2")]`. + return unsafe { avx2::words(dst, src, bit_len) }; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: forwarded from `dispatch`'s safety contract. - return unsafe { neon::words(dst, src, bit_len) }; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // SSE2 is guaranteed by `#[cfg(target_feature = "sse2")]`. + return unsafe { sse2::words(dst, src, bit_len) }; + } - #[allow(unused)] - // SAFETY: forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, bit_len) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and byte count. + // NEON is guaranteed by `#[cfg(target_feature = "neon")]`. + return unsafe { neon::words(dst, src, bit_len) }; + } + + #[allow(unused)] + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + unsafe { + scalar::words(dst, src, bit_len) + } } } @@ -290,6 +327,7 @@ mod sse2 { let mask3 = _mm_movemask_epi8(valid3) as u16; if mask0 != 0xFFFF || mask1 != 0xFFFF || mask2 != 0xFFFF || mask3 != 0xFFFF { + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. let (i, b) = unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); return Some((global_offset + i, b)); @@ -349,6 +387,8 @@ mod neon { ) -> Option<(usize, u8)> { let zero_byte = vdup_n_u8(b'0'); let one_byte = vdup_n_u8(b'1'); + // SAFETY: `BIT_MASKS` is a static array of 8 u8 values, so its pointer is + // valid for reads of 8 bytes. let bit_masks = unsafe { vld1_u8(BIT_MASKS.as_ptr()) }; let mut global_offset = 0usize; @@ -375,6 +415,7 @@ mod neon { if invalid_u64 != 0 { // Fall back to scalar for exact error position within // this 64-byte chunk. + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. let (i, b) = unsafe { scalar::words(dst, src, 64) }.expect("chunk has invalid byte"); return Some((global_offset + i, b)); diff --git a/src/bit_string/impls_for_editing/impls_for_concat.rs b/src/bit_string/impls_for_editing/impls_for_concat.rs index 3cd2fc8..92e0731 100644 --- a/src/bit_string/impls_for_editing/impls_for_concat.rs +++ b/src/bit_string/impls_for_editing/impls_for_concat.rs @@ -88,10 +88,10 @@ impl BitString { while offset > 0 { offset -= WORD_BITS.min(offset); let take = WORD_BITS.min(tail_len - offset); - let chunk = self.words.read_word_at(index + offset); + let chunk = self.words.read_word_at::(index + offset); self.words.clear_bits_at(index + shift + offset, take); self.words - .write_word_at(index + shift + offset, chunk, take); + .write_word_at::(index + shift + offset, chunk, take); } // The source region [index, index+shift) still holds the original diff --git a/src/bit_string/impls_for_hash.rs b/src/bit_string/impls_for_hash.rs index cf0568e..2593e69 100644 --- a/src/bit_string/impls_for_hash.rs +++ b/src/bit_string/impls_for_hash.rs @@ -5,6 +5,6 @@ use crate::BitString; impl Hash for BitString { #[inline] fn hash(&self, state: &mut H) { - self.as_bit_str().hash(state); + self.as_bit_str().hash_inner::(state); } } diff --git a/src/bit_string/impls_for_matching/impls_for_find.rs b/src/bit_string/impls_for_matching/impls_for_find.rs index 012cf57..7dad676 100644 --- a/src/bit_string/impls_for_matching/impls_for_find.rs +++ b/src/bit_string/impls_for_matching/impls_for_find.rs @@ -1,18 +1,63 @@ +use crate::WORD_BITS; + use super::*; impl BitString { + // ------------------------------------------------------------------- + // _str methods — needle is BitStr (hs is BitString-aligned) + // ------------------------------------------------------------------- + + /// Returns `true` if `needle` is contained within `self`. + #[inline] + pub fn contains_str(&self, needle: crate::BitStr<'_>) -> bool { + let view = self.as_bit_str(); + if needle.start % WORD_BITS == 0 { + view.contains_inner::(needle) + } else { + view.contains_inner::(needle) + } + } + + #[inline] + pub fn find_str(&self, needle: crate::BitStr<'_>) -> Option { + let view = self.as_bit_str(); + if needle.start % WORD_BITS == 0 { + view.find_inner::(needle) + } else { + view.find_inner::(needle) + } + } + + #[inline] + pub fn rfind_str(&self, needle: crate::BitStr<'_>) -> Option { + let view = self.as_bit_str(); + if needle.start % WORD_BITS == 0 { + view.rfind_inner::(needle) + } else { + view.rfind_inner::(needle) + } + } + + // ------------------------------------------------------------------- + // _string methods — both sides are BitString (double word-aligned) + // ------------------------------------------------------------------- + + /// Returns `true` if `needle` is contained within `self`. #[inline] - pub fn contains(&self, needle: crate::BitStr<'_>) -> bool { - self.as_bit_str().contains(needle) + pub fn contains_string(&self, needle: &BitString) -> bool { + self.as_bit_str() + .contains_inner::(needle.as_bit_str()) } #[inline] - pub fn find(&self, needle: crate::BitStr<'_>) -> Option { - self.as_bit_str().find(needle) + pub fn find_string(&self, needle: &BitString) -> Option { + self.as_bit_str() + .find_inner::(needle.as_bit_str()) } #[inline] - pub fn rfind(&self, needle: crate::BitStr<'_>) -> Option { - self.as_bit_str().rfind(needle) + pub fn rfind_string(&self, needle: &BitString) -> Option { + self.as_bit_str() + .rfind_inner::(needle.as_bit_str()) } } diff --git a/src/bit_string/impls_for_matching/impls_for_matches_at.rs b/src/bit_string/impls_for_matching/impls_for_matches_at.rs index 6d96876..e78ef13 100644 --- a/src/bit_string/impls_for_matching/impls_for_matches_at.rs +++ b/src/bit_string/impls_for_matching/impls_for_matches_at.rs @@ -1,21 +1,103 @@ +use crate::WORD_BITS; + use super::*; impl BitString { /// Returns `true` if `pattern` matches the bits starting at `index`. + /// + /// `BitString` is always word-aligned, so `HS_WORD_ALIGNED = true`. + #[inline] + pub fn matches_at_str(&self, index: usize, pattern: crate::BitStr<'_>) -> bool { + let view = self.as_bit_str(); + if index > view.bit_len { + return false; + } + if pattern.bit_len > view.bit_len - index { + return false; + } + if pattern.start % WORD_BITS == 0 { + view.matches_at_inner::(index, pattern) + } else { + view.matches_at_inner::(index, pattern) + } + } + + /// `matches_at_str` when `pattern` is a `BitString` (both aligned). + #[inline] + pub fn matches_at_string(&self, index: usize, pattern: &BitString) -> bool { + let view = self.as_bit_str(); + if index > view.bit_len { + return false; + } + if pattern.bit_len > view.bit_len - index { + return false; + } + view.matches_at_inner::(index, pattern.as_bit_str()) + } + + // ------------------------------------------------------------------- + // _str methods — argument is BitStr (hs is BitString-aligned) + // ------------------------------------------------------------------- + + /// Returns `true` if `prefix` is a prefix of `self`. #[inline] - pub fn matches_at(&self, index: usize, pattern: crate::BitStr<'_>) -> bool { - self.as_bit_str().matches_at(index, pattern) + pub fn starts_with_str(&self, prefix: crate::BitStr<'_>) -> bool { + let view = self.as_bit_str(); + if prefix.start % WORD_BITS == 0 { + view.starts_with_inner::(prefix) + } else { + view.starts_with_inner::(prefix) + } } + /// Returns `true` if `suffix` is a suffix of `self`. + #[inline] + pub fn ends_with_str(&self, suffix: crate::BitStr<'_>) -> bool { + let view = self.as_bit_str(); + if suffix.bit_len == 0 { + return true; + } + if suffix.bit_len > view.bit_len { + return false; + } + let offset = view.bit_len - suffix.bit_len; + let hs_aligned = offset % WORD_BITS == 0; // self.start == 0, so hs_base == offset + let nd_aligned = suffix.start % WORD_BITS == 0; + match (hs_aligned, nd_aligned) { + (true, true) => view.ends_with_inner::(suffix, offset), + (true, false) => view.ends_with_inner::(suffix, offset), + (false, true) => view.ends_with_inner::(suffix, offset), + (false, false) => view.ends_with_inner::(suffix, offset), + } + } + + // ------------------------------------------------------------------- + // _string methods — both sides are BitString (double word-aligned) + // ------------------------------------------------------------------- + /// Returns `true` if `prefix` is a prefix of `self`. #[inline] - pub fn starts_with(&self, prefix: crate::BitStr<'_>) -> bool { - self.as_bit_str().starts_with(prefix) + pub fn starts_with_string(&self, prefix: &BitString) -> bool { + self.as_bit_str() + .starts_with_inner::(prefix.as_bit_str()) } /// Returns `true` if `suffix` is a suffix of `self`. #[inline] - pub fn ends_with(&self, suffix: crate::BitStr<'_>) -> bool { - self.as_bit_str().ends_with(suffix) + pub fn ends_with_string(&self, suffix: &BitString) -> bool { + let view = self.as_bit_str(); + if suffix.bit_len == 0 { + return true; + } + if suffix.bit_len > view.bit_len { + return false; + } + let offset = view.bit_len - suffix.bit_len; + let hs_aligned = offset % WORD_BITS == 0; + if hs_aligned { + view.ends_with_inner::(suffix.as_bit_str(), offset) + } else { + view.ends_with_inner::(suffix.as_bit_str(), offset) + } } } diff --git a/src/bit_string/impls_for_matching/impls_for_strip.rs b/src/bit_string/impls_for_matching/impls_for_strip.rs index e28d350..16de53c 100644 --- a/src/bit_string/impls_for_matching/impls_for_strip.rs +++ b/src/bit_string/impls_for_matching/impls_for_strip.rs @@ -1,17 +1,48 @@ +use crate::WORD_BITS; + use super::*; impl BitString { + /// Strips `prefix` from the start, returning the remaining `BitString`. + #[inline] + pub fn strip_prefix_str(&self, prefix: crate::BitStr<'_>) -> Option { + let view = self.as_bit_str(); + let ok = if prefix.start % WORD_BITS == 0 { + view.starts_with_inner::(prefix) + } else { + view.starts_with_inner::(prefix) + }; + ok.then(|| self.slice_from(prefix.bit_len)) + } + + /// `strip_prefix_str` when both sides are `BitString`. + #[inline] + pub fn strip_prefix_string(&self, prefix: &BitString) -> Option { + let ok = self + .as_bit_str() + .starts_with_inner::(prefix.as_bit_str()); + ok.then(|| self.slice_from(prefix.as_bit_str().bit_len)) + } + + /// Strips `suffix` from the end, returning the remaining `BitString`. #[inline] - pub fn strip_prefix(&self, prefix: crate::BitStr<'_>) -> Option { - self.as_bit_str() - .starts_with(prefix) - .then(|| self.slice_from(prefix.bit_len)) + pub fn strip_suffix_str(&self, suffix: crate::BitStr<'_>) -> Option { + let view = self.as_bit_str(); + let offset = view.bit_len - suffix.bit_len; + let ok = if suffix.start % WORD_BITS == 0 { + view.ends_with_inner::(suffix, offset) + } else { + view.ends_with_inner::(suffix, offset) + }; + ok.then(|| self.slice_until(self.bit_len - suffix.bit_len)) } + /// `strip_suffix_str` when both sides are `BitString`. #[inline] - pub fn strip_suffix(&self, suffix: crate::BitStr<'_>) -> Option { - self.as_bit_str() - .ends_with(suffix) - .then(|| self.slice_until(self.bit_len - suffix.bit_len)) + pub fn strip_suffix_string(&self, suffix: &BitString) -> Option { + let view = self.as_bit_str(); + let offset = view.bit_len - suffix.as_bit_str().bit_len; + let ok = view.ends_with_inner::(suffix.as_bit_str(), offset); + ok.then(|| self.slice_until(self.bit_len - suffix.as_bit_str().bit_len)) } } diff --git a/src/bit_string/impls_for_ord.rs b/src/bit_string/impls_for_ord.rs index 1ca6698..653ecd1 100644 --- a/src/bit_string/impls_for_ord.rs +++ b/src/bit_string/impls_for_ord.rs @@ -1,17 +1,39 @@ use core::cmp::Ordering; use crate::BitString; +use crate::WORD_BITS; + +impl BitString { + /// Lexicographic comparison against a [`BitStr`](crate::BitStr). + #[inline] + pub fn cmp_str(&self, other: &crate::BitStr<'_>) -> Ordering { + let view = self.as_bit_str(); + if other.start % WORD_BITS == 0 { + view.cmp_inner::(other) + } else { + view.cmp_inner::(other) + } + } + + /// Lexicographic comparison — both sides are `BitString` + /// (double word-aligned, zero runtime alignment checks). + #[inline] + pub fn cmp_string(&self, other: &BitString) -> Ordering { + self.as_bit_str() + .cmp_inner::(&other.as_bit_str()) + } +} impl PartialOrd for BitString { #[inline] fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) + Some(self.cmp_string(other)) } } impl Ord for BitString { #[inline] fn cmp(&self, other: &Self) -> Ordering { - self.as_bit_str().cmp(&other.as_bit_str()) + self.cmp_string(other) } } diff --git a/src/consts_for_bits.rs b/src/consts_for_bits.rs index b23ed5e..48af95b 100644 --- a/src/consts_for_bits.rs +++ b/src/consts_for_bits.rs @@ -24,3 +24,10 @@ pub(crate) const SMALL_WORDS: usize = 2; all(target_arch = "aarch64", target_feature = "neon"), )))] pub(crate) const SMALL_WORDS: usize = 0; + +/// Fill-value constants for leading-/trailing-value-word scans. +/// Passed as `const FILL: u64` generic parameters so the two +/// variants (`0` / `u64::MAX`) are monomorphised separately, +/// eliminating runtime branches. +pub(crate) const FILL_ZEROS: u64 = 0; +pub(crate) const FILL_ONES: u64 = u64::MAX; diff --git a/src/cpuid.rs b/src/cpuid.rs new file mode 100644 index 0000000..3734c16 --- /dev/null +++ b/src/cpuid.rs @@ -0,0 +1,60 @@ +//! CPU feature detection — once per process, shared by all SIMD dispatch. +//! +//! Runs CPUID exactly once on first access via `OnceCell`. All runtime +//! dispatch sites read from this single cache via `crate::cpuid::features()`. + +use once_cell::sync::OnceCell; + +/// Cached CPU feature flags. All fields are `false` on non-x86 targets. +#[allow(dead_code)] +pub(crate) struct CpuFeatures { + pub(crate) avx2: bool, + pub(crate) bmi1: bool, + pub(crate) sse41: bool, + pub(crate) ssse3: bool, + pub(crate) sse2: bool, +} + +static FEATURES: OnceCell = OnceCell::new(); + +/// Returns a reference to the process-lifetime CPU feature cache. +/// +/// CPUID runs at most once, on the very first call. +#[inline] +pub(crate) fn features() -> &'static CpuFeatures { + FEATURES.get_or_init(|| { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + // SAFETY: `__cpuid_count` is always safe — read-only instruction. + #[cfg(target_arch = "x86_64")] + let (leaf1, leaf7) = unsafe { + ( + core::arch::x86_64::__cpuid_count(1, 0), + core::arch::x86_64::__cpuid_count(7, 0), + ) + }; + #[cfg(target_arch = "x86")] + let (leaf1, leaf7) = unsafe { + ( + core::arch::x86::__cpuid_count(1, 0), + core::arch::x86::__cpuid_count(7, 0), + ) + }; + CpuFeatures { + avx2: leaf7.ebx & (1 << 5) != 0, + bmi1: leaf7.ebx & (1 << 3) != 0, + sse41: leaf1.ecx & (1 << 19) != 0, + ssse3: leaf1.ecx & (1 << 9) != 0, + sse2: leaf1.edx & (1 << 26) != 0, + } + } + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + CpuFeatures { + avx2: false, + bmi1: false, + sse41: false, + ssse3: false, + sse2: false, + } + }) +} diff --git a/src/lib.rs b/src/lib.rs index e2864ca..d2248c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ extern crate alloc; mod consts_for_bits; +mod cpuid; mod funcs_for_bits; pub(crate) mod traits; diff --git a/src/traits.rs b/src/traits.rs index 2cd7831..8723483 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,13 +1,15 @@ -pub(crate) mod bit_ord; -pub(crate) mod bits_arith; -pub(crate) mod bits_edit; -pub(crate) mod bits_eq; -pub(crate) mod bits_find; -pub(crate) mod bits_ord; +pub(crate) mod word_ord; +pub(crate) mod words_arith; +pub(crate) mod words_edit; +pub(crate) mod words_eq; +pub(crate) mod words_find; +pub(crate) mod words_ord; +pub(crate) mod words_scan; -pub(crate) use bit_ord::*; -pub(crate) use bits_arith::*; -pub(crate) use bits_edit::*; -pub(crate) use bits_eq::*; -pub(crate) use bits_find::*; -pub(crate) use bits_ord::*; +pub(crate) use word_ord::*; +pub(crate) use words_arith::*; +pub(crate) use words_edit::*; +pub(crate) use words_eq::*; +pub(crate) use words_find::*; +pub(crate) use words_ord::*; +pub(crate) use words_scan::*; diff --git a/src/traits/bits_arith/funcs_for_leading_value_words.rs b/src/traits/bits_arith/funcs_for_leading_value_words.rs deleted file mode 100644 index a0a2600..0000000 --- a/src/traits/bits_arith/funcs_for_leading_value_words.rs +++ /dev/null @@ -1,391 +0,0 @@ -//! SIMD-accelerated leading-value-word scan. -//! -//! Counts consecutive words equal to `fill` from the start of a slice. -//! Used to implement both `leading_zero_words` (`fill = 0`) and -//! `leading_one_words` (`fill = !0`). - -use crate::SMALL_WORDS; - -/// Returns the number of consecutive words equal to `fill` at the start of -/// `words`. -/// -/// All words up to (but not including) the returned index equal `fill`. If -/// the return value equals `words.len()`, every word equals `fill`. -#[inline] -pub(crate) fn leading_value_words(words: &[u64], fill: u64) -> usize { - if words.len() < SMALL_WORDS { - return scalar::scan(words, fill); - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - // SAFETY: AVX2 is available (compiled with target_feature check). - unsafe { - return avx2::scan(words, fill); - } - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - // SAFETY: SSE4.1 is available. - unsafe { - return sse41::scan(words, fill); - } - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: NEON is available. - unsafe { - return neon::scan(words, fill); - } - } - - #[allow(unused)] - scalar::scan(words, fill) -} - -/// Returns the number of consecutive words equal to `fill` at the **end** of -/// `words`. -/// -/// All words from `words.len() - count` onwards equal `fill`. If the return -/// value equals `words.len()`, every word equals `fill`. -#[inline] -pub(crate) fn trailing_value_words(words: &[u64], fill: u64) -> usize { - if words.len() < SMALL_WORDS { - return scalar::scan_rev(words, fill); - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - unsafe { - return avx2::scan_rev(words, fill); - } - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - unsafe { - return sse41::scan_rev(words, fill); - } - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - unsafe { - return neon::scan_rev(words, fill); - } - } - - #[allow(unused)] - scalar::scan_rev(words, fill) -} - -// --------------------------------------------------------------------------- -// Scalar -// --------------------------------------------------------------------------- - -mod scalar { - #[inline] - pub(super) fn scan(words: &[u64], fill: u64) -> usize { - for (i, &w) in words.iter().enumerate() { - if w != fill { - return i; - } - } - words.len() - } - - #[inline] - pub(super) fn scan_rev(words: &[u64], fill: u64) -> usize { - for i in 0..words.len() { - if words[words.len() - 1 - i] != fill { - return i; - } - } - words.len() - } -} - -// --------------------------------------------------------------------------- -// AVX2 — 4 words at a time -// --------------------------------------------------------------------------- - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_set1_epi64x, - }; - - const LANES: usize = 4; - - /// AVX2 backend. - /// - /// # Safety - /// - /// Caller must ensure AVX2 is available. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn scan(words: &[u64], fill: u64) -> usize { - // SAFETY: all operations inside require AVX2, which is guaranteed by - // the `target_feature` annotation on this function. - unsafe { - let fill_vec = _mm256_set1_epi64x(fill as i64); - let chunks = words.len() / LANES; - let mut count = 0usize; - - for chunk in 0..chunks { - let offset = chunk * LANES; - // SAFETY: offset + LANES ≤ words.len(); _mm256_loadu_si256 - // supports unaligned reads. - let data = _mm256_loadu_si256(words.as_ptr().add(offset).cast::<__m256i>()); - // Compare each u64 lane to fill → -1 for equal, 0 for not. - let cmp = _mm256_cmpeq_epi64(data, fill_vec); - // movemask extracts the MSB of each byte → 32-bit mask. - // An equal-word lane (all bytes 0xFF after cmpeq) contributes - // 0xFF (8 bits of 1). A non-equal lane contributes 0x00 - // (8 bits of 0). - let mask = _mm256_movemask_epi8(cmp) as u32; - if mask != 0xFFFF_FFFF { - for k in 0..LANES { - if (mask >> (k * 8)) & 0xFF != 0xFF { - return count + k; - } - } - // SAFETY: mask != 0xFFFF_FFFF guarantees at least one - // lane is not equal to fill, so the loop always returns. - core::hint::unreachable_unchecked(); - } - count += LANES; - } - - let done = chunks * LANES; - count + super::scalar::scan(&words[done..], fill) - } - } - - /// AVX2 backend — reverse scan. - #[target_feature(enable = "avx2")] - pub(super) unsafe fn scan_rev(words: &[u64], fill: u64) -> usize { - unsafe { - let fill_vec = _mm256_set1_epi64x(fill as i64); - let full_chunks = words.len() / LANES; - let done = full_chunks * LANES; - - // Process the partial tail first. - let tail_count = super::scalar::scan_rev(&words[done..], fill); - if tail_count < words.len() - done { - return tail_count; - } - let mut count = tail_count; - - // Full chunks from right to left. - for chunk in (0..full_chunks).rev() { - let offset = chunk * LANES; - let data = _mm256_loadu_si256(words.as_ptr().add(offset).cast::<__m256i>()); - let cmp = _mm256_cmpeq_epi64(data, fill_vec); - let mask = _mm256_movemask_epi8(cmp) as u32; - if mask != 0xFFFF_FFFF { - for k in (0..LANES).rev() { - if (mask >> (k * 8)) & 0xFF != 0xFF { - return count + (LANES - 1 - k); - } - } - core::hint::unreachable_unchecked(); - } - count += LANES; - } - - count - } - } -} - -// --------------------------------------------------------------------------- -// SSE4.1 — 2 words at a time -// --------------------------------------------------------------------------- - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - }; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{ - __m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, - }; - - const LANES: usize = 2; - - /// SSE4.1 backend. - /// - /// # Safety - /// - /// Caller must ensure SSE4.1 is available. - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn scan(words: &[u64], fill: u64) -> usize { - // SAFETY: all operations inside require SSE4.1, which is guaranteed by - // the `target_feature` annotation. - unsafe { - let fill_vec = _mm_set1_epi64x(fill as i64); - let chunks = words.len() / LANES; - let mut count = 0usize; - - for chunk in 0..chunks { - let offset = chunk * LANES; - let data = _mm_loadu_si128(words.as_ptr().add(offset).cast::<__m128i>()); - let cmp = _mm_cmpeq_epi64(data, fill_vec); - let mask = _mm_movemask_epi8(cmp) as u32; - // 16-bit mask from 2 u64 lanes: 2 × 8 bytes = 16 bits. - if mask != 0xFFFF { - for k in 0..LANES { - if (mask >> (k * 8)) & 0xFF != 0xFF { - return count + k; - } - } - // SAFETY: mask != 0xFFFF guarantees at least one lane is - // not equal to fill, so the loop always returns. - core::hint::unreachable_unchecked(); - } - count += LANES; - } - - let done = chunks * LANES; - count + super::scalar::scan(&words[done..], fill) - } - } - - /// SSE4.1 backend — reverse scan. - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn scan_rev(words: &[u64], fill: u64) -> usize { - unsafe { - let fill_vec = _mm_set1_epi64x(fill as i64); - let full_chunks = words.len() / LANES; - let done = full_chunks * LANES; - - let tail_count = super::scalar::scan_rev(&words[done..], fill); - if tail_count < words.len() - done { - return tail_count; - } - let mut count = tail_count; - - for chunk in (0..full_chunks).rev() { - let offset = chunk * LANES; - let data = _mm_loadu_si128(words.as_ptr().add(offset).cast::<__m128i>()); - let cmp = _mm_cmpeq_epi64(data, fill_vec); - let mask = _mm_movemask_epi8(cmp) as u32; - if mask != 0xFFFF { - for k in (0..LANES).rev() { - if (mask >> (k * 8)) & 0xFF != 0xFF { - return count + (LANES - 1 - k); - } - } - core::hint::unreachable_unchecked(); - } - count += LANES; - } - - count - } - } -} - -// --------------------------------------------------------------------------- -// NEON — 2 words at a time -// --------------------------------------------------------------------------- - -#[allow(unused)] -#[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; - - const LANES: usize = 2; - - /// NEON backend. - /// - /// # Safety - /// - /// Caller must ensure NEON is available. - #[target_feature(enable = "neon")] - pub(super) unsafe fn scan(words: &[u64], fill: u64) -> usize { - // SAFETY: all operations inside require NEON, which is guaranteed by - // the `target_feature` annotation. - unsafe { - let fill_vec = vdupq_n_u64(fill); - let chunks = words.len() / LANES; - let mut count = 0usize; - - for chunk in 0..chunks { - let offset = chunk * LANES; - let data = vld1q_u64(words.as_ptr().add(offset)); - let cmp = vceqq_u64(data, fill_vec); - // vceqq returns all-1s (-1) for equal, all-0s for not equal. - if vgetq_lane_u64(cmp, 0) == 0 { - return count; - } - if vgetq_lane_u64(cmp, 1) == 0 { - return count + 1; - } - count += LANES; - } - - let done = chunks * LANES; - count + super::scalar::scan(&words[done..], fill) - } - } - - /// NEON backend — reverse scan. - #[target_feature(enable = "neon")] - pub(super) unsafe fn scan_rev(words: &[u64], fill: u64) -> usize { - unsafe { - let fill_vec = vdupq_n_u64(fill); - let full_chunks = words.len() / LANES; - let done = full_chunks * LANES; - - let tail_count = super::scalar::scan_rev(&words[done..], fill); - if tail_count < words.len() - done { - return tail_count; - } - let mut count = tail_count; - - for chunk in (0..full_chunks).rev() { - let offset = chunk * LANES; - let data = vld1q_u64(words.as_ptr().add(offset)); - let cmp = vceqq_u64(data, fill_vec); - if vgetq_lane_u64(cmp, 1) == 0 { - return count; - } - if vgetq_lane_u64(cmp, 0) == 0 { - return count + 1; - } - count += LANES; - } - - count - } - } -} - -#[cfg(test)] -mod tests_for_backend_equivalence; diff --git a/src/traits/bits_arith/funcs_for_leading_value_words/tests_for_backend_equivalence.rs b/src/traits/bits_arith/funcs_for_leading_value_words/tests_for_backend_equivalence.rs deleted file mode 100644 index f1fcd20..0000000 --- a/src/traits/bits_arith/funcs_for_leading_value_words/tests_for_backend_equivalence.rs +++ /dev/null @@ -1,218 +0,0 @@ -use alloc::vec; - -use super::*; - -const CASES_ZERO: &[&[u64]] = &[ - &[], - &[0], - &[u64::MAX], - &[1], - &[0, u64::MAX], - &[0, 0, u64::MAX], - &[0, 0, 0, 0, u64::MAX], - &[0, 0, 0, 0, 0, 0, 0, u64::MAX], - &[0x5555_5555_5555_5555, 0xAAAA_AAAA_AAAA_AAAA], - &[0, 1, u64::MAX, 0x0123_4567_89AB_CDEF], - &[0u64; 32], - &[u64::MAX; 16], -]; - -const CASES_ONE: &[&[u64]] = &[ - &[], - &[u64::MAX], - &[0], - &[u64::MAX - 1], - &[u64::MAX, 0], - &[u64::MAX, u64::MAX, 0], - &[u64::MAX, u64::MAX, u64::MAX, u64::MAX, 0], - &[ - u64::MAX, - u64::MAX, - u64::MAX, - u64::MAX, - u64::MAX, - u64::MAX, - u64::MAX, - 0, - ], - &[0x5555_5555_5555_5555, 0xAAAA_AAAA_AAAA_AAAA], - &[u64::MAX, 1, 0, 0x0123_4567_89AB_CDEF], - &[u64::MAX; 32], - &[0u64; 16], -]; - -fn assert_backend_matches_scalar(backend: unsafe fn(&[u64], u64) -> usize, fill: u64) { - let cases: &[&[u64]] = if fill == 0 { CASES_ZERO } else { CASES_ONE }; - - for &src in cases { - let expected = scalar::scan(src, fill); - // SAFETY: the backend requires the corresponding target feature, but - // the test is only compiled when that feature is enabled. - let actual = unsafe { backend(src, fill) }; - - assert_eq!(actual, expected, "fill=0x{fill:x} src={src:?}"); - } -} - -// Also test with random-looking data. -fn run_random(backend: unsafe fn(&[u64], u64) -> usize, fill: u64) { - for run in [0, 1, 3, 5, 7, 9, 15, 16, 17, 31] { - let mut v = vec![fill; run]; - for _ in 0..16 { - v.push(if fill == 0 { u64::MAX } else { 0 }); - } - let expected = scalar::scan(&v, fill); - let actual = unsafe { backend(&v, fill) }; - assert_eq!(actual, expected, "fill=0x{fill:x} run={run}"); - } -} - -// --------------------------------------------------------------------------- -// AVX2 -// --------------------------------------------------------------------------- - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" -))] -#[test] -fn avx2_matches_scalar_zeros() { - assert_backend_matches_scalar(avx2::scan, 0); - run_random(avx2::scan, 0); -} - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" -))] -#[test] -fn avx2_matches_scalar_ones() { - assert_backend_matches_scalar(avx2::scan, !0); - run_random(avx2::scan, !0); -} - -// --------------------------------------------------------------------------- -// SSE4.1 -// --------------------------------------------------------------------------- - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") -))] -#[test] -fn sse41_matches_scalar_zeros() { - assert_backend_matches_scalar(sse41::scan, 0); - run_random(sse41::scan, 0); -} - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") -))] -#[test] -fn sse41_matches_scalar_ones() { - assert_backend_matches_scalar(sse41::scan, !0); - run_random(sse41::scan, !0); -} - -// --------------------------------------------------------------------------- -// NEON -// --------------------------------------------------------------------------- - -#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] -#[test] -fn neon_matches_scalar_zeros() { - assert_backend_matches_scalar(neon::scan, 0); - run_random(neon::scan, 0); -} - -#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] -#[test] -fn neon_matches_scalar_ones() { - assert_backend_matches_scalar(neon::scan, !0); - run_random(neon::scan, !0); -} - -// --------------------------------------------------------------------------- -// Trailing scan (reverse) — AVX2 / SSE4.1 / NEON -// --------------------------------------------------------------------------- - -fn assert_trailing_backend_matches_scalar(backend: unsafe fn(&[u64], u64) -> usize, fill: u64) { - let cases: &[&[u64]] = if fill == 0 { CASES_ZERO } else { CASES_ONE }; - - for &src in cases { - let expected = scalar::scan_rev(src, fill); - let actual = unsafe { backend(src, fill) }; - assert_eq!(actual, expected, "fill=0x{fill:x} src={src:?}"); - } -} - -fn run_random_trailing(backend: unsafe fn(&[u64], u64) -> usize, fill: u64) { - for run in [0, 1, 3, 5, 7, 9, 15, 16, 17, 31] { - let mut v = vec![fill; run]; - for _ in 0..16 { - v.push(if fill == 0 { u64::MAX } else { 0 }); - } - let expected = scalar::scan_rev(&v, fill); - let actual = unsafe { backend(&v, fill) }; - assert_eq!(actual, expected, "fill=0x{fill:x} run={run}"); - } -} - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" -))] -#[test] -fn avx2_trailing_matches_scalar_zeros() { - assert_trailing_backend_matches_scalar(avx2::scan_rev, 0); - run_random_trailing(avx2::scan_rev, 0); -} - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" -))] -#[test] -fn avx2_trailing_matches_scalar_ones() { - assert_trailing_backend_matches_scalar(avx2::scan_rev, !0); - run_random_trailing(avx2::scan_rev, !0); -} - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") -))] -#[test] -fn sse41_trailing_matches_scalar_zeros() { - assert_trailing_backend_matches_scalar(sse41::scan_rev, 0); - run_random_trailing(sse41::scan_rev, 0); -} - -#[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") -))] -#[test] -fn sse41_trailing_matches_scalar_ones() { - assert_trailing_backend_matches_scalar(sse41::scan_rev, !0); - run_random_trailing(sse41::scan_rev, !0); -} - -#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] -#[test] -fn neon_trailing_matches_scalar_zeros() { - assert_trailing_backend_matches_scalar(neon::scan_rev, 0); - run_random_trailing(neon::scan_rev, 0); -} - -#[cfg(all(target_arch = "aarch64", target_feature = "neon"))] -#[test] -fn neon_trailing_matches_scalar_ones() { - assert_trailing_backend_matches_scalar(neon::scan_rev, !0); - run_random_trailing(neon::scan_rev, !0); -} diff --git a/src/traits/bits_eq/funcs_for_eq_words_aligned_core.rs b/src/traits/bits_eq/funcs_for_eq_words_aligned_core.rs deleted file mode 100644 index 80b7fd0..0000000 --- a/src/traits/bits_eq/funcs_for_eq_words_aligned_core.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! SIMD word-level equality — compares LANES u64 words at once -//! via `cmpeq` + `movemask`. - -use crate::SMALL_WORDS; - -// --------------------------------------------------------------------------- -// Entry point -// --------------------------------------------------------------------------- - -/// Returns `true` if the first `count` words of `src` match `other`. -/// -/// Dispatches to the best available SIMD backend at compile time. -/// Short inputs fall back to scalar. -#[inline] -pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool { - if count < SMALL_WORDS { - for i in 0..count { - if src[i] != other[i] { - return false; - } - } - return true; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] - { - return unsafe { avx2::eq_words(src, other, count) }; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] - { - return unsafe { sse41::eq_words(src, other, count) }; - } - - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return unsafe { neon::eq_words(src, other, count) }; - } - - #[allow(unused)] - { - for i in 0..count { - if src[i] != other[i] { - return false; - } - } - true - } -} - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod avx2 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; - - #[target_feature(enable = "avx2")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 4 <= len { - let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; - let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; - let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; - if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { - return false; - } - i += 4; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} - -#[allow(unused)] -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -mod sse41 { - #[cfg(target_arch = "x86")] - use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - #[cfg(target_arch = "x86_64")] - use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; - - #[target_feature(enable = "sse4.1")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 2 <= len { - let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; - let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; - let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; - if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { - return false; - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} - -#[allow(unused)] -#[cfg(target_arch = "aarch64")] -mod neon { - use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; - - #[target_feature(enable = "neon")] - pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { - let mut i = 0; - while i + 2 <= len { - let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; - let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; - let cmp = unsafe { vceqq_u64(a, b) }; - if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { - return false; - } - i += 2; - } - while i < len { - if src[i] != other[i] { - return false; - } - i += 1; - } - true - } -} - -#[cfg(test)] -mod tests_for_backend_equivalence; diff --git a/src/traits/bits_eq/impls_for_u64_slice.rs b/src/traits/bits_eq/impls_for_u64_slice.rs deleted file mode 100644 index 43fed73..0000000 --- a/src/traits/bits_eq/impls_for_u64_slice.rs +++ /dev/null @@ -1,18 +0,0 @@ -use crate::WORD_BITS; - -use super::BitsEq; - -impl BitsEq for [u64] { - #[inline] - fn eq_words(&self, other: &[u64], count: usize, offset: usize) -> bool { - let shift = offset % WORD_BITS; - let base = offset / WORD_BITS; - let sw = &self[base..]; - - if shift == 0 { - super::funcs_for_eq_words_aligned_core::eq_words_aligned(sw, other, count) - } else { - super::funcs_for_eq_words_unaligned_core::eq_words_unaligned(sw, other, count, shift) - } - } -} diff --git a/src/traits/bits_ord/impls_for_u64_slice.rs b/src/traits/bits_ord/impls_for_u64_slice.rs deleted file mode 100644 index bc3d153..0000000 --- a/src/traits/bits_ord/impls_for_u64_slice.rs +++ /dev/null @@ -1,22 +0,0 @@ -use core::cmp::Ordering; - -use crate::WORD_BITS; - -use super::BitsOrd; -use super::funcs_for_cmp_aligned_core; -use super::funcs_for_cmp_unaligned_core; - -impl BitsOrd for [u64] { - #[inline] - fn cmp_words(&self, other: &[u64], count: usize, offset: usize) -> Option { - let shift = offset % WORD_BITS; - let base = offset / WORD_BITS; - let sw = &self[base..]; - - if shift == 0 { - funcs_for_cmp_aligned_core::cmp_aligned_words(sw, other, count) - } else { - funcs_for_cmp_unaligned_core::cmp_unaligned_words(sw, other, count, shift) - } - } -} diff --git a/src/traits/bit_ord.rs b/src/traits/word_ord.rs similarity index 92% rename from src/traits/bit_ord.rs rename to src/traits/word_ord.rs index 803f42e..c18bbad 100644 --- a/src/traits/bit_ord.rs +++ b/src/traits/word_ord.rs @@ -4,11 +4,11 @@ use core::cmp::Ordering; /// /// This is the word-level primitive used by [`BitsOrd`](super::BitsOrd) /// to resolve ordering once the first differing word has been found. -pub(crate) trait BitOrd { +pub(crate) trait WordOrd { fn bitwise_cmp(self, other: Self) -> Ordering; } -impl BitOrd for u64 { +impl WordOrd for u64 { #[inline] fn bitwise_cmp(self, other: u64) -> Ordering { debug_assert!(self != other); diff --git a/src/traits/bits_arith.rs b/src/traits/words_arith.rs similarity index 59% rename from src/traits/bits_arith.rs rename to src/traits/words_arith.rs index bb1e6e5..a0e15a2 100644 --- a/src/traits/bits_arith.rs +++ b/src/traits/words_arith.rs @@ -4,7 +4,7 @@ use alloc::vec::Vec; /// /// Owned methods allocate a new `Vec` and return it. Assign methods /// operate in place. -pub(crate) trait BitsArith { +pub(crate) trait WordsArith { /// Returns `self & rhs`, allocating a new result. fn and(&self, rhs: &[u64]) -> Vec; /// Performs `self &= rhs` in place. @@ -42,41 +42,9 @@ pub(crate) trait BitsArith { /// Performs `self >>= amount` (zero-fill) in place, masking with /// `bit_len`. fn shr_assign(&mut self, bit_len: usize, amount: usize); - - /// Returns the number of ones (set bits) in the first `bit_len` bits. - /// - /// Bits beyond `bit_len` are assumed to already be zero (masked by - /// prior calls to [`BitsEdit::mask_unused_bits`]). - fn count_ones(&self, bit_len: usize) -> usize; - - /// Returns the number of consecutive zero words at the start of `self`. - /// - /// All words up to (but not including) the returned index are zero. If - /// the return value equals `self.len()`, every word is zero. - fn leading_zero_words(&self) -> usize; - - /// Returns the number of consecutive all-ones words at the start of - /// `self`. - /// - /// All words up to (but not including) the returned index are all-ones. - /// If the return value equals `self.len()`, every word is all-ones. - fn leading_one_words(&self) -> usize; - - /// Returns the number of consecutive zero words at the **end** of `self`. - /// - /// All words from `self.len() - count` onwards are zero. - fn trailing_zero_words(&self) -> usize; - - /// Returns the number of consecutive all-ones words at the **end** of - /// `self`. - /// - /// All words from `self.len() - count` onwards are all-ones. - fn trailing_one_words(&self) -> usize; } pub(crate) mod funcs_for_binary_core; -pub(crate) mod funcs_for_count_ones; -pub(crate) mod funcs_for_leading_value_words; pub(crate) mod funcs_for_not_core; pub(crate) mod funcs_for_shl_core; pub(crate) mod funcs_for_shr_core; diff --git a/src/traits/bits_arith/funcs_for_binary_core.rs b/src/traits/words_arith/funcs_for_binary_core.rs similarity index 79% rename from src/traits/bits_arith/funcs_for_binary_core.rs rename to src/traits/words_arith/funcs_for_binary_core.rs index 6b5a546..7b5c256 100644 --- a/src/traits/bits_arith/funcs_for_binary_core.rs +++ b/src/traits/words_arith/funcs_for_binary_core.rs @@ -56,44 +56,79 @@ pub(super) fn assign(lhs: &mut [u64], rhs: &[u64]) { /// - `dst` must not overlap `rhs`. #[inline] unsafe fn dispatch(dst: *mut u64, lhs: *const u64, rhs: *const u64, len: usize) { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ────────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words::(dst, lhs, rhs, len) }; - return; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. + unsafe { avx2::words::(dst, lhs, rhs, len) }; + return; + } + if f.sse2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. + unsafe { sse2::words::(dst, lhs, rhs, len) }; + return; + } + } + // Non-x86 fallbacks: NEON (aarch64), scalar. + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. + unsafe { neon::words::(dst, lhs, rhs, len) }; + return; + } + #[allow(unused)] + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + unsafe { + scalar::words::(dst, lhs, rhs, len); + } } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: existing #[cfg] cascade ─────────── + #[cfg(feature = "compile-time-dispatch")] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words::(dst, lhs, rhs, len) }; - return; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words::(dst, lhs, rhs, len) }; + return; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words::(dst, lhs, rhs, len) }; - return; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words::(dst, lhs, rhs, len) }; + return; + } - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words::(dst, lhs, rhs, len); + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words::(dst, lhs, rhs, len) }; + return; + } + + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words::(dst, lhs, rhs, len); + } } } diff --git a/src/traits/bits_arith/funcs_for_binary_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_binary_core/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_arith/funcs_for_binary_core/tests_for_backend_equivalence.rs rename to src/traits/words_arith/funcs_for_binary_core/tests_for_backend_equivalence.rs diff --git a/src/traits/bits_arith/funcs_for_not_core.rs b/src/traits/words_arith/funcs_for_not_core.rs similarity index 78% rename from src/traits/bits_arith/funcs_for_not_core.rs rename to src/traits/words_arith/funcs_for_not_core.rs index a720e92..edf5088 100644 --- a/src/traits/bits_arith/funcs_for_not_core.rs +++ b/src/traits/words_arith/funcs_for_not_core.rs @@ -52,44 +52,73 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize) { /// - be exactly equal to `src`. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u64, len: usize) { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + #[cfg(not(feature = "compile-time-dispatch"))] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words(dst, src, len) }; - return; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words(dst, src, len) }; - return; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. + unsafe { avx2::words(dst, src, len) }; + return; + } + if f.sse2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. + unsafe { sse2::words(dst, src, len) }; + return; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. + unsafe { neon::words(dst, src, len) }; + return; + } + #[allow(unused)] + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + unsafe { + scalar::words(dst, src, len) + }; } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + #[cfg(feature = "compile-time-dispatch")] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words(dst, src, len) }; - return; - } - - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, len); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words(dst, src, len) }; + return; + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words(dst, src, len) }; + return; + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words(dst, src, len) }; + return; + } + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words(dst, src, len) + }; } } diff --git a/src/traits/bits_arith/funcs_for_not_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_not_core/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_arith/funcs_for_not_core/tests_for_backend_equivalence.rs rename to src/traits/words_arith/funcs_for_not_core/tests_for_backend_equivalence.rs diff --git a/src/traits/bits_arith/funcs_for_shl_core.rs b/src/traits/words_arith/funcs_for_shl_core.rs similarity index 84% rename from src/traits/bits_arith/funcs_for_shl_core.rs rename to src/traits/words_arith/funcs_for_shl_core.rs index 9f492c3..75ca055 100644 --- a/src/traits/bits_arith/funcs_for_shl_core.rs +++ b/src/traits/words_arith/funcs_for_shl_core.rs @@ -59,44 +59,73 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize, amount: usize) { /// - be exactly equal to `src`. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + #[cfg(not(feature = "compile-time-dispatch"))] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words(dst, src, word_len, amount) }; - return; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words(dst, src, word_len, amount) }; - return; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. + unsafe { avx2::words(dst, src, word_len, amount) }; + return; + } + if f.sse2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. + unsafe { sse2::words(dst, src, word_len, amount) }; + return; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. + unsafe { neon::words(dst, src, word_len, amount) }; + return; + } + #[allow(unused)] + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + unsafe { + scalar::words(dst, src, word_len, amount) + }; } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + #[cfg(feature = "compile-time-dispatch")] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words(dst, src, word_len, amount) }; - return; - } - - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, word_len, amount); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words(dst, src, word_len, amount) }; + return; + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words(dst, src, word_len, amount) }; + return; + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words(dst, src, word_len, amount) }; + return; + } + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words(dst, src, word_len, amount) + }; } } diff --git a/src/traits/bits_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs rename to src/traits/words_arith/funcs_for_shl_core/tests_for_backend_equivalence.rs diff --git a/src/traits/bits_arith/funcs_for_shr_core.rs b/src/traits/words_arith/funcs_for_shr_core.rs similarity index 85% rename from src/traits/bits_arith/funcs_for_shr_core.rs rename to src/traits/words_arith/funcs_for_shr_core.rs index ad8f59e..020e2fa 100644 --- a/src/traits/bits_arith/funcs_for_shr_core.rs +++ b/src/traits/words_arith/funcs_for_shr_core.rs @@ -59,44 +59,73 @@ pub(super) fn assign(bits: &mut [u64], bit_len: usize, amount: usize) { /// - be exactly equal to `src`. #[inline] unsafe fn dispatch(dst: *mut u64, src: *const u64, word_len: usize, amount: usize) { - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + #[cfg(not(feature = "compile-time-dispatch"))] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - unsafe { avx2::words(dst, src, word_len, amount) }; - return; - } - - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] - { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSE2 is enabled. - unsafe { sse2::words(dst, src, word_len, amount) }; - return; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. AVX2 availability was confirmed by CPUID. + unsafe { avx2::words(dst, src, word_len, amount) }; + return; + } + if f.sse2 { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. SSE2 availability was confirmed by CPUID. + unsafe { sse2::words(dst, src, word_len, amount) }; + return; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: caller guarantees `dst`/`src` pointer validity and word length. NEON availability was confirmed by CPUID. + unsafe { neon::words(dst, src, word_len, amount) }; + return; + } + #[allow(unused)] + // SAFETY: caller guarantees pointer validity. Scalar backend is always safe. + unsafe { + scalar::words(dst, src, word_len, amount) + }; } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + #[cfg(feature = "compile-time-dispatch")] { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - unsafe { neon::words(dst, src, word_len, amount) }; - return; - } - - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::words(dst, src, word_len, amount); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when AVX2 is enabled. + unsafe { avx2::words(dst, src, word_len, amount) }; + return; + } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when SSE2 is enabled. + unsafe { sse2::words(dst, src, word_len, amount) }; + return; + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: + // - Forwarded from `dispatch`'s safety contract. + // - This branch is compiled only when NEON is enabled. + unsafe { neon::words(dst, src, word_len, amount) }; + return; + } + #[allow(unused)] + // SAFETY: Forwarded from `dispatch`'s safety contract. + unsafe { + scalar::words(dst, src, word_len, amount) + }; } } diff --git a/src/traits/bits_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs b/src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs rename to src/traits/words_arith/funcs_for_shr_core/tests_for_backend_equivalence.rs diff --git a/src/traits/bits_arith/impls_for_u64_slice.rs b/src/traits/words_arith/impls_for_u64_slice.rs similarity index 66% rename from src/traits/bits_arith/impls_for_u64_slice.rs rename to src/traits/words_arith/impls_for_u64_slice.rs index 90bcfda..a9ef10e 100644 --- a/src/traits/bits_arith/impls_for_u64_slice.rs +++ b/src/traits/words_arith/impls_for_u64_slice.rs @@ -1,14 +1,12 @@ use alloc::vec::Vec; -use super::BitsArith; +use super::WordsArith; use super::funcs_for_binary_core::{OP_AND, OP_OR, OP_XOR, assign, owned}; -use super::funcs_for_count_ones; -use super::funcs_for_leading_value_words; use super::funcs_for_not_core; use super::funcs_for_shl_core; use super::funcs_for_shr_core; -impl BitsArith for [u64] { +impl WordsArith for [u64] { #[inline] fn and(&self, rhs: &[u64]) -> Vec { owned::(self, rhs) @@ -68,29 +66,4 @@ impl BitsArith for [u64] { fn shr_assign(&mut self, bit_len: usize, amount: usize) { funcs_for_shr_core::assign(self, bit_len, amount); } - - #[inline] - fn count_ones(&self, bit_len: usize) -> usize { - funcs_for_count_ones::count_ones(self, bit_len) - } - - #[inline] - fn leading_zero_words(&self) -> usize { - funcs_for_leading_value_words::leading_value_words(self, 0) - } - - #[inline] - fn leading_one_words(&self) -> usize { - funcs_for_leading_value_words::leading_value_words(self, !0) - } - - #[inline] - fn trailing_zero_words(&self) -> usize { - funcs_for_leading_value_words::trailing_value_words(self, 0) - } - - #[inline] - fn trailing_one_words(&self) -> usize { - funcs_for_leading_value_words::trailing_value_words(self, !0) - } } diff --git a/src/traits/bits_edit.rs b/src/traits/words_edit.rs similarity index 85% rename from src/traits/bits_edit.rs rename to src/traits/words_edit.rs index d18a336..b5a28eb 100644 --- a/src/traits/bits_edit.rs +++ b/src/traits/words_edit.rs @@ -3,7 +3,7 @@ /// All bit indices are logical positions in the flat bit string (not word /// indices). Word boundaries are handled transparently — callers work with /// logical bit positions and the trait translates them to word-level accesses. -pub(crate) trait BitsEdit { +pub(crate) trait WordsEdit { /// Zeros words beyond `word_len(len)` and masks the last used word. /// /// `len` is the **total** bit-string length. Words whose bits are entirely @@ -23,7 +23,9 @@ pub(crate) trait BitsEdit { /// When `bit_start` is word-aligned the result comes from a single word. /// Otherwise it spans two consecutive words, shifting and combining them. /// Bits past the end of `self` are silently treated as zero. - fn read_word_at(&self, bit_start: usize) -> u64; + /// When `WORD_ALIGNED` is `true`, `bit_start % WORD_BITS == 0` is + /// guaranteed — the cross-word stitch is eliminated at compile time. + fn read_word_at(&self, bit_start: usize) -> u64; /// Writes the low `len` bits of `value` into `self` starting at `bit_start`. /// @@ -31,7 +33,9 @@ pub(crate) trait BitsEdit { /// bitwise OR). `len` is clamped to `WORD_BITS` via [`low_mask`]; callers /// must ensure `len <= WORD_BITS`. When `bit_start` is not word-aligned /// the value is split across two consecutive words. - fn write_word_at(&mut self, bit_start: usize, value: u64, len: usize); + /// When `WORD_ALIGNED` is `true`, `bit_start % WORD_BITS == 0` is + /// guaranteed — the cross-word spill is eliminated at compile time. + fn write_word_at(&mut self, bit_start: usize, value: u64, len: usize); /// Captures a snapshot of `len` bits starting at `start` for deferred /// paste via [`BitsCopied::paste_to`]. diff --git a/src/traits/bits_edit/bits_copied.rs b/src/traits/words_edit/bits_copied.rs similarity index 90% rename from src/traits/bits_edit/bits_copied.rs rename to src/traits/words_edit/bits_copied.rs index 679f9e0..ce59bf7 100644 --- a/src/traits/bits_edit/bits_copied.rs +++ b/src/traits/words_edit/bits_copied.rs @@ -1,13 +1,13 @@ use crate::WORD_BITS; -use super::BitsEdit; +use super::WordsEdit; mod funcs_for_copy_words_core; mod funcs_for_copy_words_shifted_core; /// Zero-cost curried copy: captures a source snapshot for deferred paste. /// -/// Created by [`BitsEdit::copy_bits`] and materialized via +/// Created by [`WordsEdit::copy_bits`] and materialized via /// [`BitsCopied::paste_to`]. pub(crate) struct BitsCopied<'a> { pub(crate) src: &'a [u64], @@ -100,16 +100,18 @@ impl BitsCopied<'_> { // then OR'd into two dst words. else { for i in 0..full_words { - let chunk = self.src.read_word_at(self.src_start + i * WORD_BITS); - dst.write_word_at(dst_start + i * WORD_BITS, chunk, WORD_BITS); + let chunk = self + .src + .read_word_at::(self.src_start + i * WORD_BITS); + dst.write_word_at::(dst_start + i * WORD_BITS, chunk, WORD_BITS); } } // Partial last word — uniform scalar handling for all paths. if remainder_bits > 0 { let offset = full_words * WORD_BITS; - let chunk = self.src.read_word_at(self.src_start + offset); - dst.write_word_at(dst_start + offset, chunk, remainder_bits); + let chunk = self.src.read_word_at::(self.src_start + offset); + dst.write_word_at::(dst_start + offset, chunk, remainder_bits); } } } diff --git a/src/traits/bits_edit/bits_copied/funcs_for_copy_words_core.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_core.rs similarity index 100% rename from src/traits/bits_edit/bits_copied/funcs_for_copy_words_core.rs rename to src/traits/words_edit/bits_copied/funcs_for_copy_words_core.rs diff --git a/src/traits/bits_edit/bits_copied/funcs_for_copy_words_core/tests_for_backend_equivalence.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_core/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_edit/bits_copied/funcs_for_copy_words_core/tests_for_backend_equivalence.rs rename to src/traits/words_edit/bits_copied/funcs_for_copy_words_core/tests_for_backend_equivalence.rs diff --git a/src/traits/bits_edit/bits_copied/funcs_for_copy_words_shifted_core.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs similarity index 55% rename from src/traits/bits_edit/bits_copied/funcs_for_copy_words_shifted_core.rs rename to src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs index 34540d2..4cb86d8 100644 --- a/src/traits/bits_edit/bits_copied/funcs_for_copy_words_shifted_core.rs +++ b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core.rs @@ -30,35 +30,73 @@ pub(super) fn copy_words_shifted(dst: &mut [u64], src: &[u64], count: usize, shi return; } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - unsafe { avx2::copy_words_shifted(dst, src, count, shift) }; - return; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). AVX2 availability was confirmed by CPUID. + unsafe { avx2::copy_words_shifted(dst, src, count, shift) }; + return; + } + if f.sse2 { + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). SSE2 availability was confirmed by CPUID. + unsafe { sse2::copy_words_shifted(dst, src, count, shift) }; + return; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). NEON availability is confirmed at compile time by `target_feature = "neon"`. + unsafe { neon::copy_words_shifted(dst, src, count, shift) }; + return; + } + #[allow(unused)] + { + for i in 0..count { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + } + } } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse2", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - unsafe { sse2::copy_words_shifted(dst, src, count, shift) }; - return; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). AVX2 availability is confirmed at compile time. + unsafe { avx2::copy_words_shifted(dst, src, count, shift) }; + return; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - unsafe { neon::copy_words_shifted(dst, src, count, shift) }; - return; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2", + not(target_feature = "avx2") + ))] + { + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). SSE2 availability is confirmed at compile time. + unsafe { sse2::copy_words_shifted(dst, src, count, shift) }; + return; + } - #[allow(unused)] - { - for i in 0..count { - dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `dst`/`src` are valid for `count` words (caller guarantee). NEON availability is confirmed at compile time. + unsafe { neon::copy_words_shifted(dst, src, count, shift) }; + return; + } + + #[allow(unused)] + { + for i in 0..count { + dst[i] = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); + } } } } @@ -90,15 +128,23 @@ mod avx2 { len: usize, shift: usize, ) { + // SAFETY: `#[target_feature(enable = "avx2")]` guarantees the CPU supports AVX2; the `unsafe fn` contract guarantees pointer validity. let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: Same as above. let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; let mut i = 0; while i + 4 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm256_loadu_si256` uses unaligned loads so alignment is not required. let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: Same as above. let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + // SAFETY: Pure register operations; no memory access. let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + // SAFETY: Same as above. let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + // SAFETY: Same as above. let window = unsafe { _mm256_or_si256(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm256_storeu_si256` uses unaligned stores. unsafe { _mm256_storeu_si256(dst.as_mut_ptr().add(i).cast::<__m256i>(), window) }; i += 4; } @@ -136,15 +182,23 @@ mod sse2 { len: usize, shift: usize, ) { + // SAFETY: `#[target_feature(enable = "sse2")]` guarantees the CPU supports SSE2; the `unsafe fn` contract guarantees pointer validity. let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: Same as above. let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; let mut i = 0; while i + 2 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee); `_mm_loadu_si128` uses unaligned loads so alignment is not required. let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: Same as above. let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + // SAFETY: Pure register operations; no memory access. let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + // SAFETY: Same as above. let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + // SAFETY: Same as above. let window = unsafe { _mm_or_si128(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee); `_mm_storeu_si128` uses unaligned stores. unsafe { _mm_storeu_si128(dst.as_mut_ptr().add(i).cast::<__m128i>(), window) }; i += 2; } @@ -172,16 +226,24 @@ mod neon { len: usize, shift: usize, ) { + // SAFETY: `#[target_feature(enable = "neon")]` guarantees the CPU supports NEON; the `unsafe fn` contract guarantees pointer validity. let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + // SAFETY: Same as above. let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; let mut i = 0; while i + 2 <= len { + // SAFETY: `src` pointers are valid for `len + 1` words (caller guarantee). let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: Same as above. let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; + // SAFETY: Pure register operations; no memory access. let lo = unsafe { vshlq_u64(w0, neg_shift) }; + // SAFETY: Same as above. let hi = unsafe { vshlq_u64(w1, pos_shift) }; + // SAFETY: Same as above. let window = unsafe { vorrq_u64(lo, hi) }; + // SAFETY: `dst` pointer is valid for `len` words (caller guarantee). unsafe { vst1q_u64(dst.as_mut_ptr().add(i), window) }; i += 2; } diff --git a/src/traits/bits_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs b/src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs rename to src/traits/words_edit/bits_copied/funcs_for_copy_words_shifted_core/tests_for_backend_equivalence.rs diff --git a/src/traits/bits_edit/bits_copied/tests_for_copy.rs b/src/traits/words_edit/bits_copied/tests_for_copy.rs similarity index 97% rename from src/traits/bits_edit/bits_copied/tests_for_copy.rs rename to src/traits/words_edit/bits_copied/tests_for_copy.rs index bd85504..e719e74 100644 --- a/src/traits/bits_edit/bits_copied/tests_for_copy.rs +++ b/src/traits/words_edit/bits_copied/tests_for_copy.rs @@ -138,12 +138,12 @@ fn scalar_paste(src: &[u64], src_start: usize, len: usize, dst: &mut [u64], dst_ let full_words = len / WORD_BITS; let rem = len % WORD_BITS; for i in 0..full_words { - let chunk = src.read_word_at(src_start + i * WORD_BITS); - dst.write_word_at(dst_start + i * WORD_BITS, chunk, WORD_BITS); + let chunk = src.read_word_at::(src_start + i * WORD_BITS); + dst.write_word_at::(dst_start + i * WORD_BITS, chunk, WORD_BITS); } if rem > 0 { - let chunk = src.read_word_at(src_start + full_words * WORD_BITS); - dst.write_word_at(dst_start + full_words * WORD_BITS, chunk, rem); + let chunk = src.read_word_at::(src_start + full_words * WORD_BITS); + dst.write_word_at::(dst_start + full_words * WORD_BITS, chunk, rem); } } diff --git a/src/traits/bits_edit/impls_for_u64_slice.rs b/src/traits/words_edit/impls_for_u64_slice.rs similarity index 87% rename from src/traits/bits_edit/impls_for_u64_slice.rs rename to src/traits/words_edit/impls_for_u64_slice.rs index d0469d6..91739cb 100644 --- a/src/traits/bits_edit/impls_for_u64_slice.rs +++ b/src/traits/words_edit/impls_for_u64_slice.rs @@ -1,9 +1,9 @@ use crate::{WORD_BITS, last_word_mask, low_mask, word_len}; use super::BitsCopied; -use super::BitsEdit; +use super::WordsEdit; -impl BitsEdit for [u64] { +impl WordsEdit for [u64] { /// Masks the last used word with /// [`last_word_mask(len)`](last_word_mask), /// and zeros any words beyond `word_len(len)`. @@ -44,17 +44,21 @@ impl BitsEdit for [u64] { /// intra-word offset. When the read crosses a word boundary the high part /// is stitched in from the next word. #[inline] - fn read_word_at(&self, bit_start: usize) -> u64 { + fn read_word_at(&self, bit_start: usize) -> u64 { let word = bit_start / WORD_BITS; - let shift = bit_start % WORD_BITS; - let lo = self.get(word).copied().unwrap_or(0) >> shift; - - if shift == 0 { - lo + if WORD_ALIGNED { + self.get(word).copied().unwrap_or(0) } else { - let hi = self.get(word + 1).copied().unwrap_or(0); - lo | (hi << (WORD_BITS - shift)) + let shift = bit_start % WORD_BITS; + let lo = self.get(word).copied().unwrap_or(0) >> shift; + + if shift == 0 { + lo + } else { + let hi = self.get(word + 1).copied().unwrap_or(0); + lo | (hi << (WORD_BITS - shift)) + } } } @@ -62,15 +66,24 @@ impl BitsEdit for [u64] { /// `self` at `bit_start`. When the write crosses a word boundary the high /// part spills into the next word. #[inline] - fn write_word_at(&mut self, bit_start: usize, value: u64, len: usize) { + fn write_word_at( + &mut self, + bit_start: usize, + value: u64, + len: usize, + ) { let value = value & low_mask(len); let word = bit_start / WORD_BITS; - let shift = bit_start % WORD_BITS; - self[word] |= value << shift; + if WORD_ALIGNED { + self[word] |= value; + } else { + let shift = bit_start % WORD_BITS; + self[word] |= value << shift; - if shift != 0 && word + 1 < self.len() { - self[word + 1] |= value >> (WORD_BITS - shift); + if shift != 0 && word + 1 < self.len() { + self[word + 1] |= value >> (WORD_BITS - shift); + } } } diff --git a/src/traits/bits_edit/impls_for_u64_slice/tests_for_bit_at.rs b/src/traits/words_edit/impls_for_u64_slice/tests_for_bit_at.rs similarity index 100% rename from src/traits/bits_edit/impls_for_u64_slice/tests_for_bit_at.rs rename to src/traits/words_edit/impls_for_u64_slice/tests_for_bit_at.rs diff --git a/src/traits/bits_edit/impls_for_u64_slice/tests_for_mask_unused.rs b/src/traits/words_edit/impls_for_u64_slice/tests_for_mask_unused.rs similarity index 100% rename from src/traits/bits_edit/impls_for_u64_slice/tests_for_mask_unused.rs rename to src/traits/words_edit/impls_for_u64_slice/tests_for_mask_unused.rs diff --git a/src/traits/bits_edit/impls_for_u64_slice/tests_for_read_chunk.rs b/src/traits/words_edit/impls_for_u64_slice/tests_for_read_chunk.rs similarity index 61% rename from src/traits/bits_edit/impls_for_u64_slice/tests_for_read_chunk.rs rename to src/traits/words_edit/impls_for_u64_slice/tests_for_read_chunk.rs index 677767a..1d3db05 100644 --- a/src/traits/bits_edit/impls_for_u64_slice/tests_for_read_chunk.rs +++ b/src/traits/words_edit/impls_for_u64_slice/tests_for_read_chunk.rs @@ -5,38 +5,38 @@ use crate::WORD_BITS; fn returns_word_when_bit_start_is_word_aligned() { let src = [0x0123_4567_89ab_cdef, 0xfedc_ba98_7654_3210]; - assert_eq!(src.read_word_at(0), src[0]); - assert_eq!(src.read_word_at(WORD_BITS), src[1]); + assert_eq!(src.read_word_at::(0), src[0]); + assert_eq!(src.read_word_at::(WORD_BITS), src[1]); } #[test] fn returns_zero_when_aligned_start_is_past_source() { let src = [0x0123_4567_89ab_cdef]; - assert_eq!(src.read_word_at(WORD_BITS), 0); - assert_eq!(src.read_word_at(WORD_BITS * 2), 0); + assert_eq!(src.read_word_at::(WORD_BITS), 0); + assert_eq!(src.read_word_at::(WORD_BITS * 2), 0); } #[test] fn reads_shifted_chunk_within_single_word() { let src = [0b1011_0100u64]; - assert_eq!(src.read_word_at(2), 0b10_1101); - assert_eq!(src.read_word_at(4), 0b1011); + assert_eq!(src.read_word_at::(2), 0b10_1101); + assert_eq!(src.read_word_at::(4), 0b1011); } #[test] fn reads_chunk_across_word_boundary() { let src = [1u64 << (WORD_BITS - 1), 0b1010u64]; - assert_eq!(src.read_word_at(WORD_BITS - 1), 0b10101); + assert_eq!(src.read_word_at::(WORD_BITS - 1), 0b10101); } #[test] fn treats_missing_high_word_as_zero() { let src = [1u64 << (WORD_BITS - 1)]; - assert_eq!(src.read_word_at(WORD_BITS - 1), 1); + assert_eq!(src.read_word_at::(WORD_BITS - 1), 1); } #[test] @@ -46,7 +46,7 @@ fn preserves_full_chunk_layout_across_boundary() { let shift = 16; let expected = (src[0] >> shift) | (src[1] << (WORD_BITS - shift)); - assert_eq!(src.read_word_at(shift), expected); + assert_eq!(src.read_word_at::(shift), expected); } #[test] @@ -55,5 +55,5 @@ fn reads_from_later_word_with_unaligned_start() { let expected = (src[1] >> 4) | (src[2] << (WORD_BITS - 4)); - assert_eq!(src.read_word_at(WORD_BITS + 4), expected); + assert_eq!(src.read_word_at::(WORD_BITS + 4), expected); } diff --git a/src/traits/bits_edit/impls_for_u64_slice/tests_for_set_bit.rs b/src/traits/words_edit/impls_for_u64_slice/tests_for_set_bit.rs similarity index 100% rename from src/traits/bits_edit/impls_for_u64_slice/tests_for_set_bit.rs rename to src/traits/words_edit/impls_for_u64_slice/tests_for_set_bit.rs diff --git a/src/traits/bits_edit/impls_for_u64_slice/tests_for_write_chunk.rs b/src/traits/words_edit/impls_for_u64_slice/tests_for_write_chunk.rs similarity index 73% rename from src/traits/bits_edit/impls_for_u64_slice/tests_for_write_chunk.rs rename to src/traits/words_edit/impls_for_u64_slice/tests_for_write_chunk.rs index 324ae9a..668fedf 100644 --- a/src/traits/bits_edit/impls_for_u64_slice/tests_for_write_chunk.rs +++ b/src/traits/words_edit/impls_for_u64_slice/tests_for_write_chunk.rs @@ -5,7 +5,7 @@ use crate::WORD_BITS; fn writes_aligned_chunk_into_single_word() { let mut dst = [0u64; 2]; - dst.write_word_at(0, 0b1011, 4); + dst.write_word_at::(0, 0b1011, 4); assert_eq!(dst[0], 0b1011); assert_eq!(dst[1], 0); @@ -15,7 +15,7 @@ fn writes_aligned_chunk_into_single_word() { fn masks_value_to_requested_len() { let mut dst = [0u64; 1]; - dst.write_word_at(4, u64::MAX, 3); + dst.write_word_at::(4, u64::MAX, 3); assert_eq!(dst[0], 0b111 << 4); } @@ -24,7 +24,7 @@ fn masks_value_to_requested_len() { fn writes_unaligned_chunk_inside_single_word() { let mut dst = [0u64; 1]; - dst.write_word_at(5, 0b10101, 5); + dst.write_word_at::(5, 0b10101, 5); assert_eq!(dst[0], 0b10101 << 5); } @@ -33,7 +33,7 @@ fn writes_unaligned_chunk_inside_single_word() { fn writes_chunk_across_word_boundary() { let mut dst = [0u64; 2]; - dst.write_word_at(WORD_BITS - 2, 0b1011, 4); + dst.write_word_at::(WORD_BITS - 2, 0b1011, 4); assert_eq!(dst[0], 0b11u64 << (WORD_BITS - 2)); assert_eq!(dst[1], 0b10); @@ -43,7 +43,7 @@ fn writes_chunk_across_word_boundary() { fn does_not_write_past_dst_when_crossing_boundary_without_next_word() { let mut dst = [0u64; 1]; - dst.write_word_at(WORD_BITS - 2, 0b1011, 4); + dst.write_word_at::(WORD_BITS - 2, 0b1011, 4); assert_eq!(dst[0], 0b11u64 << (WORD_BITS - 2)); } @@ -52,7 +52,7 @@ fn does_not_write_past_dst_when_crossing_boundary_without_next_word() { fn ors_into_existing_bits_instead_of_overwriting() { let mut dst = [0b1000u64]; - dst.write_word_at(0, 0b0011, 2); + dst.write_word_at::(0, 0b0011, 2); assert_eq!(dst[0], 0b1011); } @@ -61,7 +61,7 @@ fn ors_into_existing_bits_instead_of_overwriting() { fn zero_len_writes_nothing() { let mut dst = [0b1010u64]; - dst.write_word_at(1, u64::MAX, 0); + dst.write_word_at::(1, u64::MAX, 0); assert_eq!(dst[0], 0b1010); } @@ -70,7 +70,7 @@ fn zero_len_writes_nothing() { fn full_word_len_preserves_all_value_bits() { let mut dst = [0u64; 1]; - dst.write_word_at(0, u64::MAX, WORD_BITS); + dst.write_word_at::(0, u64::MAX, WORD_BITS); assert_eq!(dst[0], u64::MAX); } diff --git a/src/traits/bits_eq.rs b/src/traits/words_eq.rs similarity index 57% rename from src/traits/bits_eq.rs rename to src/traits/words_eq.rs index 98c49c0..f35829d 100644 --- a/src/traits/bits_eq.rs +++ b/src/traits/words_eq.rs @@ -6,13 +6,25 @@ /// - `offset % WORD_BITS != 0` — unaligned shifted-window, uses [`funcs_for_eq_words_unaligned_core`]. /// /// Short inputs fall back to scalar in both backends. -pub(crate) trait BitsEq { +pub(crate) trait WordsEq { /// Returns `true` if `other` matches `self` starting at `offset` bits. /// /// `count` is the number of full `u64` words to compare (computed from /// the needle bit length). The intra-word shift and word slicing are /// derived from `offset` internally. - fn eq_words(&self, other: &[u64], count: usize, offset: usize) -> bool; + /// `self` is pre-trimmed haystack `words[base..]`. + /// `needle` is always word-aligned (pre-trimmed by the caller). + /// `full_words` is `needle_bit_len / WORD_BITS` — how many + /// complete u64 words to compare. + /// `haystack_shift` is `original_offset % WORD_BITS`. + /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is + /// guaranteed and the aligned backend is used unconditionally. + fn eq_words( + &self, + needle: &[u64], + full_words: usize, + haystack_shift: usize, + ) -> bool; } pub(crate) mod funcs_for_eq_words_aligned_core; diff --git a/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs new file mode 100644 index 0000000..d048f40 --- /dev/null +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core.rs @@ -0,0 +1,199 @@ +//! SIMD word-level equality — compares LANES u64 words at once +//! via `cmpeq` + `movemask`. + +use crate::SMALL_WORDS; + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +/// Returns `true` if the first `count` words of `src` match `other`. +/// +/// Dispatches to the best available SIMD backend at compile time. +/// Short inputs fall back to scalar. +#[inline] +pub(super) fn eq_words_aligned(src: &[u64], other: &[u64], count: usize) -> bool { + if count < SMALL_WORDS { + for i in 0..count { + if src[i] != other[i] { + return false; + } + } + return true; + } + + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] + { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. + return unsafe { avx2::eq_words(src, other, count) }; + } + if f.sse41 { + // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. + return unsafe { sse41::eq_words(src, other, count) }; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `src`/`other` are valid for `count` words. Backend feature is enabled by the `#[cfg]` gate on this block. + return unsafe { neon::eq_words(src, other, count) }; + } + #[allow(unused)] + { + for i in 0..count { + if src[i] != other[i] { + return false; + } + } + true + } + } + + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] + { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: `src`/`other` are valid for `count` words. Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { avx2::eq_words(src, other, count) }; + } + + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] + { + // SAFETY: `src`/`other` are valid for `count` words. Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { sse41::eq_words(src, other, count) }; + } + + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `src`/`other` are valid for `count` words. Backend feature is enabled by the `#[cfg]` gate on this block. + return unsafe { neon::eq_words(src, other, count) }; + } + + #[allow(unused)] + { + for i in 0..count { + if src[i] != other[i] { + return false; + } + } + true + } + } +} + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{__m256i, _mm256_cmpeq_epi64, _mm256_loadu_si256, _mm256_movemask_pd}; + + #[target_feature(enable = "avx2")] + pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 4 <= len { + // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: `_mm256_cmpeq_epi64` and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm256_cmpeq_epi64(a, b) }; + // SAFETY: same as above + if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { + return false; + } + i += 4; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true + } +} + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse41 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{__m128i, _mm_cmpeq_epi64, _mm_loadu_si128, _mm_movemask_epi8}; + + #[target_feature(enable = "sse4.1")] + pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: `_mm_cmpeq_epi64` and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. + let cmp = unsafe { _mm_cmpeq_epi64(a, b) }; + // SAFETY: same as above + if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { + return false; + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true + } +} + +#[allow(unused)] +#[cfg(target_arch = "aarch64")] +mod neon { + use core::arch::aarch64::{uint64x2_t, vceqq_u64, vgetq_lane_u64, vld1q_u64}; + + #[target_feature(enable = "neon")] + pub(super) unsafe fn eq_words(src: &[u64], other: &[u64], len: usize) -> bool { + let mut i = 0; + while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. + // Pointers `src` and `other` are valid for `len` elements (guaranteed by caller). + let a = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: same as above; load from `other`. + let b = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + // SAFETY: `vceqq_u64` and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. + let cmp = unsafe { vceqq_u64(a, b) }; + // SAFETY: same as above + if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { + return false; + } + i += 2; + } + while i < len { + if src[i] != other[i] { + return false; + } + i += 1; + } + true + } +} + +#[cfg(test)] +mod tests_for_backend_equivalence; diff --git a/src/traits/bits_eq/funcs_for_eq_words_aligned_core/tests_for_backend_equivalence.rs b/src/traits/words_eq/funcs_for_eq_words_aligned_core/tests_for_backend_equivalence.rs similarity index 90% rename from src/traits/bits_eq/funcs_for_eq_words_aligned_core/tests_for_backend_equivalence.rs rename to src/traits/words_eq/funcs_for_eq_words_aligned_core/tests_for_backend_equivalence.rs index 61a6277..4f4384a 100644 --- a/src/traits/bits_eq/funcs_for_eq_words_aligned_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_eq/funcs_for_eq_words_aligned_core/tests_for_backend_equivalence.rs @@ -24,7 +24,7 @@ proptest! { let haystack = BitString::from_bool_iter((0..h_len).map(|i| (i * 17 + 3) % 5 == 0)); let prefix = BitString::from_bool_iter((0..p_len).map(|i| (i * 17 + 3) % 5 == 0)); - let result = haystack.starts_with(prefix.as_bit_str()); + let result = haystack.starts_with_str(prefix.as_bit_str()); let expected = (0..p_len).all(|j| haystack.get(j) == prefix.get(j)); assert_eq!(result, expected); @@ -38,7 +38,7 @@ proptest! { let haystack = BitString::from_bool_iter(h_bits); let prefix = BitString::from_bool_iter(p_bits); - let result = haystack.starts_with(prefix.as_bit_str()); + let result = haystack.starts_with_str(prefix.as_bit_str()); let expected = prefix.bit_len() <= haystack.bit_len() && (0..prefix.bit_len()).all(|j| haystack.get(j) == prefix.get(j)); diff --git a/src/traits/bits_eq/funcs_for_eq_words_unaligned_core.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs similarity index 54% rename from src/traits/bits_eq/funcs_for_eq_words_unaligned_core.rs rename to src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs index 3ac73b6..4e9e885 100644 --- a/src/traits/bits_eq/funcs_for_eq_words_unaligned_core.rs +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core.rs @@ -26,38 +26,78 @@ pub(super) fn eq_words_unaligned(src: &[u64], other: &[u64], count: usize, shift return true; } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - return unsafe { avx2::eq_words_unaligned(src, other, count, shift) }; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. + return unsafe { avx2::eq_words_unaligned(src, other, count, shift) }; + } + if f.sse41 { + // SAFETY: `src`/`other` are valid for `count` words. Backend was selected via CPUID verification. + return unsafe { sse41::eq_words_unaligned(src, other, count, shift) }; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is enabled by the `#[cfg]` gate on this block. + return unsafe { neon::eq_words_unaligned(src, other, count, shift) }; + } + #[allow(unused)] + { + for i in 0..count { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } + } + true + } } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - return unsafe { sse41::eq_words_unaligned(src, other, count, shift) }; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { avx2::eq_words_unaligned(src, other, count, shift) }; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return unsafe { neon::eq_words_unaligned(src, other, count, shift) }; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] + { + // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is guaranteed by compile-time `#[cfg]` gate. + return unsafe { sse41::eq_words_unaligned(src, other, count, shift) }; + } - #[allow(unused)] - { - for i in 0..count { - let w0 = src[i]; - let w1 = src[i + 1]; - if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { - return false; + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: `src`/`other` are valid for `count+1` words (caller ensures extra word for shifting). Backend feature is enabled by the `#[cfg]` gate on this block. + return unsafe { neon::eq_words_unaligned(src, other, count, shift) }; + } + + #[allow(unused)] + { + for i in 0..count { + let w0 = src[i]; + let w1 = src[i + 1]; + if ((w0 >> shift) | (w1 << (WORD_BITS - shift))) != other[i] { + return false; + } } + true } - true } } @@ -84,17 +124,28 @@ mod avx2 { len: usize, shift: usize, ) -> bool { + // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; AVX2 is enabled by `#[target_feature]`. let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: same as above let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; let mut i = 0; while i + 4 <= len { + // SAFETY: `#[target_feature(enable = "avx2")]` ensures AVX2 is enabled. + // Pointers `src` and `other` are valid for `len+1` elements (caller guarantees extra word for window shift). let w0 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: same as above; load from `src[i+1]`. let w1 = unsafe { _mm256_loadu_si256(src.as_ptr().add(i + 1).cast::<__m256i>()) }; + // SAFETY: `_mm256_srl_epi64`, `_mm256_sll_epi64`, `_mm256_or_si256`, `_mm256_cmpeq_epi64`, and `_mm256_movemask_pd` are pure register operations; AVX2 is enabled by `#[target_feature]`. let lo = unsafe { _mm256_srl_epi64(w0, count_lo) }; + // SAFETY: same as above let hi = unsafe { _mm256_sll_epi64(w1, count_hi) }; + // SAFETY: same as above let window = unsafe { _mm256_or_si256(lo, hi) }; + // SAFETY: AVX2 is enabled; pointer `other` is valid for `len` elements. let b = unsafe { _mm256_loadu_si256(other.as_ptr().add(i).cast::<__m256i>()) }; + // SAFETY: `_mm256_cmpeq_epi64` is a pure register operation; AVX2 is enabled by `#[target_feature]`. let cmp = unsafe { _mm256_cmpeq_epi64(window, b) }; + // SAFETY: same as above if unsafe { _mm256_movemask_pd(core::mem::transmute(cmp)) } as u32 != 0b1111 { return false; } @@ -135,17 +186,28 @@ mod sse41 { len: usize, shift: usize, ) -> bool { + // SAFETY: `shift` and `WORD_BITS - shift` fit in i64; SSE4.1 is enabled by `#[target_feature]`. let count_lo = unsafe { _mm_set1_epi64x(shift as i64) }; + // SAFETY: same as above let count_hi = unsafe { _mm_set1_epi64x((WORD_BITS - shift) as i64) }; let mut i = 0; while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "sse4.1")]` ensures SSE4.1 is enabled. + // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees). let w0 = unsafe { _mm_loadu_si128(src.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: same as above; load from `src[i+1]`. let w1 = unsafe { _mm_loadu_si128(src.as_ptr().add(i + 1).cast::<__m128i>()) }; + // SAFETY: `_mm_srl_epi64`, `_mm_sll_epi64`, `_mm_or_si128`, `_mm_cmpeq_epi64`, and `_mm_movemask_epi8` are pure register operations; SSE4.1 is enabled by `#[target_feature]`. let lo = unsafe { _mm_srl_epi64(w0, count_lo) }; + // SAFETY: same as above let hi = unsafe { _mm_sll_epi64(w1, count_hi) }; + // SAFETY: same as above let window = unsafe { _mm_or_si128(lo, hi) }; + // SAFETY: SSE4.1 is enabled; pointer `other` is valid for `len` elements. let b = unsafe { _mm_loadu_si128(other.as_ptr().add(i).cast::<__m128i>()) }; + // SAFETY: `_mm_cmpeq_epi64` is a pure register operation; SSE4.1 is enabled by `#[target_feature]`. let cmp = unsafe { _mm_cmpeq_epi64(window, b) }; + // SAFETY: same as above if unsafe { _mm_movemask_epi8(cmp) } as u32 != 0xFFFF { return false; } @@ -184,26 +246,36 @@ mod neon { // shift ∈ [1, 63] → -shift ∈ [-63, -1] // WORD_BITS - shift ∈ [1, 63] let neg_shift = unsafe { vdupq_n_s64(-(shift as i64)) }; + // SAFETY: same as above let pos_shift = unsafe { vdupq_n_s64((WORD_BITS - shift) as i64) }; // Process 2 lanes (128 bits) per iteration. let mut i = 0; while i + 2 <= len { + // SAFETY: `#[target_feature(enable = "neon")]` ensures NEON is enabled. + // Pointers `src` and `other` are valid for `len+1` and `len` elements respectively (caller guarantees extra word for window shift). // Load [src[i], src[i+1]] and [src[i+1], src[i+2]]. let w0 = unsafe { vld1q_u64(src.as_ptr().add(i)) }; + // SAFETY: same as above; load from `src[i+1]`. let w1 = unsafe { vld1q_u64(src.as_ptr().add(i + 1)) }; // Build the shifted 64-bit window for each lane: // window[k] = (src[i+k] >> shift) | (src[i+k+1] << (64 - shift)) // vshlq_u64 with a negative shift amount performs a logical right shift. + // SAFETY: `vshlq_u64`, `vorrq_u64`, `vceqq_u64`, and `vgetq_lane_u64` are pure register operations; NEON is enabled by `#[target_feature]`. let lo = unsafe { vshlq_u64(w0, neg_shift) }; + // SAFETY: same as above let hi = unsafe { vshlq_u64(w1, pos_shift) }; + // SAFETY: same as above let window = unsafe { vorrq_u64(lo, hi) }; + // SAFETY: NEON is enabled; pointer `other` is valid for `len` elements. let expected = unsafe { vld1q_u64(other.as_ptr().add(i)) }; + // SAFETY: `vceqq_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. let cmp = unsafe { vceqq_u64(window, expected) }; // Each lane is all-ones on equality → vgetq_lane_u64 returns u64::MAX. + // SAFETY: `vgetq_lane_u64` is a pure register operation; NEON is enabled by `#[target_feature]`. if unsafe { vgetq_lane_u64(cmp, 0) } == 0 || unsafe { vgetq_lane_u64(cmp, 1) } == 0 { return false; } diff --git a/src/traits/bits_eq/funcs_for_eq_words_unaligned_core/tests_for_backend_equivalence.rs b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/tests_for_backend_equivalence.rs similarity index 91% rename from src/traits/bits_eq/funcs_for_eq_words_unaligned_core/tests_for_backend_equivalence.rs rename to src/traits/words_eq/funcs_for_eq_words_unaligned_core/tests_for_backend_equivalence.rs index ea78dd8..16b61b9 100644 --- a/src/traits/bits_eq/funcs_for_eq_words_unaligned_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_eq/funcs_for_eq_words_unaligned_core/tests_for_backend_equivalence.rs @@ -24,7 +24,7 @@ proptest! { let haystack = BitString::from_bool_iter((0..h_len).map(|i| (i * 13 + 7) % 5 == 0)); let suffix = BitString::from_bool_iter((0..s_len).map(|i| (i * 13 + 7) % 5 == 0)); - let result = haystack.ends_with(suffix.as_bit_str()); + let result = haystack.ends_with_str(suffix.as_bit_str()); let start = h_len - s_len; let expected = (0..s_len).all(|j| haystack.get(start + j) == suffix.get(j)); assert_eq!(result, expected); @@ -38,7 +38,7 @@ proptest! { let haystack = BitString::from_bool_iter(h_bits); let suffix = BitString::from_bool_iter(s_bits); - let result = haystack.ends_with(suffix.as_bit_str()); + let result = haystack.ends_with_str(suffix.as_bit_str()); let expected = suffix.bit_len() <= haystack.bit_len() && { let start = haystack.bit_len() - suffix.bit_len(); diff --git a/src/traits/words_eq/impls_for_u64_slice.rs b/src/traits/words_eq/impls_for_u64_slice.rs new file mode 100644 index 0000000..9e13544 --- /dev/null +++ b/src/traits/words_eq/impls_for_u64_slice.rs @@ -0,0 +1,22 @@ +use super::WordsEq; + +impl WordsEq for [u64] { + #[inline] + fn eq_words( + &self, + needle: &[u64], + full_words: usize, + haystack_shift: usize, + ) -> bool { + if HS_WORD_ALIGNED || haystack_shift == 0 { + super::funcs_for_eq_words_aligned_core::eq_words_aligned(self, needle, full_words) + } else { + super::funcs_for_eq_words_unaligned_core::eq_words_unaligned( + self, + needle, + full_words, + haystack_shift, + ) + } + } +} diff --git a/src/traits/bits_find.rs b/src/traits/words_find.rs similarity index 98% rename from src/traits/bits_find.rs rename to src/traits/words_find.rs index 2a90ba3..4c56f63 100644 --- a/src/traits/bits_find.rs +++ b/src/traits/words_find.rs @@ -3,7 +3,7 @@ /// All positions returned are relative to the start of `self`. Callers /// with an offset view (e.g. [`BitStr`](crate::BitStr)) must shift /// positions accordingly. -pub(crate) trait BitsFind { +pub(crate) trait WordsFind { /// Returns `Some(pos)` if any 64-bit window in the haystack matches /// the first word of `needle_words` AND `verify(pos)` succeeds. /// diff --git a/src/traits/bits_find/funcs_for_contains_core.rs b/src/traits/words_find/funcs_for_contains_core.rs similarity index 77% rename from src/traits/bits_find/funcs_for_contains_core.rs rename to src/traits/words_find/funcs_for_contains_core.rs index 941bda9..28d64f0 100644 --- a/src/traits/bits_find/funcs_for_contains_core.rs +++ b/src/traits/words_find/funcs_for_contains_core.rs @@ -50,64 +50,133 @@ where ); } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - unsafe { - return avx2::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { + avx2::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; + } + if f.sse41 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { + sse41::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; + } } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { + neon::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; + } + #[allow(unused)] + scalar( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - unsafe { - return sse41::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { + avx2::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - unsafe { - return neon::find_any( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { + sse41::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; } - } - #[allow(unused)] - scalar( - haystack, - needle_first, - needle_mask, - last_start, - word_limit, - verify, - ) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { + neon::find_any( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + }; + } + + #[allow(unused)] + scalar( + haystack, + needle_first, + needle_mask, + last_start, + word_limit, + verify, + ) + } } // --------------------------------------------------------------------------- diff --git a/src/traits/bits_find/funcs_for_contains_core/tests_for_backend_equivalence.rs b/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs similarity index 93% rename from src/traits/bits_find/funcs_for_contains_core/tests_for_backend_equivalence.rs rename to src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs index ca78b2d..edd03b1 100644 --- a/src/traits/bits_find/funcs_for_contains_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_find/funcs_for_contains_core/tests_for_backend_equivalence.rs @@ -31,7 +31,7 @@ proptest! { haystack.bit_len(), needle.words(), needle.bit_len(), - &mut |pos| haystack.as_bit_str().bits_equal_at(pos, needle.as_bit_str()), + &mut |pos| haystack.as_bit_str().bits_equal_at_inner::(pos, needle.as_bit_str()), ); // Brute-force reference: find any match. @@ -75,7 +75,7 @@ proptest! { needle.words(), needle_len, &mut |pos| { - let ok = haystack.as_bit_str().bits_equal_at(pos, needle.as_bit_str()); + let ok = haystack.as_bit_str().bits_equal_at_inner::(pos, needle.as_bit_str()); if ok { any_found = true; } ok }, diff --git a/src/traits/bits_find/funcs_for_find_core.rs b/src/traits/words_find/funcs_for_find_core.rs similarity index 77% rename from src/traits/bits_find/funcs_for_find_core.rs rename to src/traits/words_find/funcs_for_find_core.rs index f4d9933..33c3ca9 100644 --- a/src/traits/bits_find/funcs_for_find_core.rs +++ b/src/traits/words_find/funcs_for_find_core.rs @@ -29,36 +29,69 @@ where return scalar_find(haystack, needle_first, needle_mask, last_start, verify); } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - unsafe { - return avx2::find(haystack, needle_first, needle_mask, last_start, verify); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { + avx2::find(haystack, needle_first, needle_mask, last_start, verify) + }; + } + if f.sse41 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { + sse41::find(haystack, needle_first, needle_mask, last_start, verify) + }; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::find(haystack, needle_first, needle_mask, last_start, verify) }; } + #[allow(unused)] + scalar_find(haystack, needle_first, needle_mask, last_start, verify) } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - unsafe { - return sse41::find(haystack, needle_first, needle_mask, last_start, verify); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { avx2::find(haystack, needle_first, needle_mask, last_start, verify) }; } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - unsafe { - return neon::find(haystack, needle_first, needle_mask, last_start, verify); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { sse41::find(haystack, needle_first, needle_mask, last_start, verify) }; } - } - #[allow(unused)] - scalar_find(haystack, needle_first, needle_mask, last_start, verify) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::find(haystack, needle_first, needle_mask, last_start, verify) }; + } + + #[allow(unused)] + scalar_find(haystack, needle_first, needle_mask, last_start, verify) + } } // --------------------------------------------------------------------------- diff --git a/src/traits/bits_find/funcs_for_find_core/tests_for_backend_equivalence.rs b/src/traits/words_find/funcs_for_find_core/tests_for_backend_equivalence.rs similarity index 98% rename from src/traits/bits_find/funcs_for_find_core/tests_for_backend_equivalence.rs rename to src/traits/words_find/funcs_for_find_core/tests_for_backend_equivalence.rs index 11257f0..7436fe0 100644 --- a/src/traits/bits_find/funcs_for_find_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_find/funcs_for_find_core/tests_for_backend_equivalence.rs @@ -31,7 +31,7 @@ proptest! { let haystack_len = haystack.bit_len(); let needle_len = needle.bit_len(); - let result = haystack.find(needle.as_bit_str()); + let result = haystack.find_str(needle.as_bit_str()); // Brute-force reference. let mut expected = None; diff --git a/src/traits/bits_find/funcs_for_rfind_core.rs b/src/traits/words_find/funcs_for_rfind_core.rs similarity index 79% rename from src/traits/bits_find/funcs_for_rfind_core.rs rename to src/traits/words_find/funcs_for_rfind_core.rs index 838db2b..761087e 100644 --- a/src/traits/bits_find/funcs_for_rfind_core.rs +++ b/src/traits/words_find/funcs_for_rfind_core.rs @@ -31,36 +31,69 @@ where return scalar_rfind(haystack, needle_key, needle_mask, last_start, verify); } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - unsafe { - return avx2::rfind(haystack, needle_key, needle_mask, last_start, verify); + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { + avx2::rfind(haystack, needle_key, needle_mask, last_start, verify) + }; + } + if f.sse41 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { + sse41::rfind(haystack, needle_key, needle_mask, last_start, verify) + }; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::rfind(haystack, needle_key, needle_mask, last_start, verify) }; } + #[allow(unused)] + scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - unsafe { - return sse41::rfind(haystack, needle_key, needle_mask, last_start, verify); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { avx2::rfind(haystack, needle_key, needle_mask, last_start, verify) }; } - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - unsafe { - return neon::rfind(haystack, needle_key, needle_mask, last_start, verify); + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { sse41::rfind(haystack, needle_key, needle_mask, last_start, verify) }; } - } - #[allow(unused)] - scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::rfind(haystack, needle_key, needle_mask, last_start, verify) }; + } + + #[allow(unused)] + scalar_rfind(haystack, needle_key, needle_mask, last_start, verify) + } } // --------------------------------------------------------------------------- diff --git a/src/traits/bits_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs b/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs similarity index 92% rename from src/traits/bits_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs rename to src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs index e14c2e5..7b84a25 100644 --- a/src/traits/bits_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs +++ b/src/traits/words_find/funcs_for_rfind_core/tests_for_backend_equivalence.rs @@ -31,7 +31,7 @@ proptest! { haystack.bit_len(), needle.words(), needle.bit_len(), - &mut |pos| haystack.as_bit_str().bits_equal_at(pos, needle.as_bit_str()), + &mut |pos| haystack.as_bit_str().bits_equal_at_inner::(pos, needle.as_bit_str()), ); // Brute-force: find rightmost match. @@ -70,7 +70,7 @@ proptest! { haystack.bit_len(), needle.words(), needle.bit_len(), - &mut |pos| haystack.as_bit_str().bits_equal_at(pos, needle.as_bit_str()), + &mut |pos| haystack.as_bit_str().bits_equal_at_inner::(pos, needle.as_bit_str()), ); let max_pos = haystack.bit_len().saturating_sub(needle.bit_len()); diff --git a/src/traits/bits_find/impls_for_u64_slice.rs b/src/traits/words_find/impls_for_u64_slice.rs similarity index 96% rename from src/traits/bits_find/impls_for_u64_slice.rs rename to src/traits/words_find/impls_for_u64_slice.rs index 23616fd..8b0d779 100644 --- a/src/traits/bits_find/impls_for_u64_slice.rs +++ b/src/traits/words_find/impls_for_u64_slice.rs @@ -1,9 +1,9 @@ -use super::BitsFind; +use super::WordsFind; use super::funcs_for_contains_core; use super::funcs_for_find_core; use super::funcs_for_rfind_core; -impl BitsFind for [u64] { +impl WordsFind for [u64] { #[inline] fn find_any_candidate( &self, diff --git a/src/traits/bits_ord.rs b/src/traits/words_ord.rs similarity index 59% rename from src/traits/bits_ord.rs rename to src/traits/words_ord.rs index 40a2498..ad99a48 100644 --- a/src/traits/bits_ord.rs +++ b/src/traits/words_ord.rs @@ -16,8 +16,19 @@ use core::cmp::Ordering; /// - NEON (aarch64, 2×u64 per iteration) /// /// Short inputs fall back to scalar in all backends. -pub(crate) trait BitsOrd { - fn cmp_words(&self, other: &[u64], count: usize, offset: usize) -> Option; +pub(crate) trait WordsOrd { + /// `self` is pre-trimmed haystack `words[base..]`. + /// `needle` is always word-aligned (pre-trimmed by the caller). + /// `full_words` is the number of complete u64 words to compare. + /// `haystack_shift` is the intra-word offset within the first word. + /// When `HS_WORD_ALIGNED` is `true`, `haystack_shift == 0` is + /// guaranteed and the aligned backend is used unconditionally. + fn cmp_words( + &self, + needle: &[u64], + full_words: usize, + haystack_shift: usize, + ) -> Option; } pub(crate) mod funcs_for_cmp_aligned_core; diff --git a/src/traits/bits_ord/funcs_for_cmp_aligned_core.rs b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs similarity index 59% rename from src/traits/bits_ord/funcs_for_cmp_aligned_core.rs rename to src/traits/words_ord/funcs_for_cmp_aligned_core.rs index 677eede..10f54b6 100644 --- a/src/traits/bits_ord/funcs_for_cmp_aligned_core.rs +++ b/src/traits/words_ord/funcs_for_cmp_aligned_core.rs @@ -1,7 +1,7 @@ use core::cmp::Ordering; use crate::SMALL_WORDS; -use crate::traits::BitOrd; +use crate::traits::WordOrd; /// Returns `Some(Ordering)` if the first `count` aligned words of `src` /// and `other` differ, otherwise `None` (all equal). @@ -14,37 +14,72 @@ pub(super) fn cmp_aligned_words(src: &[u64], other: &[u64], count: usize) -> Opt return scalar_cmp_aligned(src, other, count); } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - return unsafe { avx2::cmp_aligned(src, other, count) }; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { avx2::cmp_aligned(src, other, count) }; + } + if f.sse41 { + // SAFETY: pointer validity guaranteed by caller. Backend selected via CPUID verification. + return unsafe { sse41::cmp_aligned(src, other, count) }; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::cmp_aligned(src, other, count) }; + } + #[allow(unreachable_code)] + scalar_cmp_aligned(src, other, count) } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "sse4.1", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - return unsafe { sse41::cmp_aligned(src, other, count) }; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { avx2::cmp_aligned(src, other, count) }; + } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - return unsafe { neon::cmp_aligned(src, other, count) }; - } + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1", + not(target_feature = "avx2") + ))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { sse41::cmp_aligned(src, other, count) }; + } - #[allow(unreachable_code)] - scalar_cmp_aligned(src, other, count) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: pointer validity guaranteed by caller. Backend is always safe / + // enabled by `#[target_feature]` at compile time. + return unsafe { neon::cmp_aligned(src, other, count) }; + } + + #[allow(unreachable_code)] + scalar_cmp_aligned(src, other, count) + } } #[inline] fn scalar_cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option { for i in 0..count { if src[i] != other[i] { - return Some(BitOrd::bitwise_cmp(src[i], other[i])); + return Some(WordOrd::bitwise_cmp(src[i], other[i])); } } None @@ -59,7 +94,7 @@ fn scalar_cmp_aligned(src: &[u64], other: &[u64], count: usize) -> Option> shift) | (w1 << (WORD_BITS - shift)); if window != other[i] { - return Some(BitOrd::bitwise_cmp(window, other[i])); + return Some(WordOrd::bitwise_cmp(window, other[i])); } } None @@ -78,7 +113,7 @@ mod avx2 { use core::cmp::Ordering; use crate::WORD_BITS; - use crate::traits::BitOrd; + use crate::traits::WordOrd; #[cfg(target_arch = "x86")] use core::arch::x86::{ @@ -114,14 +149,14 @@ mod avx2 { if mask != 0b1111 { let lane = mask.trailing_ones() as usize; let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); - return Some(BitOrd::bitwise_cmp(sw, other[i + lane])); + return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); } i += 4; } while i < len { let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); if sw != other[i] { - return Some(BitOrd::bitwise_cmp(sw, other[i])); + return Some(WordOrd::bitwise_cmp(sw, other[i])); } i += 1; } @@ -139,7 +174,7 @@ mod sse41 { use core::cmp::Ordering; use crate::WORD_BITS; - use crate::traits::BitOrd; + use crate::traits::WordOrd; #[cfg(target_arch = "x86")] use core::arch::x86::{ @@ -175,14 +210,14 @@ mod sse41 { if mask != 0xFFFF { let lane = mask.trailing_ones() as usize / 8; let sw = (src[i + lane] >> shift) | (src[i + lane + 1] << (WORD_BITS - shift)); - return Some(BitOrd::bitwise_cmp(sw, other[i + lane])); + return Some(WordOrd::bitwise_cmp(sw, other[i + lane])); } i += 2; } while i < len { let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); if sw != other[i] { - return Some(BitOrd::bitwise_cmp(sw, other[i])); + return Some(WordOrd::bitwise_cmp(sw, other[i])); } i += 1; } @@ -200,7 +235,7 @@ mod neon { use core::cmp::Ordering; use crate::WORD_BITS; - use crate::traits::BitOrd; + use crate::traits::WordOrd; use core::arch::aarch64::{ vceqq_u64, vdupq_n_s64, vgetq_lane_u64, vld1q_u64, vorrq_u64, vshlq_u64, @@ -226,18 +261,18 @@ mod neon { let cmp = unsafe { vceqq_u64(window, expected) }; if unsafe { vgetq_lane_u64(cmp, 0) } == 0 { let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); - return Some(BitOrd::bitwise_cmp(sw, other[i])); + return Some(WordOrd::bitwise_cmp(sw, other[i])); } if unsafe { vgetq_lane_u64(cmp, 1) } == 0 { let sw = (src[i + 1] >> shift) | (src[i + 2] << (WORD_BITS - shift)); - return Some(BitOrd::bitwise_cmp(sw, other[i + 1])); + return Some(WordOrd::bitwise_cmp(sw, other[i + 1])); } i += 2; } while i < len { let sw = (src[i] >> shift) | (src[i + 1] << (WORD_BITS - shift)); if sw != other[i] { - return Some(BitOrd::bitwise_cmp(sw, other[i])); + return Some(WordOrd::bitwise_cmp(sw, other[i])); } i += 1; } diff --git a/src/traits/bits_ord/funcs_for_cmp_unaligned_core/tests_for_backend_equivalence.rs b/src/traits/words_ord/funcs_for_cmp_unaligned_core/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_ord/funcs_for_cmp_unaligned_core/tests_for_backend_equivalence.rs rename to src/traits/words_ord/funcs_for_cmp_unaligned_core/tests_for_backend_equivalence.rs diff --git a/src/traits/words_ord/impls_for_u64_slice.rs b/src/traits/words_ord/impls_for_u64_slice.rs new file mode 100644 index 0000000..5cf2780 --- /dev/null +++ b/src/traits/words_ord/impls_for_u64_slice.rs @@ -0,0 +1,26 @@ +use core::cmp::Ordering; + +use super::WordsOrd; +use super::funcs_for_cmp_aligned_core; +use super::funcs_for_cmp_unaligned_core; + +impl WordsOrd for [u64] { + #[inline] + fn cmp_words( + &self, + needle: &[u64], + full_words: usize, + haystack_shift: usize, + ) -> Option { + if HS_WORD_ALIGNED || haystack_shift == 0 { + funcs_for_cmp_aligned_core::cmp_aligned_words(self, needle, full_words) + } else { + funcs_for_cmp_unaligned_core::cmp_unaligned_words( + self, + needle, + full_words, + haystack_shift, + ) + } + } +} diff --git a/src/traits/words_scan.rs b/src/traits/words_scan.rs new file mode 100644 index 0000000..741c3d5 --- /dev/null +++ b/src/traits/words_scan.rs @@ -0,0 +1,30 @@ +/// Bit-counting and scanning operations on `[u64]` backing storage. +pub(crate) trait WordsScan { + /// Returns the number of ones (set bits) in the first `bit_len` bits. + fn count_ones(&self, bit_len: usize) -> usize; + + /// Returns the count of consecutive leading bits equal to `FILL`. + /// + /// `self` is pre-trimmed to `words[physical_start / WORD_BITS..]`. + /// `start_offset` is `physical_start % WORD_BITS`. + /// When `WORD_ALIGNED` is `true`, `start_offset` is guaranteed to be 0 + /// and the first-word phase is eliminated at compile time. + fn leading_value_bits( + &self, + start_offset: u32, + bit_len: usize, + ) -> usize; + + /// Returns the count of consecutive trailing bits equal to `FILL`. + /// + /// Same preconditions as [`leading_value_bits`](Self::leading_value_bits). + fn trailing_value_bits( + &self, + start_offset: u32, + bit_len: usize, + ) -> usize; +} + +pub(crate) mod funcs_for_count_ones; +pub(crate) mod funcs_for_ends; +pub(crate) mod impls_for_u64_slice; diff --git a/src/traits/bits_arith/funcs_for_count_ones.rs b/src/traits/words_scan/funcs_for_count_ones.rs similarity index 78% rename from src/traits/bits_arith/funcs_for_count_ones.rs rename to src/traits/words_scan/funcs_for_count_ones.rs index 5fde7d4..baf625d 100644 --- a/src/traits/bits_arith/funcs_for_count_ones.rs +++ b/src/traits/words_scan/funcs_for_count_ones.rs @@ -47,50 +47,78 @@ unsafe fn dispatch(src: *const u64, len: usize) -> usize { // Small inputs: skip SIMD setup overhead, go straight to scalar popcnt. // Threshold equals the backend's LANES count. - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "avx2" - ))] + // ── Default: runtime SIMD detection ───────────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] { - if len >= 4 { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when AVX2 is enabled. - return unsafe { avx2::count_words(src, len) }; + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let f = crate::cpuid::features(); + if f.avx2 { + if len >= 4 { + // SAFETY: `src` is valid for `len` words. Backend selected via CPUID verification. + return unsafe { avx2::count_words(src, len) }; + } + } + if f.ssse3 { + if len >= 2 { + // SAFETY: `src` is valid for `len` words. Backend selected via CPUID verification. + return unsafe { ssse3::count_words(src, len) }; + } + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + if len >= 2 { + // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` gate on NEON availability. + return unsafe { neon::count_words(src, len) }; + } + } + #[allow(unused)] + // SAFETY: pointer validity guaranteed by caller. Scalar backend is always safe. + unsafe { + scalar::count_words(src, len) } - // len < 4: fall through to scalar below. } - #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), - target_feature = "ssse3", - not(target_feature = "avx2") - ))] + // ── compile-time-dispatch: pure #[cfg] cascade ────────────── + #[cfg(feature = "compile-time-dispatch")] { - if len >= 2 { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when SSSE3 is enabled. - return unsafe { ssse3::count_words(src, len) }; + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + if len >= 4 { + // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. + return unsafe { avx2::count_words(src, len) }; + } } - // len < 2: fall through to scalar below. - } - #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] - { - if len >= 2 { - // SAFETY: - // - Forwarded from `dispatch`'s safety contract. - // - This branch is compiled only when NEON is enabled. - return unsafe { neon::count_words(src, len) }; + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "ssse3", + not(target_feature = "avx2") + ))] + { + if len >= 2 { + // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. + return unsafe { ssse3::count_words(src, len) }; + } } - // len < 2: fall through to scalar below. - } - #[allow(unused)] - // SAFETY: Forwarded from `dispatch`'s safety contract. - unsafe { - scalar::count_words(src, len) + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + if len >= 2 { + // SAFETY: `src` is valid for `len` words. Backend selected via `#[cfg]` feature gate. + return unsafe { neon::count_words(src, len) }; + } + } + + #[allow(unused)] + // SAFETY: pointer validity guaranteed by caller. Scalar backend is always safe. + unsafe { + scalar::count_words(src, len) + } } } diff --git a/src/traits/bits_arith/funcs_for_count_ones/tests_for_backend_equivalence.rs b/src/traits/words_scan/funcs_for_count_ones/tests_for_backend_equivalence.rs similarity index 100% rename from src/traits/bits_arith/funcs_for_count_ones/tests_for_backend_equivalence.rs rename to src/traits/words_scan/funcs_for_count_ones/tests_for_backend_equivalence.rs diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs new file mode 100644 index 0000000..a5cb6c5 --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_leading_core.rs @@ -0,0 +1,433 @@ +//! Leading value-bit count — forward scan. +//! +//! Parameterised by `const FILL: u64` and `const WORD_ALIGNED: bool`. + +use crate::{SMALL_WORDS, WORD_BITS, low_mask}; + +// ── Scalar helper ────────────────────────────────────────────────────── + +#[inline] +fn count_trailing(val: u64) -> usize { + if FILL == 0 { + val.trailing_zeros() as usize + } else { + (!val).trailing_zeros() as usize + } +} + +// ── Dispatch ─────────────────────────────────────────────────────────── + +#[inline] +pub(crate) fn leading( + bits: &[u64], + start_offset: u32, + bit_len: usize, +) -> usize { + if bit_len == 0 { + return 0; + } + + let end_offset = start_offset as usize + bit_len; + let end_rem = end_offset % WORD_BITS; + let last_wi = (end_offset - 1) / WORD_BITS; + + let mut scanned = 0usize; + let mut wi = 0usize; + + // ── Unaligned first word ─────────────────────────────────────── + if !WORD_ALIGNED && start_offset != 0 { + let first_val = bits[0] >> start_offset; + let first_limit = (WORD_BITS - start_offset as usize).min(bit_len); + let first_count = count_trailing::(first_val).min(first_limit); + if first_count < first_limit { + return first_count; + } + scanned += first_limit; + wi = 1; + } + + let mid_end = if end_rem == 0 { last_wi + 1 } else { last_wi }; + if wi < mid_end { + let total = mid_end - wi; + + // ── Tiny inputs: scalar ──────────────────────────────────── + if total < SMALL_WORDS { + for i in 0..total { + let w = bits[wi + i]; + if w != FILL { + return (scanned + count_trailing::(w)).min(bit_len); + } + scanned += WORD_BITS; + } + wi = mid_end; + } else { + // SAFETY: `wi < mid_end` and `total = mid_end - wi`, + // so `bits[wi..mid_end]` is within the input slice. + let base = unsafe { bits.as_ptr().add(wi) }; + // SAFETY: `end` is one past the last word — used only as a + // limit pointer, never dereferenced. + let end = unsafe { base.add(total) }; + + // First-word fast path — catches early non-FILL. + // SAFETY: `total > 0` (we are in the `total >= SMALL_WORDS` + // branch), so `base` is valid for at least one u64 read. + let w0 = unsafe { *base }; + if w0 != FILL { + return (scanned + count_trailing::(w0)).min(bit_len); + } + // Start SIMD from `base` (not base+1). Word 0 is + // double-checked (fast path + SIMD) but this keeps the + // iteration count a clean multiple of the SIMD stride. + let mut p = base; + + // ── Default: runtime SIMD detection ───────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] + { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + if crate::cpuid::features().avx2 { + // SAFETY: CPUID confirmed AVX2 is available. + // `p` through `end` are within the input slice. + p = unsafe { avx2::leading_scan::(p, end, base, total) }; + } else { + // SAFETY: SSE2 is baseline on x86-64. + // `p` through `end` are within the input slice. + p = unsafe { sse2::leading_scan::(p, end, total) }; + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: NEON is available per `#[cfg]` gate. + // `p` through `end` are within the input slice. + p = unsafe { neon::leading_scan::(p, end, total) }; + } + #[allow(unused)] + { + // Scalar fallback: `p` stays at base; shared tail + // below scans word-by-word. + } + } + + // ── compile-time-dispatch: pure #[cfg] cascade ────────── + #[cfg(feature = "compile-time-dispatch")] + { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + // SAFETY: AVX2 is guaranteed by compile-time + // `#[cfg]` gate. + p = unsafe { avx2::leading_scan::(p, end, base, total) }; + } + + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + any(target_feature = "sse2", target_feature = "ssse3"), + not(target_feature = "avx2") + ))] + { + // SAFETY: SSE2 is guaranteed by compile-time + // `#[cfg]` gate. + p = unsafe { sse2::leading_scan::(p, end, total) }; + } + + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + // SAFETY: NEON is guaranteed by compile-time + // `#[cfg]` gate. + p = unsafe { neon::leading_scan::(p, end, total) }; + } + + #[allow(unused)] + { + // Scalar fallback: `p` stays at base. + } + } + + // ── Post-SIMD: shared scalar remainder ───────────────── + let done_words = (p as usize - base as usize) / 8; + scanned += done_words * WORD_BITS; + + if (p as usize) >= (end as usize) && end_rem == 0 { + return scanned.min(bit_len); + } + + let rem = (end as usize - p as usize) / 8; + // SAFETY: `rem` is computed from `end - p`, so `p` through + // `p.add(rem - 1)` lies within `[base, end)`. + for _ in 0..rem { + unsafe { + if *p != FILL { + scanned += count_trailing::(*p); + return (scanned).min(bit_len); + } + scanned += WORD_BITS; + p = p.add(1); + } + } + wi = mid_end; + } + } + + if end_rem != 0 && wi == last_wi { + let last_val = bits[wi] & low_mask(end_rem); + scanned += count_trailing::(last_val).min(end_rem); + } + + scanned.min(bit_len) +} + +// ═══════════════════════════════════════════════════════════════════════ +// AVX2 backend — 256-bit / 4-lane, unaligned loads only. +// ═══════════════════════════════════════════════════════════════════════ + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m256i, _mm256_load_si256, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, + _mm256_xor_si256, + }; + + const LANES: usize = 4; + const STRIDE: usize = 8; // 2 × LANES for unrolled iteration + const ALIGN_THRESHOLD: usize = 128; + + /// AVX2 forward scan: advances `p` past all-FILL 256-bit chunks. + /// + /// For large inputs (≥ ALIGN_THRESHOLD words), aligns `p` to a + /// 32-byte boundary and uses `_mm256_load_si256` (aligned) for the + /// hot loop. Smaller inputs use `_mm256_loadu_si256` (unaligned) + /// to avoid the alignment prefix overhead. + /// + /// # Safety + /// + /// Caller must ensure AVX2 is available (checked via CPUID). + /// `p` through `end` must be valid for u64 reads. + #[target_feature(enable = "avx2")] + pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + base: *const u64, + total: usize, + ) -> *const u64 { + // SAFETY: only callable when AVX2 is available (caller verified + // via CPUID). All pointer arithmetic stays within bounds. + unsafe { + if total >= ALIGN_THRESHOLD { + // Distance in words to the next 32-byte boundary. + let words_to_align = (4usize.wrapping_sub((base as usize / 8) % 4)) % 4; + if words_to_align > 0 { + let prefix_end = base.add(words_to_align); + while p < prefix_end { + if *p != FILL { + return p; + } + p = p.add(1); + } + } + let mut iters = + (end as usize - p as usize) / (STRIDE * core::mem::size_of::()); + while iters > 0 { + if FILL == 0 { + let d0 = _mm256_load_si256(p.cast::<__m256i>()); + let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); + if _mm256_testz_si256(d0, d0) == 0 || _mm256_testz_si256(d1, d1) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_load_si256(p.cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_load_si256(p.add(LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + if _mm256_testz_si256(x0, x0) == 0 || _mm256_testz_si256(x1, x1) == 0 { + break; + } + } + p = p.add(STRIDE); + iters -= 1; + } + } else { + // 2×-unrolled unaligned path. + let mut iters = total / STRIDE; + while iters > 0 { + if FILL == 0 { + let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); + let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); + if _mm256_testz_si256(d0, d0) == 0 || _mm256_testz_si256(d1, d1) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d0 = _mm256_loadu_si256(p.cast::<__m256i>()); + let x0 = _mm256_xor_si256(d0, fill_vec); + let d1 = _mm256_loadu_si256(p.add(LANES).cast::<__m256i>()); + let x1 = _mm256_xor_si256(d1, fill_vec); + if _mm256_testz_si256(x0, x0) == 0 || _mm256_testz_si256(x1, x1) == 0 { + break; + } + } + p = p.add(STRIDE); + iters -= 1; + } + } + // Single-chunk remainder (LANES = 4 words). + let limit = end.sub(LANES); + while p <= limit { + if FILL == 0 { + let d = _mm256_loadu_si256(p.cast::<__m256i>()); + if _mm256_testz_si256(d, d) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(p.cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + if _mm256_testz_si256(x, x) == 0 { + break; + } + } + p = p.add(LANES); + } + p + } + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse2 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, + }; + + const LANES: usize = 2; + const LANES_2X: usize = LANES * 2; + + #[inline(always)] + unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // SSE2 is available. + unsafe { + let data = _mm_loadu_si128(ptr.cast::<__m128i>()); + if FILL == 0 { + let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } else { + let fill_vec = _mm_set1_epi64x(FILL as i64); + let xor = _mm_xor_si128(data, fill_vec); + let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } + } + } + + /// SSE2 forward scan: advances `p` past all-FILL chunks. + /// + /// # Safety + /// + /// Caller must ensure SSE2 is available (baseline on x86-64). + /// `p` through `end` must be valid for u64 reads. + #[target_feature(enable = "sse2")] + pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + total: usize, + ) -> *const u64 { + // SAFETY: only callable when SSE2 is available (caller verified + // via CPUID, or SSE2 is baseline). + unsafe { + let mut iters = total / LANES_2X; + while iters > 0 { + if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { + return p; + } + p = p.add(LANES_2X); + iters -= 1; + } + let limit = end.sub(LANES); + while p <= limit { + if !chunk_eq::(p) { + break; + } + p = p.add(LANES); + } + p + } + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[allow(unused)] +#[cfg(target_arch = "aarch64")] +mod neon { + use core::arch::aarch64::{uint64x2_t, vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; + + const LANES: usize = 2; + const LANES_2X: usize = LANES * 2; + + #[inline(always)] + unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // NEON is available. + unsafe { + let data = vld1q_u64(ptr); + let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); + vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 + } + } + + /// NEON forward scan: advances `p` past all-FILL chunks. + /// + /// # Safety + /// + /// Caller must ensure NEON is available. + /// `p` through `end` must be valid for u64 reads. + #[target_feature(enable = "neon")] + pub(super) unsafe fn leading_scan( + mut p: *const u64, + end: *const u64, + total: usize, + ) -> *const u64 { + // SAFETY: only callable when NEON is available. All pointer + // arithmetic stays within `[p, end)`. + unsafe { + let mut iters = total / LANES_2X; + while iters > 0 { + if !chunk_eq::(p) || !chunk_eq::(p.add(LANES)) { + return p; + } + p = p.add(LANES_2X); + iters -= 1; + } + let limit = end.sub(LANES); + while p <= limit { + if !chunk_eq::(p) { + break; + } + p = p.add(LANES); + } + p + } + } +} diff --git a/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs new file mode 100644 index 0000000..24e6f2f --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/funcs_for_trailing_core.rs @@ -0,0 +1,411 @@ +//! Trailing value-bit count — reverse scan. +//! +//! Parameterised by `const FILL: u64` and `const WORD_ALIGNED: bool`. +//! When `WORD_ALIGNED` is `true` the caller guarantees `start_offset == 0`, +//! allowing the compiler to eliminate the first-word LZCNT phase. + +use crate::{SMALL_WORDS, WORD_BITS}; + +// ── Scalar helper ────────────────────────────────────────────────────── + +/// Counts leading bits within a single u64 word that match `FILL`. +#[inline] +fn count_leading(val: u64) -> usize { + if FILL == 0 { + val.leading_zeros() as usize + } else { + (!val).leading_zeros() as usize + } +} + +// ── Dispatch ─────────────────────────────────────────────────────────── + +#[inline] +pub(crate) fn trailing( + bits: &[u64], + start_offset: u32, + bit_len: usize, +) -> usize { + if bit_len == 0 { + return 0; + } + + let end_offset = start_offset as usize + bit_len; + let end_rem = end_offset % WORD_BITS; + let last_wi = (end_offset - 1) / WORD_BITS; + + let mut scanned = 0usize; + + // ── Last partial word ───────────────────────────────────────── + if end_rem != 0 { + let last_limit = if last_wi == 0 { + end_rem - start_offset as usize + } else { + end_rem + }; + let shifted = bits[last_wi] << (WORD_BITS - end_rem); + let last_count = count_leading::(shifted).min(last_limit); + if last_count < last_limit { + return last_count; + } + scanned += last_limit; + if last_wi == 0 { + return scanned.min(bit_len); + } + } + + // ── Full middle words — reverse SIMD scan ──────────────────── + let wi_end = if end_rem != 0 { last_wi - 1 } else { last_wi }; + let mid_first = if !WORD_ALIGNED && start_offset > 0 { + 1 + } else { + 0 + }; + + if wi_end >= mid_first { + let total_words = wi_end + 1 - mid_first; + let ptr = bits.as_ptr(); + + let mut done = 0usize; + + // ── Rightmost-word fast path ───────────────────────────── + // Early exit if the answer is in the rightmost full word, + // without disrupting the SIMD stride alignment. + { + let w = bits[wi_end]; + if w != FILL { + scanned += count_leading::(w); + return scanned.min(bit_len); + } + } + + // ── Tiny inputs — simple scalar reverse scan ──────────── + if total_words < SMALL_WORDS { + while done < total_words { + let wi = wi_end - done; + if bits[wi] != FILL { + scanned += count_leading::(bits[wi]); + return scanned.min(bit_len); + } + scanned += WORD_BITS; + done += 1; + } + // All full words match FILL — skip SIMD. + } else { + // ── Default: runtime SIMD detection ─────────────────── + #[cfg(not(feature = "compile-time-dispatch"))] + { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + let done_before = done; + if crate::cpuid::features().avx2 { + // SAFETY: CPUID confirmed AVX2 is available. + done = + unsafe { avx2::trailing_scan::(ptr, wi_end, done, total_words) }; + } else { + // SAFETY: SSE2 is baseline on x86-64. + done = + unsafe { sse2::trailing_scan::(ptr, wi_end, done, total_words) }; + } + scanned += (done - done_before) * WORD_BITS; + } + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + let done_before = done; + // SAFETY: NEON is available per `#[cfg]` gate. + done = unsafe { neon::trailing_scan::(ptr, wi_end, done, total_words) }; + scanned += (done - done_before) * WORD_BITS; + } + #[allow(unused)] + { + // Scalar fallback: `done` stays unchanged; shared + // tail below scans word-by-word. + } + } + + // ── compile-time-dispatch: pure #[cfg] cascade ──────── + #[cfg(feature = "compile-time-dispatch")] + { + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "avx2" + ))] + { + let done_before = done; + // SAFETY: AVX2 is guaranteed by compile-time gate. + done = unsafe { avx2::trailing_scan::(ptr, wi_end, done, total_words) }; + scanned += (done - done_before) * WORD_BITS; + } + + #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + any(target_feature = "sse2", target_feature = "ssse3"), + not(target_feature = "avx2") + ))] + { + let done_before = done; + // SAFETY: SSE2 is guaranteed by compile-time gate. + done = unsafe { sse2::trailing_scan::(ptr, wi_end, done, total_words) }; + scanned += (done - done_before) * WORD_BITS; + } + + #[cfg(all(target_arch = "aarch64", target_feature = "neon"))] + { + let done_before = done; + // SAFETY: NEON is guaranteed by compile-time gate. + done = unsafe { neon::trailing_scan::(ptr, wi_end, done, total_words) }; + scanned += (done - done_before) * WORD_BITS; + } + + #[allow(unused)] + { + // Scalar fallback. + } + } + } // else (SIMD path) + + // ── Scalar tail ────────────────────────────────────────── + while done < total_words { + let wi = wi_end - done; + if bits[wi] != FILL { + scanned += count_leading::(bits[wi]); + return scanned.min(bit_len); + } + scanned += WORD_BITS; + done += 1; + } + } + + // ── First-word partial (trailing side) ─────────────────────── + if !WORD_ALIGNED && start_offset > 0 { + let first_limit = WORD_BITS - start_offset as usize; + let first_count = count_leading::(bits[0]).min(first_limit); + scanned += first_count; + } + + scanned.min(bit_len) +} + +// ═══════════════════════════════════════════════════════════════════════ +// AVX2 backend — 256-bit / 4-lane, unaligned loads only. +// ═══════════════════════════════════════════════════════════════════════ + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod avx2 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m256i, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, _mm256_xor_si256, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m256i, _mm256_loadu_si256, _mm256_set1_epi64x, _mm256_testz_si256, _mm256_xor_si256, + }; + + const LANES: usize = 4; + const STRIDE: usize = 8; + + /// AVX2 reverse scan: scans backwards from `wi_end` and advances `done` + /// past all-FILL 256-bit chunks. + /// + /// Returns the updated `done` count (total words consumed from right). + /// + /// # Safety + /// + /// Caller must ensure AVX2 is available (checked via CPUID). + /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. + #[target_feature(enable = "avx2")] + pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, + ) -> usize { + // SAFETY: only callable when AVX2 is available (caller verified + // via CPUID). All pointer arithmetic stays within bounds. + unsafe { + // 2×‑unrolled + while done + STRIDE <= total_words { + let chunk_start = wi_end + 1 - (done + STRIDE); + let d0_ok = if FILL == 0 { + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + _mm256_testz_si256(d, d) != 0 + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + _mm256_testz_si256(x, x) != 0 + }; + let d1_ok = if FILL == 0 { + let d = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + _mm256_testz_si256(d, d) != 0 + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(ptr.add(chunk_start + LANES).cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + _mm256_testz_si256(x, x) != 0 + }; + if !d0_ok || !d1_ok { + break; + } + done += STRIDE; + } + // Single-chunk remainder + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if FILL == 0 { + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + if _mm256_testz_si256(d, d) == 0 { + break; + } + } else { + let fill_vec = _mm256_set1_epi64x(FILL as i64); + let d = _mm256_loadu_si256(ptr.add(chunk_start).cast::<__m256i>()); + let x = _mm256_xor_si256(d, fill_vec); + if _mm256_testz_si256(x, x) == 0 { + break; + } + } + done += LANES; + } + + done + } + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// SSE2 backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[allow(unused)] +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +mod sse2 { + #[cfg(target_arch = "x86")] + use core::arch::x86::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, + }; + #[cfg(target_arch = "x86_64")] + use core::arch::x86_64::{ + __m128i, _mm_cmpeq_epi32, _mm_loadu_si128, _mm_movemask_epi8, _mm_set1_epi64x, + _mm_setzero_si128, _mm_xor_si128, + }; + + const LANES: usize = 2; + const LANES_2X: usize = LANES * 2; + + #[inline(always)] + unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // SSE2 is available. + unsafe { + let data = _mm_loadu_si128(ptr.cast::<__m128i>()); + if FILL == 0 { + let cmp = _mm_cmpeq_epi32(data, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } else { + let fill_vec = _mm_set1_epi64x(FILL as i64); + let xor = _mm_xor_si128(data, fill_vec); + let cmp = _mm_cmpeq_epi32(xor, _mm_setzero_si128()); + _mm_movemask_epi8(cmp) == 0xFFFF + } + } + } + + /// SSE2 reverse scan: advances `done` past all-FILL chunks from the right. + /// + /// # Safety + /// + /// Caller must ensure SSE2 is available (baseline on x86-64). + /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. + #[target_feature(enable = "sse2")] + pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, + ) -> usize { + // SAFETY: only callable when SSE2 is available (caller verified + // via CPUID, or SSE2 is baseline). + unsafe { + while done + LANES_2X <= total_words { + let chunk_start = wi_end + 1 - (done + LANES_2X); + if !chunk_eq::(ptr.add(chunk_start)) + || !chunk_eq::(ptr.add(chunk_start + LANES)) + { + return done; + } + done += LANES_2X; + } + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if !chunk_eq::(ptr.add(chunk_start)) { + break; + } + done += LANES; + } + done + } + } +} + +// ═══════════════════════════════════════════════════════════════════════ +// NEON backend — 128-bit / 2-lane, raw intrinsics (no chunk_eq dispatch). +// ═══════════════════════════════════════════════════════════════════════ + +#[allow(unused)] +#[cfg(target_arch = "aarch64")] +mod neon { + use core::arch::aarch64::{vceqq_u64, vdupq_n_u64, vgetq_lane_u64, vld1q_u64}; + + const LANES: usize = 2; + const LANES_2X: usize = LANES * 2; + + #[inline(always)] + unsafe fn chunk_eq(ptr: *const u64) -> bool { + // SAFETY: caller ensures `ptr` is valid for 2 u64 reads and + // NEON is available. + unsafe { + let data = vld1q_u64(ptr); + let cmp = vceqq_u64(data, vdupq_n_u64(FILL)); + vgetq_lane_u64(cmp, 0) != 0 && vgetq_lane_u64(cmp, 1) != 0 + } + } + + /// NEON reverse scan: advances `done` past all-FILL chunks from the right. + /// + /// # Safety + /// + /// Caller must ensure NEON is available. + /// `ptr` through `ptr.add(wi_end + 1)` must be valid for u64 reads. + #[target_feature(enable = "neon")] + pub(super) unsafe fn trailing_scan( + ptr: *const u64, + wi_end: usize, + mut done: usize, + total_words: usize, + ) -> usize { + // SAFETY: only callable when NEON is available. All pointer + // arithmetic stays within bounds. + unsafe { + while done + LANES_2X <= total_words { + let chunk_start = wi_end + 1 - (done + LANES_2X); + if !chunk_eq::(ptr.add(chunk_start)) + || !chunk_eq::(ptr.add(chunk_start + LANES)) + { + return done; + } + done += LANES_2X; + } + while done + LANES <= total_words { + let chunk_start = wi_end + 1 - (done + LANES); + if !chunk_eq::(ptr.add(chunk_start)) { + break; + } + done += LANES; + } + done + } + } +} diff --git a/src/traits/words_scan/funcs_for_ends/mod.rs b/src/traits/words_scan/funcs_for_ends/mod.rs new file mode 100644 index 0000000..fb50b0e --- /dev/null +++ b/src/traits/words_scan/funcs_for_ends/mod.rs @@ -0,0 +1,5 @@ +mod funcs_for_leading_core; +mod funcs_for_trailing_core; + +pub(crate) use funcs_for_leading_core::leading; +pub(crate) use funcs_for_trailing_core::trailing; diff --git a/src/traits/words_scan/impls_for_u64_slice.rs b/src/traits/words_scan/impls_for_u64_slice.rs new file mode 100644 index 0000000..a83f65c --- /dev/null +++ b/src/traits/words_scan/impls_for_u64_slice.rs @@ -0,0 +1,28 @@ +use super::WordsScan; +use super::funcs_for_count_ones; +use super::funcs_for_ends; + +impl WordsScan for [u64] { + #[inline] + fn count_ones(&self, bit_len: usize) -> usize { + funcs_for_count_ones::count_ones(self, bit_len) + } + + #[inline] + fn leading_value_bits( + &self, + start_offset: u32, + bit_len: usize, + ) -> usize { + funcs_for_ends::leading::(self, start_offset, bit_len) + } + + #[inline] + fn trailing_value_bits( + &self, + start_offset: u32, + bit_len: usize, + ) -> usize { + funcs_for_ends::trailing::(self, start_offset, bit_len) + } +} diff --git a/tests/adversarial.rs b/tests/adversarial.rs index 2994534..78c454a 100644 --- a/tests/adversarial.rs +++ b/tests/adversarial.rs @@ -71,6 +71,8 @@ mod tests_for_editing; mod tests_for_fmt; #[path = "adversarial/tests_for_iter.rs"] mod tests_for_iter; +#[path = "adversarial/tests_for_leading_trailing.rs"] +mod tests_for_leading_trailing; #[path = "adversarial/tests_for_matching.rs"] mod tests_for_matching; #[path = "adversarial/tests_for_ord_hash.rs"] diff --git a/tests/adversarial/tests_for_bugs.rs b/tests/adversarial/tests_for_bugs.rs index ce9026c..527cae1 100644 --- a/tests/adversarial/tests_for_bugs.rs +++ b/tests/adversarial/tests_for_bugs.rs @@ -31,8 +31,8 @@ fn diagnostic_find_cross_boundary_needle() { let needle: BitString = "101".parse().unwrap(); let needle_view = needle.as_bit_str(); - assert_eq!(bits.find(needle_view), Some(0)); + assert_eq!(bits.find_str(needle_view), Some(0)); let remaining = bits.as_bit_str().slice_from(3); - assert_eq!(remaining.find(needle_view), Some(60)); + assert_eq!(remaining.find_str(needle_view), Some(60)); } diff --git a/tests/adversarial/tests_for_leading_trailing.rs b/tests/adversarial/tests_for_leading_trailing.rs new file mode 100644 index 0000000..8439c64 --- /dev/null +++ b/tests/adversarial/tests_for_leading_trailing.rs @@ -0,0 +1,796 @@ +use super::*; +use int_interval::UsizeCO; + +// =========================================================================== +// A. BitString leading/trailing — exhaustive small-length coverage +// =========================================================================== + +#[test] +fn attack_bitstring_leading_trailing_exhaustive() { + // Exhaustively test every meaningful length from 0 to 256 with a + // family of patterns that stress every code path. + for len in 0..=256 { + let z = BitString::zeros(len); + assert_eq!(z.leading_zeros(), len, "zeros leading_zeros len={len}"); + assert_eq!(z.leading_ones(), 0, "zeros leading_ones len={len}"); + assert_eq!(z.trailing_zeros(), len, "zeros trailing_zeros len={len}"); + assert_eq!(z.trailing_ones(), 0, "zeros trailing_ones len={len}"); + + let o = BitString::ones(len); + assert_eq!(o.leading_zeros(), 0, "ones leading_zeros len={len}"); + assert_eq!(o.leading_ones(), len, "ones leading_ones len={len}"); + assert_eq!(o.trailing_zeros(), 0, "ones trailing_zeros len={len}"); + assert_eq!(o.trailing_ones(), len, "ones trailing_ones len={len}"); + + // Alternating starting with 0: "010101..." + let alt0 = { + let s = "01".repeat((len + 1) / 2); + bs(&s[..len]) + }; + assert_eq!( + alt0.leading_zeros(), + if len > 0 { 1 } else { 0 }, + "0101... leading_zeros len={len}" + ); + assert_eq!(alt0.leading_ones(), 0, "0101... leading_ones len={len}"); + + // Alternating starting with 1: "101010..." + let alt1 = { + let s = "10".repeat((len + 1) / 2); + bs(&s[..len]) + }; + assert_eq!(alt1.leading_zeros(), 0, "1010... leading_zeros len={len}"); + assert_eq!( + alt1.leading_ones(), + if len > 0 { 1 } else { 0 }, + "1010... leading_ones len={len}" + ); + } +} + +// =========================================================================== +// B. Single-bit and edge-position attacks +// =========================================================================== + +#[test] +fn attack_bitstring_leading_trailing_single_bit_positions() { + // Place a single 1-bit at every position from 0 to 255 and verify + // all four counts from both ends. + for pos in 0..256 { + let len = pos + 1; + let mut bits = BitString::zeros(len); + // Set the bit at `pos` (0 = MSB / leftmost in the string). + // BitString index 0 is the leftmost bit. + bits.set(pos, true); + + // Leading zeros: must be `pos` (all bits before position pos are 0). + assert_eq!( + bits.leading_zeros(), + pos, + "single 1 at pos={pos} leading_zeros" + ); + // Leading ones: always 0 (bit 0 is never 1 unless pos==0). + if pos == 0 { + assert_eq!(bits.leading_ones(), 1); + } else { + assert_eq!(bits.leading_ones(), 0); + } + + // Trailing zeros: bits after pos are all 0. + assert_eq!( + bits.trailing_zeros(), + len - pos - 1, + "single 1 at pos={pos} trailing_zeros len={len}" + ); + // Trailing ones: always 0 unless pos is the last bit. + if pos == len - 1 { + assert_eq!(bits.trailing_ones(), 1); + } else { + assert_eq!(bits.trailing_ones(), 0); + } + } +} + +// =========================================================================== +// C. Cross-word boundary patterns +// =========================================================================== + +#[test] +fn attack_bitstring_leading_crosses_word_boundary() { + // Leading FILL spans exactly k words, followed by a non-FILL bit. + // Test the SIMD scan and the scalar tail around word boundaries. + for prepend_words in [0, 1, 2, 3, 4, 5, 8, 9] { + for extra_bits in [0, 1, 63] { + let total_zeros = prepend_words * 64 + extra_bits; + if total_zeros == 0 { + continue; + } + let total_len = total_zeros + 1; + let mut bits = BitString::zeros(total_len); + // Flip the first non-zero bit to 1 — this is where leading_zeros stops. + bits.set(total_zeros, true); + assert_eq!( + bits.leading_zeros(), + total_zeros, + "leading_zeros: {} words + {} bits of zeros then 1", + prepend_words, + extra_bits + ); + assert_eq!(bits.leading_ones(), 0); + } + } +} + +#[test] +fn attack_bitstring_trailing_crosses_word_boundary() { + // Trailing FILL spans exactly k words counting backwards. + for append_words in [0, 1, 2, 3, 4, 5, 8, 9] { + for extra_bits in [0, 1, 63] { + let total_zeros = append_words * 64 + extra_bits; + if total_zeros == 0 { + continue; + } + let total_len = total_zeros + 1; + let mut bits = BitString::zeros(total_len); + // Set the leftmost bit to 1 — trailing_zeros counts from the right. + bits.set(0, true); + assert_eq!( + bits.trailing_zeros(), + total_zeros, + "trailing_zeros: 1 then {} words + {} bits of zeros", + append_words, + extra_bits + ); + assert_eq!(bits.trailing_ones(), 0); + } + } +} + +#[test] +fn attack_bitstring_leading_trailing_boundary_shifts() { + // Test patterns where the FILL/non-FILL transition lands exactly + // at ±1 from every word boundary up to 4 words. + for word_boundary in [63, 64, 65, 127, 128, 129, 191, 192, 193] { + let len = word_boundary + 10; + + // Leading: zeros up to `word_boundary`, then a 1. + let mut bits = BitString::zeros(len); + bits.set(word_boundary, true); + assert_eq!( + bits.leading_zeros(), + word_boundary, + "leading_zeros boundary={word_boundary}" + ); + + // Leading with ones. + let mut bits = BitString::ones(len); + bits.set(word_boundary, false); + assert_eq!( + bits.leading_ones(), + word_boundary, + "leading_ones boundary={word_boundary}" + ); + } +} + +// =========================================================================== +// D. Last-word masking — unused bits must not affect results +// =========================================================================== + +#[test] +fn attack_last_word_mask_does_not_affect_counts() { + // Create bitstrings of every length 1..256, fill them entirely with + // either zeros or ones, then verify counts. The last-word mask must + // zero out bits beyond `bit_len`, otherwise leading_zeros / + // trailing_zeros on a genuinely all-zero string would return the + // wrong answer. + for len in 1..=256 { + // All zeros — mask must prevent trailing junk from adding spurious + // leading/trailing zeros. + let z = BitString::zeros(len); + assert_eq!(z.leading_zeros(), len, "zeros leading_zeros len={len}"); + assert_eq!(z.trailing_zeros(), len, "zeros trailing_zeros len={len}"); + + // All ones — mask must prevent trailing junk from adding spurious + // leading/trailing ones. + let o = BitString::ones(len); + assert_eq!(o.leading_ones(), len, "ones leading_ones len={len}"); + assert_eq!(o.trailing_ones(), len, "ones trailing_ones len={len}"); + + // Verify that the invariant holds: unused bits in the last word + // are actually zeroed. + assert!(view_has_same_invariants(&z)); + assert!(view_has_same_invariants(&o)); + } +} + +// =========================================================================== +// E. BitStr oracle — systematic misaligned view comparison +// =========================================================================== + +#[test] +fn attack_bitstr_leading_trailing_oracle_misaligned() { + // For patterns long enough to span multiple words, test every + // possible start offset (0..64) combined with every bit length + // (0..remaining). Compare the BitStr result against the + // round-tripped BitString result. + let patterns: Vec<(&str, BitString)> = vec![ + ("zeros", BitString::zeros(300)), + ("ones", BitString::ones(300)), + ("0101", bs(&"01".repeat(150))), + ("1010", bs(&"10".repeat(150))), + ( + "0x20_1_zeros", + bs(&cat(&[ + "0".repeat(20).as_str(), + "1", + "0".repeat(279).as_str(), + ])), + ), + ( + "0x20_1_ones", + bs(&cat(&[ + "1".repeat(20).as_str(), + "0", + "1".repeat(279).as_str(), + ])), + ), + ]; + + for (name, bits) in &patterns { + let view = bits.as_bit_str(); + for start in 0..64.min(bits.bit_len()) { + let max_len = bits.bit_len() - start; + for len in 1..=max_len.min(130) { + let interval = UsizeCO::checked_from_start_len(start, len).unwrap(); + let sub = view.slice(interval); + let owned = sub.to_bit_string(); + + assert_eq!( + sub.leading_zeros(), + owned.leading_zeros(), + "BitStr leading_zeros mismatch pattern={name} start={start} len={len}" + ); + assert_eq!( + sub.leading_ones(), + owned.leading_ones(), + "BitStr leading_ones mismatch pattern={name} start={start} len={len}" + ); + assert_eq!( + sub.trailing_zeros(), + owned.trailing_zeros(), + "BitStr trailing_zeros mismatch pattern={name} start={start} len={len}" + ); + assert_eq!( + sub.trailing_ones(), + owned.trailing_ones(), + "BitStr trailing_ones mismatch pattern={name} start={start} len={len}" + ); + } + } + } +} + +// =========================================================================== +// F. BitStr misaligned — both start and end unaligned +// =========================================================================== + +#[test] +fn attack_bitstr_misaligned_both_ends() { + // Construct a long pattern, then take views where both `start` and + // `start + len` are unaligned (not mod 64). This stresses the + // first-word and last-word partial handling simultaneously. + let body = "01".repeat(200); // 400 bits + let bits = bs(&body); + let view = bits.as_bit_str(); + + // Test every misaligned start offset (1..63) with varied lengths. + for start in 1..64 { + for len_parts in [0, 1, 2, 3, 5, 8] { + let len = len_parts * 64 + 37; // deliberately unaligned length + if start + len > 400 { + continue; + } + let interval = UsizeCO::checked_from_start_len(start, len).unwrap(); + let sub = view.slice(interval); + let owned = sub.to_bit_string(); + + assert_eq!( + sub.leading_zeros(), + owned.leading_zeros(), + "misaligned-both leading_zeros start={start} len={len}" + ); + assert_eq!( + sub.leading_ones(), + owned.leading_ones(), + "misaligned-both leading_ones start={start} len={len}" + ); + assert_eq!( + sub.trailing_zeros(), + owned.trailing_zeros(), + "misaligned-both trailing_zeros start={start} len={len}" + ); + assert_eq!( + sub.trailing_ones(), + owned.trailing_ones(), + "misaligned-both trailing_ones start={start} len={len}" + ); + } + } +} + +// =========================================================================== +// G. Single-word views — short bit lengths with various offsets +// =========================================================================== + +#[test] +fn attack_bitstr_single_word_views() { + // Breach a view entirely inside one word of the source, with every + // possible misalignment within that word. + let bits = bs(&cat(&[ + "0".repeat(100).as_str(), + "1".repeat(100).as_str(), + "0".repeat(100).as_str(), + ])); + let view = bits.as_bit_str(); + + // Test every offset within a single word, for lengths that stay + // within one word. + for start in 0..100 { + for len in 1..=(100 - start).min(64) { + let interval = UsizeCO::checked_from_start_len(start, len).unwrap(); + let sub = view.slice(interval); + let owned = sub.to_bit_string(); + + assert_eq!( + sub.leading_zeros(), + owned.leading_zeros(), + "single-word leading_zeros start={start} len={len}" + ); + assert_eq!( + sub.trailing_zeros(), + owned.trailing_zeros(), + "single-word trailing_zeros start={start} len={len}" + ); + assert_eq!( + sub.leading_ones(), + owned.leading_ones(), + "single-word leading_ones start={start} len={len}" + ); + assert_eq!( + sub.trailing_ones(), + owned.trailing_ones(), + "single-word trailing_ones start={start} len={len}" + ); + } + } +} + +// =========================================================================== +// H. BitString leading_ones / trailing_ones on non-trivial patterns +// =========================================================================== + +// =========================================================================== +// I. Zero-length edge cases +// =========================================================================== + +#[test] +fn attack_leading_trailing_zero_length() { + // BitString + let z = BitString::zeros(0); + assert_eq!(z.leading_zeros(), 0); + assert_eq!(z.leading_ones(), 0); + assert_eq!(z.trailing_zeros(), 0); + assert_eq!(z.trailing_ones(), 0); + + // BitStr view of zero length — slice_from at the very end creates + // an empty view. + let bits = bs("01010101"); + let view = bits.as_bit_str(); + let empty = view.slice_from(8); + assert_eq!(empty.leading_zeros(), 0); + assert_eq!(empty.leading_ones(), 0); + assert_eq!(empty.trailing_zeros(), 0); + assert_eq!(empty.trailing_ones(), 0); + // Empty view at a misaligned position within the backing word. + let long = bs(&"01".repeat(100)); // 200 bits + let lv = long.as_bit_str(); + let empty_misaligned = lv.slice_from(200); + assert_eq!(empty_misaligned.bit_len(), 0); + assert_eq!(empty_misaligned.leading_zeros(), 0); + assert_eq!(empty_misaligned.leading_ones(), 0); + assert_eq!(empty_misaligned.trailing_zeros(), 0); + assert_eq!(empty_misaligned.trailing_ones(), 0); +} + +#[test] +fn attack_bitstring_leading_trailing_ones_complex() { + // Leading ones: k ones followed by a zero. + for k in [1, 2, 63, 64, 65, 127, 128, 129] { + let mut bits = BitString::ones(k + 1); + bits.set(k, false); + assert_eq!( + bits.leading_ones(), + k, + "leading_ones len={} with zero at pos={k}", + k + 1 + ); + } + + // Trailing ones: a zero followed by k ones. + for k in [1, 2, 63, 64, 65, 127, 128, 129] { + let len = k + 1; + let mut bits = BitString::ones(len); + bits.set(0, false); + assert_eq!( + bits.trailing_ones(), + k, + "trailing_ones len={len} with zero at pos=0" + ); + } +} + +// =========================================================================== +// J. Exhaustive leading_ones / trailing_ones — all lengths 0..256 +// =========================================================================== + +#[test] +fn attack_bitstring_leading_trailing_ones_exhaustive() { + // Mirror of Section A: all-ones background with a single zero at each + // position, verifying both leading_ones and trailing_ones. + for len in 0..=256 { + // Pure ones + let all = BitString::ones(len); + assert_eq!(all.leading_ones(), len, "all-ones leading_ones len={len}"); + assert_eq!(all.trailing_ones(), len, "all-ones trailing_ones len={len}"); + + // Single zero at each position + for pos in 0..len { + let mut bits = BitString::ones(len); + bits.set(pos, false); + + let expected_leading = if pos == 0 { 0 } else { pos }; + let expected_trailing = len - pos - 1; + assert_eq!( + bits.leading_ones(), + expected_leading, + "leading_ones len={len} zero@pos={pos}" + ); + assert_eq!( + bits.trailing_ones(), + expected_trailing, + "trailing_ones len={len} zero@pos={pos}" + ); + } + } +} + +// =========================================================================== +// K. Very long strings — exercise SIMD + ALIGN_THRESHOLD boundary +// =========================================================================== + +#[test] +fn attack_bitstring_leading_long_simd() { + // Exercise the AVX2 aligned-load path (total >= 128 words = 8192 bits) + // and the SSE2 unaligned path. Vary lengths above and below + // alignment-sensitive boundaries. + let lengths: &[usize] = &[ + 256, // well below ALIGN_THRESHOLD + 1023, // just under 1k bits + 4096, // 64 words — SSE2 territory, small AVX2 path + 8192, // 128 words — ALIGN_THRESHOLD boundary + 8193, // just over boundary + 10000, // odd size + 16384, // 256 words + 32768, // 512 words + 65535, // odd large + 65536, // 1024 words — well above threshold + ]; + for &len in lengths { + let z = BitString::zeros(len); + assert_eq!(z.leading_zeros(), len, "zeros leading_zeros len={len}"); + assert_eq!(z.leading_ones(), 0, "zeros leading_ones len={len}"); + assert_eq!(z.trailing_zeros(), len, "zeros trailing_zeros len={len}"); + assert_eq!(z.trailing_ones(), 0, "zeros trailing_ones len={len}"); + + let o = BitString::ones(len); + assert_eq!(o.leading_zeros(), 0, "ones leading_zeros len={len}"); + assert_eq!(o.leading_ones(), len, "ones leading_ones len={len}"); + assert_eq!(o.trailing_zeros(), 0, "ones trailing_zeros len={len}"); + assert_eq!(o.trailing_ones(), len, "ones trailing_ones len={len}"); + + // Single break near start / middle / end + for pos in [0, 1, 63, 64, 65, len / 2, len - 2, len - 1] { + if pos >= len { + continue; + } + let mut bits = BitString::ones(len); + bits.set(pos, false); + assert_eq!( + bits.leading_ones(), + pos, + "len={len} zero@pos={pos} leading_ones" + ); + assert_eq!( + bits.trailing_ones(), + len - pos - 1, + "len={len} zero@pos={pos} trailing_ones" + ); + } + } +} + +// =========================================================================== +// L. BitStr deep unaligned — start beyond first word, unaligned offset +// =========================================================================== + +#[test] +fn attack_bitstr_deep_unaligned_views() { + // Create a long backing string, then take sub-views with various + // start offsets (including deep into the backing) and lengths, + // verifying against a round-tripped BitString. + let backing_len = 2048; + let backing = { + let mut bs = BitString::zeros(backing_len); + // Put a pattern: 128 zeros, then alternating groups of 64 ones / 64 zeros + for i in 128..backing_len { + bs.set(i, (i / 64) % 2 == 1); + } + bs + }; + let full = backing.as_bit_str(); + + // Non-word-aligned start offsets deep in the backing + let start_offsets: &[usize] = &[0, 1, 3, 31, 63, 64, 65, 100, 127, 128, 129, 500, 1023]; + for &start in start_offsets { + for &len in &[1, 2, 31, 63, 64, 65, 127, 128, 129, 256] { + if start + len > backing_len { + continue; + } + let view = full.slice(UsizeCO::try_new(start, start + len).unwrap()); + // Round-trip oracle + let rt = bs(&view.to_string()); + assert_eq!( + view.leading_zeros(), + rt.leading_zeros(), + "leading_zeros start={start} len={len}" + ); + assert_eq!( + view.leading_ones(), + rt.leading_ones(), + "leading_ones start={start} len={len}" + ); + assert_eq!( + view.trailing_zeros(), + rt.trailing_zeros(), + "trailing_zeros start={start} len={len}" + ); + assert_eq!( + view.trailing_ones(), + rt.trailing_ones(), + "trailing_ones start={start} len={len}" + ); + } + } + + // All-zeros and all-ones sub-views at deep unaligned starts + for &start in &[0, 7, 63, 64, 65, 128, 129, 500] { + let zs = BitString::zeros(backing_len); + let zf = zs.as_bit_str(); + for &len in &[1, 63, 64, 65, 128, 256] { + if start + len > backing_len { + continue; + } + let zv = zf.slice(UsizeCO::try_new(start, start + len).unwrap()); + assert_eq!( + zv.leading_zeros(), + len, + "all-zeros leading_zeros start={start} len={len}" + ); + assert_eq!( + zv.trailing_zeros(), + len, + "all-zeros trailing_zeros start={start} len={len}" + ); + assert_eq!( + zv.leading_ones(), + 0, + "all-zeros leading_ones start={start} len={len}" + ); + } + } +} + +// =========================================================================== +// M. Leading/trailing cross-word boundary — ones variant +// =========================================================================== + +#[test] +fn attack_bitstring_leading_trailing_ones_cross_word() { + // Mirror Section C for ones: all-ones background with a zero that + // crosses word boundaries. + for n_words in 0..10 { + for extra_bits in [0, 1, 63] { + let k = n_words * 64 + extra_bits; + if k == 0 { + continue; + } + + // leading_ones: k ones, then a zero + let mut bits = BitString::ones(k + 1); + bits.set(k, false); + assert_eq!( + bits.leading_ones(), + k, + "leading_ones k={k} ({} words + {} bits)", + n_words, + extra_bits + ); + + // trailing_ones: a zero, then k ones + let mut bits = BitString::ones(k + 1); + bits.set(0, false); + assert_eq!( + bits.trailing_ones(), + k, + "trailing_ones k={k} ({} words + {} bits)", + n_words, + extra_bits + ); + } + } +} + +// =========================================================================== +// N. Randomized cross-operation invariants +// =========================================================================== + +/// Simple LCG for deterministic pseudo-random bits. +fn lcg_step(state: &mut u64) { + *state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); +} + +#[test] +fn attack_leading_trailing_random_invariants() { + // Generate 200 pseudo-random BitStrings and verify cross-operation + // invariants on both BitString and BitStr views. + let mut rng: u64 = 0xdead_beef_cafe_babe; + for _ in 0..200 { + lcg_step(&mut rng); + let seed = rng; + lcg_step(&mut rng); + let len = (rng as usize % 1024) + 1; + let mut bits = BitString::zeros(len); + for i in 0..len { + lcg_step(&mut rng); + bits.set(i, rng & 1 != 0); + } + + // BitString invariants + let lz = bits.leading_zeros(); + let lo = bits.leading_ones(); + let tz = bits.trailing_zeros(); + let to = bits.trailing_ones(); + assert!(lz <= len, "lz={lz} > len={len} seed={seed}"); + assert!(lo <= len, "lo={lo} > len={len} seed={seed}"); + assert!(tz <= len, "tz={tz} > len={len} seed={seed}"); + assert!(to <= len, "to={to} > len={len} seed={seed}"); + // At least one of lz, lo is 0 (first bit is either 0 or 1) + assert!( + lz == 0 || lo == 0, + "lz={lz} lo={lo} seed={seed} — first bit cannot be both 0 and 1" + ); + // At least one of tz, to is 0 (last bit is either 0 or 1) + assert!( + tz == 0 || to == 0, + "tz={tz} to={to} seed={seed} — last bit cannot be both 0 and 1" + ); + + // BitStr oracle comparison for random sub-views + let full = bits.as_bit_str(); + lcg_step(&mut rng); + let start = (rng as usize) % len; + lcg_step(&mut rng); + let sub_len = if start < len { + ((rng as usize) % (len - start)).max(1) + } else { + 1 + }; + if start + sub_len <= len { + let view = full.slice(UsizeCO::try_new(start, start + sub_len).unwrap()); + let rt = bs(&view.to_string()); + assert_eq!( + view.leading_zeros(), + rt.leading_zeros(), + "random view lz seed={seed} start={start} len={sub_len}" + ); + assert_eq!( + view.trailing_zeros(), + rt.trailing_zeros(), + "random view tz seed={seed} start={start} len={sub_len}" + ); + assert_eq!( + view.leading_ones(), + rt.leading_ones(), + "random view lo seed={seed} start={start} len={sub_len}" + ); + assert_eq!( + view.trailing_ones(), + rt.trailing_ones(), + "random view to seed={seed} start={start} len={sub_len}" + ); + } + } +} + +// =========================================================================== +// O. Stress interleaving mutation with leading/trailing counts +// =========================================================================== + +#[test] +fn attack_leading_trailing_after_mutation() { + // Verify that mutating a BitString (set, push, pop, slice, truncate) + // produces correct counts afterward. This stresses the last-word + // mask invariant and boundary recalculations. + for word_len in [1, 2, 63, 64, 65, 128, 129] { + // Build: word_len zeros, then a single 1 + let mut bits = BitString::zeros(word_len + 1); + bits.set(word_len, true); + + // push more zeros after the 1 + bits.push(false); + bits.push(false); + assert_eq!( + bits.leading_zeros(), + word_len, + "after push zeros, word_len={word_len}" + ); + assert_eq!( + bits.trailing_zeros(), + 2, + "after push zeros trailing, word_len={word_len}" + ); + + // pop them back + bits.pop(); + bits.pop(); + assert_eq!( + bits.leading_zeros(), + word_len, + "after pop, word_len={word_len}" + ); + assert_eq!( + bits.trailing_zeros(), + 0, + "after pop trailing, word_len={word_len}" + ); + + // truncate to the 1-bit + bits.truncate(word_len); + assert_eq!( + bits.leading_zeros(), + word_len, + "after truncate, word_len={word_len}" + ); + assert_eq!( + bits.trailing_zeros(), + word_len, + "after truncate trailing, word_len={word_len}" + ); + + // set a bit near a word boundary + bits.set(0, true); + assert_eq!( + bits.leading_zeros(), + 0, + "after set pos=0, word_len={word_len}" + ); + assert_eq!( + bits.leading_ones(), + 1, + "after set pos=0 ones, word_len={word_len}" + ); + } +} diff --git a/tests/adversarial/tests_for_matching.rs b/tests/adversarial/tests_for_matching.rs index 4b6acec..6a9864a 100644 --- a/tests/adversarial/tests_for_matching.rs +++ b/tests/adversarial/tests_for_matching.rs @@ -7,19 +7,19 @@ fn attack_matches_at_oob() { let pattern = bits.as_bit_str(); // index beyond len - assert!(!bits.matches_at(5, pattern)); - assert!(!bits.matches_at(usize::MAX, pattern)); + assert!(!bits.matches_at_str(5, pattern)); + assert!(!bits.matches_at_str(usize::MAX, pattern)); // pattern longer than remaining let binding = bs("101010"); let pattern = binding.as_bit_str(); // len 6 - assert!(!bits.matches_at(0, pattern)); + assert!(!bits.matches_at_str(0, pattern)); // Empty pattern always matches let empty_binding = BitString::new(); let empty = empty_binding.as_bit_str(); - assert!(bits.matches_at(0, empty)); - assert!(bits.matches_at(5, empty)); + assert!(bits.matches_at_str(0, empty)); + assert!(bits.matches_at_str(5, empty)); } #[test] @@ -27,16 +27,16 @@ fn attack_starts_with_ends_with_edge() { let bits = bs("10101"); // Empty always matches - assert!(bits.starts_with(BitString::new().as_bit_str())); - assert!(bits.ends_with(BitString::new().as_bit_str())); + assert!(bits.starts_with_str(BitString::new().as_bit_str())); + assert!(bits.ends_with_str(BitString::new().as_bit_str())); // Longer than self - assert!(!bits.starts_with(bs("101010").as_bit_str())); - assert!(!bits.ends_with(bs("101010").as_bit_str())); + assert!(!bits.starts_with_str(bs("101010").as_bit_str())); + assert!(!bits.ends_with_str(bs("101010").as_bit_str())); // Exact match - assert!(bits.starts_with(bits.as_bit_str())); - assert!(bits.ends_with(bits.as_bit_str())); + assert!(bits.starts_with_str(bits.as_bit_str())); + assert!(bits.ends_with_str(bits.as_bit_str())); } #[test] @@ -45,8 +45,8 @@ fn attack_find_empty_needle() { let empty_binding2 = BitString::new(); let needle = empty_binding2.as_bit_str(); - assert_eq!(bits.find(needle), Some(0)); - assert_eq!(bits.rfind(needle), Some(5)); // rfind with empty returns len + assert_eq!(bits.find_str(needle), Some(0)); + assert_eq!(bits.rfind_str(needle), Some(5)); // rfind with empty returns len } #[test] @@ -54,9 +54,9 @@ fn attack_find_needle_longer_than_haystack() { let bits = bs("10"); let binding3 = bs("101"); let needle = binding3.as_bit_str(); - assert_eq!(bits.find(needle), None); - assert_eq!(bits.rfind(needle), None); - assert!(!bits.contains(needle)); + assert_eq!(bits.find_str(needle), None); + assert_eq!(bits.rfind_str(needle), None); + assert!(!bits.contains_str(needle)); } #[test] @@ -68,9 +68,9 @@ fn attack_find_at_word_boundaries() { let binding4 = bs("101"); let needle = binding4.as_bit_str(); - assert_eq!(bits.find(needle), Some(128)); - assert_eq!(bits.rfind(needle), Some(128)); - assert!(bits.contains(needle)); + assert_eq!(bits.find_str(needle), Some(128)); + assert_eq!(bits.rfind_str(needle), Some(128)); + assert!(bits.contains_str(needle)); } #[test] @@ -81,10 +81,10 @@ fn attack_find_pattern_at_word_edge() { let pattern = bs("110011"); bits.replace_assign(offset, &pattern); - let found = bits.find(pattern.as_bit_str()); + let found = bits.find_str(pattern.as_bit_str()); assert_eq!(found, Some(offset), "find failed at offset {offset}"); - let rfound = bits.rfind(pattern.as_bit_str()); + let rfound = bits.rfind_str(pattern.as_bit_str()); assert_eq!(rfound, Some(offset), "rfind failed at offset {offset}"); } } @@ -95,13 +95,13 @@ fn attack_find_multiple_occurrences() { let binding5 = bs("101"); let needle = binding5.as_bit_str(); - assert_eq!(bits.find(needle), Some(0)); - assert_eq!(bits.rfind(needle), Some(126)); // 0 + 3 + 60 + 3 + 60 = 126 + assert_eq!(bits.find_str(needle), Some(0)); + assert_eq!(bits.rfind_str(needle), Some(126)); // 0 + 3 + 60 + 3 + 60 = 126 // Find all occurrences let mut pos = 0; let mut count = 0; - while let Some(p) = bits.as_bit_str().slice_from(pos).find(needle) { + while let Some(p) = bits.as_bit_str().slice_from(pos).find_str(needle) { count += 1; pos += p + needle.bit_len(); } @@ -114,23 +114,25 @@ fn attack_strip_prefix_suffix() { // Strip empty assert_eq!( - bits.strip_prefix(BitString::new().as_bit_str()).unwrap(), + bits.strip_prefix_str(BitString::new().as_bit_str()) + .unwrap(), bits ); assert_eq!( - bits.strip_suffix(BitString::new().as_bit_str()).unwrap(), + bits.strip_suffix_str(BitString::new().as_bit_str()) + .unwrap(), bits ); // Mismatch - assert!(bits.strip_prefix(bs("0").as_bit_str()).is_none()); - assert!(bits.strip_suffix(bs("0").as_bit_str()).is_none()); + assert!(bits.strip_prefix_str(bs("0").as_bit_str()).is_none()); + assert!(bits.strip_suffix_str(bs("0").as_bit_str()).is_none()); // Valid strip - let stripped = bits.strip_prefix(bs("10").as_bit_str()).unwrap(); + let stripped = bits.strip_prefix_str(bs("10").as_bit_str()).unwrap(); assert_eq!(stripped.to_string(), "101"); - let stripped = bits.strip_suffix(bs("01").as_bit_str()).unwrap(); + let stripped = bits.strip_suffix_str(bs("01").as_bit_str()).unwrap(); assert_eq!(stripped.to_string(), "101"); } @@ -149,9 +151,9 @@ fn attack_find_on_unaligned_haystack_aligned_needle() { .as_bit_str() .slice(UsizeCO::checked_from_start_len(3, 20).unwrap()); let needle = bs("101"); - assert_eq!(view.find(needle.as_bit_str()), Some(7)); - assert_eq!(view.rfind(needle.as_bit_str()), Some(7)); - assert!(view.contains(needle.as_bit_str())); + assert_eq!(view.find_str(needle.as_bit_str()), Some(7)); + assert_eq!(view.rfind_str(needle.as_bit_str()), Some(7)); + assert!(view.contains_str(needle.as_bit_str())); } #[test] @@ -172,8 +174,8 @@ fn attack_find_on_unaligned_haystack_unaligned_needle() { let hay_view = hay .as_bit_str() .slice(UsizeCO::checked_from_start_len(1, 18).unwrap()); - assert_eq!(hay_view.find(needle_view), Some(4)); - assert!(hay_view.contains(needle_view)); + assert_eq!(hay_view.find_str(needle_view), Some(4)); + assert!(hay_view.contains_str(needle_view)); } #[test] @@ -191,11 +193,11 @@ fn attack_find_unaligned_multiple_occurrences() { .as_bit_str() .slice(UsizeCO::checked_from_start_len(2, 132).unwrap()); let needle = bs("101"); - assert_eq!(view.find(needle.as_bit_str()), Some(3)); - assert_eq!(view.rfind(needle.as_bit_str()), Some(129)); + assert_eq!(view.find_str(needle.as_bit_str()), Some(3)); + assert_eq!(view.rfind_str(needle.as_bit_str()), Some(129)); let mut pos = 0; let mut count = 0; - while let Some(p) = view.slice_from(pos).find(needle.as_bit_str()) { + while let Some(p) = view.slice_from(pos).find_str(needle.as_bit_str()) { count += 1; pos += p + 3; } @@ -212,8 +214,8 @@ fn attack_contains_on_unaligned_short_haystack() { let view = a .as_bit_str() .slice(UsizeCO::checked_from_start_len(37, 20).unwrap()); - assert!(view.contains(bs("101").as_bit_str())); - assert!(!view.contains(bs("111").as_bit_str())); + assert!(view.contains_str(bs("101").as_bit_str())); + assert!(!view.contains_str(bs("111").as_bit_str())); } // =========================================================================== @@ -230,7 +232,7 @@ fn attack_strip_prefix_unaligned() { let view = a .as_bit_str() .slice(UsizeCO::checked_from_start_len(3, 20).unwrap()); - let result = view.strip_prefix(bs("0000000").as_bit_str()); + let result = view.strip_prefix_str(bs("0000000").as_bit_str()); assert!(result.is_some()); assert_eq!(result.unwrap().to_bit_string().to_string(), "1100110000000"); } @@ -245,7 +247,7 @@ fn attack_strip_suffix_unaligned() { let view = a .as_bit_str() .slice(UsizeCO::checked_from_start_len(7, 15).unwrap()); - let result = view.strip_suffix(bs("000000").as_bit_str()); + let result = view.strip_suffix_str(bs("000000").as_bit_str()); assert!(result.is_some()); assert_eq!(result.unwrap().to_bit_string().to_string(), "000101101"); } @@ -280,10 +282,10 @@ fn attack_matches_at_both_unaligned() { .slice(UsizeCO::checked_from_start_len(6, 4).unwrap()); // "0011" // "0011" at relative offset 3 (source 8-11 = "0011") and offset 7 (source 12-15) - assert!(hay_view.matches_at(3, nd_view)); - assert!(hay_view.matches_at(7, nd_view)); + assert!(hay_view.matches_at_str(3, nd_view)); + assert!(hay_view.matches_at_str(7, nd_view)); // Wrong position: "1100" at offset 5 - assert!(!hay_view.matches_at(5, nd_view)); + assert!(!hay_view.matches_at_str(5, nd_view)); } #[test] @@ -300,7 +302,7 @@ fn attack_starts_with_both_unaligned() { let nd_view = nd_src .as_bit_str() .slice(UsizeCO::checked_from_start_len(2, 4).unwrap()); - assert!(view.starts_with(nd_view)); + assert!(view.starts_with_str(nd_view)); } #[test] @@ -313,5 +315,5 @@ fn attack_ends_with_both_unaligned() { let nd_view = nd_src .as_bit_str() .slice(UsizeCO::checked_from_start_len(3, 4).unwrap()); - assert!(view.ends_with(nd_view)); + assert!(view.ends_with_str(nd_view)); }