From 409fd8c3dcbab89f2147b60fd657c50745197b4f Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 15:40:45 -0500 Subject: [PATCH 01/11] Update MSRV --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index faa009581..2a4cbc73a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ keywords = ["hash", "no_std", "hashmap", "swisstable"] categories = ["data-structures", "no-std"] exclude = [".github", "/ci/*"] edition = "2021" -rust-version = "1.65.0" +rust-version = "1.84.0" [dependencies] # For the default hasher From 8c6b6a070ade435a1576b1444c2b24045a3f328a Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 15:22:42 -0500 Subject: [PATCH 02/11] taplo fmt --- Cargo.toml | 17 +++++++++-------- clippy.toml | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2a4cbc73a..3df5bfada 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-all # Support for allocators that use allocator-api2 allocator-api2 = { version = "0.2.9", optional = true, default-features = false, features = [ - "alloc", + "alloc", ] } # Equivalent trait which can be shared with other hash table implementations. @@ -50,7 +50,13 @@ bumpalo = { version = "3.13.0", features = ["allocator-api2"] } libc = "0.2.155" [features] -default = ["default-hasher", "inline-more", "allocator-api2", "equivalent", "raw-entry"] +default = [ + "default-hasher", + "inline-more", + "allocator-api2", + "equivalent", + "raw-entry", +] # Enables use of nightly features. This is only guaranteed to work on the latest # version of nightly Rust. @@ -60,12 +66,7 @@ nightly = ["foldhash?/nightly", "bumpalo/allocator_api"] rustc-internal-api = [] # Internal feature used when building as part of the standard library. -rustc-dep-of-std = [ - "nightly", - "core", - "alloc", - "rustc-internal-api", -] +rustc-dep-of-std = ["nightly", "core", "alloc", "rustc-internal-api"] # Enables serde support. serde = ["dep:serde_core", "dep:serde"] diff --git a/clippy.toml b/clippy.toml index d98bf2c09..10719990a 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1 +1 @@ -doc-valid-idents = [ "CppCon", "SwissTable", "SipHash", "HashDoS" ] +doc-valid-idents = ["CppCon", "SwissTable", "SipHash", "HashDoS"] From ab070dee26a7697fae4995ba16a561fd24eaa26f Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 15:37:35 -0500 Subject: [PATCH 03/11] Move lints to Cargo.toml --- Cargo.toml | 20 ++++++++++++++++++++ ci/tools.sh | 2 +- src/lib.rs | 19 ++----------------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3df5bfada..f25bf2c87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,26 @@ exclude = [".github", "/ci/*"] edition = "2021" rust-version = "1.84.0" +[lints.rust] +missing_docs = "warn" + +# rust_2018_idioms +bare_trait_objects = "warn" +elided_lifetimes_in_paths = "warn" +ellipsis_inclusive_range_patterns = "warn" +explicit_outlives_requirements = "warn" +unused_extern_crates = "warn" + +[lints.clippy] +doc_markdown = "allow" +manual_map = "allow" +missing_errors_doc = "allow" +missing_safety_doc = "allow" +module_name_repetitions = "allow" +must_use_candidate = "allow" +option_if_let_else = "allow" +redundant_else = "allow" + [dependencies] # For the default hasher foldhash = { version = "0.2.0", default-features = false, optional = true } diff --git a/ci/tools.sh b/ci/tools.sh index b23e2d71c..1b4d55c8d 100644 --- a/ci/tools.sh +++ b/ci/tools.sh @@ -30,7 +30,7 @@ if retry rustup component add rustfmt ; then fi if retry rustup component add clippy ; then - cargo clippy --all --tests --features serde,rayon -- -D clippy::all + cargo clippy --all --tests --features serde,rayon -- -D warnings fi if command -v shellcheck ; then diff --git a/src/lib.rs b/src/lib.rs index c5faec43a..d6ba501f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,24 +25,9 @@ strict_provenance_lints ) )] -#![cfg_attr(feature = "rustc-dep-of-std", feature(rustc_attrs))] -#![allow( - clippy::doc_markdown, - clippy::module_name_repetitions, - clippy::must_use_candidate, - clippy::option_if_let_else, - clippy::redundant_else, - clippy::manual_map, - clippy::missing_safety_doc, - clippy::missing_errors_doc -)] -#![warn(missing_docs)] -#![warn(rust_2018_idioms)] #![cfg_attr(feature = "nightly", warn(fuzzy_provenance_casts))] -#![cfg_attr( - feature = "nightly", - allow(clippy::incompatible_msrv, internal_features) -)] +#![cfg_attr(feature = "rustc-dep-of-std", feature(rustc_attrs))] +#![cfg_attr(feature = "nightly", allow(internal_features))] #![cfg_attr( all(feature = "nightly", target_arch = "loongarch64"), feature(stdarch_loongarch) From 49dd86e5db5eec29843fe3eb5e41895f7fac519b Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 15:23:30 -0500 Subject: [PATCH 04/11] Clippy fix --- benches/bench.rs | 8 ++++---- benches/with_capacity.rs | 1 + src/external_trait_impls/rayon/map.rs | 4 ++-- src/external_trait_impls/rayon/set.rs | 2 +- src/map.rs | 20 +++----------------- src/raw/mod.rs | 2 +- tests/equivalent_trait.rs | 2 ++ tests/rayon.rs | 1 + tests/serde.rs | 1 + tests/set.rs | 1 + 10 files changed, 17 insertions(+), 25 deletions(-) diff --git a/benches/bench.rs b/benches/bench.rs index ce3aee5ce..14bf1aeb9 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -1,7 +1,7 @@ -// This benchmark suite contains some benchmarks along a set of dimensions: -// Hasher: std default (SipHash) and crate default (foldhash). -// Int key distribution: low bit heavy, top bit heavy, and random. -// Task: basic functionality: insert, insert_erase, lookup, lookup_fail, iter +//! This benchmark suite contains some benchmarks along a set of dimensions: +//! * Hasher: std default (SipHash) and crate default (foldhash). +//! * Int key distribution: low bit heavy, top bit heavy, and random. +//! * Task: basic functionality: insert, insert_erase, lookup, lookup_fail, iter #![feature(test)] extern crate test; diff --git a/benches/with_capacity.rs b/benches/with_capacity.rs index eeb85b59a..933276387 100644 --- a/benches/with_capacity.rs +++ b/benches/with_capacity.rs @@ -1,3 +1,4 @@ +#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![feature(test)] extern crate test; diff --git a/src/external_trait_impls/rayon/map.rs b/src/external_trait_impls/rayon/map.rs index 9623ca747..e12080d0d 100644 --- a/src/external_trait_impls/rayon/map.rs +++ b/src/external_trait_impls/rayon/map.rs @@ -346,7 +346,7 @@ where self.len() == other.len() && self .into_par_iter() - .all(|(key, value)| other.get(key).map_or(false, |v| *value == *v)) + .all(|(key, value)| other.get(key).is_some_and(|v| *value == *v)) } } @@ -455,7 +455,7 @@ where // Reserve the entire length if the map is empty. // Otherwise reserve half the length (rounded up), so the map // will only resize twice in the worst case. - let reserve = if map.is_empty() { len } else { (len + 1) / 2 }; + let reserve = if map.is_empty() { len } else { len.div_ceil(2) }; map.reserve(reserve); for vec in list { map.extend(vec); diff --git a/src/external_trait_impls/rayon/set.rs b/src/external_trait_impls/rayon/set.rs index 3de98fccb..8a3bfa481 100644 --- a/src/external_trait_impls/rayon/set.rs +++ b/src/external_trait_impls/rayon/set.rs @@ -384,7 +384,7 @@ where // Reserve the entire length if the set is empty. // Otherwise reserve half the length (rounded up), so the set // will only resize twice in the worst case. - let reserve = if set.is_empty() { len } else { (len + 1) / 2 }; + let reserve = if set.is_empty() { len } else { len.div_ceil(2) }; set.reserve(reserve); for vec in list { set.extend(vec); diff --git a/src/map.rs b/src/map.rs index f37bafa02..c4430a86f 100644 --- a/src/map.rs +++ b/src/map.rs @@ -236,20 +236,6 @@ where move |x| k.equivalent(x) } -#[cfg(not(feature = "nightly"))] -#[cfg_attr(feature = "inline-more", inline)] -pub(crate) fn make_hash(hash_builder: &S, val: &Q) -> u64 -where - Q: Hash + ?Sized, - S: BuildHasher, -{ - use core::hash::Hasher; - let mut state = hash_builder.build_hasher(); - val.hash(&mut state); - state.finish() -} - -#[cfg(feature = "nightly")] #[cfg_attr(feature = "inline-more", inline)] pub(crate) fn make_hash(hash_builder: &S, val: &Q) -> u64 where @@ -2074,7 +2060,7 @@ where } self.iter() - .all(|(key, value)| other.get(key).map_or(false, |v| *value == *v)) + .all(|(key, value)| other.get(key).is_some_and(|v| *value == *v)) } } @@ -4732,7 +4718,7 @@ where let reserve = if self.is_empty() { iter.size_hint().0 } else { - (iter.size_hint().0 + 1) / 2 + iter.size_hint().0.div_ceil(2) }; self.reserve(reserve); iter.for_each(move |(k, v)| { @@ -4756,7 +4742,7 @@ where let reserve = if self.is_empty() { additional } else { - (additional + 1) / 2 + additional.div_ceil(2) }; self.reserve(reserve); } diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 1f21feca5..9809adcce 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -4561,7 +4561,7 @@ mod test_map { }), }); - for (idx, panic_in_clone) in core::iter::repeat(DISARMED).take(7).enumerate() { + for (idx, panic_in_clone) in core::iter::repeat_n(DISARMED, 7).enumerate() { let idx = idx as u64; table.insert( idx, diff --git a/tests/equivalent_trait.rs b/tests/equivalent_trait.rs index 713dddd53..7ffa8d006 100644 --- a/tests/equivalent_trait.rs +++ b/tests/equivalent_trait.rs @@ -1,3 +1,5 @@ +#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 + use hashbrown::Equivalent; use hashbrown::HashMap; diff --git a/tests/rayon.rs b/tests/rayon.rs index d55e5a980..98cdf42f3 100644 --- a/tests/rayon.rs +++ b/tests/rayon.rs @@ -1,3 +1,4 @@ +#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![cfg(feature = "rayon")] #[macro_use] diff --git a/tests/serde.rs b/tests/serde.rs index a642348b3..d792b17ae 100644 --- a/tests/serde.rs +++ b/tests/serde.rs @@ -1,3 +1,4 @@ +#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![cfg(feature = "serde")] use core::hash::BuildHasherDefault; diff --git a/tests/set.rs b/tests/set.rs index d25f3d459..f5bb9fae8 100644 --- a/tests/set.rs +++ b/tests/set.rs @@ -1,3 +1,4 @@ +#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![cfg(not(miri))] // FIXME: takes too long use hashbrown::HashSet; From fa49f3fcd22253eccfc5d50bd4e3a34a473a2a4e Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 16:27:56 -0500 Subject: [PATCH 05/11] Apparently CI hard-coded 1.80 as a fixed point, now it's just 1.84 --- .github/workflows/rust.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d2f61c3b3..9ed7c9ef7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -54,7 +54,7 @@ jobs: # Disabled due to issues with cross-rs #x86_64-pc-windows-gnu, ] - channel: [1.80.0, nightly] + channel: [1.84.0, nightly] include: - os: macos-latest target: aarch64-apple-darwin @@ -64,10 +64,10 @@ jobs: channel: nightly - os: macos-latest target: aarch64-apple-darwin - channel: 1.80.0 + channel: 1.84.0 - os: windows-latest target: x86_64-pc-windows-msvc - channel: 1.80.0 + channel: 1.84.0 - os: ubuntu-latest target: x86_64-unknown-linux-gnu channel: beta From 9839a9f72de653f0ef1ae7a2af1e1d3cb6a8c753 Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 16:31:56 -0500 Subject: [PATCH 06/11] Enable unsafe_op_in_unsafe_fn for 2024 compatibility, clippy fix --- Cargo.toml | 1 + src/control/group/generic.rs | 14 +- src/control/group/lsx.rs | 14 +- src/control/group/neon.rs | 14 +- src/control/group/sse2.rs | 14 +- src/external_trait_impls/rayon/raw.rs | 10 +- src/map.rs | 40 +- src/raw/alloc.rs | 4 +- src/raw/mod.rs | 1673 +++++++++++++------------ src/set.rs | 2 +- src/table.rs | 16 +- 11 files changed, 965 insertions(+), 837 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f25bf2c87..cf2440710 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ rust-version = "1.84.0" [lints.rust] missing_docs = "warn" +unsafe_op_in_unsafe_fn = "warn" # rust_2018_idioms bare_trait_objects = "warn" diff --git a/src/control/group/generic.rs b/src/control/group/generic.rs index ecb81d278..de8c9abba 100644 --- a/src/control/group/generic.rs +++ b/src/control/group/generic.rs @@ -72,7 +72,7 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { - Group(ptr::read_unaligned(ptr.cast())) + unsafe { Group(ptr::read_unaligned(ptr.cast())) } } /// Loads a group of tags starting at the given address, which must be @@ -80,8 +80,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(ptr::read(ptr.cast())) + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + Group(ptr::read(ptr.cast())) + } } /// Stores the group of tags to the given address, which must be @@ -89,8 +91,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - ptr::write(ptr.cast(), self.0); + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + ptr::write(ptr.cast(), self.0); + } } /// Returns a `BitMask` indicating all tags in the group which *may* diff --git a/src/control/group/lsx.rs b/src/control/group/lsx.rs index dcac09a0c..2cd053b6c 100644 --- a/src/control/group/lsx.rs +++ b/src/control/group/lsx.rs @@ -46,7 +46,7 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { - Group(lsx_vld::<0>(ptr.cast())) + unsafe { Group(lsx_vld::<0>(ptr.cast())) } } /// Loads a group of tags starting at the given address, which must be @@ -54,8 +54,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(lsx_vld::<0>(ptr.cast())) + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + Group(lsx_vld::<0>(ptr.cast())) + } } /// Stores the group of tags to the given address, which must be @@ -63,8 +65,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - lsx_vst::<0>(self.0, ptr.cast()); + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + lsx_vst::<0>(self.0, ptr.cast()); + } } /// Returns a `BitMask` indicating all tags in the group which have diff --git a/src/control/group/neon.rs b/src/control/group/neon.rs index 9374cb388..99c6c888f 100644 --- a/src/control/group/neon.rs +++ b/src/control/group/neon.rs @@ -43,7 +43,7 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { - Group(neon::vld1_u8(ptr.cast())) + unsafe { Group(neon::vld1_u8(ptr.cast())) } } /// Loads a group of tags starting at the given address, which must be @@ -51,8 +51,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(neon::vld1_u8(ptr.cast())) + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + Group(neon::vld1_u8(ptr.cast())) + } } /// Stores the group of tags to the given address, which must be @@ -60,8 +62,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - neon::vst1_u8(ptr.cast(), self.0); + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + neon::vst1_u8(ptr.cast(), self.0); + } } /// Returns a `BitMask` indicating all tags in the group which *may* diff --git a/src/control/group/sse2.rs b/src/control/group/sse2.rs index 0d4b10822..95f537b1a 100644 --- a/src/control/group/sse2.rs +++ b/src/control/group/sse2.rs @@ -49,7 +49,7 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { - Group(x86::_mm_loadu_si128(ptr.cast())) + unsafe { Group(x86::_mm_loadu_si128(ptr.cast())) } } /// Loads a group of tags starting at the given address, which must be @@ -57,8 +57,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(x86::_mm_load_si128(ptr.cast())) + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + Group(x86::_mm_load_si128(ptr.cast())) + } } /// Stores the group of tags to the given address, which must be @@ -66,8 +68,10 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - x86::_mm_store_si128(ptr.cast(), self.0); + unsafe { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + x86::_mm_store_si128(ptr.cast(), self.0); + } } /// Returns a `BitMask` indicating all tags in the group which have diff --git a/src/external_trait_impls/rayon/raw.rs b/src/external_trait_impls/rayon/raw.rs index dbe1cdef7..1e2a5cc7f 100644 --- a/src/external_trait_impls/rayon/raw.rs +++ b/src/external_trait_impls/rayon/raw.rs @@ -82,7 +82,7 @@ pub(crate) struct RawIntoParIter { impl RawIntoParIter { #[cfg_attr(feature = "inline-more", inline)] pub(super) unsafe fn par_iter(&self) -> RawParIter { - self.table.par_iter() + unsafe { self.table.par_iter() } } } @@ -120,7 +120,7 @@ unsafe impl Send for RawParDrain<'_, T, A> {} impl RawParDrain<'_, T, A> { #[cfg_attr(feature = "inline-more", inline)] pub(super) unsafe fn par_iter(&self) -> RawParIter { - self.table.as_ref().par_iter() + unsafe { self.table.as_ref().par_iter() } } } @@ -207,8 +207,10 @@ impl RawTable { /// Returns a parallel iterator over the elements in a `RawTable`. #[cfg_attr(feature = "inline-more", inline)] pub(crate) unsafe fn par_iter(&self) -> RawParIter { - RawParIter { - iter: self.iter().iter, + unsafe { + RawParIter { + iter: self.iter().iter, + } } } diff --git a/src/map.rs b/src/map.rs index c4430a86f..5afbe2c23 100644 --- a/src/map.rs +++ b/src/map.rs @@ -1594,8 +1594,10 @@ where where Q: Hash + Equivalent + ?Sized, { - self.get_disjoint_unchecked_mut_inner(ks) - .map(|res| res.map(|(_, v)| v)) + unsafe { + self.get_disjoint_unchecked_mut_inner(ks) + .map(|res| res.map(|(_, v)| v)) + } } /// Attempts to get mutable references to `N` values in the map at once, without validating that @@ -1608,7 +1610,7 @@ where where Q: Hash + Equivalent + ?Sized, { - self.get_disjoint_unchecked_mut(ks) + unsafe { self.get_disjoint_unchecked_mut(ks) } } /// Attempts to get mutable references to `N` values in the map at once, with immutable @@ -1746,8 +1748,10 @@ where where Q: Hash + Equivalent + ?Sized, { - self.get_disjoint_unchecked_mut_inner(ks) - .map(|res| res.map(|(k, v)| (&*k, v))) + unsafe { + self.get_disjoint_unchecked_mut_inner(ks) + .map(|res| res.map(|(k, v)| (&*k, v))) + } } /// Attempts to get mutable references to `N` values in the map at once, with immutable @@ -1760,7 +1764,7 @@ where where Q: Hash + Equivalent + ?Sized, { - self.get_disjoint_key_value_unchecked_mut(ks) + unsafe { self.get_disjoint_key_value_unchecked_mut(ks) } } fn get_disjoint_mut_inner( @@ -1782,9 +1786,11 @@ where where Q: Hash + Equivalent + ?Sized, { - let hashes = self.build_hashes_inner(ks); - self.table - .get_disjoint_unchecked_mut(hashes, |i, (k, _)| ks[i].equivalent(k)) + unsafe { + let hashes = self.build_hashes_inner(ks); + self.table + .get_disjoint_unchecked_mut(hashes, |i, (k, _)| ks[i].equivalent(k)) + } } fn build_hashes_inner(&self, ks: [&Q; N]) -> [u64; N] @@ -6375,8 +6381,10 @@ mod test_map { } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - let g = Global; - g.deallocate(ptr, layout) + unsafe { + let g = Global; + g.deallocate(ptr, layout) + } } } @@ -6830,10 +6838,12 @@ mod test_map_with_mmap_allocations { } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - // If they allocated it with this layout, it must round correctly. - let size = self.fit_to_page_size(layout.size()).unwrap(); - let _result = libc::munmap(ptr.as_ptr().cast(), size); - debug_assert_eq!(0, _result) + unsafe { + // If they allocated it with this layout, it must round correctly. + let size = self.fit_to_page_size(layout.size()).unwrap(); + let _result = libc::munmap(ptr.as_ptr().cast(), size); + debug_assert_eq!(0, _result) + } } } diff --git a/src/raw/alloc.rs b/src/raw/alloc.rs index 936d13b7c..eabafa0f6 100644 --- a/src/raw/alloc.rs +++ b/src/raw/alloc.rs @@ -86,7 +86,9 @@ mod inner { } #[inline] unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - dealloc(ptr.as_ptr(), layout); + unsafe { + dealloc(ptr.as_ptr(), layout); + } } } diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 9809adcce..f322788fa 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -18,7 +18,7 @@ pub(crate) use self::alloc::{do_alloc, Allocator, Global}; #[inline] unsafe fn offset_from(to: *const T, from: *const T) -> usize { - to.offset_from(from) as usize + unsafe { to.offset_from(from) as usize } } /// Whether memory allocation errors should return an error or abort. @@ -310,36 +310,38 @@ impl Bucket { /// [`RawTableInner::num_buckets`]: RawTableInner::num_buckets #[inline] unsafe fn from_base_index(base: NonNull, index: usize) -> Self { - // If mem::size_of::() != 0 then return a pointer to an `element` in - // the data part of the table (we start counting from "0", so that - // in the expression T[last], the "last" index actually one less than the - // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"): - // - // `from_base_index(base, 1).as_ptr()` returns a pointer that - // points here in the data part of the table - // (to the start of T1) - // | - // | `base: NonNull` must point here - // | (to the end of T0 or to the start of C0) - // v v - // [Padding], Tlast, ..., |T1|, T0, |C0, C1, ..., Clast - // ^ - // `from_base_index(base, 1)` returns a pointer - // that points here in the data part of the table - // (to the end of T1) - // - // where: T0...Tlast - our stored data; C0...Clast - control bytes - // or metadata for data. - let ptr = if T::IS_ZERO_SIZED { - // won't overflow because index must be less than length (bucket_mask) - // and bucket_mask is guaranteed to be less than `isize::MAX` - // (see TableLayout::calculate_layout_for method) - invalid_mut(index + 1) - } else { - base.as_ptr().sub(index) - }; - Self { - ptr: NonNull::new_unchecked(ptr), + unsafe { + // If mem::size_of::() != 0 then return a pointer to an `element` in + // the data part of the table (we start counting from "0", so that + // in the expression T[last], the "last" index actually one less than the + // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"): + // + // `from_base_index(base, 1).as_ptr()` returns a pointer that + // points here in the data part of the table + // (to the start of T1) + // | + // | `base: NonNull` must point here + // | (to the end of T0 or to the start of C0) + // v v + // [Padding], Tlast, ..., |T1|, T0, |C0, C1, ..., Clast + // ^ + // `from_base_index(base, 1)` returns a pointer + // that points here in the data part of the table + // (to the end of T1) + // + // where: T0...Tlast - our stored data; C0...Clast - control bytes + // or metadata for data. + let ptr = if T::IS_ZERO_SIZED { + // won't overflow because index must be less than length (bucket_mask) + // and bucket_mask is guaranteed to be less than `isize::MAX` + // (see TableLayout::calculate_layout_for method) + invalid_mut(index + 1) + } else { + base.as_ptr().sub(index) + }; + Self { + ptr: NonNull::new_unchecked(ptr), + } } } @@ -383,32 +385,34 @@ impl Bucket { /// [`<*const T>::offset_from`]: https://doc.rust-lang.org/nightly/core/primitive.pointer.html#method.offset_from #[inline] unsafe fn to_base_index(&self, base: NonNull) -> usize { - // If mem::size_of::() != 0 then return an index under which we used to store the - // `element` in the data part of the table (we start counting from "0", so - // that in the expression T[last], the "last" index actually is one less than the - // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"). - // For example for 5th element in table calculation is performed like this: - // - // mem::size_of::() - // | - // | `self = from_base_index(base, 5)` that returns pointer - // | that points here in the data part of the table - // | (to the end of T5) - // | | `base: NonNull` must point here - // v | (to the end of T0 or to the start of C0) - // /???\ v v - // [Padding], Tlast, ..., |T10|, ..., T5|, T4, T3, T2, T1, T0, |C0, C1, C2, C3, C4, C5, ..., C10, ..., Clast - // \__________ __________/ - // \/ - // `bucket.to_base_index(base)` = 5 - // (base.as_ptr() as usize - self.ptr.as_ptr() as usize) / mem::size_of::() - // - // where: T0...Tlast - our stored data; C0...Clast - control bytes or metadata for data. - if T::IS_ZERO_SIZED { - // this can not be UB - self.ptr.as_ptr() as usize - 1 - } else { - offset_from(base.as_ptr(), self.ptr.as_ptr()) + unsafe { + // If mem::size_of::() != 0 then return an index under which we used to store the + // `element` in the data part of the table (we start counting from "0", so + // that in the expression T[last], the "last" index actually is one less than the + // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"). + // For example for 5th element in table calculation is performed like this: + // + // mem::size_of::() + // | + // | `self = from_base_index(base, 5)` that returns pointer + // | that points here in the data part of the table + // | (to the end of T5) + // | | `base: NonNull` must point here + // v | (to the end of T0 or to the start of C0) + // /???\ v v + // [Padding], Tlast, ..., |T10|, ..., T5|, T4, T3, T2, T1, T0, |C0, C1, C2, C3, C4, C5, ..., C10, ..., Clast + // \__________ __________/ + // \/ + // `bucket.to_base_index(base)` = 5 + // (base.as_ptr() as usize - self.ptr.as_ptr() as usize) / mem::size_of::() + // + // where: T0...Tlast - our stored data; C0...Clast - control bytes or metadata for data. + if T::IS_ZERO_SIZED { + // this can not be UB + self.ptr.as_ptr() as usize - 1 + } else { + offset_from(base.as_ptr(), self.ptr.as_ptr()) + } } } @@ -488,14 +492,16 @@ impl Bucket { /// [`RawTableInner::num_buckets`]: RawTableInner::num_buckets #[inline] unsafe fn next_n(&self, offset: usize) -> Self { - let ptr = if T::IS_ZERO_SIZED { - // invalid pointer is good enough for ZST - invalid_mut(self.ptr.as_ptr() as usize + offset) - } else { - self.ptr.as_ptr().sub(offset) - }; - Self { - ptr: NonNull::new_unchecked(ptr), + unsafe { + let ptr = if T::IS_ZERO_SIZED { + // invalid pointer is good enough for ZST + invalid_mut(self.ptr.as_ptr() as usize + offset) + } else { + self.ptr.as_ptr().sub(offset) + }; + Self { + ptr: NonNull::new_unchecked(ptr), + } } } @@ -516,7 +522,9 @@ impl Bucket { /// [`RawTable::erase`]: crate::raw::RawTable::erase #[cfg_attr(feature = "inline-more", inline)] pub(crate) unsafe fn drop(&self) { - self.as_ptr().drop_in_place(); + unsafe { + self.as_ptr().drop_in_place(); + } } /// Reads the `value` from `self` without moving it. This leaves the @@ -537,7 +545,7 @@ impl Bucket { /// [`RawTable::remove`]: crate::raw::RawTable::remove #[inline] pub(crate) unsafe fn read(&self) -> T { - self.as_ptr().read() + unsafe { self.as_ptr().read() } } /// Overwrites a memory location with the given `value` without reading @@ -559,7 +567,9 @@ impl Bucket { /// [`Eq`]: https://doc.rust-lang.org/core/cmp/trait.Eq.html #[inline] pub(crate) unsafe fn write(&self, val: T) { - self.as_ptr().write(val); + unsafe { + self.as_ptr().write(val); + } } /// Returns a shared immutable reference to the `value`. @@ -571,7 +581,7 @@ impl Bucket { /// [`NonNull::as_ref`]: https://doc.rust-lang.org/core/ptr/struct.NonNull.html#method.as_ref #[inline] pub(crate) unsafe fn as_ref<'a>(&self) -> &'a T { - &*self.as_ptr() + unsafe { &*self.as_ptr() } } /// Returns a unique mutable reference to the `value`. @@ -592,7 +602,7 @@ impl Bucket { /// [`Eq`]: https://doc.rust-lang.org/core/cmp/trait.Eq.html #[inline] pub(crate) unsafe fn as_mut<'a>(&self) -> &'a mut T { - &mut *self.as_ptr() + unsafe { &mut *self.as_ptr() } } } @@ -673,18 +683,20 @@ impl RawTable { buckets: usize, fallibility: Fallibility, ) -> Result { - debug_assert!(buckets.is_power_of_two()); - - Ok(Self { - table: RawTableInner::new_uninitialized( - &alloc, - Self::TABLE_LAYOUT, - buckets, - fallibility, - )?, - alloc, - marker: PhantomData, - }) + unsafe { + debug_assert!(buckets.is_power_of_two()); + + Ok(Self { + table: RawTableInner::new_uninitialized( + &alloc, + Self::TABLE_LAYOUT, + buckets, + fallibility, + )?, + alloc, + marker: PhantomData, + }) + } } /// Allocates a new hash table using the given allocator, with at least enough capacity for @@ -736,7 +748,7 @@ impl RawTable { #[inline] #[cfg(feature = "nightly")] pub(crate) unsafe fn data_start(&self) -> NonNull { - NonNull::new_unchecked(self.data_end().as_ptr().wrapping_sub(self.num_buckets())) + unsafe { NonNull::new_unchecked(self.data_end().as_ptr().wrapping_sub(self.num_buckets())) } } /// Returns the total amount of memory allocated internally by the hash @@ -754,7 +766,7 @@ impl RawTable { /// Returns the index of a bucket from a `Bucket`. #[inline] pub(crate) unsafe fn bucket_index(&self, bucket: &Bucket) -> usize { - bucket.to_base_index(self.data_end()) + unsafe { bucket.to_base_index(self.data_end()) } } /// Returns a pointer to an element in the table. @@ -783,49 +795,55 @@ impl RawTable { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] pub(crate) unsafe fn bucket(&self, index: usize) -> Bucket { - // If mem::size_of::() != 0 then return a pointer to the `element` in the `data part` of the table - // (we start counting from "0", so that in the expression T[n], the "n" index actually one less than - // the "buckets" number of our `RawTable`, i.e. "n = RawTable::num_buckets() - 1"): - // - // `table.bucket(3).as_ptr()` returns a pointer that points here in the `data` - // part of the `RawTable`, i.e. to the start of T3 (see `Bucket::as_ptr`) - // | - // | `base = self.data_end()` points here - // | (to the start of CT0 or to the end of T0) - // v v - // [Pad], T_n, ..., |T3|, T2, T1, T0, |CT0, CT1, CT2, CT3, ..., CT_n, CTa_0, CTa_1, ..., CTa_m - // ^ \__________ __________/ - // `table.bucket(3)` returns a pointer that points \/ - // here in the `data` part of the `RawTable` (to additional control bytes - // the end of T3) `m = Group::WIDTH - 1` - // - // where: T0...T_n - our stored data; - // CT0...CT_n - control bytes or metadata for `data`; - // CTa_0...CTa_m - additional control bytes (so that the search with loading `Group` bytes from - // the heap works properly, even if the result of `h1(hash) & self.table.bucket_mask` - // is equal to `self.table.bucket_mask`). See also `RawTableInner::set_ctrl` function. - // - // P.S. `h1(hash) & self.table.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number - // of buckets is a power of two, and `self.table.bucket_mask = self.num_buckets() - 1`. - debug_assert_ne!(self.table.bucket_mask, 0); - debug_assert!(index < self.num_buckets()); - Bucket::from_base_index(self.data_end(), index) + unsafe { + // If mem::size_of::() != 0 then return a pointer to the `element` in the `data part` of the table + // (we start counting from "0", so that in the expression T[n], the "n" index actually one less than + // the "buckets" number of our `RawTable`, i.e. "n = RawTable::num_buckets() - 1"): + // + // `table.bucket(3).as_ptr()` returns a pointer that points here in the `data` + // part of the `RawTable`, i.e. to the start of T3 (see `Bucket::as_ptr`) + // | + // | `base = self.data_end()` points here + // | (to the start of CT0 or to the end of T0) + // v v + // [Pad], T_n, ..., |T3|, T2, T1, T0, |CT0, CT1, CT2, CT3, ..., CT_n, CTa_0, CTa_1, ..., CTa_m + // ^ \__________ __________/ + // `table.bucket(3)` returns a pointer that points \/ + // here in the `data` part of the `RawTable` (to additional control bytes + // the end of T3) `m = Group::WIDTH - 1` + // + // where: T0...T_n - our stored data; + // CT0...CT_n - control bytes or metadata for `data`; + // CTa_0...CTa_m - additional control bytes (so that the search with loading `Group` bytes from + // the heap works properly, even if the result of `h1(hash) & self.table.bucket_mask` + // is equal to `self.table.bucket_mask`). See also `RawTableInner::set_ctrl` function. + // + // P.S. `h1(hash) & self.table.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number + // of buckets is a power of two, and `self.table.bucket_mask = self.num_buckets() - 1`. + debug_assert_ne!(self.table.bucket_mask, 0); + debug_assert!(index < self.num_buckets()); + Bucket::from_base_index(self.data_end(), index) + } } /// Erases an element from the table without dropping it. #[cfg_attr(feature = "inline-more", inline)] unsafe fn erase_no_drop(&mut self, item: &Bucket) { - let index = self.bucket_index(item); - self.table.erase(index); + unsafe { + let index = self.bucket_index(item); + self.table.erase(index); + } } /// Erases an element from the table, dropping it in place. #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::needless_pass_by_value)] pub(crate) unsafe fn erase(&mut self, item: Bucket) { - // Erase the element from the table first since drop might panic. - self.erase_no_drop(&item); - item.drop(); + unsafe { + // Erase the element from the table first since drop might panic. + self.erase_no_drop(&item); + item.drop(); + } } /// Removes an element from the table, returning it. @@ -834,8 +852,10 @@ impl RawTable { #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::needless_pass_by_value)] pub(crate) unsafe fn remove(&mut self, item: Bucket) -> (T, usize) { - self.erase_no_drop(&item); - (item.read(), self.bucket_index(&item)) + unsafe { + self.erase_no_drop(&item); + (item.read(), self.bucket_index(&item)) + } } /// Removes an element from the table, returning it. @@ -845,10 +865,12 @@ impl RawTable { #[cfg_attr(feature = "inline-more", inline)] #[allow(clippy::needless_pass_by_value)] pub(crate) unsafe fn remove_tagged(&mut self, item: Bucket) -> (T, usize, Tag) { - let index = self.bucket_index(&item); - let tag = *self.table.ctrl(index); - self.table.erase(index); - (item.read(), index, tag) + unsafe { + let index = self.bucket_index(&item); + let tag = *self.table.ctrl(index); + self.table.erase(index); + (item.read(), index, tag) + } } /// Finds and removes an element from the table, returning it. @@ -1052,19 +1074,21 @@ impl RawTable { hasher: impl Fn(&T) -> u64, fallibility: Fallibility, ) -> Result<(), TryReserveError> { - // SAFETY: - // 1. The caller of this function guarantees that `capacity >= self.table.items`. - // 2. We know for sure that `alloc` and `layout` matches the [`Allocator`] and - // [`TableLayout`] that were used to allocate this table. - // 3. The caller ensures that the control bytes of the `RawTableInner` - // are already initialized. - self.table.resize_inner( - &self.alloc, - capacity, - &|table, index| hasher(table.bucket::(index).as_ref()), - fallibility, - Self::TABLE_LAYOUT, - ) + unsafe { + // SAFETY: + // 1. The caller of this function guarantees that `capacity >= self.table.items`. + // 2. We know for sure that `alloc` and `layout` matches the [`Allocator`] and + // [`TableLayout`] that were used to allocate this table. + // 3. The caller ensures that the control bytes of the `RawTableInner` + // are already initialized. + self.table.resize_inner( + &self.alloc, + capacity, + &|table, index| hasher(table.bucket::(index).as_ref()), + fallibility, + Self::TABLE_LAYOUT, + ) + } } /// Inserts a new element into the table, and returns its raw bucket. @@ -1117,16 +1141,18 @@ impl RawTable { #[cfg_attr(feature = "inline-more", inline)] #[cfg(feature = "rustc-internal-api")] pub(crate) unsafe fn insert_no_grow(&mut self, hash: u64, value: T) -> Bucket { - let (index, old_ctrl) = self.table.prepare_insert_index(hash); - let bucket = self.table.bucket(index); + unsafe { + let (index, old_ctrl) = self.table.prepare_insert_index(hash); + let bucket = self.table.bucket(index); - // If we are replacing a DELETED entry then we don't need to update - // the load counter. - self.table.growth_left -= old_ctrl.special_is_empty() as usize; + // If we are replacing a DELETED entry then we don't need to update + // the load counter. + self.table.growth_left -= old_ctrl.special_is_empty() as usize; - bucket.write(value); - self.table.items += 1; - bucket + bucket.write(value); + self.table.items += 1; + bucket + } } /// Temporarily removes a bucket, applying the given function to the removed @@ -1140,19 +1166,21 @@ impl RawTable { where F: FnOnce(T) -> Option, { - let index = self.bucket_index(&bucket); - let old_ctrl = *self.table.ctrl(index); - debug_assert!(self.is_bucket_full(index)); - let old_growth_left = self.table.growth_left; - let item = self.remove(bucket).0; - if let Some(new_item) = f(item) { - self.table.growth_left = old_growth_left; - self.table.set_ctrl(index, old_ctrl); - self.table.items += 1; - self.bucket(index).write(new_item); - None - } else { - Some(old_ctrl) + unsafe { + let index = self.bucket_index(&bucket); + let old_ctrl = *self.table.ctrl(index); + debug_assert!(self.is_bucket_full(index)); + let old_growth_left = self.table.growth_left; + let item = self.remove(bucket).0; + if let Some(new_item) = f(item) { + self.table.growth_left = old_growth_left; + self.table.set_ctrl(index, old_ctrl); + self.table.items += 1; + self.bucket(index).write(new_item); + None + } else { + Some(old_ctrl) + } } } @@ -1205,7 +1233,7 @@ impl RawTable { index: usize, value: T, ) -> Bucket { - self.insert_tagged_at_index(Tag::full(hash), index, value) + unsafe { self.insert_tagged_at_index(Tag::full(hash), index, value) } } /// Inserts a new element into the table at the given index with the given tag, @@ -1223,12 +1251,14 @@ impl RawTable { index: usize, value: T, ) -> Bucket { - let old_ctrl = *self.table.ctrl(index); - self.table.record_item_insert_at(index, old_ctrl, tag); + unsafe { + let old_ctrl = *self.table.ctrl(index); + self.table.record_item_insert_at(index, old_ctrl, tag); - let bucket = self.bucket(index); - bucket.write(value); - bucket + let bucket = self.bucket(index); + bucket.write(value); + bucket + } } /// Searches for an element in the table. @@ -1345,8 +1375,10 @@ impl RawTable { hashes: [u64; N], eq: impl FnMut(usize, &T) -> bool, ) -> [Option<&'_ mut T>; N] { - let ptrs = self.get_disjoint_mut_pointers(hashes, eq); - ptrs.map(|ptr| ptr.map(|mut ptr| ptr.as_mut())) + unsafe { + let ptrs = self.get_disjoint_mut_pointers(hashes, eq); + ptrs.map(|ptr| ptr.map(|mut ptr| ptr.as_mut())) + } } unsafe fn get_disjoint_mut_pointers( @@ -1394,7 +1426,7 @@ impl RawTable { /// The caller must ensure `index` is less than the number of buckets. #[inline] pub(crate) unsafe fn is_bucket_full(&self, index: usize) -> bool { - self.table.is_bucket_full(index) + unsafe { self.table.is_bucket_full(index) } } /// Returns an iterator over every element in the table. It is up to @@ -1403,11 +1435,13 @@ impl RawTable { /// struct, we have to make the `iter` method unsafe. #[inline] pub(crate) unsafe fn iter(&self) -> RawIter { - // SAFETY: - // 1. The caller must uphold the safety contract for `iter` method. - // 2. The [`RawTableInner`] must already have properly initialized control bytes since - // we will never expose RawTable::new_uninitialized in a public API. - self.table.iter() + unsafe { + // SAFETY: + // 1. The caller must uphold the safety contract for `iter` method. + // 2. The [`RawTableInner`] must already have properly initialized control bytes since + // we will never expose RawTable::new_uninitialized in a public API. + self.table.iter() + } } /// Returns an iterator over occupied buckets that could match a given hash. @@ -1421,7 +1455,7 @@ impl RawTable { /// `RawIterHash` struct, we have to make the `iter_hash` method unsafe. #[cfg_attr(feature = "inline-more", inline)] pub(crate) unsafe fn iter_hash(&self, hash: u64) -> RawIterHash { - RawIterHash::new(self, hash) + unsafe { RawIterHash::new(self, hash) } } /// Returns an iterator over occupied bucket indices that could match a given hash. @@ -1435,7 +1469,7 @@ impl RawTable { /// `RawIterHashIndices` struct, we have to make the `iter_hash_buckets` method unsafe. #[cfg_attr(feature = "inline-more", inline)] pub(crate) unsafe fn iter_hash_buckets(&self, hash: u64) -> RawIterHashIndices { - RawIterHashIndices::new(&self.table, hash) + unsafe { RawIterHashIndices::new(&self.table, hash) } } /// Returns an iterator over full buckets indices in the table. @@ -1443,7 +1477,7 @@ impl RawTable { /// See [`RawTableInner::full_buckets_indices`] for safety conditions. #[inline(always)] pub(crate) unsafe fn full_buckets_indices(&self) -> FullBucketsIndices { - self.table.full_buckets_indices() + unsafe { self.table.full_buckets_indices() } } /// Returns an iterator which removes all elements from the table without @@ -1612,47 +1646,49 @@ impl RawTableInner { where A: Allocator, { - debug_assert!(buckets.is_power_of_two()); - - // Avoid `Option::ok_or_else` because it bloats LLVM IR. - let (layout, mut ctrl_offset) = match table_layout.calculate_layout_for(buckets) { - Some(lco) => lco, - None => return Err(fallibility.capacity_overflow()), - }; + unsafe { + debug_assert!(buckets.is_power_of_two()); + + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let (layout, mut ctrl_offset) = match table_layout.calculate_layout_for(buckets) { + Some(lco) => lco, + None => return Err(fallibility.capacity_overflow()), + }; + + let ptr: NonNull = match do_alloc(alloc, layout) { + Ok(block) => { + // The allocator can't return a value smaller than was + // requested, so this can be != instead of >=. + if block.len() != layout.size() { + // Utilize over-sized allocations. + let x = maximum_buckets_in(block.len(), table_layout, Group::WIDTH); + debug_assert!(x >= buckets); + // Calculate the new ctrl_offset. + let (oversized_layout, oversized_ctrl_offset) = + match table_layout.calculate_layout_for(x) { + Some(lco) => lco, + None => unsafe { hint::unreachable_unchecked() }, + }; + debug_assert!(oversized_layout.size() <= block.len()); + debug_assert!(oversized_ctrl_offset >= ctrl_offset); + ctrl_offset = oversized_ctrl_offset; + buckets = x; + } - let ptr: NonNull = match do_alloc(alloc, layout) { - Ok(block) => { - // The allocator can't return a value smaller than was - // requested, so this can be != instead of >=. - if block.len() != layout.size() { - // Utilize over-sized allocations. - let x = maximum_buckets_in(block.len(), table_layout, Group::WIDTH); - debug_assert!(x >= buckets); - // Calculate the new ctrl_offset. - let (oversized_layout, oversized_ctrl_offset) = - match table_layout.calculate_layout_for(x) { - Some(lco) => lco, - None => unsafe { hint::unreachable_unchecked() }, - }; - debug_assert!(oversized_layout.size() <= block.len()); - debug_assert!(oversized_ctrl_offset >= ctrl_offset); - ctrl_offset = oversized_ctrl_offset; - buckets = x; + block.cast() } - - block.cast() - } - Err(_) => return Err(fallibility.alloc_err(layout)), - }; - - // SAFETY: null pointer will be caught in above check - let ctrl = NonNull::new_unchecked(ptr.as_ptr().add(ctrl_offset)); - Ok(Self { - ctrl, - bucket_mask: buckets - 1, - items: 0, - growth_left: bucket_mask_to_capacity(buckets - 1), - }) + Err(_) => return Err(fallibility.alloc_err(layout)), + }; + + // SAFETY: null pointer will be caught in above check + let ctrl = NonNull::new_unchecked(ptr.as_ptr().add(ctrl_offset)); + Ok(Self { + ctrl, + bucket_mask: buckets - 1, + items: 0, + growth_left: bucket_mask_to_capacity(buckets - 1), + }) + } } /// Attempts to allocate a new [`RawTableInner`] with at least enough @@ -1758,32 +1794,34 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn fix_insert_index(&self, mut index: usize) -> usize { - // SAFETY: The caller of this function ensures that `index` is in the range `0..=self.bucket_mask`. - if unlikely(self.is_bucket_full(index)) { - debug_assert!(self.bucket_mask < Group::WIDTH); - // SAFETY: - // - // * Since the caller of this function ensures that the control bytes are properly - // initialized and `ptr = self.ctrl(0)` points to the start of the array of control - // bytes, therefore: `ctrl` is valid for reads, properly aligned to `Group::WIDTH` - // and points to the properly initialized control bytes (see also - // `TableLayout::calculate_layout_for` and `ptr::read`); - // - // * Because the caller of this function ensures that the index was provided by the - // `self.find_insert_index_in_group()` function, so for for tables larger than the - // group width (self.num_buckets() >= Group::WIDTH), we will never end up in the given - // branch, since `(probe_seq.pos + bit) & self.bucket_mask` in `find_insert_index_in_group` - // cannot return a full bucket index. For tables smaller than the group width, calling - // the `unwrap_unchecked` function is also safe, as the trailing control bytes outside - // the range of the table are filled with EMPTY bytes (and we know for sure that there - // is at least one FULL bucket), so this second scan either finds an empty slot (due to - // the load factor) or hits the trailing control bytes (containing EMPTY). - index = Group::load_aligned(self.ctrl(0)) - .match_empty_or_deleted() - .lowest_set_bit() - .unwrap_unchecked(); + unsafe { + // SAFETY: The caller of this function ensures that `index` is in the range `0..=self.bucket_mask`. + if unlikely(self.is_bucket_full(index)) { + debug_assert!(self.bucket_mask < Group::WIDTH); + // SAFETY: + // + // * Since the caller of this function ensures that the control bytes are properly + // initialized and `ptr = self.ctrl(0)` points to the start of the array of control + // bytes, therefore: `ctrl` is valid for reads, properly aligned to `Group::WIDTH` + // and points to the properly initialized control bytes (see also + // `TableLayout::calculate_layout_for` and `ptr::read`); + // + // * Because the caller of this function ensures that the index was provided by the + // `self.find_insert_index_in_group()` function, so for for tables larger than the + // group width (self.num_buckets() >= Group::WIDTH), we will never end up in the given + // branch, since `(probe_seq.pos + bit) & self.bucket_mask` in `find_insert_index_in_group` + // cannot return a full bucket index. For tables smaller than the group width, calling + // the `unwrap_unchecked` function is also safe, as the trailing control bytes outside + // the range of the table are filled with EMPTY bytes (and we know for sure that there + // is at least one FULL bucket), so this second scan either finds an empty slot (due to + // the load factor) or hits the trailing control bytes (containing EMPTY). + index = Group::load_aligned(self.ctrl(0)) + .match_empty_or_deleted() + .lowest_set_bit() + .unwrap_unchecked(); + } + index } - index } /// Finds the position to insert something in a group. @@ -1958,17 +1996,19 @@ impl RawTableInner { /// [`RawTableInner::find_insert_index`]: RawTableInner::find_insert_index #[inline] unsafe fn prepare_insert_index(&mut self, hash: u64) -> (usize, Tag) { - // SAFETY: Caller of this function ensures that the control bytes are properly initialized. - let index: usize = self.find_insert_index(hash); - // SAFETY: - // 1. The `find_insert_index` function either returns an `index` less than or - // equal to `self.num_buckets() = self.bucket_mask + 1` of the table, or never - // returns if it cannot find an empty or deleted slot. - // 2. The caller of this function guarantees that the table has already been - // allocated - let old_ctrl = *self.ctrl(index); - self.set_ctrl_hash(index, hash); - (index, old_ctrl) + unsafe { + // SAFETY: Caller of this function ensures that the control bytes are properly initialized. + let index: usize = self.find_insert_index(hash); + // SAFETY: + // 1. The `find_insert_index` function either returns an `index` less than or + // equal to `self.num_buckets() = self.bucket_mask + 1` of the table, or never + // returns if it cannot find an empty or deleted slot. + // 2. The caller of this function guarantees that the table has already been + // allocated + let old_ctrl = *self.ctrl(index); + self.set_ctrl_hash(index, hash); + (index, old_ctrl) + } } /// Searches for an empty or deleted bucket which is suitable for inserting @@ -2131,39 +2171,41 @@ impl RawTableInner { #[allow(clippy::mut_mut)] #[inline] unsafe fn prepare_rehash_in_place(&mut self) { - // Bulk convert all full control bytes to DELETED, and all DELETED control bytes to EMPTY. - // This effectively frees up all buckets containing a DELETED entry. - // - // SAFETY: - // 1. `i` is guaranteed to be within bounds since we are iterating from zero to `buckets - 1`; - // 2. Even if `i` will be `i == self.bucket_mask`, it is safe to call `Group::load_aligned` - // due to the extended control bytes range, which is `self.bucket_mask + 1 + Group::WIDTH`; - // 3. The caller of this function guarantees that [`RawTableInner`] has already been allocated; - // 4. We can use `Group::load_aligned` and `Group::store_aligned` here since we start from 0 - // and go to the end with a step equal to `Group::WIDTH` (see TableLayout::calculate_layout_for). - for i in (0..self.num_buckets()).step_by(Group::WIDTH) { - let group = Group::load_aligned(self.ctrl(i)); - let group = group.convert_special_to_empty_and_full_to_deleted(); - group.store_aligned(self.ctrl(i)); - } + unsafe { + // Bulk convert all full control bytes to DELETED, and all DELETED control bytes to EMPTY. + // This effectively frees up all buckets containing a DELETED entry. + // + // SAFETY: + // 1. `i` is guaranteed to be within bounds since we are iterating from zero to `buckets - 1`; + // 2. Even if `i` will be `i == self.bucket_mask`, it is safe to call `Group::load_aligned` + // due to the extended control bytes range, which is `self.bucket_mask + 1 + Group::WIDTH`; + // 3. The caller of this function guarantees that [`RawTableInner`] has already been allocated; + // 4. We can use `Group::load_aligned` and `Group::store_aligned` here since we start from 0 + // and go to the end with a step equal to `Group::WIDTH` (see TableLayout::calculate_layout_for). + for i in (0..self.num_buckets()).step_by(Group::WIDTH) { + let group = Group::load_aligned(self.ctrl(i)); + let group = group.convert_special_to_empty_and_full_to_deleted(); + group.store_aligned(self.ctrl(i)); + } - // Fix up the trailing control bytes. See the comments in set_ctrl - // for the handling of tables smaller than the group width. - // - // SAFETY: The caller of this function guarantees that [`RawTableInner`] - // has already been allocated - if unlikely(self.num_buckets() < Group::WIDTH) { - // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of control bytes, - // so copying `self.num_buckets() == self.bucket_mask + 1` bytes with offset equal to - // `Group::WIDTH` is safe - self.ctrl(0) - .copy_to(self.ctrl(Group::WIDTH), self.num_buckets()); - } else { - // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of - // control bytes,so copying `Group::WIDTH` bytes with offset equal - // to `self.num_buckets() == self.bucket_mask + 1` is safe - self.ctrl(0) - .copy_to(self.ctrl(self.num_buckets()), Group::WIDTH); + // Fix up the trailing control bytes. See the comments in set_ctrl + // for the handling of tables smaller than the group width. + // + // SAFETY: The caller of this function guarantees that [`RawTableInner`] + // has already been allocated + if unlikely(self.num_buckets() < Group::WIDTH) { + // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of control bytes, + // so copying `self.num_buckets() == self.bucket_mask + 1` bytes with offset equal to + // `Group::WIDTH` is safe + self.ctrl(0) + .copy_to(self.ctrl(Group::WIDTH), self.num_buckets()); + } else { + // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of + // control bytes,so copying `Group::WIDTH` bytes with offset equal + // to `self.num_buckets() == self.bucket_mask + 1` is safe + self.ctrl(0) + .copy_to(self.ctrl(self.num_buckets()), Group::WIDTH); + } } } @@ -2186,38 +2228,40 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn iter(&self) -> RawIter { - // SAFETY: - // 1. Since the caller of this function ensures that the control bytes - // are properly initialized and `self.data_end()` points to the start - // of the array of control bytes, therefore: `ctrl` is valid for reads, - // properly aligned to `Group::WIDTH` and points to the properly initialized - // control bytes. - // 2. `data` bucket index in the table is equal to the `ctrl` index (i.e. - // equal to zero). - // 3. We pass the exact value of buckets of the table to the function. - // - // `ctrl` points here (to the start - // of the first control byte `CT0`) - // ∨ - // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, CTa_0, CTa_1, ..., CTa_m - // \________ ________/ - // \/ - // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` - // - // where: T0...T_n - our stored data; - // CT0...CT_n - control bytes or metadata for `data`. - // CTa_0...CTa_m - additional control bytes, where `m = Group::WIDTH - 1` (so that the search - // with loading `Group` bytes from the heap works properly, even if the result - // of `h1(hash) & self.bucket_mask` is equal to `self.bucket_mask`). See also - // `RawTableInner::set_ctrl` function. - // - // P.S. `h1(hash) & self.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number - // of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. - let data = Bucket::from_base_index(self.data_end(), 0); - RawIter { - // SAFETY: See explanation above - iter: RawIterRange::new(self.ctrl.as_ptr(), data, self.num_buckets()), - items: self.items, + unsafe { + // SAFETY: + // 1. Since the caller of this function ensures that the control bytes + // are properly initialized and `self.data_end()` points to the start + // of the array of control bytes, therefore: `ctrl` is valid for reads, + // properly aligned to `Group::WIDTH` and points to the properly initialized + // control bytes. + // 2. `data` bucket index in the table is equal to the `ctrl` index (i.e. + // equal to zero). + // 3. We pass the exact value of buckets of the table to the function. + // + // `ctrl` points here (to the start + // of the first control byte `CT0`) + // ∨ + // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, CTa_0, CTa_1, ..., CTa_m + // \________ ________/ + // \/ + // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` + // + // where: T0...T_n - our stored data; + // CT0...CT_n - control bytes or metadata for `data`. + // CTa_0...CTa_m - additional control bytes, where `m = Group::WIDTH - 1` (so that the search + // with loading `Group` bytes from the heap works properly, even if the result + // of `h1(hash) & self.bucket_mask` is equal to `self.bucket_mask`). See also + // `RawTableInner::set_ctrl` function. + // + // P.S. `h1(hash) & self.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number + // of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. + let data = Bucket::from_base_index(self.data_end(), 0); + RawIter { + // SAFETY: See explanation above + iter: RawIterRange::new(self.ctrl.as_ptr(), data, self.num_buckets()), + items: self.items, + } } } @@ -2258,16 +2302,18 @@ impl RawTableInner { /// [`clear_no_drop`]: RawTableInner::clear_no_drop /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html unsafe fn drop_elements(&mut self) { - // Check that `self.items != 0`. Protects against the possibility - // of creating an iterator on an table with uninitialized control bytes. - if T::NEEDS_DROP && self.items != 0 { - // SAFETY: We know for sure that RawTableInner will outlive the - // returned `RawIter` iterator, and the caller of this function - // must uphold the safety contract for `drop_elements` method. - for item in self.iter::() { - // SAFETY: The caller must uphold the safety contract for - // `drop_elements` method. - item.drop(); + unsafe { + // Check that `self.items != 0`. Protects against the possibility + // of creating an iterator on an table with uninitialized control bytes. + if T::NEEDS_DROP && self.items != 0 { + // SAFETY: We know for sure that RawTableInner will outlive the + // returned `RawIter` iterator, and the caller of this function + // must uphold the safety contract for `drop_elements` method. + for item in self.iter::() { + // SAFETY: The caller must uphold the safety contract for + // `drop_elements` method. + item.drop(); + } } } } @@ -2390,9 +2436,11 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn bucket(&self, index: usize) -> Bucket { - debug_assert_ne!(self.bucket_mask, 0); - debug_assert!(index < self.num_buckets()); - Bucket::from_base_index(self.data_end(), index) + unsafe { + debug_assert_ne!(self.bucket_mask, 0); + debug_assert!(index < self.num_buckets()); + Bucket::from_base_index(self.data_end(), index) + } } /// Returns a raw `*mut u8` pointer to the start of the `data` element in the table @@ -2443,10 +2491,12 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn bucket_ptr(&self, index: usize, size_of: usize) -> *mut u8 { - debug_assert_ne!(self.bucket_mask, 0); - debug_assert!(index < self.num_buckets()); - let base: *mut u8 = self.data_end().as_ptr(); - base.sub((index + 1) * size_of) + unsafe { + debug_assert_ne!(self.bucket_mask, 0); + debug_assert!(index < self.num_buckets()); + let base: *mut u8 = self.data_end().as_ptr(); + base.sub((index + 1) * size_of) + } } /// Returns pointer to one past last `data` element in the table as viewed from @@ -2506,9 +2556,11 @@ impl RawTableInner { #[inline] unsafe fn record_item_insert_at(&mut self, index: usize, old_ctrl: Tag, new_ctrl: Tag) { - self.growth_left -= usize::from(old_ctrl.special_is_empty()); - self.set_ctrl(index, new_ctrl); - self.items += 1; + unsafe { + self.growth_left -= usize::from(old_ctrl.special_is_empty()); + self.set_ctrl(index, new_ctrl); + self.items += 1; + } } #[inline] @@ -2548,8 +2600,10 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn set_ctrl_hash(&mut self, index: usize, hash: u64) { - // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::set_ctrl_hash`] - self.set_ctrl(index, Tag::full(hash)); + unsafe { + // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::set_ctrl_hash`] + self.set_ctrl(index, Tag::full(hash)); + } } /// Replaces the hash in the control byte at the given index with the provided one, @@ -2582,10 +2636,12 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn replace_ctrl_hash(&mut self, index: usize, hash: u64) -> Tag { - // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::replace_ctrl_hash`] - let prev_ctrl = *self.ctrl(index); - self.set_ctrl_hash(index, hash); - prev_ctrl + unsafe { + // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::replace_ctrl_hash`] + let prev_ctrl = *self.ctrl(index); + self.set_ctrl_hash(index, hash); + prev_ctrl + } } /// Sets a control byte, and possibly also the replicated control byte at @@ -2614,35 +2670,37 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn set_ctrl(&mut self, index: usize, ctrl: Tag) { - // Replicate the first Group::WIDTH control bytes at the end of - // the array without using a branch. If the tables smaller than - // the group width (self.num_buckets() < Group::WIDTH), - // `index2 = Group::WIDTH + index`, otherwise `index2` is: - // - // - If index >= Group::WIDTH then index == index2. - // - Otherwise index2 == self.bucket_mask + 1 + index. - // - // The very last replicated control byte is never actually read because - // we mask the initial index for unaligned loads, but we write it - // anyways because it makes the set_ctrl implementation simpler. - // - // If there are fewer buckets than Group::WIDTH then this code will - // replicate the buckets at the end of the trailing group. For example - // with 2 buckets and a group size of 4, the control bytes will look - // like this: - // - // Real | Replicated - // --------------------------------------------- - // | [A] | [B] | [Tag::EMPTY] | [EMPTY] | [A] | [B] | - // --------------------------------------------- - - // This is the same as `(index.wrapping_sub(Group::WIDTH)) % self.num_buckets() + Group::WIDTH` - // because the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. - let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; - - // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::set_ctrl`] - *self.ctrl(index) = ctrl; - *self.ctrl(index2) = ctrl; + unsafe { + // Replicate the first Group::WIDTH control bytes at the end of + // the array without using a branch. If the tables smaller than + // the group width (self.num_buckets() < Group::WIDTH), + // `index2 = Group::WIDTH + index`, otherwise `index2` is: + // + // - If index >= Group::WIDTH then index == index2. + // - Otherwise index2 == self.bucket_mask + 1 + index. + // + // The very last replicated control byte is never actually read because + // we mask the initial index for unaligned loads, but we write it + // anyways because it makes the set_ctrl implementation simpler. + // + // If there are fewer buckets than Group::WIDTH then this code will + // replicate the buckets at the end of the trailing group. For example + // with 2 buckets and a group size of 4, the control bytes will look + // like this: + // + // Real | Replicated + // --------------------------------------------- + // | [A] | [B] | [Tag::EMPTY] | [EMPTY] | [A] | [B] | + // --------------------------------------------- + + // This is the same as `(index.wrapping_sub(Group::WIDTH)) % self.num_buckets() + Group::WIDTH` + // because the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. + let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; + + // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::set_ctrl`] + *self.ctrl(index) = ctrl; + *self.ctrl(index2) = ctrl; + } } /// Returns a pointer to a control byte. @@ -2670,9 +2728,11 @@ impl RawTableInner { /// [`Undefined Behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn ctrl(&self, index: usize) -> *mut Tag { - debug_assert!(index < self.num_ctrl_bytes()); - // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::ctrl`] - self.ctrl.as_ptr().add(index).cast() + unsafe { + debug_assert!(index < self.num_ctrl_bytes()); + // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::ctrl`] + self.ctrl.as_ptr().add(index).cast() + } } /// Gets the slice of all control bytes. @@ -2693,8 +2753,10 @@ impl RawTableInner { /// The caller must ensure `index` is less than the number of buckets. #[inline] unsafe fn is_bucket_full(&self, index: usize) -> bool { - debug_assert!(index < self.num_buckets()); - (*self.ctrl(index)).is_full() + unsafe { + debug_assert!(index < self.num_buckets()); + (*self.ctrl(index)).is_full() + } } #[inline] @@ -2800,44 +2862,46 @@ impl RawTableInner { where A: Allocator, { - // Avoid `Option::ok_or_else` because it bloats LLVM IR. - let new_items = match self.items.checked_add(additional) { - Some(new_items) => new_items, - None => return Err(fallibility.capacity_overflow()), - }; - let full_capacity = bucket_mask_to_capacity(self.bucket_mask); - if new_items <= full_capacity / 2 { - // Rehash in-place without re-allocating if we have plenty of spare - // capacity that is locked up due to DELETED entries. + unsafe { + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let new_items = match self.items.checked_add(additional) { + Some(new_items) => new_items, + None => return Err(fallibility.capacity_overflow()), + }; + let full_capacity = bucket_mask_to_capacity(self.bucket_mask); + if new_items <= full_capacity / 2 { + // Rehash in-place without re-allocating if we have plenty of spare + // capacity that is locked up due to DELETED entries. - // SAFETY: - // 1. We know for sure that `[`RawTableInner`]` has already been allocated - // (since new_items <= full_capacity / 2); - // 2. The caller ensures that `drop` function is the actual drop function of - // the elements stored in the table. - // 3. The caller ensures that `layout` matches the [`TableLayout`] that was - // used to allocate this table. - // 4. The caller ensures that the control bytes of the `RawTableInner` - // are already initialized. - self.rehash_in_place(hasher, layout.size, drop); - Ok(()) - } else { - // Otherwise, conservatively resize to at least the next size up - // to avoid churning deletes into frequent rehashes. - // - // SAFETY: - // 1. We know for sure that `capacity >= self.items`. - // 2. The caller ensures that `alloc` and `layout` matches the [`Allocator`] and - // [`TableLayout`] that were used to allocate this table. - // 3. The caller ensures that the control bytes of the `RawTableInner` - // are already initialized. - self.resize_inner( - alloc, - usize::max(new_items, full_capacity + 1), - hasher, - fallibility, - layout, - ) + // SAFETY: + // 1. We know for sure that `[`RawTableInner`]` has already been allocated + // (since new_items <= full_capacity / 2); + // 2. The caller ensures that `drop` function is the actual drop function of + // the elements stored in the table. + // 3. The caller ensures that `layout` matches the [`TableLayout`] that was + // used to allocate this table. + // 4. The caller ensures that the control bytes of the `RawTableInner` + // are already initialized. + self.rehash_in_place(hasher, layout.size, drop); + Ok(()) + } else { + // Otherwise, conservatively resize to at least the next size up + // to avoid churning deletes into frequent rehashes. + // + // SAFETY: + // 1. We know for sure that `capacity >= self.items`. + // 2. The caller ensures that `alloc` and `layout` matches the [`Allocator`] and + // [`TableLayout`] that were used to allocate this table. + // 3. The caller ensures that the control bytes of the `RawTableInner` + // are already initialized. + self.resize_inner( + alloc, + usize::max(new_items, full_capacity + 1), + hasher, + fallibility, + layout, + ) + } } } @@ -2855,36 +2919,38 @@ impl RawTableInner { /// * The [`RawTableInner`] must have properly initialized control bytes. #[inline(always)] unsafe fn full_buckets_indices(&self) -> FullBucketsIndices { - // SAFETY: - // 1. Since the caller of this function ensures that the control bytes - // are properly initialized and `self.ctrl(0)` points to the start - // of the array of control bytes, therefore: `ctrl` is valid for reads, - // properly aligned to `Group::WIDTH` and points to the properly initialized - // control bytes. - // 2. The value of `items` is equal to the amount of data (values) added - // to the table. - // - // `ctrl` points here (to the start - // of the first control byte `CT0`) - // ∨ - // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, Group::WIDTH - // \________ ________/ - // \/ - // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` - // - // where: T0...T_n - our stored data; - // CT0...CT_n - control bytes or metadata for `data`. - let ctrl = NonNull::new_unchecked(self.ctrl(0).cast::()); - - FullBucketsIndices { - // Load the first group - // SAFETY: See explanation above. - current_group: Group::load_aligned(ctrl.as_ptr().cast()) - .match_full() - .into_iter(), - group_first_index: 0, - ctrl, - items: self.items, + unsafe { + // SAFETY: + // 1. Since the caller of this function ensures that the control bytes + // are properly initialized and `self.ctrl(0)` points to the start + // of the array of control bytes, therefore: `ctrl` is valid for reads, + // properly aligned to `Group::WIDTH` and points to the properly initialized + // control bytes. + // 2. The value of `items` is equal to the amount of data (values) added + // to the table. + // + // `ctrl` points here (to the start + // of the first control byte `CT0`) + // ∨ + // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, Group::WIDTH + // \________ ________/ + // \/ + // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` + // + // where: T0...T_n - our stored data; + // CT0...CT_n - control bytes or metadata for `data`. + let ctrl = NonNull::new_unchecked(self.ctrl(0).cast::()); + + FullBucketsIndices { + // Load the first group + // SAFETY: See explanation above. + current_group: Group::load_aligned(ctrl.as_ptr().cast()) + .match_full() + .into_iter(), + group_first_index: 0, + ctrl, + items: self.items, + } } } @@ -2941,65 +3007,67 @@ impl RawTableInner { where A: Allocator, { - // SAFETY: We know for sure that `alloc` and `layout` matches the [`Allocator`] and [`TableLayout`] - // that were used to allocate this table. - let mut new_table = self.prepare_resize(alloc, layout, capacity, fallibility)?; + unsafe { + // SAFETY: We know for sure that `alloc` and `layout` matches the [`Allocator`] and [`TableLayout`] + // that were used to allocate this table. + let mut new_table = self.prepare_resize(alloc, layout, capacity, fallibility)?; - // SAFETY: We know for sure that RawTableInner will outlive the - // returned `FullBucketsIndices` iterator, and the caller of this - // function ensures that the control bytes are properly initialized. - for full_byte_index in self.full_buckets_indices() { - // This may panic. - let hash = hasher(self, full_byte_index); + // SAFETY: We know for sure that RawTableInner will outlive the + // returned `FullBucketsIndices` iterator, and the caller of this + // function ensures that the control bytes are properly initialized. + for full_byte_index in self.full_buckets_indices() { + // This may panic. + let hash = hasher(self, full_byte_index); - // SAFETY: - // We can use a simpler version of insert() here since: - // 1. There are no DELETED entries. - // 2. We know there is enough space in the table. - // 3. All elements are unique. - // 4. The caller of this function guarantees that `capacity > 0` - // so `new_table` must already have some allocated memory. - // 5. We set `growth_left` and `items` fields of the new table - // after the loop. - // 6. We insert into the table, at the returned index, the data - // matching the given hash immediately after calling this function. - let (new_index, _) = new_table.prepare_insert_index(hash); + // SAFETY: + // We can use a simpler version of insert() here since: + // 1. There are no DELETED entries. + // 2. We know there is enough space in the table. + // 3. All elements are unique. + // 4. The caller of this function guarantees that `capacity > 0` + // so `new_table` must already have some allocated memory. + // 5. We set `growth_left` and `items` fields of the new table + // after the loop. + // 6. We insert into the table, at the returned index, the data + // matching the given hash immediately after calling this function. + let (new_index, _) = new_table.prepare_insert_index(hash); - // SAFETY: - // - // * `src` is valid for reads of `layout.size` bytes, since the - // table is alive and the `full_byte_index` is guaranteed to be - // within bounds (see `FullBucketsIndices::next_impl`); - // - // * `dst` is valid for writes of `layout.size` bytes, since the - // caller ensures that `table_layout` matches the [`TableLayout`] - // that was used to allocate old table and we have the `new_index` - // returned by `prepare_insert_index`. - // - // * Both `src` and `dst` are properly aligned. - // - // * Both `src` and `dst` point to different region of memory. - ptr::copy_nonoverlapping( - self.bucket_ptr(full_byte_index, layout.size), - new_table.bucket_ptr(new_index, layout.size), - layout.size, - ); - } + // SAFETY: + // + // * `src` is valid for reads of `layout.size` bytes, since the + // table is alive and the `full_byte_index` is guaranteed to be + // within bounds (see `FullBucketsIndices::next_impl`); + // + // * `dst` is valid for writes of `layout.size` bytes, since the + // caller ensures that `table_layout` matches the [`TableLayout`] + // that was used to allocate old table and we have the `new_index` + // returned by `prepare_insert_index`. + // + // * Both `src` and `dst` are properly aligned. + // + // * Both `src` and `dst` point to different region of memory. + ptr::copy_nonoverlapping( + self.bucket_ptr(full_byte_index, layout.size), + new_table.bucket_ptr(new_index, layout.size), + layout.size, + ); + } - // The hash function didn't panic, so we can safely set the - // `growth_left` and `items` fields of the new table. - new_table.growth_left -= self.items; - new_table.items = self.items; + // The hash function didn't panic, so we can safely set the + // `growth_left` and `items` fields of the new table. + new_table.growth_left -= self.items; + new_table.items = self.items; - // We successfully copied all elements without panicking. Now replace - // self with the new table. The old table will have its memory freed but - // the items will not be dropped (since they have been moved into the - // new table). - // SAFETY: The caller ensures that `table_layout` matches the [`TableLayout`] - // that was used to allocate this table. - mem::swap(self, &mut new_table); + // We successfully copied all elements without panicking. Now replace + // self with the new table. The old table will have its memory freed but + // the items will not be dropped (since they have been moved into the + // new table). + // SAFETY: The caller ensures that `table_layout` matches the [`TableLayout`] + // that was used to allocate this table. + mem::swap(self, &mut new_table); - Ok(()) + Ok(()) + } } /// Rehashes the contents of the table in place (i.e. without changing the @@ -3033,81 +3101,83 @@ impl RawTableInner { size_of: usize, drop: Option, ) { - // If the hash function panics then properly clean up any elements - // that we haven't rehashed yet. We unfortunately can't preserve the - // element since we lost their hash and have no way of recovering it - // without risking another panic. - self.prepare_rehash_in_place(); - - let mut guard = guard(self, move |self_| { - if let Some(drop) = drop { - for i in 0..self_.num_buckets() { - if *self_.ctrl(i) == Tag::DELETED { - self_.set_ctrl(i, Tag::EMPTY); - drop(self_.bucket_ptr(i, size_of)); - self_.items -= 1; + unsafe { + // If the hash function panics then properly clean up any elements + // that we haven't rehashed yet. We unfortunately can't preserve the + // element since we lost their hash and have no way of recovering it + // without risking another panic. + self.prepare_rehash_in_place(); + + let mut guard = guard(self, move |self_| { + if let Some(drop) = drop { + for i in 0..self_.num_buckets() { + if *self_.ctrl(i) == Tag::DELETED { + self_.set_ctrl(i, Tag::EMPTY); + drop(self_.bucket_ptr(i, size_of)); + self_.items -= 1; + } } } - } - self_.growth_left = bucket_mask_to_capacity(self_.bucket_mask) - self_.items; - }); - - // At this point, DELETED elements are elements that we haven't - // rehashed yet. Find them and re-insert them at their ideal - // position. - 'outer: for i in 0..guard.num_buckets() { - if *guard.ctrl(i) != Tag::DELETED { - continue; - } - - let i_p = guard.bucket_ptr(i, size_of); - - 'inner: loop { - // Hash the current item - let hash = hasher(*guard, i); + self_.growth_left = bucket_mask_to_capacity(self_.bucket_mask) - self_.items; + }); - // Search for a suitable place to put it - // - // SAFETY: Caller of this function ensures that the control bytes - // are properly initialized. - let new_i = guard.find_insert_index(hash); - - // Probing works by scanning through all of the control - // bytes in groups, which may not be aligned to the group - // size. If both the new and old position fall within the - // same unaligned group, then there is no benefit in moving - // it and we can just continue to the next item. - if likely(guard.is_in_same_group(i, new_i, hash)) { - guard.set_ctrl_hash(i, hash); - continue 'outer; + // At this point, DELETED elements are elements that we haven't + // rehashed yet. Find them and re-insert them at their ideal + // position. + 'outer: for i in 0..guard.num_buckets() { + if *guard.ctrl(i) != Tag::DELETED { + continue; } - let new_i_p = guard.bucket_ptr(new_i, size_of); - - // We are moving the current item to a new position. Write - // our H2 to the control byte of the new position. - let prev_ctrl = guard.replace_ctrl_hash(new_i, hash); - if prev_ctrl == Tag::EMPTY { - guard.set_ctrl(i, Tag::EMPTY); - // If the target slot is empty, simply move the current - // element into the new slot and clear the old control - // byte. - ptr::copy_nonoverlapping(i_p, new_i_p, size_of); - continue 'outer; - } else { - // If the target slot is occupied, swap the two elements - // and then continue processing the element that we just - // swapped into the old slot. - debug_assert_eq!(prev_ctrl, Tag::DELETED); - ptr::swap_nonoverlapping(i_p, new_i_p, size_of); - continue 'inner; + let i_p = guard.bucket_ptr(i, size_of); + + 'inner: loop { + // Hash the current item + let hash = hasher(*guard, i); + + // Search for a suitable place to put it + // + // SAFETY: Caller of this function ensures that the control bytes + // are properly initialized. + let new_i = guard.find_insert_index(hash); + + // Probing works by scanning through all of the control + // bytes in groups, which may not be aligned to the group + // size. If both the new and old position fall within the + // same unaligned group, then there is no benefit in moving + // it and we can just continue to the next item. + if likely(guard.is_in_same_group(i, new_i, hash)) { + guard.set_ctrl_hash(i, hash); + continue 'outer; + } + + let new_i_p = guard.bucket_ptr(new_i, size_of); + + // We are moving the current item to a new position. Write + // our H2 to the control byte of the new position. + let prev_ctrl = guard.replace_ctrl_hash(new_i, hash); + if prev_ctrl == Tag::EMPTY { + guard.set_ctrl(i, Tag::EMPTY); + // If the target slot is empty, simply move the current + // element into the new slot and clear the old control + // byte. + ptr::copy_nonoverlapping(i_p, new_i_p, size_of); + continue 'outer; + } else { + // If the target slot is occupied, swap the two elements + // and then continue processing the element that we just + // swapped into the old slot. + debug_assert_eq!(prev_ctrl, Tag::DELETED); + ptr::swap_nonoverlapping(i_p, new_i_p, size_of); + continue 'inner; + } } } - } - guard.growth_left = bucket_mask_to_capacity(guard.bucket_mask) - guard.items; + guard.growth_left = bucket_mask_to_capacity(guard.bucket_mask) - guard.items; - mem::forget(guard); + mem::forget(guard); + } } /// Deallocates the table without dropping any entries. @@ -3141,10 +3211,12 @@ impl RawTableInner { where A: Allocator, { - // SAFETY: The caller must uphold the safety contract for `free_buckets` - // method. - let (ptr, layout) = self.allocation_info(table_layout); - alloc.deallocate(ptr, layout); + unsafe { + // SAFETY: The caller must uphold the safety contract for `free_buckets` + // method. + let (ptr, layout) = self.allocation_info(table_layout); + alloc.deallocate(ptr, layout); + } } /// Returns a pointer to the allocated memory and the layout that was used to @@ -3257,62 +3329,65 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn erase(&mut self, index: usize) { - debug_assert!(self.is_bucket_full(index)); - - // This is the same as `index.wrapping_sub(Group::WIDTH) % self.num_buckets()` because - // the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. - let index_before = index.wrapping_sub(Group::WIDTH) & self.bucket_mask; - // SAFETY: - // - The caller must uphold the safety contract for `erase` method; - // - `index_before` is guaranteed to be in range due to masking with `self.bucket_mask` - let empty_before = Group::load(self.ctrl(index_before)).match_empty(); - let empty_after = Group::load(self.ctrl(index)).match_empty(); - - // Inserting and searching in the map is performed by two key functions: - // - // - The `find_insert_index` function that looks up the index of any `Tag::EMPTY` or `Tag::DELETED` - // slot in a group to be able to insert. If it doesn't find an `Tag::EMPTY` or `Tag::DELETED` - // slot immediately in the first group, it jumps to the next `Group` looking for it, - // and so on until it has gone through all the groups in the control bytes. - // - // - The `find_inner` function that looks for the index of the desired element by looking - // at all the `FULL` bytes in the group. If it did not find the element right away, and - // there is no `Tag::EMPTY` byte in the group, then this means that the `find_insert_index` - // function may have found a suitable slot in the next group. Therefore, `find_inner` - // jumps further, and if it does not find the desired element and again there is no `Tag::EMPTY` - // byte, then it jumps further, and so on. The search stops only if `find_inner` function - // finds the desired element or hits an `Tag::EMPTY` slot/byte. - // - // Accordingly, this leads to two consequences: - // - // - The map must have `Tag::EMPTY` slots (bytes); - // - // - You can't just mark the byte to be erased as `Tag::EMPTY`, because otherwise the `find_inner` - // function may stumble upon an `Tag::EMPTY` byte before finding the desired element and stop - // searching. - // - // Thus it is necessary to check all bytes after and before the erased element. If we are in - // a contiguous `Group` of `FULL` or `Tag::DELETED` bytes (the number of `FULL` or `Tag::DELETED` bytes - // before and after is greater than or equal to `Group::WIDTH`), then we must mark our byte as - // `Tag::DELETED` in order for the `find_inner` function to go further. On the other hand, if there - // is at least one `Tag::EMPTY` slot in the `Group`, then the `find_inner` function will still stumble - // upon an `Tag::EMPTY` byte, so we can safely mark our erased byte as `Tag::EMPTY` as well. - // - // Finally, since `index_before == (index.wrapping_sub(Group::WIDTH) & self.bucket_mask) == index` - // and given all of the above, tables smaller than the group width (self.num_buckets() < Group::WIDTH) - // cannot have `Tag::DELETED` bytes. - // - // Note that in this context `leading_zeros` refers to the bytes at the end of a group, while - // `trailing_zeros` refers to the bytes at the beginning of a group. - let ctrl = if empty_before.leading_zeros() + empty_after.trailing_zeros() >= Group::WIDTH { - Tag::DELETED - } else { - self.growth_left += 1; - Tag::EMPTY - }; - // SAFETY: the caller must uphold the safety contract for `erase` method. - self.set_ctrl(index, ctrl); - self.items -= 1; + unsafe { + debug_assert!(self.is_bucket_full(index)); + + // This is the same as `index.wrapping_sub(Group::WIDTH) % self.num_buckets()` because + // the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. + let index_before = index.wrapping_sub(Group::WIDTH) & self.bucket_mask; + // SAFETY: + // - The caller must uphold the safety contract for `erase` method; + // - `index_before` is guaranteed to be in range due to masking with `self.bucket_mask` + let empty_before = Group::load(self.ctrl(index_before)).match_empty(); + let empty_after = Group::load(self.ctrl(index)).match_empty(); + + // Inserting and searching in the map is performed by two key functions: + // + // - The `find_insert_index` function that looks up the index of any `Tag::EMPTY` or `Tag::DELETED` + // slot in a group to be able to insert. If it doesn't find an `Tag::EMPTY` or `Tag::DELETED` + // slot immediately in the first group, it jumps to the next `Group` looking for it, + // and so on until it has gone through all the groups in the control bytes. + // + // - The `find_inner` function that looks for the index of the desired element by looking + // at all the `FULL` bytes in the group. If it did not find the element right away, and + // there is no `Tag::EMPTY` byte in the group, then this means that the `find_insert_index` + // function may have found a suitable slot in the next group. Therefore, `find_inner` + // jumps further, and if it does not find the desired element and again there is no `Tag::EMPTY` + // byte, then it jumps further, and so on. The search stops only if `find_inner` function + // finds the desired element or hits an `Tag::EMPTY` slot/byte. + // + // Accordingly, this leads to two consequences: + // + // - The map must have `Tag::EMPTY` slots (bytes); + // + // - You can't just mark the byte to be erased as `Tag::EMPTY`, because otherwise the `find_inner` + // function may stumble upon an `Tag::EMPTY` byte before finding the desired element and stop + // searching. + // + // Thus it is necessary to check all bytes after and before the erased element. If we are in + // a contiguous `Group` of `FULL` or `Tag::DELETED` bytes (the number of `FULL` or `Tag::DELETED` bytes + // before and after is greater than or equal to `Group::WIDTH`), then we must mark our byte as + // `Tag::DELETED` in order for the `find_inner` function to go further. On the other hand, if there + // is at least one `Tag::EMPTY` slot in the `Group`, then the `find_inner` function will still stumble + // upon an `Tag::EMPTY` byte, so we can safely mark our erased byte as `Tag::EMPTY` as well. + // + // Finally, since `index_before == (index.wrapping_sub(Group::WIDTH) & self.bucket_mask) == index` + // and given all of the above, tables smaller than the group width (self.num_buckets() < Group::WIDTH) + // cannot have `Tag::DELETED` bytes. + // + // Note that in this context `leading_zeros` refers to the bytes at the end of a group, while + // `trailing_zeros` refers to the bytes at the beginning of a group. + let ctrl = + if empty_before.leading_zeros() + empty_after.trailing_zeros() >= Group::WIDTH { + Tag::DELETED + } else { + self.growth_left += 1; + Tag::EMPTY + }; + // SAFETY: the caller must uphold the safety contract for `erase` method. + self.set_ctrl(index, ctrl); + self.items -= 1; + } } } @@ -3424,26 +3499,28 @@ trait RawTableClone { impl RawTableClone for RawTable { default_fn! { #[cfg_attr(feature = "inline-more", inline)] - unsafe fn clone_from_spec(&mut self, source: &Self) { + unsafe fn clone_from_spec(&mut self, source: &Self) { unsafe { self.clone_from_impl(source); - } + }} } } #[cfg(feature = "nightly")] impl RawTableClone for RawTable { #[cfg_attr(feature = "inline-more", inline)] unsafe fn clone_from_spec(&mut self, source: &Self) { - source - .table - .ctrl(0) - .copy_to_nonoverlapping(self.table.ctrl(0), self.table.num_ctrl_bytes()); - source - .data_start() - .as_ptr() - .copy_to_nonoverlapping(self.data_start().as_ptr(), self.table.num_buckets()); - - self.table.items = source.table.items; - self.table.growth_left = source.table.growth_left; + unsafe { + source + .table + .ctrl(0) + .copy_to_nonoverlapping(self.table.ctrl(0), self.table.num_ctrl_bytes()); + source + .data_start() + .as_ptr() + .copy_to_nonoverlapping(self.data_start().as_ptr(), self.table.num_buckets()); + + self.table.items = source.table.items; + self.table.growth_left = source.table.growth_left; + } } } @@ -3454,39 +3531,41 @@ impl RawTable { /// - The control bytes are not initialized yet. #[cfg_attr(feature = "inline-more", inline)] unsafe fn clone_from_impl(&mut self, source: &Self) { - // Copy the control bytes unchanged. We do this in a single pass - source - .table - .ctrl(0) - .copy_to_nonoverlapping(self.table.ctrl(0), self.table.num_ctrl_bytes()); - - // The cloning of elements may panic, in which case we need - // to make sure we drop only the elements that have been - // cloned so far. - let mut guard = guard((0, &mut *self), |(index, self_)| { - if T::NEEDS_DROP { - for i in 0..*index { - if self_.is_bucket_full(i) { - self_.bucket(i).drop(); + unsafe { + // Copy the control bytes unchanged. We do this in a single pass + source + .table + .ctrl(0) + .copy_to_nonoverlapping(self.table.ctrl(0), self.table.num_ctrl_bytes()); + + // The cloning of elements may panic, in which case we need + // to make sure we drop only the elements that have been + // cloned so far. + let mut guard = guard((0, &mut *self), |(index, self_)| { + if T::NEEDS_DROP { + for i in 0..*index { + if self_.is_bucket_full(i) { + self_.bucket(i).drop(); + } } } - } - }); + }); - for from in source.iter() { - let index = source.bucket_index(&from); - let to = guard.1.bucket(index); - to.write(from.as_ref().clone()); + for from in source.iter() { + let index = source.bucket_index(&from); + let to = guard.1.bucket(index); + to.write(from.as_ref().clone()); - // Update the index in case we need to unwind. - guard.0 = index + 1; - } + // Update the index in case we need to unwind. + guard.0 = index + 1; + } - // Successfully cloned all items, no need to clean up. - mem::forget(guard); + // Successfully cloned all items, no need to clean up. + mem::forget(guard); - self.table.items = source.table.items; - self.table.growth_left = source.table.growth_left; + self.table.items = source.table.items; + self.table.growth_left = source.table.growth_left; + } } } @@ -3592,21 +3671,23 @@ impl RawIterRange { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[cfg_attr(feature = "inline-more", inline)] unsafe fn new(ctrl: *const u8, data: Bucket, len: usize) -> Self { - debug_assert_ne!(len, 0); - debug_assert_eq!(ctrl as usize % Group::WIDTH, 0); - // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] - let end = ctrl.add(len); - - // Load the first group and advance ctrl to point to the next group - // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] - let current_group = Group::load_aligned(ctrl.cast()).match_full(); - let next_ctrl = ctrl.add(Group::WIDTH); - - Self { - current_group: current_group.into_iter(), - data, - next_ctrl, - end, + unsafe { + debug_assert_ne!(len, 0); + debug_assert_eq!(ctrl as usize % Group::WIDTH, 0); + // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] + let end = ctrl.add(len); + + // Load the first group and advance ctrl to point to the next group + // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] + let current_group = Group::load_aligned(ctrl.cast()).match_full(); + let next_ctrl = ctrl.add(Group::WIDTH); + + Self { + current_group: current_group.into_iter(), + data, + next_ctrl, + end, + } } } @@ -3659,25 +3740,27 @@ impl RawIterRange { /// after yielding all elements. #[cfg_attr(feature = "inline-more", inline)] unsafe fn next_impl(&mut self) -> Option> { - loop { - if let Some(index) = self.current_group.next() { - return Some(self.data.next_n(index)); - } + unsafe { + loop { + if let Some(index) = self.current_group.next() { + return Some(self.data.next_n(index)); + } - if DO_CHECK_PTR_RANGE && self.next_ctrl >= self.end { - return None; - } + if DO_CHECK_PTR_RANGE && self.next_ctrl >= self.end { + return None; + } - // We might read past self.end up to the next group boundary, - // but this is fine because it only occurs on tables smaller - // than the group size where the trailing control bytes are all - // EMPTY. On larger tables self.end is guaranteed to be aligned - // to the group size (since tables are power-of-two sized). - self.current_group = Group::load_aligned(self.next_ctrl.cast()) - .match_full() - .into_iter(); - self.data = self.data.next_n(Group::WIDTH); - self.next_ctrl = self.next_ctrl.add(Group::WIDTH); + // We might read past self.end up to the next group boundary, + // but this is fine because it only occurs on tables smaller + // than the group size where the trailing control bytes are all + // EMPTY. On larger tables self.end is guaranteed to be aligned + // to the group size (since tables are power-of-two sized). + self.current_group = Group::load_aligned(self.next_ctrl.cast()) + .match_full() + .into_iter(); + self.data = self.data.next_n(Group::WIDTH); + self.next_ctrl = self.next_ctrl.add(Group::WIDTH); + } } } @@ -3712,53 +3795,55 @@ impl RawIterRange { where F: FnMut(B, Bucket) -> B, { - loop { - while let Some(index) = self.current_group.next() { - // The returned `index` will always be in the range `0..Group::WIDTH`, - // so that calling `self.data.next_n(index)` is safe (see detailed explanation below). - debug_assert!(n != 0); - let bucket = self.data.next_n(index); - acc = f(acc, bucket); - n -= 1; - } + unsafe { + loop { + while let Some(index) = self.current_group.next() { + // The returned `index` will always be in the range `0..Group::WIDTH`, + // so that calling `self.data.next_n(index)` is safe (see detailed explanation below). + debug_assert!(n != 0); + let bucket = self.data.next_n(index); + acc = f(acc, bucket); + n -= 1; + } - if n == 0 { - return acc; - } + if n == 0 { + return acc; + } - // SAFETY: The caller of this function ensures that: - // - // 1. The provided `n` value matches the actual number of items in the table; - // 2. The table is alive and did not moved. - // - // Taking the above into account, we always stay within the bounds, because: - // - // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), - // we will never end up in the given branch, since we should have already - // yielded all the elements of the table. - // - // 2. For tables larger than the group width. The number of buckets is a - // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since - // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the - // start of the array of control bytes, and never try to iterate after - // getting all the elements, the last `self.current_group` will read bytes - // from the `self.num_buckets() - Group::WIDTH` index. We know also that - // `self.current_group.next()` will always return indices within the range - // `0..Group::WIDTH`. - // - // Knowing all of the above and taking into account that we are synchronizing - // the `self.data` index with the index we used to read the `self.current_group`, - // the subsequent `self.data.next_n(index)` will always return a bucket with - // an index number less than `self.num_buckets()`. - // - // The last `self.next_ctrl`, whose index would be `self.num_buckets()`, will never - // actually be read, since we should have already yielded all the elements of - // the table. - self.current_group = Group::load_aligned(self.next_ctrl.cast()) - .match_full() - .into_iter(); - self.data = self.data.next_n(Group::WIDTH); - self.next_ctrl = self.next_ctrl.add(Group::WIDTH); + // SAFETY: The caller of this function ensures that: + // + // 1. The provided `n` value matches the actual number of items in the table; + // 2. The table is alive and did not moved. + // + // Taking the above into account, we always stay within the bounds, because: + // + // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), + // we will never end up in the given branch, since we should have already + // yielded all the elements of the table. + // + // 2. For tables larger than the group width. The number of buckets is a + // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since + // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the + // start of the array of control bytes, and never try to iterate after + // getting all the elements, the last `self.current_group` will read bytes + // from the `self.num_buckets() - Group::WIDTH` index. We know also that + // `self.current_group.next()` will always return indices within the range + // `0..Group::WIDTH`. + // + // Knowing all of the above and taking into account that we are synchronizing + // the `self.data` index with the index we used to read the `self.current_group`, + // the subsequent `self.data.next_n(index)` will always return a bucket with + // an index number less than `self.num_buckets()`. + // + // The last `self.next_ctrl`, whose index would be `self.num_buckets()`, will never + // actually be read, since we should have already yielded all the elements of + // the table. + self.current_group = Group::load_aligned(self.next_ctrl.cast()) + .match_full() + .into_iter(); + self.data = self.data.next_n(Group::WIDTH); + self.next_ctrl = self.next_ctrl.add(Group::WIDTH); + } } } } @@ -3826,9 +3911,11 @@ pub(crate) struct RawIter { impl RawIter { unsafe fn drop_elements(&mut self) { - if T::NEEDS_DROP && self.items != 0 { - for item in self { - item.drop(); + unsafe { + if T::NEEDS_DROP && self.items != 0 { + for item in self { + item.drop(); + } } } } @@ -3945,41 +4032,43 @@ impl FullBucketsIndices { /// [`Undefined Behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline(always)] unsafe fn next_impl(&mut self) -> Option { - loop { - if let Some(index) = self.current_group.next() { - // The returned `self.group_first_index + index` will always - // be in the range `0..self.num_buckets()`. See explanation below. - return Some(self.group_first_index + index); - } + unsafe { + loop { + if let Some(index) = self.current_group.next() { + // The returned `self.group_first_index + index` will always + // be in the range `0..self.num_buckets()`. See explanation below. + return Some(self.group_first_index + index); + } - // SAFETY: The caller of this function ensures that: - // - // 1. It never tries to iterate after getting all the elements; - // 2. The table is alive and did not moved; - // 3. The first `self.ctrl` pointed to the start of the array of control bytes. - // - // Taking the above into account, we always stay within the bounds, because: - // - // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), - // we will never end up in the given branch, since we should have already - // yielded all the elements of the table. - // - // 2. For tables larger than the group width. The number of buckets is a - // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since - // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the - // the start of the array of control bytes, and never try to iterate after - // getting all the elements, the last `self.ctrl` will be equal to - // the `self.num_buckets() - Group::WIDTH`, so `self.current_group.next()` - // will always contains indices within the range `0..Group::WIDTH`, - // and subsequent `self.group_first_index + index` will always return a - // number less than `self.num_buckets()`. - self.ctrl = NonNull::new_unchecked(self.ctrl.as_ptr().add(Group::WIDTH)); - - // SAFETY: See explanation above. - self.current_group = Group::load_aligned(self.ctrl.as_ptr().cast()) - .match_full() - .into_iter(); - self.group_first_index += Group::WIDTH; + // SAFETY: The caller of this function ensures that: + // + // 1. It never tries to iterate after getting all the elements; + // 2. The table is alive and did not moved; + // 3. The first `self.ctrl` pointed to the start of the array of control bytes. + // + // Taking the above into account, we always stay within the bounds, because: + // + // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), + // we will never end up in the given branch, since we should have already + // yielded all the elements of the table. + // + // 2. For tables larger than the group width. The number of buckets is a + // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since + // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the + // the start of the array of control bytes, and never try to iterate after + // getting all the elements, the last `self.ctrl` will be equal to + // the `self.num_buckets() - Group::WIDTH`, so `self.current_group.next()` + // will always contains indices within the range `0..Group::WIDTH`, + // and subsequent `self.group_first_index + index` will always return a + // number less than `self.num_buckets()`. + self.ctrl = NonNull::new_unchecked(self.ctrl.as_ptr().add(Group::WIDTH)); + + // SAFETY: See explanation above. + self.current_group = Group::load_aligned(self.ctrl.as_ptr().cast()) + .match_full() + .into_iter(); + self.group_first_index += Group::WIDTH; + } } } } @@ -4221,9 +4310,11 @@ pub(crate) struct RawIterHashIndices { impl RawIterHash { #[cfg_attr(feature = "inline-more", inline)] unsafe fn new(table: &RawTable, hash: u64) -> Self { - RawIterHash { - inner: RawIterHashIndices::new(&table.table, hash), - _marker: PhantomData, + unsafe { + RawIterHash { + inner: RawIterHashIndices::new(&table.table, hash), + _marker: PhantomData, + } } } } @@ -4259,18 +4350,20 @@ impl Default for RawIterHashIndices { impl RawIterHashIndices { #[cfg_attr(feature = "inline-more", inline)] unsafe fn new(table: &RawTableInner, hash: u64) -> Self { - let tag_hash = Tag::full(hash); - let probe_seq = table.probe_seq(hash); - let group = Group::load(table.ctrl(probe_seq.pos)); - let bitmask = group.match_tag(tag_hash).into_iter(); - - RawIterHashIndices { - bucket_mask: table.bucket_mask, - ctrl: table.ctrl, - tag_hash, - probe_seq, - group, - bitmask, + unsafe { + let tag_hash = Tag::full(hash); + let probe_seq = table.probe_seq(hash); + let group = Group::load(table.ctrl(probe_seq.pos)); + let bitmask = group.match_tag(tag_hash).into_iter(); + + RawIterHashIndices { + bucket_mask: table.bucket_mask, + ctrl: table.ctrl, + tag_hash, + probe_seq, + group, + bitmask, + } } } } @@ -4517,8 +4610,10 @@ mod test_map { } unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { - let g = Global; - g.deallocate(ptr, layout) + unsafe { + let g = Global; + g.deallocate(ptr, layout) + } } } diff --git a/src/set.rs b/src/set.rs index 36fb60a36..9b3703272 100644 --- a/src/set.rs +++ b/src/set.rs @@ -1114,7 +1114,7 @@ where /// correctly, and would cause unsoundness as a result. #[cfg_attr(feature = "inline-more", inline)] pub unsafe fn insert_unique_unchecked(&mut self, value: T) -> &T { - self.map.insert_unique_unchecked(value, ()).0 + unsafe { self.map.insert_unique_unchecked(value, ()).0 } } /// Adds a value to the set, replacing the existing value, if any, that is equal to the given diff --git a/src/table.rs b/src/table.rs index a890e29c2..a3cf9c282 100644 --- a/src/table.rs +++ b/src/table.rs @@ -511,9 +511,11 @@ where /// ``` #[inline] pub unsafe fn get_bucket_entry_unchecked(&mut self, index: usize) -> OccupiedEntry<'_, T, A> { - OccupiedEntry { - bucket: self.raw.bucket(index), - table: self, + unsafe { + OccupiedEntry { + bucket: self.raw.bucket(index), + table: self, + } } } @@ -588,7 +590,7 @@ where /// ``` #[inline] pub unsafe fn get_bucket_unchecked(&self, index: usize) -> &T { - self.raw.bucket(index).as_ref() + unsafe { self.raw.bucket(index).as_ref() } } /// Gets a mutable reference to an entry in the table at the given bucket index, @@ -666,7 +668,7 @@ where /// ``` #[inline] pub unsafe fn get_bucket_unchecked_mut(&mut self, index: usize) -> &mut T { - self.raw.bucket(index).as_mut() + unsafe { self.raw.bucket(index).as_mut() } } /// Inserts an element into the `HashTable` with the given hash value, but @@ -1547,7 +1549,7 @@ where hashes: [u64; N], eq: impl FnMut(usize, &T) -> bool, ) -> [Option<&'_ mut T>; N] { - self.raw.get_disjoint_unchecked_mut(hashes, eq) + unsafe { self.raw.get_disjoint_unchecked_mut(hashes, eq) } } /// Attempts to get mutable references to `N` values in the map at once, without validating that @@ -1558,7 +1560,7 @@ where hashes: [u64; N], eq: impl FnMut(usize, &T) -> bool, ) -> [Option<&'_ mut T>; N] { - self.raw.get_disjoint_unchecked_mut(hashes, eq) + unsafe { self.raw.get_disjoint_unchecked_mut(hashes, eq) } } /// Returns the total amount of memory allocated internally by the hash From fcafbd5bd73e4e56a51c3834344a8ca5f1fdb754 Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 16:34:58 -0500 Subject: [PATCH 07/11] Manually tweak unsafe block additions --- src/control/group/generic.rs | 8 +- src/control/group/lsx.rs | 8 +- src/control/group/neon.rs | 8 +- src/control/group/sse2.rs | 8 +- src/raw/mod.rs | 1259 +++++++++++++++++----------------- src/table.rs | 8 +- 6 files changed, 643 insertions(+), 656 deletions(-) diff --git a/src/control/group/generic.rs b/src/control/group/generic.rs index de8c9abba..d401a4c6f 100644 --- a/src/control/group/generic.rs +++ b/src/control/group/generic.rs @@ -80,10 +80,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(ptr::read(ptr.cast())) - } + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + unsafe { Group(ptr::read(ptr.cast())) } } /// Stores the group of tags to the given address, which must be @@ -91,8 +89,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); ptr::write(ptr.cast(), self.0); } } diff --git a/src/control/group/lsx.rs b/src/control/group/lsx.rs index 2cd053b6c..ee2540043 100644 --- a/src/control/group/lsx.rs +++ b/src/control/group/lsx.rs @@ -54,10 +54,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(lsx_vld::<0>(ptr.cast())) - } + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + unsafe { Group(lsx_vld::<0>(ptr.cast())) } } /// Stores the group of tags to the given address, which must be @@ -65,8 +63,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); lsx_vst::<0>(self.0, ptr.cast()); } } diff --git a/src/control/group/neon.rs b/src/control/group/neon.rs index 99c6c888f..02e5fd491 100644 --- a/src/control/group/neon.rs +++ b/src/control/group/neon.rs @@ -51,10 +51,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(neon::vld1_u8(ptr.cast())) - } + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + unsafe { Group(neon::vld1_u8(ptr.cast())) } } /// Stores the group of tags to the given address, which must be @@ -62,8 +60,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); neon::vst1_u8(ptr.cast(), self.0); } } diff --git a/src/control/group/sse2.rs b/src/control/group/sse2.rs index 95f537b1a..651dc6098 100644 --- a/src/control/group/sse2.rs +++ b/src/control/group/sse2.rs @@ -57,10 +57,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { - unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); - Group(x86::_mm_load_si128(ptr.cast())) - } + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); + unsafe { Group(x86::_mm_load_si128(ptr.cast())) } } /// Stores the group of tags to the given address, which must be @@ -68,8 +66,8 @@ impl Group { #[inline] #[allow(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { + debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { - debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); x86::_mm_store_si128(ptr.cast(), self.0); } } diff --git a/src/raw/mod.rs b/src/raw/mod.rs index f322788fa..81dd3012c 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -310,38 +310,36 @@ impl Bucket { /// [`RawTableInner::num_buckets`]: RawTableInner::num_buckets #[inline] unsafe fn from_base_index(base: NonNull, index: usize) -> Self { - unsafe { - // If mem::size_of::() != 0 then return a pointer to an `element` in - // the data part of the table (we start counting from "0", so that - // in the expression T[last], the "last" index actually one less than the - // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"): - // - // `from_base_index(base, 1).as_ptr()` returns a pointer that - // points here in the data part of the table - // (to the start of T1) - // | - // | `base: NonNull` must point here - // | (to the end of T0 or to the start of C0) - // v v - // [Padding], Tlast, ..., |T1|, T0, |C0, C1, ..., Clast - // ^ - // `from_base_index(base, 1)` returns a pointer - // that points here in the data part of the table - // (to the end of T1) - // - // where: T0...Tlast - our stored data; C0...Clast - control bytes - // or metadata for data. - let ptr = if T::IS_ZERO_SIZED { - // won't overflow because index must be less than length (bucket_mask) - // and bucket_mask is guaranteed to be less than `isize::MAX` - // (see TableLayout::calculate_layout_for method) - invalid_mut(index + 1) - } else { - base.as_ptr().sub(index) - }; - Self { - ptr: NonNull::new_unchecked(ptr), - } + // If mem::size_of::() != 0 then return a pointer to an `element` in + // the data part of the table (we start counting from "0", so that + // in the expression T[last], the "last" index actually one less than the + // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"): + // + // `from_base_index(base, 1).as_ptr()` returns a pointer that + // points here in the data part of the table + // (to the start of T1) + // | + // | `base: NonNull` must point here + // | (to the end of T0 or to the start of C0) + // v v + // [Padding], Tlast, ..., |T1|, T0, |C0, C1, ..., Clast + // ^ + // `from_base_index(base, 1)` returns a pointer + // that points here in the data part of the table + // (to the end of T1) + // + // where: T0...Tlast - our stored data; C0...Clast - control bytes + // or metadata for data. + let ptr = if T::IS_ZERO_SIZED { + // won't overflow because index must be less than length (bucket_mask) + // and bucket_mask is guaranteed to be less than `isize::MAX` + // (see TableLayout::calculate_layout_for method) + invalid_mut(index + 1) + } else { + unsafe { base.as_ptr().sub(index) } + }; + Self { + ptr: unsafe { NonNull::new_unchecked(ptr) }, } } @@ -385,34 +383,32 @@ impl Bucket { /// [`<*const T>::offset_from`]: https://doc.rust-lang.org/nightly/core/primitive.pointer.html#method.offset_from #[inline] unsafe fn to_base_index(&self, base: NonNull) -> usize { - unsafe { - // If mem::size_of::() != 0 then return an index under which we used to store the - // `element` in the data part of the table (we start counting from "0", so - // that in the expression T[last], the "last" index actually is one less than the - // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"). - // For example for 5th element in table calculation is performed like this: - // - // mem::size_of::() - // | - // | `self = from_base_index(base, 5)` that returns pointer - // | that points here in the data part of the table - // | (to the end of T5) - // | | `base: NonNull` must point here - // v | (to the end of T0 or to the start of C0) - // /???\ v v - // [Padding], Tlast, ..., |T10|, ..., T5|, T4, T3, T2, T1, T0, |C0, C1, C2, C3, C4, C5, ..., C10, ..., Clast - // \__________ __________/ - // \/ - // `bucket.to_base_index(base)` = 5 - // (base.as_ptr() as usize - self.ptr.as_ptr() as usize) / mem::size_of::() - // - // where: T0...Tlast - our stored data; C0...Clast - control bytes or metadata for data. - if T::IS_ZERO_SIZED { - // this can not be UB - self.ptr.as_ptr() as usize - 1 - } else { - offset_from(base.as_ptr(), self.ptr.as_ptr()) - } + // If mem::size_of::() != 0 then return an index under which we used to store the + // `element` in the data part of the table (we start counting from "0", so + // that in the expression T[last], the "last" index actually is one less than the + // "buckets" number in the table, i.e. "last = RawTableInner.bucket_mask"). + // For example for 5th element in table calculation is performed like this: + // + // mem::size_of::() + // | + // | `self = from_base_index(base, 5)` that returns pointer + // | that points here in the data part of the table + // | (to the end of T5) + // | | `base: NonNull` must point here + // v | (to the end of T0 or to the start of C0) + // /???\ v v + // [Padding], Tlast, ..., |T10|, ..., T5|, T4, T3, T2, T1, T0, |C0, C1, C2, C3, C4, C5, ..., C10, ..., Clast + // \__________ __________/ + // \/ + // `bucket.to_base_index(base)` = 5 + // (base.as_ptr() as usize - self.ptr.as_ptr() as usize) / mem::size_of::() + // + // where: T0...Tlast - our stored data; C0...Clast - control bytes or metadata for data. + if T::IS_ZERO_SIZED { + // this can not be UB + self.ptr.as_ptr() as usize - 1 + } else { + unsafe { offset_from(base.as_ptr(), self.ptr.as_ptr()) } } } @@ -492,16 +488,14 @@ impl Bucket { /// [`RawTableInner::num_buckets`]: RawTableInner::num_buckets #[inline] unsafe fn next_n(&self, offset: usize) -> Self { - unsafe { - let ptr = if T::IS_ZERO_SIZED { - // invalid pointer is good enough for ZST - invalid_mut(self.ptr.as_ptr() as usize + offset) - } else { - self.ptr.as_ptr().sub(offset) - }; - Self { - ptr: NonNull::new_unchecked(ptr), - } + let ptr = if T::IS_ZERO_SIZED { + // invalid pointer is good enough for ZST + invalid_mut(self.ptr.as_ptr() as usize + offset) + } else { + unsafe { self.ptr.as_ptr().sub(offset) } + }; + Self { + ptr: unsafe { NonNull::new_unchecked(ptr) }, } } @@ -683,20 +677,15 @@ impl RawTable { buckets: usize, fallibility: Fallibility, ) -> Result { - unsafe { - debug_assert!(buckets.is_power_of_two()); + debug_assert!(buckets.is_power_of_two()); - Ok(Self { - table: RawTableInner::new_uninitialized( - &alloc, - Self::TABLE_LAYOUT, - buckets, - fallibility, - )?, - alloc, - marker: PhantomData, - }) - } + Ok(Self { + table: unsafe { + RawTableInner::new_uninitialized(&alloc, Self::TABLE_LAYOUT, buckets, fallibility) + }?, + alloc, + marker: PhantomData, + }) } /// Allocates a new hash table using the given allocator, with at least enough capacity for @@ -795,35 +784,33 @@ impl RawTable { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] pub(crate) unsafe fn bucket(&self, index: usize) -> Bucket { - unsafe { - // If mem::size_of::() != 0 then return a pointer to the `element` in the `data part` of the table - // (we start counting from "0", so that in the expression T[n], the "n" index actually one less than - // the "buckets" number of our `RawTable`, i.e. "n = RawTable::num_buckets() - 1"): - // - // `table.bucket(3).as_ptr()` returns a pointer that points here in the `data` - // part of the `RawTable`, i.e. to the start of T3 (see `Bucket::as_ptr`) - // | - // | `base = self.data_end()` points here - // | (to the start of CT0 or to the end of T0) - // v v - // [Pad], T_n, ..., |T3|, T2, T1, T0, |CT0, CT1, CT2, CT3, ..., CT_n, CTa_0, CTa_1, ..., CTa_m - // ^ \__________ __________/ - // `table.bucket(3)` returns a pointer that points \/ - // here in the `data` part of the `RawTable` (to additional control bytes - // the end of T3) `m = Group::WIDTH - 1` - // - // where: T0...T_n - our stored data; - // CT0...CT_n - control bytes or metadata for `data`; - // CTa_0...CTa_m - additional control bytes (so that the search with loading `Group` bytes from - // the heap works properly, even if the result of `h1(hash) & self.table.bucket_mask` - // is equal to `self.table.bucket_mask`). See also `RawTableInner::set_ctrl` function. - // - // P.S. `h1(hash) & self.table.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number - // of buckets is a power of two, and `self.table.bucket_mask = self.num_buckets() - 1`. - debug_assert_ne!(self.table.bucket_mask, 0); - debug_assert!(index < self.num_buckets()); - Bucket::from_base_index(self.data_end(), index) - } + // If mem::size_of::() != 0 then return a pointer to the `element` in the `data part` of the table + // (we start counting from "0", so that in the expression T[n], the "n" index actually one less than + // the "buckets" number of our `RawTable`, i.e. "n = RawTable::num_buckets() - 1"): + // + // `table.bucket(3).as_ptr()` returns a pointer that points here in the `data` + // part of the `RawTable`, i.e. to the start of T3 (see `Bucket::as_ptr`) + // | + // | `base = self.data_end()` points here + // | (to the start of CT0 or to the end of T0) + // v v + // [Pad], T_n, ..., |T3|, T2, T1, T0, |CT0, CT1, CT2, CT3, ..., CT_n, CTa_0, CTa_1, ..., CTa_m + // ^ \__________ __________/ + // `table.bucket(3)` returns a pointer that points \/ + // here in the `data` part of the `RawTable` (to additional control bytes + // the end of T3) `m = Group::WIDTH - 1` + // + // where: T0...T_n - our stored data; + // CT0...CT_n - control bytes or metadata for `data`; + // CTa_0...CTa_m - additional control bytes (so that the search with loading `Group` bytes from + // the heap works properly, even if the result of `h1(hash) & self.table.bucket_mask` + // is equal to `self.table.bucket_mask`). See also `RawTableInner::set_ctrl` function. + // + // P.S. `h1(hash) & self.table.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number + // of buckets is a power of two, and `self.table.bucket_mask = self.num_buckets() - 1`. + debug_assert_ne!(self.table.bucket_mask, 0); + debug_assert!(index < self.num_buckets()); + unsafe { Bucket::from_base_index(self.data_end(), index) } } /// Erases an element from the table without dropping it. @@ -1074,13 +1061,13 @@ impl RawTable { hasher: impl Fn(&T) -> u64, fallibility: Fallibility, ) -> Result<(), TryReserveError> { + // SAFETY: + // 1. The caller of this function guarantees that `capacity >= self.table.items`. + // 2. We know for sure that `alloc` and `layout` matches the [`Allocator`] and + // [`TableLayout`] that were used to allocate this table. + // 3. The caller ensures that the control bytes of the `RawTableInner` + // are already initialized. unsafe { - // SAFETY: - // 1. The caller of this function guarantees that `capacity >= self.table.items`. - // 2. We know for sure that `alloc` and `layout` matches the [`Allocator`] and - // [`TableLayout`] that were used to allocate this table. - // 3. The caller ensures that the control bytes of the `RawTableInner` - // are already initialized. self.table.resize_inner( &self.alloc, capacity, @@ -1375,10 +1362,8 @@ impl RawTable { hashes: [u64; N], eq: impl FnMut(usize, &T) -> bool, ) -> [Option<&'_ mut T>; N] { - unsafe { - let ptrs = self.get_disjoint_mut_pointers(hashes, eq); - ptrs.map(|ptr| ptr.map(|mut ptr| ptr.as_mut())) - } + let ptrs = unsafe { self.get_disjoint_mut_pointers(hashes, eq) }; + ptrs.map(|ptr| ptr.map(|mut ptr| unsafe { ptr.as_mut() })) } unsafe fn get_disjoint_mut_pointers( @@ -1435,13 +1420,11 @@ impl RawTable { /// struct, we have to make the `iter` method unsafe. #[inline] pub(crate) unsafe fn iter(&self) -> RawIter { - unsafe { - // SAFETY: - // 1. The caller must uphold the safety contract for `iter` method. - // 2. The [`RawTableInner`] must already have properly initialized control bytes since - // we will never expose RawTable::new_uninitialized in a public API. - self.table.iter() - } + // SAFETY: + // 1. The caller must uphold the safety contract for `iter` method. + // 2. The [`RawTableInner`] must already have properly initialized control bytes since + // we will never expose RawTable::new_uninitialized in a public API. + unsafe { self.table.iter() } } /// Returns an iterator over occupied buckets that could match a given hash. @@ -1646,49 +1629,47 @@ impl RawTableInner { where A: Allocator, { - unsafe { - debug_assert!(buckets.is_power_of_two()); - - // Avoid `Option::ok_or_else` because it bloats LLVM IR. - let (layout, mut ctrl_offset) = match table_layout.calculate_layout_for(buckets) { - Some(lco) => lco, - None => return Err(fallibility.capacity_overflow()), - }; + debug_assert!(buckets.is_power_of_two()); - let ptr: NonNull = match do_alloc(alloc, layout) { - Ok(block) => { - // The allocator can't return a value smaller than was - // requested, so this can be != instead of >=. - if block.len() != layout.size() { - // Utilize over-sized allocations. - let x = maximum_buckets_in(block.len(), table_layout, Group::WIDTH); - debug_assert!(x >= buckets); - // Calculate the new ctrl_offset. - let (oversized_layout, oversized_ctrl_offset) = - match table_layout.calculate_layout_for(x) { - Some(lco) => lco, - None => unsafe { hint::unreachable_unchecked() }, - }; - debug_assert!(oversized_layout.size() <= block.len()); - debug_assert!(oversized_ctrl_offset >= ctrl_offset); - ctrl_offset = oversized_ctrl_offset; - buckets = x; - } + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let (layout, mut ctrl_offset) = match table_layout.calculate_layout_for(buckets) { + Some(lco) => lco, + None => return Err(fallibility.capacity_overflow()), + }; - block.cast() + let ptr: NonNull = match do_alloc(alloc, layout) { + Ok(block) => { + // The allocator can't return a value smaller than was + // requested, so this can be != instead of >=. + if block.len() != layout.size() { + // Utilize over-sized allocations. + let x = maximum_buckets_in(block.len(), table_layout, Group::WIDTH); + debug_assert!(x >= buckets); + // Calculate the new ctrl_offset. + let (oversized_layout, oversized_ctrl_offset) = + match table_layout.calculate_layout_for(x) { + Some(lco) => lco, + None => unsafe { hint::unreachable_unchecked() }, + }; + debug_assert!(oversized_layout.size() <= block.len()); + debug_assert!(oversized_ctrl_offset >= ctrl_offset); + ctrl_offset = oversized_ctrl_offset; + buckets = x; } - Err(_) => return Err(fallibility.alloc_err(layout)), - }; - // SAFETY: null pointer will be caught in above check - let ctrl = NonNull::new_unchecked(ptr.as_ptr().add(ctrl_offset)); - Ok(Self { - ctrl, - bucket_mask: buckets - 1, - items: 0, - growth_left: bucket_mask_to_capacity(buckets - 1), - }) - } + block.cast() + } + Err(_) => return Err(fallibility.alloc_err(layout)), + }; + + // SAFETY: null pointer will be caught in above check + let ctrl = unsafe { NonNull::new_unchecked(ptr.as_ptr().add(ctrl_offset)) }; + Ok(Self { + ctrl, + bucket_mask: buckets - 1, + items: 0, + growth_left: bucket_mask_to_capacity(buckets - 1), + }) } /// Attempts to allocate a new [`RawTableInner`] with at least enough @@ -1794,34 +1775,34 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn fix_insert_index(&self, mut index: usize) -> usize { - unsafe { - // SAFETY: The caller of this function ensures that `index` is in the range `0..=self.bucket_mask`. - if unlikely(self.is_bucket_full(index)) { - debug_assert!(self.bucket_mask < Group::WIDTH); - // SAFETY: - // - // * Since the caller of this function ensures that the control bytes are properly - // initialized and `ptr = self.ctrl(0)` points to the start of the array of control - // bytes, therefore: `ctrl` is valid for reads, properly aligned to `Group::WIDTH` - // and points to the properly initialized control bytes (see also - // `TableLayout::calculate_layout_for` and `ptr::read`); - // - // * Because the caller of this function ensures that the index was provided by the - // `self.find_insert_index_in_group()` function, so for for tables larger than the - // group width (self.num_buckets() >= Group::WIDTH), we will never end up in the given - // branch, since `(probe_seq.pos + bit) & self.bucket_mask` in `find_insert_index_in_group` - // cannot return a full bucket index. For tables smaller than the group width, calling - // the `unwrap_unchecked` function is also safe, as the trailing control bytes outside - // the range of the table are filled with EMPTY bytes (and we know for sure that there - // is at least one FULL bucket), so this second scan either finds an empty slot (due to - // the load factor) or hits the trailing control bytes (containing EMPTY). - index = Group::load_aligned(self.ctrl(0)) + // SAFETY: The caller of this function ensures that `index` is in the range `0..=self.bucket_mask`. + if unlikely(unsafe { self.is_bucket_full(index) }) { + debug_assert!(self.bucket_mask < Group::WIDTH); + // SAFETY: + // + // * Since the caller of this function ensures that the control bytes are properly + // initialized and `ptr = self.ctrl(0)` points to the start of the array of control + // bytes, therefore: `ctrl` is valid for reads, properly aligned to `Group::WIDTH` + // and points to the properly initialized control bytes (see also + // `TableLayout::calculate_layout_for` and `ptr::read`); + // + // * Because the caller of this function ensures that the index was provided by the + // `self.find_insert_index_in_group()` function, so for for tables larger than the + // group width (self.num_buckets() >= Group::WIDTH), we will never end up in the given + // branch, since `(probe_seq.pos + bit) & self.bucket_mask` in `find_insert_index_in_group` + // cannot return a full bucket index. For tables smaller than the group width, calling + // the `unwrap_unchecked` function is also safe, as the trailing control bytes outside + // the range of the table are filled with EMPTY bytes (and we know for sure that there + // is at least one FULL bucket), so this second scan either finds an empty slot (due to + // the load factor) or hits the trailing control bytes (containing EMPTY). + index = unsafe { + Group::load_aligned(self.ctrl(0)) .match_empty_or_deleted() .lowest_set_bit() - .unwrap_unchecked(); - } - index + .unwrap_unchecked() + }; } + index } /// Finds the position to insert something in a group. @@ -2171,38 +2152,39 @@ impl RawTableInner { #[allow(clippy::mut_mut)] #[inline] unsafe fn prepare_rehash_in_place(&mut self) { + // Bulk convert all full control bytes to DELETED, and all DELETED control bytes to EMPTY. + // This effectively frees up all buckets containing a DELETED entry. + // + // SAFETY: + // 1. `i` is guaranteed to be within bounds since we are iterating from zero to `buckets - 1`; + // 2. Even if `i` will be `i == self.bucket_mask`, it is safe to call `Group::load_aligned` + // due to the extended control bytes range, which is `self.bucket_mask + 1 + Group::WIDTH`; + // 3. The caller of this function guarantees that [`RawTableInner`] has already been allocated; + // 4. We can use `Group::load_aligned` and `Group::store_aligned` here since we start from 0 + // and go to the end with a step equal to `Group::WIDTH` (see TableLayout::calculate_layout_for). unsafe { - // Bulk convert all full control bytes to DELETED, and all DELETED control bytes to EMPTY. - // This effectively frees up all buckets containing a DELETED entry. - // - // SAFETY: - // 1. `i` is guaranteed to be within bounds since we are iterating from zero to `buckets - 1`; - // 2. Even if `i` will be `i == self.bucket_mask`, it is safe to call `Group::load_aligned` - // due to the extended control bytes range, which is `self.bucket_mask + 1 + Group::WIDTH`; - // 3. The caller of this function guarantees that [`RawTableInner`] has already been allocated; - // 4. We can use `Group::load_aligned` and `Group::store_aligned` here since we start from 0 - // and go to the end with a step equal to `Group::WIDTH` (see TableLayout::calculate_layout_for). for i in (0..self.num_buckets()).step_by(Group::WIDTH) { let group = Group::load_aligned(self.ctrl(i)); let group = group.convert_special_to_empty_and_full_to_deleted(); group.store_aligned(self.ctrl(i)); } + } - // Fix up the trailing control bytes. See the comments in set_ctrl - // for the handling of tables smaller than the group width. - // - // SAFETY: The caller of this function guarantees that [`RawTableInner`] - // has already been allocated - if unlikely(self.num_buckets() < Group::WIDTH) { - // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of control bytes, - // so copying `self.num_buckets() == self.bucket_mask + 1` bytes with offset equal to - // `Group::WIDTH` is safe + // Fix up the trailing control bytes. See the comments in set_ctrl + // for the handling of tables smaller than the group width. + if unlikely(self.num_buckets() < Group::WIDTH) { + // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of control bytes, + // so copying `self.num_buckets() == self.bucket_mask + 1` bytes with offset equal to + // `Group::WIDTH` is safe + unsafe { self.ctrl(0) .copy_to(self.ctrl(Group::WIDTH), self.num_buckets()); - } else { - // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of - // control bytes,so copying `Group::WIDTH` bytes with offset equal - // to `self.num_buckets() == self.bucket_mask + 1` is safe + } + } else { + // SAFETY: We have `self.bucket_mask + 1 + Group::WIDTH` number of + // control bytes,so copying `Group::WIDTH` bytes with offset equal + // to `self.num_buckets() == self.bucket_mask + 1` is safe + unsafe { self.ctrl(0) .copy_to(self.ctrl(self.num_buckets()), Group::WIDTH); } @@ -2228,34 +2210,34 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn iter(&self) -> RawIter { + // SAFETY: + // 1. Since the caller of this function ensures that the control bytes + // are properly initialized and `self.data_end()` points to the start + // of the array of control bytes, therefore: `ctrl` is valid for reads, + // properly aligned to `Group::WIDTH` and points to the properly initialized + // control bytes. + // 2. `data` bucket index in the table is equal to the `ctrl` index (i.e. + // equal to zero). + // 3. We pass the exact value of buckets of the table to the function. + // + // `ctrl` points here (to the start + // of the first control byte `CT0`) + // ∨ + // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, CTa_0, CTa_1, ..., CTa_m + // \________ ________/ + // \/ + // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` + // + // where: T0...T_n - our stored data; + // CT0...CT_n - control bytes or metadata for `data`. + // CTa_0...CTa_m - additional control bytes, where `m = Group::WIDTH - 1` (so that the search + // with loading `Group` bytes from the heap works properly, even if the result + // of `h1(hash) & self.bucket_mask` is equal to `self.bucket_mask`). See also + // `RawTableInner::set_ctrl` function. + // + // P.S. `h1(hash) & self.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number + // of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. unsafe { - // SAFETY: - // 1. Since the caller of this function ensures that the control bytes - // are properly initialized and `self.data_end()` points to the start - // of the array of control bytes, therefore: `ctrl` is valid for reads, - // properly aligned to `Group::WIDTH` and points to the properly initialized - // control bytes. - // 2. `data` bucket index in the table is equal to the `ctrl` index (i.e. - // equal to zero). - // 3. We pass the exact value of buckets of the table to the function. - // - // `ctrl` points here (to the start - // of the first control byte `CT0`) - // ∨ - // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, CTa_0, CTa_1, ..., CTa_m - // \________ ________/ - // \/ - // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` - // - // where: T0...T_n - our stored data; - // CT0...CT_n - control bytes or metadata for `data`. - // CTa_0...CTa_m - additional control bytes, where `m = Group::WIDTH - 1` (so that the search - // with loading `Group` bytes from the heap works properly, even if the result - // of `h1(hash) & self.bucket_mask` is equal to `self.bucket_mask`). See also - // `RawTableInner::set_ctrl` function. - // - // P.S. `h1(hash) & self.bucket_mask` is the same as `hash as usize % self.num_buckets()` because the number - // of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. let data = Bucket::from_base_index(self.data_end(), 0); RawIter { // SAFETY: See explanation above @@ -2302,13 +2284,13 @@ impl RawTableInner { /// [`clear_no_drop`]: RawTableInner::clear_no_drop /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html unsafe fn drop_elements(&mut self) { - unsafe { - // Check that `self.items != 0`. Protects against the possibility - // of creating an iterator on an table with uninitialized control bytes. - if T::NEEDS_DROP && self.items != 0 { - // SAFETY: We know for sure that RawTableInner will outlive the - // returned `RawIter` iterator, and the caller of this function - // must uphold the safety contract for `drop_elements` method. + // Check that `self.items != 0`. Protects against the possibility + // of creating an iterator on an table with uninitialized control bytes. + if T::NEEDS_DROP && self.items != 0 { + // SAFETY: We know for sure that RawTableInner will outlive the + // returned `RawIter` iterator, and the caller of this function + // must uphold the safety contract for `drop_elements` method. + unsafe { for item in self.iter::() { // SAFETY: The caller must uphold the safety contract for // `drop_elements` method. @@ -2366,12 +2348,14 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html unsafe fn drop_inner_table(&mut self, alloc: &A, table_layout: TableLayout) { if !self.is_empty_singleton() { + // SAFETY: The caller must uphold the safety contract for `drop_inner_table` method. unsafe { - // SAFETY: The caller must uphold the safety contract for `drop_inner_table` method. self.drop_elements::(); - // SAFETY: - // 1. We have checked that our table is allocated. - // 2. The caller must uphold the safety contract for `drop_inner_table` method. + } + // SAFETY: + // 1. We have checked that our table is allocated. + // 2. The caller must uphold the safety contract for `drop_inner_table` method. + unsafe { self.free_buckets(alloc, table_layout); } } @@ -2436,11 +2420,9 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn bucket(&self, index: usize) -> Bucket { - unsafe { - debug_assert_ne!(self.bucket_mask, 0); - debug_assert!(index < self.num_buckets()); - Bucket::from_base_index(self.data_end(), index) - } + debug_assert_ne!(self.bucket_mask, 0); + debug_assert!(index < self.num_buckets()); + unsafe { Bucket::from_base_index(self.data_end(), index) } } /// Returns a raw `*mut u8` pointer to the start of the `data` element in the table @@ -2491,9 +2473,9 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn bucket_ptr(&self, index: usize, size_of: usize) -> *mut u8 { + debug_assert_ne!(self.bucket_mask, 0); + debug_assert!(index < self.num_buckets()); unsafe { - debug_assert_ne!(self.bucket_mask, 0); - debug_assert!(index < self.num_buckets()); let base: *mut u8 = self.data_end().as_ptr(); base.sub((index + 1) * size_of) } @@ -2556,11 +2538,11 @@ impl RawTableInner { #[inline] unsafe fn record_item_insert_at(&mut self, index: usize, old_ctrl: Tag, new_ctrl: Tag) { + self.growth_left -= usize::from(old_ctrl.special_is_empty()); unsafe { - self.growth_left -= usize::from(old_ctrl.special_is_empty()); self.set_ctrl(index, new_ctrl); - self.items += 1; } + self.items += 1; } #[inline] @@ -2670,34 +2652,34 @@ impl RawTableInner { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn set_ctrl(&mut self, index: usize, ctrl: Tag) { - unsafe { - // Replicate the first Group::WIDTH control bytes at the end of - // the array without using a branch. If the tables smaller than - // the group width (self.num_buckets() < Group::WIDTH), - // `index2 = Group::WIDTH + index`, otherwise `index2` is: - // - // - If index >= Group::WIDTH then index == index2. - // - Otherwise index2 == self.bucket_mask + 1 + index. - // - // The very last replicated control byte is never actually read because - // we mask the initial index for unaligned loads, but we write it - // anyways because it makes the set_ctrl implementation simpler. - // - // If there are fewer buckets than Group::WIDTH then this code will - // replicate the buckets at the end of the trailing group. For example - // with 2 buckets and a group size of 4, the control bytes will look - // like this: - // - // Real | Replicated - // --------------------------------------------- - // | [A] | [B] | [Tag::EMPTY] | [EMPTY] | [A] | [B] | - // --------------------------------------------- + // Replicate the first Group::WIDTH control bytes at the end of + // the array without using a branch. If the tables smaller than + // the group width (self.num_buckets() < Group::WIDTH), + // `index2 = Group::WIDTH + index`, otherwise `index2` is: + // + // - If index >= Group::WIDTH then index == index2. + // - Otherwise index2 == self.bucket_mask + 1 + index. + // + // The very last replicated control byte is never actually read because + // we mask the initial index for unaligned loads, but we write it + // anyways because it makes the set_ctrl implementation simpler. + // + // If there are fewer buckets than Group::WIDTH then this code will + // replicate the buckets at the end of the trailing group. For example + // with 2 buckets and a group size of 4, the control bytes will look + // like this: + // + // Real | Replicated + // --------------------------------------------- + // | [A] | [B] | [Tag::EMPTY] | [EMPTY] | [A] | [B] | + // --------------------------------------------- - // This is the same as `(index.wrapping_sub(Group::WIDTH)) % self.num_buckets() + Group::WIDTH` - // because the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. - let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; + // This is the same as `(index.wrapping_sub(Group::WIDTH)) % self.num_buckets() + Group::WIDTH` + // because the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. + let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; - // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::set_ctrl`] + // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::set_ctrl`] + unsafe { *self.ctrl(index) = ctrl; *self.ctrl(index2) = ctrl; } @@ -2728,11 +2710,9 @@ impl RawTableInner { /// [`Undefined Behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline] unsafe fn ctrl(&self, index: usize) -> *mut Tag { - unsafe { - debug_assert!(index < self.num_ctrl_bytes()); - // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::ctrl`] - self.ctrl.as_ptr().add(index).cast() - } + debug_assert!(index < self.num_ctrl_bytes()); + // SAFETY: The caller must uphold the safety rules for the [`RawTableInner::ctrl`] + unsafe { self.ctrl.as_ptr().add(index).cast() } } /// Gets the slice of all control bytes. @@ -2753,10 +2733,8 @@ impl RawTableInner { /// The caller must ensure `index` is less than the number of buckets. #[inline] unsafe fn is_bucket_full(&self, index: usize) -> bool { - unsafe { - debug_assert!(index < self.num_buckets()); - (*self.ctrl(index)).is_full() - } + debug_assert!(index < self.num_buckets()); + unsafe { (*self.ctrl(index)).is_full() } } #[inline] @@ -2862,38 +2840,40 @@ impl RawTableInner { where A: Allocator, { - unsafe { - // Avoid `Option::ok_or_else` because it bloats LLVM IR. - let new_items = match self.items.checked_add(additional) { - Some(new_items) => new_items, - None => return Err(fallibility.capacity_overflow()), - }; - let full_capacity = bucket_mask_to_capacity(self.bucket_mask); - if new_items <= full_capacity / 2 { - // Rehash in-place without re-allocating if we have plenty of spare - // capacity that is locked up due to DELETED entries. + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let new_items = match self.items.checked_add(additional) { + Some(new_items) => new_items, + None => return Err(fallibility.capacity_overflow()), + }; + let full_capacity = bucket_mask_to_capacity(self.bucket_mask); + if new_items <= full_capacity / 2 { + // Rehash in-place without re-allocating if we have plenty of spare + // capacity that is locked up due to DELETED entries. - // SAFETY: - // 1. We know for sure that `[`RawTableInner`]` has already been allocated - // (since new_items <= full_capacity / 2); - // 2. The caller ensures that `drop` function is the actual drop function of - // the elements stored in the table. - // 3. The caller ensures that `layout` matches the [`TableLayout`] that was - // used to allocate this table. - // 4. The caller ensures that the control bytes of the `RawTableInner` - // are already initialized. + // SAFETY: + // 1. We know for sure that `[`RawTableInner`]` has already been allocated + // (since new_items <= full_capacity / 2); + // 2. The caller ensures that `drop` function is the actual drop function of + // the elements stored in the table. + // 3. The caller ensures that `layout` matches the [`TableLayout`] that was + // used to allocate this table. + // 4. The caller ensures that the control bytes of the `RawTableInner` + // are already initialized. + unsafe { self.rehash_in_place(hasher, layout.size, drop); - Ok(()) - } else { - // Otherwise, conservatively resize to at least the next size up - // to avoid churning deletes into frequent rehashes. - // - // SAFETY: - // 1. We know for sure that `capacity >= self.items`. - // 2. The caller ensures that `alloc` and `layout` matches the [`Allocator`] and - // [`TableLayout`] that were used to allocate this table. - // 3. The caller ensures that the control bytes of the `RawTableInner` - // are already initialized. + } + Ok(()) + } else { + // Otherwise, conservatively resize to at least the next size up + // to avoid churning deletes into frequent rehashes. + // + // SAFETY: + // 1. We know for sure that `capacity >= self.items`. + // 2. The caller ensures that `alloc` and `layout` matches the [`Allocator`] and + // [`TableLayout`] that were used to allocate this table. + // 3. The caller ensures that the control bytes of the `RawTableInner` + // are already initialized. + unsafe { self.resize_inner( alloc, usize::max(new_items, full_capacity + 1), @@ -2919,26 +2899,26 @@ impl RawTableInner { /// * The [`RawTableInner`] must have properly initialized control bytes. #[inline(always)] unsafe fn full_buckets_indices(&self) -> FullBucketsIndices { + // SAFETY: + // 1. Since the caller of this function ensures that the control bytes + // are properly initialized and `self.ctrl(0)` points to the start + // of the array of control bytes, therefore: `ctrl` is valid for reads, + // properly aligned to `Group::WIDTH` and points to the properly initialized + // control bytes. + // 2. The value of `items` is equal to the amount of data (values) added + // to the table. + // + // `ctrl` points here (to the start + // of the first control byte `CT0`) + // ∨ + // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, Group::WIDTH + // \________ ________/ + // \/ + // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` + // + // where: T0...T_n - our stored data; + // CT0...CT_n - control bytes or metadata for `data`. unsafe { - // SAFETY: - // 1. Since the caller of this function ensures that the control bytes - // are properly initialized and `self.ctrl(0)` points to the start - // of the array of control bytes, therefore: `ctrl` is valid for reads, - // properly aligned to `Group::WIDTH` and points to the properly initialized - // control bytes. - // 2. The value of `items` is equal to the amount of data (values) added - // to the table. - // - // `ctrl` points here (to the start - // of the first control byte `CT0`) - // ∨ - // [Pad], T_n, ..., T1, T0, |CT0, CT1, ..., CT_n|, Group::WIDTH - // \________ ________/ - // \/ - // `n = buckets - 1`, i.e. `RawTableInner::num_buckets() - 1` - // - // where: T0...T_n - our stored data; - // CT0...CT_n - control bytes or metadata for `data`. let ctrl = NonNull::new_unchecked(self.ctrl(0).cast::()); FullBucketsIndices { @@ -3007,14 +2987,14 @@ impl RawTableInner { where A: Allocator, { - unsafe { - // SAFETY: We know for sure that `alloc` and `layout` matches the [`Allocator`] and [`TableLayout`] - // that were used to allocate this table. - let mut new_table = self.prepare_resize(alloc, layout, capacity, fallibility)?; + // SAFETY: We know for sure that `alloc` and `layout` matches the [`Allocator`] and [`TableLayout`] + // that were used to allocate this table. + let mut new_table = self.prepare_resize(alloc, layout, capacity, fallibility)?; - // SAFETY: We know for sure that RawTableInner will outlive the - // returned `FullBucketsIndices` iterator, and the caller of this - // function ensures that the control bytes are properly initialized. + // SAFETY: We know for sure that RawTableInner will outlive the + // returned `FullBucketsIndices` iterator, and the caller of this + // function ensures that the control bytes are properly initialized. + unsafe { for full_byte_index in self.full_buckets_indices() { // This may panic. let hash = hasher(self, full_byte_index); @@ -3052,22 +3032,22 @@ impl RawTableInner { layout.size, ); } + } - // The hash function didn't panic, so we can safely set the - // `growth_left` and `items` fields of the new table. - new_table.growth_left -= self.items; - new_table.items = self.items; + // The hash function didn't panic, so we can safely set the + // `growth_left` and `items` fields of the new table. + new_table.growth_left -= self.items; + new_table.items = self.items; - // We successfully copied all elements without panicking. Now replace - // self with the new table. The old table will have its memory freed but - // the items will not be dropped (since they have been moved into the - // new table). - // SAFETY: The caller ensures that `table_layout` matches the [`TableLayout`] - // that was used to allocate this table. - mem::swap(self, &mut new_table); + // We successfully copied all elements without panicking. Now replace + // self with the new table. The old table will have its memory freed but + // the items will not be dropped (since they have been moved into the + // new table). + // SAFETY: The caller ensures that `table_layout` matches the [`TableLayout`] + // that was used to allocate this table. + mem::swap(self, &mut new_table); - Ok(()) - } + Ok(()) } /// Rehashes the contents of the table in place (i.e. without changing the @@ -3101,16 +3081,18 @@ impl RawTableInner { size_of: usize, drop: Option, ) { + // If the hash function panics then properly clean up any elements + // that we haven't rehashed yet. We unfortunately can't preserve the + // element since we lost their hash and have no way of recovering it + // without risking another panic. unsafe { - // If the hash function panics then properly clean up any elements - // that we haven't rehashed yet. We unfortunately can't preserve the - // element since we lost their hash and have no way of recovering it - // without risking another panic. self.prepare_rehash_in_place(); + } - let mut guard = guard(self, move |self_| { - if let Some(drop) = drop { - for i in 0..self_.num_buckets() { + let mut guard = guard(self, move |self_| { + if let Some(drop) = drop { + for i in 0..self_.num_buckets() { + unsafe { if *self_.ctrl(i) == Tag::DELETED { self_.set_ctrl(i, Tag::EMPTY); drop(self_.bucket_ptr(i, size_of)); @@ -3118,66 +3100,72 @@ impl RawTableInner { } } } - self_.growth_left = bucket_mask_to_capacity(self_.bucket_mask) - self_.items; - }); + } + self_.growth_left = bucket_mask_to_capacity(self_.bucket_mask) - self_.items; + }); - // At this point, DELETED elements are elements that we haven't - // rehashed yet. Find them and re-insert them at their ideal - // position. - 'outer: for i in 0..guard.num_buckets() { + // At this point, DELETED elements are elements that we haven't + // rehashed yet. Find them and re-insert them at their ideal + // position. + 'outer: for i in 0..guard.num_buckets() { + unsafe { if *guard.ctrl(i) != Tag::DELETED { continue; } + } - let i_p = guard.bucket_ptr(i, size_of); - - 'inner: loop { - // Hash the current item - let hash = hasher(*guard, i); - - // Search for a suitable place to put it - // - // SAFETY: Caller of this function ensures that the control bytes - // are properly initialized. - let new_i = guard.find_insert_index(hash); - - // Probing works by scanning through all of the control - // bytes in groups, which may not be aligned to the group - // size. If both the new and old position fall within the - // same unaligned group, then there is no benefit in moving - // it and we can just continue to the next item. - if likely(guard.is_in_same_group(i, new_i, hash)) { - guard.set_ctrl_hash(i, hash); - continue 'outer; - } + let i_p = unsafe { guard.bucket_ptr(i, size_of) }; + + 'inner: loop { + // Hash the current item + let hash = hasher(*guard, i); + + // Search for a suitable place to put it + // + // SAFETY: Caller of this function ensures that the control bytes + // are properly initialized. + let new_i = unsafe { guard.find_insert_index(hash) }; + + // Probing works by scanning through all of the control + // bytes in groups, which may not be aligned to the group + // size. If both the new and old position fall within the + // same unaligned group, then there is no benefit in moving + // it and we can just continue to the next item. + if likely(guard.is_in_same_group(i, new_i, hash)) { + unsafe { guard.set_ctrl_hash(i, hash) }; + continue 'outer; + } - let new_i_p = guard.bucket_ptr(new_i, size_of); + let new_i_p = unsafe { guard.bucket_ptr(new_i, size_of) }; - // We are moving the current item to a new position. Write - // our H2 to the control byte of the new position. - let prev_ctrl = guard.replace_ctrl_hash(new_i, hash); - if prev_ctrl == Tag::EMPTY { - guard.set_ctrl(i, Tag::EMPTY); - // If the target slot is empty, simply move the current - // element into the new slot and clear the old control - // byte. + // We are moving the current item to a new position. Write + // our H2 to the control byte of the new position. + let prev_ctrl = unsafe { guard.replace_ctrl_hash(new_i, hash) }; + if prev_ctrl == Tag::EMPTY { + unsafe { guard.set_ctrl(i, Tag::EMPTY) }; + // If the target slot is empty, simply move the current + // element into the new slot and clear the old control + // byte. + unsafe { ptr::copy_nonoverlapping(i_p, new_i_p, size_of); - continue 'outer; - } else { - // If the target slot is occupied, swap the two elements - // and then continue processing the element that we just - // swapped into the old slot. - debug_assert_eq!(prev_ctrl, Tag::DELETED); + } + continue 'outer; + } else { + // If the target slot is occupied, swap the two elements + // and then continue processing the element that we just + // swapped into the old slot. + debug_assert_eq!(prev_ctrl, Tag::DELETED); + unsafe { ptr::swap_nonoverlapping(i_p, new_i_p, size_of); - continue 'inner; } + continue 'inner; } } + } - guard.growth_left = bucket_mask_to_capacity(guard.bucket_mask) - guard.items; + guard.growth_left = bucket_mask_to_capacity(guard.bucket_mask) - guard.items; - mem::forget(guard); - } + mem::forget(guard); } /// Deallocates the table without dropping any entries. @@ -3331,63 +3319,68 @@ impl RawTableInner { unsafe fn erase(&mut self, index: usize) { unsafe { debug_assert!(self.is_bucket_full(index)); + } - // This is the same as `index.wrapping_sub(Group::WIDTH) % self.num_buckets()` because - // the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. - let index_before = index.wrapping_sub(Group::WIDTH) & self.bucket_mask; - // SAFETY: - // - The caller must uphold the safety contract for `erase` method; - // - `index_before` is guaranteed to be in range due to masking with `self.bucket_mask` - let empty_before = Group::load(self.ctrl(index_before)).match_empty(); - let empty_after = Group::load(self.ctrl(index)).match_empty(); + // This is the same as `index.wrapping_sub(Group::WIDTH) % self.num_buckets()` because + // the number of buckets is a power of two, and `self.bucket_mask = self.num_buckets() - 1`. + let index_before = index.wrapping_sub(Group::WIDTH) & self.bucket_mask; + // SAFETY: + // - The caller must uphold the safety contract for `erase` method; + // - `index_before` is guaranteed to be in range due to masking with `self.bucket_mask` + let (empty_before, empty_after) = unsafe { + ( + Group::load(self.ctrl(index_before)).match_empty(), + Group::load(self.ctrl(index)).match_empty(), + ) + }; - // Inserting and searching in the map is performed by two key functions: - // - // - The `find_insert_index` function that looks up the index of any `Tag::EMPTY` or `Tag::DELETED` - // slot in a group to be able to insert. If it doesn't find an `Tag::EMPTY` or `Tag::DELETED` - // slot immediately in the first group, it jumps to the next `Group` looking for it, - // and so on until it has gone through all the groups in the control bytes. - // - // - The `find_inner` function that looks for the index of the desired element by looking - // at all the `FULL` bytes in the group. If it did not find the element right away, and - // there is no `Tag::EMPTY` byte in the group, then this means that the `find_insert_index` - // function may have found a suitable slot in the next group. Therefore, `find_inner` - // jumps further, and if it does not find the desired element and again there is no `Tag::EMPTY` - // byte, then it jumps further, and so on. The search stops only if `find_inner` function - // finds the desired element or hits an `Tag::EMPTY` slot/byte. - // - // Accordingly, this leads to two consequences: - // - // - The map must have `Tag::EMPTY` slots (bytes); - // - // - You can't just mark the byte to be erased as `Tag::EMPTY`, because otherwise the `find_inner` - // function may stumble upon an `Tag::EMPTY` byte before finding the desired element and stop - // searching. - // - // Thus it is necessary to check all bytes after and before the erased element. If we are in - // a contiguous `Group` of `FULL` or `Tag::DELETED` bytes (the number of `FULL` or `Tag::DELETED` bytes - // before and after is greater than or equal to `Group::WIDTH`), then we must mark our byte as - // `Tag::DELETED` in order for the `find_inner` function to go further. On the other hand, if there - // is at least one `Tag::EMPTY` slot in the `Group`, then the `find_inner` function will still stumble - // upon an `Tag::EMPTY` byte, so we can safely mark our erased byte as `Tag::EMPTY` as well. - // - // Finally, since `index_before == (index.wrapping_sub(Group::WIDTH) & self.bucket_mask) == index` - // and given all of the above, tables smaller than the group width (self.num_buckets() < Group::WIDTH) - // cannot have `Tag::DELETED` bytes. - // - // Note that in this context `leading_zeros` refers to the bytes at the end of a group, while - // `trailing_zeros` refers to the bytes at the beginning of a group. - let ctrl = - if empty_before.leading_zeros() + empty_after.trailing_zeros() >= Group::WIDTH { - Tag::DELETED - } else { - self.growth_left += 1; - Tag::EMPTY - }; - // SAFETY: the caller must uphold the safety contract for `erase` method. + // Inserting and searching in the map is performed by two key functions: + // + // - The `find_insert_index` function that looks up the index of any `Tag::EMPTY` or `Tag::DELETED` + // slot in a group to be able to insert. If it doesn't find an `Tag::EMPTY` or `Tag::DELETED` + // slot immediately in the first group, it jumps to the next `Group` looking for it, + // and so on until it has gone through all the groups in the control bytes. + // + // - The `find_inner` function that looks for the index of the desired element by looking + // at all the `FULL` bytes in the group. If it did not find the element right away, and + // there is no `Tag::EMPTY` byte in the group, then this means that the `find_insert_index` + // function may have found a suitable slot in the next group. Therefore, `find_inner` + // jumps further, and if it does not find the desired element and again there is no `Tag::EMPTY` + // byte, then it jumps further, and so on. The search stops only if `find_inner` function + // finds the desired element or hits an `Tag::EMPTY` slot/byte. + // + // Accordingly, this leads to two consequences: + // + // - The map must have `Tag::EMPTY` slots (bytes); + // + // - You can't just mark the byte to be erased as `Tag::EMPTY`, because otherwise the `find_inner` + // function may stumble upon an `Tag::EMPTY` byte before finding the desired element and stop + // searching. + // + // Thus it is necessary to check all bytes after and before the erased element. If we are in + // a contiguous `Group` of `FULL` or `Tag::DELETED` bytes (the number of `FULL` or `Tag::DELETED` bytes + // before and after is greater than or equal to `Group::WIDTH`), then we must mark our byte as + // `Tag::DELETED` in order for the `find_inner` function to go further. On the other hand, if there + // is at least one `Tag::EMPTY` slot in the `Group`, then the `find_inner` function will still stumble + // upon an `Tag::EMPTY` byte, so we can safely mark our erased byte as `Tag::EMPTY` as well. + // + // Finally, since `index_before == (index.wrapping_sub(Group::WIDTH) & self.bucket_mask) == index` + // and given all of the above, tables smaller than the group width (self.num_buckets() < Group::WIDTH) + // cannot have `Tag::DELETED` bytes. + // + // Note that in this context `leading_zeros` refers to the bytes at the end of a group, while + // `trailing_zeros` refers to the bytes at the beginning of a group. + let ctrl = if empty_before.leading_zeros() + empty_after.trailing_zeros() >= Group::WIDTH { + Tag::DELETED + } else { + self.growth_left += 1; + Tag::EMPTY + }; + // SAFETY: the caller must uphold the safety contract for `erase` method. + unsafe { self.set_ctrl(index, ctrl); - self.items -= 1; } + self.items -= 1; } } @@ -3499,9 +3492,11 @@ trait RawTableClone { impl RawTableClone for RawTable { default_fn! { #[cfg_attr(feature = "inline-more", inline)] - unsafe fn clone_from_spec(&mut self, source: &Self) { unsafe { - self.clone_from_impl(source); - }} + unsafe fn clone_from_spec(&mut self, source: &Self) { + unsafe { + self.clone_from_impl(source); + } + } } } #[cfg(feature = "nightly")] @@ -3517,10 +3512,10 @@ impl RawTableClone for RawTa .data_start() .as_ptr() .copy_to_nonoverlapping(self.data_start().as_ptr(), self.table.num_buckets()); - - self.table.items = source.table.items; - self.table.growth_left = source.table.growth_left; } + + self.table.items = source.table.items; + self.table.growth_left = source.table.growth_left; } } @@ -3531,26 +3526,30 @@ impl RawTable { /// - The control bytes are not initialized yet. #[cfg_attr(feature = "inline-more", inline)] unsafe fn clone_from_impl(&mut self, source: &Self) { + // Copy the control bytes unchanged. We do this in a single pass unsafe { - // Copy the control bytes unchanged. We do this in a single pass source .table .ctrl(0) .copy_to_nonoverlapping(self.table.ctrl(0), self.table.num_ctrl_bytes()); + } - // The cloning of elements may panic, in which case we need - // to make sure we drop only the elements that have been - // cloned so far. - let mut guard = guard((0, &mut *self), |(index, self_)| { - if T::NEEDS_DROP { - for i in 0..*index { + // The cloning of elements may panic, in which case we need + // to make sure we drop only the elements that have been + // cloned so far. + let mut guard = guard((0, &mut *self), |(index, self_)| { + if T::NEEDS_DROP { + for i in 0..*index { + unsafe { if self_.is_bucket_full(i) { self_.bucket(i).drop(); } } } - }); + } + }); + unsafe { for from in source.iter() { let index = source.bucket_index(&from); let to = guard.1.bucket(index); @@ -3559,13 +3558,13 @@ impl RawTable { // Update the index in case we need to unwind. guard.0 = index + 1; } + } - // Successfully cloned all items, no need to clean up. - mem::forget(guard); + // Successfully cloned all items, no need to clean up. + mem::forget(guard); - self.table.items = source.table.items; - self.table.growth_left = source.table.growth_left; - } + self.table.items = source.table.items; + self.table.growth_left = source.table.growth_left; } } @@ -3580,14 +3579,14 @@ impl Default for RawTable { unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawTable { #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { + // SAFETY: + // 1. We call the function only once; + // 2. We know for sure that `alloc` and `table_layout` matches the [`Allocator`] + // and [`TableLayout`] that were used to allocate this table. + // 3. If the drop function of any elements fails, then only a memory leak will occur, + // and we don't care because we are inside the `Drop` function of the `RawTable`, + // so there won't be any table left in an inconsistent state. unsafe { - // SAFETY: - // 1. We call the function only once; - // 2. We know for sure that `alloc` and `table_layout` matches the [`Allocator`] - // and [`TableLayout`] that were used to allocate this table. - // 3. If the drop function of any elements fails, then only a memory leak will occur, - // and we don't care because we are inside the `Drop` function of the `RawTable`, - // so there won't be any table left in an inconsistent state. self.table .drop_inner_table::(&self.alloc, Self::TABLE_LAYOUT); } @@ -3597,14 +3596,14 @@ unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawTable { impl Drop for RawTable { #[cfg_attr(feature = "inline-more", inline)] fn drop(&mut self) { + // SAFETY: + // 1. We call the function only once; + // 2. We know for sure that `alloc` and `table_layout` matches the [`Allocator`] + // and [`TableLayout`] that were used to allocate this table. + // 3. If the drop function of any elements fails, then only a memory leak will occur, + // and we don't care because we are inside the `Drop` function of the `RawTable`, + // so there won't be any table left in an inconsistent state. unsafe { - // SAFETY: - // 1. We call the function only once; - // 2. We know for sure that `alloc` and `table_layout` matches the [`Allocator`] - // and [`TableLayout`] that were used to allocate this table. - // 3. If the drop function of any elements fails, then only a memory leak will occur, - // and we don't care because we are inside the `Drop` function of the `RawTable`, - // so there won't be any table left in an inconsistent state. self.table .drop_inner_table::(&self.alloc, Self::TABLE_LAYOUT); } @@ -3671,23 +3670,25 @@ impl RawIterRange { /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[cfg_attr(feature = "inline-more", inline)] unsafe fn new(ctrl: *const u8, data: Bucket, len: usize) -> Self { - unsafe { - debug_assert_ne!(len, 0); - debug_assert_eq!(ctrl as usize % Group::WIDTH, 0); - // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] - let end = ctrl.add(len); - - // Load the first group and advance ctrl to point to the next group - // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] - let current_group = Group::load_aligned(ctrl.cast()).match_full(); - let next_ctrl = ctrl.add(Group::WIDTH); - - Self { - current_group: current_group.into_iter(), - data, - next_ctrl, - end, - } + debug_assert_ne!(len, 0); + debug_assert_eq!(ctrl as usize % Group::WIDTH, 0); + // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] + let end = unsafe { ctrl.add(len) }; + + // Load the first group and advance ctrl to point to the next group + // SAFETY: The caller must uphold the safety rules for the [`RawIterRange::new`] + let (current_group, next_ctrl) = unsafe { + ( + Group::load_aligned(ctrl.cast()).match_full(), + ctrl.add(Group::WIDTH), + ) + }; + + Self { + current_group: current_group.into_iter(), + data, + next_ctrl, + end, } } @@ -3740,21 +3741,21 @@ impl RawIterRange { /// after yielding all elements. #[cfg_attr(feature = "inline-more", inline)] unsafe fn next_impl(&mut self) -> Option> { - unsafe { - loop { - if let Some(index) = self.current_group.next() { - return Some(self.data.next_n(index)); - } + loop { + if let Some(index) = self.current_group.next() { + return Some(unsafe { self.data.next_n(index) }); + } - if DO_CHECK_PTR_RANGE && self.next_ctrl >= self.end { - return None; - } + if DO_CHECK_PTR_RANGE && self.next_ctrl >= self.end { + return None; + } - // We might read past self.end up to the next group boundary, - // but this is fine because it only occurs on tables smaller - // than the group size where the trailing control bytes are all - // EMPTY. On larger tables self.end is guaranteed to be aligned - // to the group size (since tables are power-of-two sized). + // We might read past self.end up to the next group boundary, + // but this is fine because it only occurs on tables smaller + // than the group size where the trailing control bytes are all + // EMPTY. On larger tables self.end is guaranteed to be aligned + // to the group size (since tables are power-of-two sized). + unsafe { self.current_group = Group::load_aligned(self.next_ctrl.cast()) .match_full() .into_iter(); @@ -3795,49 +3796,49 @@ impl RawIterRange { where F: FnMut(B, Bucket) -> B, { - unsafe { - loop { - while let Some(index) = self.current_group.next() { - // The returned `index` will always be in the range `0..Group::WIDTH`, - // so that calling `self.data.next_n(index)` is safe (see detailed explanation below). - debug_assert!(n != 0); - let bucket = self.data.next_n(index); - acc = f(acc, bucket); - n -= 1; - } + loop { + while let Some(index) = self.current_group.next() { + // The returned `index` will always be in the range `0..Group::WIDTH`, + // so that calling `self.data.next_n(index)` is safe (see detailed explanation below). + debug_assert!(n != 0); + let bucket = unsafe { self.data.next_n(index) }; + acc = f(acc, bucket); + n -= 1; + } - if n == 0 { - return acc; - } + if n == 0 { + return acc; + } - // SAFETY: The caller of this function ensures that: - // - // 1. The provided `n` value matches the actual number of items in the table; - // 2. The table is alive and did not moved. - // - // Taking the above into account, we always stay within the bounds, because: - // - // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), - // we will never end up in the given branch, since we should have already - // yielded all the elements of the table. - // - // 2. For tables larger than the group width. The number of buckets is a - // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since - // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the - // start of the array of control bytes, and never try to iterate after - // getting all the elements, the last `self.current_group` will read bytes - // from the `self.num_buckets() - Group::WIDTH` index. We know also that - // `self.current_group.next()` will always return indices within the range - // `0..Group::WIDTH`. - // - // Knowing all of the above and taking into account that we are synchronizing - // the `self.data` index with the index we used to read the `self.current_group`, - // the subsequent `self.data.next_n(index)` will always return a bucket with - // an index number less than `self.num_buckets()`. - // - // The last `self.next_ctrl`, whose index would be `self.num_buckets()`, will never - // actually be read, since we should have already yielded all the elements of - // the table. + // SAFETY: The caller of this function ensures that: + // + // 1. The provided `n` value matches the actual number of items in the table; + // 2. The table is alive and did not moved. + // + // Taking the above into account, we always stay within the bounds, because: + // + // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), + // we will never end up in the given branch, since we should have already + // yielded all the elements of the table. + // + // 2. For tables larger than the group width. The number of buckets is a + // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since + // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the + // start of the array of control bytes, and never try to iterate after + // getting all the elements, the last `self.current_group` will read bytes + // from the `self.num_buckets() - Group::WIDTH` index. We know also that + // `self.current_group.next()` will always return indices within the range + // `0..Group::WIDTH`. + // + // Knowing all of the above and taking into account that we are synchronizing + // the `self.data` index with the index we used to read the `self.current_group`, + // the subsequent `self.data.next_n(index)` will always return a bucket with + // an index number less than `self.num_buckets()`. + // + // The last `self.next_ctrl`, whose index would be `self.num_buckets()`, will never + // actually be read, since we should have already yielded all the elements of + // the table. + unsafe { self.current_group = Group::load_aligned(self.next_ctrl.cast()) .match_full() .into_iter(); @@ -4032,38 +4033,40 @@ impl FullBucketsIndices { /// [`Undefined Behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html #[inline(always)] unsafe fn next_impl(&mut self) -> Option { - unsafe { - loop { - if let Some(index) = self.current_group.next() { - // The returned `self.group_first_index + index` will always - // be in the range `0..self.num_buckets()`. See explanation below. - return Some(self.group_first_index + index); - } + loop { + if let Some(index) = self.current_group.next() { + // The returned `self.group_first_index + index` will always + // be in the range `0..self.num_buckets()`. See explanation below. + return Some(self.group_first_index + index); + } - // SAFETY: The caller of this function ensures that: - // - // 1. It never tries to iterate after getting all the elements; - // 2. The table is alive and did not moved; - // 3. The first `self.ctrl` pointed to the start of the array of control bytes. - // - // Taking the above into account, we always stay within the bounds, because: - // - // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), - // we will never end up in the given branch, since we should have already - // yielded all the elements of the table. - // - // 2. For tables larger than the group width. The number of buckets is a - // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since - // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the - // the start of the array of control bytes, and never try to iterate after - // getting all the elements, the last `self.ctrl` will be equal to - // the `self.num_buckets() - Group::WIDTH`, so `self.current_group.next()` - // will always contains indices within the range `0..Group::WIDTH`, - // and subsequent `self.group_first_index + index` will always return a - // number less than `self.num_buckets()`. + // SAFETY: The caller of this function ensures that: + // + // 1. It never tries to iterate after getting all the elements; + // 2. The table is alive and did not moved; + // 3. The first `self.ctrl` pointed to the start of the array of control bytes. + // + // Taking the above into account, we always stay within the bounds, because: + // + // 1. For tables smaller than the group width (self.num_buckets() <= Group::WIDTH), + // we will never end up in the given branch, since we should have already + // yielded all the elements of the table. + // + // 2. For tables larger than the group width. The number of buckets is a + // power of two (2 ^ n), Group::WIDTH is also power of two (2 ^ k). Since + // `(2 ^ n) > (2 ^ k)`, than `(2 ^ n) % (2 ^ k) = 0`. As we start from the + // the start of the array of control bytes, and never try to iterate after + // getting all the elements, the last `self.ctrl` will be equal to + // the `self.num_buckets() - Group::WIDTH`, so `self.current_group.next()` + // will always contains indices within the range `0..Group::WIDTH`, + // and subsequent `self.group_first_index + index` will always return a + // number less than `self.num_buckets()`. + unsafe { self.ctrl = NonNull::new_unchecked(self.ctrl.as_ptr().add(Group::WIDTH)); + } - // SAFETY: See explanation above. + // SAFETY: See explanation above. + unsafe { self.current_group = Group::load_aligned(self.ctrl.as_ptr().cast()) .match_full() .into_iter(); @@ -4086,12 +4089,10 @@ impl Iterator for FullBucketsIndices { return None; } - let nxt = unsafe { - // SAFETY: - // 1. We check number of items to yield using `items` field. - // 2. The caller ensures that the table is alive and has not moved. - self.next_impl() - }; + // SAFETY: + // 1. We check number of items to yield using `items` field. + // 2. The caller ensures that the table is alive and has not moved. + let nxt = unsafe { self.next_impl() }; debug_assert!(nxt.is_some()); self.items -= 1; @@ -4310,11 +4311,9 @@ pub(crate) struct RawIterHashIndices { impl RawIterHash { #[cfg_attr(feature = "inline-more", inline)] unsafe fn new(table: &RawTable, hash: u64) -> Self { - unsafe { - RawIterHash { - inner: RawIterHashIndices::new(&table.table, hash), - _marker: PhantomData, - } + RawIterHash { + inner: unsafe { RawIterHashIndices::new(&table.table, hash) }, + _marker: PhantomData, } } } @@ -4350,20 +4349,18 @@ impl Default for RawIterHashIndices { impl RawIterHashIndices { #[cfg_attr(feature = "inline-more", inline)] unsafe fn new(table: &RawTableInner, hash: u64) -> Self { - unsafe { - let tag_hash = Tag::full(hash); - let probe_seq = table.probe_seq(hash); - let group = Group::load(table.ctrl(probe_seq.pos)); - let bitmask = group.match_tag(tag_hash).into_iter(); - - RawIterHashIndices { - bucket_mask: table.bucket_mask, - ctrl: table.ctrl, - tag_hash, - probe_seq, - group, - bitmask, - } + let tag_hash = Tag::full(hash); + let probe_seq = table.probe_seq(hash); + let group = unsafe { Group::load(table.ctrl(probe_seq.pos)) }; + let bitmask = group.match_tag(tag_hash).into_iter(); + + RawIterHashIndices { + bucket_mask: table.bucket_mask, + ctrl: table.ctrl, + tag_hash, + probe_seq, + group, + bitmask, } } } diff --git a/src/table.rs b/src/table.rs index a3cf9c282..82fc6ffae 100644 --- a/src/table.rs +++ b/src/table.rs @@ -511,11 +511,9 @@ where /// ``` #[inline] pub unsafe fn get_bucket_entry_unchecked(&mut self, index: usize) -> OccupiedEntry<'_, T, A> { - unsafe { - OccupiedEntry { - bucket: self.raw.bucket(index), - table: self, - } + OccupiedEntry { + bucket: unsafe { self.raw.bucket(index) }, + table: self, } } From ac2e45443b4322ba42d4eb5443b7c1ee71d1b3a2 Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 16:47:42 -0500 Subject: [PATCH 08/11] Deny unreachable_pub --- Cargo.toml | 1 + src/raw/alloc.rs | 4 ++-- src/raw/mod.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cf2440710..69f0d8bd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ rust-version = "1.84.0" [lints.rust] missing_docs = "warn" +unreachable_pub = "warn" unsafe_op_in_unsafe_fn = "warn" # rust_2018_idioms diff --git a/src/raw/alloc.rs b/src/raw/alloc.rs index eabafa0f6..bb224fd85 100644 --- a/src/raw/alloc.rs +++ b/src/raw/alloc.rs @@ -9,7 +9,7 @@ pub(crate) use self::inner::{do_alloc, Allocator, Global}; #[cfg(feature = "nightly")] mod inner { #[cfg(test)] - pub use crate::alloc::alloc::AllocError; + pub(crate) use crate::alloc::alloc::AllocError; use crate::alloc::alloc::Layout; pub(crate) use crate::alloc::alloc::{Allocator, Global}; use core::ptr::NonNull; @@ -33,7 +33,7 @@ mod inner { mod inner { use crate::alloc::alloc::Layout; #[cfg(test)] - pub use allocator_api2::alloc::AllocError; + pub(crate) use allocator_api2::alloc::AllocError; pub(crate) use allocator_api2::alloc::{Allocator, Global}; use core::ptr::NonNull; diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 81dd3012c..0bf34700b 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -242,7 +242,7 @@ impl TableLayout { /// This is usually just a pointer to the element itself. However if the element /// is a ZST, then we instead track the index of the element in the table so /// that `erase` works properly. -pub struct Bucket { +pub(crate) struct Bucket { // Actually it is pointer to next element than element itself // this is needed to maintain pointer arithmetic invariants // keeping direct pointer to element introduces difficulty. From 4ea80121eda6029a1aedfbb0fc62e6d602670b5a Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 17:32:56 -0500 Subject: [PATCH 09/11] Replace allow with expect where possible --- benches/with_capacity.rs | 2 +- src/control/bitmask.rs | 4 ++-- src/control/group/generic.rs | 10 +++++----- src/control/group/lsx.rs | 10 +++++----- src/control/group/neon.rs | 8 ++++---- src/control/group/sse2.rs | 16 ++++++++-------- src/control/tag.rs | 2 +- src/external_trait_impls/rayon/helpers.rs | 2 +- src/external_trait_impls/serde.rs | 2 +- src/lib.rs | 2 +- src/macros.rs | 2 +- src/map.rs | 18 ++++++++---------- src/raw/alloc.rs | 5 ++--- src/raw/mod.rs | 20 ++++++++++---------- src/raw_entry.rs | 14 +++++++------- src/set.rs | 5 ++--- src/util.rs | 2 +- tests/equivalent_trait.rs | 2 +- tests/rayon.rs | 2 +- tests/serde.rs | 2 +- tests/set.rs | 4 ++-- 21 files changed, 65 insertions(+), 69 deletions(-) diff --git a/benches/with_capacity.rs b/benches/with_capacity.rs index 933276387..6689dac79 100644 --- a/benches/with_capacity.rs +++ b/benches/with_capacity.rs @@ -1,4 +1,4 @@ -#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 +#![expect(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![feature(test)] extern crate test; diff --git a/src/control/bitmask.rs b/src/control/bitmask.rs index cfacfce67..32d1bdaaf 100644 --- a/src/control/bitmask.rs +++ b/src/control/bitmask.rs @@ -21,12 +21,12 @@ use super::group::{ #[derive(Copy, Clone)] pub(crate) struct BitMask(pub(crate) BitMaskWord); -#[allow(clippy::use_self)] +#[expect(clippy::use_self)] impl BitMask { /// Returns a new `BitMask` with all bits inverted. #[inline] #[must_use] - #[allow(dead_code)] + #[expect(dead_code)] pub(crate) fn invert(self) -> Self { BitMask(self.0 ^ BITMASK_MASK) } diff --git a/src/control/group/generic.rs b/src/control/group/generic.rs index d401a4c6f..0d3982559 100644 --- a/src/control/group/generic.rs +++ b/src/control/group/generic.rs @@ -24,7 +24,7 @@ pub(crate) type BitMaskWord = GroupWord; pub(crate) type NonZeroBitMaskWord = NonZeroGroupWord; pub(crate) const BITMASK_STRIDE: usize = 8; // We only care about the highest bit of each tag for the mask. -#[allow(clippy::cast_possible_truncation, clippy::unnecessary_cast)] +#[expect(clippy::cast_possible_truncation, clippy::unnecessary_cast)] pub(crate) const BITMASK_MASK: BitMaskWord = u64::from_ne_bytes([Tag::DELETED.0; 8]) as GroupWord; pub(crate) const BITMASK_ITER_MASK: BitMaskWord = !0; @@ -45,7 +45,7 @@ pub(crate) struct Group(GroupWord); // little-endian just before creating a BitMask. The can potentially // enable the compiler to eliminate unnecessary byte swaps if we are // only checking whether a BitMask is empty. -#[allow(clippy::use_self)] +#[expect(clippy::use_self)] impl Group { /// Number of bytes in the group. pub(crate) const WIDTH: usize = mem::size_of::(); @@ -70,7 +70,7 @@ impl Group { /// Loads a group of tags starting at the given address. #[inline] - #[allow(clippy::cast_ptr_alignment)] // unaligned load + #[expect(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { unsafe { Group(ptr::read_unaligned(ptr.cast())) } } @@ -78,7 +78,7 @@ impl Group { /// Loads a group of tags starting at the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { Group(ptr::read(ptr.cast())) } @@ -87,7 +87,7 @@ impl Group { /// Stores the group of tags to the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { diff --git a/src/control/group/lsx.rs b/src/control/group/lsx.rs index ee2540043..b117addf4 100644 --- a/src/control/group/lsx.rs +++ b/src/control/group/lsx.rs @@ -18,7 +18,7 @@ pub(crate) const BITMASK_ITER_MASK: BitMaskWord = !0; pub(crate) struct Group(m128i); // FIXME: https://github.com/rust-lang/rust-clippy/issues/3859 -#[allow(clippy::use_self)] +#[expect(clippy::use_self)] impl Group { /// Number of bytes in the group. pub(crate) const WIDTH: usize = mem::size_of::(); @@ -28,7 +28,7 @@ impl Group { /// /// This is guaranteed to be aligned to the group size. #[inline] - #[allow(clippy::items_after_statements)] + #[expect(clippy::items_after_statements)] pub(crate) const fn static_empty() -> &'static [Tag; Group::WIDTH] { #[repr(C)] struct AlignedTags { @@ -44,7 +44,7 @@ impl Group { /// Loads a group of tags starting at the given address. #[inline] - #[allow(clippy::cast_ptr_alignment)] // unaligned load + #[expect(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { unsafe { Group(lsx_vld::<0>(ptr.cast())) } } @@ -52,7 +52,7 @@ impl Group { /// Loads a group of tags starting at the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { Group(lsx_vld::<0>(ptr.cast())) } @@ -61,7 +61,7 @@ impl Group { /// Stores the group of tags to the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { diff --git a/src/control/group/neon.rs b/src/control/group/neon.rs index 02e5fd491..8273063a5 100644 --- a/src/control/group/neon.rs +++ b/src/control/group/neon.rs @@ -16,7 +16,7 @@ pub(crate) const BITMASK_ITER_MASK: BitMaskWord = 0x8080_8080_8080_8080; #[derive(Copy, Clone)] pub(crate) struct Group(neon::uint8x8_t); -#[allow(clippy::use_self)] +#[expect(clippy::use_self)] impl Group { /// Number of bytes in the group. pub(crate) const WIDTH: usize = mem::size_of::(); @@ -41,7 +41,7 @@ impl Group { /// Loads a group of tags starting at the given address. #[inline] - #[allow(clippy::cast_ptr_alignment)] // unaligned load + #[expect(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { unsafe { Group(neon::vld1_u8(ptr.cast())) } } @@ -49,7 +49,7 @@ impl Group { /// Loads a group of tags starting at the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { Group(neon::vld1_u8(ptr.cast())) } @@ -58,7 +58,7 @@ impl Group { /// Stores the group of tags to the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { diff --git a/src/control/group/sse2.rs b/src/control/group/sse2.rs index 651dc6098..ee807168b 100644 --- a/src/control/group/sse2.rs +++ b/src/control/group/sse2.rs @@ -21,7 +21,7 @@ pub(crate) const BITMASK_ITER_MASK: BitMaskWord = !0; pub(crate) struct Group(x86::__m128i); // FIXME: https://github.com/rust-lang/rust-clippy/issues/3859 -#[allow(clippy::use_self)] +#[expect(clippy::use_self)] impl Group { /// Number of bytes in the group. pub(crate) const WIDTH: usize = mem::size_of::(); @@ -31,7 +31,7 @@ impl Group { /// /// This is guaranteed to be aligned to the group size. #[inline] - #[allow(clippy::items_after_statements)] + #[expect(clippy::items_after_statements)] pub(crate) const fn static_empty() -> &'static [Tag; Group::WIDTH] { #[repr(C)] struct AlignedTags { @@ -47,7 +47,7 @@ impl Group { /// Loads a group of tags starting at the given address. #[inline] - #[allow(clippy::cast_ptr_alignment)] // unaligned load + #[expect(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { unsafe { Group(x86::_mm_loadu_si128(ptr.cast())) } } @@ -55,7 +55,7 @@ impl Group { /// Loads a group of tags starting at the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { Group(x86::_mm_load_si128(ptr.cast())) } @@ -64,7 +64,7 @@ impl Group { /// Stores the group of tags to the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[allow(clippy::cast_ptr_alignment)] + #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { @@ -76,7 +76,7 @@ impl Group { /// the given value. #[inline] pub(crate) fn match_tag(self, tag: Tag) -> BitMask { - #[allow( + #[expect( clippy::cast_possible_wrap, // tag.0: Tag as i8 // tag: i32 as u16 // note: _mm_movemask_epi8 returns a 16-bit mask in a i32, the @@ -101,7 +101,7 @@ impl Group { /// `EMPTY` or `DELETED`. #[inline] pub(crate) fn match_empty_or_deleted(self) -> BitMask { - #[allow( + #[expect( // tag: i32 as u16 // note: _mm_movemask_epi8 returns a 16-bit mask in a i32, the // upper 16-bits of the i32 are zeroed: @@ -133,7 +133,7 @@ impl Group { // let special = 0 > tag = 1111_1111 (true) or 0000_0000 (false) // 1111_1111 | 1000_0000 = 1111_1111 // 0000_0000 | 1000_0000 = 1000_0000 - #[allow( + #[expect( clippy::cast_possible_wrap, // tag: Tag::DELETED.0 as i8 )] unsafe { diff --git a/src/control/tag.rs b/src/control/tag.rs index 817dd55cd..d08477e43 100644 --- a/src/control/tag.rs +++ b/src/control/tag.rs @@ -32,7 +32,7 @@ impl Tag { /// Creates a control tag representing a full bucket with the given hash. #[inline] - #[allow(clippy::cast_possible_truncation)] + #[expect(clippy::cast_possible_truncation)] pub(crate) const fn full(hash: u64) -> Tag { // Constant for function that grabs the top 7 bits of the hash. const MIN_HASH_LEN: usize = if mem::size_of::() < mem::size_of::() { diff --git a/src/external_trait_impls/rayon/helpers.rs b/src/external_trait_impls/rayon/helpers.rs index 4420af3a2..d934264c0 100644 --- a/src/external_trait_impls/rayon/helpers.rs +++ b/src/external_trait_impls/rayon/helpers.rs @@ -4,7 +4,7 @@ use alloc::vec::Vec; use rayon::iter::{IntoParallelIterator, ParallelIterator}; /// Helper for collecting parallel iterators to an intermediary -#[allow(clippy::linkedlist)] // yes, we need linked list here for efficient appending! +#[expect(clippy::linkedlist)] // yes, we need linked list here for efficient appending! pub(super) fn collect(iter: I) -> (LinkedList>, usize) { let list = iter.into_par_iter().collect_vec_list(); diff --git a/src/external_trait_impls/serde.rs b/src/external_trait_impls/serde.rs index 85b6170b8..4515489d3 100644 --- a/src/external_trait_impls/serde.rs +++ b/src/external_trait_impls/serde.rs @@ -177,7 +177,7 @@ mod set { deserializer.deserialize_seq(visitor) } - #[allow(clippy::missing_errors_doc)] + #[expect(clippy::missing_errors_doc)] fn deserialize_in_place(deserializer: D, place: &mut Self) -> Result<(), D::Error> where D: Deserializer<'de>, diff --git a/src/lib.rs b/src/lib.rs index d6ba501f6..2b3539cdb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,7 +27,7 @@ )] #![cfg_attr(feature = "nightly", warn(fuzzy_provenance_casts))] #![cfg_attr(feature = "rustc-dep-of-std", feature(rustc_attrs))] -#![cfg_attr(feature = "nightly", allow(internal_features))] +#![cfg_attr(feature = "nightly", expect(internal_features))] #![cfg_attr( all(feature = "nightly", target_arch = "loongarch64"), feature(stdarch_loongarch) diff --git a/src/macros.rs b/src/macros.rs index eaba6bed1..7177d2212 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -1,5 +1,5 @@ // See the cfg-if crate. -#[allow(unused_macro_rules)] +#[expect(unused_macro_rules)] macro_rules! cfg_if { // match if/else chains with a final `else` ($( diff --git a/src/map.rs b/src/map.rs index 5afbe2c23..14bd28dd7 100644 --- a/src/map.rs +++ b/src/map.rs @@ -228,7 +228,7 @@ where /// Ensures that a single closure type across uses of this which, in turn prevents multiple /// instances of any functions like `RawTable::reserve` from being generated #[cfg_attr(feature = "inline-more", inline)] -#[allow(dead_code)] +#[expect(dead_code)] pub(crate) fn equivalent(k: &Q) -> impl Fn(&K) -> bool + '_ where Q: Equivalent + ?Sized, @@ -4879,7 +4879,7 @@ where } } -#[allow(dead_code)] +#[expect(dead_code)] fn assert_covariance() { fn map_key<'new>(v: HashMap<&'static str, u8>) -> HashMap<&'new str, u8> { v @@ -5001,7 +5001,6 @@ mod test_map { assert_eq!(m.len(), 1); assert!(m.insert(2, 4).is_none()); assert_eq!(m.len(), 2); - #[allow(clippy::redundant_clone)] let m2 = m.clone(); assert_eq!(*m2.get(&1).unwrap(), 2); assert_eq!(*m2.get(&2).unwrap(), 4); @@ -5800,8 +5799,8 @@ mod test_map { #[test] fn test_entry_take_doesnt_corrupt() { - #![allow(deprecated)] //rand - // Test for #19292 + #![expect(deprecated)] //rand + // Test for #19292 fn check(m: &HashMap) { for k in m.keys() { assert!(m.contains_key(k), "{k} is in keys() but not in the map?"); @@ -5836,8 +5835,8 @@ mod test_map { #[test] fn test_entry_ref_take_doesnt_corrupt() { - #![allow(deprecated)] //rand - // Test for #19292 + #![expect(deprecated)] //rand + // Test for #19292 fn check(m: &HashMap) { for k in m.keys() { assert!(m.contains_key(k), "{k} is in keys() but not in the map?"); @@ -5889,7 +5888,6 @@ mod test_map { } #[test] - #[allow(clippy::needless_borrow)] fn test_extend_ref_kv_tuple() { use std::ops::AddAssign; let mut a = HashMap::new(); @@ -6111,8 +6109,8 @@ mod test_map { #[test] fn test_replace_entry_with_doesnt_corrupt() { - #![allow(deprecated)] //rand - // Test for #19292 + #![expect(deprecated)] //rand + // Test for #19292 fn check(m: &HashMap) { for k in m.keys() { assert!(m.contains_key(k), "{k} is in keys() but not in the map?"); diff --git a/src/raw/alloc.rs b/src/raw/alloc.rs index bb224fd85..09652b377 100644 --- a/src/raw/alloc.rs +++ b/src/raw/alloc.rs @@ -14,7 +14,6 @@ mod inner { pub(crate) use crate::alloc::alloc::{Allocator, Global}; use core::ptr::NonNull; - #[allow(clippy::map_err_ignore)] pub(crate) fn do_alloc(alloc: &A, layout: Layout) -> Result, ()> { match alloc.allocate(layout) { Ok(ptr) => Ok(ptr), @@ -37,7 +36,7 @@ mod inner { pub(crate) use allocator_api2::alloc::{Allocator, Global}; use core::ptr::NonNull; - #[allow(clippy::map_err_ignore)] + #[expect(clippy::map_err_ignore)] pub(crate) fn do_alloc(alloc: &A, layout: Layout) -> Result, ()> { match alloc.allocate(layout) { Ok(ptr) => Ok(ptr), @@ -59,7 +58,7 @@ mod inner { use crate::alloc::alloc::{alloc, dealloc, Layout}; use core::ptr::NonNull; - #[allow(clippy::missing_safety_doc)] // not exposed outside of this crate + #[expect(clippy::missing_safety_doc)] // not exposed outside of this crate pub unsafe trait Allocator { fn allocate(&self, layout: Layout) -> Result, ()>; unsafe fn deallocate(&self, ptr: NonNull, layout: Layout); diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 0bf34700b..231ddc0e9 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -57,7 +57,7 @@ impl SizedTypeProperties for T {} /// Primary hash function, used to select the initial bucket to probe from. #[inline] -#[allow(clippy::cast_possible_truncation)] +#[expect(clippy::cast_possible_truncation)] fn h1(hash: u64) -> usize { // On 32-bit platforms we simply ignore the higher hash bits. hash as usize @@ -824,7 +824,7 @@ impl RawTable { /// Erases an element from the table, dropping it in place. #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::needless_pass_by_value)] + #[expect(clippy::needless_pass_by_value)] pub(crate) unsafe fn erase(&mut self, item: Bucket) { unsafe { // Erase the element from the table first since drop might panic. @@ -837,7 +837,7 @@ impl RawTable { /// /// This also returns an index to the newly free bucket. #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::needless_pass_by_value)] + #[expect(clippy::needless_pass_by_value)] pub(crate) unsafe fn remove(&mut self, item: Bucket) -> (T, usize) { unsafe { self.erase_no_drop(&item); @@ -850,7 +850,7 @@ impl RawTable { /// This also returns an index to the newly free bucket /// and the former `Tag` for that bucket. #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::needless_pass_by_value)] + #[expect(clippy::needless_pass_by_value)] pub(crate) unsafe fn remove_tagged(&mut self, item: Bucket) -> (T, usize, Tag) { unsafe { let index = self.bucket_index(&item); @@ -2149,7 +2149,7 @@ impl RawTableInner { /// /// [`Bucket::as_ptr`]: Bucket::as_ptr /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[allow(clippy::mut_mut)] + #[expect(clippy::mut_mut)] #[inline] unsafe fn prepare_rehash_in_place(&mut self) { // Bulk convert all full control bytes to DELETED, and all DELETED control bytes to EMPTY. @@ -2769,7 +2769,7 @@ impl RawTableInner { /// by this function results in [`undefined behavior`]. /// /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[allow(clippy::mut_mut)] + #[expect(clippy::mut_mut)] #[inline] fn prepare_resize<'a, A>( &self, @@ -2826,7 +2826,7 @@ impl RawTableInner { /// * The [`RawTableInner`] must have properly initialized control bytes. /// /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[allow(clippy::inline_always)] + #[expect(clippy::inline_always)] #[inline(always)] unsafe fn reserve_rehash_inner( &mut self, @@ -2974,7 +2974,7 @@ impl RawTableInner { /// /// [`RawTableInner::find_insert_index`]: RawTableInner::find_insert_index /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[allow(clippy::inline_always)] + #[expect(clippy::inline_always)] #[inline(always)] unsafe fn resize_inner( &mut self, @@ -3072,7 +3072,7 @@ impl RawTableInner { /// * The [`RawTableInner`] must have properly initialized control bytes. /// /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[allow(clippy::inline_always)] + #[expect(clippy::inline_always)] #[cfg_attr(feature = "inline-more", inline(always))] #[cfg_attr(not(feature = "inline-more"), inline)] unsafe fn rehash_in_place( @@ -3790,7 +3790,7 @@ impl RawIterRange { /// in the table. /// /// [`Undefined Behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[allow(clippy::while_let_on_iterator)] + #[expect(clippy::while_let_on_iterator)] #[cfg_attr(feature = "inline-more", inline)] unsafe fn fold_impl(mut self, mut n: usize, mut acc: B, mut f: F) -> B where diff --git a/src/raw_entry.rs b/src/raw_entry.rs index 20623a83b..2a7560d53 100644 --- a/src/raw_entry.rs +++ b/src/raw_entry.rs @@ -521,7 +521,7 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilderMut<'a, K, V, S, A> { /// assert_eq!(map[&"a"], 100); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::wrong_self_convention)] + #[expect(clippy::wrong_self_convention)] pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V, S, A> where S: BuildHasher, @@ -554,7 +554,7 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilderMut<'a, K, V, S, A> { /// assert_eq!(map[&"a"], 100); /// ``` #[inline] - #[allow(clippy::wrong_self_convention)] + #[expect(clippy::wrong_self_convention)] pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> RawEntryMut<'a, K, V, S, A> where Q: Equivalent + ?Sized, @@ -587,7 +587,7 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilderMut<'a, K, V, S, A> { /// assert_eq!(map[&"a"], 100); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::wrong_self_convention)] + #[expect(clippy::wrong_self_convention)] pub fn from_hash(self, hash: u64, is_match: F) -> RawEntryMut<'a, K, V, S, A> where for<'b> F: FnMut(&'b K) -> bool, @@ -627,7 +627,7 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilder<'a, K, V, S, A> { /// assert_eq!(map.raw_entry().from_key(&key), Some((&"a", &100))); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::wrong_self_convention)] + #[expect(clippy::wrong_self_convention)] pub fn from_key(self, k: &Q) -> Option<(&'a K, &'a V)> where S: BuildHasher, @@ -658,7 +658,7 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilder<'a, K, V, S, A> { /// assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &key), Some((&"a", &100))); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::wrong_self_convention)] + #[expect(clippy::wrong_self_convention)] pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> Option<(&'a K, &'a V)> where Q: Equivalent + ?Sized, @@ -698,7 +698,7 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilder<'a, K, V, S, A> { /// assert_eq!(map.raw_entry().from_hash(hash, |k| k == &key), Some((&"a", &100))); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::wrong_self_convention)] + #[expect(clippy::wrong_self_convention)] pub fn from_hash(self, hash: u64, is_match: F) -> Option<(&'a K, &'a V)> where F: FnMut(&K) -> bool, @@ -1359,7 +1359,7 @@ impl<'a, K, V, S, A: Allocator> RawVacantEntryMut<'a, K, V, S, A> { /// assert_eq!(map[&"c"], 300); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[allow(clippy::shadow_unrelated)] + #[expect(clippy::shadow_unrelated)] pub fn insert_hashed_nocheck(self, hash: u64, key: K, value: V) -> (&'a mut K, &'a mut V) where K: Hash, diff --git a/src/set.rs b/src/set.rs index 9b3703272..5ece27f77 100644 --- a/src/set.rs +++ b/src/set.rs @@ -2530,7 +2530,7 @@ impl<'a, T, S, A: Allocator> VacantEntry<'a, T, S, A> { } } -#[allow(dead_code)] +#[expect(dead_code)] fn assert_covariance() { fn set<'new>(v: HashSet<&'static str>) -> HashSet<&'new str> { v @@ -2918,7 +2918,7 @@ mod test_set { use core::hash; #[derive(Debug)] - #[allow(dead_code)] + #[expect(dead_code)] struct Foo(&'static str, i32); impl PartialEq for Foo { @@ -2947,7 +2947,6 @@ mod test_set { } #[test] - #[allow(clippy::needless_borrow)] fn test_extend_ref() { let mut a = HashSet::new(); a.insert(1); diff --git a/src/util.rs b/src/util.rs index 90a8df311..1217e56b6 100644 --- a/src/util.rs +++ b/src/util.rs @@ -32,7 +32,7 @@ pub(crate) fn unlikely(b: bool) -> bool { // FIXME: use strict provenance functions once they are stable. // Implement it with a transmute for now. #[inline(always)] -#[allow(clippy::useless_transmute)] // clippy is wrong, cast and transmute are different here +#[expect(clippy::useless_transmute)] // clippy is wrong, cast and transmute are different here pub(crate) fn invalid_mut(addr: usize) -> *mut T { unsafe { core::mem::transmute(addr) } } diff --git a/tests/equivalent_trait.rs b/tests/equivalent_trait.rs index 7ffa8d006..31f0a63e2 100644 --- a/tests/equivalent_trait.rs +++ b/tests/equivalent_trait.rs @@ -1,4 +1,4 @@ -#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 +#![expect(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 use hashbrown::Equivalent; use hashbrown::HashMap; diff --git a/tests/rayon.rs b/tests/rayon.rs index 98cdf42f3..e9ba040a3 100644 --- a/tests/rayon.rs +++ b/tests/rayon.rs @@ -1,4 +1,4 @@ -#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 +#![expect(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![cfg(feature = "rayon")] #[macro_use] diff --git a/tests/serde.rs b/tests/serde.rs index d792b17ae..ef91d76b7 100644 --- a/tests/serde.rs +++ b/tests/serde.rs @@ -1,4 +1,4 @@ -#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 +#![expect(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![cfg(feature = "serde")] use core::hash::BuildHasherDefault; diff --git a/tests/set.rs b/tests/set.rs index f5bb9fae8..cf9204e79 100644 --- a/tests/set.rs +++ b/tests/set.rs @@ -1,4 +1,4 @@ -#![allow(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 +#![expect(missing_docs)] // https://github.com/rust-lang/rust/issues/137561 #![cfg(not(miri))] // FIXME: takes too long use hashbrown::HashSet; @@ -21,7 +21,7 @@ fn test_hashset_insert_remove() { .collect(); // more readable with explicit `true` / `false` - #[allow(clippy::bool_assert_comparison)] + #[expect(clippy::bool_assert_comparison)] for _ in 0..32 { for x in &tx { assert_eq!(m.contains(x), false); From 673129dc72e25b1ee2eddaf9cc273a7bc56d547d Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 17:38:06 -0500 Subject: [PATCH 10/11] Remove expectations that no longer apply --- src/control/bitmask.rs | 1 - src/control/group/sse2.rs | 4 ---- src/control/tag.rs | 1 - src/external_trait_impls/serde.rs | 1 - src/map.rs | 2 +- src/raw/alloc.rs | 1 - src/raw/mod.rs | 4 +--- src/raw_entry.rs | 7 ------- src/util.rs | 1 - 9 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/control/bitmask.rs b/src/control/bitmask.rs index 32d1bdaaf..f8c079183 100644 --- a/src/control/bitmask.rs +++ b/src/control/bitmask.rs @@ -26,7 +26,6 @@ impl BitMask { /// Returns a new `BitMask` with all bits inverted. #[inline] #[must_use] - #[expect(dead_code)] pub(crate) fn invert(self) -> Self { BitMask(self.0 ^ BITMASK_MASK) } diff --git a/src/control/group/sse2.rs b/src/control/group/sse2.rs index ee807168b..4677f1ac0 100644 --- a/src/control/group/sse2.rs +++ b/src/control/group/sse2.rs @@ -31,7 +31,6 @@ impl Group { /// /// This is guaranteed to be aligned to the group size. #[inline] - #[expect(clippy::items_after_statements)] pub(crate) const fn static_empty() -> &'static [Tag; Group::WIDTH] { #[repr(C)] struct AlignedTags { @@ -47,7 +46,6 @@ impl Group { /// Loads a group of tags starting at the given address. #[inline] - #[expect(clippy::cast_ptr_alignment)] // unaligned load pub(crate) unsafe fn load(ptr: *const Tag) -> Self { unsafe { Group(x86::_mm_loadu_si128(ptr.cast())) } } @@ -55,7 +53,6 @@ impl Group { /// Loads a group of tags starting at the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn load_aligned(ptr: *const Tag) -> Self { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { Group(x86::_mm_load_si128(ptr.cast())) } @@ -64,7 +61,6 @@ impl Group { /// Stores the group of tags to the given address, which must be /// aligned to `mem::align_of::()`. #[inline] - #[expect(clippy::cast_ptr_alignment)] pub(crate) unsafe fn store_aligned(self, ptr: *mut Tag) { debug_assert_eq!(ptr.align_offset(mem::align_of::()), 0); unsafe { diff --git a/src/control/tag.rs b/src/control/tag.rs index d08477e43..95c7640cb 100644 --- a/src/control/tag.rs +++ b/src/control/tag.rs @@ -32,7 +32,6 @@ impl Tag { /// Creates a control tag representing a full bucket with the given hash. #[inline] - #[expect(clippy::cast_possible_truncation)] pub(crate) const fn full(hash: u64) -> Tag { // Constant for function that grabs the top 7 bits of the hash. const MIN_HASH_LEN: usize = if mem::size_of::() < mem::size_of::() { diff --git a/src/external_trait_impls/serde.rs b/src/external_trait_impls/serde.rs index 4515489d3..92614fe04 100644 --- a/src/external_trait_impls/serde.rs +++ b/src/external_trait_impls/serde.rs @@ -177,7 +177,6 @@ mod set { deserializer.deserialize_seq(visitor) } - #[expect(clippy::missing_errors_doc)] fn deserialize_in_place(deserializer: D, place: &mut Self) -> Result<(), D::Error> where D: Deserializer<'de>, diff --git a/src/map.rs b/src/map.rs index 14bd28dd7..76c5c2b59 100644 --- a/src/map.rs +++ b/src/map.rs @@ -228,7 +228,7 @@ where /// Ensures that a single closure type across uses of this which, in turn prevents multiple /// instances of any functions like `RawTable::reserve` from being generated #[cfg_attr(feature = "inline-more", inline)] -#[expect(dead_code)] +#[cfg(feature = "raw-entry")] pub(crate) fn equivalent(k: &Q) -> impl Fn(&K) -> bool + '_ where Q: Equivalent + ?Sized, diff --git a/src/raw/alloc.rs b/src/raw/alloc.rs index 09652b377..9bb45a5dc 100644 --- a/src/raw/alloc.rs +++ b/src/raw/alloc.rs @@ -36,7 +36,6 @@ mod inner { pub(crate) use allocator_api2::alloc::{Allocator, Global}; use core::ptr::NonNull; - #[expect(clippy::map_err_ignore)] pub(crate) fn do_alloc(alloc: &A, layout: Layout) -> Result, ()> { match alloc.allocate(layout) { Ok(ptr) => Ok(ptr), diff --git a/src/raw/mod.rs b/src/raw/mod.rs index 231ddc0e9..0b35670af 100644 --- a/src/raw/mod.rs +++ b/src/raw/mod.rs @@ -2149,7 +2149,6 @@ impl RawTableInner { /// /// [`Bucket::as_ptr`]: Bucket::as_ptr /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[expect(clippy::mut_mut)] #[inline] unsafe fn prepare_rehash_in_place(&mut self) { // Bulk convert all full control bytes to DELETED, and all DELETED control bytes to EMPTY. @@ -2769,7 +2768,6 @@ impl RawTableInner { /// by this function results in [`undefined behavior`]. /// /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[expect(clippy::mut_mut)] #[inline] fn prepare_resize<'a, A>( &self, @@ -3072,7 +3070,7 @@ impl RawTableInner { /// * The [`RawTableInner`] must have properly initialized control bytes. /// /// [`undefined behavior`]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html - #[expect(clippy::inline_always)] + #[cfg_attr(feature = "inline-more", expect(clippy::inline_always))] #[cfg_attr(feature = "inline-more", inline(always))] #[cfg_attr(not(feature = "inline-more"), inline)] unsafe fn rehash_in_place( diff --git a/src/raw_entry.rs b/src/raw_entry.rs index 2a7560d53..d8dfdc550 100644 --- a/src/raw_entry.rs +++ b/src/raw_entry.rs @@ -521,7 +521,6 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilderMut<'a, K, V, S, A> { /// assert_eq!(map[&"a"], 100); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[expect(clippy::wrong_self_convention)] pub fn from_key(self, k: &Q) -> RawEntryMut<'a, K, V, S, A> where S: BuildHasher, @@ -554,7 +553,6 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilderMut<'a, K, V, S, A> { /// assert_eq!(map[&"a"], 100); /// ``` #[inline] - #[expect(clippy::wrong_self_convention)] pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> RawEntryMut<'a, K, V, S, A> where Q: Equivalent + ?Sized, @@ -587,7 +585,6 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilderMut<'a, K, V, S, A> { /// assert_eq!(map[&"a"], 100); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[expect(clippy::wrong_self_convention)] pub fn from_hash(self, hash: u64, is_match: F) -> RawEntryMut<'a, K, V, S, A> where for<'b> F: FnMut(&'b K) -> bool, @@ -627,7 +624,6 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilder<'a, K, V, S, A> { /// assert_eq!(map.raw_entry().from_key(&key), Some((&"a", &100))); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[expect(clippy::wrong_self_convention)] pub fn from_key(self, k: &Q) -> Option<(&'a K, &'a V)> where S: BuildHasher, @@ -658,7 +654,6 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilder<'a, K, V, S, A> { /// assert_eq!(map.raw_entry().from_key_hashed_nocheck(hash, &key), Some((&"a", &100))); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[expect(clippy::wrong_self_convention)] pub fn from_key_hashed_nocheck(self, hash: u64, k: &Q) -> Option<(&'a K, &'a V)> where Q: Equivalent + ?Sized, @@ -698,7 +693,6 @@ impl<'a, K, V, S, A: Allocator> RawEntryBuilder<'a, K, V, S, A> { /// assert_eq!(map.raw_entry().from_hash(hash, |k| k == &key), Some((&"a", &100))); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[expect(clippy::wrong_self_convention)] pub fn from_hash(self, hash: u64, is_match: F) -> Option<(&'a K, &'a V)> where F: FnMut(&K) -> bool, @@ -1359,7 +1353,6 @@ impl<'a, K, V, S, A: Allocator> RawVacantEntryMut<'a, K, V, S, A> { /// assert_eq!(map[&"c"], 300); /// ``` #[cfg_attr(feature = "inline-more", inline)] - #[expect(clippy::shadow_unrelated)] pub fn insert_hashed_nocheck(self, hash: u64, key: K, value: V) -> (&'a mut K, &'a mut V) where K: Hash, diff --git a/src/util.rs b/src/util.rs index 1217e56b6..8667f34dc 100644 --- a/src/util.rs +++ b/src/util.rs @@ -32,7 +32,6 @@ pub(crate) fn unlikely(b: bool) -> bool { // FIXME: use strict provenance functions once they are stable. // Implement it with a transmute for now. #[inline(always)] -#[expect(clippy::useless_transmute)] // clippy is wrong, cast and transmute are different here pub(crate) fn invalid_mut(addr: usize) -> *mut T { unsafe { core::mem::transmute(addr) } } From c74adb5200e64ea98a0a6e68b633c1ed0c9c48ab Mon Sep 17 00:00:00 2001 From: ltdk Date: Tue, 23 Dec 2025 19:33:00 -0500 Subject: [PATCH 11/11] BitMask::invert isn't used in all cases; just inline it to avoid allowing a lint --- src/control/bitmask.rs | 11 +---------- src/control/group/generic.rs | 4 ++-- src/control/group/lsx.rs | 1 - src/control/group/mod.rs | 4 +--- src/control/group/neon.rs | 1 - src/control/group/sse2.rs | 3 +-- 6 files changed, 5 insertions(+), 19 deletions(-) diff --git a/src/control/bitmask.rs b/src/control/bitmask.rs index f8c079183..15f97a18f 100644 --- a/src/control/bitmask.rs +++ b/src/control/bitmask.rs @@ -1,6 +1,4 @@ -use super::group::{ - BitMaskWord, NonZeroBitMaskWord, BITMASK_ITER_MASK, BITMASK_MASK, BITMASK_STRIDE, -}; +use super::group::{BitMaskWord, NonZeroBitMaskWord, BITMASK_ITER_MASK, BITMASK_STRIDE}; /// A bit mask which contains the result of a `Match` operation on a `Group` and /// allows iterating through them. @@ -23,13 +21,6 @@ pub(crate) struct BitMask(pub(crate) BitMaskWord); #[expect(clippy::use_self)] impl BitMask { - /// Returns a new `BitMask` with all bits inverted. - #[inline] - #[must_use] - pub(crate) fn invert(self) -> Self { - BitMask(self.0 ^ BITMASK_MASK) - } - /// Returns a new `BitMask` with the lowest bit removed. #[inline] #[must_use] diff --git a/src/control/group/generic.rs b/src/control/group/generic.rs index 0d3982559..3af8229ca 100644 --- a/src/control/group/generic.rs +++ b/src/control/group/generic.rs @@ -25,7 +25,7 @@ pub(crate) type NonZeroBitMaskWord = NonZeroGroupWord; pub(crate) const BITMASK_STRIDE: usize = 8; // We only care about the highest bit of each tag for the mask. #[expect(clippy::cast_possible_truncation, clippy::unnecessary_cast)] -pub(crate) const BITMASK_MASK: BitMaskWord = u64::from_ne_bytes([Tag::DELETED.0; 8]) as GroupWord; +const BITMASK_MASK: BitMaskWord = u64::from_ne_bytes([Tag::DELETED.0; 8]) as GroupWord; pub(crate) const BITMASK_ITER_MASK: BitMaskWord = !0; /// Helper function to replicate a tag across a `GroupWord`. @@ -134,7 +134,7 @@ impl Group { /// Returns a `BitMask` indicating all tags in the group which are full. #[inline] pub(crate) fn match_full(self) -> BitMask { - self.match_empty_or_deleted().invert() + BitMask(self.match_empty_or_deleted().0 ^ BITMASK_MASK) } /// Performs the following transformation on all tags in the group: diff --git a/src/control/group/lsx.rs b/src/control/group/lsx.rs index b117addf4..842c410af 100644 --- a/src/control/group/lsx.rs +++ b/src/control/group/lsx.rs @@ -7,7 +7,6 @@ use core::arch::loongarch64::*; pub(crate) type BitMaskWord = u16; pub(crate) type NonZeroBitMaskWord = NonZeroU16; pub(crate) const BITMASK_STRIDE: usize = 1; -pub(crate) const BITMASK_MASK: BitMaskWord = 0xffff; pub(crate) const BITMASK_ITER_MASK: BitMaskWord = !0; /// Abstraction over a group of control tags which can be scanned in diff --git a/src/control/group/mod.rs b/src/control/group/mod.rs index fe2d77483..063f912af 100644 --- a/src/control/group/mod.rs +++ b/src/control/group/mod.rs @@ -38,6 +38,4 @@ cfg_if! { } } pub(crate) use self::imp::Group; -pub(super) use self::imp::{ - BitMaskWord, NonZeroBitMaskWord, BITMASK_ITER_MASK, BITMASK_MASK, BITMASK_STRIDE, -}; +pub(super) use self::imp::{BitMaskWord, NonZeroBitMaskWord, BITMASK_ITER_MASK, BITMASK_STRIDE}; diff --git a/src/control/group/neon.rs b/src/control/group/neon.rs index 8273063a5..6c5327e2a 100644 --- a/src/control/group/neon.rs +++ b/src/control/group/neon.rs @@ -6,7 +6,6 @@ use core::num::NonZeroU64; pub(crate) type BitMaskWord = u64; pub(crate) type NonZeroBitMaskWord = NonZeroU64; pub(crate) const BITMASK_STRIDE: usize = 8; -pub(crate) const BITMASK_MASK: BitMaskWord = !0; pub(crate) const BITMASK_ITER_MASK: BitMaskWord = 0x8080_8080_8080_8080; /// Abstraction over a group of control tags which can be scanned in diff --git a/src/control/group/sse2.rs b/src/control/group/sse2.rs index 4677f1ac0..2b12c0104 100644 --- a/src/control/group/sse2.rs +++ b/src/control/group/sse2.rs @@ -10,7 +10,6 @@ use core::arch::x86_64 as x86; pub(crate) type BitMaskWord = u16; pub(crate) type NonZeroBitMaskWord = NonZeroU16; pub(crate) const BITMASK_STRIDE: usize = 1; -pub(crate) const BITMASK_MASK: BitMaskWord = 0xffff; pub(crate) const BITMASK_ITER_MASK: BitMaskWord = !0; /// Abstraction over a group of control tags which can be scanned in @@ -113,7 +112,7 @@ impl Group { /// Returns a `BitMask` indicating all tags in the group which are full. #[inline] pub(crate) fn match_full(&self) -> BitMask { - self.match_empty_or_deleted().invert() + BitMask(!self.match_empty_or_deleted().0) } /// Performs the following transformation on all tags in the group: