diff --git a/benches/bench.rs b/benches/bench.rs index 23864009..3eced39e 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -1,14 +1,12 @@ #![feature(test)] #![allow(deprecated)] - extern crate test; - -use smallvec::{smallvec, SmallVec}; -use test::Bencher; - +use { + smallvec::{smallvec, SmallVec}, + test::Bencher, +}; const VEC_SIZE: usize = 16; const SPILLED_SIZE: usize = 100; - trait Vector: for<'a> From<&'a [T]> + Extend { fn new() -> Self; fn push(&mut self, val: T); @@ -19,75 +17,58 @@ trait Vector: for<'a> From<&'a [T]> + Extend { fn from_elems(val: &[T]) -> Self; fn extend_from_slice(&mut self, other: &[T]); } - impl Vector for Vec { fn new() -> Self { Self::with_capacity(VEC_SIZE) } - fn push(&mut self, val: T) { self.push(val) } - fn pop(&mut self) -> Option { self.pop() } - fn remove(&mut self, p: usize) -> T { self.remove(p) } - fn insert(&mut self, n: usize, val: T) { self.insert(n, val) } - fn from_elem(val: T, n: usize) -> Self { vec![val; n] } - fn from_elems(val: &[T]) -> Self { val.to_owned() } - fn extend_from_slice(&mut self, other: &[T]) { Vec::extend_from_slice(self, other) } } - impl Vector for SmallVec { fn new() -> Self { Self::new() } - fn push(&mut self, val: T) { self.push(val) } - fn pop(&mut self) -> Option { self.pop() } - fn remove(&mut self, p: usize) -> T { self.remove(p) } - fn insert(&mut self, n: usize, val: T) { self.insert(n, val) } - fn from_elem(val: T, n: usize) -> Self { smallvec![val; n] } - fn from_elems(val: &[T]) -> Self { SmallVec::from(val) } - fn extend_from_slice(&mut self, other: &[T]) { SmallVec::extend_from_slice(self, other) } } - macro_rules! make_benches { ($typ:ty { $($b_name:ident => $g_name:ident($($args:expr),*),)* }) => { $( @@ -98,7 +79,6 @@ macro_rules! make_benches { )* } } - make_benches! { SmallVec { bench_push => gen_push(SPILLED_SIZE as _), @@ -124,7 +104,6 @@ make_benches! { bench_pushpop => gen_pushpop(), } } - make_benches! { Vec { bench_push_vec => gen_push(SPILLED_SIZE as _), @@ -150,13 +129,11 @@ make_benches! { bench_pushpop_vec => gen_pushpop(), } } - fn gen_push>(n: u64, b: &mut Bencher) { #[inline(never)] fn push_noinline>(vec: &mut V, x: u64) { vec.push(x); } - b.iter(|| { let mut vec = V::new(); for x in 0..n { @@ -165,13 +142,11 @@ fn gen_push>(n: u64, b: &mut Bencher) { vec }); } - fn gen_insert_push>(n: u64, b: &mut Bencher) { #[inline(never)] fn insert_push_noinline>(vec: &mut V, x: u64) { vec.insert(x as usize, x); } - b.iter(|| { let mut vec = V::new(); for x in 0..n { @@ -180,13 +155,11 @@ fn gen_insert_push>(n: u64, b: &mut Bencher) { vec }); } - fn gen_insert>(n: u64, b: &mut Bencher) { #[inline(never)] fn insert_noinline>(vec: &mut V, p: usize, x: u64) { vec.insert(p, x) } - b.iter(|| { let mut vec = V::new(); // Always insert at position 0 so that we are subject to shifts of @@ -198,22 +171,18 @@ fn gen_insert>(n: u64, b: &mut Bencher) { vec }); } - fn gen_remove>(n: usize, b: &mut Bencher) { #[inline(never)] fn remove_noinline>(vec: &mut V, p: usize) -> u64 { vec.remove(p) } - b.iter(|| { let mut vec = V::from_elem(0, n as _); - for _ in 0..n { remove_noinline(&mut vec, 0); } }); } - fn gen_extend>(n: u64, b: &mut Bencher) { b.iter(|| { let mut vec = V::new(); @@ -221,7 +190,6 @@ fn gen_extend>(n: u64, b: &mut Bencher) { vec }); } - fn gen_extend_filtered>(n: u64, b: &mut Bencher) { b.iter(|| { let mut vec = V::new(); @@ -229,7 +197,6 @@ fn gen_extend_filtered>(n: u64, b: &mut Bencher) { vec }); } - fn gen_from_iter>(n: u64, b: &mut Bencher) { let v: Vec = (0..n).collect(); b.iter(|| { @@ -237,7 +204,6 @@ fn gen_from_iter>(n: u64, b: &mut Bencher) { vec }); } - fn gen_from_slice>(n: u64, b: &mut Bencher) { let v: Vec = (0..n).collect(); b.iter(|| { @@ -245,7 +211,6 @@ fn gen_from_slice>(n: u64, b: &mut Bencher) { vec }); } - fn gen_extend_from_slice>(n: u64, b: &mut Bencher) { let v: Vec = (0..n).collect(); b.iter(|| { @@ -254,14 +219,12 @@ fn gen_extend_from_slice>(n: u64, b: &mut Bencher) { vec }); } - fn gen_pushpop>(b: &mut Bencher) { #[inline(never)] fn pushpop_noinline>(vec: &mut V, x: u64) -> Option { vec.push(x); vec.pop() } - b.iter(|| { let mut vec = V::new(); for x in 0..SPILLED_SIZE as _ { @@ -270,14 +233,12 @@ fn gen_pushpop>(b: &mut Bencher) { vec }); } - fn gen_from_elem>(n: usize, b: &mut Bencher) { b.iter(|| { let vec = V::from_elem(42, n); vec }); } - #[bench] fn bench_macro_from_list(b: &mut Bencher) { b.iter(|| { @@ -289,7 +250,6 @@ fn bench_macro_from_list(b: &mut Bencher) { vec }); } - #[bench] fn bench_macro_from_list_vec(b: &mut Bencher) { b.iter(|| { diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..6643b4d1 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,10 @@ +wrap_comments = true +imports_granularity = "One" +group_imports = "One" +format_code_in_doc_comments = true +match_arm_blocks = false +blank_lines_lower_bound = 0 +blank_lines_upper_bound = 0 +condense_wildcard_suffixes = true +error_on_unformatted = true +error_on_line_overflow = true \ No newline at end of file diff --git a/src/allocationerror.rs b/src/allocationerror.rs new file mode 100644 index 00000000..c28f38c2 --- /dev/null +++ b/src/allocationerror.rs @@ -0,0 +1,24 @@ +use { + alloc::alloc::Layout, + core::{ + error::Error, + fmt::{Display, Formatter, Result as Format}, + }, +}; +/// Error type for APIs with fallible heap allocation +#[derive(Debug)] +pub enum AllocationError { + /// Overflow `usize::MAX` or other error during size computation + CapacityOverflow, + /// The allocator return an error + Failure { + /// The layout that was passed to the allocator + layout: Layout, + }, +} +impl Display for AllocationError { + fn fmt(&self, f: &mut Formatter) -> Format { + write!(f, "Allocation error: {:?}", self) + } +} +impl Error for AllocationError {} diff --git a/src/bytes.rs b/src/bytes.rs new file mode 100644 index 00000000..11f7fbc0 --- /dev/null +++ b/src/bytes.rs @@ -0,0 +1,60 @@ +use { + super::SmallVec, + bytes::{buf::UninitSlice, BufMut}, +}; +unsafe impl BufMut for SmallVec { + #[inline] + fn remaining_mut(&self) -> usize { + // A vector can never have more than isize::MAX bytes + isize::MAX as usize - self.len() + } + #[inline] + unsafe fn advance_mut(&mut self, cnt: usize) { + let len = self.len(); + let remaining = self.capacity() - len; + if remaining < cnt { + panic!("advance out of bounds: the len is {remaining} but advancing by {cnt}"); + } + // Addition will not overflow since the sum is at most the capacity. + self.set_len(len + cnt); + } + #[inline] + fn chunk_mut(&mut self) -> &mut UninitSlice { + if self.capacity() == self.len() { + self.reserve(64); // Grow the smallvec + } + let cap = self.capacity(); + let len = self.len(); + let ptr = self.as_mut_ptr(); + // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be + // valid for `cap - len` bytes. The subtraction will not underflow since + // `len <= cap`. + unsafe { UninitSlice::from_raw_parts_mut(ptr.add(len), cap - len) } + } + // Specialize these methods so they can skip checking `remaining_mut` + // and `advance_mut`. + #[inline] + fn put(&mut self, mut src: T) + where + Self: Sized, + { + // In case the src isn't contiguous, reserve upfront. + self.reserve(src.remaining()); + while src.has_remaining() { + let s = src.chunk(); + let l = s.len(); + self.extend_from_slice(s); + src.advance(l); + } + } + #[inline] + fn put_slice(&mut self, src: &[u8]) { + self.extend_from_slice(src); + } + #[inline] + fn put_bytes(&mut self, val: u8, cnt: usize) { + // If the addition overflows, then the `resize` will fail. + let new_len = self.len().saturating_add(cnt); + self.resize(new_len, val); + } +} diff --git a/src/comparisons.rs b/src/comparisons.rs new file mode 100644 index 00000000..f4ca1d3f --- /dev/null +++ b/src/comparisons.rs @@ -0,0 +1,74 @@ +use super::SmallVec; +impl PartialEq> for SmallVec +where + T: PartialEq, +{ + #[inline] + fn eq(&self, other: &SmallVec) -> bool { + self.as_slice().eq(other.as_slice()) + } +} +impl Eq for SmallVec where T: Eq {} +impl PartialEq<[U; M]> for SmallVec +where + T: PartialEq, +{ + #[inline] + fn eq(&self, other: &[U; M]) -> bool { + self[..] == other[..] + } +} +impl PartialEq<&[U; M]> for SmallVec +where + T: PartialEq, +{ + #[inline] + fn eq(&self, other: &&[U; M]) -> bool { + self[..] == other[..] + } +} +impl PartialEq<[U]> for SmallVec +where + T: PartialEq, +{ + #[inline] + fn eq(&self, other: &[U]) -> bool { + self[..] == other[..] + } +} +impl PartialEq<&[U]> for SmallVec +where + T: PartialEq, +{ + #[inline] + fn eq(&self, other: &&[U]) -> bool { + self[..] == other[..] + } +} +impl PartialEq<&mut [U]> for SmallVec +where + T: PartialEq, +{ + #[inline] + fn eq(&self, other: &&mut [U]) -> bool { + self[..] == other[..] + } +} +impl PartialOrd for SmallVec +where + T: PartialOrd, +{ + #[inline] + fn partial_cmp(&self, other: &SmallVec) -> Option { + self.as_slice().partial_cmp(other.as_slice()) + } +} +impl Ord for SmallVec +where + T: Ord, +{ + #[inline] + fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { + self.as_slice().cmp(other.as_slice()) + } +} diff --git a/src/conversions.rs b/src/conversions.rs new file mode 100644 index 00000000..18da6b4c --- /dev/null +++ b/src/conversions.rs @@ -0,0 +1,71 @@ +#[cfg(feature = "specialization")] +use super::spec_traits; +use { + super::SmallVec, + alloc::vec::Vec, + core::{mem::ManuallyDrop, ptr::copy_nonoverlapping}, +}; +impl From<&mut [T; M]> for SmallVec { + #[inline] + fn from(slice: &mut [T; M]) -> Self { + Self::from(slice as &[T]) + } +} +impl From<[T; M]> for SmallVec { + fn from(array: [T; M]) -> Self { + if M > N { + // If M > N, we'd have to heap allocate anyway, + // so delegate for Vec for the allocation. + Self::from(Vec::from(array)) + } else { + // M <= N + let mut this = Self::new(); + debug_assert!(M <= this.capacity()); + let array = ManuallyDrop::new(array); + // SAFETY: M <= this.capacity() + unsafe { + copy_nonoverlapping(array.as_ptr(), this.as_mut_ptr(), M); + this.set_len(M); + } + this + } + } +} +impl From> for SmallVec { + fn from(array: Vec) -> Self { + Self::from_vec(array) + } +} +impl From<&[T]> for SmallVec { + #[inline] + fn from(slice: &[T]) -> Self { + if slice.len() > Self::inline_size() { + // Standard Rust vectors are already specialized. + Self::from_vec(Vec::from(slice)) + } else { + // SAFETY: The precondition is checked in the initial comparison above. + unsafe { + #[cfg(feature = "specialization")] + { + >::spec_from(slice) + } + #[cfg(not(feature = "specialization"))] + { + Self::from_slice_fallback(slice) + } + } + } + } +} +impl From<&mut [T]> for SmallVec { + #[inline] + fn from(slice: &mut [T]) -> Self { + Self::from(slice as &[T]) + } +} +impl From<&[T; M]> for SmallVec { + #[inline] + fn from(slice: &[T; M]) -> Self { + Self::from(slice as &[T]) + } +} diff --git a/src/lib.rs b/src/lib.rs index f7b1bba1..7b4c1ef6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,12 +1,7 @@ -// Licensed under the Apache License, Version 2.0 or the MIT license -// , at your -// option. This file may not be copied, modified, or distributed -// except according to those terms. - -//! Small vectors in various sizes. These store a certain number of elements inline, and fall back -//! to the heap for larger allocations. This can be a useful optimization for improving cache -//! locality and reducing allocator traffic for workloads that fit within the inline buffer. +//! Small vectors in various sizes. These store a certain number of elements +//! inline, and fall back to the heap for larger allocations. This can be a +//! useful optimization for improving cache locality and reducing allocator +//! traffic for workloads that fit within the inline buffer. //! //! ## `no_std` support //! @@ -21,120 +16,81 @@ //! When this feature is enabled, traits available from `std` are implemented: //! //! * `SmallVec` implements the [`std::io::Write`] trait. -//! * [`CollectionAllocErr`] implements [`std::error::Error`]. //! //! This feature is not compatible with `#![no_std]` programs. //! //! ### `serde` //! -//! When this optional dependency is enabled, `SmallVec` implements the `serde::Serialize` and -//! `serde::Deserialize` traits. +//! When this optional dependency is enabled, `SmallVec` implements the +//! `serde::Serialize` and `serde::Deserialize` traits. //! //! ### `specialization` //! -//! **This feature is unstable and requires a nightly build of the Rust toolchain.** +//! **This feature is unstable and requires a nightly build of the Rust +//! toolchain.** //! -//! When this feature is enabled, `SmallVec::from(slice)` has improved performance for slices -//! of `Copy` types. (Without this feature, you can use `SmallVec::from_slice` to get optimal -//! performance for `Copy` types.) +//! When this feature is enabled, `SmallVec::from(slice)` has improved +//! performance for slices of `Copy` types. (Without this feature, you can use +//! `SmallVec::from_slice` to get optimal performance for `Copy` types.) //! //! Tracking issue: [rust-lang/rust#31844](https://github.com/rust-lang/rust/issues/31844) //! //! ### `may_dangle` //! -//! **This feature is unstable and requires a nightly build of the Rust toolchain.** +//! **This feature is unstable and requires a nightly build of the Rust +//! toolchain.** //! -//! This feature makes the Rust compiler less strict about use of vectors that contain borrowed -//! references. For details, see the +//! This feature makes the Rust compiler less strict about use of vectors that +//! contain borrowed references. For details, see the //! [Rustonomicon](https://doc.rust-lang.org/1.42.0/nomicon/dropck.html#an-escape-hatch). //! //! Tracking issue: [rust-lang/rust#34761](https://github.com/rust-lang/rust/issues/34761) #![no_std] -#![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(feature = "specialization", allow(incomplete_features))] #![cfg_attr(feature = "specialization", feature(specialization, trusted_len))] #![cfg_attr(feature = "may_dangle", feature(dropck_eyepatch))] - #[doc(hidden)] pub extern crate alloc; - -#[cfg(any(test, feature = "std"))] -extern crate std; - -mod rawsmallvec; -#[cfg(test)] -mod tests; - -use alloc::boxed::Box; -use alloc::vec; -use alloc::vec::Vec; - -use alloc::alloc::Layout; -use core::borrow::Borrow; -use core::borrow::BorrowMut; -use core::fmt::Debug; -use core::hash::{Hash, Hasher}; -use core::marker::PhantomData; -use core::mem::align_of; -use core::mem::size_of; -use core::mem::ManuallyDrop; -use core::mem::MaybeUninit; -use core::ptr::copy; -use core::ptr::copy_nonoverlapping; -use core::ptr::NonNull; - +mod allocationerror; #[cfg(feature = "bytes")] -use bytes::{buf::UninitSlice, BufMut}; +mod bytes; +mod comparisons; +mod conversions; #[cfg(feature = "malloc_size_of")] -use malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}; +mod mallocsizeof; +mod rawsmallvec; +mod references; #[cfg(feature = "serde")] -use serde_core::{ - de::{Deserialize, Deserializer, SeqAccess, Visitor}, - ser::{Serialize, SerializeSeq, Serializer}, -}; +mod serde; #[cfg(feature = "std")] -use std::io; - +mod std; +mod taggedlen; +#[cfg(test)] +mod tests; +pub use allocationerror::AllocationError; +use { + alloc::{alloc::Layout, boxed::Box, vec::Vec}, + core::{ + fmt::Debug, + hash::{Hash, Hasher}, + marker::PhantomData, + mem::{align_of, size_of, ManuallyDrop, MaybeUninit}, + ptr::{copy, copy_nonoverlapping, NonNull}, + }, +}; #[cfg(feature = "internals")] -pub use rawsmallvec::RawSmallVec; +pub use {rawsmallvec::RawSmallVec, taggedlen::TaggedLen}; #[cfg(not(feature = "internals"))] -use rawsmallvec::RawSmallVec; - -/// Error type for APIs with fallible heap allocation -#[derive(Debug)] -pub enum CollectionAllocErr { - /// Overflow `usize::MAX` or other error during size computation - CapacityOverflow, - /// The allocator return an error - AllocErr { - /// The layout that was passed to the allocator - layout: Layout, - }, -} -impl core::fmt::Display for CollectionAllocErr { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "Allocation error: {:?}", self) - } -} - -impl core::error::Error for CollectionAllocErr {} - +use {rawsmallvec::RawSmallVec, taggedlen::TaggedLen}; #[inline] -fn infallible(result: Result) -> T { +fn infallible(result: Result) -> T { match result { Ok(x) => x, - Err(CollectionAllocErr::CapacityOverflow) => panic!("capacity overflow"), - Err(CollectionAllocErr::AllocErr { layout }) => alloc::alloc::handle_alloc_error(layout), + Err(AllocationError::CapacityOverflow) => panic!("capacity overflow"), + Err(AllocationError::Failure { layout }) => alloc::alloc::handle_alloc_error(layout), } } - -/// Helper function to check if a type is a ZST. -#[inline] -const fn is_zst() -> bool { - const { size_of::() == 0 } -} - #[inline] /// A local copy of [`core::slice::range`]. The latter function is unstable /// and thus cannot be used yet. @@ -143,7 +99,6 @@ where R: core::ops::RangeBounds, { let len = bounds.end; - let start = match range.start_bound() { core::ops::Bound::Included(&start) => start, core::ops::Bound::Excluded(start) => start @@ -151,7 +106,6 @@ where .unwrap_or_else(|| panic!("attempted to index slice from after maximum usize")), core::ops::Bound::Unbounded => 0, }; - let end = match range.end_bound() { core::ops::Bound::Included(end) => end .checked_add(1) @@ -159,199 +113,30 @@ where core::ops::Bound::Excluded(&end) => end, core::ops::Bound::Unbounded => len, }; - if start > end { panic!("slice index starts at {start} but ends at {end}"); } if end > len { panic!("range end index {end} out of range for slice of length {len}"); } - core::ops::Range { start, end } } - -impl RawSmallVec { - const IS_ZST: bool = is_zst::(); - - #[inline] - const fn new() -> Self { - Self::new_inline(MaybeUninit::uninit()) - } - #[inline] - const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { - Self { - inline: ManuallyDrop::new(inline), - } - } - #[inline] - const fn new_heap(ptr: NonNull, capacity: usize) -> Self { - Self { - heap: (ptr, capacity), - } - } - - #[inline] - const fn as_ptr_inline(&self) -> *const T { - // SAFETY: it is safe because we aren't reading the value, just getting a - // reference to it. reading it would be UB potentially, but for that downstream - // unsafe is required - (unsafe { &raw const self.inline }) as *mut T - } - - #[inline] - const fn as_mut_ptr_inline(&mut self) -> *mut T { - // SAFETY: same as above - (unsafe { &raw mut self.inline }) as *mut T - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_ptr_heap(&self) -> *const T { - self.heap.0.as_ptr() - } - - /// # Safety - /// - /// The vector must be on the heap - #[inline] - const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { - self.heap.0.as_ptr() - } - - /// # Safety - /// - /// `new_capacity` must be non zero, and greater or equal to the length. - /// T must not be a ZST. - unsafe fn try_grow_raw( - &mut self, - len: TaggedLen, - new_capacity: usize, - ) -> Result<(), CollectionAllocErr> { - use alloc::alloc::{alloc, realloc}; - debug_assert!(!Self::IS_ZST); - debug_assert!(new_capacity > 0); - debug_assert!(new_capacity >= len.value()); - - let was_on_heap = len.on_heap(); - let ptr = if was_on_heap { - self.as_mut_ptr_heap() - } else { - self.as_mut_ptr_inline() - }; - let len = len.value(); - - let new_layout = - Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; - if new_layout.size() > isize::MAX as usize { - return Err(CollectionAllocErr::CapacityOverflow); - } - - let new_ptr = if len == 0 || !was_on_heap { - // get a fresh allocation - let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. - let new_ptr = - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })?; - copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); - new_ptr - } else { - // use realloc - - // this can't overflow since we already constructed an equivalent layout during - // the previous allocation - let old_layout = - Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); - - // SAFETY: ptr was allocated with this allocator - // old_layout is the same as the layout used to allocate the previous memory block - // new_layout.size() is greater than zero - // does not overflow when rounded up to alignment. since it was constructed - // with Layout::array - let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; - NonNull::new(new_ptr).ok_or(CollectionAllocErr::AllocErr { layout: new_layout })? - }; - *self = Self::new_heap(new_ptr, new_capacity); - Ok(()) - } -} - -/// Vec guarantees that its length is always less than [`isize::MAX`] in *bytes*. -/// -/// For a non ZST, this means that the length is less than `isize::MAX` objects, which implies we -/// have at least one free bit we can use. We use the least significant bit for the tag. And store -/// the length in the `usize::BITS - 1` most significant bits. -/// -/// For a ZST, we never use the heap, so we just store the length directly. -#[repr(transparent)] -struct TaggedLen(usize, PhantomData); - -// Clone and Copy must be manually implemented because the generic interferes with the derive attribute implementations. -impl Clone for TaggedLen { - #[inline] - fn clone(&self) -> Self { - Self(self.0, PhantomData) - } - - #[inline] - fn clone_from(&mut self, source: &Self) { - self.0 = source.0; - } -} - -impl Copy for TaggedLen {} - -impl TaggedLen { - const IS_ZST: bool = is_zst::(); - #[inline] - pub const fn new(len: usize, on_heap: bool) -> Self { - if Self::IS_ZST { - debug_assert!(!on_heap); - Self(len, PhantomData) - } else { - debug_assert!(len < isize::MAX as usize); - Self((len << 1) | on_heap as usize, PhantomData) - } - } - - #[inline] - #[must_use] - pub const fn on_heap(self) -> bool { - if Self::IS_ZST { - false - } else { - (self.0 & 1_usize) == 1 - } - } - - #[inline] - pub const fn value(self) -> usize { - if Self::IS_ZST { - self.0 - } else { - self.0 >> 1 - } - } -} - #[repr(C)] pub struct SmallVec { len: TaggedLen, raw: RawSmallVec, _marker: PhantomData, } - unsafe impl Send for SmallVec {} unsafe impl Sync for SmallVec {} - impl Default for SmallVec { #[inline] fn default() -> Self { Self::new() } } - -/// An iterator that removes the items from a `SmallVec` and yields them by value. +/// An iterator that removes the items from a `SmallVec` and yields them by +/// value. /// /// Returned from [`SmallVec::drain`][1]. /// @@ -369,25 +154,21 @@ pub struct Drain<'a, T: 'a, const N: usize> { iter: core::slice::Iter<'a, T>, vec: core::ptr::NonNull>, } - impl<'a, T: 'a, const N: usize> Iterator for Drain<'a, T, N> { type Item = T; - #[inline] fn next(&mut self) -> Option { - // SAFETY: we shrunk the length of the vector so it no longer owns these items, and we can - // take ownership of them. + // SAFETY: we shrunk the length of the vector so it no longer owns these items, + // and we can take ownership of them. self.iter .next() .map(|reference| unsafe { core::ptr::read(reference) }) } - #[inline] fn size_hint(&self) -> (usize, Option) { self.iter.size_hint() } } - impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { #[inline] fn next_back(&mut self) -> Option { @@ -397,21 +178,17 @@ impl<'a, T: 'a, const N: usize> DoubleEndedIterator for Drain<'a, T, N> { .map(|reference| unsafe { core::ptr::read(reference) }) } } - impl ExactSizeIterator for Drain<'_, T, N> { #[inline] fn len(&self) -> usize { self.iter.len() } } - impl core::iter::FusedIterator for Drain<'_, T, N> {} - impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { fn drop(&mut self) { /// Moves back the un-`Drain`ed elements to restore the original `Vec`. struct DropGuard<'r, 'a, T, const N: usize>(&'r mut Drain<'a, T, N>); - impl<'r, 'a, T, const N: usize> Drop for DropGuard<'r, 'a, T, N> { fn drop(&mut self) { if self.0.tail_len > 0 { @@ -431,43 +208,38 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { } } } - let iter = core::mem::take(&mut self.iter); let drop_len = iter.len(); - let mut vec = self.vec; - if SmallVec::::IS_ZST { - // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. - // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + // ZSTs have no identity, so we don't need to move them around, we only need to + // drop the correct amount. this can be achieved by manipulating the + // Vec length instead of moving values out from `iter`. unsafe { let vec = vec.as_mut(); let old_len = vec.len(); vec.set_len(old_len + drop_len + self.tail_len); vec.truncate(old_len + self.tail_len); } - return; } - - // ensure elements are moved back into their appropriate places, even when drop_in_place panics + // ensure elements are moved back into their appropriate places, even when + // drop_in_place panics let _guard = DropGuard(self); - if drop_len == 0 { return; } - // as_slice() must only be called when iter.len() is > 0 because // it also gets touched by vec::Splice which may turn it into a dangling pointer - // which would make it and the vec pointer point to different allocations which would - // lead to invalid pointer arithmetic below. + // which would make it and the vec pointer point to different allocations which + // would lead to invalid pointer arithmetic below. let drop_ptr = iter.as_slice().as_ptr(); - unsafe { - // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place - // a pointer with mutable provenance is necessary. Therefore we must reconstruct - // it from the original vec but also avoid creating a &mut to the front since that could - // invalidate raw pointers to it which some unsafe code might rely on. + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for + // drop_in_place a pointer with mutable provenance is necessary. + // Therefore we must reconstruct it from the original vec but also + // avoid creating a &mut to the front since that could invalidate + // raw pointers to it which some unsafe code might rely on. let vec_ptr = vec.as_mut().as_mut_ptr(); // May be replaced with the line below later, once this crate's MSRV is >= 1.87. //let drop_offset = drop_ptr.offset_from_unsigned(vec_ptr); @@ -477,17 +249,16 @@ impl<'a, T: 'a, const N: usize> Drop for Drain<'a, T, N> { } } } - impl Drain<'_, T, N> { #[must_use] pub fn as_slice(&self) -> &[T] { self.iter.as_slice() } - /// The range from `self.vec.len` to `self.tail_start` contains elements /// that have been moved out. - /// Fill that range as much as possible with new elements from the `replace_with` iterator. - /// Returns `true` if we filled the entire range. (`replace_with.next()` didn’t return `None`.) + /// Fill that range as much as possible with new elements from the + /// `replace_with` iterator. Returns `true` if we filled the entire + /// range. (`replace_with.next()` didn’t return `None`.) unsafe fn fill>(&mut self, replace_with: &mut I) -> bool { let vec = unsafe { self.vec.as_mut() }; let range_start = vec.len(); @@ -498,7 +269,6 @@ impl Drain<'_, T, N> { range_end - range_start, ) }; - for place in range_slice { if let Some(new_item) = replace_with.next() { unsafe { core::ptr::write(place, new_item) }; @@ -509,19 +279,16 @@ impl Drain<'_, T, N> { } true } - /// Makes room for inserting more elements before the tail. #[track_caller] unsafe fn move_tail(&mut self, additional: usize) { let vec = unsafe { self.vec.as_mut() }; let len = self.tail_start + self.tail_len; - // Test let old_len = vec.len(); vec.set_len(len); vec.reserve(additional); vec.set_len(old_len); - let new_tail_start = self.tail_start + additional; unsafe { let src = vec.as_ptr().add(self.tail_start); @@ -531,8 +298,8 @@ impl Drain<'_, T, N> { self.tail_start = new_tail_start; } } - -/// An iterator which uses a closure to determine if an element should be removed. +/// An iterator which uses a closure to determine if an element should be +/// removed. /// /// Returned from [`SmallVec::extract_if`][1]. /// @@ -544,7 +311,8 @@ where vec: &'a mut SmallVec, /// The index of the item that will be inspected by the next call to `next`. idx: usize, - /// Elements at and beyond this point will be retained. Must be equal or smaller than `old_len`. + /// Elements at and beyond this point will be retained. Must be equal or + /// smaller than `old_len`. end: usize, /// The number of items that have been drained (removed) thus far. del: usize, @@ -553,7 +321,6 @@ where /// The filter test predicate. pred: F, } - impl core::fmt::Debug for ExtractIf<'_, T, N, F> where F: FnMut(&mut T) -> bool, @@ -565,13 +332,11 @@ where .finish() } } - impl Iterator for ExtractIf<'_, T, N, F> where F: FnMut(&mut T) -> bool, { type Item = T; - fn next(&mut self) -> Option { unsafe { while self.idx < self.end { @@ -595,12 +360,10 @@ where None } } - fn size_hint(&self) -> (usize, Option) { (0, Some(self.end - self.idx)) } } - impl Drop for ExtractIf<'_, T, N, F> where F: FnMut(&mut T) -> bool, @@ -624,12 +387,10 @@ where } } } - pub struct Splice<'a, I: Iterator + 'a, const N: usize> { drain: Drain<'a, I::Item, N>, replace_with: I, } - impl<'a, I, const N: usize> core::fmt::Debug for Splice<'a, I, N> where I: Debug + Iterator + 'a, @@ -639,48 +400,39 @@ where f.debug_tuple("Splice").field(&self.drain).finish() } } - impl Iterator for Splice<'_, I, N> { type Item = I::Item; - fn next(&mut self) -> Option { self.drain.next() } - fn size_hint(&self) -> (usize, Option) { self.drain.size_hint() } } - impl DoubleEndedIterator for Splice<'_, I, N> { fn next_back(&mut self) -> Option { self.drain.next_back() } } - impl ExactSizeIterator for Splice<'_, I, N> {} - impl Drop for Splice<'_, I, N> { fn drop(&mut self) { self.drain.by_ref().for_each(drop); // At this point draining is done and the only remaining tasks are splicing // and moving things into the final place. - // Which means we can replace the slice::Iter with pointers that won't point to deallocated - // memory, so that Drain::drop is still allowed to call iter.len(), otherwise it would break - // the ptr.sub_ptr contract. + // Which means we can replace the slice::Iter with pointers that won't point to + // deallocated memory, so that Drain::drop is still allowed to call + // iter.len(), otherwise it would break the ptr.sub_ptr contract. self.drain.iter = [].iter(); - unsafe { if self.drain.tail_len == 0 { self.drain.vec.as_mut().extend(self.replace_with.by_ref()); return; } - // First fill the range left by drain(). if !self.drain.fill(&mut self.replace_with) { return; } - // There may be more elements. Use the lower bound as an estimate. // FIXME: Is the upper bound a better guess? Or something else? let (lower_bound, _upper_bound) = self.replace_with.size_hint(); @@ -690,7 +442,6 @@ impl Drop for Splice<'_, I, N> { return; } } - // Collect any remaining elements. let mut collected = self .replace_with @@ -705,10 +456,10 @@ impl Drop for Splice<'_, I, N> { debug_assert_eq!(collected.len(), 0); } } - // Let `Drain::drop` move the tail back if necessary and restore `vec.len`. + // Let `Drain::drop` move the tail back if necessary and restore + // `vec.len`. } } - /// An iterator that consumes a `SmallVec` and yields its items by value. /// /// Returned from [`SmallVec::into_iter`][1]. @@ -725,12 +476,10 @@ pub struct IntoIter { end: TaggedLen, _marker: PhantomData, } - -// SAFETY: IntoIter has unique ownership of its contents. Sending (or sharing) an `IntoIter` -// is equivalent to sending (or sharing) a `SmallVec`. +// SAFETY: IntoIter has unique ownership of its contents. Sending (or sharing) +// an `IntoIter` is equivalent to sending (or sharing) a `SmallVec`. unsafe impl Send for IntoIter where T: Send {} unsafe impl Sync for IntoIter where T: Sync {} - impl IntoIter { #[inline] const fn as_ptr(&self) -> *const T { @@ -742,7 +491,6 @@ impl IntoIter { self.raw.as_ptr_inline() } } - #[inline] const fn as_mut_ptr(&mut self) -> *mut T { let on_heap = self.end.on_heap(); @@ -753,7 +501,6 @@ impl IntoIter { self.raw.as_mut_ptr_inline() } } - #[inline] pub const fn as_slice(&self) -> &[T] { // SAFETY: The members in self.begin..self.end.value() are all initialized @@ -763,7 +510,6 @@ impl IntoIter { core::slice::from_raw_parts(ptr.add(self.begin), self.end.value() - self.begin) } } - #[inline] pub const fn as_mut_slice(&mut self) -> &mut [T] { // SAFETY: see above @@ -773,10 +519,8 @@ impl IntoIter { } } } - impl Iterator for IntoIter { type Item = T; - #[inline] fn next(&mut self) -> Option { if self.begin == self.end.value() { @@ -791,14 +535,12 @@ impl Iterator for IntoIter { } } } - #[inline] fn size_hint(&self) -> (usize, Option) { let size = self.end.value() - self.begin; (size, Some(size)) } } - impl DoubleEndedIterator for IntoIter { #[inline] fn next_back(&mut self) -> Option { @@ -820,7 +562,6 @@ impl DoubleEndedIterator for IntoIter { } impl ExactSizeIterator for IntoIter {} impl core::iter::FusedIterator for IntoIter {} - impl SmallVec { #[inline] pub const fn new() -> SmallVec { @@ -830,7 +571,6 @@ impl SmallVec { _marker: PhantomData, } } - #[inline] pub fn with_capacity(capacity: usize) -> Self { let mut this = Self::new(); @@ -839,27 +579,23 @@ impl SmallVec { } this } - #[inline] pub const fn from_buf(elements: [T; S]) -> Self { const { assert!(S <= N); } - // Although we create a new buffer, since S and N are known at compile time, - // even with `-C opt-level=1`, it gets optimized as best as it could be. (Checked with ) + // even with `-C opt-level=1`, it gets optimized as best as it could be. + // (Checked with ) let mut buf: MaybeUninit<[T; N]> = MaybeUninit::uninit(); - // SAFETY: buf and elements do not overlap, are aligned and have space // for at least S elements since S <= N. // We will drop the elements only once since we do forget(elements). unsafe { copy_nonoverlapping(elements.as_ptr(), buf.as_mut_ptr() as *mut T, S); } - // `elements` have been moved into buf and will be dropped by SmallVec core::mem::forget(elements); - // SAFETY: all the members in 0..S are initialized Self { len: TaggedLen::new(S, false), @@ -867,7 +603,6 @@ impl SmallVec { _marker: PhantomData, } } - #[inline] pub fn from_buf_and_len(buf: [T; N], len: usize) -> Self { assert!(len <= N); @@ -879,33 +614,28 @@ impl SmallVec { }; // Deallocate the remaining elements so no memory is leaked. unsafe { - // SAFETY: both the input and output pointers are in range of the stack allocation + // SAFETY: both the input and output pointers are in range of the stack + // allocation let remainder_ptr = vec.raw.as_mut_ptr_inline().add(len); let remainder_len = N - len; - // SAFETY: the values are initialized, so dropping them here is fine. core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut( remainder_ptr, remainder_len, )); } - vec } - - /// Constructs a new `SmallVec` on the stack from an A without copying elements. Also sets the length. The user is responsible for ensuring that `len <= A::size()`. + /// Constructs a new `SmallVec` on the stack from an A without copying + /// elements. Also sets the length. The user is responsible for ensuring + /// that `len <= A::size()`. /// /// # Examples /// /// ``` - /// use smallvec::SmallVec; - /// use std::mem::MaybeUninit; - /// + /// use {smallvec::SmallVec, std::mem::MaybeUninit}; /// let buf = [1, 2, 3, 4, 5, 0, 0, 0]; - /// let small_vec = unsafe { - /// SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) - /// }; - /// + /// let small_vec = unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) }; /// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); /// ``` /// @@ -922,24 +652,20 @@ impl SmallVec { } } } - impl SmallVec { - const IS_ZST: bool = is_zst::(); - + const IS_ZST: bool = size_of::() == 0; #[inline] pub fn from_vec(vec: Vec) -> Self { if vec.capacity() == 0 { return Self::new(); } - if Self::IS_ZST { // "Move" elements to stack buffer. They're ZST so we don't actually have to do // anything. Just make sure they're not dropped. - // We don't wrap the vector in ManuallyDrop so that when it's dropped, the memory is - // deallocated, if it needs to be. + // We don't wrap the vector in ManuallyDrop so that when it's dropped, the + // memory is deallocated, if it needs to be. let mut vec = vec; let len = vec.len(); - // SAFETY: `0` is less than the vector's capacity. // old_len..new_len is an empty range. So there are no uninitialized elements unsafe { vec.set_len(0) }; @@ -955,7 +681,6 @@ impl SmallVec { // SAFETY: vec.capacity is not `0` (checked above), so the pointer // can not dangle and thus specifically cannot be null. let ptr = unsafe { NonNull::new_unchecked(vec.as_mut_ptr()) }; - Self { len: TaggedLen::new(len, true), raw: RawSmallVec::new_heap(ptr, cap), @@ -963,7 +688,6 @@ impl SmallVec { } } } - /// Sets the tag to be on the heap /// /// # Safety @@ -973,7 +697,6 @@ impl SmallVec { unsafe fn set_on_heap(&mut self) { self.len = TaggedLen::new(self.len(), true); } - /// Sets the tag to be inline /// /// # Safety @@ -983,23 +706,22 @@ impl SmallVec { unsafe fn set_inline(&mut self) { self.len = TaggedLen::new(self.len(), false); } - /// Sets the length of a vector. /// - /// This will explicitly set the size of the vector, without actually modifying its buffers, so - /// it is up to the caller to ensure that the vector is actually the specified size. + /// This will explicitly set the size of the vector, without actually + /// modifying its buffers, so it is up to the caller to ensure that the + /// vector is actually the specified size. /// /// # Safety /// - /// `new_len <= self.capacity()` must be true, and all the elements in the range `..self.len` - /// must be initialized. + /// `new_len <= self.capacity()` must be true, and all the elements in the + /// range `..self.len` must be initialized. #[inline] pub unsafe fn set_len(&mut self, new_len: usize) { debug_assert!(new_len <= self.capacity()); let on_heap = self.len.on_heap(); self.len = TaggedLen::new(new_len, on_heap); } - #[inline] pub const fn inline_size() -> usize { if Self::IS_ZST { @@ -1008,18 +730,15 @@ impl SmallVec { N } } - #[inline] pub const fn len(&self) -> usize { self.len.value() } - #[must_use] #[inline] pub const fn is_empty(&self) -> bool { self.len() == 0 } - #[inline] pub const fn capacity(&self) -> usize { if self.len.on_heap() { @@ -1029,12 +748,10 @@ impl SmallVec { Self::inline_size() } } - #[inline] pub const fn spilled(&self) -> bool { self.len.on_heap() } - /// Splits the collection into two at the given index. /// /// Returns a newly allocated vector containing the elements in the range @@ -1043,9 +760,11 @@ impl SmallVec { /// /// - If you want to take ownership of the entire contents and capacity of /// the vector, see [`core::mem::take`] or [`core::mem::replace`]. - /// - If you don't need the returned vector at all, see [`SmallVec::truncate`]. + /// - If you don't need the returned vector at all, see + /// [`SmallVec::truncate`]. /// - If you want to take ownership of an arbitrary subslice, or you don't - /// necessarily want to store the removed items in a vector, see [`SmallVec::drain`]. + /// necessarily want to store the removed items in a vector, see + /// [`SmallVec::drain`]. /// /// # Panics /// @@ -1063,58 +782,54 @@ impl SmallVec { pub fn split_off(&mut self, at: usize) -> Self { let len = self.len(); assert!(at <= len); - let other_len = len - at; let mut other = Self::with_capacity(other_len); - // Unsafely `set_len` and copy items to `other`. unsafe { self.set_len(at); other.set_len(other_len); - core::ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other_len); } other } - pub fn drain(&mut self, range: R) -> Drain<'_, T, N> where R: core::ops::RangeBounds, { let len = self.len(); let core::ops::Range { start, end } = slice_range(range, ..len); - unsafe { // SAFETY: `start <= len` self.set_len(start); - // SAFETY: all the elements in `start..end` are initialized let range_slice = core::slice::from_raw_parts(self.as_ptr().add(start), end - start); - // SAFETY: all the elements in `end..len` are initialized Drain { tail_start: end, tail_len: len - end, iter: range_slice.iter(), - // Since self is a &mut, passing it to a function would invalidate the slice iterator. + // Since self is a &mut, passing it to a function would invalidate the slice + // iterator. vec: core::ptr::NonNull::new_unchecked(self as *mut _), //vec: core::ptr::NonNull::from(self), } } } - - /// Creates an iterator which uses a closure to determine if element in the range should be removed. + /// Creates an iterator which uses a closure to determine if element in the + /// range should be removed. /// /// If the closure returns true, then the element is removed and yielded. - /// If the closure returns false, the element will remain in the vector and will not be yielded - /// by the iterator. + /// If the closure returns false, the element will remain in the vector and + /// will not be yielded by the iterator. /// - /// Only elements that fall in the provided range are considered for extraction, but any elements - /// after the range will still have to be moved if any element has been extracted. + /// Only elements that fall in the provided range are considered for + /// extraction, but any elements after the range will still have to be + /// moved if any element has been extracted. /// - /// If the returned `ExtractIf` is not exhausted, e.g. because it is dropped without iterating - /// or the iteration short-circuits, then the remaining elements will be retained. - /// Use [`retain`] with a negated predicate if you do not need the returned iterator. + /// If the returned `ExtractIf` is not exhausted, e.g. because it is dropped + /// without iterating or the iteration short-circuits, then the + /// remaining elements will be retained. Use [`retain`] with a negated + /// predicate if you do not need the returned iterator. /// /// [`retain`]: SmallVec::retain /// @@ -1141,8 +856,9 @@ impl SmallVec { /// But `extract_if` is easier to use. `extract_if` is also more efficient, /// because it can backshift the elements of the array in bulk. /// - /// Note that `extract_if` also lets you mutate the elements passed to the filter closure, - /// regardless of whether you choose to keep or remove them. + /// Note that `extract_if` also lets you mutate the elements passed to the + /// filter closure, regardless of whether you choose to keep or remove + /// them. /// /// # Panics /// @@ -1154,13 +870,17 @@ impl SmallVec { /// /// ``` /// # use smallvec::SmallVec; - /// let mut numbers: SmallVec = SmallVec::from(&[1i32, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]); - /// - /// let evens = numbers.extract_if(.., |x| *x % 2 == 0).collect::>(); + /// let mut numbers: SmallVec = + /// SmallVec::from(&[1i32, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15]); + /// let evens = numbers + /// .extract_if(.., |x| *x % 2 == 0) + /// .collect::>(); /// let odds = numbers; - /// /// assert_eq!(evens, SmallVec::::from(&[2i32, 4, 6, 8, 14])); - /// assert_eq!(odds, SmallVec::::from(&[1i32, 3, 5, 9, 11, 13, 15])); + /// assert_eq!( + /// odds, + /// SmallVec::::from(&[1i32, 3, 5, 9, 11, 13, 15]) + /// ); /// ``` /// /// Using the range argument to only process a part of the vector: @@ -1168,8 +888,13 @@ impl SmallVec { /// ``` /// # use smallvec::SmallVec; /// let mut items: SmallVec = SmallVec::from(&[0, 0, 0, 0, 0, 0, 0, 1, 2, 1, 2, 1, 2]); - /// let ones = items.extract_if(7.., |x| *x == 1).collect::>(); - /// assert_eq!(items, SmallVec::::from(&[0, 0, 0, 0, 0, 0, 0, 2, 2, 2])); + /// let ones = items + /// .extract_if(7.., |x| *x == 1) + /// .collect::>(); + /// assert_eq!( + /// items, + /// SmallVec::::from(&[0, 0, 0, 0, 0, 0, 0, 2, 2, 2]) + /// ); /// assert_eq!(ones.len(), 3); /// ``` pub fn extract_if(&mut self, range: R, filter: F) -> ExtractIf<'_, T, N, F> @@ -1179,12 +904,10 @@ impl SmallVec { { let old_len = self.len(); let core::ops::Range { start, end } = slice_range(range, ..old_len); - // Guard against us getting leaked (leak amplification) unsafe { self.set_len(0); } - ExtractIf { vec: self, idx: start, @@ -1194,7 +917,6 @@ impl SmallVec { pred: filter, } } - pub fn splice(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, N> where R: core::ops::RangeBounds, @@ -1205,7 +927,6 @@ impl SmallVec { replace_with: replace_with.into_iter(), } } - #[inline] pub fn push(&mut self, value: T) { let len = self.len(); @@ -1214,27 +935,26 @@ impl SmallVec { } // SAFETY: both the input and output are within the allocation let ptr = unsafe { self.as_mut_ptr().add(len) }; - // SAFETY: we allocated enough space in case it wasn't enough, so the address is valid for - // writes. + // SAFETY: we allocated enough space in case it wasn't enough, so the address is + // valid for writes. unsafe { ptr.write(value) }; unsafe { self.set_len(len + 1) } } - #[inline] pub fn pop(&mut self) -> Option { if self.is_empty() { None } else { let len = self.len() - 1; - // SAFETY: len < old_len since this can't overflow, because the old length is non zero + // SAFETY: len < old_len since this can't overflow, because the old length is + // non zero unsafe { self.set_len(len) }; - // SAFETY: this element was initialized and we just gave up ownership of it, so we can - // give it away + // SAFETY: this element was initialized and we just gave up ownership of it, so + // we can give it away let value = unsafe { self.as_mut_ptr().add(len).read() }; Some(value) } } - #[inline] pub fn pop_if(&mut self, predicate: impl FnOnce(&mut T) -> bool) -> Option { let last = self.last_mut()?; @@ -1244,44 +964,38 @@ impl SmallVec { None } } - #[inline] pub fn append(&mut self, other: &mut SmallVec) { - // can't overflow since both are smaller than isize::MAX and 2 * isize::MAX < usize::MAX + // can't overflow since both are smaller than isize::MAX and 2 * isize::MAX < + // usize::MAX let len = self.len(); let other_len = other.len(); let total_len = len + other_len; if total_len > self.capacity() { self.reserve(other_len); } - // SAFETY: see `Self::push` let ptr = unsafe { self.as_mut_ptr().add(len) }; unsafe { other.set_len(0) } - // SAFETY: we have a mutable reference to each vector and each uniquely owns its memory. - // so the ranges can't overlap + // SAFETY: we have a mutable reference to each vector and each uniquely owns its + // memory. so the ranges can't overlap unsafe { copy_nonoverlapping(other.as_ptr(), ptr, other_len) }; unsafe { self.set_len(total_len) } } - #[inline] pub fn grow(&mut self, new_capacity: usize) { infallible(self.try_grow(new_capacity)); } - #[cold] - pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), CollectionAllocErr> { + pub fn try_grow(&mut self, new_capacity: usize) -> Result<(), AllocationError> { if Self::IS_ZST { return Ok(()); } - let len = self.len(); assert!(new_capacity >= len); - if new_capacity > Self::inline_size() { // SAFETY: we checked all the preconditions let result = unsafe { self.raw.try_grow_raw(self.len, new_capacity) }; - if result.is_ok() { // SAFETY: the allocation succeeded, so self.raw.heap is now active unsafe { self.set_on_heap() }; @@ -1294,7 +1008,6 @@ impl SmallVec { // SAFETY: heap member is active let (ptr, old_cap) = self.raw.heap; // inline member is now active - // SAFETY: len <= new_capacity <= Self::inline_size() // so the copy is within bounds of the inline member copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len); @@ -1309,7 +1022,6 @@ impl SmallVec { Ok(()) } } - #[inline] pub fn reserve(&mut self, additional: usize) { // can't overflow since len <= capacity @@ -1318,26 +1030,24 @@ impl SmallVec { self.len() .checked_add(additional) .and_then(usize::checked_next_power_of_two) - .ok_or(CollectionAllocErr::CapacityOverflow), + .ok_or(AllocationError::CapacityOverflow), ); self.grow(new_capacity); } } - #[inline] - pub fn try_reserve(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { + pub fn try_reserve(&mut self, additional: usize) -> Result<(), AllocationError> { if additional > self.capacity() - self.len() { let new_capacity = self .len() .checked_add(additional) .and_then(usize::checked_next_power_of_two) - .ok_or(CollectionAllocErr::CapacityOverflow)?; + .ok_or(AllocationError::CapacityOverflow)?; self.try_grow(new_capacity) } else { Ok(()) } } - #[inline] pub fn reserve_exact(&mut self, additional: usize) { // can't overflow since len <= capacity @@ -1345,25 +1055,23 @@ impl SmallVec { let new_capacity = infallible( self.len() .checked_add(additional) - .ok_or(CollectionAllocErr::CapacityOverflow), + .ok_or(AllocationError::CapacityOverflow), ); self.grow(new_capacity); } } - #[inline] - pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), CollectionAllocErr> { + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), AllocationError> { if additional > self.capacity() - self.len() { let new_capacity = self .len() .checked_add(additional) - .ok_or(CollectionAllocErr::CapacityOverflow)?; + .ok_or(AllocationError::CapacityOverflow)?; self.try_grow(new_capacity) } else { Ok(()) } } - #[inline] pub fn shrink_to_fit(&mut self) { if !self.spilled() { @@ -1389,7 +1097,6 @@ impl SmallVec { unsafe { infallible(self.raw.try_grow_raw(self.len, len)) }; } } - #[inline] pub fn shrink_to(&mut self, min_capacity: usize) { if !self.spilled() { @@ -1421,7 +1128,6 @@ impl SmallVec { } } } - #[inline] pub fn truncate(&mut self, len: usize) { let old_len = self.len(); @@ -1437,7 +1143,6 @@ impl SmallVec { } } } - #[inline] pub fn swap_remove(&mut self, index: usize) -> T { let len = self.len(); @@ -1458,7 +1163,6 @@ impl SmallVec { value } } - #[inline] pub fn clear(&mut self) { // SAFETY: we set `len` to a smaller value @@ -1472,7 +1176,6 @@ impl SmallVec { )); } } - #[inline] pub fn remove(&mut self, index: usize) -> T { let len = self.len(); @@ -1492,7 +1195,6 @@ impl SmallVec { ith_item } } - #[inline] pub fn insert(&mut self, index: usize, value: T) { let len = self.len(); @@ -1509,12 +1211,10 @@ impl SmallVec { } // the element at `index` is now initialized ptr.add(index).write(value); - // SAFETY: all the elements are initialized self.set_len(len + 1); } } - #[inline] pub const fn as_slice(&self) -> &[T] { let len = self.len(); @@ -1522,7 +1222,6 @@ impl SmallVec { // SAFETY: all the elements in `..len` are initialized unsafe { core::slice::from_raw_parts(ptr, len) } } - #[inline] pub const fn as_mut_slice(&mut self) -> &mut [T] { let len = self.len(); @@ -1530,7 +1229,6 @@ impl SmallVec { // SAFETY: see above unsafe { core::slice::from_raw_parts_mut(ptr, len) } } - #[inline] pub const fn as_ptr(&self) -> *const T { if self.len.on_heap() { @@ -1540,7 +1238,6 @@ impl SmallVec { self.raw.as_ptr_inline() } } - #[inline] pub const fn as_mut_ptr(&mut self) -> *mut T { if self.len.on_heap() { @@ -1550,15 +1247,14 @@ impl SmallVec { self.raw.as_mut_ptr_inline() } } - #[inline] pub fn into_vec(self) -> Vec { let len = self.len(); if !self.spilled() { let mut vec = Vec::with_capacity(len); let this = ManuallyDrop::new(self); - // SAFETY: we create a new vector with sufficient capacity, copy our elements into it - // to transfer ownership and then set the length + // SAFETY: we create a new vector with sufficient capacity, copy our elements + // into it to transfer ownership and then set the length // we don't drop the elements we previously held unsafe { copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len); @@ -1580,12 +1276,10 @@ impl SmallVec { } } } - #[inline] pub fn into_boxed_slice(self) -> Box<[T]> { self.into_vec().into_boxed_slice() } - #[inline] pub fn into_inner(self) -> Result<[T; N], Self> { if self.len() != N { @@ -1602,12 +1296,10 @@ impl SmallVec { unsafe { Ok(ptr.read()) } } } - #[inline] pub fn retain bool>(&mut self, mut f: F) { self.retain_mut(|elem| f(elem)) } - #[inline] pub fn retain_mut bool>(&mut self, mut f: F) { let mut del = 0; @@ -1626,7 +1318,6 @@ impl SmallVec { } self.truncate(len - del); } - #[inline] pub fn dedup(&mut self) where @@ -1634,7 +1325,6 @@ impl SmallVec { { self.dedup_by(|a, b| a == b); } - #[inline] pub fn dedup_by_key(&mut self, mut key: F) where @@ -1643,7 +1333,6 @@ impl SmallVec { { self.dedup_by(|a, b| key(a) == key(b)); } - #[inline] pub fn dedup_by(&mut self, mut same_bucket: F) where @@ -1655,10 +1344,8 @@ impl SmallVec { if len <= 1 { return; } - let ptr = self.as_mut_ptr(); let mut w: usize = 1; - unsafe { for r in 1..len { let p_r = ptr.add(r); @@ -1672,10 +1359,8 @@ impl SmallVec { } } } - self.truncate(w); } - pub fn resize_with(&mut self, new_len: usize, f: F) where F: FnMut() -> T, @@ -1692,7 +1377,6 @@ impl SmallVec { self.truncate(new_len); } } - pub fn leak<'a>(self) -> &'a mut [T] { if !self.spilled() { panic!( @@ -1702,7 +1386,6 @@ impl SmallVec { let mut me = ManuallyDrop::new(self); unsafe { core::slice::from_raw_parts_mut(me.as_mut_ptr(), me.len()) } } - /// Returns the remaining spare capacity of the vector as a slice of /// `MaybeUninit`. /// @@ -1718,42 +1401,49 @@ impl SmallVec { ) } } - - /// Creates a `SmallVec` directly from the raw components of another `SmallVec`. + /// Creates a `SmallVec` directly from the raw components of another + /// `SmallVec`. /// /// # Safety /// - /// This is highly unsafe, due to the number of invariants that aren’t checked: + /// This is highly unsafe, due to the number of invariants that aren’t + /// checked: /// - /// - `ptr` needs to have been previously allocated via `SmallVec` from its spilled storage (at least, it’s highly likely to be incorrect if it wasn’t). - /// - `ptr`’s `A::Item` type needs to be the same size and alignment that it was allocated with + /// - `ptr` needs to have been previously allocated via `SmallVec` from its + /// spilled storage (at least, it’s highly likely to be incorrect if it + /// wasn’t). + /// - `ptr`’s `A::Item` type needs to be the same size and alignment that it + /// was allocated with /// - `length` needs to be less than or equal to `capacity`. - /// - `capacity` needs to be the capacity that the pointer was allocated with. + /// - `capacity` needs to be the capacity that the pointer was allocated + /// with. /// - /// Violating these may cause problems like corrupting the allocator’s internal data structures. + /// Violating these may cause problems like corrupting the allocator’s + /// internal data structures. /// - /// Additionally, `capacity` must be greater than the amount of inline storage `A` has; that is, the new `SmallVec` must need to spill over into heap allocated storage. This condition is asserted against. + /// Additionally, `capacity` must be greater than the amount of inline + /// storage `A` has; that is, the new `SmallVec` must need to spill over + /// into heap allocated storage. This condition is asserted against. /// - /// The ownership of `ptr` is effectively transferred to the `SmallVec` which may then deallocate, reallocate or change the contents of memory pointed to by the pointer at will. Ensure that nothing else uses the pointer after calling this function. + /// The ownership of `ptr` is effectively transferred to the `SmallVec` + /// which may then deallocate, reallocate or change the contents of memory + /// pointed to by the pointer at will. Ensure that nothing else uses the + /// pointer after calling this function. /// /// # Examples /// /// ``` - /// use smallvec::{SmallVec, smallvec}; - /// + /// use smallvec::{smallvec, SmallVec}; /// let mut v: SmallVec<_, 1> = smallvec![1, 2, 3]; - /// /// // Pull out the important parts of `v`. /// let p = v.as_mut_ptr(); /// let len = v.len(); /// let cap = v.capacity(); /// let spilled = v.spilled(); - /// /// unsafe { /// // Forget all about `v`. The heap allocation that stored the /// // three values won't be deallocated. /// std::mem::forget(v); - /// /// // Overwrite memory with [4, 5, 6]. /// // /// // This is only safe if `spilled` is true! Otherwise, we are @@ -1763,7 +1453,6 @@ impl SmallVec { /// for i in 0..len { /// std::ptr::write(p.add(i), 4 + i); /// } - /// /// // Put everything back together into a SmallVec with a different /// // amount of inline storage, but which is still less than `cap`. /// let rebuilt = SmallVec::<_, 2>::from_raw_parts(p, len, cap); @@ -1773,14 +1462,12 @@ impl SmallVec { #[inline] pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> SmallVec { assert!(!Self::IS_ZST); - // SAFETY: We require caller to provide same ptr as we alloc // and we never alloc null pointer. let ptr = unsafe { debug_assert!(!ptr.is_null(), "Called `from_raw_parts` with null pointer."); NonNull::new_unchecked(ptr) }; - SmallVec { len: TaggedLen::new(length, true), raw: RawSmallVec::new_heap(ptr, capacity), @@ -1788,7 +1475,6 @@ impl SmallVec { } } } - impl SmallVec { #[inline] pub fn resize(&mut self, len: usize, value: T) { @@ -1799,19 +1485,16 @@ impl SmallVec { self.truncate(len); } } - #[inline] pub fn extend_from_slice(&mut self, other: &[T]) { self.extend(other.iter()) } - pub fn extend_from_within(&mut self, src: R) where R: core::ops::RangeBounds, { let src = slice_range(src, ..self.len()); self.reserve(src.len()); - // SAFETY: The call to `reserve` ensures that the capacity is large enough. // The range is within bounds through the use of `core::slice::range`. unsafe { @@ -1819,14 +1502,12 @@ impl SmallVec { { >::spec_extend_from_within(self, src); } - #[cfg(not(feature = "specialization"))] { self.extend_from_within_fallback(src); } } } - #[inline] pub fn extend_from_slice_copy(&mut self, other: &[T]) where @@ -1834,10 +1515,8 @@ impl SmallVec { { let len = other.len(); let src = other.as_ptr(); - let l = self.len(); self.reserve(len); - // SAFETY: Additional memory has been reserved, // therefore the pointer access is valid. unsafe { @@ -1846,7 +1525,6 @@ impl SmallVec { self.set_len(l + len); } } - pub fn extend_from_within_copy(&mut self, src: R) where R: core::ops::RangeBounds, @@ -1856,7 +1534,6 @@ impl SmallVec { let core::ops::Range { start, end } = src; let len = end - start; self.reserve(len); - // SAFETY: The call to `reserve` ensures that the capacity is large enough. // The range is within bounds through the use of `core::slice::range`. unsafe { @@ -1866,7 +1543,6 @@ impl SmallVec { self.set_len(l + len); } } - pub fn insert_from_slice_copy(&mut self, index: usize, other: &[T]) where T: Copy, @@ -1883,12 +1559,10 @@ impl SmallVec { copy(ith_ptr, shifted_ptr, l - index); // elements at `index..index + other_len` are now initialized copy_nonoverlapping(other.as_ptr(), ith_ptr, len); - // SAFETY: all the elements are initialized self.set_len(l + len); } } - /// A function for creating [`SmallVec`] values out of slices /// for types with the [`Copy`] trait. pub fn from_slice_copy(slice: &[T]) -> Self @@ -1898,18 +1572,15 @@ impl SmallVec { let src = slice.as_ptr(); let len = slice.len(); let mut result = Self::with_capacity(len); - // SAFETY: By using `with_capacity`, the pointer will point to valid memory. unsafe { let dst = result.as_mut_ptr(); copy_nonoverlapping(src, dst, len); result.set_len(len); } - result } } - struct DropGuard { ptr: *mut T, len: usize, @@ -1922,13 +1593,11 @@ impl Drop for DropGuard { } } } - struct DropDealloc { ptr: NonNull, size_bytes: usize, align: usize, } - impl Drop for DropDealloc { #[inline] fn drop(&mut self) { @@ -1942,15 +1611,14 @@ impl Drop for DropDealloc { } } } - #[cfg(feature = "may_dangle")] unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { fn drop(&mut self) { let on_heap = self.spilled(); let len = self.len(); let ptr = self.as_mut_ptr(); - // SAFETY: we first drop the elements, then `_drop_dealloc` is dropped, releasing memory we - // used to own + // SAFETY: we first drop the elements, then `_drop_dealloc` is dropped, + // releasing memory we used to own unsafe { let _drop_dealloc = if on_heap { let capacity = self.capacity(); @@ -1966,7 +1634,6 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { } } } - #[cfg(not(feature = "may_dangle"))] impl Drop for SmallVec { fn drop(&mut self) { @@ -1989,7 +1656,6 @@ impl Drop for SmallVec { } } } - impl Drop for IntoIter { fn drop(&mut self) { // SAFETY: see above @@ -2012,22 +1678,6 @@ impl Drop for IntoIter { } } } - -impl core::ops::Deref for SmallVec { - type Target = [T]; - - #[inline] - fn deref(&self) -> &Self::Target { - self.as_slice() - } -} -impl core::ops::DerefMut for SmallVec { - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - self.as_mut_slice() - } -} - /// This function is used in the [`smallvec`] macro. /// It is recommended to use the macro instead of using thís function. #[doc(hidden)] @@ -2035,14 +1685,14 @@ impl core::ops::DerefMut for SmallVec { pub fn from_elem(elem: T, n: usize) -> SmallVec { if n > SmallVec::::inline_size() { // Standard Rust vectors are already specialized. - SmallVec::::from_vec(vec![elem; n]) + use core::iter::repeat_n; + SmallVec::from(Vec::from_iter(repeat_n(elem, n))) } else { #[cfg(feature = "specialization")] { // SAFETY: The precondition is checked in the initial comparison above. unsafe { as spec_traits::SpecFromElem>::spec_from_elem(elem, n) } } - #[cfg(not(feature = "specialization"))] { // SAFETY: The precondition is checked in the initial comparison above. @@ -2050,11 +1700,9 @@ pub fn from_elem(elem: T, n: usize) -> SmallVec } } } - #[cfg(feature = "specialization")] mod spec_traits { use super::*; - /// A trait for specializing the implementation of [`from_elem`]. /// /// [`from_elem`]: crate::from_elem @@ -2067,7 +1715,6 @@ mod spec_traits { /// The caller must ensure that `n <= Self::inline_size()`. unsafe fn spec_from_elem(elem: T, n: usize) -> Self; } - impl SpecFromElem for SmallVec { #[inline] default unsafe fn spec_from_elem(elem: T, n: usize) -> Self { @@ -2075,14 +1722,11 @@ mod spec_traits { unsafe { SmallVec::from_elem_fallback(elem, n) } } } - impl SpecFromElem for SmallVec { unsafe fn spec_from_elem(elem: T, n: usize) -> Self { let mut result = Self::new(); - if n > 0 { let ptr = result.raw.as_mut_ptr_inline(); - // SAFETY: The caller ensures that the first `n` // is smaller than the inline size. unsafe { @@ -2091,24 +1735,21 @@ mod spec_traits { } } } - // SAFETY: The first `n` elements of the vector // have been initialized in the loop above. unsafe { result.set_len(n); } - result } } - - /// A trait for specializing the implementations of [`Extend`] and [`extend_from_slice`]. + /// A trait for specializing the implementations of [`Extend`] and + /// [`extend_from_slice`]. /// /// [`extend_from_slice`]: crate::SmallVec::extend_from_slice pub(crate) trait SpecExtend { fn spec_extend(&mut self, iter: I); } - impl SpecExtend for SmallVec where I: Iterator, @@ -2118,7 +1759,6 @@ mod spec_traits { self.extend_fallback(iter); } } - impl SpecExtend for SmallVec where I: core::iter::TrustedLen, @@ -2128,7 +1768,6 @@ mod spec_traits { panic!("capacity overflow") }; self.reserve(additional); - // SAFETY: A `TrustedLen` iterator provides accurate information // about its size, which was used to reserve additional memory. // This ensures that the access operations inside the loop always @@ -2137,27 +1776,22 @@ mod spec_traits { let len = self.len(); let ptr = self.as_mut_ptr().add(len); let mut guard = DropGuard { ptr, len: 0 }; - for x in iter { ptr.add(guard.len).write(x); guard.len += 1; } - // The elements have been initialized in the loop above. self.set_len(len + guard.len); core::mem::forget(guard); } } } - impl SpecExtend> for SmallVec { fn spec_extend(&mut self, mut iter: IntoIter) { let slice = iter.as_slice(); let len = slice.len(); let old_len = self.len(); - self.reserve(len); - // SAFETY: Additional memory has been reserved above. // Therefore, the copy operates on valid memory. unsafe { @@ -2165,17 +1799,14 @@ mod spec_traits { let src = slice.as_ptr(); copy_nonoverlapping(src, dst, len); } - // SAFETY: The elements were initialized above. unsafe { self.set_len(old_len + len); } - // Mark the iterator as fully consumed. iter.begin = iter.end.value(); } } - impl<'a, T: 'a, const N: usize, I> SpecExtend<&'a T, I> for SmallVec where I: Iterator, @@ -2186,7 +1817,6 @@ mod spec_traits { self.spec_extend(iterator.cloned()) } } - impl<'a, T: 'a, const N: usize> SpecExtend<&'a T, core::slice::Iter<'a, T>> for SmallVec where T: Copy, @@ -2195,9 +1825,7 @@ mod spec_traits { let slice = iter.as_slice(); let len = slice.len(); let old_len = self.len(); - self.reserve(len); - // SAFETY: Additional memory has been reserved above. // Therefore, the copy operates on valid memory. unsafe { @@ -2205,14 +1833,12 @@ mod spec_traits { let src = slice.as_ptr(); copy_nonoverlapping(src, dst, len); } - // SAFETY: The elements were initialized above. unsafe { self.set_len(old_len + len); } } } - /// A trait for specializing the implementation of [`extend_from_within`]. /// /// [`extend_from_within`]: crate::SmallVec::extend_from_within @@ -2222,12 +1848,12 @@ mod spec_traits { /// # Safety /// /// * The length of the vector is larger than or equal to `src.len()`. - /// * The spare capacity of the vector is larger than or equal to `src.len()`. + /// * The spare capacity of the vector is larger than or equal to + /// `src.len()`. /// /// [`extend_from_within`]: SmallVec::extend_from_within unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range); } - impl SpecExtendFromWithin for SmallVec { default unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { // SAFETY: Safety conditions are identical. @@ -2236,14 +1862,11 @@ mod spec_traits { } } } - impl SpecExtendFromWithin for SmallVec { unsafe fn spec_extend_from_within(&mut self, src: core::ops::Range) { let old_len = self.len(); - let start = src.start; let len = src.len(); - // SAFETY: The caller ensures that the vector has spare capacity // for at least `src.len()` elements. This is also the amount of memory // accessed when the data is copied. @@ -2253,21 +1876,18 @@ mod spec_traits { let src = ptr.add(start); copy_nonoverlapping(src, dst, len); } - // SAFETY: The elements were initialized above. unsafe { self.set_len(old_len + len); } } } - /// A trait for specializing the implementation of [`FromIterator`]. /// /// [`clone_from`]: Clone::clone_from pub(crate) trait SpecFromIterator { fn spec_from_iter(iter: I) -> Self; } - impl SpecFromIterator for SmallVec where I: Iterator, @@ -2277,7 +1897,6 @@ mod spec_traits { Self::from_iter_fallback(iter) } } - impl SpecFromIterator for SmallVec where I: core::iter::TrustedLen, @@ -2296,28 +1915,24 @@ mod spec_traits { v } } - /// A trait for specializing the implementation of [`clone_from`]. /// /// [`clone_from`]: Clone::clone_from pub(crate) trait SpecCloneFrom { fn spec_clone_from(&mut self, source: &[T]); } - impl SpecCloneFrom for SmallVec { #[inline] default fn spec_clone_from(&mut self, source: &[T]) { self.clone_from_fallback(source); } } - impl SpecCloneFrom for SmallVec { fn spec_clone_from(&mut self, source: &[T]) { self.clear(); self.extend_from_slice(source); } } - /// A trait for specializing the implementation of [`From`] /// with the source type being slices. pub(crate) trait SpecFromSlice { @@ -2329,40 +1944,34 @@ mod spec_traits { /// The caller must ensure that `slice.len() <= Self::inline_size()`. unsafe fn spec_from(slice: &[T]) -> Self; } - impl SpecFromSlice for SmallVec { default unsafe fn spec_from(slice: &[T]) -> Self { // SAFETY: Safety conditions are identical. unsafe { Self::from_slice_fallback(slice) } } } - impl SpecFromSlice for SmallVec { unsafe fn spec_from(slice: &[T]) -> Self { let mut v = Self::new(); - let src = slice.as_ptr(); let len = slice.len(); let dst = v.as_mut_ptr(); - // SAFETY: The caller ensures that the slice length is smaller // than or equal to the inline length. unsafe { copy_nonoverlapping(src, dst, len); } - // SAFETY: The elements were initialized above. unsafe { v.set_len(len); } - v } } } - /// Fallback functions for various specialized methods. These are kept in -/// a separate implementation block for easy access whenever specialization is disabled. +/// a separate implementation block for easy access whenever specialization is +/// disabled. impl SmallVec { /// Creates a `Smallvec` value where `elem` is repeated `n` times. /// This will use the inline storage, not the heap. @@ -2375,11 +1984,9 @@ impl SmallVec { T: Clone, { let mut result = Self::new(); - if n > 0 { let ptr = result.raw.as_mut_ptr_inline(); let mut guard = DropGuard { ptr, len: 0 }; - // SAFETY: The caller ensures that the first `n` // is smaller than the inline size. unsafe { @@ -2391,16 +1998,13 @@ impl SmallVec { ptr.add(n - 1).write(elem); } } - // SAFETY: The first `n` elements of the vector // have been initialized in the loop above. unsafe { result.set_len(n); } - result } - fn extend_fallback(&mut self, iter: I) where I: IntoIterator, @@ -2412,13 +2016,13 @@ impl SmallVec { self.push(x); } } - /// Main worker for [`extend_from_within`]. /// /// # Safety /// /// * The length of the vector is larger than or equal to `src.len()`. - /// * The spare capacity of the vector is larger than or equal to `src.len()`. + /// * The spare capacity of the vector is larger than or equal to + /// `src.len()`. /// /// [`extend_from_within`]: SmallVec::extend_from_within unsafe fn extend_from_within_fallback(&mut self, src: core::ops::Range) @@ -2426,10 +2030,8 @@ impl SmallVec { T: Clone, { let old_len = self.len(); - let start = src.start; let len = src.len(); - // SAFETY: The caller ensures that the vector has spare capacity // for at least `src.len()` elements. This implies that the loop // operates on valid memory. @@ -2437,7 +2039,6 @@ impl SmallVec { let ptr = self.as_mut_ptr(); let dst = ptr.add(old_len); let src = ptr.add(start); - let mut guard = DropGuard { ptr: dst, len: 0 }; for i in 0..len { let val = (*src.add(i)).clone(); @@ -2446,13 +2047,11 @@ impl SmallVec { } core::mem::forget(guard); } - // SAFETY: The elements were initialized in the loop above. unsafe { self.set_len(old_len + len); } } - fn from_iter_fallback(iter: I) -> Self where I: Iterator, @@ -2464,25 +2063,20 @@ impl SmallVec { } v } - fn clone_from_fallback(&mut self, source: &[T]) where T: Clone, { // Inspired from `impl Clone for Vec`. - // Drop anything that will not be overwritten. self.truncate(source.len()); - // SAFETY: self.len <= other.len due to the truncate above, so the // slices here are always in-bounds. let (init, tail) = unsafe { source.split_at_unchecked(self.len()) }; - // Reuse the contained values' allocations/resources. self.clone_from_slice(init); self.extend(tail.iter().cloned()); } - /// Creates a `SmallVec` value based on the contents of `slice`. /// This will use the inline storage, not the heap. /// @@ -2494,11 +2088,9 @@ impl SmallVec { T: Clone, { let mut v = Self::new(); - let src = slice.as_ptr(); let len = slice.len(); let dst = v.as_mut_ptr(); - // SAFETY: The caller ensures that the slice length is smaller // than or equal to the inline length. unsafe { @@ -2510,114 +2102,36 @@ impl SmallVec { } core::mem::forget(guard); } - // SAFETY: The elements were initialized in the loop above. unsafe { v.set_len(len); } - v } } - -impl From<&[T]> for SmallVec { - #[inline] - fn from(slice: &[T]) -> Self { - if slice.len() > Self::inline_size() { - // Standard Rust vectors are already specialized. - Self::from_vec(Vec::from(slice)) - } else { - // SAFETY: The precondition is checked in the initial comparison above. - unsafe { - #[cfg(feature = "specialization")] - { - >::spec_from(slice) - } - - #[cfg(not(feature = "specialization"))] - { - Self::from_slice_fallback(slice) - } - } - } - } -} - -impl From<&mut [T]> for SmallVec { - #[inline] - fn from(slice: &mut [T]) -> Self { - Self::from(slice as &[T]) - } -} - -impl From<&[T; M]> for SmallVec { - #[inline] - fn from(slice: &[T; M]) -> Self { - Self::from(slice as &[T]) - } -} - -impl From<&mut [T; M]> for SmallVec { - #[inline] - fn from(slice: &mut [T; M]) -> Self { - Self::from(slice as &[T]) - } -} - -impl From<[T; M]> for SmallVec { - fn from(array: [T; M]) -> Self { - if M > N { - // If M > N, we'd have to heap allocate anyway, - // so delegate for Vec for the allocation. - Self::from(Vec::from(array)) - } else { - // M <= N - let mut this = Self::new(); - debug_assert!(M <= this.capacity()); - let array = ManuallyDrop::new(array); - // SAFETY: M <= this.capacity() - unsafe { - copy_nonoverlapping(array.as_ptr(), this.as_mut_ptr(), M); - this.set_len(M); - } - this - } - } -} - -impl From> for SmallVec { - fn from(array: Vec) -> Self { - Self::from_vec(array) - } -} - impl Clone for SmallVec { #[inline] fn clone(&self) -> SmallVec { SmallVec::from(self.as_slice()) } - #[inline] fn clone_from(&mut self, source: &Self) { #[cfg(feature = "specialization")] { >::spec_clone_from(self, source); } - #[cfg(not(feature = "specialization"))] { self.clone_from_fallback(&*source); } } } - impl Clone for IntoIter { #[inline] fn clone(&self) -> IntoIter { SmallVec::from(self.as_slice()).into_iter() } } - impl Extend for SmallVec { #[inline] fn extend>(&mut self, iter: I) { @@ -2625,14 +2139,12 @@ impl Extend for SmallVec { { spec_traits::SpecExtend::::spec_extend(self, iter.into_iter()); } - #[cfg(not(feature = "specialization"))] { self.extend_fallback(iter); } } } - impl<'a, T: Clone + 'a, const N: usize> Extend<&'a T> for SmallVec { #[inline] fn extend>(&mut self, iter: I) { @@ -2640,14 +2152,12 @@ impl<'a, T: Clone + 'a, const N: usize> Extend<&'a T> for SmallVec { { spec_traits::SpecExtend::<&'a T, _>::spec_extend(self, iter.into_iter()); } - #[cfg(not(feature = "specialization"))] { self.extend_fallback(iter.into_iter().cloned()); } } } - impl core::iter::FromIterator for SmallVec { #[inline] fn from_iter>(iter: I) -> Self { @@ -2655,14 +2165,12 @@ impl core::iter::FromIterator for SmallVec { { spec_traits::SpecFromIterator::::spec_from_iter(iter.into_iter()) } - #[cfg(not(feature = "specialization"))] { Self::from_iter_fallback(iter.into_iter()) } } } - #[macro_export] macro_rules! smallvec { ($elem:expr; $n:expr) => ({ @@ -2672,7 +2180,6 @@ macro_rules! smallvec { $crate::SmallVec::from([$($($x),+)?]) }); } - #[macro_export] macro_rules! smallvec_inline { // count helper: transform any expression into 1 @@ -2685,13 +2192,12 @@ macro_rules! smallvec_inline { $crate::SmallVec::<_, N>::from_buf([$($x),*]) }); } - impl IntoIterator for SmallVec { type IntoIter = IntoIter; type Item = T; fn into_iter(self) -> Self::IntoIter { - // SAFETY: we move out of this.raw by reading the value at its address, which is fine since - // we don't drop it + // SAFETY: we move out of this.raw by reading the value at its address, which is + // fine since we don't drop it unsafe { // Set SmallVec len to zero as `IntoIter` drop handles dropping of the elements let this = ManuallyDrop::new(self); @@ -2704,7 +2210,6 @@ impl IntoIterator for SmallVec { } } } - impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { type IntoIter = core::slice::Iter<'a, T>; type Item = &'a T; @@ -2712,7 +2217,6 @@ impl<'a, T, const N: usize> IntoIterator for &'a SmallVec { self.iter() } } - impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { type IntoIter = core::slice::IterMut<'a, T>; type Item = &'a mut T; @@ -2720,308 +2224,23 @@ impl<'a, T, const N: usize> IntoIterator for &'a mut SmallVec { self.iter_mut() } } - -impl PartialEq> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &SmallVec) -> bool { - self.as_slice().eq(other.as_slice()) - } -} -impl Eq for SmallVec where T: Eq {} - -impl PartialEq<[U; M]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &[U; M]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<&[U; M]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &&[U; M]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<[U]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &[U]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<&[U]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &&[U]) -> bool { - self[..] == other[..] - } -} - -impl PartialEq<&mut [U]> for SmallVec -where - T: PartialEq, -{ - #[inline] - fn eq(&self, other: &&mut [U]) -> bool { - self[..] == other[..] - } -} - -impl PartialOrd for SmallVec -where - T: PartialOrd, -{ - #[inline] - fn partial_cmp(&self, other: &SmallVec) -> Option { - self.as_slice().partial_cmp(other.as_slice()) - } -} - -impl Ord for SmallVec -where - T: Ord, -{ - #[inline] - fn cmp(&self, other: &SmallVec) -> core::cmp::Ordering { - self.as_slice().cmp(other.as_slice()) - } -} - impl Hash for SmallVec { fn hash(&self, state: &mut H) { self.as_slice().hash(state) } } - -impl Borrow<[T]> for SmallVec { - #[inline] - fn borrow(&self) -> &[T] { - self.as_slice() - } -} - -impl BorrowMut<[T]> for SmallVec { - #[inline] - fn borrow_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } -} - -impl AsRef<[T]> for SmallVec { - #[inline] - fn as_ref(&self) -> &[T] { - self.as_slice() - } -} - -impl AsMut<[T]> for SmallVec { - #[inline] - fn as_mut(&mut self) -> &mut [T] { - self.as_mut_slice() - } -} - impl Debug for SmallVec { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_list().entries(self.iter()).finish() } } - impl Debug for IntoIter { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_tuple("IntoIter").field(&self.as_slice()).finish() } } - impl Debug for Drain<'_, T, N> { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() } } - -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl Serialize for SmallVec -where - T: Serialize, -{ - fn serialize(&self, serializer: S) -> Result { - let mut state = serializer.serialize_seq(Some(self.len()))?; - for item in self { - state.serialize_element(item)?; - } - state.end() - } -} - -#[cfg(feature = "serde")] -#[cfg_attr(docsrs, doc(cfg(feature = "serde")))] -impl<'de, T, const N: usize> Deserialize<'de> for SmallVec -where - T: Deserialize<'de>, -{ - fn deserialize>(deserializer: D) -> Result { - deserializer.deserialize_seq(SmallVecVisitor { - phantom: PhantomData, - }) - } -} - -#[cfg(feature = "serde")] -struct SmallVecVisitor { - phantom: PhantomData, -} - -#[cfg(feature = "serde")] -impl<'de, T, const N: usize> Visitor<'de> for SmallVecVisitor -where - T: Deserialize<'de>, -{ - type Value = SmallVec; - - fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - formatter.write_str("a sequence") - } - - fn visit_seq(self, mut seq: B) -> Result - where - B: SeqAccess<'de>, - { - use serde_core::de::Error; - let len = seq.size_hint().unwrap_or(0); - let mut values = SmallVec::new(); - values.try_reserve(len).map_err(B::Error::custom)?; - - while let Some(value) = seq.next_element()? { - values.push(value); - } - - Ok(values) - } -} - -#[cfg(feature = "malloc_size_of")] -impl MallocShallowSizeOf for SmallVec { - fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - if self.spilled() { - unsafe { ops.malloc_size_of(self.as_ptr()) } - } else { - 0 - } - } -} - -#[cfg(feature = "malloc_size_of")] -impl MallocSizeOf for SmallVec { - fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { - let mut n = self.shallow_size_of(ops); - for elem in self.iter() { - n += elem.size_of(ops); - } - n - } -} - -#[cfg(feature = "std")] -#[cfg_attr(docsrs, doc(cfg(feature = "std")))] -impl io::Write for SmallVec { - #[inline] - fn write(&mut self, buf: &[u8]) -> io::Result { - self.extend_from_slice(buf); - Ok(buf.len()) - } - - #[inline] - fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { - self.extend_from_slice(buf); - Ok(()) - } - - #[inline] - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -#[cfg(feature = "bytes")] -unsafe impl BufMut for SmallVec { - #[inline] - fn remaining_mut(&self) -> usize { - // A vector can never have more than isize::MAX bytes - isize::MAX as usize - self.len() - } - - #[inline] - unsafe fn advance_mut(&mut self, cnt: usize) { - let len = self.len(); - let remaining = self.capacity() - len; - - if remaining < cnt { - panic!("advance out of bounds: the len is {remaining} but advancing by {cnt}"); - } - - // Addition will not overflow since the sum is at most the capacity. - self.set_len(len + cnt); - } - - #[inline] - fn chunk_mut(&mut self) -> &mut UninitSlice { - if self.capacity() == self.len() { - self.reserve(64); // Grow the smallvec - } - - let cap = self.capacity(); - let len = self.len(); - - let ptr = self.as_mut_ptr(); - // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be - // valid for `cap - len` bytes. The subtraction will not underflow since - // `len <= cap`. - unsafe { UninitSlice::from_raw_parts_mut(ptr.add(len), cap - len) } - } - - // Specialize these methods so they can skip checking `remaining_mut` - // and `advance_mut`. - #[inline] - fn put(&mut self, mut src: T) - where - Self: Sized, - { - // In case the src isn't contiguous, reserve upfront. - self.reserve(src.remaining()); - - while src.has_remaining() { - let s = src.chunk(); - let l = s.len(); - self.extend_from_slice(s); - src.advance(l); - } - } - - #[inline] - fn put_slice(&mut self, src: &[u8]) { - self.extend_from_slice(src); - } - - #[inline] - fn put_bytes(&mut self, val: u8, cnt: usize) { - // If the addition overflows, then the `resize` will fail. - let new_len = self.len().saturating_add(cnt); - self.resize(new_len, val); - } -} diff --git a/src/mallocsizeof.rs b/src/mallocsizeof.rs new file mode 100644 index 00000000..7a49d865 --- /dev/null +++ b/src/mallocsizeof.rs @@ -0,0 +1,22 @@ +use { + super::SmallVec, + malloc_size_of::{MallocShallowSizeOf, MallocSizeOf, MallocSizeOfOps}, +}; +impl MallocShallowSizeOf for SmallVec { + fn shallow_size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + if self.spilled() { + unsafe { ops.malloc_size_of(self.as_ptr()) } + } else { + 0 + } + } +} +impl MallocSizeOf for SmallVec { + fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize { + let mut n = self.shallow_size_of(ops); + for elem in self.iter() { + n += elem.size_of(ops); + } + n + } +} diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index dadbf958..2f501f0e 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -1,6 +1,11 @@ -use core::mem::{ManuallyDrop, MaybeUninit}; -use core::ptr::NonNull; - +use { + super::{allocationerror::AllocationError, TaggedLen}, + alloc::alloc::Layout, + core::{ + mem::{ManuallyDrop, MaybeUninit}, + ptr::{copy_nonoverlapping, NonNull}, + }, +}; /// Either a stack array with `length <= N` or a heap array /// whose pointer and capacity are stored here. /// @@ -11,3 +16,99 @@ pub union RawSmallVec { pub inline: ManuallyDrop>, pub heap: (NonNull, usize), } +impl RawSmallVec { + pub const IS_ZST: bool = size_of::() == 0; + #[inline] + pub const fn new() -> Self { + Self::new_inline(MaybeUninit::uninit()) + } + #[inline] + pub const fn new_inline(inline: MaybeUninit<[T; N]>) -> Self { + Self { + inline: ManuallyDrop::new(inline), + } + } + #[inline] + pub const fn new_heap(ptr: NonNull, capacity: usize) -> Self { + Self { + heap: (ptr, capacity), + } + } + #[inline] + pub const fn as_ptr_inline(&self) -> *const T { + // SAFETY: it is safe because we aren't reading the value, just getting a + // reference to it. reading it would be UB potentially, but for that downstream + // unsafe is required + #[allow(unused_unsafe, reason = "requires unsafe in MSRV")] + (unsafe { &raw const self.inline }).cast::() + } + #[inline] + pub const fn as_mut_ptr_inline(&mut self) -> *mut T { + // SAFETY: same as above + #[allow(unused_unsafe, reason = "requires unsafe in MSRV")] + (unsafe { &raw mut self.inline }).cast::() + } + /// # Safety + /// + /// The vector must be on the heap + #[inline] + pub const unsafe fn as_ptr_heap(&self) -> *const T { + self.heap.0.as_ptr() + } + /// # Safety + /// + /// The vector must be on the heap + #[inline] + pub const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { + self.heap.0.as_ptr() + } + /// # Safety + /// + /// `new_capacity` must be non zero, and greater or equal to the length. + /// T must not be a ZST. + pub unsafe fn try_grow_raw( + &mut self, + len: TaggedLen, + new_capacity: usize, + ) -> Result<(), AllocationError> { + use alloc::alloc::{alloc, realloc}; + debug_assert!(!Self::IS_ZST); + debug_assert!(new_capacity > 0); + debug_assert!(new_capacity >= len.value()); + let was_on_heap = len.on_heap(); + let ptr = if was_on_heap { + self.as_mut_ptr_heap() + } else { + self.as_mut_ptr_inline() + }; + let len = len.value(); + let new_layout = + Layout::array::(new_capacity).map_err(|_| AllocationError::CapacityOverflow)?; + if new_layout.size() > isize::MAX as usize { + return Err(AllocationError::CapacityOverflow); + } + let new_ptr = if len == 0 || !was_on_heap { + // get a fresh allocation + let new_ptr = alloc(new_layout) as *mut T; // `new_layout` has nonzero size. + let new_ptr = + NonNull::new(new_ptr).ok_or(AllocationError::Failure { layout: new_layout })?; + copy_nonoverlapping(ptr, new_ptr.as_ptr(), len); + new_ptr + } else { + // use realloc + // this can't overflow since we already constructed an equivalent layout during + // the previous allocation + let old_layout = + Layout::from_size_align_unchecked(self.heap.1 * size_of::(), align_of::()); + // SAFETY: ptr was allocated with this allocator + // old_layout is the same as the layout used to allocate the previous memory + // block new_layout.size() is greater than zero + // does not overflow when rounded up to alignment. since it was constructed + // with Layout::array + let new_ptr = realloc(ptr as *mut u8, old_layout, new_layout.size()) as *mut T; + NonNull::new(new_ptr).ok_or(AllocationError::Failure { layout: new_layout })? + }; + *self = Self::new_heap(new_ptr, new_capacity); + Ok(()) + } +} diff --git a/src/references.rs b/src/references.rs new file mode 100644 index 00000000..fff4b4ba --- /dev/null +++ b/src/references.rs @@ -0,0 +1,41 @@ +use { + super::SmallVec, + core::borrow::{Borrow, BorrowMut}, +}; +impl core::ops::Deref for SmallVec { + type Target = [T]; + #[inline] + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} +impl core::ops::DerefMut for SmallVec { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + self.as_mut_slice() + } +} +impl AsRef<[T]> for SmallVec { + #[inline] + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} +impl AsMut<[T]> for SmallVec { + #[inline] + fn as_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} +impl Borrow<[T]> for SmallVec { + #[inline] + fn borrow(&self) -> &[T] { + self.as_slice() + } +} +impl BorrowMut<[T]> for SmallVec { + #[inline] + fn borrow_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} diff --git a/src/serde.rs b/src/serde.rs new file mode 100644 index 00000000..18fb2a39 --- /dev/null +++ b/src/serde.rs @@ -0,0 +1,56 @@ +use { + super::SmallVec, + core::marker::PhantomData, + serde_core::{ + de::{SeqAccess, Visitor}, + ser::SerializeSeq, + Deserialize, Deserializer, Serialize, Serializer, + }, +}; +impl Serialize for SmallVec +where + T: Serialize, +{ + fn serialize(&self, serializer: S) -> Result { + let mut state = serializer.serialize_seq(Some(self.len()))?; + for item in self { + state.serialize_element(item)?; + } + state.end() + } +} +impl<'de, T, const N: usize> Deserialize<'de> for SmallVec +where + T: Deserialize<'de>, +{ + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_seq(SmallVecVisitor { + phantom: PhantomData, + }) + } +} +struct SmallVecVisitor { + phantom: PhantomData, +} +impl<'de, T, const N: usize> Visitor<'de> for SmallVecVisitor +where + T: Deserialize<'de>, +{ + type Value = SmallVec; + fn expecting(&self, formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + formatter.write_str("a sequence") + } + fn visit_seq(self, mut seq: B) -> Result + where + B: SeqAccess<'de>, + { + use serde_core::de::Error; + let len = seq.size_hint().unwrap_or(0); + let mut values = SmallVec::new(); + values.try_reserve(len).map_err(B::Error::custom)?; + while let Some(value) = seq.next_element()? { + values.push(value); + } + Ok(values) + } +} diff --git a/src/std.rs b/src/std.rs new file mode 100644 index 00000000..b9e25685 --- /dev/null +++ b/src/std.rs @@ -0,0 +1,20 @@ +extern crate std; +use {super::SmallVec, std::io}; +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +impl io::Write for SmallVec { + #[inline] + fn write(&mut self, buf: &[u8]) -> io::Result { + self.extend_from_slice(buf); + Ok(buf.len()) + } + #[inline] + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.extend_from_slice(buf); + Ok(()) + } + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} diff --git a/src/taggedlen.rs b/src/taggedlen.rs new file mode 100644 index 00000000..acc97fa0 --- /dev/null +++ b/src/taggedlen.rs @@ -0,0 +1,55 @@ +use core::marker::PhantomData; +/// Vec guarantees that its length is always less than [`isize::MAX`] in +/// *bytes*. +/// +/// For a non ZST, this means that the length is less than `isize::MAX` objects, +/// which implies we have at least one free bit we can use. We use the least +/// significant bit for the tag. And store the length in the `usize::BITS - 1` +/// most significant bits. +/// +/// For a ZST, we never use the heap, so we just store the length directly. +#[repr(transparent)] +pub struct TaggedLen(usize, PhantomData); +// Clone and Copy must be manually implemented because the generic interferes +// with the derive attribute implementations. +impl Clone for TaggedLen { + #[inline] + fn clone(&self) -> Self { + Self(self.0, PhantomData) + } + #[inline] + fn clone_from(&mut self, source: &Self) { + self.0 = source.0; + } +} +impl Copy for TaggedLen {} +impl TaggedLen { + const IS_ZST: bool = size_of::() == 0; + #[inline] + pub const fn new(len: usize, on_heap: bool) -> Self { + if Self::IS_ZST { + debug_assert!(!on_heap); + Self(len, PhantomData) + } else { + debug_assert!(len < isize::MAX as usize); + Self((len << 1) | on_heap as usize, PhantomData) + } + } + #[inline] + #[must_use] + pub const fn on_heap(self) -> bool { + if Self::IS_ZST { + false + } else { + (self.0 & 1_usize) == 1 + } + } + #[inline] + pub const fn value(self) -> usize { + if Self::IS_ZST { + self.0 + } else { + self.0 >> 1 + } + } +} diff --git a/src/tests.rs b/src/tests.rs index cfd801b4..d4af1a44 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -1,13 +1,9 @@ -use crate::{smallvec, SmallVec}; - -use core::hash::Hasher; -use core::iter::FromIterator; - -use alloc::borrow::ToOwned; -use alloc::boxed::Box; -use alloc::rc::Rc; -use alloc::{vec, vec::Vec}; - +extern crate std; +use { + crate::{smallvec, SmallVec}, + alloc::{borrow::ToOwned, boxed::Box, rc::Rc, vec::Vec}, + core::{hash::Hasher, iter::FromIterator}, +}; #[test] pub fn test_zero() { let mut v = SmallVec::<_, 0>::new(); @@ -16,9 +12,8 @@ pub fn test_zero() { assert!(v.spilled()); assert_eq!(&*v, &[0]); } - -// We heap allocate all these strings so that double frees will show up under valgrind. - +// We heap allocate all these strings so that double frees will show up under +// valgrind. #[test] pub fn test_inline() { let mut v = SmallVec::<_, 16>::new(); @@ -26,7 +21,6 @@ pub fn test_inline() { v.push("there".to_owned()); assert_eq!(&*v, &["hello".to_owned(), "there".to_owned(),][..]); } - #[test] pub fn test_spill() { let mut v = SmallVec::<_, 2>::new(); @@ -46,7 +40,6 @@ pub fn test_spill() { ][..] ); } - #[test] pub fn test_double_spill() { let mut v = SmallVec::<_, 2>::new(); @@ -72,38 +65,32 @@ pub fn test_double_spill() { ][..] ); } - // https://github.com/servo/rust-smallvec/issues/4 #[test] fn issue_4() { SmallVec::, 2>::new(); } - // https://github.com/servo/rust-smallvec/issues/5 #[test] fn issue_5() { assert!(Some(SmallVec::<&u32, 2>::new()).is_some()); } - #[test] fn test_with_capacity() { let v: SmallVec = SmallVec::with_capacity(1); assert!(v.is_empty()); assert!(!v.spilled()); assert_eq!(v.capacity(), 3); - let v: SmallVec = SmallVec::with_capacity(10); assert!(v.is_empty()); assert!(v.spilled()); assert_eq!(v.capacity(), 10); } - #[test] fn drain() { let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.drain(..).collect::>(), &[3]); - // spilling the vec v.push(3); v.push(4); @@ -112,7 +99,6 @@ fn drain() { assert_eq!(v.drain(1..).collect::>(), &[4, 5]); // drain should not change the capacity assert_eq!(v.capacity(), old_capacity); - // Exercise the tail-shifting code when in the inline state // This has the potential to produce UB due to aliasing let mut v: SmallVec = SmallVec::new(); @@ -120,27 +106,23 @@ fn drain() { v.push(2); assert_eq!(v.drain(..1).collect::>(), &[1]); } - #[test] fn drain_rev() { let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.drain(..).rev().collect::>(), &[3]); - // spilling the vec v.push(3); v.push(4); v.push(5); assert_eq!(v.drain(..).rev().collect::>(), &[5, 4, 3]); } - #[test] fn drain_forget() { let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6, 7]; std::mem::forget(v.drain(2..5)); assert_eq!(v.len(), 2); } - #[test] fn splice() { // The range starts right before the end. @@ -149,14 +131,12 @@ fn splice() { let u: SmallVec = v.splice(6.., new).collect(); assert_eq!(v, [0, 1, 2, 3, 4, 5, 7, 8, 9, 10]); assert_eq!(u, [6]); - // The range is empty. let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; let new = [7, 8, 9, 10]; let u: SmallVec = v.splice(1..1, new).collect(); assert_eq!(v, [0, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6]); assert_eq!(u, [0u8; 0]); - // The range is at the beginning and nonempty. let mut v: SmallVec = smallvec![0, 1, 2, 3, 4, 5, 6]; let new = [7, 8, 9, 10]; @@ -164,13 +144,11 @@ fn splice() { assert_eq!(v, [7, 8, 9, 10, 3, 4, 5, 6]); assert_eq!(u, [0, 1, 2]); } - #[test] fn into_iter() { let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.into_iter().collect::>(), &[3]); - // spilling the vec let mut v: SmallVec = SmallVec::new(); v.push(3); @@ -178,13 +156,11 @@ fn into_iter() { v.push(5); assert_eq!(v.into_iter().collect::>(), &[3, 4, 5]); } - #[test] fn into_iter_rev() { let mut v: SmallVec = SmallVec::new(); v.push(3); assert_eq!(v.into_iter().rev().collect::>(), &[3]); - // spilling the vec let mut v: SmallVec = SmallVec::new(); v.push(3); @@ -192,19 +168,15 @@ fn into_iter_rev() { v.push(5); assert_eq!(v.into_iter().rev().collect::>(), &[5, 4, 3]); } - #[test] fn into_iter_drop() { use std::cell::Cell; - struct DropCounter<'a>(&'a Cell); - impl<'a> Drop for DropCounter<'a> { fn drop(&mut self) { self.0.set(self.0.get() + 1); } } - { let cell = Cell::new(0); let mut v: SmallVec, 2> = SmallVec::new(); @@ -212,7 +184,6 @@ fn into_iter_drop() { v.into_iter(); assert_eq!(cell.get(), 1); } - { let cell = Cell::new(0); let mut v: SmallVec, 2> = SmallVec::new(); @@ -221,7 +192,6 @@ fn into_iter_drop() { assert!(v.into_iter().next().is_some()); assert_eq!(cell.get(), 2); } - { let cell = Cell::new(0); let mut v: SmallVec, 2> = SmallVec::new(); @@ -245,79 +215,62 @@ fn into_iter_drop() { assert_eq!(cell.get(), 3); } } - #[test] fn test_capacity() { let mut v: SmallVec = SmallVec::new(); v.reserve(1); assert_eq!(v.capacity(), 2); assert!(!v.spilled()); - v.reserve_exact(0x100); assert!(v.capacity() >= 0x100); - v.push(0); v.push(1); v.push(2); v.push(3); - v.shrink_to_fit(); assert!(v.capacity() < 0x100); } - #[test] fn test_truncate() { let mut v: SmallVec, 8> = SmallVec::new(); - for x in 0..8 { v.push(Box::new(x)); } v.truncate(4); - assert_eq!(v.len(), 4); assert!(!v.spilled()); - assert_eq!(*v.swap_remove(1), 1); assert_eq!(*v.remove(1), 3); v.insert(1, Box::new(3)); - assert_eq!(&v.iter().map(|v| **v).collect::>(), &[0, 3, 2]); } - #[test] fn test_truncate_references() { - let mut v = vec![0, 1, 2, 3, 4, 5, 6, 7]; + let mut v = Vec::from([0, 1, 2, 3, 4, 5, 6, 7]); let mut i = 8; let mut v: SmallVec<&mut u8, 8> = v.iter_mut().collect(); - v.truncate(4); - assert_eq!(v.len(), 4); assert!(!v.spilled()); - assert_eq!(*v.swap_remove(1), 1); assert_eq!(*v.remove(1), 3); v.insert(1, &mut i); - assert_eq!( &v.iter_mut().map(|v| &mut **v).collect::>(), &[&mut 0, &mut 8, &mut 2] ); } - #[test] fn test_split_off() { let mut vec: SmallVec = smallvec![1, 2, 3, 4, 5, 6]; let orig_ptr = vec.as_ptr(); let orig_capacity = vec.capacity(); - let split_off = vec.split_off(4); assert_eq!(&vec[..], &[1, 2, 3, 4]); assert_eq!(&split_off[..], &[5, 6]); assert_eq!(vec.capacity(), orig_capacity); assert_eq!(vec.as_ptr(), orig_ptr); } - #[test] fn test_split_off_take_all() { // Allocate enough capacity that we can tell whether the split-off vector's @@ -326,19 +279,16 @@ fn test_split_off_take_all() { vec.extend([1, 2, 3, 4, 5, 6]); let orig_ptr = vec.as_ptr(); let orig_capacity: usize = vec.capacity(); - let split_off = vec.split_off(0); assert_eq!(&vec[..], &[0u32; 0]); assert_eq!(&split_off[..], &[1, 2, 3, 4, 5, 6]); assert_eq!(vec.capacity(), orig_capacity); assert_eq!(vec.as_ptr(), orig_ptr); - // The split-off vector should be newly-allocated, and should not have // stolen the original vector's allocation. assert!(split_off.capacity() < orig_capacity); assert_ne!(split_off.as_ptr(), orig_ptr); } - #[test] fn test_append() { let mut v: SmallVec = SmallVec::new(); @@ -346,18 +296,15 @@ fn test_append() { v.push(x); } assert_eq!(v.len(), 4); - let mut n: SmallVec = SmallVec::from_buf([5, 6]); v.append(&mut n); assert_eq!(v.len(), 6); assert_eq!(n.len(), 0); - assert_eq!( &v.iter().map(|v| *v).collect::>(), &[0, 1, 2, 3, 5, 6] ); } - #[test] #[should_panic] fn test_invalid_grow() { @@ -365,14 +312,12 @@ fn test_invalid_grow() { v.extend(0..8); v.grow(5); } - #[test] #[should_panic] fn drain_overflow() { let mut v: SmallVec = smallvec![0]; v.drain(..=usize::MAX); } - #[test] fn test_extend_from_slice() { let mut v: SmallVec = SmallVec::new(); @@ -386,7 +331,6 @@ fn test_extend_from_slice() { &[0, 1, 2, 3, 5, 6] ); } - #[test] fn test_extend_from_within() { let mut v: SmallVec = smallvec![0, 1, 2, 3]; @@ -396,24 +340,20 @@ fn test_extend_from_within() { &[0, 1, 2, 3, 1, 2], ); } - #[test] #[should_panic] fn test_drop_panic_smallvec() { // This test should only panic once, and not double panic, // which would mean a double drop struct DropPanic; - impl Drop for DropPanic { fn drop(&mut self) { panic!("drop"); } } - let mut v = SmallVec::<_, 1>::new(); v.push(DropPanic); } - #[test] fn test_eq() { let mut a: SmallVec = SmallVec::new(); @@ -428,11 +368,9 @@ fn test_eq() { // c = [3, 4] c.push(3); c.push(4); - assert!(a == b); assert!(a != c); } - #[test] fn test_ord() { let mut a: SmallVec = SmallVec::new(); @@ -446,31 +384,25 @@ fn test_ord() { // c = [1, 2] c.push(1); c.push(2); - assert!(a < b); assert!(b > a); assert!(b < c); assert!(c > b); } - #[test] fn test_hash() { - use std::collections::hash_map::DefaultHasher; - use std::hash::Hash; - + use std::{collections::hash_map::DefaultHasher, hash::Hash}; fn hash(value: impl Hash) -> u64 { let mut hasher = DefaultHasher::new(); value.hash(&mut hasher); hasher.finish() } - { let mut a: SmallVec = SmallVec::new(); let b = [1, 2]; a.extend(b.iter().cloned()); assert_eq!(hash(a), hash(b)); } - { let mut a: SmallVec = SmallVec::new(); let b = [1, 2, 11, 12]; @@ -478,7 +410,6 @@ fn test_hash() { assert_eq!(hash(a), hash(b)); } } - #[test] fn test_as_ref() { let mut a: SmallVec = SmallVec::new(); @@ -489,7 +420,6 @@ fn test_as_ref() { a.push(3); assert_eq!(a.as_ref(), [1, 2, 3]); } - #[test] fn test_as_mut() { let mut a: SmallVec = SmallVec::new(); @@ -502,11 +432,9 @@ fn test_as_mut() { a.as_mut()[1] = 4; assert_eq!(a.as_mut(), [1, 4, 3]); } - #[test] fn test_borrow() { use std::borrow::Borrow; - let mut a: SmallVec = SmallVec::new(); a.push(1); assert_eq!(a.borrow(), [1]); @@ -515,11 +443,9 @@ fn test_borrow() { a.push(3); assert_eq!(a.borrow(), [1, 2, 3]); } - #[test] fn test_borrow_mut() { use std::borrow::BorrowMut; - let mut a: SmallVec = SmallVec::new(); a.push(1); assert_eq!(a.borrow_mut(), [1]); @@ -530,66 +456,54 @@ fn test_borrow_mut() { BorrowMut::<[u32]>::borrow_mut(&mut a)[1] = 4; assert_eq!(a.borrow_mut(), [1, 4, 3]); } - #[test] fn test_from() { assert_eq!(&SmallVec::::from(&[1][..])[..], [1]); assert_eq!(&SmallVec::::from(&[1, 2, 3][..])[..], [1, 2, 3]); - - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); - - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); - let array = [1]; let small_vec: SmallVec = SmallVec::from(array); assert_eq!(&*small_vec, &[1]); drop(small_vec); - let array = [99; 128]; let small_vec: SmallVec = SmallVec::from(array); - assert_eq!(&*small_vec, vec![99u8; 128].as_slice()); + assert_eq!(&*small_vec, Vec::from([99u8; 128]).as_slice()); drop(small_vec); - #[derive(PartialEq, Eq, Debug)] struct NoClone(u8); let array = [NoClone(42)]; let small_vec: SmallVec = SmallVec::from(array); assert_eq!(&*small_vec, &[NoClone(42)]); drop(small_vec); - - let vec = vec![NoClone(42)]; + let vec = Vec::from([NoClone(42)]); let small_vec: SmallVec = SmallVec::from(vec); assert_eq!(&*small_vec, &[NoClone(42)]); drop(small_vec); - let array = [1; 128]; let small_vec: SmallVec = SmallVec::from(array); - assert_eq!(&*small_vec, vec![1; 128].as_slice()); + assert_eq!(&*small_vec, Vec::from([1; 128]).as_slice()); drop(small_vec); - let array = [99]; let small_vec: SmallVec = SmallVec::from(array); assert_eq!(&*small_vec, &[99u8]); drop(small_vec); } - #[test] fn test_from_slice() { assert_eq!(&SmallVec::::from(&[1][..])[..], [1]); assert_eq!(&SmallVec::::from(&[1, 2, 3][..])[..], [1, 2, 3]); } - #[test] fn test_exact_size_iterator() { let mut vec = SmallVec::::from(&[1, 2, 3][..]); @@ -597,7 +511,6 @@ fn test_exact_size_iterator() { assert_eq!(vec.drain(..2).len(), 2); assert_eq!(vec.into_iter().len(), 1); } - #[test] fn test_into_iter_as_slice() { let vec = SmallVec::::from(&[1, 2, 3][..]); @@ -611,11 +524,10 @@ fn test_into_iter_as_slice() { assert_eq!(iter.as_slice(), &[2]); assert_eq!(iter.as_mut_slice(), &[2]); } - #[test] fn test_into_iter_clone() { - // Test that the cloned iterator yields identical elements and that it owns its own copy - // (i.e. no use after move errors). + // Test that the cloned iterator yields identical elements and that it owns its + // own copy (i.e. no use after move errors). let mut iter = SmallVec::::from_iter(0..3).into_iter(); let mut clone_iter = iter.clone(); while let Some(x) = iter.next() { @@ -623,10 +535,10 @@ fn test_into_iter_clone() { } assert_eq!(clone_iter.next(), None); } - #[test] fn test_into_iter_clone_partially_consumed_iterator() { - // Test that the cloned iterator only contains the remaining elements of the original iterator. + // Test that the cloned iterator only contains the remaining elements of the + // original iterator. let mut iter = SmallVec::::from_iter(0..3).into_iter().skip(1); let mut clone_iter = iter.clone(); while let Some(x) = iter.next() { @@ -634,7 +546,6 @@ fn test_into_iter_clone_partially_consumed_iterator() { } assert_eq!(clone_iter.next(), None); } - #[test] fn test_into_iter_clone_empty_smallvec() { let mut iter = SmallVec::::new().into_iter(); @@ -642,7 +553,6 @@ fn test_into_iter_clone_empty_smallvec() { assert_eq!(iter.next(), None); assert_eq!(clone_iter.next(), None); } - #[test] fn shrink_to_fit_unspill() { let mut vec = SmallVec::::from_iter(0..3); @@ -651,68 +561,55 @@ fn shrink_to_fit_unspill() { vec.shrink_to_fit(); assert!(!vec.spilled(), "shrink_to_fit will un-spill if possible"); } - #[test] fn shrink_after_from_empty_vec() { - let mut v = SmallVec::::from_vec(vec![]); + let mut v = SmallVec::::from_vec(Vec::new()); v.shrink_to_fit(); assert!(!v.spilled()) } - #[test] fn test_into_vec() { let vec = SmallVec::::from_iter(0..2); - assert_eq!(vec.into_vec(), vec![0, 1]); - + assert_eq!(vec.into_vec(), Vec::from([0, 1])); let vec = SmallVec::::from_iter(0..3); - assert_eq!(vec.into_vec(), vec![0, 1, 2]); + assert_eq!(vec.into_vec(), Vec::from([0, 1, 2])); } - #[test] fn test_into_inner() { let vec = SmallVec::::from_iter(0..2); assert_eq!(vec.into_inner(), Ok([0, 1])); - let vec = SmallVec::::from_iter(0..1); assert_eq!(vec.clone().into_inner(), Err(vec)); - let vec = SmallVec::::from_iter(0..3); assert_eq!(vec.clone().into_inner(), Err(vec)); } - #[test] fn test_from_vec() { - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - - let vec = vec![]; + let vec = Vec::new(); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[0u8; 0]); drop(small_vec); - - let vec = vec![1]; + let vec = Vec::from([1]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1]); drop(small_vec); - - let vec = vec![1, 2, 3]; + let vec = Vec::from([1, 2, 3]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3]); drop(small_vec); - - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); - - let vec = vec![1, 2, 3, 4, 5]; + let vec = Vec::from([1, 2, 3, 4, 5]); let small_vec: SmallVec = SmallVec::from_vec(vec); assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]); drop(small_vec); } - #[test] fn test_retain() { // Test inline data storage @@ -722,7 +619,6 @@ fn test_retain() { assert_eq!(sv.pop(), Some(2)); assert_eq!(sv.pop(), Some(1)); assert_eq!(sv.pop(), None); - // Test spilled data storage let mut sv: SmallVec = SmallVec::from(&[1, 2, 3, 3, 4]); sv.retain(|&i| i != 3); @@ -730,7 +626,6 @@ fn test_retain() { assert_eq!(sv.pop(), Some(2)); assert_eq!(sv.pop(), Some(1)); assert_eq!(sv.pop(), None); - // Test that drop implementations are called for inline. let one = Rc::new(1); let mut sv: SmallVec, 3> = SmallVec::new(); @@ -738,7 +633,6 @@ fn test_retain() { assert_eq!(Rc::strong_count(&one), 2); sv.retain(|_| false); assert_eq!(Rc::strong_count(&one), 1); - // Test that drop implementations are called for spilled data. let mut sv: SmallVec, 1> = SmallVec::new(); sv.push(Rc::clone(&one)); @@ -747,54 +641,43 @@ fn test_retain() { sv.retain(|_| false); assert_eq!(Rc::strong_count(&one), 1); } - #[test] fn test_dedup() { let mut dupes: SmallVec = SmallVec::from(&[1, 1, 2, 3, 3]); dupes.dedup(); assert_eq!(&*dupes, &[1, 2, 3]); - let mut empty: SmallVec = SmallVec::new(); empty.dedup(); assert!(empty.is_empty()); - let mut all_ones: SmallVec = SmallVec::from(&[1, 1, 1, 1, 1]); all_ones.dedup(); assert_eq!(all_ones.len(), 1); - let mut no_dupes: SmallVec = SmallVec::from(&[1, 2, 3, 4, 5]); no_dupes.dedup(); assert_eq!(no_dupes.len(), 5); } - #[test] fn test_resize() { let mut v: SmallVec = SmallVec::new(); v.push(1); v.resize(5, 0); assert_eq!(v[..], [1, 0, 0, 0, 0][..]); - v.resize(2, -1); assert_eq!(v[..], [1, 0][..]); } - #[cfg(feature = "std")] #[test] fn test_write() { use std::io::Write; - let data = [1, 2, 3, 4, 5]; - let mut small_vec: SmallVec = SmallVec::new(); let len = small_vec.write(&data[..]).unwrap(); assert_eq!(len, 5); assert_eq!(small_vec.as_ref(), data.as_ref()); - let mut small_vec: SmallVec = SmallVec::new(); small_vec.write_all(&data[..]).unwrap(); assert_eq!(small_vec.as_ref(), data.as_ref()); } - #[cfg(feature = "serde")] #[test] fn test_serde() { @@ -819,7 +702,6 @@ fn test_serde() { ], ); } - #[test] fn grow_to_shrink() { let mut v: SmallVec = SmallVec::new(); @@ -836,7 +718,6 @@ fn grow_to_shrink() { v.push(4); assert_eq!(v[..], [4]); } - #[test] fn resumable_extend() { let s = "a b c"; @@ -848,14 +729,12 @@ fn resumable_extend() { v.extend(it); assert_eq!(v[..], ['a']); } - // #139 #[test] fn uninhabited() { enum Void {} let _sv = SmallVec::::new(); } - #[test] fn grow_spilled_same_size() { let mut v: SmallVec = SmallVec::new(); @@ -869,12 +748,10 @@ fn grow_spilled_same_size() { assert_eq!(v.capacity(), 4); assert_eq!(v[..], [0, 1, 2]); } - #[test] fn const_generics() { let _v = SmallVec::::default(); } - #[test] fn const_new() { let v = const_new_inner(); @@ -899,111 +776,93 @@ const fn const_new_inline_sized() -> SmallVec { const fn const_new_inline_args() -> SmallVec { crate::smallvec_inline![1, 4] } - #[test] fn empty_macro() { let _v: SmallVec = smallvec![]; } - #[test] fn zero_size_items() { SmallVec::<(), 0>::new().push(()); } - #[test] fn test_clone_from() { let mut a: SmallVec = SmallVec::new(); a.push(1); a.push(2); a.push(3); - let mut b: SmallVec = SmallVec::new(); b.push(10); - let mut c: SmallVec = SmallVec::new(); c.push(20); c.push(21); c.push(22); - a.clone_from(&b); assert_eq!(&*a, &[10]); - b.clone_from(&c); assert_eq!(&*b, &[20, 21, 22]); } - #[test] fn test_extract_if() { let mut a: SmallVec = smallvec![0, 1u8, 2, 3, 4, 5, 6, 7, 8, 0]; - let b: SmallVec = a.extract_if(1..9, |x| *x % 3 == 0).collect(); - assert_eq!(a, SmallVec::::from(&[0, 1u8, 2, 4, 5, 7, 8, 0])); assert_eq!(b, SmallVec::::from(&[3u8, 6])); } - -/// This assortment of tests, in combination with miri, verifies we handle UB on fishy arguments -/// given to SmallVec. Draining and extending the allocation are fairly well-tested earlier, but -/// `smallvec.insert(usize::MAX, val)` once slipped by! +/// This assortment of tests, in combination with miri, verifies we handle UB on +/// fishy arguments given to SmallVec. Draining and extending the allocation are +/// fairly well-tested earlier, but `smallvec.insert(usize::MAX, val)` once +/// slipped by! /// -/// All code that indexes into SmallVecs should be tested with such "trivially wrong" args. +/// All code that indexes into SmallVecs should be tested with such "trivially +/// wrong" args. #[test] fn max_dont_panic() { let mut sv: SmallVec = smallvec![0]; let _ = sv.get(usize::MAX); sv.truncate(usize::MAX); } - #[test] #[should_panic] fn max_remove() { let mut sv: SmallVec = smallvec![0]; sv.remove(usize::MAX); } - #[test] #[should_panic] fn max_swap_remove() { let mut sv: SmallVec = smallvec![0]; sv.swap_remove(usize::MAX); } - #[test] #[should_panic] fn max_insert() { let mut sv: SmallVec = smallvec![0]; sv.insert(usize::MAX, 0); } - #[test] fn collect_from_iter() { // Regression test for https://github.com/servo/rust-smallvec/issues/353 struct IterNoHint(I); - impl Iterator for IterNoHint { type Item = I::Item; fn next(&mut self) -> Option { self.0.next() } - - // no implementation of size_hint means it returns (0, None) - which forces from_iter to - // grow the allocated space iteratively. + // no implementation of size_hint means it returns (0, None) - which forces + // from_iter to grow the allocated space iteratively. } - - // A length of 3 is fine to trigger this bug under valgrind, but making the vector 1 million - // elements makes it crash - which is much easier to detect. + // A length of 3 is fine to trigger this bug under valgrind, but making the + // vector 1 million elements makes it crash - which is much easier to + // detect. let iter = IterNoHint(std::iter::repeat(1u8).take(1_000_000)); - let _y: SmallVec = SmallVec::from_iter(iter); } - #[test] fn test_collect_with_spill() { let input = "0123456"; let collected: SmallVec = input.chars().collect(); assert_eq!(collected, &['0', '1', '2', '3', '4', '5', '6']); } - #[test] fn test_spare_capacity_mut() { let mut v: SmallVec = SmallVec::new(); @@ -1011,55 +870,41 @@ fn test_spare_capacity_mut() { let spare = v.spare_capacity_mut(); assert_eq!(spare.len(), 2); assert_eq!(spare.as_ptr().cast::(), v.as_ptr()); - v.push(1); assert!(!v.spilled()); let spare = v.spare_capacity_mut(); assert_eq!(spare.len(), 1); assert_eq!(spare.as_ptr().cast::(), unsafe { v.as_ptr().add(1) }); - v.push(2); assert!(!v.spilled()); let spare = v.spare_capacity_mut(); assert_eq!(spare.len(), 0); assert_eq!(spare.as_ptr().cast::(), unsafe { v.as_ptr().add(2) }); - v.push(3); assert!(v.spilled()); let spare = v.spare_capacity_mut(); assert!(spare.len() >= 1); assert_eq!(spare.as_ptr().cast::(), unsafe { v.as_ptr().add(3) }); } - // Adopted from `tests/test_buf_mut.rs` in the `bytes` crate. #[cfg(feature = "bytes")] mod buf_mut { use bytes::BufMut as _; - type SmallVec = crate::SmallVec; - #[test] fn test_smallvec_as_mut_buf() { let mut buf = SmallVec::with_capacity(64); - assert_eq!(buf.remaining_mut(), isize::MAX as usize); - assert!(buf.chunk_mut().len() >= 64); - buf.put(&b"zomg"[..]); - assert_eq!(&buf, b"zomg"); - assert_eq!(buf.remaining_mut(), isize::MAX as usize - 4); assert_eq!(buf.capacity(), 64); - for _ in 0..16 { buf.put(&b"zomg"[..]); } - assert_eq!(buf.len(), 68); } - #[test] fn test_smallvec_put_bytes() { let mut buf = SmallVec::new(); @@ -1067,53 +912,45 @@ mod buf_mut { buf.put_bytes(19, 2); assert_eq!([17, 19, 19], &buf[..]); } - #[test] fn test_put_u8() { let mut buf = SmallVec::with_capacity(8); buf.put_u8(33); assert_eq!(b"\x21", &buf[..]); } - #[test] fn test_put_u16() { let mut buf = SmallVec::with_capacity(8); buf.put_u16(8532); assert_eq!(b"\x21\x54", &buf[..]); - buf.clear(); buf.put_u16_le(8532); assert_eq!(b"\x54\x21", &buf[..]); } - #[test] fn test_put_int() { let mut buf = SmallVec::with_capacity(8); buf.put_int(0x1020304050607080, 3); assert_eq!(b"\x60\x70\x80", &buf[..]); } - #[test] #[should_panic] fn test_put_int_nbytes_overflow() { let mut buf = SmallVec::with_capacity(8); buf.put_int(0x1020304050607080, 9); } - #[test] fn test_put_int_le() { let mut buf = SmallVec::with_capacity(8); buf.put_int_le(0x1020304050607080, 3); assert_eq!(b"\x80\x70\x60", &buf[..]); } - #[test] #[should_panic] fn test_put_int_le_nbytes_overflow() { let mut buf = SmallVec::with_capacity(8); buf.put_int_le(0x1020304050607080, 9); } - #[test] #[should_panic(expected = "advance out of bounds: the len is 8 but advancing by 12")] fn test_smallvec_advance_mut() { diff --git a/tests/macro.rs b/tests/macro.rs index a5d3a71f..66e183d5 100644 --- a/tests/macro.rs +++ b/tests/macro.rs @@ -4,19 +4,16 @@ #[test] fn smallvec() { let mut vec: smallvec::SmallVec; - macro_rules! check { ($init:tt) => { vec = smallvec::smallvec! $init; assert_eq!(*vec, *vec! $init); } } - check!([0; 0]); check!([1; 1]); check!([2; 2]); check!([3; 3]); - check!([]); check!([1]); check!([1, 2]);