diff --git a/diskann-wide/src/arch/emulated/mod.rs b/diskann-wide/src/arch/emulated/mod.rs index 283c9a720e..4c41b94389 100644 --- a/diskann-wide/src/arch/emulated/mod.rs +++ b/diskann-wide/src/arch/emulated/mod.rs @@ -133,7 +133,7 @@ impl Architecture for Scalar { // SAFETY: It's always safe to construct the `Scalar` architecture. Additionally, // since `Scalar` is a `Copy` zero-sized type, it is safe to wink into existence - // and is ABI compattible with `Hidden`. + // and is ABI compatible with `Hidden`. unsafe { arch::hide1(f) } } @@ -147,7 +147,7 @@ impl Architecture for Scalar { // SAFETY: It's always safe to construct the `Scalar` architecture. Additionally, // since `Scalar` is a `Copy` zero-sized type, it is safe to wink into existence - // and is ABI compattible with `Hidden`. + // and is ABI compatible with `Hidden`. unsafe { arch::hide2(f) } } @@ -163,7 +163,7 @@ impl Architecture for Scalar { // SAFETY: It's always safe to construct the `Scalar` architecture. Additionally, // since `Scalar` is a `Copy` zero-sized type, it is safe to wink into existence - // and is ABI compattible with `Hidden`. + // and is ABI compatible with `Hidden`. unsafe { arch::hide3(f) } } } diff --git a/diskann-wide/src/arch/mod.rs b/diskann-wide/src/arch/mod.rs index 820e391853..4b030c9a3e 100644 --- a/diskann-wide/src/arch/mod.rs +++ b/diskann-wide/src/arch/mod.rs @@ -190,7 +190,7 @@ //! The consequence of this is that we need to take an **unsafe** function pointer so we //! can dispatch call directly to the implementation. //! -//! Reason 2: Even if the above approach worked, the [`Architecture`] is sill present in the +//! Reason 2: Even if the above approach worked, the [`Architecture`] is still present in the //! signature of the `fn`, meaning we haven't really hidden the micro-architecture //! information. //! @@ -690,7 +690,7 @@ pub trait Architecture: sealed::Sealed { /// Run the provided closure targeting this architecture. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. fn run(self, f: F) -> R where F: Target; @@ -698,9 +698,9 @@ pub trait Architecture: sealed::Sealed { /// Run the provided closure targeting this architecture with an inlining hint. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. /// - /// Note that although an inline hint is applied, it is not a guaranteed that this call + /// Note that although an inline hint is applied, it is not guaranteed that this call /// will be inlined due to the interaction of `target_features`. If you really need `F` /// to be inlined, you can call its `Target` method directly, but care must be taken /// because this will not reapply `target_features`. @@ -711,7 +711,7 @@ pub trait Architecture: sealed::Sealed { /// Run the provided closure targeting this architecture with an additional argument. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. fn run1(self, f: F, x0: T0) -> R where F: Target1; @@ -720,9 +720,9 @@ pub trait Architecture: sealed::Sealed { /// an inlining hint. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. /// - /// Note that although an inline hint is applied, it is not a guaranteed that this call + /// Note that although an inline hint is applied, it is not guaranteed that this call /// will be inlined due to the interaction of `target_features`. If you really need `F` /// to be inlined, you can call its `Target1` method directly, but care must be taken /// because this will not reapply `target_features`. @@ -733,7 +733,7 @@ pub trait Architecture: sealed::Sealed { /// Run the provided closure targeting this architecture with two additional arguments. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. fn run2(self, f: F, x0: T0, x1: T1) -> R where F: Target2; @@ -742,9 +742,9 @@ pub trait Architecture: sealed::Sealed { /// and an inlining hint. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. /// - /// Note that although an inline hint is applied, it is not a guaranteed that this call + /// Note that although an inline hint is applied, it is not guaranteed that this call /// will be inlined due to the interaction of `target_features`. If you really need `F` /// to be inlined, you can call its `Target2` method directly, but care must be taken /// because this will not reapply `target_features`. @@ -755,7 +755,7 @@ pub trait Architecture: sealed::Sealed { /// Run the provided closure targeting this architecture with three additional arguments. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. fn run3(self, f: F, x0: T0, x1: T1, x2: T2) -> R where F: Target3; @@ -764,9 +764,9 @@ pub trait Architecture: sealed::Sealed { /// and an inlining hint. /// /// This function is always safe to call, but the function `f` likely needs to be - /// inlined into `run` in for the correct target features to be applied. + /// inlined into `run` for the correct target features to be applied. /// - /// Note that although an inline hint is applied, it is not a guaranteed that this call + /// Note that although an inline hint is applied, it is not guaranteed that this call /// will be inlined due to the interaction of `target_features`. If you really need `F` /// to be inlined, you can call its `Target3` method directly, but care must be taken /// because this will not reapply `target_features`. @@ -893,7 +893,7 @@ where /// A variation of [`Target1`] that uses an associated function instead of a method. /// -/// This is useful used in the function pointer API. +/// This is used in the function pointer API. pub trait FTarget1 where A: Architecture, @@ -903,7 +903,7 @@ where /// A variation of [`Target2`] that uses an associated function instead of a method. /// -/// This is useful used in the function pointer API. +/// This is used in the function pointer API. pub trait FTarget2 where A: Architecture, @@ -913,7 +913,7 @@ where /// A variation of [`Target3`] that uses an associated function instead of a method. /// -/// This is useful used in the function pointer API. +/// This is used in the function pointer API. pub trait FTarget3 where A: Architecture, @@ -994,7 +994,7 @@ const _ASSERT_ALIGNED: () = assert!( macro_rules! dispatched { ($name:ident, { $($Ts:ident )* }, { $($xs:ident )* }, { $($lt:lifetime )* }) => { /// A function pointer that calls directly into a micro-architecture optimized - /// function, returning a value of type `R` and accepting the speficied number of + /// function, returning a value of type `R` and accepting the specified number of /// arguments. /// /// Arguments are mapped using the [`AddLifetime`] trait to enable passing structs @@ -1065,7 +1065,7 @@ dispatched!(Dispatched1, { T0 }, { x0 }, { 'a0 }); dispatched!(Dispatched2, { T0 T1 }, { x0 x1 }, { 'a0 'a1 }); dispatched!(Dispatched3, { T0 T1 T2 }, { x0 x1 x2 }, { 'a0 'a1 'a2 }); -/// This macro stamps out the function-pointer tranmute trick we use to type-erase +/// This macro stamps out the function-pointer transmute trick we use to type-erase /// architecture in the function-pointer API. macro_rules! hide { ($name:ident, $dispatched:ident, { $($Ts:ident )* }) => { @@ -1082,9 +1082,9 @@ macro_rules! hide { /// We can do this because Rust guarantees that zero sized types are ABI /// compatible. /// - /// The caller must ensure that winking into existance and instance of `A` is + /// The caller must ensure that winking into existence an instance of `A` is /// a safe operation. For [`Architectures`], this means that the requirements - /// of `A::new()` are uphelf. + /// of `A::new()` are upheld. /// /// Put plainly: /// diff --git a/diskann-wide/src/arch/x86_64/algorithms.rs b/diskann-wide/src/arch/x86_64/algorithms.rs index 3807927053..fe8ff105a3 100644 --- a/diskann-wide/src/arch/x86_64/algorithms.rs +++ b/diskann-wide/src/arch/x86_64/algorithms.rs @@ -74,7 +74,7 @@ pub(crate) unsafe fn __load_first_of_16_bytes(arch: V3, ptr: *const u8, first: u // SAFETY: // * Pointer Cast: The instruction `_mm_loadu_si128` does not have any alignment // restrictions, so if `[ptr, ptr + first)` is valid, the cast will be valid. - // * `_mm_loadu_si128`: Use of the intrinsic is gated by the `cfg` macro. + // * `_mm_loadu_si128`: The intrinsic requires SSE2, implied by V3. // The load is valid since the caller passed a value greater than 16. // *`__m128i` and `u128` are both the same size, do not own any resources, and are // valid for all bit patterns. diff --git a/diskann-wide/src/arch/x86_64/common.rs b/diskann-wide/src/arch/x86_64/common.rs index 1192b739e3..7d875cc929 100644 --- a/diskann-wide/src/arch/x86_64/common.rs +++ b/diskann-wide/src/arch/x86_64/common.rs @@ -13,21 +13,21 @@ pub(crate) trait AllOnes { impl AllOnes for __m128i { fn all_ones() -> Self { - // SAFETY: Gated by CFG + // SAFETY: `_mm_set1_epi32` requires SSE2, which is baseline for x86_64. unsafe { _mm_set1_epi32(-1) } } } impl AllOnes for __m256i { fn all_ones() -> Self { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_set1_epi32` requires AVX, implied by the caller's architecture. unsafe { _mm256_set1_epi32(-1) } } } impl AllOnes for __m512i { fn all_ones() -> Self { - // SAFETY: Gated by CFG + // SAFETY: `_mm512_set1_epi32` requires AVX-512F, implied by the caller's architecture. unsafe { _mm512_set1_epi32(-1) } } } diff --git a/diskann-wide/src/arch/x86_64/macros.rs b/diskann-wide/src/arch/x86_64/macros.rs index a446027bd6..6d9fac31a6 100644 --- a/diskann-wide/src/arch/x86_64/macros.rs +++ b/diskann-wide/src/arch/x86_64/macros.rs @@ -65,7 +65,7 @@ macro_rules! x86_define_register { #[inline(always)] fn to_array(self) -> [$scalar; $lanes] { // SAFETY: Provided the scalar type is an integer or floating point, - // then all bit pattens are valid between source and destination types. + // then all bit patterns are valid between source and destination types. // (provided an x86 intrinsic is one of the transmuted types). // // The source argument is taken by value (no reference conversion) and @@ -79,7 +79,7 @@ macro_rules! x86_define_register { #[inline(always)] fn from_array(_: $arch, x: [$scalar; $lanes]) -> Self { // SAFETY: Provided the scalar type is an integer or floating point, - // then all bit pattens are valid between source and destination types. + // then all bit patterns are valid between source and destination types. // (provided an x86 intrinsic is one of the transmuted types). // // The source argument is taken by value (no reference conversion) and @@ -208,7 +208,7 @@ macro_rules! x86_retarget { /// Utility macro for defining `X86Splat`. /// /// SAFETY: It is the invoker's responsibility to ensure that the intrinsic is safe to call. -/// That is - any intrinsics invoked must be compatbiel with `$type`'s associated architecture. +/// That is - any intrinsics invoked must be compatible with `$type`'s associated architecture. macro_rules! x86_define_splat { ($type:ty, $intrinsic:expr, $requires:literal) => { impl X86Splat for $type { @@ -241,7 +241,7 @@ macro_rules! x86_define_splat { /// Utility macro for defining `X86Default`. /// /// SAFETY: It is the invoker's responsibility to ensure that the intrinsic is safe to call. -/// That is - any intrinsics invoked must be compatbiel with `$type`'s associated architecture. +/// That is - any intrinsics invoked must be compatible with `$type`'s associated architecture. macro_rules! x86_define_default { ($type:ty, $intrinsic:expr, $requires:literal) => { impl X86Default for $type { @@ -258,9 +258,9 @@ macro_rules! x86_define_default { } /// SAFETY: It is the invoker's responsibility to ensure that the provided intrinsics are -/// safe to call. T +/// safe to call. /// -/// hat is - any intrinsics invoked must be compatbiel with `$type`'s associated architecture. +/// That is - any intrinsics invoked must be compatible with `$type`'s associated architecture. macro_rules! x86_splitjoin { (__m512i, $type:path, $half:path) => { impl $crate::SplitJoin for $type { diff --git a/diskann-wide/src/arch/x86_64/mod.rs b/diskann-wide/src/arch/x86_64/mod.rs index f84d16f48f..1e15b66370 100644 --- a/diskann-wide/src/arch/x86_64/mod.rs +++ b/diskann-wide/src/arch/x86_64/mod.rs @@ -80,8 +80,9 @@ cfg_if::cfg_if! { // We cache a single enum and use it to indicate the version with the following meaning: // // 0: Uninitialized -// 1: V3 -// 2 and above: Scalar +// 1: Scalar +// 2: V3 +// 3: V4 static ARCH_NUMBER: AtomicU64 = AtomicU64::new(ARCH_UNINITIALIZED); // NOTE: Architecture must be properly nested in ascending order so compatibility checks @@ -445,7 +446,7 @@ mod tests { // These tests reach directly into the dispatch mechanism. // // There should only be a single test (this one) that does this, and all other tests - // involving dispatch should either be configured to work properly regarless of the + // involving dispatch should either be configured to work properly regardless of the // backend architecture, or be run in their own process. #[test] fn test_dispatch() { diff --git a/diskann-wide/src/arch/x86_64/v3/conversion.rs b/diskann-wide/src/arch/x86_64/v3/conversion.rs index cb71cc6c2c..e3212079e7 100644 --- a/diskann-wide/src/arch/x86_64/v3/conversion.rs +++ b/diskann-wide/src/arch/x86_64/v3/conversion.rs @@ -88,7 +88,7 @@ helpers::unsafe_map_conversion!(i8x16, i16x16, _mm256_cvtepi8_epi16, "avx2"); helpers::unsafe_map_conversion!(u8x16, i16x16, _mm256_cvtepu8_epi16, "avx2"); // i32 to f32 -helpers::unsafe_map_cast!(i32x8 => (f32, f32x8), _mm256_cvtepi32_ps, "avx2"); +helpers::unsafe_map_cast!(i32x8 => (f32, f32x8), _mm256_cvtepi32_ps, "avx"); ////////////////// // Reinterprets // diff --git a/diskann-wide/src/arch/x86_64/v3/f16x16_.rs b/diskann-wide/src/arch/x86_64/v3/f16x16_.rs index 70c790c58c..80500df604 100644 --- a/diskann-wide/src/arch/x86_64/v3/f16x16_.rs +++ b/diskann-wide/src/arch/x86_64/v3/f16x16_.rs @@ -46,7 +46,7 @@ impl X86Splat for f16x16 { // (1) .to_bits() -> Returns the underlying `u16` from the `f16`. // (2) as i16 -> Bit-cast to `i16` to give to the intrinsic. // - // SAFETY: `_mm256_set1_epi` requires AVX - implied by V3. + // SAFETY: `_mm256_set1_epi16` requires AVX - implied by V3. Self(unsafe { _mm256_set1_epi16(value.to_bits() as i16) }) } } @@ -56,7 +56,7 @@ impl X86LoadStore for f16x16 { unsafe fn load_simd(_: V3, ptr: *const f16) -> Self { // SAFETY: Pointer access guaranteed by caller. // - // `_mm256_loadu_si256` requires AVX - implied by V4. + // `_mm256_loadu_si256` requires AVX - implied by V3. Self(unsafe { _mm256_loadu_si256(ptr as *const Self::Underlying) }) } @@ -80,7 +80,7 @@ impl X86LoadStore for f16x16 { unsafe fn store_simd(self, ptr: *mut f16) { // SAFETY: Pointer access guaranteed by caller. // - // `_mm256_storeu_si256` requires AVX - implied by V4. + // `_mm256_storeu_si256` requires AVX - implied by V3. unsafe { _mm256_storeu_si256(ptr as *mut Self::Underlying, self.to_underlying()) } } diff --git a/diskann-wide/src/arch/x86_64/v3/f16x8_.rs b/diskann-wide/src/arch/x86_64/v3/f16x8_.rs index 793c5a71c1..606deb64cb 100644 --- a/diskann-wide/src/arch/x86_64/v3/f16x8_.rs +++ b/diskann-wide/src/arch/x86_64/v3/f16x8_.rs @@ -38,8 +38,7 @@ impl X86Splat for f16x8 { // (1) .to_bits() -> Returns the underlying `u16` from the `f16`. // (2) as i16 -> Bit-cast to `i16` to give to the intrinsic. // - // SAFETY: Safe invocation of this function is gated by the CFG macro conditionally - // compiling this implementation. + // SAFETY: `_mm_set1_epi16` requires SSE2, implied by V3. Self(unsafe { _mm_set1_epi16(value.to_bits() as i16) }) } } diff --git a/diskann-wide/src/arch/x86_64/v3/f32x4_.rs b/diskann-wide/src/arch/x86_64/v3/f32x4_.rs index d9905e531f..60ffa4477c 100644 --- a/diskann-wide/src/arch/x86_64/v3/f32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v3/f32x4_.rs @@ -35,7 +35,8 @@ helpers::unsafe_map_binary_op!(f32x4, std::ops::Mul, mul, _mm_mul_ps, "sse"); impl f32x4 { #[inline(always)] fn is_nan(self) -> mask32x4 { - // NOTE: `_CMP_UNORD_Q` returns `true` only if both arguments are NAN. + // NOTE: `_CMP_UNORD_Q` returns `true` if either argument is NaN. Since we compare + // `self` with `self`, this returns `true` exactly when `self` is NaN. mask32x4::from_underlying( self.arch(), // SAFETY: `_mm_castps_si128` requires SSE2 and `_mm_cmp_ps` requires AVX, @@ -133,14 +134,14 @@ impl X86LoadStore for f32x4 { impl SIMDPartialEq for f32x4 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_castps_si128` and `_mm_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm_castps_si128(_mm_cmp_ps(self.0, other.0, _CMP_EQ_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_castps_si128` and `_mm_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm_castps_si128(_mm_cmp_ps(self.0, other.0, _CMP_NEQ_UQ)) }; Self::Mask::from_underlying(self.arch(), m) } @@ -149,28 +150,28 @@ impl SIMDPartialEq for f32x4 { impl SIMDPartialOrd for f32x4 { #[inline(always)] fn lt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm_castps_si128` and `_mm_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm_castps_si128(_mm_cmp_ps(self.0, other.0, _CMP_LT_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm_castps_si128` and `_mm_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm_castps_si128(_mm_cmp_ps(self.0, other.0, _CMP_LE_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn gt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm_castps_si128` and `_mm_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm_castps_si128(_mm_cmp_ps(self.0, other.0, _CMP_GT_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn ge_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm_castps_si128` and `_mm_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm_castps_si128(_mm_cmp_ps(self.0, other.0, _CMP_GE_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } @@ -180,7 +181,7 @@ impl SIMDSumTree for f32x4 { #[inline(always)] fn sum_tree(self) -> f32 { let x = self.to_underlying(); - // SAFETY: Gated by CFG. + // SAFETY: These intrinsics require SSE, implied by V3. unsafe { // loDual = ( -, -, x1, x0 ) let lo_dual = x; diff --git a/diskann-wide/src/arch/x86_64/v3/f32x8_.rs b/diskann-wide/src/arch/x86_64/v3/f32x8_.rs index e135e6caa6..054b249e8f 100644 --- a/diskann-wide/src/arch/x86_64/v3/f32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v3/f32x8_.rs @@ -37,7 +37,8 @@ helpers::unsafe_map_binary_op!(f32x8, std::ops::Mul, mul, _mm256_mul_ps, "avx"); impl f32x8 { #[inline(always)] fn is_nan(self) -> mask32x8 { - // NOTE: `_CMP_UNORD_Q` returns `true` only if both arguments are NAN. + // NOTE: `_CMP_UNORD_Q` returns `true` if either argument is NaN. Since we compare + // `self` with `self`, this returns `true` exactly when `self` is NaN. mask32x8::from_underlying( self.arch(), // SAFETY: `_mm256_castps_si256` and `_mm256_cmp_ps` requires AVX, which is @@ -138,14 +139,14 @@ impl X86LoadStore for f32x8 { impl SIMDPartialEq for f32x8 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_castps_si256` and `_mm256_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm256_castps_si256(_mm256_cmp_ps(self.0, other.0, _CMP_EQ_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_castps_si256` and `_mm256_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm256_castps_si256(_mm256_cmp_ps(self.0, other.0, _CMP_NEQ_UQ)) }; Self::Mask::from_underlying(self.arch(), m) } @@ -154,28 +155,28 @@ impl SIMDPartialEq for f32x8 { impl SIMDPartialOrd for f32x8 { #[inline(always)] fn lt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm256_castps_si256` and `_mm256_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm256_castps_si256(_mm256_cmp_ps(self.0, other.0, _CMP_LT_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm256_castps_si256` and `_mm256_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm256_castps_si256(_mm256_cmp_ps(self.0, other.0, _CMP_LE_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn gt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm256_castps_si256` and `_mm256_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm256_castps_si256(_mm256_cmp_ps(self.0, other.0, _CMP_GT_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn ge_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG. + // SAFETY: `_mm256_castps_si256` and `_mm256_cmp_ps` require AVX, implied by V3. let m = unsafe { _mm256_castps_si256(_mm256_cmp_ps(self.0, other.0, _CMP_GE_OQ)) }; Self::Mask::from_underlying(self.arch(), m) } @@ -185,7 +186,9 @@ impl SIMDSumTree for f32x8 { #[inline(always)] fn sum_tree(self) -> f32 { let x = self.to_underlying(); - // SAFETY: Gated by CFG. + // SAFETY: `_mm256_extractf128_ps` and `_mm256_castps256_ps128` require AVX; + // `_mm_add_ps`, `_mm_movehl_ps`, `_mm_shuffle_ps`, `_mm_add_ss`, and + // `_mm_cvtss_f32` require SSE, implied by V3. unsafe { // hiQuad = ( x7, x6, x5, x4 ) let hi_quad = _mm256_extractf128_ps(x, 1); diff --git a/diskann-wide/src/arch/x86_64/v3/i16x32_.rs b/diskann-wide/src/arch/x86_64/v3/i16x32_.rs index bf68cff6f9..8174f28184 100644 --- a/diskann-wide/src/arch/x86_64/v3/i16x32_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i16x32_.rs @@ -7,9 +7,9 @@ use super::{i8x32, i16x16, u8x32}; use crate::{LoHi, SplitJoin, doubled}; -//////////////////// -// 16-bit integer // -//////////////////// +/////////////////// +// 16-bit signed // +/////////////////// doubled::double_vector!(i16, 32, i16x16); doubled::double_scalar_shift!(Doubled); diff --git a/diskann-wide/src/arch/x86_64/v3/i16x8_.rs b/diskann-wide/src/arch/x86_64/v3/i16x8_.rs index 2279f55465..9fb85bd845 100644 --- a/diskann-wide/src/arch/x86_64/v3/i16x8_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i16x8_.rs @@ -35,7 +35,7 @@ macros::x86_define_default!(i16x8, _mm_setzero_si128, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::Add, add, _mm_add_epi16, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::Sub, sub, _mm_sub_epi16, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::Mul, mul, _mm_mullo_epi16, "sse2"); -helpers::unsafe_map_unary_op!(i16x8, SIMDAbs, abs_simd, _mm_abs_epi16, "sse3"); +helpers::unsafe_map_unary_op!(i16x8, SIMDAbs, abs_simd, _mm_abs_epi16, "ssse3"); helpers::unsafe_map_binary_op!(i16x8, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::BitOr, bitor, _mm_or_si128, "sse2"); diff --git a/diskann-wide/src/arch/x86_64/v3/i32x16_.rs b/diskann-wide/src/arch/x86_64/v3/i32x16_.rs index 11be4b6a10..c355ce9a22 100644 --- a/diskann-wide/src/arch/x86_64/v3/i32x16_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i32x16_.rs @@ -7,9 +7,9 @@ use super::i32x8; use crate::doubled; -///////////////////// -// 32-bit floating // -///////////////////// +/////////////////// +// 32-bit signed // +/////////////////// doubled::double_vector!(i32, 16, i32x8); doubled::double_scalar_shift!(Doubled); diff --git a/diskann-wide/src/arch/x86_64/v3/i32x4_.rs b/diskann-wide/src/arch/x86_64/v3/i32x4_.rs index b2e6bb081f..191d38e49c 100644 --- a/diskann-wide/src/arch/x86_64/v3/i32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i32x4_.rs @@ -19,7 +19,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit signed integer ///// macros::x86_define_register!(i32x4, __m128i, mask32x4, i32, 4, V3); @@ -29,7 +29,7 @@ macros::x86_define_default!(i32x4, _mm_setzero_si128, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::Add, add, _mm_add_epi32, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::Sub, sub, _mm_sub_epi32, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::Mul, mul, _mm_mullo_epi32, "sse4.1"); -helpers::unsafe_map_unary_op!(i32x4, SIMDAbs, abs_simd, _mm_abs_epi32, "sse3"); +helpers::unsafe_map_unary_op!(i32x4, SIMDAbs, abs_simd, _mm_abs_epi32, "ssse3"); helpers::unsafe_map_binary_op!(i32x4, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::BitOr, bitor, _mm_or_si128, "sse2"); @@ -56,13 +56,13 @@ impl SIMDMulAdd for i32x4 { impl SIMDPartialEq for i32x4 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi32` requires SSE2, implied by V3. Self::Mask::from_underlying(self.arch(), unsafe { _mm_cmpeq_epi32(self.0, other.0) }) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi32` and `_mm_xor_si128` require SSE2, implied by V3. let m = unsafe { _mm_xor_si128(_mm_cmpeq_epi32(self.0, other.0), __m128i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) } @@ -71,13 +71,13 @@ impl SIMDPartialEq for i32x4 { impl SIMDPartialOrd for i32x4 { #[inline(always)] fn lt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpgt_epi32` requires SSE2, implied by V3. Self::Mask::from_underlying(self.arch(), unsafe { _mm_cmpgt_epi32(other.0, self.0) }) } #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi32` requires SSE2 and `_mm_min_epi32` requires SSE4.1, implied by V3. let m = unsafe { _mm_cmpeq_epi32(self.0, _mm_min_epi32(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) } diff --git a/diskann-wide/src/arch/x86_64/v3/i32x8_.rs b/diskann-wide/src/arch/x86_64/v3/i32x8_.rs index 5bb5f00b34..409d9d1ab6 100644 --- a/diskann-wide/src/arch/x86_64/v3/i32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i32x8_.rs @@ -23,7 +23,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit signed integer ///// macros::x86_define_register!(i32x8, __m256i, mask32x8, i32, 8, V3); @@ -40,7 +40,7 @@ macros::x86_splitjoin!( helpers::unsafe_map_binary_op!(i32x8, std::ops::Add, add, _mm256_add_epi32, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::Sub, sub, _mm256_sub_epi32, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::Mul, mul, _mm256_mullo_epi32, "avx2"); -helpers::unsafe_map_unary_op!(i32x8, SIMDAbs, abs_simd, _mm256_abs_epi32, "sse3"); +helpers::unsafe_map_unary_op!(i32x8, SIMDAbs, abs_simd, _mm256_abs_epi32, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::BitAnd, bitand, _mm256_and_si256, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::BitOr, bitor, _mm256_or_si256, "avx2"); @@ -108,13 +108,13 @@ impl X86LoadStore for i32x8 { impl SIMDPartialEq for i32x8 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_cmpeq_epi32` requires AVX2, implied by V3. Self::Mask::from_underlying(self.arch(), unsafe { _mm256_cmpeq_epi32(self.0, other.0) }) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_xor_si256` and `_mm256_cmpeq_epi32` require AVX2, implied by V3. let m = unsafe { _mm256_xor_si256(_mm256_cmpeq_epi32(self.0, other.0), __m256i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) @@ -124,13 +124,13 @@ impl SIMDPartialEq for i32x8 { impl SIMDPartialOrd for i32x8 { #[inline(always)] fn lt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_cmpgt_epi32` requires AVX2, implied by V3. Self::Mask::from_underlying(self.arch(), unsafe { _mm256_cmpgt_epi32(other.0, self.0) }) } #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_cmpeq_epi32` and `_mm256_min_epi32` require AVX2, implied by V3. let m = unsafe { _mm256_cmpeq_epi32(self.0, _mm256_min_epi32(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) } @@ -187,7 +187,7 @@ impl SIMDDotProduct for i32x8 { fn dot_simd(self, left: i16x16, right: i16x16) -> Self { self + Self::from_underlying( self.arch(), - // SAFETY: Gated by CFG. + // SAFETY: `_mm256_madd_epi16` requires AVX2, implied by V3. unsafe { _mm256_madd_epi16(left.to_underlying(), right.to_underlying()) }, ) } diff --git a/diskann-wide/src/arch/x86_64/v3/i8x16_.rs b/diskann-wide/src/arch/x86_64/v3/i8x16_.rs index 9f7acd4648..779b0efbb1 100644 --- a/diskann-wide/src/arch/x86_64/v3/i8x16_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i8x16_.rs @@ -33,7 +33,7 @@ macros::x86_define_default!(i8x16, _mm_setzero_si128, "sse2"); helpers::unsafe_map_binary_op!(i8x16, std::ops::Add, add, _mm_add_epi8, "sse2"); helpers::unsafe_map_binary_op!(i8x16, std::ops::Sub, sub, _mm_sub_epi8, "sse2"); -helpers::unsafe_map_unary_op!(i8x16, SIMDAbs, abs_simd, _mm_abs_epi8, "sse3"); +helpers::unsafe_map_unary_op!(i8x16, SIMDAbs, abs_simd, _mm_abs_epi8, "ssse3"); impl std::ops::Mul for i8x16 { type Output = Self; @@ -147,13 +147,13 @@ impl X86LoadStore for i8x16 { impl SIMDPartialEq for i8x16 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: The presence of `Self` attest the intrinsic can be used. + // SAFETY: The presence of `Self` attests the intrinsic can be used. Self::Mask::from_underlying(self.arch(), unsafe { _mm_cmpeq_epi8(self.0, other.0) }) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: The presence of `Self` attest the intrinsic can be used. + // SAFETY: The presence of `Self` attests the intrinsic can be used. let m = unsafe { _mm_xor_si128(_mm_cmpeq_epi8(self.0, other.0), __m128i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) } @@ -162,13 +162,13 @@ impl SIMDPartialEq for i8x16 { impl SIMDPartialOrd for i8x16 { #[inline(always)] fn lt_simd(self, other: Self) -> Self::Mask { - // SAFETY: The presence of `Self` attest the intrinsic can be used. + // SAFETY: The presence of `Self` attests the intrinsic can be used. Self::Mask::from_underlying(self.arch(), unsafe { _mm_cmpgt_epi8(other.0, self.0) }) } #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: The presence of `Self` attest the intrinsic can be used. + // SAFETY: The presence of `Self` attests the intrinsic can be used. let m = unsafe { _mm_cmpeq_epi8(self.0, _mm_min_epi8(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) } diff --git a/diskann-wide/src/arch/x86_64/v3/i8x32_.rs b/diskann-wide/src/arch/x86_64/v3/i8x32_.rs index 8cd4b3763b..8964288a4c 100644 --- a/diskann-wide/src/arch/x86_64/v3/i8x32_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i8x32_.rs @@ -23,9 +23,9 @@ use crate::{ traits::{AsSIMD, SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDVector}, }; -//////////////////// -// 8-bit unsigned // -//////////////////// +////////////////// +// 8-bit signed // +////////////////// macros::x86_define_register!(i8x32, __m256i, mask8x32, i8, 32, V3); macros::x86_define_splat!(i8x32 as i8, _mm256_set1_epi8, "avx"); @@ -150,7 +150,7 @@ impl SIMDPartialOrd for i8x32 { fn lt_simd(self, other: Self) -> Self::Mask { // Check that each lane in `self` is not equal to the element-wise maximum. // - // SAFETY: The intrinsics `_mm256_max_epi8`, `_mm256_empeq_epi8`, and + // SAFETY: The intrinsics `_mm256_max_epi8`, `_mm256_cmpeq_epi8`, and // `_mm256_xor_si256` require AVX2 - all of which are implied by `V3`. let m = unsafe { let max = _mm256_max_epi8(self.0, other.0); @@ -163,7 +163,7 @@ impl SIMDPartialOrd for i8x32 { fn le_simd(self, other: Self) -> Self::Mask { // Check that each lane in `self` is not equal to the element-wise maximum. // - // SAFETY: The intrinsics `_mm256_min_epi8` and `_mm256_empeq_epi8` require AVX2 + // SAFETY: The intrinsics `_mm256_min_epi8` and `_mm256_cmpeq_epi8` require AVX2 // - implied by `V3`. let m = unsafe { _mm256_cmpeq_epi8(self.0, _mm256_min_epi8(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) diff --git a/diskann-wide/src/arch/x86_64/v3/i8x64_.rs b/diskann-wide/src/arch/x86_64/v3/i8x64_.rs index e7bc50530c..b06cfd3cc2 100644 --- a/diskann-wide/src/arch/x86_64/v3/i8x64_.rs +++ b/diskann-wide/src/arch/x86_64/v3/i8x64_.rs @@ -7,9 +7,9 @@ use super::i8x32; use crate::doubled; -/////////////////// -// 8-bit integer // -/////////////////// +////////////////// +// 8-bit signed // +////////////////// doubled::double_vector!(i8, 64, i8x32); doubled::double_scalar_shift!(Doubled); diff --git a/diskann-wide/src/arch/x86_64/v3/masks.rs b/diskann-wide/src/arch/x86_64/v3/masks.rs index 8b2c7eb14e..c9a5288d3d 100644 --- a/diskann-wide/src/arch/x86_64/v3/masks.rs +++ b/diskann-wide/src/arch/x86_64/v3/masks.rs @@ -300,7 +300,7 @@ impl SIMDMask for mask32x4 { #[inline(always)] fn arch(self) -> V3 { - // SAFETY: The existence `Self` proves its architecture is safe. + // SAFETY: The existence of `Self` proves its architecture is safe. unsafe { V3::new() } } @@ -392,7 +392,7 @@ impl SIMDMask for mask32x8 { #[inline(always)] fn arch(self) -> V3 { - // SAFETY: The existence `Self` proves its architecture is safe. + // SAFETY: The existence of `Self` proves its architecture is safe. unsafe { V3::new() } } @@ -495,7 +495,7 @@ impl SIMDMask for mask64x2 { #[inline(always)] fn arch(self) -> V3 { - // SAFETY: The existence `Self` proves its architecture is safe. + // SAFETY: The existence of `Self` proves its architecture is safe. unsafe { V3::new() } } @@ -586,7 +586,7 @@ impl SIMDMask for mask64x4 { #[inline(always)] fn arch(self) -> V3 { - // SAFETY: The existence `Self` proves its architecture is safe. + // SAFETY: The existence of `Self` proves its architecture is safe. unsafe { V3::new() } } diff --git a/diskann-wide/src/arch/x86_64/v3/mod.rs b/diskann-wide/src/arch/x86_64/v3/mod.rs index 29d1d1d8ab..1219c02dc0 100644 --- a/diskann-wide/src/arch/x86_64/v3/mod.rs +++ b/diskann-wide/src/arch/x86_64/v3/mod.rs @@ -159,7 +159,7 @@ impl V3 { // just check if the user specified level would have instantiated it. if unsafe { Self::new_checked_with(super::test_arch_number()) }.is_some() { panic!( - "V4 architecture was requested but is not compatible on the current hardare" + "V3 architecture was requested but is not compatible on the current hardware" ); } else { None @@ -170,7 +170,7 @@ impl V3 { /// # Safety /// - /// The architecture number `version` can only be `ARCH_V3` is runtime checks have + /// The architecture number `version` can only be `ARCH_V3` if runtime checks have /// verified that it is safe to do so. unsafe fn new_checked_with(version: u64) -> Option { if version >= super::ARCH_V3 { @@ -434,9 +434,9 @@ impl Architecture for V3 { { let f: unsafe fn(Self, T0::Of<'_>) -> R = Self::run_function_with_1::; - // SAFETY: The present of `self` as an argument attests that it is safe to construct + // SAFETY: The presence of `self` as an argument attests that it is safe to construct // A `V3` architecture. Additionally, since `V3` is a `Copy` zero-sized type, - // it is safe to wink into existence and is ABI compattible with `Hidden`. + // it is safe to wink into existence and is ABI compatible with `Hidden`. unsafe { arch::hide1(f) } } @@ -449,9 +449,9 @@ impl Architecture for V3 { let f: unsafe fn(Self, T0::Of<'_>, T1::Of<'_>) -> R = Self::run_function_with_2::; - // SAFETY: The present of `self` as an argument attests that it is safe to construct + // SAFETY: The presence of `self` as an argument attests that it is safe to construct // A `V3` architecture. Additionally, since `V3` is a `Copy` zero-sized type, - // it is safe to wink into existence and is ABI compattible with `Hidden`. + // it is safe to wink into existence and is ABI compatible with `Hidden`. unsafe { arch::hide2(f) } } @@ -465,9 +465,9 @@ impl Architecture for V3 { let f: unsafe fn(Self, T0::Of<'_>, T1::Of<'_>, T2::Of<'_>) -> R = Self::run_function_with_3::; - // SAFETY: The present of `self` as an argument attests that it is safe to construct + // SAFETY: The presence of `self` as an argument attests that it is safe to construct // A `V3` architecture. Additionally, since `V3` is a `Copy` zero-sized type, - // it is safe to wink into existence and is ABI compattible with `Hidden`. + // it is safe to wink into existence and is ABI compatible with `Hidden`. unsafe { arch::hide3(f) } } } diff --git a/diskann-wide/src/arch/x86_64/v3/u32x16_.rs b/diskann-wide/src/arch/x86_64/v3/u32x16_.rs index 9be2e494af..f39105d90c 100644 --- a/diskann-wide/src/arch/x86_64/v3/u32x16_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u32x16_.rs @@ -8,7 +8,7 @@ use super::u32x8; use crate::doubled; ///////////////////// -// 32-bit floating // +// 32-bit unsigned // ///////////////////// doubled::double_vector!(u32, 16, u32x8); diff --git a/diskann-wide/src/arch/x86_64/v3/u32x4_.rs b/diskann-wide/src/arch/x86_64/v3/u32x4_.rs index 4de58555dd..02963076e4 100644 --- a/diskann-wide/src/arch/x86_64/v3/u32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u32x4_.rs @@ -21,7 +21,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit unsigned integer ///// macros::x86_define_register!(u32x4, __m128i, mask32x4, u32, 4, V3); @@ -96,13 +96,13 @@ impl X86LoadStore for u32x4 { impl SIMDPartialEq for u32x4 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi32` requires SSE2, implied by V3. Self::Mask::from_underlying(self.arch(), unsafe { _mm_cmpeq_epi32(self.0, other.0) }) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi32` and `_mm_xor_si128` require SSE2, implied by V3. let m = unsafe { _mm_xor_si128(_mm_cmpeq_epi32(self.0, other.0), __m128i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) } @@ -111,7 +111,7 @@ impl SIMDPartialEq for u32x4 { impl SIMDPartialOrd for u32x4 { #[inline(always)] fn lt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_max_epu32` requires SSE4.1 and `_mm_cmpeq_epi32` and `_mm_xor_si128` require SSE2, implied by V3. let m = unsafe { let max = _mm_max_epu32(self.0, other.0); _mm_xor_si128(_mm_cmpeq_epi32(self.0, max), __m128i::all_ones()) @@ -121,7 +121,7 @@ impl SIMDPartialOrd for u32x4 { #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi32` requires SSE2 and `_mm_min_epu32` requires SSE4.1, implied by V3. let m = unsafe { _mm_cmpeq_epi32(self.0, _mm_min_epu32(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) } diff --git a/diskann-wide/src/arch/x86_64/v3/u32x8_.rs b/diskann-wide/src/arch/x86_64/v3/u32x8_.rs index 17e6d22b78..744c2b7533 100644 --- a/diskann-wide/src/arch/x86_64/v3/u32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u32x8_.rs @@ -21,7 +21,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit unsigned integer ///// macros::x86_define_register!(u32x8, __m256i, mask32x8, u32, 8, V3); @@ -105,13 +105,13 @@ impl X86LoadStore for u32x8 { impl SIMDPartialEq for u32x8 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_cmpeq_epi32` requires AVX2, implied by V3. Self::Mask::from_underlying(self.arch(), unsafe { _mm256_cmpeq_epi32(self.0, other.0) }) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_xor_si256` and `_mm256_cmpeq_epi32` require AVX2, implied by V3. let m = unsafe { _mm256_xor_si256(_mm256_cmpeq_epi32(self.0, other.0), __m256i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) @@ -121,7 +121,7 @@ impl SIMDPartialEq for u32x8 { impl SIMDPartialOrd for u32x8 { #[inline(always)] fn lt_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_max_epu32`, `_mm256_cmpeq_epi32`, and `_mm256_xor_si256` require AVX2, implied by V3. let m = unsafe { let max = _mm256_max_epu32(self.0, other.0); _mm256_xor_si256(_mm256_cmpeq_epi32(self.0, max), __m256i::all_ones()) @@ -131,7 +131,7 @@ impl SIMDPartialOrd for u32x8 { #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_cmpeq_epi32` and `_mm256_min_epu32` require AVX2, implied by V3. let m = unsafe { _mm256_cmpeq_epi32(self.0, _mm256_min_epu32(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) } diff --git a/diskann-wide/src/arch/x86_64/v3/u64x2_.rs b/diskann-wide/src/arch/x86_64/v3/u64x2_.rs index 58e09769b7..6e1ae7cb11 100644 --- a/diskann-wide/src/arch/x86_64/v3/u64x2_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u64x2_.rs @@ -100,14 +100,14 @@ impl X86LoadStore for u64x2 { impl SIMDPartialEq for u64x2 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi64` requires SSE4.1, implied by V3. let m = unsafe { _mm_cmpeq_epi64(self.0, other.0) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_xor_si128` requires SSE2 and `_mm_cmpeq_epi64` requires SSE4.1, implied by V3. let m = unsafe { _mm_xor_si128(_mm_cmpeq_epi64(self.0, other.0), __m128i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) } diff --git a/diskann-wide/src/arch/x86_64/v3/u64x4_.rs b/diskann-wide/src/arch/x86_64/v3/u64x4_.rs index c895462f56..1e16b458c5 100644 --- a/diskann-wide/src/arch/x86_64/v3/u64x4_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u64x4_.rs @@ -107,14 +107,14 @@ impl X86LoadStore for u64x4 { impl SIMDPartialEq for u64x4 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_cmpeq_epi64` requires AVX2, implied by V3. let m = unsafe { _mm256_cmpeq_epi64(self.0, other.0) }; Self::Mask::from_underlying(self.arch(), m) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm256_xor_si256` and `_mm256_cmpeq_epi64` require AVX2, implied by V3. let m = unsafe { _mm256_xor_si256(_mm256_cmpeq_epi64(self.0, other.0), __m256i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) diff --git a/diskann-wide/src/arch/x86_64/v3/u8x16_.rs b/diskann-wide/src/arch/x86_64/v3/u8x16_.rs index 5034a9c759..50d6e7a33b 100644 --- a/diskann-wide/src/arch/x86_64/v3/u8x16_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u8x16_.rs @@ -165,7 +165,7 @@ impl SIMDPartialOrd for u8x16 { fn lt_simd(self, other: Self) -> Self::Mask { // Check that each lane in `self` is not equal to the element-wise maximum. // - // SAFETY: Gated by CFG + // SAFETY: `_mm_max_epu8`, `_mm_cmpeq_epi8`, and `_mm_xor_si128` require SSE2, implied by V3. let m = unsafe { let max = _mm_max_epu8(self.0, other.0); _mm_xor_si128(_mm_cmpeq_epi8(self.0, max), __m128i::all_ones()) @@ -175,7 +175,7 @@ impl SIMDPartialOrd for u8x16 { #[inline(always)] fn le_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi8` and `_mm_min_epu8` require SSE2, implied by V3. let m = unsafe { _mm_cmpeq_epi8(self.0, _mm_min_epu8(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) } @@ -184,13 +184,13 @@ impl SIMDPartialOrd for u8x16 { impl SIMDPartialEq for u8x16 { #[inline(always)] fn eq_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_cmpeq_epi8` requires SSE2, implied by V3. Self::Mask::from_underlying(self.arch(), unsafe { _mm_cmpeq_epi8(self.0, other.0) }) } #[inline(always)] fn ne_simd(self, other: Self) -> Self::Mask { - // SAFETY: Gated by CFG + // SAFETY: `_mm_xor_si128` and `_mm_cmpeq_epi8` require SSE2, implied by V3. let m = unsafe { _mm_xor_si128(_mm_cmpeq_epi8(self.0, other.0), __m128i::all_ones()) }; Self::Mask::from_underlying(self.arch(), m) } diff --git a/diskann-wide/src/arch/x86_64/v3/u8x32_.rs b/diskann-wide/src/arch/x86_64/v3/u8x32_.rs index ae8ddcc4ef..4ad808567b 100644 --- a/diskann-wide/src/arch/x86_64/v3/u8x32_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u8x32_.rs @@ -119,7 +119,7 @@ impl X86LoadStore for u8x32 { unsafe fn store_simd(self, ptr: *mut u8) { // SAFETY: The pointer access is guaranteed by the caller. // - // `_mm256_storeuu_si256` requires AVX - implied by `V3`. + // `_mm256_storeu_si256` requires AVX - implied by `V3`. unsafe { _mm256_storeu_si256(ptr.cast::<__m256i>(), self.to_underlying()) } } @@ -149,7 +149,7 @@ impl SIMDPartialOrd for u8x32 { fn lt_simd(self, other: Self) -> Self::Mask { // Check that each lane in `self` is not equal to the element-wise maximum. // - // SAFETY: The intrinsics `_mm256_max_epu8`, `_mm256_empeq_epi8`, and + // SAFETY: The intrinsics `_mm256_max_epu8`, `_mm256_cmpeq_epi8`, and // `_mm256_xor_si256` require AVX2 - all of which are implied by `V3`. let m = unsafe { let max = _mm256_max_epu8(self.0, other.0); @@ -162,7 +162,7 @@ impl SIMDPartialOrd for u8x32 { fn le_simd(self, other: Self) -> Self::Mask { // Check that each lane in `self` is not equal to the element-wise maximum. // - // SAFETY: The intrinsics `_mm256_min_epu8` and `_mm256_empeq_epi8` require AVX2 + // SAFETY: The intrinsics `_mm256_min_epu8` and `_mm256_cmpeq_epi8` require AVX2 // - implied by `V3`. let m = unsafe { _mm256_cmpeq_epi8(self.0, _mm256_min_epu8(self.0, other.0)) }; Self::Mask::from_underlying(self.arch(), m) diff --git a/diskann-wide/src/arch/x86_64/v3/u8x64_.rs b/diskann-wide/src/arch/x86_64/v3/u8x64_.rs index 1233ddddbd..7df0f05292 100644 --- a/diskann-wide/src/arch/x86_64/v3/u8x64_.rs +++ b/diskann-wide/src/arch/x86_64/v3/u8x64_.rs @@ -7,9 +7,9 @@ use super::u8x32; use crate::doubled; -/////////////////// -// 8-bit integer // -/////////////////// +//////////////////// +// 8-bit unsigned // +//////////////////// doubled::double_vector!(u8, 64, u8x32); doubled::double_scalar_shift!(Doubled); diff --git a/diskann-wide/src/arch/x86_64/v4/conversion.rs b/diskann-wide/src/arch/x86_64/v4/conversion.rs index 15800df362..4f6c4ae736 100644 --- a/diskann-wide/src/arch/x86_64/v4/conversion.rs +++ b/diskann-wide/src/arch/x86_64/v4/conversion.rs @@ -77,7 +77,7 @@ helpers::unsafe_map_conversion!(u8x16, i16x16, _mm256_cvtepu8_epi16, "avx2"); helpers::unsafe_map_conversion!(u8x32, i16x32, _mm512_cvtepu8_epi16, "avx512bw"); // i32 to f32 -helpers::unsafe_map_cast!(i32x8 => (f32, f32x8), _mm256_cvtepi32_ps, "avx2"); +helpers::unsafe_map_cast!(i32x8 => (f32, f32x8), _mm256_cvtepi32_ps, "avx"); helpers::unsafe_map_cast!(i16x16 => (u8, u8x16), _mm256_cvtepi16_epi8, "avx512bw,avx512vl"); helpers::unsafe_map_cast!(i16x16 => (i8, i8x16), _mm256_cvtepi16_epi8, "avx512bw,avx512vl"); diff --git a/diskann-wide/src/arch/x86_64/v4/f16x16_.rs b/diskann-wide/src/arch/x86_64/v4/f16x16_.rs index 989522c3f5..cbb2614b9d 100644 --- a/diskann-wide/src/arch/x86_64/v4/f16x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/f16x16_.rs @@ -25,7 +25,7 @@ use crate::{ ///////////////////// macros::x86_define_register!(f16x16, __m256i, BitMask<16, V4>, f16, 16, V4); -macros::x86_define_default!(f16x16, _mm256_setzero_si256, "sse2"); +macros::x86_define_default!(f16x16, _mm256_setzero_si256, "avx"); macros::x86_retarget!(f16x16 => v3::f16x16); macros::x86_splitjoin!( f16x16, @@ -77,7 +77,7 @@ impl X86LoadStore for f16x16 { unsafe fn store_simd_masked_logical(self, ptr: *mut f16, mask: Self::Mask) { // SAFETY: Pointer access guaranteed by caller. // - // `_mm256_maskz_loadu_epi16` requires AVX512BW + AVX512VL - implied by V4. + // `_mm256_mask_storeu_epi16` requires AVX512BW + AVX512VL - implied by V4. unsafe { _mm256_mask_storeu_epi16(ptr.cast(), mask.0, self.0) } } } diff --git a/diskann-wide/src/arch/x86_64/v4/f16x8_.rs b/diskann-wide/src/arch/x86_64/v4/f16x8_.rs index 3b5d497e44..0f92096d5f 100644 --- a/diskann-wide/src/arch/x86_64/v4/f16x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/f16x8_.rs @@ -69,7 +69,7 @@ impl X86LoadStore for f16x8 { unsafe fn store_simd_masked_logical(self, ptr: *mut f16, mask: Self::Mask) { // SAFETY: Pointer access guaranteed by caller. // - // `_mm_maskz_loadu_epi16` requires AVX512BW + AVX512VL - implied by V4. + // `_mm_mask_storeu_epi16` requires AVX512BW + AVX512VL - implied by V4. unsafe { _mm_mask_storeu_epi16(ptr.cast(), mask.0, self.0) } } } diff --git a/diskann-wide/src/arch/x86_64/v4/f32x16_.rs b/diskann-wide/src/arch/x86_64/v4/f32x16_.rs index 961f005aba..d38465f906 100644 --- a/diskann-wide/src/arch/x86_64/v4/f32x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/f32x16_.rs @@ -46,7 +46,6 @@ impl crate::SplitJoin for f32x16 { #[inline(always)] fn join(lohi: crate::LoHi) -> Self { - // SAFETY: Required by instantiator. let v = Self::default(lohi.lo.arch()).to_underlying(); // SAFETY: `_mm512_insertf32x8` requires `AVX512DQ` - implied by V4. @@ -55,14 +54,15 @@ impl crate::SplitJoin for f32x16 { } } -helpers::unsafe_map_binary_op!(f32x16, std::ops::Add, add, _mm512_add_ps, "avx"); -helpers::unsafe_map_binary_op!(f32x16, std::ops::Sub, sub, _mm512_sub_ps, "avx"); -helpers::unsafe_map_binary_op!(f32x16, std::ops::Mul, mul, _mm512_mul_ps, "avx"); +helpers::unsafe_map_binary_op!(f32x16, std::ops::Add, add, _mm512_add_ps, "avx512f"); +helpers::unsafe_map_binary_op!(f32x16, std::ops::Sub, sub, _mm512_sub_ps, "avx512f"); +helpers::unsafe_map_binary_op!(f32x16, std::ops::Mul, mul, _mm512_mul_ps, "avx512f"); impl f32x16 { #[inline(always)] fn is_nan(self) -> BitMask<16, V4> { - // NOTE: `_CMP_UNORD_Q` returns `true` only if both arguments are NAN. + // NOTE: `_CMP_UNORD_Q` returns `true` if either argument is NaN. Since we compare + // `self` with `self`, this returns `true` exactly when `self` is NaN. BitMask::from_underlying( self.arch(), // SAFETY: `_mm512_cmp_ps_mask` requires AVX512F, which is implied by the diff --git a/diskann-wide/src/arch/x86_64/v4/f32x4_.rs b/diskann-wide/src/arch/x86_64/v4/f32x4_.rs index ff191ac160..328dba4d26 100644 --- a/diskann-wide/src/arch/x86_64/v4/f32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v4/f32x4_.rs @@ -37,7 +37,8 @@ helpers::unsafe_map_binary_op!(f32x4, std::ops::Mul, mul, _mm_mul_ps, "sse"); impl f32x4 { #[inline(always)] fn is_nan(self) -> BitMask<4, V4> { - // NOTE: `_CMP_UNORD_Q` returns `true` only if both arguments are NAN. + // NOTE: `_CMP_UNORD_Q` returns `true` if either argument is NaN. Since we compare + // `self` with `self`, this returns `true` exactly when `self` is NaN. BitMask::from_underlying( self.arch(), // SAFETY: `_mm_cmp_ps_mask` requires AVX512F + AVX512VL, both of which diff --git a/diskann-wide/src/arch/x86_64/v4/f32x8_.rs b/diskann-wide/src/arch/x86_64/v4/f32x8_.rs index f09e909b01..3158ffc1dd 100644 --- a/diskann-wide/src/arch/x86_64/v4/f32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/f32x8_.rs @@ -30,7 +30,7 @@ use crate::{ macros::x86_define_register!(f32x8, __m256, BitMask<8, V4>, f32, 8, V4); macros::x86_define_splat!(f32x8, _mm256_set1_ps, "avx"); macros::x86_define_default!(f32x8, _mm256_setzero_ps, "avx"); -macros::x86_splitjoin!(f32x8, f32x4, _mm256_extractf128_ps, _mm256_set_m128, "avx2"); +macros::x86_splitjoin!(f32x8, f32x4, _mm256_extractf128_ps, _mm256_set_m128, "avx"); macros::x86_retarget!(f32x8 => v3::f32x8); helpers::unsafe_map_binary_op!(f32x8, std::ops::Add, add, _mm256_add_ps, "avx"); @@ -40,7 +40,8 @@ helpers::unsafe_map_binary_op!(f32x8, std::ops::Mul, mul, _mm256_mul_ps, "avx"); impl f32x8 { #[inline(always)] fn is_nan(self) -> BitMask<8, V4> { - // NOTE: `_CMP_UNORD_Q` returns `true` only if both arguments are NAN. + // NOTE: `_CMP_UNORD_Q` returns `true` if either argument is NaN. Since we compare + // `self` with `self`, this returns `true` exactly when `self` is NaN. BitMask::from_underlying( self.arch(), // SAFETY: `_mm256_cmp_ps_mask` requires AVX512F + AVX512VL, both of which diff --git a/diskann-wide/src/arch/x86_64/v4/i16x16_.rs b/diskann-wide/src/arch/x86_64/v4/i16x16_.rs index 34252bad87..e542fd5092 100644 --- a/diskann-wide/src/arch/x86_64/v4/i16x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i16x16_.rs @@ -46,8 +46,20 @@ helpers::unsafe_map_binary_op!(i16x16, std::ops::BitAnd, bitand, _mm256_and_si25 helpers::unsafe_map_binary_op!(i16x16, std::ops::BitOr, bitor, _mm256_or_si256, "avx2"); helpers::unsafe_map_binary_op!(i16x16, std::ops::BitXor, bitxor, _mm256_xor_si256, "avx2"); -helpers::unsafe_map_binary_op!(i16x16, std::ops::Shr, shr, _mm256_srav_epi16, "avx2"); -helpers::unsafe_map_binary_op!(i16x16, std::ops::Shl, shl, _mm256_sllv_epi16, "avx2"); +helpers::unsafe_map_binary_op!( + i16x16, + std::ops::Shr, + shr, + _mm256_srav_epi16, + "avx512bw,avx512vl" +); +helpers::unsafe_map_binary_op!( + i16x16, + std::ops::Shl, + shl, + _mm256_sllv_epi16, + "avx512bw,avx512vl" +); helpers::scalar_shift_by_splat!(i16x16, i16); impl std::ops::Not for i16x16 { diff --git a/diskann-wide/src/arch/x86_64/v4/i16x32_.rs b/diskann-wide/src/arch/x86_64/v4/i16x32_.rs index ee54a94b9d..19f10f89ad 100644 --- a/diskann-wide/src/arch/x86_64/v4/i16x32_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i16x32_.rs @@ -69,7 +69,7 @@ impl SIMDMulAdd for i16x32 { } } -macros::x86_avx512_int_comparisons!(i16x32, _mm512_cmp_epi16_mask, "avx512bw,avx512vl"); +macros::x86_avx512_int_comparisons!(i16x32, _mm512_cmp_epi16_mask, "avx512bw"); macros::x86_avx512_load_store!( i16x32, _mm512_loadu_epi16, diff --git a/diskann-wide/src/arch/x86_64/v4/i16x8_.rs b/diskann-wide/src/arch/x86_64/v4/i16x8_.rs index 60733b72e7..58683875d8 100644 --- a/diskann-wide/src/arch/x86_64/v4/i16x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i16x8_.rs @@ -32,13 +32,25 @@ macros::x86_retarget!(i16x8 => v3::i16x8); helpers::unsafe_map_binary_op!(i16x8, std::ops::Add, add, _mm_add_epi16, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::Sub, sub, _mm_sub_epi16, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::Mul, mul, _mm_mullo_epi16, "sse2"); -helpers::unsafe_map_unary_op!(i16x8, SIMDAbs, abs_simd, _mm_abs_epi16, "sse3"); +helpers::unsafe_map_unary_op!(i16x8, SIMDAbs, abs_simd, _mm_abs_epi16, "ssse3"); helpers::unsafe_map_binary_op!(i16x8, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::BitOr, bitor, _mm_or_si128, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::BitXor, bitxor, _mm_xor_si128, "sse2"); -helpers::unsafe_map_binary_op!(i16x8, std::ops::Shr, shr, _mm_srav_epi16, "avx2"); -helpers::unsafe_map_binary_op!(i16x8, std::ops::Shl, shl, _mm_sllv_epi16, "avx2"); +helpers::unsafe_map_binary_op!( + i16x8, + std::ops::Shr, + shr, + _mm_srav_epi16, + "avx512bw,avx512vl" +); +helpers::unsafe_map_binary_op!( + i16x8, + std::ops::Shl, + shl, + _mm_sllv_epi16, + "avx512bw,avx512vl" +); helpers::scalar_shift_by_splat!(i16x8, i16); impl std::ops::Not for i16x8 { diff --git a/diskann-wide/src/arch/x86_64/v4/i32x16_.rs b/diskann-wide/src/arch/x86_64/v4/i32x16_.rs index da1e535e8d..2f4603a5b8 100644 --- a/diskann-wide/src/arch/x86_64/v4/i32x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i32x16_.rs @@ -90,7 +90,7 @@ impl SIMDSumTree for i32x16 { impl SIMDSelect for BitMask<16, V4> { #[inline(always)] fn select(self, x: i32x16, y: i32x16) -> i32x16 { - // SAFETY: `_mm512_mask_blend_epi32` requires AVX512F + AVX512VL - implied by V4 + // SAFETY: `_mm512_mask_blend_epi32` requires AVX512F - implied by V4 i32x16::from_underlying(self.arch(), unsafe { _mm512_mask_blend_epi32(self.to_underlying(), y.to_underlying(), x.to_underlying()) }) diff --git a/diskann-wide/src/arch/x86_64/v4/i32x4_.rs b/diskann-wide/src/arch/x86_64/v4/i32x4_.rs index e420abce0c..66dbec7b5d 100644 --- a/diskann-wide/src/arch/x86_64/v4/i32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i32x4_.rs @@ -20,7 +20,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit signed integer ///// macros::x86_define_register!(i32x4, __m128i, BitMask<4, V4>, i32, 4, V4); @@ -31,7 +31,7 @@ macros::x86_retarget!(i32x4 => v3::i32x4); helpers::unsafe_map_binary_op!(i32x4, std::ops::Add, add, _mm_add_epi32, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::Sub, sub, _mm_sub_epi32, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::Mul, mul, _mm_mullo_epi32, "sse4.1"); -helpers::unsafe_map_unary_op!(i32x4, SIMDAbs, abs_simd, _mm_abs_epi32, "sse3"); +helpers::unsafe_map_unary_op!(i32x4, SIMDAbs, abs_simd, _mm_abs_epi32, "ssse3"); helpers::unsafe_map_binary_op!(i32x4, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::BitOr, bitor, _mm_or_si128, "sse2"); diff --git a/diskann-wide/src/arch/x86_64/v4/i32x8_.rs b/diskann-wide/src/arch/x86_64/v4/i32x8_.rs index 7305d25ba5..55ee62ad7c 100644 --- a/diskann-wide/src/arch/x86_64/v4/i32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i32x8_.rs @@ -21,7 +21,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit signed integer ///// macros::x86_define_register!(i32x8, __m256i, BitMask<8, V4>, i32, 8, V4); @@ -39,7 +39,7 @@ macros::x86_splitjoin!( helpers::unsafe_map_binary_op!(i32x8, std::ops::Add, add, _mm256_add_epi32, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::Sub, sub, _mm256_sub_epi32, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::Mul, mul, _mm256_mullo_epi32, "avx2"); -helpers::unsafe_map_unary_op!(i32x8, SIMDAbs, abs_simd, _mm256_abs_epi32, "sse3"); +helpers::unsafe_map_unary_op!(i32x8, SIMDAbs, abs_simd, _mm256_abs_epi32, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::BitAnd, bitand, _mm256_and_si256, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::BitOr, bitor, _mm256_or_si256, "avx2"); @@ -108,7 +108,7 @@ impl SIMDDotProduct for i32x8 { impl SIMDDotProduct for i32x8 { #[inline(always)] fn dot_simd(self, left: u8x32, right: i8x32) -> Self { - // SAFETY: `_mm512_dpbusd_epi32` requires AVX512_VNNI - implied by V4 + // SAFETY: `_mm256_dpbusd_epi32` requires AVX512_VNNI + AVX512VL - implied by V4 let r = unsafe { _mm256_dpbusd_epi32(self.0, left.0, right.0) }; Self::from_underlying(self.arch(), r) } diff --git a/diskann-wide/src/arch/x86_64/v4/i8x16_.rs b/diskann-wide/src/arch/x86_64/v4/i8x16_.rs index 41936b61b3..55e1491ea1 100644 --- a/diskann-wide/src/arch/x86_64/v4/i8x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i8x16_.rs @@ -30,7 +30,7 @@ macros::x86_retarget!(i8x16 => v3::i8x16); helpers::unsafe_map_binary_op!(i8x16, std::ops::Add, add, _mm_add_epi8, "sse2"); helpers::unsafe_map_binary_op!(i8x16, std::ops::Sub, sub, _mm_sub_epi8, "sse2"); -helpers::unsafe_map_unary_op!(i8x16, SIMDAbs, abs_simd, _mm_abs_epi8, "sse3"); +helpers::unsafe_map_unary_op!(i8x16, SIMDAbs, abs_simd, _mm_abs_epi8, "ssse3"); impl std::ops::Mul for i8x16 { type Output = Self; diff --git a/diskann-wide/src/arch/x86_64/v4/i8x32_.rs b/diskann-wide/src/arch/x86_64/v4/i8x32_.rs index 5944d756a8..7b5f411f4c 100644 --- a/diskann-wide/src/arch/x86_64/v4/i8x32_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i8x32_.rs @@ -19,9 +19,9 @@ use crate::{ traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDVector}, }; -//////////////////// -// 8-bit unsigned // -//////////////////// +////////////////// +// 8-bit signed // +////////////////// macros::x86_define_register!(i8x32, __m256i, BitMask<32, V4>, i8, 32, V4); macros::x86_define_splat!(i8x32 as i8, _mm256_set1_epi8, "avx"); diff --git a/diskann-wide/src/arch/x86_64/v4/i8x64_.rs b/diskann-wide/src/arch/x86_64/v4/i8x64_.rs index 657304ebfd..b1bf8445f2 100644 --- a/diskann-wide/src/arch/x86_64/v4/i8x64_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i8x64_.rs @@ -19,9 +19,9 @@ use crate::{ traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDVector}, }; -//////////////////// -// 8-bit unsigned // -//////////////////// +////////////////// +// 8-bit signed // +////////////////// macros::x86_define_register!(i8x64, __m512i, BitMask<64, V4>, i8, 64, V4); macros::x86_define_splat!(i8x64 as i8, _mm512_set1_epi8, "avx512f"); diff --git a/diskann-wide/src/arch/x86_64/v4/mod.rs b/diskann-wide/src/arch/x86_64/v4/mod.rs index baa76f29be..e61f9e9c98 100644 --- a/diskann-wide/src/arch/x86_64/v4/mod.rs +++ b/diskann-wide/src/arch/x86_64/v4/mod.rs @@ -5,7 +5,7 @@ //! # AVX-512 Support //! -//! Access to AVX-512 instrinsics is provided by the [`V4`] backend. +//! Access to AVX-512 intrinsics is provided by the [`V4`] backend. //! //! This backend corresponds to the features required for the [`V3`] architecture in addition //! to: @@ -34,7 +34,7 @@ //! However, Miri is a great tool for assisting with writing kernels when it comes to //! validating bounds-checking. The [`V4`] supports running under Miri with a few caveats. //! First, when using `#[cfg(miri)]`, the type aliases such as [`f32x4`] will be mapped -//! to emulated variants (i.e., [Emulated`). Use of the standard SIMD functions +//! to emulated variants (i.e., [`Emulated`]). Use of the standard SIMD functions //! like arithmetic, loading, storing, etc. will execute as expected, enabling Miri assisted //! bounds checking for loads and stores. //! @@ -55,7 +55,7 @@ //! these are aliased to `Emulated` instead of the native intrinsic types. //! //! This means that regardless of Miri, the native intrinsic types are still defined and -//! useable. They are just not mapped via `V4 as Architecture`. Thus, when working in the +//! usable. They are just not mapped via `V4 as Architecture`. Thus, when working in the //! submodules like `f32x8_` that reference types like `f32x4`, make sure that the symbol //! `f32x4` is imported via `f32x4_::f32x4` to ensure compilation when using Miri. //! @@ -186,7 +186,7 @@ pub struct V4(Hidden); impl arch::Sealed for V4 {} -// `miri` does not have the ability to dynamically determing micro-architecture. Thus, to +// `miri` does not have the ability to dynamically determine micro-architecture. Thus, to // run V4 kernels under `Miri` without compiling the whole application of `x86-64-v4`, we // drop the `target_features`. // @@ -239,7 +239,7 @@ impl V4 { /// Retarget for a more conservative architecture. pub fn retarget(self) -> V3 { - // SAFETY: `V4` is a superset of `V3` - so an instance of `V4` asserts the present. + // SAFETY: `V4` is a superset of `V3` - so an instance of `V4` asserts the presence of `V3`. unsafe { V3::new() } } @@ -260,7 +260,7 @@ impl V4 { // just check if the user specified level would have instantiated it. if unsafe { Self::new_checked_with(super::test_arch_number()) }.is_some() { panic!( - "V4 architecture was requested but is not compatible on the current hardare" + "V4 architecture was requested but is not compatible on the current hardware" ); } else { None @@ -285,7 +285,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. pub(super) unsafe fn run_with(self, f: F) -> R where F: Target, @@ -297,7 +297,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. #[inline] pub(super) unsafe fn run_with_inline(self, f: F) -> R where @@ -310,7 +310,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. pub(super) unsafe fn run_with_1(self, f: F, x0: T0) -> R where F: Target1, @@ -322,7 +322,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. #[inline] pub(super) unsafe fn run_with_1_inline(self, f: F, x0: T0) -> R where @@ -335,7 +335,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. pub(super) unsafe fn run_with_2(self, f: F, x0: T0, x1: T1) -> R where F: Target2, @@ -347,7 +347,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. #[inline] pub(super) unsafe fn run_with_2_inline(self, f: F, x0: T0, x1: T1) -> R where @@ -360,7 +360,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. pub(super) unsafe fn run_with_3(self, f: F, x0: T0, x1: T1, x2: T2) -> R where F: Target3, @@ -372,7 +372,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. #[inline] pub(super) unsafe fn run_with_3_inline( self, @@ -391,7 +391,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. pub(super) unsafe fn run_function_with_1(self, x0: T0::Of<'_>) -> R where T0: AddLifetime, @@ -404,7 +404,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. pub(super) unsafe fn run_function_with_2( self, x0: T0::Of<'_>, @@ -422,7 +422,7 @@ impl V4 { v4_features! { // # Safety // - // The current machine must have all the documented features required for V3. + // The current machine must have all the documented features required for V4. pub(super) unsafe fn run_function_with_3( self, x0: T0::Of<'_>, @@ -453,7 +453,7 @@ impl Architecture for V4 { where F: Target, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { Self::run_with(self, f) } } @@ -463,7 +463,7 @@ impl Architecture for V4 { where F: Target, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { Self::run_with_inline(self, f) } } @@ -473,7 +473,7 @@ impl Architecture for V4 { where F: Target1, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { self.run_with_1(f, x0) } } @@ -483,7 +483,7 @@ impl Architecture for V4 { where F: Target1, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { self.run_with_1_inline(f, x0) } } @@ -493,7 +493,7 @@ impl Architecture for V4 { where F: Target2, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { self.run_with_2(f, x0, x1) } } @@ -503,7 +503,7 @@ impl Architecture for V4 { where F: Target2, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { self.run_with_2_inline(f, x0, x1) } } @@ -513,7 +513,7 @@ impl Architecture for V4 { where F: Target3, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { self.run_with_3(f, x0, x1, x2) } } @@ -523,7 +523,7 @@ impl Architecture for V4 { where F: Target3, { - // SAFETY: The existence of `self` implies that we are V3 compatible and therefore + // SAFETY: The existence of `self` implies that we are V4 compatible and therefore // have all the required features. unsafe { self.run_with_3_inline(f, x0, x1, x2) } } @@ -535,9 +535,9 @@ impl Architecture for V4 { { let f: unsafe fn(Self, T0::Of<'_>) -> R = Self::run_function_with_1::; - // SAFETY: The present of `self` as an argument attests that it is safe to construct + // SAFETY: The presence of `self` as an argument attests that it is safe to construct // A `V4` architecture. Additionally, since `V4` is a `Copy` zero-sized type, - // it is safe to wink into existence and is ABI compattible with `Hidden`. + // it is safe to wink into existence and is ABI compatible with `Hidden`. unsafe { arch::hide1(f) } } @@ -550,9 +550,9 @@ impl Architecture for V4 { let f: unsafe fn(Self, T0::Of<'_>, T1::Of<'_>) -> R = Self::run_function_with_2::; - // SAFETY: The present of `self` as an argument attests that it is safe to construct + // SAFETY: The presence of `self` as an argument attests that it is safe to construct // A `V4` architecture. Additionally, since `V4` is a `Copy` zero-sized type, - // it is safe to wink into existence and is ABI compattible with `Hidden`. + // it is safe to wink into existence and is ABI compatible with `Hidden`. unsafe { arch::hide2(f) } } @@ -566,9 +566,9 @@ impl Architecture for V4 { let f: unsafe fn(Self, T0::Of<'_>, T1::Of<'_>, T2::Of<'_>) -> R = Self::run_function_with_3::; - // SAFETY: The present of `self` as an argument attests that it is safe to construct + // SAFETY: The presence of `self` as an argument attests that it is safe to construct // A `V4` architecture. Additionally, since `V4` is a `Copy` zero-sized type, - // it is safe to wink into existence and is ABI compattible with `Hidden`. + // it is safe to wink into existence and is ABI compatible with `Hidden`. unsafe { arch::hide3(f) } } } diff --git a/diskann-wide/src/arch/x86_64/v4/u32x16_.rs b/diskann-wide/src/arch/x86_64/v4/u32x16_.rs index 63a435a806..113afb8637 100644 --- a/diskann-wide/src/arch/x86_64/v4/u32x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u32x16_.rs @@ -20,7 +20,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit unsigned integer ///// macros::x86_define_register!(u32x16, __m512i, BitMask<16, V4>, u32, 16, V4); @@ -89,7 +89,7 @@ impl SIMDSumTree for u32x16 { impl SIMDSelect for BitMask<16, V4> { #[inline(always)] fn select(self, x: u32x16, y: u32x16) -> u32x16 { - // SAFETY: `_mm512_mask_blend_epi32` requires AVX512F + AVX512VL - implied by V4 + // SAFETY: `_mm512_mask_blend_epi32` requires AVX512F - implied by V4 u32x16::from_underlying(self.arch(), unsafe { _mm512_mask_blend_epi32(self.to_underlying(), y.to_underlying(), x.to_underlying()) }) diff --git a/diskann-wide/src/arch/x86_64/v4/u32x4_.rs b/diskann-wide/src/arch/x86_64/v4/u32x4_.rs index 10d2174157..938a5ab7e8 100644 --- a/diskann-wide/src/arch/x86_64/v4/u32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u32x4_.rs @@ -20,7 +20,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit unsigned integer ///// macros::x86_define_register!(u32x4, __m128i, BitMask<4, V4>, u32, 4, V4); diff --git a/diskann-wide/src/arch/x86_64/v4/u32x8_.rs b/diskann-wide/src/arch/x86_64/v4/u32x8_.rs index 73ab9b26e0..4946e1b837 100644 --- a/diskann-wide/src/arch/x86_64/v4/u32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u32x8_.rs @@ -21,7 +21,7 @@ use crate::{ }; ///// -///// 32-bit floating point +///// 32-bit unsigned integer ///// macros::x86_define_register!(u32x8, __m256i, BitMask<8, V4>, u32, 8, V4); diff --git a/diskann-wide/src/arch/x86_64/v4/u64x2_.rs b/diskann-wide/src/arch/x86_64/v4/u64x2_.rs index 67d30074f4..43fd5c30e5 100644 --- a/diskann-wide/src/arch/x86_64/v4/u64x2_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u64x2_.rs @@ -20,7 +20,7 @@ use crate::{ }; ///// -///// 64-bit unsigned integerA +///// 64-bit unsigned integer ///// macros::x86_define_register!(u64x2, __m128i, BitMask<2, V4>, u64, 2, V4); diff --git a/diskann-wide/src/emulated.rs b/diskann-wide/src/emulated.rs index 11971ceb01..f9a454698f 100644 --- a/diskann-wide/src/emulated.rs +++ b/diskann-wide/src/emulated.rs @@ -23,15 +23,15 @@ use super::{ /// using loops over arrays rather than dispatching to platform specific instructions. /// /// The idea behind this type is that it can be used on architecture where explicit backend -/// support has not been added, or when an architecture does not support a given type/lengh +/// support has not been added, or when an architecture does not support a given type/length /// pair well. /// /// Furthermore, it can be used when developing new back-ends to provide fallback -/// implementations. This allows new back-ends to be developed one piece as a time instead -/// of all at onces. +/// implementations. This allows new back-ends to be developed one piece at a time instead +/// of all at once. /// /// NOTE: The alignment requirements of an emulated vector *will* be different than the -/// alignment requirements an actual intrinsic. +/// alignment requirements of an actual intrinsic. /// /// Higher level code *must not* rely on alignments being compatible across architectures! #[derive(Debug, Clone, Copy)] @@ -108,7 +108,7 @@ where ) } - /// Only load values then the corresponding mask lane is set. + /// Only load values when the corresponding mask lane is set. unsafe fn load_simd_masked_logical(arch: A, ptr: *const T, mask: Self::Mask) -> Self { Self::from_arch_fn(arch, |i| { if mask.get_unchecked(i) { @@ -142,7 +142,7 @@ where unsafe { ptr.cast::<[T; N]>().write_unaligned(self.0) } } - /// Only store values then the corresponding mask lane is set. + /// Only store values when the corresponding mask lane is set. unsafe fn store_simd_masked_logical(self, ptr: *mut T, mask: Self::Mask) { for (i, v) in self.0.iter().enumerate() { if mask.get_unchecked(i) { @@ -377,7 +377,7 @@ where // i16 to i32 macro_rules! impl_simd_dot_product_i16_to_i32 { ($N:literal, $TwoN:literal) => { - /// Promote intermediate values to `i32` and then perform accuulation. + /// Promote intermediate values to `i32` and then perform accumulation. impl SIMDDotProduct> for Emulated where A: arch::Sealed, @@ -403,7 +403,7 @@ macro_rules! impl_simd_dot_product_i16_to_i32 { //i8/u8 to i32 macro_rules! impl_simd_dot_product_iu8_to_i32 { ($N:literal, $TwoN:literal) => { - /// Promote intermediate values to `i32` and then perform accuulation. + /// Promote intermediate values to `i32` and then perform accumulation. impl SIMDDotProduct, Emulated> for Emulated where @@ -549,7 +549,7 @@ macro_rules! impl_little_endian_transmute_cast { { fn reinterpret_simd(self) -> Emulated<$to, $Nto, A> { let array = self.0; - // # SAFETY: This is only ever instantiated with arrays of primitive + // SAFETY: This is only ever instantiated with arrays of primitive // types that hold no resources, no padding, and are valid for all // possible bit-patterns. let casted = unsafe { std::mem::transmute::<[$from; $Nfrom], [$to; $Nto]>(array) }; @@ -676,7 +676,7 @@ mod test_emulated { test_utils::test_load_simd::>(Scalar); test_utils::test_load_simd::>(Scalar); - // Unsigned Integers + // Signed Integers test_utils::test_load_simd::>(Scalar); test_utils::test_load_simd::>(Scalar); @@ -709,7 +709,7 @@ mod test_emulated { test_utils::test_store_simd::>(Scalar); test_utils::test_store_simd::>(Scalar); - // Unsigned Integers + // Signed Integers test_utils::test_store_simd::>(Scalar); test_utils::test_store_simd::>(Scalar); diff --git a/diskann-wide/src/lib.rs b/diskann-wide/src/lib.rs index d9c1a845bc..5e382775eb 100644 --- a/diskann-wide/src/lib.rs +++ b/diskann-wide/src/lib.rs @@ -104,7 +104,7 @@ //! invokes the reference implementation may be all that's needed. //! //! More complicated operations may require their own test harness (see -//! `test_tuils/dot_product.rs`). +//! `test_utils/dot_product.rs`). //! //! Tests should go through the utilities in `test_utils::driver` to ensure adequate //! coverage and low compile time. @@ -170,8 +170,6 @@ pub mod arch; pub use arch::Architecture; /// The current architecture that is the closest fit for the current compilation target. -/// -/// The type [`Wide`] is always configured to use this as its associated architecture type. pub const ARCH: arch::Current = arch::current(); /////////////////////// diff --git a/diskann-wide/src/lifetime.rs b/diskann-wide/src/lifetime.rs index 5bfed6d397..09b1f021fb 100644 --- a/diskann-wide/src/lifetime.rs +++ b/diskann-wide/src/lifetime.rs @@ -3,7 +3,7 @@ * Licensed under the MIT license. */ -//! Tools to pass objects with lifetimes across the funtion pointer API. +//! Tools to pass objects with lifetimes across the function pointer API. //! //! Useful helpers include //! @@ -17,8 +17,8 @@ use std::marker::PhantomData; /// A lifetime annotator for the function pointer API of [`crate::Architecture`]. /// -/// This mainly works around limitations the Rust compiler's ability to infer the proper -/// of dispatched function pointers. +/// This mainly works around limitations in the Rust compiler's ability to infer the proper +/// lifetimes of dispatched function pointers. pub trait AddLifetime: 'static { /// The type with a lifetime (if any). type Of<'a>; diff --git a/diskann-wide/src/splitjoin.rs b/diskann-wide/src/splitjoin.rs index 2f27af0831..1ec771ba5e 100644 --- a/diskann-wide/src/splitjoin.rs +++ b/diskann-wide/src/splitjoin.rs @@ -42,7 +42,7 @@ impl LoHi { U::join(self) } - /// Return a new [`LoHi`] with the function `f` applied to the pairwise members of + /// Return a new [`LoHi`] with the function `f` applied to the pairwise members /// of `self` and `x`. /// /// If it does not panic, `f` will be invoked exactly twice, first on `lo`, then on `hi`. diff --git a/diskann-wide/src/test_utils/distribution.rs b/diskann-wide/src/test_utils/distribution.rs index d3b2f3ee2d..f336e257a3 100644 --- a/diskann-wide/src/test_utils/distribution.rs +++ b/diskann-wide/src/test_utils/distribution.rs @@ -58,7 +58,7 @@ macro_rules! finite { /// Generate floating point numbers spread more-or-less uniformly across the /// distribution of floating point numbers. /// - /// Note that this is *not* a methematical uniform distribution over some range. + /// Note that this is *not* a mathematical uniform distribution over some range. /// For normal values, every normal floating point number is generated with equal /// probability. /// diff --git a/diskann-wide/src/test_utils/load.rs b/diskann-wide/src/test_utils/load.rs index 0d43db8cb4..a8e7cf12b3 100644 --- a/diskann-wide/src/test_utils/load.rs +++ b/diskann-wide/src/test_utils/load.rs @@ -26,7 +26,7 @@ where V: SIMDVector>, { // Test loads for all alignments. - // Our strategy to to create an array of twice the underlying vector width and perform a + // Our strategy is to create an array of twice the underlying vector width and perform a // full-width load on each offset. let mut input = vec![T::default(); 2 * N]; iota_slice(input.as_mut_slice()); diff --git a/diskann-wide/src/test_utils/mask.rs b/diskann-wide/src/test_utils/mask.rs index 360c0677c5..d164548442 100644 --- a/diskann-wide/src/test_utils/mask.rs +++ b/diskann-wide/src/test_utils/mask.rs @@ -8,7 +8,7 @@ use crate::{Architecture, BitMask, Const, SIMDMask, SupportedLaneCount}; // The maximum length supported for BitMasks. const MAXLEN: usize = 64; -/// Utilities for checking compliance an implementation of the `SIMDMask` trait. +/// Utilities for checking compliance of an implementation of the `SIMDMask` trait. pub(crate) fn test_keep_first(arch: A, mut f: F) where A: Architecture, diff --git a/diskann-wide/src/test_utils/ops.rs b/diskann-wide/src/test_utils/ops.rs index b81037bdbc..7a38b09e48 100644 --- a/diskann-wide/src/test_utils/ops.rs +++ b/diskann-wide/src/test_utils/ops.rs @@ -148,7 +148,7 @@ fn check_minmax_standard_f32(got: f32, standard: f32) -> Result<(), String> { /// /// The [`FastMax`] checker will accept either the IEEE result or `NaN` in such situations. /// -/// If the expected result is +/-0.0, than a zero of either sign will be accepted. +/// If the expected result is +/-0.0, then a zero of either sign will be accepted. /// /// If neither argument is `NaN`, then the results must match the IEEE result. #[derive(Debug, Clone, Copy)] @@ -163,10 +163,10 @@ impl CheckBinary for FastMax { /// A checker for Rust standard compliant max implementations. /// -/// If one argument is NaN, than the other is returned. A NaN is only returned if both +/// If one argument is NaN, then the other is returned. A NaN is only returned if both /// arguments are NaN. /// -/// If the expected result is +/-0.0, than a zero of either sign will be accepted. +/// If the expected result is +/-0.0, then a zero of either sign will be accepted. /// /// If neither argument is `NaN`, then the results must match the IEEE result. #[derive(Debug, Clone, Copy)] @@ -185,9 +185,9 @@ impl CheckBinary for StandardMax { /// IEEE specifies that `min` between a NaN and non-NaN value `x` should return `x`. However, /// it is common for Intel hardware to return `NaN` instead for such operations. /// -/// The [`FastMax`] checker will accept either the IEEE result or `NaN` in such situations. +/// The [`FastMin`] checker will accept either the IEEE result or `NaN` in such situations. /// -/// If the expected result is +/-0.0, than a zero of either sign will be accepted. +/// If the expected result is +/-0.0, then a zero of either sign will be accepted. /// /// If neither argument is `NaN`, then the results must match the IEEE result. #[derive(Debug, Clone, Copy)] @@ -202,10 +202,10 @@ impl CheckBinary for FastMin { /// A checker for Rust standard compliant min implementations. /// -/// If one argument is NaN, than the other is returned. A NaN is only returned if both +/// If one argument is NaN, then the other is returned. A NaN is only returned if both /// arguments are NaN. /// -/// If the expected result is +/-0.0, than a zero of either sign will be accepted. +/// If the expected result is +/-0.0, then a zero of either sign will be accepted. /// /// If neither argument is `NaN`, then the results must match the IEEE result. #[derive(Debug, Clone, Copy)] diff --git a/diskann-wide/src/test_utils/store.rs b/diskann-wide/src/test_utils/store.rs index 6a58584a7c..3aaeb893d4 100644 --- a/diskann-wide/src/test_utils/store.rs +++ b/diskann-wide/src/test_utils/store.rs @@ -26,7 +26,7 @@ where V: SIMDVector>, { // Test stores for all alignments. - // Our strategy to to create an array of twice the underlying vector width and perform a + // Our strategy is to create an array of twice the underlying vector width and perform a // full-width stores on each offset. let mut output = vec![T::default(); 2 * N]; diff --git a/diskann-wide/src/traits.rs b/diskann-wide/src/traits.rs index 63f6f7b02e..bc19d4e64c 100644 --- a/diskann-wide/src/traits.rs +++ b/diskann-wide/src/traits.rs @@ -40,9 +40,9 @@ where } /// Stable Rust does not allow expressions involving compile-time computation with -/// const generic parameters: +/// const generic parameters: /// -/// This makes is difficult to go a const parameter defining the number of SIMD lanes +/// This makes it difficult to go from a const parameter defining the number of SIMD lanes /// to an appropriately sized mask. /// /// This helper trait provides a level of indirection to map SIMD representations to @@ -80,7 +80,7 @@ where /// bit is set to 1. /// /// * On AVX-512 systems, the story is much simpler as the masks used in that instruction -/// set are simply the correponsing bit mask. +/// set are simply the corresponding bit mask. /// /// So a mask for 8-wide operations is simply an 8-bit unsigned integer. /// @@ -149,7 +149,7 @@ pub trait SIMDMask: Copy + std::fmt::Debug { } } - /// Construct a mask based on the result of invoking `f` once each element in the range + /// Construct a mask based on the result of invoking `f` once for each element in the range /// `0..Self::LANES` in order. /// /// In the returned mask `m`, `m.get(0)` corresponds to the value of `f(0)`. Similarly, @@ -163,7 +163,7 @@ pub trait SIMDMask: Copy + std::fmt::Debug { Self::BitMask::from_fn(arch, f).into() } - /// Return `true` if any lane in the mask is set. Otherwise, return `false. + /// Return `true` if any lane in the mask is set. Otherwise, return `false`. #[inline(always)] fn any(self) -> bool { // Recurse to BitMask. @@ -177,7 +177,7 @@ pub trait SIMDMask: Copy + std::fmt::Debug { >::from(self).all() } - /// Return `true` if all lanes in the mask are set. Otherwise, return `false`. + /// Return `true` if no lanes in the mask are set. Otherwise, return `false`. #[inline(always)] fn none(self) -> bool { !self.any() @@ -244,7 +244,7 @@ pub trait SIMDVector: Copy + std::fmt::Debug { /// a compatible architecture. fn arch(self) -> Self::Arch; - /// Return the default value for the type. This is always the numberic 0 for the + /// Return the default value for the type. This is always the numeric 0 for the /// associated scalar type. fn default(arch: Self::Arch) -> Self; @@ -298,7 +298,7 @@ pub trait SIMDVector: Copy + std::fmt::Debug { /// /// # Safety /// - /// Offsets from the `ptr` where the mask evaluates to true must be dereferencable to + /// Offsets from the `ptr` where the mask evaluates to true must be dereferenceable to /// the underlying scalar type. unsafe fn load_simd_masked_logical( arch: Self::Arch, @@ -312,7 +312,7 @@ pub trait SIMDVector: Copy + std::fmt::Debug { /// /// # Safety /// - /// Offsets from the `ptr` where the mask evaluates to true must be dereferencable to + /// Offsets from the `ptr` where the mask evaluates to true must be dereferenceable to /// the underlying scalar type. For implementations using the provided default, the /// conversion from the bitmask to the actual mask must be correct. #[inline(always)] @@ -322,7 +322,7 @@ pub trait SIMDVector: Copy + std::fmt::Debug { mask: <::ConstLanes as BitMaskType>::Type, ) -> Self { // SAFETY: Bitmasks must be convertible to their corresponding logical mask. - // When the logical mask **is** a bitbask, this is a no-op. + // When the logical mask **is** a bitmask, this is a no-op. unsafe { Self::load_simd_masked_logical(arch, ptr, mask.into()) } } @@ -379,14 +379,14 @@ pub trait SIMDVector: Copy + std::fmt::Debug { /// The pointed-to memory must adhere to Rust's exclusive reference rules. /// /// Offsets from the `ptr` where the mask evaluates to true must be mutably - /// dereferencable to the underlying scalar type. + /// dereferenceable to the underlying scalar type. unsafe fn store_simd_masked_logical( self, ptr: *mut ::Scalar, mask: ::Mask, ); - /// The same as `load_simd_masked_logical` but taking a BitMask instead. + /// The same as `store_simd_masked_logical` but taking a BitMask instead. /// /// No store attempt will be made to lanes that are masked out. /// @@ -395,7 +395,7 @@ pub trait SIMDVector: Copy + std::fmt::Debug { /// The pointed-to memory must adhere to Rust's exclusive reference rules. /// /// Offsets from the `ptr` where the mask evaluates to true must be mutably - /// dereferencable to the underlying scalar type. + /// dereferenceable to the underlying scalar type. /// /// For implementations using the provided default, the conversion from the bitmask to /// the actual mask must be correct. @@ -406,7 +406,7 @@ pub trait SIMDVector: Copy + std::fmt::Debug { mask: <::ConstLanes as BitMaskType>::Type, ) { // SAFETY: Bitmasks must be convertible to their corresponding logical mask. - // When the logical mask **is** a bitbask, this is a no-op. + // When the logical mask **is** a bitmask, this is a no-op. unsafe { self.store_simd_masked_logical(ptr, mask.into()) } } @@ -448,7 +448,7 @@ pub trait SIMDVector: Copy + std::fmt::Debug { /// ```ignore /// self * rhs + accumulator /// ``` -/// with the following semantics dependant on the associated scalar type. +/// with the following semantics dependent on the associated scalar type. /// /// * floating point: Perform a fused multiply-add, implementing the operation with only a /// single rounding instance. @@ -473,7 +473,7 @@ pub trait SIMDMulAdd { /// be returned. /// /// * Fast (unsuffixed): Compute the minimum or maximum using the fastest possible method -/// on the given architecture with non-standard NaN handing. +/// on the given architecture with non-standard NaN handling. /// /// When the scalar type is integral, the behavior is the same as the standard /// implementations. @@ -524,7 +524,7 @@ pub trait SIMDAbs { /// A SIMD equivalent of `std::cmp::PartialEq`. /// -/// Instead of a boolean, return `Self::Mask` containin the result of the element-wise +/// Instead of a boolean, return `Self::Mask` containing the result of the element-wise /// comparison of the two vectors. pub trait SIMDPartialEq: SIMDVector { /// SIMD equivalent of `std::cmp::PartialEq::eq`, applying the latter trait to each @@ -536,7 +536,7 @@ pub trait SIMDPartialEq: SIMDVector { fn ne_simd(self, other: Self) -> Self::Mask; } -/// A SIMD equaivalent of `std::cmp::PartialOrd`. +/// A SIMD equivalent of `std::cmp::PartialOrd`. /// /// Instead of a boolean, return `Self::Mask` containing the result of the element-wise /// comparisons of the two vectors. @@ -617,7 +617,7 @@ pub trait SIMDSelect: SIMDMask { /// 1. Perform multiplication as `i32x32 x i32x32` as if converting each lane to `i32`, /// resulting in effectively `i32x32`. No overflow can happen. /// 2. Sum together consecutive groups of 4 in the resulting `i32x32` to yield `i32x8`. -/// 3. Add the resulting `i342x8` into `Self`. +/// 3. Add the resulting `i32x8` into `Self`. /// /// The same applies when the order of `u8x32` and `i8x32` are swapped and for types that /// are twice as wide.