nprasad2077/oxc
1
0 Fork 0
mirror of https://github.com/oxc-project/oxc.git synced 2026-11-04 10:29:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(lexer): add all common methods to Vec128Impl and Vec256Impl traits (#28768)

Browse Source 
Pure refactor - no change to codegen.

Preparatory step for compiling all of the lexer's code on all platforms (#28772).

Previously, the methods of `Vec128` and `Vec256` were inherent methods, defined separately for each CPU type in the `avx2` and `neon` modules. Code which is generic over the CPU type couldn't call them.

Move every method which has both AVX2 and Neon versions into the `Vec128Impl` and `Vec256Impl` traits (introduced in #28766) - 31 methods for `Vec128`, and 36 for `Vec256`. Each CPU type implements the traits in its own module.

`Vec128<C>` and `Vec256<C>` also have generic inherent methods with the same names, which call the trait methods. So callers don't need the traits in scope, and call sites don't change.

The structs, the traits and the generic methods are in new `simd/vec128.rs` and `simd/vec256.rs` files.

The `!`, `&`, `|` and `^` operators, and the `Operand128` / `Operand256` traits, are generic over the CPU type too. The operators move from `bit_traits.rs` into `vec128.rs` and `vec256.rs`. The `CpuType` alias, which picked the platform's CPU type with `#[cfg]`, is removed.

Methods which only one CPU has stay inherent methods of that CPU's types:

- AVX2 only: `movemask` and `Vec256::shuffle_u32s`.
- Neon only: `shl_u8`, `shr_u8`, `pairwise_add_u8` and `test_u8`.

`Vec256::shuffle_u8s` stays separate for each CPU too, because it does different things on each: 2 lookups in 16-entry tables on AVX2, 1 lookup in a 32-entry table on Neon.

Neon's `Vec256` gets the methods it didn't have yet, which work on its 2 `Vec128` halves.

Tests of the common methods move from `avx2/tests.rs` and `neon/tests.rs` into `simd/tests.rs`, and run with the CPU type which the platform has. `avx2/tests.rs` and `neon/tests.rs` keep the tests for methods which only one CPU has, or which behave differently on each.
This commit is contained in:
overlookmotel committed 2026-11-03 18:02:32 +00:00
1 parent f0d84e7514
commit b03fadad8d
14 files changed
+3006 -1733
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/oxc_lexer/src/simd/avx2/tests.rs
+33 -557
View File
@@ -1,32 +1,20 @@
//! Tests for the methods of `Vec128<Avx2>` and `Vec256<Avx2>`.
//! Tests for the methods of `Vec128<Avx2>` and `Vec256<Avx2>`
//! which only AVX2 has, or which behave differently on AVX2.
//!
//! `simd::tests` has the tests which apply to every CPU type.
//!
//! Each test returns early without checking anything if the CPU doesn't support AVX2.
use std::{array, mem::transmute};
use std::array;
use crate::{
cpu::{Avx2, DynCpu},
cpu::DynCpu,
simd::{
Vec128, Vec256, VecData128, VecData256,
operand::Operand128,
tests::{map2, pattern},
Vec128, Vec256,
tests::{pattern, to_bytes128, to_bytes256, to_u32s},
},
};
/// Wrapper to align a value on 32.
#[repr(align(32))]
struct Aligned<T>(T);
/// Transform an array of bytes as if it was an array of `u16`s.
fn map_u16(bytes: [u8; 32], f: impl Fn(u16) -> u16) -> [u8; 32] {
let mut out = [0; 32];
for i in (0..32).step_by(2) {
let value = f(u16::from_le_bytes([bytes[i], bytes[i + 1]]));
out[i..i + 2].copy_from_slice(&value.to_le_bytes());
}
out
}
/// Get the scalar equivalent of `movemask`.
fn top_bits(bytes: &[u8]) -> u32 {
bytes.iter().enumerate().fold(0, |mask, (i, byte)| mask | (u32::from(byte >> 7) << i))
@@ -38,265 +26,54 @@ fn shuffle(bytes: &[u8], control: &[u8]) -> Vec<u8> {
control.iter().map(select).collect()
}
fn to_bytes256(v: Vec256<Avx2>) -> [u8; 32] {
let mut out = [0; 32];
// SAFETY: `out` is valid for writing 32 bytes
unsafe { v.store_unaligned(out.as_mut_ptr()) };
out
}
fn to_u32s(v: Vec256<Avx2>) -> [u32; 8] {
let mut out = [0; 8];
// SAFETY: `out` is valid for writing 32 bytes
unsafe { v.store_unaligned(out.as_mut_ptr().cast::<u8>()) };
out
}
fn to_bytes128(v: Vec128<Avx2>) -> [u8; 16] {
let mut out = [0; 16];
// SAFETY: `out` is valid for writing 16 bytes
unsafe { v.store_unaligned(out.as_mut_ptr()) };
out
}
#[test]
fn vec256_create() {
fn vec128_movemask() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
assert_eq!(to_bytes256(Vec256::zero(cpu)), [0; 32]);
assert_eq!(to_bytes256(Vec256::zero(Vec256::zero(cpu).cpu())), [0; 32]);
assert_eq!(to_bytes256(Vec256::splat(0x85, cpu)), [0x85; 32]);
assert_eq!(to_bytes256(Vec256::splat(0x85, cpu).eq_u8(0x85)), [0xFF; 32]);
assert_eq!(to_u32s(Vec256::splat_u32(0x8234_5678, cpu)), [0x8234_5678; 8]);
let (a, b): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
let bytes = pattern(37, 11);
assert_eq!(to_bytes256(Vec256::read_unaligned(&bytes, cpu)), bytes);
let values = [1, 2, 3, 0x8000_0000, 5, 6, 7, u32::MAX];
assert_eq!(to_u32s(Vec256::read_unaligned(&values, cpu)), values);
assert_eq!(va.movemask(), top_bits(&a));
assert_eq!(vb.movemask(), top_bits(&b));
}
#[test]
fn vec256_from_vec128() {
fn vec128_shuffle_u8s() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 16] = pattern(37, 11);
let v = Vec128::read_unaligned(&bytes, cpu);
let splatted = to_bytes256(Vec256::splat_v128(v));
assert_eq!(splatted[..16], bytes);
assert_eq!(splatted[16..], bytes);
let expected: [u32; 8] = array::from_fn(|i| u32::from(bytes[i]));
assert_eq!(to_u32s(Vec256::widen_u8_to_u32(v)), expected);
let control: [u8; 16] = pattern(23, 5);
assert!(control.iter().any(|c| c & 0x80 != 0) && control.iter().any(|c| c & 0x80 == 0));
let shuffled = to_bytes128(v.shuffle_u8s(Vec128::read_unaligned(&control, cpu)));
assert_eq!(shuffled[..], shuffle(&bytes, &control));
}
#[test]
fn vec256_read_unaligned() {
fn vec128_blend() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 32] = pattern(37, 11);
// SAFETY: These types are all 32 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec256::read_unaligned(&bytes, cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i8; 32]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u16; 16]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i16; 16]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u32; 8]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i32; 8]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u64; 4]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i64; 4]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u128; 2]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i128; 2]>(bytes), cpu),
]
};
for v in vectors {
assert_eq!(to_bytes256(v), bytes);
}
let (a, b, mask) = (pattern(37, 11), pattern(101, 200), pattern(23, 5));
let v = Vec128::read_unaligned(&a, cpu)
.blend(Vec128::read_unaligned(&b, cpu), Vec128::read_unaligned(&mask, cpu));
let expected: [u8; 16] = array::from_fn(|i| if mask[i] & 0x80 != 0 { b[i] } else { a[i] });
assert_eq!(to_bytes128(v), expected);
let v = Vec128::read_unaligned(&a, cpu).blend(0x85, Vec128::read_unaligned(&mask, cpu));
let expected: [u8; 16] = array::from_fn(|i| if mask[i] & 0x80 != 0 { 0x85 } else { a[i] });
assert_eq!(to_bytes128(v), expected);
}
#[test]
fn vec256_load_store() {
fn vec256_movemask() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let src = Aligned(pattern::<32>(37, 11));
let mut dest = Aligned([0; 32]);
// SAFETY: `src` and `dest` are 32 bytes and aligned on 32
unsafe { Vec256::load(src.0.as_ptr(), cpu).store(dest.0.as_mut_ptr()) };
assert_eq!(dest.0, src.0);
let src = VecData256::from_bytes(pattern(37, 11));
let mut dest = VecData256::from_bytes([0; 32]);
Vec256::read(&src, cpu).write(&mut dest);
assert_eq!(*dest.as_bytes(), *src.as_bytes());
assert_eq!(to_bytes256(src.read(cpu)), *src.as_bytes());
let src = pattern::<34>(37, 11);
let mut dest = [0; 34];
// SAFETY: `src` and `dest` are valid for 32 bytes starting at offset 1
unsafe {
let v = Vec256::load_unaligned(src.as_ptr().add(1), cpu);
v.store_unaligned(dest.as_mut_ptr().add(1));
}
assert_eq!(dest[1..33], src[1..33]);
assert_eq!((dest[0], dest[33]), (0, 0));
let values = [1, u64::MAX, 0x0102_0304_0506_0708, 1 << 63];
let bytes = to_bytes256(Vec256::read_unaligned(&values, cpu));
assert_eq!(bytes[..], *values.map(u64::to_le_bytes).as_flattened());
}
#[test]
fn vec256_bitwise() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (a, b): ([u8; 32], [u8; 32]) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
assert_eq!(to_bytes256(va & vb), map2(a, b, |x, y| x & y));
assert_eq!(to_bytes256(va | vb), map2(a, b, |x, y| x | y));
assert_eq!(to_bytes256(va ^ vb), map2(a, b, |x, y| x ^ y));
assert_eq!(to_bytes256(!va), a.map(|x| !x));
assert_eq!(to_bytes256(va & 0x3C), a.map(|x| x & 0x3C));
assert_eq!(to_bytes256(va | 0x3C), a.map(|x| x | 0x3C));
assert_eq!(to_bytes256(va ^ 0x3C), a.map(|x| x ^ 0x3C));
let mut v = va;
v &= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x & y));
let mut v = va;
v |= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x | y));
let mut v = va;
v ^= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x ^ y));
let mut v = va;
v &= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x & 0x3C));
let mut v = va;
v |= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x | 0x3C));
let mut v = va;
v ^= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x ^ 0x3C));
}
#[test]
fn vec256_is_zero() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
assert!(Vec256::zero(cpu).is_zero());
assert!(!Vec256::splat(0xFF, cpu).is_zero());
// Check every byte, with only the lowest or highest bit set
for i in 0..32 {
for bit in [0x01, 0x80] {
let mut bytes = [0u8; 32];
bytes[i] = bit;
assert!(!Vec256::read_unaligned(&bytes, cpu).is_zero());
}
}
}
#[test]
fn vec256_add_u32() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let a: [u32; 8] = [1, 2, 3, 4, 5, 6, 7, u32::MAX];
let b: [u32; 8] = [10, 20, 30, 0x1_0000, 50, 60, 70, 2];
let sum = Vec256::read_unaligned(&a, cpu).add_u32(Vec256::read_unaligned(&b, cpu));
assert_eq!(to_u32s(sum), [11, 22, 33, 0x1_0004, 55, 66, 77, 1]);
let sum = Vec256::read_unaligned(&a, cpu).add_u32(16);
assert_eq!(to_u32s(sum), [17, 18, 19, 20, 21, 22, 23, 15]);
}
#[test]
fn vec256_add_sub() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
assert_eq!(to_bytes256(va.add_u8(vb)), map2(a, b, u8::wrapping_add));
assert_eq!(to_bytes256(va.saturating_add_u8(vb)), map2(a, b, u8::saturating_add));
assert_eq!(to_bytes256(va.sub_u8(vb)), map2(a, b, u8::wrapping_sub));
assert_eq!(to_bytes256(va.saturating_sub_u8(vb)), map2(a, b, u8::saturating_sub));
assert_eq!(to_bytes256(va.add_u8(0x90)), a.map(|x| x.wrapping_add(0x90)));
assert_eq!(to_bytes256(va.saturating_add_u8(0x90)), a.map(|x| x.saturating_add(0x90)));
assert_eq!(to_bytes256(va.sub_u8(0x90)), a.map(|x| x.wrapping_sub(0x90)));
assert_eq!(to_bytes256(va.saturating_sub_u8(0x90)), a.map(|x| x.saturating_sub(0x90)));
}
#[test]
fn vec256_compare() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, mut b): ([u8; 32], [u8; 32]) = (pattern(37, 11), pattern(101, 200));
b[3] = a[3];
b[30] = a[30];
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
let eq = to_bytes256(va.eq_u8(vb));
assert_eq!(eq, map2(a, b, |x, y| if x == y { 0xFF } else { 0 }));
assert_eq!(eq.iter().filter(|&&x| x != 0).count(), 2);
let eq = to_bytes256(va.eq_u8(a[5]));
assert_eq!(eq, a.map(|x| if x == a[5] { 0xFF } else { 0 }));
// `0xFF` is -1, so less than 1 as a signed integer
let gt = to_bytes256(va.gt_i8(vb));
assert_eq!(gt, map2(a, b, |x, y| if x.cast_signed() > y.cast_signed() { 0xFF } else { 0 }));
let lt = to_bytes256(va.lt_i8(vb));
assert_eq!(lt, map2(a, b, |x, y| if x.cast_signed() < y.cast_signed() { 0xFF } else { 0 }));
// Include values either side of the thresholds
let c: [u8; 32] = array::from_fn(|i| [0, 1, 0x1F, 0x20, 0x21, 0x7F, 0x80, 0xFF][i % 8]);
let vc = Vec256::read_unaligned(&c, cpu);
for n in [0, 0x20] {
let gt = to_bytes256(vc.gt_i8(n));
assert_eq!(gt, c.map(|x| if x.cast_signed() > n { 0xFF } else { 0 }));
let lt = to_bytes256(vc.lt_i8(n));
assert_eq!(lt, c.map(|x| if x.cast_signed() < n { 0xFF } else { 0 }));
}
}
#[test]
fn vec256_all_eq_u8() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let a: [u8; 32] = pattern(37, 11);
let va = Vec256::read_unaligned(&a, cpu);
assert!(va.all_eq_u8(va));
assert!(!va.all_eq_u8(a[0]));
assert!(Vec256::splat(0x85, cpu).all_eq_u8(0x85));
assert!(!Vec256::splat(0x85, cpu).all_eq_u8(0x84));
// Check every byte, with a difference of only the lowest or highest bit
for i in 0..32 {
for bit in [0x01, 0x80] {
let mut b = a;
b[i] ^= bit;
assert!(!va.all_eq_u8(Vec256::read_unaligned(&b, cpu)));
}
}
}
#[test]
fn vec256_shift() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let bytes = pattern(37, 11);
let v = Vec256::read_unaligned(&bytes, cpu);
assert_eq!(to_bytes256(v.shl_u16::<3>()), map_u16(bytes, |x| x << 3));
assert_eq!(to_bytes256(v.shr_u16::<4>()), map_u16(bytes, |x| x >> 4));
assert_eq!(va.movemask(), top_bits(&a));
assert_eq!(vb.movemask(), top_bits(&b));
}
#[test]
@@ -329,313 +106,12 @@ fn vec256_blend() {
}
#[test]
fn vec256_rearrange_u32s() {
fn vec256_shuffle_u32s() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let s = Vec256::read_unaligned(&[0u32, 1, 2, 3, 4, 5, 6, 7], cpu);
let o = Vec256::read_unaligned(&[10u32, 11, 12, 13, 14, 15, 16, 17], cpu);
// Only bottom 3 bits of indices are used
let indices = Vec256::read_unaligned(&[7u32, 0, 3, 3, 8 + 1, 2, 6, 4], cpu);
assert_eq!(to_u32s(s.shuffle_u32s(indices)), [7, 0, 3, 3, 1, 2, 6, 4]);
assert_eq!(to_u32s(s.unpack_lo_u32(o)), [0, 10, 1, 11, 4, 14, 5, 15]);
assert_eq!(to_u32s(s.unpack_hi_u32(o)), [2, 12, 3, 13, 6, 16, 7, 17]);
assert_eq!(to_u32s(s.join_lo_halves(o)), [0, 1, 2, 3, 10, 11, 12, 13]);
assert_eq!(to_u32s(s.join_hi_halves(o)), [4, 5, 6, 7, 14, 15, 16, 17]);
}
#[test]
fn vec256_movemask() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, b): ([u8; 32], [u8; 32]) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
assert_eq!(va.movemask(), top_bits(&a));
assert_eq!(vb.movemask(), top_bits(&b));
}
#[test]
fn vec128_create() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
assert_eq!(to_bytes128(Vec128::zero(cpu)), [0; 16]);
assert_eq!(to_bytes128(Vec128::splat(0x85, cpu)), [0x85; 16]);
let bytes = to_bytes128(Vec128::splat_u32(0x8234_5678, cpu));
assert_eq!(bytes[..], *[0x8234_5678u32; 4].map(u32::to_le_bytes).as_flattened());
assert_eq!(to_bytes128(0x8234_5678u32.into_vec128(cpu)), bytes);
let bytes = pattern(37, 11);
assert_eq!(to_bytes128(Vec128::read_unaligned(&bytes, cpu)), bytes);
let bytes: [u8; 8] = pattern(37, 11);
let v = to_bytes128(Vec128::read_low_unaligned(&bytes, cpu));
assert_eq!(v[..8], bytes);
assert_eq!(v[8..], [0; 8]);
}
#[test]
fn vec128_read_unaligned() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 16] = pattern(37, 11);
// SAFETY: These types are all 16 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec128::read_unaligned(&bytes, cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i8; 16]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u16; 8]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i16; 8]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u32; 4]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i32; 4]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u64; 2]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i64; 2]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u128; 1]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i128; 1]>(bytes), cpu),
]
};
for v in vectors {
assert_eq!(to_bytes128(v), bytes);
}
}
#[test]
fn vec128_read_low_unaligned() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 8] = pattern(37, 11);
// SAFETY: These types are all 8 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec128::read_low_unaligned(&bytes, cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i8; 8]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u16; 4]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i16; 4]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u32; 2]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i32; 2]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u64; 1]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i64; 1]>(bytes), cpu),
]
};
for v in vectors {
let result = to_bytes128(v);
assert_eq!(result[..8], bytes);
assert_eq!(result[8..], [0; 8]);
}
}
#[test]
fn vec128_load_store() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let src = Aligned(pattern::<16>(37, 11));
let mut dest = Aligned([0; 16]);
// SAFETY: `src` and `dest` are 16 bytes and aligned on 32, which is also aligned on 16
unsafe { Vec128::load(src.0.as_ptr(), cpu).store(dest.0.as_mut_ptr()) };
assert_eq!(dest.0, src.0);
let src = VecData128::from_bytes(pattern(37, 11));
let mut dest = VecData128::from_bytes([0; 16]);
Vec128::read(&src, cpu).write(&mut dest);
assert_eq!(*dest.as_bytes(), *src.as_bytes());
assert_eq!(to_bytes128(src.read(cpu)), *src.as_bytes());
let src = pattern::<18>(37, 11);
let mut dest = [0; 18];
// SAFETY: `src` and `dest` are valid for 16 bytes starting at offset 1
unsafe {
let v = Vec128::load_unaligned(src.as_ptr().add(1), cpu);
v.store_unaligned(dest.as_mut_ptr().add(1));
}
assert_eq!(dest[1..17], src[1..17]);
assert_eq!((dest[0], dest[17]), (0, 0));
let src = pattern::<10>(37, 11);
// SAFETY: `src` is valid for reading 8 bytes starting at offset 1
let v = unsafe { Vec128::load_low_unaligned(src.as_ptr().add(1), cpu) };
let bytes = to_bytes128(v);
assert_eq!(bytes[..8], src[1..9]);
assert_eq!(bytes[8..], [0; 8]);
let mut dest = [0; 10];
// SAFETY: `dest` is valid for writing 8 bytes starting at offset 1
unsafe { Vec128::splat(0x85, cpu).store_low_unaligned(dest.as_mut_ptr().add(1)) };
assert_eq!(dest[1..9], [0x85; 8]);
assert_eq!((dest[0], dest[9]), (0, 0));
let values: [u64; 2] = [0x0102_0304_0506_0708, 1 << 63];
let bytes = to_bytes128(Vec128::read_unaligned(&values, cpu));
assert_eq!(bytes[..], *values.map(u64::to_le_bytes).as_flattened());
}
#[test]
fn vec128_bitwise() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
assert_eq!(to_bytes128(va & vb), map2(a, b, |x, y| x & y));
assert_eq!(to_bytes128(va | vb), map2(a, b, |x, y| x | y));
assert_eq!(to_bytes128(va ^ vb), map2(a, b, |x, y| x ^ y));
assert_eq!(to_bytes128(!va), a.map(|x| !x));
assert_eq!(to_bytes128(va & 0x3C), a.map(|x| x & 0x3C));
assert_eq!(to_bytes128(va | 0x3C), a.map(|x| x | 0x3C));
assert_eq!(to_bytes128(va ^ 0x3C), a.map(|x| x ^ 0x3C));
let mut v = va;
v &= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x & y));
let mut v = va;
v |= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x | y));
let mut v = va;
v ^= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x ^ y));
let mut v = va;
v &= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x & 0x3C));
let mut v = va;
v |= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x | 0x3C));
let mut v = va;
v ^= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x ^ 0x3C));
}
#[test]
fn vec128_is_zero() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
assert!(Vec128::zero(cpu).is_zero());
assert!(!Vec128::splat(0xFF, cpu).is_zero());
// Check every byte, with only the lowest or highest bit set
for i in 0..16 {
for bit in [0x01, 0x80] {
let mut bytes = [0u8; 16];
bytes[i] = bit;
assert!(!Vec128::read_unaligned(&bytes, cpu).is_zero());
}
}
}
#[test]
fn vec128_add_sub() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
assert_eq!(to_bytes128(va.add_u8(vb)), map2(a, b, u8::wrapping_add));
assert_eq!(to_bytes128(va.saturating_add_u8(vb)), map2(a, b, u8::saturating_add));
assert_eq!(to_bytes128(va.sub_u8(vb)), map2(a, b, u8::wrapping_sub));
assert_eq!(to_bytes128(va.saturating_sub_u8(vb)), map2(a, b, u8::saturating_sub));
assert_eq!(to_bytes128(va.add_u8(0x90)), a.map(|x| x.wrapping_add(0x90)));
assert_eq!(to_bytes128(va.saturating_add_u8(0x90)), a.map(|x| x.saturating_add(0x90)));
assert_eq!(to_bytes128(va.sub_u8(0x90)), a.map(|x| x.wrapping_sub(0x90)));
assert_eq!(to_bytes128(va.saturating_sub_u8(0x90)), a.map(|x| x.saturating_sub(0x90)));
}
#[test]
fn vec128_compare() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, mut b): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
b[3] = a[3];
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
let eq = to_bytes128(va.eq_u8(vb));
assert_eq!(eq, map2(a, b, |x, y| if x == y { 0xFF } else { 0 }));
assert_eq!(eq[3], 0xFF);
let eq = to_bytes128(va.eq_u8(a[5]));
assert_eq!(eq, a.map(|x| if x == a[5] { 0xFF } else { 0 }));
let gt = to_bytes128(va.gt_i8(vb));
assert_eq!(gt, map2(a, b, |x, y| if x.cast_signed() > y.cast_signed() { 0xFF } else { 0 }));
let lt = to_bytes128(va.lt_i8(vb));
assert_eq!(lt, map2(a, b, |x, y| if x.cast_signed() < y.cast_signed() { 0xFF } else { 0 }));
// Include values either side of the thresholds
let c: [u8; 16] = array::from_fn(|i| [0, 1, 0x1F, 0x20, 0x21, 0x7F, 0x80, 0xFF][i % 8]);
let vc = Vec128::read_unaligned(&c, cpu);
for n in [0, 0x20] {
let gt = to_bytes128(vc.gt_i8(n));
assert_eq!(gt, c.map(|x| if x.cast_signed() > n { 0xFF } else { 0 }));
let lt = to_bytes128(vc.lt_i8(n));
assert_eq!(lt, c.map(|x| if x.cast_signed() < n { 0xFF } else { 0 }));
}
}
#[test]
fn vec128_all_eq_u8() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let a: [u8; 16] = pattern(37, 11);
let va = Vec128::read_unaligned(&a, cpu);
assert!(va.all_eq_u8(va));
assert!(!va.all_eq_u8(a[0]));
assert!(Vec128::splat(0x85, cpu).all_eq_u8(0x85));
assert!(!Vec128::splat(0x85, cpu).all_eq_u8(0x84));
// Check every byte, with a difference of only the lowest or highest bit
for i in 0..16 {
for bit in [0x01, 0x80] {
let mut b = a;
b[i] ^= bit;
assert!(!va.all_eq_u8(Vec128::read_unaligned(&b, cpu)));
}
}
}
#[test]
fn vec128_rearrange_bytes() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 16] = pattern(37, 11);
let v = Vec128::read_unaligned(&bytes, cpu);
let shifted = to_bytes128(v.shr_bytes::<3>());
assert_eq!(shifted[..13], bytes[3..]);
assert_eq!(shifted[13..], [0; 3]);
let control: [u8; 16] = pattern(23, 5);
assert!(control.iter().any(|c| c & 0x80 != 0) && control.iter().any(|c| c & 0x80 == 0));
let shuffled = to_bytes128(v.shuffle_u8s(Vec128::read_unaligned(&control, cpu)));
assert_eq!(shuffled[..], shuffle(&bytes, &control));
}
#[test]
fn vec128_blend() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, b, mask) = (pattern(37, 11), pattern(101, 200), pattern(23, 5));
let v = Vec128::read_unaligned(&a, cpu)
.blend(Vec128::read_unaligned(&b, cpu), Vec128::read_unaligned(&mask, cpu));
let expected: [u8; 16] = array::from_fn(|i| if mask[i] & 0x80 != 0 { b[i] } else { a[i] });
assert_eq!(to_bytes128(v), expected);
let v = Vec128::read_unaligned(&a, cpu).blend(0x85, Vec128::read_unaligned(&mask, cpu));
let expected: [u8; 16] = array::from_fn(|i| if mask[i] & 0x80 != 0 { 0x85 } else { a[i] });
assert_eq!(to_bytes128(v), expected);
}
#[test]
fn vec128_movemask() {
let DynCpu::Avx2(cpu) = DynCpu::detect() else { return };
let (a, b): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
assert_eq!(va.movemask(), top_bits(&a));
assert_eq!(vb.movemask(), top_bits(&b));
}
crates/oxc_lexer/src/simd/avx2/vec128.rs
+91 -74
View File
@@ -1,5 +1,7 @@
//! Methods of `Vec128<Avx2>`, which wraps an [`__m128i`].
//!
//! Implements [`Vec128Impl`], and has the methods which only AVX2 has.
//!
//! These are 128-bit operations, so the intrinsics are from SSE2 to SSE4.1,
//! which every CPU with AVX2 supports.
//!
@@ -21,48 +23,59 @@ use crate::{
},
};
/// Creation methods.
impl Vec128<Avx2> {
/// Create a [`Vec128`] wrapping an [`__m128i`].
///
/// Internal method only.
/// Internal method only. `pub(super)` so `Vec256` methods can use it.
#[inline(always)]
fn new(inner: __m128i, cpu: Avx2) -> Self {
pub(super) fn new(inner: __m128i, cpu: Avx2) -> Self {
Self { inner, cpu }
}
}
/// Get [`Avx2`] token from this [`Vec128`].
#[expect(dead_code)]
#[inline(always)]
pub fn cpu(self) -> Avx2 {
self.cpu
}
impl Vec128Impl for Vec128<Avx2> {
type Cpu = Avx2;
type Inner = __m128i;
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec128`] with all bits zero.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn zero(cpu: Avx2) -> Self {
fn zero(cpu: Avx2) -> Self {
// SAFETY: `Avx2` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_setzero_si128() }, cpu)
}
/// Create a [`Vec128`] by repeating `n` 16 times.
#[inline(always)]
pub fn splat(n: u8, cpu: Avx2) -> Self {
fn splat(n: u8, cpu: Avx2) -> Self {
// SAFETY: `Avx2` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_set1_epi8(n.cast_signed()) }, cpu)
}
/// Create a [`Vec128`] by repeating `n` 4 times.
#[inline(always)]
pub fn splat_u32(n: u32, cpu: Avx2) -> Self {
fn splat_u32(n: u32, cpu: Avx2) -> Self {
// SAFETY: `Avx2` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_set1_epi32(n.cast_signed()) }, cpu)
}
}
/// Loads and stores.
impl Vec128<Avx2> {
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get [`Avx2`] token from this [`Vec128`].
#[inline(always)]
fn cpu(self) -> Avx2 {
self.cpu
}
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec128`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
@@ -70,7 +83,7 @@ impl Vec128<Avx2> {
/// * `ptr` must be valid for reading 16 bytes.
/// * `ptr` must be aligned on 16.
#[inline(always)]
pub unsafe fn load(ptr: *const u8, cpu: Avx2) -> Self {
unsafe fn load(ptr: *const u8, cpu: Avx2) -> Self {
#[expect(clippy::cast_ptr_alignment, reason = "caller guarantees `ptr` is aligned")]
let ptr = ptr.cast::<__m128i>();
@@ -85,7 +98,7 @@ impl Vec128<Avx2> {
///
/// `ptr` must be valid for reading 16 bytes.
#[inline(always)]
pub unsafe fn load_unaligned(ptr: *const u8, cpu: Avx2) -> Self {
unsafe fn load_unaligned(ptr: *const u8, cpu: Avx2) -> Self {
#[expect(clippy::cast_ptr_alignment, reason = "intrinsic doesn't require alignment")]
let ptr = ptr.cast::<__m128i>();
@@ -101,7 +114,7 @@ impl Vec128<Avx2> {
///
/// `ptr` must be valid for reading 8 bytes.
#[inline(always)]
pub unsafe fn load_low_unaligned(ptr: *const u8, cpu: Avx2) -> Self {
unsafe fn load_low_unaligned(ptr: *const u8, cpu: Avx2) -> Self {
#[expect(clippy::cast_ptr_alignment, reason = "intrinsic doesn't require alignment")]
let ptr = ptr.cast::<__m128i>();
@@ -112,7 +125,7 @@ impl Vec128<Avx2> {
/// Read a [`Vec128`] from a [`VecData128`], which is aligned on 16.
#[inline(always)]
pub fn read(data: &VecData128, cpu: Avx2) -> Self {
fn read(data: &VecData128, cpu: Avx2) -> Self {
// SAFETY: `VecData128` guarantees `data.as_ptr()`
// is valid for aligned reads of 16 bytes
unsafe { Self::load(data.as_ptr(), cpu) }
@@ -123,7 +136,7 @@ impl Vec128<Avx2> {
/// `slice` can be any slice of integers which is 16 bytes in total
/// (e.g. `&[u8; 16]`, `&[i64; 2]`).
#[inline(always)]
pub fn read_unaligned(slice: &impl IntArray128, cpu: Avx2) -> Self {
fn read_unaligned(slice: &impl IntArray128, cpu: Avx2) -> Self {
// SAFETY: `IntArray128` guarantees `slice.as_ptr()`
// is valid for unaligned reads of 16 bytes
unsafe { Self::load_unaligned(slice.as_ptr(), cpu) }
@@ -135,7 +148,7 @@ impl Vec128<Avx2> {
/// `slice` can be any slice of integers which is 8 bytes in total
/// (e.g. `&[u8; 8]`, `&[i32; 2]`).
#[inline(always)]
pub fn read_low_unaligned(slice: &impl IntArray64, cpu: Avx2) -> Self {
fn read_low_unaligned(slice: &impl IntArray64, cpu: Avx2) -> Self {
// SAFETY: `IntArray64` guarantees `slice.as_ptr()`
// is valid for unaligned reads of 8 bytes
unsafe { Self::load_low_unaligned(slice.as_ptr(), cpu) }
@@ -147,9 +160,8 @@ impl Vec128<Avx2> {
///
/// * `ptr` must be valid for writing 16 bytes.
/// * `ptr` must be aligned on 16.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub unsafe fn store(self, ptr: *mut u8) {
unsafe fn store(self, ptr: *mut u8) {
#[expect(clippy::cast_ptr_alignment, reason = "caller guarantees `ptr` is aligned")]
let ptr = ptr.cast::<__m128i>();
@@ -164,7 +176,7 @@ impl Vec128<Avx2> {
///
/// `ptr` must be valid for writing 16 bytes.
#[inline(always)]
pub unsafe fn store_unaligned(self, ptr: *mut u8) {
unsafe fn store_unaligned(self, ptr: *mut u8) {
#[expect(clippy::cast_ptr_alignment, reason = "intrinsic doesn't require alignment")]
let ptr = ptr.cast::<__m128i>();
@@ -179,7 +191,7 @@ impl Vec128<Avx2> {
///
/// `ptr` must be valid for writing 8 bytes.
#[inline(always)]
pub unsafe fn store_low_unaligned(self, ptr: *mut u8) {
unsafe fn store_low_unaligned(self, ptr: *mut u8) {
#[expect(clippy::cast_ptr_alignment, reason = "intrinsic doesn't require alignment")]
let ptr = ptr.cast::<__m128i>();
@@ -189,42 +201,33 @@ impl Vec128<Avx2> {
}
/// Write a [`Vec128`] into a [`VecData128`], which is aligned on 16.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn write(self, dst: &mut VecData128) {
fn write(self, dst: &mut VecData128) {
// SAFETY: `VecData128` guarantees `dst.as_mut_ptr()`
// is valid for aligned writes of 16 bytes
unsafe { self.store(dst.as_mut_ptr()) }
}
}
/// Bitwise ops.
impl Vec128<Avx2> {
/// Gather the top bit of each byte of this [`Vec128`] into a 16-bit mask.
///
/// Bit N of the result is the top bit of byte N. Bits 16-31 of the result are zero.
#[inline(always)]
pub fn movemask(self) -> u32 {
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
unsafe { _mm_movemask_epi8(self.inner).cast_unsigned() }
}
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec128`] are zero.
#[inline(always)]
pub fn is_zero(self) -> bool {
fn is_zero(self) -> bool {
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE4.1
unsafe { _mm_testz_si128(self.inner, self.inner) != 0 }
}
/// Invert all the bits of this [`Vec128`].
#[inline(always)]
pub fn not(self) -> Self {
fn not(self) -> Self {
self.xor(Self::splat(0xFF, self.cpu))
}
/// AND this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn and(self, other: impl Operand128<u8>) -> Self {
fn and(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_and_si128(self.inner, other.inner) }, self.cpu)
@@ -232,7 +235,7 @@ impl Vec128<Avx2> {
/// OR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn or(self, other: impl Operand128<u8>) -> Self {
fn or(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_or_si128(self.inner, other.inner) }, self.cpu)
@@ -240,20 +243,20 @@ impl Vec128<Avx2> {
/// XOR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn xor(self, other: impl Operand128<u8>) -> Self {
fn xor(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_xor_si128(self.inner, other.inner) }, self.cpu)
}
}
/// Arithmetic ops.
impl Vec128<Avx2> {
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// wrapping on overflow.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn add_u8(self, other: impl Operand128<u8>) -> Self {
fn add_u8(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_add_epi8(self.inner, other.inner) }, self.cpu)
@@ -261,9 +264,8 @@ impl Vec128<Avx2> {
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// stopping at 255.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn saturating_add_u8(self, other: impl Operand128<u8>) -> Self {
fn saturating_add_u8(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_adds_epu8(self.inner, other.inner) }, self.cpu)
@@ -272,7 +274,7 @@ impl Vec128<Avx2> {
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// wrapping on overflow.
#[inline(always)]
pub fn sub_u8(self, other: impl Operand128<u8>) -> Self {
fn sub_u8(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_sub_epi8(self.inner, other.inner) }, self.cpu)
@@ -281,20 +283,21 @@ impl Vec128<Avx2> {
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// stopping at 0.
#[inline(always)]
pub fn saturating_sub_u8(self, other: impl Operand128<u8>) -> Self {
fn saturating_sub_u8(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_subs_epu8(self.inner, other.inner) }, self.cpu)
}
}
/// Comparison.
impl Vec128<Avx2> {
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec128`] to either another [`Vec128`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[inline(always)]
pub fn eq_u8(self, other: impl Operand128<u8>) -> Self {
fn eq_u8(self, other: impl Operand128<u8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_cmpeq_epi8(self.inner, other.inner) }, self.cpu)
@@ -302,7 +305,7 @@ impl Vec128<Avx2> {
/// Check if every byte of this [`Vec128`] equals either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn all_eq_u8(self, other: impl Operand128<u8>) -> bool {
fn all_eq_u8(self, other: impl Operand128<u8, Avx2>) -> bool {
self.xor(other).is_zero()
}
@@ -310,9 +313,8 @@ impl Vec128<Avx2> {
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn lt_i8(self, other: impl Operand128<i8>) -> Self {
fn lt_i8(self, other: impl Operand128<i8, Avx2>) -> Self {
other.into_vec128(self.cpu).gt_i8(self)
}
@@ -321,20 +323,21 @@ impl Vec128<Avx2> {
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[inline(always)]
pub fn gt_i8(self, other: impl Operand128<i8>) -> Self {
fn gt_i8(self, other: impl Operand128<i8, Avx2>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_cmpgt_epi8(self.inner, other.inner) }, self.cpu)
}
}
/// Bytewise ops.
impl Vec128<Avx2> {
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Shift the whole of this [`Vec128`] right by `BYTES` bytes, filling with zeros.
///
/// Byte `N` moves to byte 0.
#[inline(always)]
pub fn shr_bytes<const BYTES: i32>(self) -> Self {
fn shr_bytes<const BYTES: i32>(self) -> Self {
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
Self::new(unsafe { _mm_srli_si128::<BYTES>(self.inner) }, self.cpu)
}
@@ -346,7 +349,7 @@ impl Vec128<Avx2> {
/// * If the top bit of `control[i]` is set, `result[i]` is 0.
/// * Otherwise, `result[i]` is `self[control[i] & 0x0F]`.
#[inline(always)]
pub fn shuffle_u8s(self, control: Self) -> Self {
fn shuffle_u8s(self, control: Self) -> Self {
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSSE3
Self::new(unsafe { _mm_shuffle_epi8(self.inner, control.inner) }, self.cpu)
}
@@ -355,21 +358,35 @@ impl Vec128<Avx2> {
///
/// Each byte of the result comes from `other` if the top bit of the same byte of `mask` is set,
/// otherwise from `self`. Other bits of `mask` are ignored.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn blend(self, other: impl Operand128<u8>, mask: Self) -> Self {
fn blend(self, other: impl Operand128<u8, Avx2>, mask: Self) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE4.1
Self::new(unsafe { _mm_blendv_epi8(self.inner, other.inner, mask.inner) }, self.cpu)
}
}
impl Vec128Impl for Vec128<Avx2> {
type Cpu = Avx2;
// ----------------------------------------
// `u64`-wise ops
// ----------------------------------------
/// Get one of the 2 `u64`s of this [`Vec128`].
///
/// `INDEX` is 0 for the low 8 bytes, or 1 for the high 8 bytes.
#[inline(always)]
fn read(data: &VecData128, cpu: Avx2) -> Self {
// Inherent methods take priority, so this calls `Vec128::read` above, not itself
Vec128::read(data, cpu)
fn get_u64<const INDEX: i32>(self) -> u64 {
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE4.1
unsafe { _mm_extract_epi64::<INDEX>(self.inner).cast_unsigned() }
}
}
/// Methods which are specific to this architecture.
impl Vec128<Avx2> {
/// Gather the top bit of each byte of this [`Vec128`] into a 16-bit mask.
///
/// Bit N of the result is the top bit of byte N. Bits 16-31 of the result are zero.
#[inline(always)]
pub fn movemask(self) -> u32 {
// SAFETY: `Vec128` type guarantees platform supports AVX2, which includes SSE2
unsafe { _mm_movemask_epi8(self.inner).cast_unsigned() }
}
}
crates/oxc_lexer/src/simd/avx2/vec256.rs
+262 -236
View File
@@ -1,5 +1,7 @@
//! Methods of `Vec256<Avx2>`, which wraps an [`__m256i`].
//!
//! Implements [`Vec256Impl`], and has the methods which only AVX2 has.
//!
//! Some AVX2 instructions treat a 256-bit vector as 2 separate 128-bit halves
//! (e.g. `shuffle_u8s`, `unpack_lo_u32`). The docs of those methods say so.
//!
@@ -17,7 +19,6 @@ use crate::{
simd::{Vec128, Vec256, Vec256Impl, VecData256, arrays::IntArray256, operand::Operand256},
};
/// Creation methods.
impl Vec256<Avx2> {
/// Create a [`Vec256`] wrapping an [`__m256i`].
///
@@ -26,51 +27,65 @@ impl Vec256<Avx2> {
fn new(inner: __m256i, cpu: Avx2) -> Self {
Self { inner, cpu }
}
}
/// Get [`Avx2`] token from this [`Vec256`].
#[inline(always)]
pub fn cpu(self) -> Avx2 {
self.cpu
}
impl Vec256Impl for Vec256<Avx2> {
type Cpu = Avx2;
type Inner = __m256i;
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec256`] with all bits zero.
#[inline(always)]
pub fn zero(cpu: Avx2) -> Self {
fn zero(cpu: Avx2) -> Self {
// SAFETY: `Avx2` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_setzero_si256() }, cpu)
}
/// Create a [`Vec256`] by repeating `n` 32 times.
#[inline(always)]
pub fn splat(n: u8, cpu: Avx2) -> Self {
fn splat(n: u8, cpu: Avx2) -> Self {
// SAFETY: `Avx2` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_set1_epi8(n.cast_signed()) }, cpu)
}
/// Create a [`Vec256`] by repeating `n` 8 times.
#[inline(always)]
pub fn splat_u32(n: u32, cpu: Avx2) -> Self {
fn splat_u32(n: u32, cpu: Avx2) -> Self {
// SAFETY: `Avx2` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_set1_epi32(n.cast_signed()) }, cpu)
}
/// Create a [`Vec256`] by repeating `v` twice.
#[inline(always)]
pub fn splat_v128(v: Vec128<Avx2>) -> Self {
fn splat_v128(v: Vec128<Avx2>) -> Self {
// SAFETY: `Vec128` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_broadcastsi128_si256(v.inner) }, v.cpu)
}
/// Create a [`Vec256`] by zero-extending the low 8 bytes of `v` to 8 `u32`s.
#[inline(always)]
pub fn widen_u8_to_u32(v: Vec128<Avx2>) -> Self {
fn widen_u8_to_u32(v: Vec128<Avx2>) -> Self {
// SAFETY: `Vec128` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_cvtepu8_epi32(v.inner) }, v.cpu)
}
}
/// Loads and stores.
impl Vec256<Avx2> {
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get [`Avx2`] token from this [`Vec256`].
#[inline(always)]
fn cpu(self) -> Avx2 {
self.cpu
}
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec256`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
@@ -78,7 +93,7 @@ impl Vec256<Avx2> {
/// * `ptr` must be valid for reading 32 bytes.
/// * `ptr` must be aligned on 32.
#[inline(always)]
pub unsafe fn load(ptr: *const u8, cpu: Avx2) -> Self {
unsafe fn load(ptr: *const u8, cpu: Avx2) -> Self {
#[expect(clippy::cast_ptr_alignment, reason = "caller guarantees `ptr` is aligned")]
let ptr = ptr.cast::<__m256i>();
@@ -93,7 +108,7 @@ impl Vec256<Avx2> {
///
/// `ptr` must be valid for reading 32 bytes.
#[inline(always)]
pub unsafe fn load_unaligned(ptr: *const u8, cpu: Avx2) -> Self {
unsafe fn load_unaligned(ptr: *const u8, cpu: Avx2) -> Self {
#[expect(clippy::cast_ptr_alignment, reason = "intrinsic doesn't require alignment")]
let ptr = ptr.cast::<__m256i>();
@@ -104,7 +119,7 @@ impl Vec256<Avx2> {
/// Read a [`Vec256`] from a [`VecData256`], which is aligned on 32.
#[inline(always)]
pub fn read(data: &VecData256, cpu: Avx2) -> Self {
fn read(data: &VecData256, cpu: Avx2) -> Self {
// SAFETY: `VecData256` guarantees `data.as_ptr()`
// is valid for aligned reads of 32 bytes
unsafe { Self::load(data.as_ptr(), cpu) }
@@ -115,7 +130,7 @@ impl Vec256<Avx2> {
/// `slice` can be any slice of integers which is 32 bytes in total
/// (e.g. `&[u8; 32]`, `&[i64; 4]`).
#[inline(always)]
pub fn read_unaligned(slice: &impl IntArray256, cpu: Avx2) -> Self {
fn read_unaligned(slice: &impl IntArray256, cpu: Avx2) -> Self {
// SAFETY: `IntArray256` guarantees `slice.as_ptr()`
// is valid for unaligned reads of 32 bytes
unsafe { Self::load_unaligned(slice.as_ptr(), cpu) }
@@ -127,9 +142,8 @@ impl Vec256<Avx2> {
///
/// * `ptr` must be valid for writing 32 bytes.
/// * `ptr` must be aligned on 32.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub unsafe fn store(self, ptr: *mut u8) {
unsafe fn store(self, ptr: *mut u8) {
#[expect(clippy::cast_ptr_alignment, reason = "caller guarantees `ptr` is aligned")]
let ptr = ptr.cast::<__m256i>();
@@ -144,7 +158,7 @@ impl Vec256<Avx2> {
///
/// `ptr` must be valid for writing 32 bytes.
#[inline(always)]
pub unsafe fn store_unaligned(self, ptr: *mut u8) {
unsafe fn store_unaligned(self, ptr: *mut u8) {
#[expect(clippy::cast_ptr_alignment, reason = "intrinsic doesn't require alignment")]
let ptr = ptr.cast::<__m256i>();
@@ -153,19 +167,240 @@ impl Vec256<Avx2> {
unsafe { _mm256_storeu_si256(ptr, self.inner) }
}
/// Write a [`Vec256`] into a [`VecData256`].
///
/// This is an aligned write.
#[cfg_attr(not(test), expect(dead_code))]
/// Write a [`Vec256`] into a [`VecData256`], which is aligned on 32.
#[inline(always)]
pub fn write(self, dst: &mut VecData256) {
fn write(self, dst: &mut VecData256) {
// SAFETY: `VecData256` guarantees `dst.as_mut_ptr()`
// is valid for aligned writes of 32 bytes
unsafe { self.store(dst.as_mut_ptr()) }
}
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec256`] are zero.
#[inline(always)]
fn is_zero(self) -> bool {
// SAFETY: `Vec256` type guarantees platform supports AVX2
unsafe { _mm256_testz_si256(self.inner, self.inner) != 0 }
}
/// Invert all the bits of this [`Vec256`].
#[inline(always)]
fn not(self) -> Self {
self.xor(Self::splat(0xFF, self.cpu))
}
/// AND this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
fn and(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_and_si256(self.inner, other.inner) }, self.cpu)
}
/// OR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
fn or(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_or_si256(self.inner, other.inner) }, self.cpu)
}
/// XOR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
fn xor(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_xor_si256(self.inner, other.inner) }, self.cpu)
}
/// Shift each of the 16 `u16`s of this [`Vec256`] left by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
#[inline(always)]
fn shl_u16<const BITS: i32>(self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_slli_epi16::<BITS>(self.inner) }, self.cpu)
}
/// Shift each of the 16 `u16`s of this [`Vec256`] right by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
#[inline(always)]
fn shr_u16<const BITS: i32>(self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_srli_epi16::<BITS>(self.inner) }, self.cpu)
}
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// wrapping on overflow.
#[inline(always)]
fn add_u8(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_add_epi8(self.inner, other.inner) }, self.cpu)
}
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// stopping at 255.
#[inline(always)]
fn saturating_add_u8(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_adds_epu8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// wrapping on overflow.
#[inline(always)]
fn sub_u8(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_sub_epi8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// stopping at 0.
#[inline(always)]
fn saturating_sub_u8(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_subs_epu8(self.inner, other.inner) }, self.cpu)
}
/// Add either another [`Vec256`] or a `u32` to every `u32` of this [`Vec256`],
/// wrapping on overflow.
#[inline(always)]
fn add_u32(self, other: impl Operand256<u32, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_add_epi32(self.inner, other.inner) }, self.cpu)
}
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec256`] to either another [`Vec256`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[inline(always)]
fn eq_u8(self, other: impl Operand256<u8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_cmpeq_epi8(self.inner, other.inner) }, self.cpu)
}
/// Check if every byte of this [`Vec256`] equals either another [`Vec256`] or a `u8`.
#[inline(always)]
fn all_eq_u8(self, other: impl Operand256<u8, Avx2>) -> bool {
self.xor(other).is_zero()
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[inline(always)]
fn lt_i8(self, other: impl Operand256<i8, Avx2>) -> Self {
other.into_vec256(self.cpu).gt_i8(self)
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[inline(always)]
fn gt_i8(self, other: impl Operand256<i8, Avx2>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_cmpgt_epi8(self.inner, other.inner) }, self.cpu)
}
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Combine the bytes of this [`Vec256`] and either another [`Vec256`] or a `u8`.
///
/// Each byte of the result comes from `other` if the top bit of the same byte of `mask` is set,
/// otherwise from `self`. Other bits of `mask` are ignored.
#[inline(always)]
fn blend(self, other: impl Operand256<u8, Avx2>, mask: Self) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_blendv_epi8(self.inner, other.inner, mask.inner) }, self.cpu)
}
// ----------------------------------------
// `u32`-wise ops
// ----------------------------------------
/// Interleave the low 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s0, o0, s1, o1, s4, o4, s5, o5]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[inline(always)]
fn unpack_lo_u32(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_unpacklo_epi32(self.inner, other.inner) }, self.cpu)
}
/// Interleave the high 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s2, o2, s3, o3, s6, o6, s7, o7]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[inline(always)]
fn unpack_hi_u32(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_unpackhi_epi32(self.inner, other.inner) }, self.cpu)
}
// ----------------------------------------
// Operations on 128-bit halves
// ----------------------------------------
/// Create a [`Vec256`] from its low and high 128 bits.
#[inline(always)]
fn from_halves(low: Vec128<Avx2>, high: Vec128<Avx2>) -> Self {
// SAFETY: `Vec128` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_set_m128i(high.inner, low.inner) }, low.cpu)
}
/// Get the low and high 128 bits of this [`Vec256`].
#[inline(always)]
fn into_halves(self) -> (Vec128<Avx2>, Vec128<Avx2>) {
// SAFETY: `Vec256` type guarantees platform supports AVX2
let (low, high) = unsafe {
(_mm256_castsi256_si128(self.inner), _mm256_extracti128_si256::<1>(self.inner))
};
(Vec128::new(low, self.cpu), Vec128::new(high, self.cpu))
}
/// Get a [`Vec256`] made of the low 128 bits of this [`Vec256`],
/// followed by the low 128 bits of `other`.
#[inline(always)]
fn join_lo_halves(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_permute2x128_si256::<0x20>(self.inner, other.inner) }, self.cpu)
}
/// Get a [`Vec256`] made of the high 128 bits of this [`Vec256`],
/// followed by the high 128 bits of `other`.
#[inline(always)]
fn join_hi_halves(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_permute2x128_si256::<0x31>(self.inner, other.inner) }, self.cpu)
}
}
/// Bitwise ops.
/// Methods which are specific to this architecture.
impl Vec256<Avx2> {
/// Gather the top bit of each byte of this [`Vec256`] into a 32-bit mask.
///
@@ -176,152 +411,6 @@ impl Vec256<Avx2> {
unsafe { _mm256_movemask_epi8(self.inner).cast_unsigned() }
}
/// Check if all the bits of this [`Vec256`] are zero.
#[inline(always)]
pub fn is_zero(self) -> bool {
// SAFETY: `Vec256` type guarantees platform supports AVX2
unsafe { _mm256_testz_si256(self.inner, self.inner) != 0 }
}
/// Invert all the bits of this [`Vec256`].
#[inline(always)]
pub fn not(self) -> Self {
self.xor(Self::splat(0xFF, self.cpu))
}
/// Shift each of the 16 `u16`s of this [`Vec256`] left by `BITS` bits.
#[inline(always)]
pub fn shl_u16<const BITS: i32>(self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_slli_epi16::<BITS>(self.inner) }, self.cpu)
}
/// Shift each of the 16 `u16`s of this [`Vec256`] right by `BITS` bits.
#[inline(always)]
pub fn shr_u16<const BITS: i32>(self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_srli_epi16::<BITS>(self.inner) }, self.cpu)
}
/// AND this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn and(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_and_si256(self.inner, other.inner) }, self.cpu)
}
/// OR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn or(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_or_si256(self.inner, other.inner) }, self.cpu)
}
/// XOR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn xor(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_xor_si256(self.inner, other.inner) }, self.cpu)
}
}
/// Arithmetic ops.
impl Vec256<Avx2> {
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// wrapping on overflow.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn add_u8(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_add_epi8(self.inner, other.inner) }, self.cpu)
}
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// stopping at 255.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn saturating_add_u8(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_adds_epu8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// wrapping on overflow.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn sub_u8(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_sub_epi8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// stopping at 0.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn saturating_sub_u8(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_subs_epu8(self.inner, other.inner) }, self.cpu)
}
/// Add either another [`Vec256`] or a `u32` to every `u32` of this [`Vec256`],
/// wrapping on overflow.
#[inline(always)]
pub fn add_u32(self, other: impl Operand256<u32>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_add_epi32(self.inner, other.inner) }, self.cpu)
}
}
/// Comparison.
impl Vec256<Avx2> {
/// Compare every byte of this [`Vec256`] to either another [`Vec256`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[inline(always)]
pub fn eq_u8(self, other: impl Operand256<u8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_cmpeq_epi8(self.inner, other.inner) }, self.cpu)
}
/// Check if every byte of this [`Vec256`] equals either another [`Vec256`] or a `u8`.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn all_eq_u8(self, other: impl Operand256<u8>) -> bool {
self.xor(other).is_zero()
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[inline(always)]
pub fn lt_i8(self, other: impl Operand256<i8>) -> Self {
other.into_vec256(self.cpu).gt_i8(self)
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[inline(always)]
pub fn gt_i8(self, other: impl Operand256<i8>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_cmpgt_epi8(self.inner, other.inner) }, self.cpu)
}
}
/// Bytewise ops.
impl Vec256<Avx2> {
/// Rearrange the bytes of this [`Vec256`].
///
/// The low and high 128 bits are treated as 2 separate 16-byte vectors,
@@ -339,20 +428,6 @@ impl Vec256<Avx2> {
Self::new(unsafe { _mm256_shuffle_epi8(self.inner, control.inner) }, self.cpu)
}
/// Combine the bytes of this [`Vec256`] and either another [`Vec256`] or a `u8`.
///
/// Each byte of the result comes from `other` if the top bit of the same byte of `mask` is set,
/// otherwise from `self`. Other bits of `mask` are ignored.
#[inline(always)]
pub fn blend(self, other: impl Operand256<u8>, mask: Self) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_blendv_epi8(self.inner, other.inner, mask.inner) }, self.cpu)
}
}
/// `u32`-wise ops.
impl Vec256<Avx2> {
/// Rearrange the 8 `u32`s of this [`Vec256`].
///
/// For each `u32` `i`, `result[i]` is `self[indices[i] & 7]`.
@@ -365,53 +440,4 @@ impl Vec256<Avx2> {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_permutevar8x32_epi32(self.inner, indices.inner) }, self.cpu)
}
/// Interleave the low 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s0, o0, s1, o1, s4, o4, s5, o5]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[inline(always)]
pub fn unpack_lo_u32(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_unpacklo_epi32(self.inner, other.inner) }, self.cpu)
}
/// Interleave the high 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s2, o2, s3, o3, s6, o6, s7, o7]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[inline(always)]
pub fn unpack_hi_u32(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_unpackhi_epi32(self.inner, other.inner) }, self.cpu)
}
}
/// Operations on 128-bit halves.
impl Vec256<Avx2> {
/// Get a [`Vec256`] made of the low 128 bits of this [`Vec256`],
/// followed by the low 128 bits of `other`.
#[inline(always)]
pub fn join_lo_halves(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_permute2x128_si256::<0x20>(self.inner, other.inner) }, self.cpu)
}
/// Get a [`Vec256`] made of the high 128 bits of this [`Vec256`],
/// followed by the high 128 bits of `other`.
#[inline(always)]
pub fn join_hi_halves(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports AVX2
Self::new(unsafe { _mm256_permute2x128_si256::<0x31>(self.inner, other.inner) }, self.cpu)
}
}
impl Vec256Impl for Vec256<Avx2> {
type Cpu = Avx2;
#[inline(always)]
fn read(data: &VecData256, cpu: Avx2) -> Self {
// Inherent methods take priority, so this calls `Vec256::read` above, not itself
Vec256::read(data, cpu)
}
}
crates/oxc_lexer/src/simd/bit_traits.rs
-138
View File
@@ -1,138 +0,0 @@
//! Bitwise operators for [`Vec128`] and [`Vec256`].
//!
//! `!v`, `v & x`, `v | x` and `v ^ x` call the vector's `not`, `and`, `or` and `xor` methods,
//! and so do `&=`, `|=` and `^=`. As with those methods, `x` can be a vector or a `u8`,
//! so `v & 0x0f` works.
//!
//! The operators are only implemented for vectors of `CpuType`,
//! which is the CPU feature set with vector types on this platform.
#![expect(
clippy::inline_always,
reason = "operators call vector methods, which must inline into the function calling them"
)]
use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not};
use super::{
CpuType, Vec128, Vec256,
operand::{Operand128, Operand256},
};
// Vec128
impl Not for Vec128<CpuType> {
type Output = Self;
#[inline(always)]
fn not(self) -> Self {
Vec128::not(self)
}
}
impl<T: Operand128<u8>> BitAnd<T> for Vec128<CpuType> {
type Output = Self;
#[inline(always)]
fn bitand(self, rhs: T) -> Self {
self.and(rhs)
}
}
impl<T: Operand128<u8>> BitAndAssign<T> for Vec128<CpuType> {
#[inline(always)]
fn bitand_assign(&mut self, rhs: T) {
*self = self.and(rhs);
}
}
impl<T: Operand128<u8>> BitOr<T> for Vec128<CpuType> {
type Output = Self;
#[inline(always)]
fn bitor(self, rhs: T) -> Self {
self.or(rhs)
}
}
impl<T: Operand128<u8>> BitOrAssign<T> for Vec128<CpuType> {
#[inline(always)]
fn bitor_assign(&mut self, rhs: T) {
*self = self.or(rhs);
}
}
impl<T: Operand128<u8>> BitXor<T> for Vec128<CpuType> {
type Output = Self;
#[inline(always)]
fn bitxor(self, rhs: T) -> Self {
self.xor(rhs)
}
}
impl<T: Operand128<u8>> BitXorAssign<T> for Vec128<CpuType> {
#[inline(always)]
fn bitxor_assign(&mut self, rhs: T) {
*self = self.xor(rhs);
}
}
// Vec256
impl Not for Vec256<CpuType> {
type Output = Self;
#[inline(always)]
fn not(self) -> Self {
Vec256::not(self)
}
}
impl<T: Operand256<u8>> BitAnd<T> for Vec256<CpuType> {
type Output = Self;
#[inline(always)]
fn bitand(self, rhs: T) -> Self {
self.and(rhs)
}
}
impl<T: Operand256<u8>> BitAndAssign<T> for Vec256<CpuType> {
#[inline(always)]
fn bitand_assign(&mut self, rhs: T) {
*self = self.and(rhs);
}
}
impl<T: Operand256<u8>> BitOr<T> for Vec256<CpuType> {
type Output = Self;
#[inline(always)]
fn bitor(self, rhs: T) -> Self {
self.or(rhs)
}
}
impl<T: Operand256<u8>> BitOrAssign<T> for Vec256<CpuType> {
#[inline(always)]
fn bitor_assign(&mut self, rhs: T) {
*self = self.or(rhs);
}
}
impl<T: Operand256<u8>> BitXor<T> for Vec256<CpuType> {
type Output = Self;
#[inline(always)]
fn bitxor(self, rhs: T) -> Self {
self.xor(rhs)
}
}
impl<T: Operand256<u8>> BitXorAssign<T> for Vec256<CpuType> {
#[inline(always)]
fn bitxor_assign(&mut self, rhs: T) {
*self = self.xor(rhs);
}
}
crates/oxc_lexer/src/simd/data.rs
-2
View File
@@ -103,7 +103,6 @@ impl VecData128 {
/// Get a `*mut u8` pointing to start of the array.
///
/// This pointer is guaranteed to be valid for aligned writes of 16 bytes.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub const fn as_mut_ptr(&mut self) -> *mut u8 {
self.0.as_mut_ptr()
@@ -223,7 +222,6 @@ impl VecData256 {
/// Get a `*mut u8` pointing to start of the array.
///
/// This pointer is guaranteed to be valid for aligned writes of 32 bytes.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub const fn as_mut_ptr(&mut self) -> *mut u8 {
self.0.as_mut_ptr()
crates/oxc_lexer/src/simd/mod.rs
+17 -71
View File
@@ -5,7 +5,11 @@
//! A vector holds a `C`, so it can only exist if the CPU supports that feature set.
//! So only the methods which load or store via a raw pointer are unsafe.
//!
//! Each feature set's methods are in its own module, `avx2` on x86_64 and `neon` on aarch64.
//! [`Vec128`] and [`Vec256`] are in the `vec128` and `vec256` modules, along with the
//! [`Vec128Impl`] and [`Vec256Impl`] traits, which hold the methods which every feature set has,
//! and the `!`, `&`, `|` and `^` operators.
//! Each feature set implements the traits in its own module,
//! `avx2` on x86_64 and `neon` on aarch64, along with the methods which only it has.
//! Other platforms have no vector types.
//!
//! The other modules:
@@ -13,15 +17,6 @@
//! - `data` has [`VecData128`] and [`VecData256`], aligned byte arrays to read into vectors.
//! - `arrays` has traits for arrays of integers which vectors can read without alignment.
//! - `operand` has traits which let an operation take either a vector or a scalar.
//! - `bit_traits` implements the `!`, `&`, `|` and `^` operators for vectors.
//!
//! `operand` and `bit_traits` only cover vectors of `CpuType`,
//! which is the CPU feature set with vector types on this platform.
#[cfg(target_arch = "x86_64")]
use crate::cpu::Avx2 as CpuType;
#[cfg(target_arch = "aarch64")]
use crate::cpu::Neon as CpuType;
#[cfg(target_arch = "x86_64")]
mod avx2;
@@ -30,15 +25,25 @@ mod neon;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
mod arrays;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
mod bit_traits;
#[cfg(any(test, target_arch = "x86_64", target_arch = "aarch64"))]
mod data;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
mod operand;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
mod vec128;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
mod vec256;
#[cfg(any(test, target_arch = "x86_64", target_arch = "aarch64"))]
pub use data::{VecData128, VecData256};
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
pub use vec128::Vec128;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
use vec128::Vec128Impl;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
pub use vec256::Vec256;
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
use vec256::Vec256Impl;
#[cfg(all(test, any(target_arch = "x86_64", target_arch = "aarch64")))]
mod tests;
@@ -50,75 +55,16 @@ pub trait CpuSimd: Copy {
type Vec128: Vec128Impl<Cpu = Self>;
/// [`Vec256`] for this CPU feature set.
type Vec256: Vec256Impl<Cpu = Self>;
/// Type which a [`Vec128`] wraps.
type Vec128Inner: Copy;
/// Type which a [`Vec256`] wraps.
type Vec256Inner: Copy;
}
#[cfg(target_arch = "x86_64")]
impl CpuSimd for crate::cpu::Avx2 {
type Vec128 = Vec128<Self>;
type Vec256 = Vec256<Self>;
type Vec128Inner = std::arch::x86_64::__m128i;
type Vec256Inner = std::arch::x86_64::__m256i;
}
#[cfg(target_arch = "aarch64")]
impl CpuSimd for crate::cpu::Neon {
type Vec128 = Vec128<Self>;
type Vec256 = Vec256<Self>;
type Vec128Inner = std::arch::aarch64::uint8x16_t;
type Vec256Inner = std::arch::aarch64::uint8x16x2_t;
}
/// Methods of [`Vec128`] which code that is generic over the CPU type can call.
///
/// Each method is also an inherent method of [`Vec128`], which can be called without this trait in scope.
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
pub trait Vec128Impl {
/// CPU feature set which this vector is for.
type Cpu: CpuSimd;
/// Read a [`Vec128`] from a [`VecData128`].
fn read(data: &VecData128, cpu: Self::Cpu) -> Self;
}
/// Methods of [`Vec256`] which code that is generic over the CPU type can call.
///
/// Each method is also an inherent method of [`Vec256`], which can be called without this trait in scope.
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
pub trait Vec256Impl {
/// CPU feature set which this vector is for.
type Cpu: CpuSimd;
/// Read a [`Vec256`] from a [`VecData256`].
fn read(data: &VecData256, cpu: Self::Cpu) -> Self;
}
/// 128-bit SIMD vector.
///
/// Holds a `C`, so a vector can only exist if the CPU supports that feature set.
/// `C` is zero-sized, so [`Vec128`] is the same size as the type it wraps.
///
/// The methods are in the `avx2` and `neon` modules.
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
#[derive(Clone, Copy)]
pub struct Vec128<C: CpuSimd> {
inner: C::Vec128Inner,
cpu: C,
}
/// 256-bit SIMD vector.
///
/// Holds a `C`, so a vector can only exist if the CPU supports that feature set.
/// `C` is zero-sized, so [`Vec256`] is the same size as the type it wraps.
///
/// The methods are in the `avx2` and `neon` modules.
#[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))]
#[derive(Clone, Copy)]
pub struct Vec256<C: CpuSimd> {
inner: C::Vec256Inner,
cpu: C,
}
crates/oxc_lexer/src/simd/neon/mod.rs
+8 -7
View File
@@ -15,19 +15,21 @@
//!
//! # Differences from AVX2
//!
//! Methods have the same names as the methods of `Vec128<Avx2>` which do the same job,
//! but these differ in behavior:
//! Methods have the same names as the methods of `Vec128<Avx2>` and `Vec256<Avx2>`
//! which do the same job, but these differ in behavior:
//!
//! * [`shuffle_u8s`] reads all 8 bits of each control byte.
//! On AVX2, a control byte with the top bit set gives 0, and otherwise only its low 4 bits count.
//! * [`blend`] selects each bit separately. On AVX2, it selects whole bytes, using their top bits.
//! * `blend` selects each bit separately. On AVX2, it selects whole bytes, using their top bits.
//! * [`Vec256::shuffle_u8s`] is 1 lookup in a 32-entry table, with a 16-byte control.
//! On AVX2, it's 2 separate lookups in 16-entry tables, with a 32-byte control.
//! * `load`, `store`, `read` and `write` use the same instructions as their unaligned versions.
//! Neon has no separate aligned load and store instructions.
//!
//! `Vec128<Avx2>` also has some methods which `Vec128<Neon>` doesn't:
//! Some methods only exist for one CPU, because the other CPU has no matching instruction:
//!
//! * `movemask`. Neon has no equivalent of x86's `pmovmskb`, so it would be several instructions.
//! * Aligned `load` and `store`. Aligned loads and stores have no advantage on Neon.
//! * AVX2 only: `movemask` (Neon has no equivalent of x86's `pmovmskb`) and `Vec256::shuffle_u32s`.
//! * Neon only: `shl_u8`, `shr_u8`, `pairwise_add_u8` and `test_u8`.
//!
//! # Performance
//!
@@ -37,7 +39,6 @@
//!
//! [`Neon`]: crate::cpu::Neon
//! [`Vec256::shuffle_u8s`]: super::Vec256::shuffle_u8s
//! [`blend`]: super::Vec128::blend
//! [`shuffle_u8s`]: super::Vec128::shuffle_u8s
//! [`uint8x16_t`]: std::arch::aarch64::uint8x16_t
//! [`uint8x16x2_t`]: std::arch::aarch64::uint8x16x2_t
crates/oxc_lexer/src/simd/neon/tests.rs
+34 -378
View File
@@ -1,180 +1,20 @@
//! Tests for the methods of `Vec128<Neon>` and `Vec256<Neon>`.
//! Tests for the methods of `Vec128<Neon>` and `Vec256<Neon>`
//! which only Neon has, or which behave differently on Neon.
//!
//! `simd::tests` has the tests which apply to every CPU type.
//!
//! Each test returns early without checking anything if the CPU doesn't support Neon.
use std::{array, mem::transmute};
use std::array;
use crate::{
cpu::{DynCpu, Neon},
cpu::DynCpu,
simd::{
Vec128, Vec256, VecData128, VecData256,
operand::Operand128,
tests::{map2, pattern},
Vec128, Vec256,
tests::{map2, pattern, to_bytes128, to_bytes256},
},
};
fn to_bytes128(v: Vec128<Neon>) -> [u8; 16] {
let mut out = [0; 16];
// SAFETY: `out` is valid for writing 16 bytes
unsafe { v.store_unaligned(out.as_mut_ptr()) };
out
}
/// Get the bytes of a [`Vec256`].
fn to_bytes256(v: Vec256<Neon>) -> [u8; 32] {
let (low, high) = v.into_halves();
let (low, high) = (to_bytes128(low), to_bytes128(high));
array::from_fn(|i| if i < 16 { low[i] } else { high[i - 16] })
}
#[test]
fn vec128_create() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
assert_eq!(to_bytes128(Vec128::zero(cpu)), [0; 16]);
assert_eq!(to_bytes128(Vec128::splat(0x85, cpu)), [0x85; 16]);
let bytes = to_bytes128(Vec128::splat_u32(0x8234_5678, cpu));
assert_eq!(bytes[..], *[0x8234_5678u32; 4].map(u32::to_le_bytes).as_flattened());
assert_eq!(to_bytes128(0x8234_5678u32.into_vec128(cpu)), bytes);
let bytes = pattern(37, 11);
assert_eq!(to_bytes128(Vec128::read_unaligned(&bytes, cpu)), bytes);
let bytes: [u8; 8] = pattern(37, 11);
let v = to_bytes128(Vec128::read_low_unaligned(&bytes, cpu));
assert_eq!(v[..8], bytes);
assert_eq!(v[8..], [0; 8]);
}
#[test]
fn vec128_read_unaligned() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 16] = pattern(37, 11);
// SAFETY: These types are all 16 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec128::read_unaligned(&bytes, cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i8; 16]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u16; 8]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i16; 8]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u32; 4]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i32; 4]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u64; 2]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i64; 2]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u128; 1]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i128; 1]>(bytes), cpu),
]
};
for v in vectors {
assert_eq!(to_bytes128(v), bytes);
}
}
#[test]
fn vec128_read_low_unaligned() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 8] = pattern(37, 11);
// SAFETY: These types are all 8 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec128::read_low_unaligned(&bytes, cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i8; 8]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u16; 4]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i16; 4]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u32; 2]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i32; 2]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u64; 1]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i64; 1]>(bytes), cpu),
]
};
for v in vectors {
let result = to_bytes128(v);
assert_eq!(result[..8], bytes);
assert_eq!(result[8..], [0; 8]);
}
}
#[test]
fn vec128_load_store() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let src = VecData128::from_bytes(pattern(37, 11));
let mut dest = VecData128::from_bytes([0; 16]);
Vec128::read(&src, cpu).write(&mut dest);
assert_eq!(*dest.as_bytes(), *src.as_bytes());
assert_eq!(to_bytes128(src.read(cpu)), *src.as_bytes());
let src = pattern::<18>(37, 11);
let mut dest = [0; 18];
// SAFETY: `src` and `dest` are valid for 16 bytes starting at offset 1
unsafe {
let v = Vec128::load_unaligned(src.as_ptr().add(1), cpu);
v.store_unaligned(dest.as_mut_ptr().add(1));
}
assert_eq!(dest[1..17], src[1..17]);
assert_eq!((dest[0], dest[17]), (0, 0));
let src = pattern::<10>(37, 11);
// SAFETY: `src` is valid for reading 8 bytes starting at offset 1
let v = unsafe { Vec128::load_low_unaligned(src.as_ptr().add(1), cpu) };
let bytes = to_bytes128(v);
assert_eq!(bytes[..8], src[1..9]);
assert_eq!(bytes[8..], [0; 8]);
let mut dest = [0; 10];
// SAFETY: `dest` is valid for writing 8 bytes starting at offset 1
unsafe { Vec128::splat(0x85, cpu).store_low_unaligned(dest.as_mut_ptr().add(1)) };
assert_eq!(dest[1..9], [0x85; 8]);
assert_eq!((dest[0], dest[9]), (0, 0));
let values: [u64; 2] = [0x0102_0304_0506_0708, 1 << 63];
let bytes = to_bytes128(Vec128::read_unaligned(&values, cpu));
assert_eq!(bytes[..], *values.map(u64::to_le_bytes).as_flattened());
}
#[test]
fn vec128_bitwise() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
assert_eq!(to_bytes128(va & vb), map2(a, b, |x, y| x & y));
assert_eq!(to_bytes128(va | vb), map2(a, b, |x, y| x | y));
assert_eq!(to_bytes128(va ^ vb), map2(a, b, |x, y| x ^ y));
assert_eq!(to_bytes128(!va), a.map(|x| !x));
assert_eq!(to_bytes128(va & 0x3C), a.map(|x| x & 0x3C));
assert_eq!(to_bytes128(va | 0x3C), a.map(|x| x | 0x3C));
assert_eq!(to_bytes128(va ^ 0x3C), a.map(|x| x ^ 0x3C));
let mut v = va;
v &= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x & y));
let mut v = va;
v |= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x | y));
let mut v = va;
v ^= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x ^ y));
let mut v = va;
v &= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x & 0x3C));
let mut v = va;
v |= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x | 0x3C));
let mut v = va;
v ^= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x ^ 0x3C));
}
#[test]
fn vec128_shift() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
@@ -186,41 +26,6 @@ fn vec128_shift() {
assert_eq!(to_bytes128(v.shr_u8::<3>()), bytes.map(|x| x >> 3));
}
#[test]
fn vec128_is_zero() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
assert!(Vec128::zero(cpu).is_zero());
assert!(!Vec128::splat(0xFF, cpu).is_zero());
// Check every byte, with only the lowest or highest bit set
for i in 0..16 {
for bit in [0x01, 0x80] {
let mut bytes = [0u8; 16];
bytes[i] = bit;
assert!(!Vec128::read_unaligned(&bytes, cpu).is_zero());
}
}
}
#[test]
fn vec128_add_sub() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
assert_eq!(to_bytes128(va.add_u8(vb)), map2(a, b, u8::wrapping_add));
assert_eq!(to_bytes128(va.saturating_add_u8(vb)), map2(a, b, u8::saturating_add));
assert_eq!(to_bytes128(va.sub_u8(vb)), map2(a, b, u8::wrapping_sub));
assert_eq!(to_bytes128(va.saturating_sub_u8(vb)), map2(a, b, u8::saturating_sub));
assert_eq!(to_bytes128(va.add_u8(0x90)), a.map(|x| x.wrapping_add(0x90)));
assert_eq!(to_bytes128(va.saturating_add_u8(0x90)), a.map(|x| x.saturating_add(0x90)));
assert_eq!(to_bytes128(va.sub_u8(0x90)), a.map(|x| x.wrapping_sub(0x90)));
assert_eq!(to_bytes128(va.saturating_sub_u8(0x90)), a.map(|x| x.saturating_sub(0x90)));
}
#[test]
fn vec128_pairwise_add_u8() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
@@ -236,199 +41,32 @@ fn vec128_pairwise_add_u8() {
}
#[test]
fn vec128_compare() {
fn vec128_test_u8() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let (a, mut b): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
b[3] = a[3];
let (a, b): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
let eq = to_bytes128(va.eq_u8(vb));
assert_eq!(eq, map2(a, b, |x, y| if x == y { 0xFF } else { 0 }));
assert_eq!(eq[3], 0xFF);
let eq = to_bytes128(va.eq_u8(a[5]));
assert_eq!(eq, a.map(|x| if x == a[5] { 0xFF } else { 0 }));
let test = to_bytes128(va.test_u8(vb));
assert_eq!(test, map2(a, b, |x, y| if x & y != 0 { 0xFF } else { 0 }));
let test = to_bytes128(va.test_u8(0x88));
assert_eq!(test, a.map(|x| if x & 0x88 != 0 { 0xFF } else { 0 }));
let gt = to_bytes128(va.gt_i8(vb));
assert_eq!(gt, map2(a, b, |x, y| if x.cast_signed() > y.cast_signed() { 0xFF } else { 0 }));
let lt = to_bytes128(va.lt_i8(vb));
assert_eq!(lt, map2(a, b, |x, y| if x.cast_signed() < y.cast_signed() { 0xFF } else { 0 }));
// Include values either side of the thresholds
let c: [u8; 16] = array::from_fn(|i| [0, 1, 0x1F, 0x20, 0x21, 0x7F, 0x80, 0xFF][i % 8]);
let vc = Vec128::read_unaligned(&c, cpu);
for n in [0, 0x20] {
let gt = to_bytes128(vc.gt_i8(n));
assert_eq!(gt, c.map(|x| if x.cast_signed() > n { 0xFF } else { 0 }));
let lt = to_bytes128(vc.lt_i8(n));
assert_eq!(lt, c.map(|x| if x.cast_signed() < n { 0xFF } else { 0 }));
}
}
#[test]
fn vec128_all_eq_u8() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let a: [u8; 16] = pattern(37, 11);
let va = Vec128::read_unaligned(&a, cpu);
assert!(va.all_eq_u8(va));
assert!(!va.all_eq_u8(a[0]));
assert!(Vec128::splat(0x85, cpu).all_eq_u8(0x85));
assert!(!Vec128::splat(0x85, cpu).all_eq_u8(0x84));
// Check every byte, with a difference of only the lowest or highest bit
for i in 0..16 {
for bit in [0x01, 0x80] {
let mut b = a;
b[i] ^= bit;
assert!(!va.all_eq_u8(Vec128::read_unaligned(&b, cpu)));
}
}
}
#[test]
fn vec128_rearrange_bytes() {
fn vec128_shuffle_u8s() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 16] = pattern(37, 11);
let v = Vec128::read_unaligned(&bytes, cpu);
let shifted = to_bytes128(v.shr_bytes::<3>());
assert_eq!(shifted[..13], bytes[3..]);
assert_eq!(shifted[13..], [0; 3]);
// Include 16, 0x1F and 0x7F, which AVX2 would treat as in range
let control: [u8; 16] = [15, 0, 16, 0x1F, 0x7F, 0x80, 0xFF, 7, 8, 1, 2, 3, 14, 13, 0x8F, 9];
let shuffled = to_bytes128(v.shuffle_u8s(Vec128::read_unaligned(&control, cpu)));
assert_eq!(shuffled, control.map(|c| if c < 16 { bytes[c as usize] } else { 0 }));
}
#[test]
fn vec256_create() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
assert_eq!(to_bytes256(Vec256::splat(0x85, cpu)), [0x85; 32]);
let bytes = to_bytes256(Vec256::splat_u32(0x8234_5678, cpu));
assert_eq!(bytes[..], *[0x8234_5678u32; 8].map(u32::to_le_bytes).as_flattened());
}
#[test]
fn vec256_load_unaligned() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let src = pattern::<34>(37, 11);
// SAFETY: `src` is valid for reading 32 bytes starting at offset 1
let v = unsafe { Vec256::load_unaligned(src.as_ptr().add(1), cpu) };
assert_eq!(to_bytes256(v)[..], src[1..33]);
}
#[test]
fn vec256_read() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let src = VecData256::from_bytes(pattern(37, 11));
assert_eq!(to_bytes256(Vec256::read(&src, cpu)), *src.as_bytes());
assert_eq!(to_bytes256(src.read(cpu)), *src.as_bytes());
}
#[test]
fn vec256_read_unaligned() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let bytes: [u8; 32] = pattern(37, 11);
// SAFETY: These types are all 32 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec256::read_unaligned(&bytes, cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i8; 32]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u16; 16]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i16; 16]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u32; 8]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i32; 8]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u64; 4]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i64; 4]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u128; 2]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i128; 2]>(bytes), cpu),
]
};
for v in vectors {
assert_eq!(to_bytes256(v), bytes);
}
}
#[test]
fn vec256_bitwise() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let (a, b): ([u8; 32], [u8; 32]) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
assert_eq!(to_bytes256(va & vb), map2(a, b, |x, y| x & y));
assert_eq!(to_bytes256(va | vb), map2(a, b, |x, y| x | y));
assert_eq!(to_bytes256(va ^ vb), map2(a, b, |x, y| x ^ y));
assert_eq!(to_bytes256(!va), a.map(|x| !x));
assert_eq!(to_bytes256(va & 0x3C), a.map(|x| x & 0x3C));
assert_eq!(to_bytes256(va | 0x3C), a.map(|x| x | 0x3C));
assert_eq!(to_bytes256(va ^ 0x3C), a.map(|x| x ^ 0x3C));
let mut v = va;
v &= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x & y));
let mut v = va;
v |= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x | y));
let mut v = va;
v ^= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x ^ y));
let mut v = va;
v &= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x & 0x3C));
let mut v = va;
v |= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x | 0x3C));
let mut v = va;
v ^= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x ^ 0x3C));
}
#[test]
fn vec256_halves() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let (low, high): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
let (vlow, vhigh) = (Vec128::read_unaligned(&low, cpu), Vec128::read_unaligned(&high, cpu));
let (out_low, out_high) = Vec256::from_halves(vlow, vhigh).into_halves();
assert_eq!(to_bytes128(out_low), low);
assert_eq!(to_bytes128(out_high), high);
}
#[test]
fn vec256_shuffle_u8s() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let table: [u8; 32] = pattern(37, 11);
let v = Vec256::read_unaligned(&table, cpu);
let control: [u8; 16] = [31, 0, 16, 15, 32, 0x7F, 0x80, 0xFF, 17, 8, 24, 33, 47, 1, 30, 14];
let expected = control.map(|c| if c < 32 { table[c as usize] } else { 0 });
let shuffled = v.shuffle_u8s(Vec128::read_unaligned(&control, cpu));
assert_eq!(to_bytes128(shuffled), expected);
}
#[test]
fn vec128_blend() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
@@ -445,11 +83,29 @@ fn vec128_blend() {
}
#[test]
fn vec128_get_u64() {
fn vec256_shuffle_u8s() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let values: [u64; 2] = [0x0102_0304_0506_0708, 1 << 63];
let v = Vec128::read_unaligned(&values, cpu);
assert_eq!(v.get_u64::<0>(), values[0]);
assert_eq!(v.get_u64::<1>(), values[1]);
let table: [u8; 32] = pattern(37, 11);
let v = Vec256::read_unaligned(&table, cpu);
let control: [u8; 16] = [31, 0, 16, 15, 32, 0x7F, 0x80, 0xFF, 17, 8, 24, 33, 47, 1, 30, 14];
let expected = control.map(|c| if c < 32 { table[c as usize] } else { 0 });
let shuffled = v.shuffle_u8s(Vec128::read_unaligned(&control, cpu));
assert_eq!(to_bytes128(shuffled), expected);
}
#[test]
fn vec256_blend() {
let DynCpu::Neon(cpu) = DynCpu::detect() else { return };
let (a, b, mask) = (pattern(37, 11), pattern(101, 200), pattern(23, 5));
let v = Vec256::read_unaligned(&a, cpu)
.blend(Vec256::read_unaligned(&b, cpu), Vec256::read_unaligned(&mask, cpu));
let expected: [u8; 32] = array::from_fn(|i| (a[i] & !mask[i]) | (b[i] & mask[i]));
assert_eq!(to_bytes256(v), expected);
let v = Vec256::read_unaligned(&a, cpu).blend(0x85, Vec256::read_unaligned(&mask, cpu));
let expected: [u8; 32] = array::from_fn(|i| (a[i] & !mask[i]) | (0x85 & mask[i]));
assert_eq!(to_bytes256(v), expected);
}
crates/oxc_lexer/src/simd/neon/vec128.rs
+230 -205
View File
@@ -1,7 +1,7 @@
//! Methods of `Vec128<Neon>`, which wraps a [`uint8x16_t`].
//!
//! Methods have the same names as the methods of `Vec128<Avx2>` which do the same job.
//! The `neon` module docs list the ones which behave differently.
//! Implements [`Vec128Impl`], and has the methods which only Neon has.
//! The `neon` module docs list the methods which behave differently from AVX2.
//!
//! [`uint8x16_t`]: std::arch::aarch64::uint8x16_t
@@ -21,7 +21,6 @@ use crate::{
},
};
/// Creation methods.
impl Vec128<Neon> {
/// Create a [`Vec128`] wrapping a [`uint8x16_t`].
///
@@ -30,43 +29,70 @@ impl Vec128<Neon> {
pub(super) fn new(inner: uint8x16_t, cpu: Neon) -> Self {
Self { inner, cpu }
}
}
/// Get [`Neon`] token from this [`Vec128`].
#[inline(always)]
pub fn cpu(self) -> Neon {
self.cpu
}
impl Vec128Impl for Vec128<Neon> {
type Cpu = Neon;
type Inner = uint8x16_t;
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec128`] with all bits zero.
#[inline(always)]
pub fn zero(cpu: Neon) -> Self {
fn zero(cpu: Neon) -> Self {
Self::splat(0, cpu)
}
/// Create a [`Vec128`] by repeating `n` 16 times.
#[inline(always)]
pub fn splat(n: u8, cpu: Neon) -> Self {
fn splat(n: u8, cpu: Neon) -> Self {
// SAFETY: `Neon` type guarantees platform supports Neon
Self::new(unsafe { vdupq_n_u8(n) }, cpu)
}
/// Create a [`Vec128`] by repeating `n` 4 times.
#[inline(always)]
pub fn splat_u32(n: u32, cpu: Neon) -> Self {
fn splat_u32(n: u32, cpu: Neon) -> Self {
// SAFETY: `Neon` type guarantees platform supports Neon
Self::new(unsafe { vreinterpretq_u8_u32(vdupq_n_u32(n)) }, cpu)
}
}
/// Loads and stores.
impl Vec128<Neon> {
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get [`Neon`] token from this [`Vec128`].
#[inline(always)]
fn cpu(self) -> Neon {
self.cpu
}
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec128`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for reading 16 bytes.
/// * `ptr` must be aligned on 16.
#[inline(always)]
unsafe fn load(ptr: *const u8, cpu: Neon) -> Self {
// Neon has no separate aligned load instruction, so this is the same as `load_unaligned`.
// SAFETY: Caller guarantees `ptr` is valid for reading 16 bytes.
unsafe { Self::load_unaligned(ptr, cpu) }
}
/// Read a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 16 bytes.
#[inline(always)]
pub unsafe fn load_unaligned(ptr: *const u8, cpu: Neon) -> Self {
unsafe fn load_unaligned(ptr: *const u8, cpu: Neon) -> Self {
// SAFETY: Caller guarantees `ptr` is valid for reading 16 bytes.
// `Neon` type guarantees platform supports Neon.
Self::new(unsafe { vld1q_u8(ptr) }, cpu)
@@ -78,21 +104,17 @@ impl Vec128<Neon> {
/// # SAFETY
///
/// `ptr` must be valid for reading 8 bytes.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub unsafe fn load_low_unaligned(ptr: *const u8, cpu: Neon) -> Self {
unsafe fn load_low_unaligned(ptr: *const u8, cpu: Neon) -> Self {
// SAFETY: Caller guarantees `ptr` is valid for reading 8 bytes.
// `Neon` type guarantees platform supports Neon.
Self::new(unsafe { vcombine_u8(vld1_u8(ptr), vdup_n_u8(0)) }, cpu)
}
/// Read a [`Vec128`] from a [`VecData128`], which is aligned on 16.
///
/// Neon has no separate aligned load instruction, so this is the same as [`read_unaligned`].
///
/// [`read_unaligned`]: Self::read_unaligned
#[inline(always)]
pub fn read(data: &VecData128, cpu: Neon) -> Self {
fn read(data: &VecData128, cpu: Neon) -> Self {
// Neon has no separate aligned load instruction, so this is the same as `read_unaligned`
Self::read_unaligned(data, cpu)
}
@@ -101,7 +123,7 @@ impl Vec128<Neon> {
/// `slice` can be any slice of integers which is 16 bytes in total
/// (e.g. `&[u8; 16]`, `&[i64; 2]`).
#[inline(always)]
pub fn read_unaligned(slice: &impl IntArray128, cpu: Neon) -> Self {
fn read_unaligned(slice: &impl IntArray128, cpu: Neon) -> Self {
// SAFETY: `IntArray128` guarantees `slice.as_ptr()`
// is valid for unaligned reads of 16 bytes
unsafe { Self::load_unaligned(slice.as_ptr(), cpu) }
@@ -112,21 +134,33 @@ impl Vec128<Neon> {
///
/// `slice` can be any slice of integers which is 8 bytes in total
/// (e.g. `&[u8; 8]`, `&[i32; 2]`).
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn read_low_unaligned(slice: &impl IntArray64, cpu: Neon) -> Self {
fn read_low_unaligned(slice: &impl IntArray64, cpu: Neon) -> Self {
// SAFETY: `IntArray64` guarantees `slice.as_ptr()`
// is valid for unaligned reads of 8 bytes
unsafe { Self::load_low_unaligned(slice.as_ptr(), cpu) }
}
/// Store a [`Vec128`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for writing 16 bytes.
/// * `ptr` must be aligned on 16.
#[inline(always)]
unsafe fn store(self, ptr: *mut u8) {
// Neon has no separate aligned store instruction, so this is the same as `store_unaligned`.
// SAFETY: Caller guarantees `ptr` is valid for writing 16 bytes.
unsafe { self.store_unaligned(ptr) }
}
/// Store a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 16 bytes.
#[inline(always)]
pub unsafe fn store_unaligned(self, ptr: *mut u8) {
unsafe fn store_unaligned(self, ptr: *mut u8) {
// This is all `vst1q_u8` does. Calling it would give the same instructions,
// but it's a function which the compiler has to instantiate and then inline.
// SAFETY: Caller guarantees `ptr` is valid for writing 16 bytes.
@@ -138,9 +172,8 @@ impl Vec128<Neon> {
/// # SAFETY
///
/// `ptr` must be valid for writing 8 bytes.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub unsafe fn store_low_unaligned(self, ptr: *mut u8) {
unsafe fn store_low_unaligned(self, ptr: *mut u8) {
// SAFETY: Caller guarantees `ptr` is valid for writing 8 bytes.
// `Vec128` type guarantees platform supports Neon.
unsafe { vst1_u8(ptr, vget_low_u8(self.inner)) }
@@ -151,19 +184,19 @@ impl Vec128<Neon> {
/// Neon has no separate aligned store instruction, so this does the same as [`store_unaligned`].
///
/// [`store_unaligned`]: Self::store_unaligned
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn write(self, dst: &mut VecData128) {
fn write(self, dst: &mut VecData128) {
// SAFETY: `VecData128` guarantees `dst.as_mut_ptr()` is valid for writes of 16 bytes
unsafe { self.store_unaligned(dst.as_mut_ptr()) }
}
}
/// Bitwise ops.
impl Vec128<Neon> {
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec128`] are zero.
#[inline(always)]
pub fn is_zero(self) -> bool {
fn is_zero(self) -> bool {
// Neon has no equivalent of x86's `ptest`, so combine the 2 halves,
// and test the result for 0
(self.get_u64::<0>() | self.get_u64::<1>()) == 0
@@ -171,11 +204,172 @@ impl Vec128<Neon> {
/// Invert all the bits of this [`Vec128`].
#[inline(always)]
pub fn not(self) -> Self {
fn not(self) -> Self {
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vmvnq_u8(self.inner) }, self.cpu)
}
/// AND this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
fn and(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vandq_u8(self.inner, other.inner) }, self.cpu)
}
/// OR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
fn or(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vorrq_u8(self.inner, other.inner) }, self.cpu)
}
/// XOR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
fn xor(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { veorq_u8(self.inner, other.inner) }, self.cpu)
}
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// wrapping on overflow.
#[inline(always)]
fn add_u8(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vaddq_u8(self.inner, other.inner) }, self.cpu)
}
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// stopping at 255.
#[inline(always)]
fn saturating_add_u8(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vqaddq_u8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// wrapping on overflow.
#[inline(always)]
fn sub_u8(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vsubq_u8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// stopping at 0.
#[inline(always)]
fn saturating_sub_u8(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vqsubq_u8(self.inner, other.inner) }, self.cpu)
}
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec128`] to either another [`Vec128`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[inline(always)]
fn eq_u8(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vceqq_u8(self.inner, other.inner) }, self.cpu)
}
/// Check if every byte of this [`Vec128`] equals either another [`Vec128`] or a `u8`.
#[inline(always)]
fn all_eq_u8(self, other: impl Operand128<u8, Neon>) -> bool {
self.xor(other).is_zero()
}
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[inline(always)]
fn lt_i8(self, other: impl Operand128<i8, Neon>) -> Self {
other.into_vec128(self.cpu).gt_i8(self)
}
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[inline(always)]
fn gt_i8(self, other: impl Operand128<i8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
let (left, right) = (self.inner, other.inner);
// SAFETY: `Vec128` type guarantees platform supports Neon
let result = unsafe { vcgtq_s8(vreinterpretq_s8_u8(left), vreinterpretq_s8_u8(right)) };
Self::new(result, self.cpu)
}
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Shift the whole of this [`Vec128`] right by `BYTES` bytes, filling with zeros.
///
/// Byte `N` moves to byte 0.
#[inline(always)]
fn shr_bytes<const BYTES: i32>(self) -> Self {
let zero = Self::zero(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vextq_u8::<BYTES>(self.inner, zero.inner) }, self.cpu)
}
/// Rearrange the bytes of this [`Vec128`].
///
/// For each byte `i`:
///
/// * If `control[i]` is 16 or more, `result[i]` is 0.
/// * Otherwise, `result[i]` is `self[control[i]]`.
///
/// Often used as a 16-entry lookup table, with `self` as the table.
#[inline(always)]
fn shuffle_u8s(self, control: Self) -> Self {
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vqtbl1q_u8(self.inner, control.inner) }, self.cpu)
}
/// Combine the bits of this [`Vec128`] and either another [`Vec128`] or a `u8`.
///
/// Each bit of the result comes from `other` if the same bit of `mask` is set,
/// otherwise from `self`.
#[inline(always)]
fn blend(self, other: impl Operand128<u8, Neon>, mask: Self) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vbslq_u8(mask.inner, other.inner, self.inner) }, self.cpu)
}
// ----------------------------------------
// `u64`-wise ops
// ----------------------------------------
/// Get one of the 2 `u64`s of this [`Vec128`].
///
/// `INDEX` is 0 for the low 8 bytes, or 1 for the high 8 bytes.
#[inline(always)]
fn get_u64<const INDEX: i32>(self) -> u64 {
// SAFETY: `Vec128` type guarantees platform supports Neon
unsafe { vgetq_lane_u64::<INDEX>(vreinterpretq_u64_u8(self.inner)) }
}
}
/// Methods which are specific to this architecture.
impl Vec128<Neon> {
/// Shift each of the 16 bytes of this [`Vec128`] left by `BITS` bits.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
@@ -191,73 +385,6 @@ impl Vec128<Neon> {
Self::new(unsafe { vshrq_n_u8::<BITS>(self.inner) }, self.cpu)
}
/// AND this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn and(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vandq_u8(self.inner, other.inner) }, self.cpu)
}
/// OR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn or(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vorrq_u8(self.inner, other.inner) }, self.cpu)
}
/// XOR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn xor(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { veorq_u8(self.inner, other.inner) }, self.cpu)
}
}
/// Arithmetic ops.
impl Vec128<Neon> {
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// wrapping on overflow.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn add_u8(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vaddq_u8(self.inner, other.inner) }, self.cpu)
}
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// stopping at 255.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn saturating_add_u8(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vqaddq_u8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// wrapping on overflow.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn sub_u8(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vsubq_u8(self.inner, other.inner) }, self.cpu)
}
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// stopping at 0.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn saturating_sub_u8(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vqsubq_u8(self.inner, other.inner) }, self.cpu)
}
/// Add adjacent pairs of bytes in this [`Vec128`] and `other`, wrapping on overflow.
///
/// The result is `[s0 + s1, s2 + s3, .., s14 + s15, o0 + o1, o2 + o3, .., o14 + o15]`,
@@ -267,116 +394,14 @@ impl Vec128<Neon> {
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vpaddq_u8(self.inner, other.inner) }, self.cpu)
}
}
/// Comparison.
impl Vec128<Neon> {
/// Compare every byte of this [`Vec128`] to either another [`Vec128`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[inline(always)]
pub fn eq_u8(self, other: impl Operand128<u8>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vceqq_u8(self.inner, other.inner) }, self.cpu)
}
/// Check if every byte of this [`Vec128`] equals either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn all_eq_u8(self, other: impl Operand128<u8>) -> bool {
self.xor(other).is_zero()
}
/// Check every byte of this [`Vec128`] against either another [`Vec128`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes share any set bits, otherwise `0x00`.
#[inline(always)]
pub fn test_u8(self, other: impl Operand128<u8>) -> Self {
pub fn test_u8(self, other: impl Operand128<u8, Neon>) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vtstq_u8(self.inner, other.inner) }, self.cpu)
}
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[inline(always)]
pub fn lt_i8(self, other: impl Operand128<i8>) -> Self {
other.into_vec128(self.cpu).gt_i8(self)
}
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[inline(always)]
pub fn gt_i8(self, other: impl Operand128<i8>) -> Self {
let other = other.into_vec128(self.cpu);
let (left, right) = (self.inner, other.inner);
// SAFETY: `Vec128` type guarantees platform supports Neon
let result = unsafe { vcgtq_s8(vreinterpretq_s8_u8(left), vreinterpretq_s8_u8(right)) };
Self::new(result, self.cpu)
}
}
/// Bytewise ops.
impl Vec128<Neon> {
/// Shift the whole of this [`Vec128`] right by `BYTES` bytes, filling with zeros.
///
/// Byte `N` moves to byte 0.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn shr_bytes<const BYTES: i32>(self) -> Self {
let zero = Self::zero(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vextq_u8::<BYTES>(self.inner, zero.inner) }, self.cpu)
}
/// Rearrange the bytes of this [`Vec128`].
///
/// For each byte `i`:
///
/// * If `control[i]` is 16 or more, `result[i]` is 0.
/// * Otherwise, `result[i]` is `self[control[i]]`.
///
/// Often used as a 16-entry lookup table, with `self` as the table.
#[inline(always)]
pub fn shuffle_u8s(self, control: Self) -> Self {
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vqtbl1q_u8(self.inner, control.inner) }, self.cpu)
}
/// Combine the bits of this [`Vec128`] and either another [`Vec128`] or a `u8`.
///
/// Each bit of the result comes from `other` if the same bit of `mask` is set,
/// otherwise from `self`.
#[inline(always)]
pub fn blend(self, other: impl Operand128<u8>, mask: Self) -> Self {
let other = other.into_vec128(self.cpu);
// SAFETY: `Vec128` type guarantees platform supports Neon
Self::new(unsafe { vbslq_u8(mask.inner, other.inner, self.inner) }, self.cpu)
}
}
/// `u64`-wise ops.
impl Vec128<Neon> {
/// Get one of the 2 `u64`s of this [`Vec128`].
///
/// `INDEX` is 0 for the low 8 bytes, or 1 for the high 8 bytes.
#[inline(always)]
pub fn get_u64<const INDEX: i32>(self) -> u64 {
// SAFETY: `Vec128` type guarantees platform supports Neon
unsafe { vgetq_lane_u64::<INDEX>(vreinterpretq_u64_u8(self.inner)) }
}
}
impl Vec128Impl for Vec128<Neon> {
type Cpu = Neon;
#[inline(always)]
fn read(data: &VecData128, cpu: Neon) -> Self {
// Inherent methods take priority, so this calls `Vec128::read` above, not itself
Vec128::read(data, cpu)
}
}
crates/oxc_lexer/src/simd/neon/vec256.rs
+323 -37
View File
@@ -1,5 +1,7 @@
//! Methods of `Vec256<Neon>`, which wraps a [`uint8x16x2_t`].
//!
//! Implements [`Vec256Impl`], and has the methods which only Neon has.
//!
//! Neon registers are 128 bits, so a `Vec256<Neon>` is a pair of registers,
//! and its bitwise methods work on each half separately.
//! It only has the few methods which Neon code needs, far fewer than `Vec256<Avx2>`.
@@ -18,7 +20,6 @@ use crate::{
simd::{Vec128, Vec256, Vec256Impl, VecData256, arrays::IntArray256, operand::Operand256},
};
/// Creation methods.
impl Vec256<Neon> {
/// Create a [`Vec256`] wrapping a [`uint8x16x2_t`].
///
@@ -27,43 +28,89 @@ impl Vec256<Neon> {
fn new(inner: uint8x16x2_t, cpu: Neon) -> Self {
Self { inner, cpu }
}
}
impl Vec256Impl for Vec256<Neon> {
type Cpu = Neon;
type Inner = uint8x16x2_t;
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec256`] with all bits zero.
#[inline(always)]
fn zero(cpu: Neon) -> Self {
Self::splat(0, cpu)
}
/// Create a [`Vec256`] by repeating `n` 32 times.
#[inline(always)]
pub fn splat(n: u8, cpu: Neon) -> Self {
fn splat(n: u8, cpu: Neon) -> Self {
let half = Vec128::splat(n, cpu);
Self::from_halves(half, half)
}
/// Create a [`Vec256`] by repeating `n` 8 times.
#[inline(always)]
pub fn splat_u32(n: u32, cpu: Neon) -> Self {
fn splat_u32(n: u32, cpu: Neon) -> Self {
let half = Vec128::splat_u32(n, cpu);
Self::from_halves(half, half)
}
/// Create a [`Vec256`] from its low and high 128 bits.
/// Create a [`Vec256`] by repeating `v` twice.
#[inline(always)]
pub fn from_halves(low: Vec128<Neon>, high: Vec128<Neon>) -> Self {
Self::new(uint8x16x2_t(low.inner, high.inner), low.cpu)
fn splat_v128(v: Vec128<Neon>) -> Self {
Self::from_halves(v, v)
}
/// Get the low and high 128 bits of this [`Vec256`].
/// Create a [`Vec256`] by zero-extending the low 8 bytes of `v` to 8 `u32`s.
#[inline(always)]
pub fn into_halves(self) -> (Vec128<Neon>, Vec128<Neon>) {
(Vec128::new(self.inner.0, self.cpu), Vec128::new(self.inner.1, self.cpu))
fn widen_u8_to_u32(v: Vec128<Neon>) -> Self {
// SAFETY: `Vec128` type guarantees platform supports Neon
let inner = unsafe {
let u16s = vmovl_u8(vget_low_u8(v.inner));
let low = vmovl_u16(vget_low_u16(u16s));
let high = vmovl_u16(vget_high_u16(u16s));
uint8x16x2_t(vreinterpretq_u8_u32(low), vreinterpretq_u8_u32(high))
};
Self::new(inner, v.cpu)
}
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get [`Neon`] token from this [`Vec256`].
#[inline(always)]
fn cpu(self) -> Neon {
self.cpu
}
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec256`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for reading 32 bytes.
/// * `ptr` must be aligned on 32.
#[inline(always)]
unsafe fn load(ptr: *const u8, cpu: Neon) -> Self {
// Neon has no separate aligned load instruction, so this is the same as `load_unaligned`.
// SAFETY: Caller guarantees `ptr` is valid for reading 32 bytes.
unsafe { Self::load_unaligned(ptr, cpu) }
}
}
/// Loads.
impl Vec256<Neon> {
/// Read a [`Vec256`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 32 bytes.
#[inline(always)]
pub unsafe fn load_unaligned(ptr: *const u8, cpu: Neon) -> Self {
unsafe fn load_unaligned(ptr: *const u8, cpu: Neon) -> Self {
// This is all `vld1q_u8_x2` does. Calling it would give the same instructions,
// but it's a function which the compiler has to instantiate and then inline.
// SAFETY: Caller guarantees `ptr` is valid for reading 32 bytes.
@@ -72,12 +119,9 @@ impl Vec256<Neon> {
}
/// Read a [`Vec256`] from a [`VecData256`], which is aligned on 32.
///
/// Neon has no separate aligned load instruction, so this is the same as [`read_unaligned`].
///
/// [`read_unaligned`]: Self::read_unaligned
#[inline(always)]
pub fn read(data: &VecData256, cpu: Neon) -> Self {
fn read(data: &VecData256, cpu: Neon) -> Self {
// Neon has no separate aligned load instruction, so this is the same as `read_unaligned`
Self::read_unaligned(data, cpu)
}
@@ -86,25 +130,67 @@ impl Vec256<Neon> {
/// `slice` can be any slice of integers which is 32 bytes in total
/// (e.g. `&[u8; 32]`, `&[i64; 4]`).
#[inline(always)]
pub fn read_unaligned(slice: &impl IntArray256, cpu: Neon) -> Self {
fn read_unaligned(slice: &impl IntArray256, cpu: Neon) -> Self {
// SAFETY: `IntArray256` guarantees `slice.as_ptr()`
// is valid for unaligned reads of 32 bytes
unsafe { Self::load_unaligned(slice.as_ptr(), cpu) }
}
}
/// Bitwise ops.
impl Vec256<Neon> {
/// Store a [`Vec256`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for writing 32 bytes.
/// * `ptr` must be aligned on 32.
#[inline(always)]
unsafe fn store(self, ptr: *mut u8) {
// Neon has no separate aligned store instruction, so this is the same as `store_unaligned`.
// SAFETY: Caller guarantees `ptr` is valid for writing 32 bytes.
unsafe { self.store_unaligned(ptr) }
}
/// Store a [`Vec256`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 32 bytes.
#[inline(always)]
unsafe fn store_unaligned(self, ptr: *mut u8) {
// This is all `vst1q_u8_x2` does. Calling it would give the same instructions,
// but it's a function which the compiler has to instantiate and then inline.
// SAFETY: Caller guarantees `ptr` is valid for writing 32 bytes.
unsafe { ptr.cast::<uint8x16x2_t>().write_unaligned(self.inner) }
}
/// Write a [`Vec256`] into a [`VecData256`], which is aligned on 32.
#[inline(always)]
fn write(self, dst: &mut VecData256) {
// Neon has no separate aligned store instruction, so this is the same as `store_unaligned`.
// SAFETY: `VecData256` guarantees `dst.as_mut_ptr()` is valid for writes of 32 bytes.
unsafe { self.store_unaligned(dst.as_mut_ptr()) }
}
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec256`] are zero.
#[inline(always)]
fn is_zero(self) -> bool {
let (low, high) = self.into_halves();
(low | high).is_zero()
}
/// Invert all the bits of this [`Vec256`].
#[inline(always)]
pub fn not(self) -> Self {
fn not(self) -> Self {
let (low, high) = self.into_halves();
Self::from_halves(!low, !high)
}
/// AND this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn and(self, other: impl Operand256<u8>) -> Self {
fn and(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low & other_low, high & other_high)
@@ -112,7 +198,7 @@ impl Vec256<Neon> {
/// OR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn or(self, other: impl Operand256<u8>) -> Self {
fn or(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low | other_low, high | other_high)
@@ -120,14 +206,224 @@ impl Vec256<Neon> {
/// XOR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn xor(self, other: impl Operand256<u8>) -> Self {
fn xor(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low ^ other_low, high ^ other_high)
}
/// Shift each of the 16 `u16`s of this [`Vec256`] left by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
#[inline(always)]
fn shl_u16<const BITS: i32>(self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports Neon
let inner = unsafe {
let low = vshlq_n_u16::<BITS>(vreinterpretq_u16_u8(self.inner.0));
let high = vshlq_n_u16::<BITS>(vreinterpretq_u16_u8(self.inner.1));
uint8x16x2_t(vreinterpretq_u8_u16(low), vreinterpretq_u8_u16(high))
};
Self::new(inner, self.cpu)
}
/// Shift each of the 16 `u16`s of this [`Vec256`] right by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
#[inline(always)]
fn shr_u16<const BITS: i32>(self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports Neon
let inner = unsafe {
let low = vshrq_n_u16::<BITS>(vreinterpretq_u16_u8(self.inner.0));
let high = vshrq_n_u16::<BITS>(vreinterpretq_u16_u8(self.inner.1));
uint8x16x2_t(vreinterpretq_u8_u16(low), vreinterpretq_u8_u16(high))
};
Self::new(inner, self.cpu)
}
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// wrapping on overflow.
#[inline(always)]
fn add_u8(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low.add_u8(other_low), high.add_u8(other_high))
}
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// stopping at 255.
#[inline(always)]
fn saturating_add_u8(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low.saturating_add_u8(other_low), high.saturating_add_u8(other_high))
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// wrapping on overflow.
#[inline(always)]
fn sub_u8(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low.sub_u8(other_low), high.sub_u8(other_high))
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// stopping at 0.
#[inline(always)]
fn saturating_sub_u8(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low.saturating_sub_u8(other_low), high.saturating_sub_u8(other_high))
}
/// Add either another [`Vec256`] or a `u32` to every `u32` of this [`Vec256`],
/// wrapping on overflow.
#[inline(always)]
fn add_u32(self, other: impl Operand256<u32, Neon>) -> Self {
let other = other.into_vec256(self.cpu);
// SAFETY: `Vec256` type guarantees platform supports Neon
let inner = unsafe {
let low =
vaddq_u32(vreinterpretq_u32_u8(self.inner.0), vreinterpretq_u32_u8(other.inner.0));
let high =
vaddq_u32(vreinterpretq_u32_u8(self.inner.1), vreinterpretq_u32_u8(other.inner.1));
uint8x16x2_t(vreinterpretq_u8_u32(low), vreinterpretq_u8_u32(high))
};
Self::new(inner, self.cpu)
}
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec256`] to either another [`Vec256`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[inline(always)]
fn eq_u8(self, other: impl Operand256<u8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low.eq_u8(other_low), high.eq_u8(other_high))
}
/// Check if every byte of this [`Vec256`] equals either another [`Vec256`] or a `u8`.
#[inline(always)]
fn all_eq_u8(self, other: impl Operand256<u8, Neon>) -> bool {
self.xor(other).is_zero()
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[inline(always)]
fn lt_i8(self, other: impl Operand256<i8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low.lt_i8(other_low), high.lt_i8(other_high))
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[inline(always)]
fn gt_i8(self, other: impl Operand256<i8, Neon>) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
Self::from_halves(low.gt_i8(other_low), high.gt_i8(other_high))
}
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Combine the bits of this [`Vec256`] and either another [`Vec256`] or a `u8`.
///
/// Each bit of the result comes from `other` if the same bit of `mask` is set,
/// otherwise from `self`.
#[inline(always)]
fn blend(self, other: impl Operand256<u8, Neon>, mask: Self) -> Self {
let (low, high) = self.into_halves();
let (other_low, other_high) = other.into_vec256(self.cpu).into_halves();
let (mask_low, mask_high) = mask.into_halves();
Self::from_halves(low.blend(other_low, mask_low), high.blend(other_high, mask_high))
}
// ----------------------------------------
// `u32`-wise ops
// ----------------------------------------
/// Interleave the low 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s0, o0, s1, o1, s4, o4, s5, o5]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[inline(always)]
fn unpack_lo_u32(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports Neon
let inner = unsafe {
let low =
vzip1q_u32(vreinterpretq_u32_u8(self.inner.0), vreinterpretq_u32_u8(other.inner.0));
let high =
vzip1q_u32(vreinterpretq_u32_u8(self.inner.1), vreinterpretq_u32_u8(other.inner.1));
uint8x16x2_t(vreinterpretq_u8_u32(low), vreinterpretq_u8_u32(high))
};
Self::new(inner, self.cpu)
}
/// Interleave the high 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s2, o2, s3, o3, s6, o6, s7, o7]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[inline(always)]
fn unpack_hi_u32(self, other: Self) -> Self {
// SAFETY: `Vec256` type guarantees platform supports Neon
let inner = unsafe {
let low =
vzip2q_u32(vreinterpretq_u32_u8(self.inner.0), vreinterpretq_u32_u8(other.inner.0));
let high =
vzip2q_u32(vreinterpretq_u32_u8(self.inner.1), vreinterpretq_u32_u8(other.inner.1));
uint8x16x2_t(vreinterpretq_u8_u32(low), vreinterpretq_u8_u32(high))
};
Self::new(inner, self.cpu)
}
// ----------------------------------------
// Operations on 128-bit halves
// ----------------------------------------
/// Create a [`Vec256`] from its low and high 128 bits.
#[inline(always)]
fn from_halves(low: Vec128<Neon>, high: Vec128<Neon>) -> Self {
Self::new(uint8x16x2_t(low.inner, high.inner), low.cpu)
}
/// Get the low and high 128 bits of this [`Vec256`].
#[inline(always)]
fn into_halves(self) -> (Vec128<Neon>, Vec128<Neon>) {
(Vec128::new(self.inner.0, self.cpu), Vec128::new(self.inner.1, self.cpu))
}
/// Get a [`Vec256`] made of the low 128 bits of this [`Vec256`],
/// followed by the low 128 bits of `other`.
#[inline(always)]
fn join_lo_halves(self, other: Self) -> Self {
Self::from_halves(self.into_halves().0, other.into_halves().0)
}
/// Get a [`Vec256`] made of the high 128 bits of this [`Vec256`],
/// followed by the high 128 bits of `other`.
#[inline(always)]
fn join_hi_halves(self, other: Self) -> Self {
Self::from_halves(self.into_halves().1, other.into_halves().1)
}
}
/// Bytewise ops.
/// Methods which are specific to this architecture.
impl Vec256<Neon> {
/// Rearrange the bytes of this [`Vec256`] into a [`Vec128`].
///
@@ -143,13 +439,3 @@ impl Vec256<Neon> {
Vec128::new(unsafe { vqtbl2q_u8(self.inner, control.inner) }, self.cpu)
}
}
impl Vec256Impl for Vec256<Neon> {
type Cpu = Neon;
#[inline(always)]
fn read(data: &VecData256, cpu: Neon) -> Self {
// Inherent methods take priority, so this calls `Vec256::read` above, not itself
Vec256::read(data, cpu)
}
}
crates/oxc_lexer/src/simd/operand.rs
+45 -26
View File
@@ -2,54 +2,62 @@
//!
//! e.g. `v.and(w)` ANDs 2 vectors, and `v.and(0x0f)` ANDs every byte of `v` with `0x0f`.
//! A scalar is repeated to fill a vector.
//!
//! The traits only cover vectors of `CpuType`,
//! which is the CPU feature set with vector types on this platform.
#![expect(
clippy::inline_always,
reason = "conversions call vector methods, which must inline into the function calling them"
)]
use super::{CpuType, Vec128, Vec256};
use super::{CpuSimd, Vec128, Vec128Impl, Vec256, Vec256Impl};
/// Right-hand side of a [`Vec128`] operation.
///
/// Either another [`Vec128`], or a scalar of type `T` which is repeated to fill a [`Vec128`].
///
/// `T` is a parameter so that only 1 integer type implements `Operand128<T>` for each `T`.
/// `T` is a parameter so that only 1 integer type implements `Operand128<T, C>` for each `T`.
/// That's what allows integer literals as operands without a suffix (e.g. `v & 0x0f`).
/// If an operation accepted both `u8` and `i8`, a literal's type would be ambiguous,
/// so it would fall back to `i32`, and wouldn't compile.
pub trait Operand128<T> {
///
/// `C` is the CPU feature set of the [`Vec128`].
pub trait Operand128<T, C: CpuSimd> {
/// Convert to a [`Vec128`].
fn into_vec128(self, cpu: CpuType) -> Vec128<CpuType>;
fn into_vec128(self, cpu: C) -> Vec128<C>;
}
impl<T> Operand128<T> for Vec128<CpuType> {
impl<T, C: CpuSimd> Operand128<T, C> for Vec128<C> {
#[inline(always)]
fn into_vec128(self, _cpu: CpuType) -> Vec128<CpuType> {
fn into_vec128(self, _cpu: C) -> Vec128<C> {
self
}
}
impl Operand128<u8> for u8 {
impl<C: CpuSimd> Operand128<u8, C> for u8
where
Vec128<C>: Vec128Impl<Cpu = C>,
{
#[inline(always)]
fn into_vec128(self, cpu: CpuType) -> Vec128<CpuType> {
fn into_vec128(self, cpu: C) -> Vec128<C> {
Vec128::splat(self, cpu)
}
}
impl Operand128<i8> for i8 {
impl<C: CpuSimd> Operand128<i8, C> for i8
where
Vec128<C>: Vec128Impl<Cpu = C>,
{
#[inline(always)]
fn into_vec128(self, cpu: CpuType) -> Vec128<CpuType> {
fn into_vec128(self, cpu: C) -> Vec128<C> {
Vec128::splat(self.cast_unsigned(), cpu)
}
}
impl Operand128<u32> for u32 {
impl<C: CpuSimd> Operand128<u32, C> for u32
where
Vec128<C>: Vec128Impl<Cpu = C>,
{
#[inline(always)]
fn into_vec128(self, cpu: CpuType) -> Vec128<CpuType> {
fn into_vec128(self, cpu: C) -> Vec128<C> {
Vec128::splat_u32(self, cpu)
}
}
@@ -58,39 +66,50 @@ impl Operand128<u32> for u32 {
///
/// Either another [`Vec256`], or a scalar of type `T` which is repeated to fill a [`Vec256`].
///
/// `T` is a parameter so that only 1 integer type implements `Operand256<T>` for each `T`.
/// `T` is a parameter so that only 1 integer type implements `Operand256<T, C>` for each `T`.
/// That's what allows integer literals as operands without a suffix (e.g. `v & 0x0f`).
/// If an operation accepted both `u8` and `i8`, a literal's type would be ambiguous,
/// so it would fall back to `i32`, and wouldn't compile.
pub trait Operand256<T> {
///
/// `C` is the CPU feature set of the [`Vec256`].
pub trait Operand256<T, C: CpuSimd> {
/// Convert to a [`Vec256`].
fn into_vec256(self, cpu: CpuType) -> Vec256<CpuType>;
fn into_vec256(self, cpu: C) -> Vec256<C>;
}
impl<T> Operand256<T> for Vec256<CpuType> {
impl<T, C: CpuSimd> Operand256<T, C> for Vec256<C> {
#[inline(always)]
fn into_vec256(self, _cpu: CpuType) -> Vec256<CpuType> {
fn into_vec256(self, _cpu: C) -> Vec256<C> {
self
}
}
impl Operand256<u8> for u8 {
impl<C: CpuSimd> Operand256<u8, C> for u8
where
Vec256<C>: Vec256Impl<Cpu = C>,
{
#[inline(always)]
fn into_vec256(self, cpu: CpuType) -> Vec256<CpuType> {
fn into_vec256(self, cpu: C) -> Vec256<C> {
Vec256::splat(self, cpu)
}
}
impl Operand256<i8> for i8 {
impl<C: CpuSimd> Operand256<i8, C> for i8
where
Vec256<C>: Vec256Impl<Cpu = C>,
{
#[inline(always)]
fn into_vec256(self, cpu: CpuType) -> Vec256<CpuType> {
fn into_vec256(self, cpu: C) -> Vec256<C> {
Vec256::splat(self.cast_unsigned(), cpu)
}
}
impl Operand256<u32> for u32 {
impl<C: CpuSimd> Operand256<u32, C> for u32
where
Vec256<C>: Vec256Impl<Cpu = C>,
{
#[inline(always)]
fn into_vec256(self, cpu: CpuType) -> Vec256<CpuType> {
fn into_vec256(self, cpu: C) -> Vec256<C> {
Vec256::splat_u32(self, cpu)
}
}
crates/oxc_lexer/src/simd/tests.rs
+658 -2
View File
@@ -1,6 +1,75 @@
//! Helper functions for AVX2 and Neon tests.
//! Tests for [`Vec128`] and [`Vec256`] which apply to every CPU type,
//! and helper functions for them and for the AVX2 and Neon tests.
//!
//! Each test runs on this platform's CPU type with SIMD vectors (AVX2 on x86_64, Neon on aarch64),
//! and returns early without checking anything if the CPU doesn't support it.
//!
//! The `avx2` and `neon` modules test the methods which only one CPU type has,
//! or which behave differently on different CPU types.
use std::array;
use std::{array, mem::transmute};
use crate::{
cpu::DynCpu,
simd::{
CpuSimd, Vec128, Vec256, VecData128, VecData256, operand::Operand128, vec128::Vec128Impl,
vec256::Vec256Impl,
},
};
#[cfg(target_arch = "x86_64")]
use crate::cpu::Avx2;
#[cfg(target_arch = "aarch64")]
use crate::cpu::Neon;
/// Get this platform's CPU type with SIMD vectors, if the CPU supports it.
pub(super) fn simd_cpu() -> Option<
cfg_select! {
target_arch = "x86_64" => Avx2,
target_arch = "aarch64" => Neon,
},
> {
match DynCpu::detect() {
#[cfg(target_arch = "x86_64")]
DynCpu::Avx2(cpu) => Some(cpu),
#[cfg(target_arch = "aarch64")]
DynCpu::Neon(cpu) => Some(cpu),
_ => None,
}
}
/// Get the bytes of a [`Vec128`].
pub(super) fn to_bytes128<C: CpuSimd>(v: Vec128<C>) -> [u8; 16]
where
Vec128<C>: Vec128Impl<Cpu = C>,
{
let mut out = [0; 16];
// SAFETY: `out` is valid for writing 16 bytes
unsafe { v.store_unaligned(out.as_mut_ptr()) };
out
}
/// Get the bytes of a [`Vec256`].
pub(super) fn to_bytes256<C: CpuSimd>(v: Vec256<C>) -> [u8; 32]
where
Vec256<C>: Vec256Impl<Cpu = C>,
{
let mut out = [0; 32];
// SAFETY: `out` is valid for writing 32 bytes
unsafe { v.store_unaligned(out.as_mut_ptr()) };
out
}
/// Get the `u32`s of a [`Vec256`].
pub(super) fn to_u32s<C: CpuSimd>(v: Vec256<C>) -> [u32; 8]
where
Vec256<C>: Vec256Impl<Cpu = C>,
{
let mut out = [0; 8];
// SAFETY: `out` is valid for writing 32 bytes
unsafe { v.store_unaligned(out.as_mut_ptr().cast::<u8>()) };
out
}
/// Get `N` bytes which cover a spread of values, with and without the top bit set.
pub(super) fn pattern<const N: usize>(multiplier: u8, offset: u8) -> [u8; N] {
@@ -11,3 +80,590 @@ pub(super) fn pattern<const N: usize>(multiplier: u8, offset: u8) -> [u8; N] {
pub(super) fn map2<const N: usize>(a: [u8; N], b: [u8; N], f: impl Fn(u8, u8) -> u8) -> [u8; N] {
array::from_fn(|i| f(a[i], b[i]))
}
/// Transform an array of bytes as if it was an array of `u16`s.
pub(super) fn map_u16(bytes: [u8; 32], f: impl Fn(u16) -> u16) -> [u8; 32] {
let mut out = [0; 32];
for i in (0..32).step_by(2) {
let value = f(u16::from_le_bytes([bytes[i], bytes[i + 1]]));
out[i..i + 2].copy_from_slice(&value.to_le_bytes());
}
out
}
#[test]
fn vec128_create() {
let Some(cpu) = simd_cpu() else { return };
assert_eq!(to_bytes128(Vec128::zero(cpu)), [0; 16]);
assert_eq!(to_bytes128(Vec128::splat(0x85, cpu)), [0x85; 16]);
let bytes = to_bytes128(Vec128::splat_u32(0x8234_5678, cpu));
assert_eq!(bytes[..], *[0x8234_5678u32; 4].map(u32::to_le_bytes).as_flattened());
assert_eq!(to_bytes128(0x8234_5678u32.into_vec128(cpu)), bytes);
let bytes = pattern(37, 11);
assert_eq!(to_bytes128(Vec128::read_unaligned(&bytes, cpu)), bytes);
let bytes: [u8; 8] = pattern(37, 11);
let v = to_bytes128(Vec128::read_low_unaligned(&bytes, cpu));
assert_eq!(v[..8], bytes);
assert_eq!(v[8..], [0; 8]);
}
#[test]
fn vec128_load_store() {
let Some(cpu) = simd_cpu() else { return };
let src = VecData128::from_bytes(pattern(37, 11));
let mut dest = VecData128::from_bytes([0; 16]);
// SAFETY: `src` and `dest` are 16 bytes and aligned on 16
unsafe { Vec128::load(src.as_ptr(), cpu).store(dest.as_mut_ptr()) };
assert_eq!(*dest.as_bytes(), *src.as_bytes());
let mut dest = VecData128::from_bytes([0; 16]);
Vec128::read(&src, cpu).write(&mut dest);
assert_eq!(*dest.as_bytes(), *src.as_bytes());
assert_eq!(to_bytes128(src.read(cpu)), *src.as_bytes());
let src = pattern::<18>(37, 11);
let mut dest = [0; 18];
// SAFETY: `src` and `dest` are valid for 16 bytes starting at offset 1
unsafe {
let v = Vec128::load_unaligned(src.as_ptr().add(1), cpu);
v.store_unaligned(dest.as_mut_ptr().add(1));
}
assert_eq!(dest[1..17], src[1..17]);
assert_eq!((dest[0], dest[17]), (0, 0));
let src = pattern::<10>(37, 11);
// SAFETY: `src` is valid for reading 8 bytes starting at offset 1
let v = unsafe { Vec128::load_low_unaligned(src.as_ptr().add(1), cpu) };
let bytes = to_bytes128(v);
assert_eq!(bytes[..8], src[1..9]);
assert_eq!(bytes[8..], [0; 8]);
let mut dest = [0; 10];
// SAFETY: `dest` is valid for writing 8 bytes starting at offset 1
unsafe { Vec128::splat(0x85, cpu).store_low_unaligned(dest.as_mut_ptr().add(1)) };
assert_eq!(dest[1..9], [0x85; 8]);
assert_eq!((dest[0], dest[9]), (0, 0));
let values: [u64; 2] = [0x0102_0304_0506_0708, 1 << 63];
let bytes = to_bytes128(Vec128::read_unaligned(&values, cpu));
assert_eq!(bytes[..], *values.map(u64::to_le_bytes).as_flattened());
}
#[test]
fn vec128_read_unaligned() {
let Some(cpu) = simd_cpu() else { return };
let bytes: [u8; 16] = pattern(37, 11);
// SAFETY: These types are all 16 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec128::read_unaligned(&bytes, cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i8; 16]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u16; 8]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i16; 8]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u32; 4]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i32; 4]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u64; 2]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i64; 2]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [u128; 1]>(bytes), cpu),
Vec128::read_unaligned(&transmute::<[u8; 16], [i128; 1]>(bytes), cpu),
]
};
for v in vectors {
assert_eq!(to_bytes128(v), bytes);
}
}
#[test]
fn vec128_read_low_unaligned() {
let Some(cpu) = simd_cpu() else { return };
let bytes: [u8; 8] = pattern(37, 11);
// SAFETY: These types are all 8 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec128::read_low_unaligned(&bytes, cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i8; 8]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u16; 4]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i16; 4]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u32; 2]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i32; 2]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [u64; 1]>(bytes), cpu),
Vec128::read_low_unaligned(&transmute::<[u8; 8], [i64; 1]>(bytes), cpu),
]
};
for v in vectors {
let result = to_bytes128(v);
assert_eq!(result[..8], bytes);
assert_eq!(result[8..], [0; 8]);
}
}
#[test]
fn vec128_is_zero() {
let Some(cpu) = simd_cpu() else { return };
assert!(Vec128::zero(cpu).is_zero());
assert!(!Vec128::splat(0xFF, cpu).is_zero());
// Check every byte, with only the lowest or highest bit set
for i in 0..16 {
for bit in [0x01, 0x80] {
let mut bytes = [0u8; 16];
bytes[i] = bit;
assert!(!Vec128::read_unaligned(&bytes, cpu).is_zero());
}
}
}
#[test]
fn vec128_bitwise() {
let Some(cpu) = simd_cpu() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
assert_eq!(to_bytes128(va & vb), map2(a, b, |x, y| x & y));
assert_eq!(to_bytes128(va | vb), map2(a, b, |x, y| x | y));
assert_eq!(to_bytes128(va ^ vb), map2(a, b, |x, y| x ^ y));
assert_eq!(to_bytes128(!va), a.map(|x| !x));
assert_eq!(to_bytes128(va & 0x3C), a.map(|x| x & 0x3C));
assert_eq!(to_bytes128(va | 0x3C), a.map(|x| x | 0x3C));
assert_eq!(to_bytes128(va ^ 0x3C), a.map(|x| x ^ 0x3C));
let mut v = va;
v &= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x & y));
let mut v = va;
v |= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x | y));
let mut v = va;
v ^= vb;
assert_eq!(to_bytes128(v), map2(a, b, |x, y| x ^ y));
let mut v = va;
v &= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x & 0x3C));
let mut v = va;
v |= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x | 0x3C));
let mut v = va;
v ^= 0x3C;
assert_eq!(to_bytes128(v), a.map(|x| x ^ 0x3C));
}
#[test]
fn vec128_add_sub() {
let Some(cpu) = simd_cpu() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
assert_eq!(to_bytes128(va.add_u8(vb)), map2(a, b, u8::wrapping_add));
assert_eq!(to_bytes128(va.saturating_add_u8(vb)), map2(a, b, u8::saturating_add));
assert_eq!(to_bytes128(va.sub_u8(vb)), map2(a, b, u8::wrapping_sub));
assert_eq!(to_bytes128(va.saturating_sub_u8(vb)), map2(a, b, u8::saturating_sub));
assert_eq!(to_bytes128(va.add_u8(0x90)), a.map(|x| x.wrapping_add(0x90)));
assert_eq!(to_bytes128(va.saturating_add_u8(0x90)), a.map(|x| x.saturating_add(0x90)));
assert_eq!(to_bytes128(va.sub_u8(0x90)), a.map(|x| x.wrapping_sub(0x90)));
assert_eq!(to_bytes128(va.saturating_sub_u8(0x90)), a.map(|x| x.saturating_sub(0x90)));
}
#[test]
fn vec128_compare() {
let Some(cpu) = simd_cpu() else { return };
let (a, mut b): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
b[3] = a[3];
let (va, vb) = (Vec128::read_unaligned(&a, cpu), Vec128::read_unaligned(&b, cpu));
let eq = to_bytes128(va.eq_u8(vb));
assert_eq!(eq, map2(a, b, |x, y| if x == y { 0xFF } else { 0 }));
assert_eq!(eq[3], 0xFF);
let eq = to_bytes128(va.eq_u8(a[5]));
assert_eq!(eq, a.map(|x| if x == a[5] { 0xFF } else { 0 }));
let gt = to_bytes128(va.gt_i8(vb));
assert_eq!(gt, map2(a, b, |x, y| if x.cast_signed() > y.cast_signed() { 0xFF } else { 0 }));
let lt = to_bytes128(va.lt_i8(vb));
assert_eq!(lt, map2(a, b, |x, y| if x.cast_signed() < y.cast_signed() { 0xFF } else { 0 }));
// Include values either side of the thresholds
let c: [u8; 16] = array::from_fn(|i| [0, 1, 0x1F, 0x20, 0x21, 0x7F, 0x80, 0xFF][i % 8]);
let vc = Vec128::read_unaligned(&c, cpu);
for n in [0, 0x20] {
let gt = to_bytes128(vc.gt_i8(n));
assert_eq!(gt, c.map(|x| if x.cast_signed() > n { 0xFF } else { 0 }));
let lt = to_bytes128(vc.lt_i8(n));
assert_eq!(lt, c.map(|x| if x.cast_signed() < n { 0xFF } else { 0 }));
}
}
#[test]
fn vec128_all_eq_u8() {
let Some(cpu) = simd_cpu() else { return };
let a: [u8; 16] = pattern(37, 11);
let va = Vec128::read_unaligned(&a, cpu);
assert!(va.all_eq_u8(va));
assert!(!va.all_eq_u8(a[0]));
assert!(Vec128::splat(0x85, cpu).all_eq_u8(0x85));
assert!(!Vec128::splat(0x85, cpu).all_eq_u8(0x84));
// Check every byte, with a difference of only the lowest or highest bit
for i in 0..16 {
for bit in [0x01, 0x80] {
let mut b = a;
b[i] ^= bit;
assert!(!va.all_eq_u8(Vec128::read_unaligned(&b, cpu)));
}
}
}
#[test]
fn vec128_rearrange_bytes() {
let Some(cpu) = simd_cpu() else { return };
let bytes: [u8; 16] = pattern(37, 11);
let v = Vec128::read_unaligned(&bytes, cpu);
let shifted = to_bytes128(v.shr_bytes::<3>());
assert_eq!(shifted[..13], bytes[3..]);
assert_eq!(shifted[13..], [0; 3]);
// Only control bytes which every CPU treats the same: less than 16, or with the top bit set
let control: [u8; 16] = [15, 0, 0x80, 7, 0xFF, 8, 1, 0x8F, 2, 3, 14, 0x90, 13, 9, 0xC5, 4];
let shuffled = to_bytes128(v.shuffle_u8s(Vec128::read_unaligned(&control, cpu)));
assert_eq!(shuffled, control.map(|c| if c < 16 { bytes[c as usize] } else { 0 }));
}
#[test]
fn vec128_blend() {
let Some(cpu) = simd_cpu() else { return };
// Only mask bytes which every CPU treats the same: `0x00` or `0xFF`
let (a, b): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
let mask: [u8; 16] = pattern::<16>(23, 5).map(|x| if x & 0x80 != 0 { 0xFF } else { 0 });
assert!(mask.contains(&0xFF) && mask.contains(&0));
let (va, vb, vmask) = (
Vec128::read_unaligned(&a, cpu),
Vec128::read_unaligned(&b, cpu),
Vec128::read_unaligned(&mask, cpu),
);
let expected: [u8; 16] = array::from_fn(|i| if mask[i] == 0xFF { b[i] } else { a[i] });
assert_eq!(to_bytes128(va.blend(vb, vmask)), expected);
let expected: [u8; 16] = array::from_fn(|i| if mask[i] == 0xFF { 0x85 } else { a[i] });
assert_eq!(to_bytes128(va.blend(0x85, vmask)), expected);
}
#[test]
fn vec128_get_u64() {
let Some(cpu) = simd_cpu() else { return };
let values: [u64; 2] = [0x0102_0304_0506_0708, 1 << 63];
let v = Vec128::read_unaligned(&values, cpu);
assert_eq!(v.get_u64::<0>(), values[0]);
assert_eq!(v.get_u64::<1>(), values[1]);
}
#[test]
fn vec256_create() {
let Some(cpu) = simd_cpu() else { return };
assert_eq!(to_bytes256(Vec256::zero(cpu)), [0; 32]);
assert_eq!(to_bytes256(Vec256::zero(Vec256::zero(cpu).cpu())), [0; 32]);
assert_eq!(to_bytes256(Vec256::splat(0x85, cpu)), [0x85; 32]);
assert_eq!(to_bytes256(Vec256::splat(0x85, cpu).eq_u8(0x85)), [0xFF; 32]);
assert_eq!(to_u32s(Vec256::splat_u32(0x8234_5678, cpu)), [0x8234_5678; 8]);
let bytes = pattern(37, 11);
assert_eq!(to_bytes256(Vec256::read_unaligned(&bytes, cpu)), bytes);
let values = [1, 2, 3, 0x8000_0000, 5, 6, 7, u32::MAX];
assert_eq!(to_u32s(Vec256::read_unaligned(&values, cpu)), values);
}
#[test]
fn vec256_from_vec128() {
let Some(cpu) = simd_cpu() else { return };
let bytes: [u8; 16] = pattern(37, 11);
let v = Vec128::read_unaligned(&bytes, cpu);
let splatted = to_bytes256(Vec256::splat_v128(v));
assert_eq!(splatted[..16], bytes);
assert_eq!(splatted[16..], bytes);
let expected: [u32; 8] = array::from_fn(|i| u32::from(bytes[i]));
assert_eq!(to_u32s(Vec256::widen_u8_to_u32(v)), expected);
}
#[test]
fn vec256_load_store() {
let Some(cpu) = simd_cpu() else { return };
let src = VecData256::from_bytes(pattern(37, 11));
let mut dest = VecData256::from_bytes([0; 32]);
// SAFETY: `src` and `dest` are 32 bytes and aligned on 32
unsafe { Vec256::load(src.as_ptr(), cpu).store(dest.as_mut_ptr()) };
assert_eq!(*dest.as_bytes(), *src.as_bytes());
let mut dest = VecData256::from_bytes([0; 32]);
Vec256::read(&src, cpu).write(&mut dest);
assert_eq!(*dest.as_bytes(), *src.as_bytes());
assert_eq!(to_bytes256(src.read(cpu)), *src.as_bytes());
let src = pattern::<34>(37, 11);
let mut dest = [0; 34];
// SAFETY: `src` and `dest` are valid for 32 bytes starting at offset 1
unsafe {
let v = Vec256::load_unaligned(src.as_ptr().add(1), cpu);
v.store_unaligned(dest.as_mut_ptr().add(1));
}
assert_eq!(dest[1..33], src[1..33]);
assert_eq!((dest[0], dest[33]), (0, 0));
let values = [1, u64::MAX, 0x0102_0304_0506_0708, 1 << 63];
let bytes = to_bytes256(Vec256::read_unaligned(&values, cpu));
assert_eq!(bytes[..], *values.map(u64::to_le_bytes).as_flattened());
}
#[test]
fn vec256_read_unaligned() {
let Some(cpu) = simd_cpu() else { return };
let bytes: [u8; 32] = pattern(37, 11);
// SAFETY: These types are all 32 bytes, and all bit patterns are valid for them
let vectors = unsafe {
[
Vec256::read_unaligned(&bytes, cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i8; 32]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u16; 16]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i16; 16]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u32; 8]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i32; 8]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u64; 4]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i64; 4]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [u128; 2]>(bytes), cpu),
Vec256::read_unaligned(&transmute::<[u8; 32], [i128; 2]>(bytes), cpu),
]
};
for v in vectors {
assert_eq!(to_bytes256(v), bytes);
}
}
#[test]
fn vec256_is_zero() {
let Some(cpu) = simd_cpu() else { return };
assert!(Vec256::zero(cpu).is_zero());
assert!(!Vec256::splat(0xFF, cpu).is_zero());
// Check every byte, with only the lowest or highest bit set
for i in 0..32 {
for bit in [0x01, 0x80] {
let mut bytes = [0u8; 32];
bytes[i] = bit;
assert!(!Vec256::read_unaligned(&bytes, cpu).is_zero());
}
}
}
#[test]
fn vec256_bitwise() {
let Some(cpu) = simd_cpu() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
assert_eq!(to_bytes256(va & vb), map2(a, b, |x, y| x & y));
assert_eq!(to_bytes256(va | vb), map2(a, b, |x, y| x | y));
assert_eq!(to_bytes256(va ^ vb), map2(a, b, |x, y| x ^ y));
assert_eq!(to_bytes256(!va), a.map(|x| !x));
assert_eq!(to_bytes256(va & 0x3C), a.map(|x| x & 0x3C));
assert_eq!(to_bytes256(va | 0x3C), a.map(|x| x | 0x3C));
assert_eq!(to_bytes256(va ^ 0x3C), a.map(|x| x ^ 0x3C));
let mut v = va;
v &= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x & y));
let mut v = va;
v |= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x | y));
let mut v = va;
v ^= vb;
assert_eq!(to_bytes256(v), map2(a, b, |x, y| x ^ y));
let mut v = va;
v &= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x & 0x3C));
let mut v = va;
v |= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x | 0x3C));
let mut v = va;
v ^= 0x3C;
assert_eq!(to_bytes256(v), a.map(|x| x ^ 0x3C));
}
#[test]
fn vec256_shift() {
let Some(cpu) = simd_cpu() else { return };
let bytes = pattern(37, 11);
let v = Vec256::read_unaligned(&bytes, cpu);
assert_eq!(to_bytes256(v.shl_u16::<3>()), map_u16(bytes, |x| x << 3));
assert_eq!(to_bytes256(v.shr_u16::<4>()), map_u16(bytes, |x| x >> 4));
}
#[test]
fn vec256_add_sub() {
let Some(cpu) = simd_cpu() else { return };
let (a, b) = (pattern(37, 11), pattern(101, 200));
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
assert_eq!(to_bytes256(va.add_u8(vb)), map2(a, b, u8::wrapping_add));
assert_eq!(to_bytes256(va.saturating_add_u8(vb)), map2(a, b, u8::saturating_add));
assert_eq!(to_bytes256(va.sub_u8(vb)), map2(a, b, u8::wrapping_sub));
assert_eq!(to_bytes256(va.saturating_sub_u8(vb)), map2(a, b, u8::saturating_sub));
assert_eq!(to_bytes256(va.add_u8(0x90)), a.map(|x| x.wrapping_add(0x90)));
assert_eq!(to_bytes256(va.saturating_add_u8(0x90)), a.map(|x| x.saturating_add(0x90)));
assert_eq!(to_bytes256(va.sub_u8(0x90)), a.map(|x| x.wrapping_sub(0x90)));
assert_eq!(to_bytes256(va.saturating_sub_u8(0x90)), a.map(|x| x.saturating_sub(0x90)));
}
#[test]
fn vec256_add_u32() {
let Some(cpu) = simd_cpu() else { return };
let a: [u32; 8] = [1, 2, 3, 4, 5, 6, 7, u32::MAX];
let b: [u32; 8] = [10, 20, 30, 0x1_0000, 50, 60, 70, 2];
let sum = Vec256::read_unaligned(&a, cpu).add_u32(Vec256::read_unaligned(&b, cpu));
assert_eq!(to_u32s(sum), [11, 22, 33, 0x1_0004, 55, 66, 77, 1]);
let sum = Vec256::read_unaligned(&a, cpu).add_u32(16);
assert_eq!(to_u32s(sum), [17, 18, 19, 20, 21, 22, 23, 15]);
}
#[test]
fn vec256_compare() {
let Some(cpu) = simd_cpu() else { return };
let (a, mut b): ([u8; 32], [u8; 32]) = (pattern(37, 11), pattern(101, 200));
b[3] = a[3];
b[30] = a[30];
let (va, vb) = (Vec256::read_unaligned(&a, cpu), Vec256::read_unaligned(&b, cpu));
let eq = to_bytes256(va.eq_u8(vb));
assert_eq!(eq, map2(a, b, |x, y| if x == y { 0xFF } else { 0 }));
assert_eq!(eq.iter().filter(|&&x| x != 0).count(), 2);
let eq = to_bytes256(va.eq_u8(a[5]));
assert_eq!(eq, a.map(|x| if x == a[5] { 0xFF } else { 0 }));
// `0xFF` is -1, so less than 1 as a signed integer
let gt = to_bytes256(va.gt_i8(vb));
assert_eq!(gt, map2(a, b, |x, y| if x.cast_signed() > y.cast_signed() { 0xFF } else { 0 }));
let lt = to_bytes256(va.lt_i8(vb));
assert_eq!(lt, map2(a, b, |x, y| if x.cast_signed() < y.cast_signed() { 0xFF } else { 0 }));
// Include values either side of the thresholds
let c: [u8; 32] = array::from_fn(|i| [0, 1, 0x1F, 0x20, 0x21, 0x7F, 0x80, 0xFF][i % 8]);
let vc = Vec256::read_unaligned(&c, cpu);
for n in [0, 0x20] {
let gt = to_bytes256(vc.gt_i8(n));
assert_eq!(gt, c.map(|x| if x.cast_signed() > n { 0xFF } else { 0 }));
let lt = to_bytes256(vc.lt_i8(n));
assert_eq!(lt, c.map(|x| if x.cast_signed() < n { 0xFF } else { 0 }));
}
}
#[test]
fn vec256_all_eq_u8() {
let Some(cpu) = simd_cpu() else { return };
let a: [u8; 32] = pattern(37, 11);
let va = Vec256::read_unaligned(&a, cpu);
assert!(va.all_eq_u8(va));
assert!(!va.all_eq_u8(a[0]));
assert!(Vec256::splat(0x85, cpu).all_eq_u8(0x85));
assert!(!Vec256::splat(0x85, cpu).all_eq_u8(0x84));
// Check every byte, with a difference of only the lowest or highest bit
for i in 0..32 {
for bit in [0x01, 0x80] {
let mut b = a;
b[i] ^= bit;
assert!(!va.all_eq_u8(Vec256::read_unaligned(&b, cpu)));
}
}
}
#[test]
fn vec256_blend() {
let Some(cpu) = simd_cpu() else { return };
// Only mask bytes which every CPU treats the same: `0x00` or `0xFF`
let (a, b): ([u8; 32], [u8; 32]) = (pattern(37, 11), pattern(101, 200));
let mask: [u8; 32] = pattern::<32>(23, 5).map(|x| if x & 0x80 != 0 { 0xFF } else { 0 });
assert!(mask.contains(&0xFF) && mask.contains(&0));
let (va, vb, vmask) = (
Vec256::read_unaligned(&a, cpu),
Vec256::read_unaligned(&b, cpu),
Vec256::read_unaligned(&mask, cpu),
);
let expected: [u8; 32] = array::from_fn(|i| if mask[i] == 0xFF { b[i] } else { a[i] });
assert_eq!(to_bytes256(va.blend(vb, vmask)), expected);
let expected: [u8; 32] = array::from_fn(|i| if mask[i] == 0xFF { 0x85 } else { a[i] });
assert_eq!(to_bytes256(va.blend(0x85, vmask)), expected);
}
#[test]
fn vec256_rearrange_u32s() {
let Some(cpu) = simd_cpu() else { return };
let s = Vec256::read_unaligned(&[0u32, 1, 2, 3, 4, 5, 6, 7], cpu);
let o = Vec256::read_unaligned(&[10u32, 11, 12, 13, 14, 15, 16, 17], cpu);
assert_eq!(to_u32s(s.unpack_lo_u32(o)), [0, 10, 1, 11, 4, 14, 5, 15]);
assert_eq!(to_u32s(s.unpack_hi_u32(o)), [2, 12, 3, 13, 6, 16, 7, 17]);
assert_eq!(to_u32s(s.join_lo_halves(o)), [0, 1, 2, 3, 10, 11, 12, 13]);
assert_eq!(to_u32s(s.join_hi_halves(o)), [4, 5, 6, 7, 14, 15, 16, 17]);
}
#[test]
fn vec256_halves() {
let Some(cpu) = simd_cpu() else { return };
let (low, high): ([u8; 16], [u8; 16]) = (pattern(37, 11), pattern(101, 200));
let (vlow, vhigh) = (Vec128::read_unaligned(&low, cpu), Vec128::read_unaligned(&high, cpu));
let v = Vec256::from_halves(vlow, vhigh);
assert_eq!(to_bytes256(v)[..16], low);
assert_eq!(to_bytes256(v)[16..], high);
let (out_low, out_high) = v.into_halves();
assert_eq!(to_bytes128(out_low), low);
assert_eq!(to_bytes128(out_high), high);
}
crates/oxc_lexer/src/simd/vec128.rs
+635
View File
@@ -0,0 +1,635 @@
//! [`Vec128`], a 128-bit SIMD vector which is generic over the CPU type.
//!
//! [`Vec128Impl`] holds the methods which every CPU type's [`Vec128`] has.
//! The inherent methods here are generic over the CPU type. Each calls the trait method
//! of the same name, so code can call the methods without the trait in scope.
//! Each CPU type implements the trait in its own module (`avx2` or `neon`),
//! along with the methods which only it has.
//!
//! The `!`, `&`, `|` and `^` operators for [`Vec128`] are here too.
#![expect(clippy::inline_always, reason = "wrappers must inline into the function calling them")]
use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not};
use super::{
CpuSimd, VecData128,
arrays::{IntArray64, IntArray128},
operand::Operand128,
};
/// 128-bit SIMD vector.
///
/// Holds a `C`, so a vector can only exist if the CPU supports that feature set.
/// `C` is zero-sized, so [`Vec128`] is the same size as the type it wraps.
///
/// The methods which every CPU type has are in [`Vec128Impl`], and are also inherent methods.
/// The `avx2` and `neon` modules have the methods which only one CPU type has.
pub struct Vec128<C: CpuSimd> {
pub(super) inner: <C::Vec128 as Vec128Impl>::Inner,
pub(super) cpu: C,
}
// `Clone` and `Copy` are implemented by hand, because `#[derive(Copy)]` would add a bound
// requiring the type of `inner` to be `Copy`. Checking that bound requires
// `Vec128<C>: Vec128Impl`, which requires `Vec128<C>: Copy`, which is a cycle.
// `#[derive(Clone)]` has the same problem.
#[expect(clippy::expl_impl_clone_on_copy)]
impl<C: CpuSimd> Clone for Vec128<C> {
/// Copy this [`Vec128`].
#[inline(always)]
fn clone(&self) -> Self {
*self
}
}
impl<C: CpuSimd> Copy for Vec128<C> {}
/// Methods of [`Vec128`] which code that is generic over the CPU type can call.
///
/// Each CPU type's [`Vec128`] implements this trait.
/// Each method is also an inherent method of [`Vec128`],
/// which can be called without this trait in scope.
pub trait Vec128Impl: Copy {
/// CPU feature set which this vector is for.
type Cpu: CpuSimd;
/// Type which this vector wraps.
type Inner: Copy;
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec128`] with all bits zero.
fn zero(cpu: Self::Cpu) -> Self;
/// Create a [`Vec128`] by repeating `n` 16 times.
fn splat(n: u8, cpu: Self::Cpu) -> Self;
/// Create a [`Vec128`] by repeating `n` 4 times.
fn splat_u32(n: u32, cpu: Self::Cpu) -> Self;
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get the [`CpuSimd`] token from this [`Vec128`].
fn cpu(self) -> Self::Cpu;
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec128`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for reading 16 bytes.
/// * `ptr` must be aligned on 16.
unsafe fn load(ptr: *const u8, cpu: Self::Cpu) -> Self;
/// Read a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 16 bytes.
unsafe fn load_unaligned(ptr: *const u8, cpu: Self::Cpu) -> Self;
/// Read 8 bytes at pointer `ptr` into the low half of a [`Vec128`],
/// with no alignment requirement. The high half is set to zero.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 8 bytes.
unsafe fn load_low_unaligned(ptr: *const u8, cpu: Self::Cpu) -> Self;
/// Read a [`Vec128`] from a [`VecData128`], which is aligned on 16.
fn read(data: &VecData128, cpu: Self::Cpu) -> Self;
/// Read a [`Vec128`] from a slice of integers, with no alignment requirement.
///
/// `slice` can be any slice of integers which is 16 bytes in total
/// (e.g. `&[u8; 16]`, `&[i64; 2]`).
fn read_unaligned(slice: &impl IntArray128, cpu: Self::Cpu) -> Self;
/// Read a slice of integers into the low half of a [`Vec128`],
/// with no alignment requirement. The high half is set to zero.
///
/// `slice` can be any slice of integers which is 8 bytes in total
/// (e.g. `&[u8; 8]`, `&[i32; 2]`).
fn read_low_unaligned(slice: &impl IntArray64, cpu: Self::Cpu) -> Self;
/// Store a [`Vec128`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for writing 16 bytes.
/// * `ptr` must be aligned on 16.
unsafe fn store(self, ptr: *mut u8);
/// Store a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 16 bytes.
unsafe fn store_unaligned(self, ptr: *mut u8);
/// Store the low 8 bytes of a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 8 bytes.
unsafe fn store_low_unaligned(self, ptr: *mut u8);
/// Write a [`Vec128`] into a [`VecData128`], which is aligned on 16.
fn write(self, dst: &mut VecData128);
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec128`] are zero.
fn is_zero(self) -> bool;
/// Invert all the bits of this [`Vec128`].
fn not(self) -> Self;
/// AND this [`Vec128`] with either another [`Vec128`] or a `u8`.
fn and(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
/// OR this [`Vec128`] with either another [`Vec128`] or a `u8`.
fn or(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
/// XOR this [`Vec128`] with either another [`Vec128`] or a `u8`.
fn xor(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// wrapping on overflow.
fn add_u8(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// stopping at 255.
fn saturating_add_u8(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// wrapping on overflow.
fn sub_u8(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// stopping at 0.
fn saturating_sub_u8(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec128`] to either another [`Vec128`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
fn eq_u8(self, other: impl Operand128<u8, Self::Cpu>) -> Self;
/// Check if every byte of this [`Vec128`] equals either another [`Vec128`] or a `u8`.
fn all_eq_u8(self, other: impl Operand128<u8, Self::Cpu>) -> bool;
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
fn lt_i8(self, other: impl Operand128<i8, Self::Cpu>) -> Self;
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
fn gt_i8(self, other: impl Operand128<i8, Self::Cpu>) -> Self;
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Shift the whole of this [`Vec128`] right by `BYTES` bytes, filling with zeros.
///
/// Byte `N` moves to byte 0.
fn shr_bytes<const BYTES: i32>(self) -> Self;
/// Rearrange the bytes of this [`Vec128`].
///
/// For each byte `i`:
///
/// - If `control[i]` is less than 16, `result[i]` is `self[control[i]]`.
/// - If the top bit of `control[i]` is set, `result[i]` is 0.
/// - Otherwise, `result[i]` depends on the CPU.
///
/// Often used as a 16-entry lookup table, with `self` as the table.
fn shuffle_u8s(self, control: Self) -> Self;
/// Combine the bytes of this [`Vec128`] and either another [`Vec128`] or a `u8`.
///
/// Each byte of the result comes from `other` if the same byte of `mask` is `0xFF`,
/// or from `self` if it's `0x00`.
/// Other values in `mask` give different results on different CPUs.
fn blend(self, other: impl Operand128<u8, Self::Cpu>, mask: Self) -> Self;
// ----------------------------------------
// `u64`-wise ops
// ----------------------------------------
/// Get one of the 2 `u64`s of this [`Vec128`].
///
/// `INDEX` is 0 for the low 8 bytes, or 1 for the high 8 bytes.
fn get_u64<const INDEX: i32>(self) -> u64;
}
/// Methods which are generic over the CPU type.
///
/// Each calls the [`Vec128Impl`] method of the same name, which each CPU type implements.
/// Calling them doesn't need [`Vec128Impl`] in scope.
impl<C: CpuSimd> Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec128`] with all bits zero.
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn zero(cpu: C) -> Self {
<Self as Vec128Impl>::zero(cpu)
}
/// Create a [`Vec128`] by repeating `n` 16 times.
#[inline(always)]
pub fn splat(n: u8, cpu: C) -> Self {
<Self as Vec128Impl>::splat(n, cpu)
}
/// Create a [`Vec128`] by repeating `n` 4 times.
#[inline(always)]
pub fn splat_u32(n: u32, cpu: C) -> Self {
<Self as Vec128Impl>::splat_u32(n, cpu)
}
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get the [`CpuSimd`] token from this [`Vec128`].
#[cfg_attr(target_arch = "x86_64", expect(dead_code))]
#[inline(always)]
pub fn cpu(self) -> C {
<Self as Vec128Impl>::cpu(self)
}
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec128`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for reading 16 bytes.
/// * `ptr` must be aligned on 16.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub unsafe fn load(ptr: *const u8, cpu: C) -> Self {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec128Impl>::load(ptr, cpu) }
}
/// Read a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 16 bytes.
#[inline(always)]
pub unsafe fn load_unaligned(ptr: *const u8, cpu: C) -> Self {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec128Impl>::load_unaligned(ptr, cpu) }
}
/// Read 8 bytes at pointer `ptr` into the low half of a [`Vec128`],
/// with no alignment requirement. The high half is set to zero.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 8 bytes.
#[inline(always)]
pub unsafe fn load_low_unaligned(ptr: *const u8, cpu: C) -> Self {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec128Impl>::load_low_unaligned(ptr, cpu) }
}
/// Read a [`Vec128`] from a [`VecData128`], which is aligned on 16.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn read(data: &VecData128, cpu: C) -> Self {
<Self as Vec128Impl>::read(data, cpu)
}
/// Read a [`Vec128`] from a slice of integers, with no alignment requirement.
///
/// `slice` can be any slice of integers which is 16 bytes in total
/// (e.g. `&[u8; 16]`, `&[i64; 2]`).
#[inline(always)]
pub fn read_unaligned(slice: &impl IntArray128, cpu: C) -> Self {
<Self as Vec128Impl>::read_unaligned(slice, cpu)
}
/// Read a slice of integers into the low half of a [`Vec128`],
/// with no alignment requirement. The high half is set to zero.
///
/// `slice` can be any slice of integers which is 8 bytes in total
/// (e.g. `&[u8; 8]`, `&[i32; 2]`).
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn read_low_unaligned(slice: &impl IntArray64, cpu: C) -> Self {
<Self as Vec128Impl>::read_low_unaligned(slice, cpu)
}
/// Store a [`Vec128`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for writing 16 bytes.
/// * `ptr` must be aligned on 16.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub unsafe fn store(self, ptr: *mut u8) {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec128Impl>::store(self, ptr) }
}
/// Store a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 16 bytes.
#[inline(always)]
pub unsafe fn store_unaligned(self, ptr: *mut u8) {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec128Impl>::store_unaligned(self, ptr) }
}
/// Store the low 8 bytes of a [`Vec128`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 8 bytes.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub unsafe fn store_low_unaligned(self, ptr: *mut u8) {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec128Impl>::store_low_unaligned(self, ptr) }
}
/// Write a [`Vec128`] into a [`VecData128`], which is aligned on 16.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn write(self, dst: &mut VecData128) {
<Self as Vec128Impl>::write(self, dst);
}
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec128`] are zero.
#[inline(always)]
pub fn is_zero(self) -> bool {
<Self as Vec128Impl>::is_zero(self)
}
/// Invert all the bits of this [`Vec128`].
#[inline(always)]
pub fn not(self) -> Self {
<Self as Vec128Impl>::not(self)
}
/// AND this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn and(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::and(self, other)
}
/// OR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn or(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::or(self, other)
}
/// XOR this [`Vec128`] with either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn xor(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::xor(self, other)
}
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// wrapping on overflow.
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn add_u8(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::add_u8(self, other)
}
/// Add either another [`Vec128`] or a `u8` to every byte of this [`Vec128`],
/// stopping at 255.
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn saturating_add_u8(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::saturating_add_u8(self, other)
}
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// wrapping on overflow.
#[inline(always)]
pub fn sub_u8(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::sub_u8(self, other)
}
/// Subtract either another [`Vec128`] or a `u8` from every byte of this [`Vec128`],
/// stopping at 0.
#[inline(always)]
pub fn saturating_sub_u8(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::saturating_sub_u8(self, other)
}
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec128`] to either another [`Vec128`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[inline(always)]
pub fn eq_u8(self, other: impl Operand128<u8, C>) -> Self {
<Self as Vec128Impl>::eq_u8(self, other)
}
/// Check if every byte of this [`Vec128`] equals either another [`Vec128`] or a `u8`.
#[inline(always)]
pub fn all_eq_u8(self, other: impl Operand128<u8, C>) -> bool {
<Self as Vec128Impl>::all_eq_u8(self, other)
}
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn lt_i8(self, other: impl Operand128<i8, C>) -> Self {
<Self as Vec128Impl>::lt_i8(self, other)
}
/// Compare every byte of this [`Vec128`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec128`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[inline(always)]
pub fn gt_i8(self, other: impl Operand128<i8, C>) -> Self {
<Self as Vec128Impl>::gt_i8(self, other)
}
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Shift the whole of this [`Vec128`] right by `BYTES` bytes, filling with zeros.
///
/// Byte `N` moves to byte 0.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn shr_bytes<const BYTES: i32>(self) -> Self {
<Self as Vec128Impl>::shr_bytes::<BYTES>(self)
}
/// Rearrange the bytes of this [`Vec128`].
///
/// For each byte `i`:
///
/// - If `control[i]` is less than 16, `result[i]` is `self[control[i]]`.
/// - If the top bit of `control[i]` is set, `result[i]` is 0.
/// - Otherwise, `result[i]` depends on the CPU.
///
/// Often used as a 16-entry lookup table, with `self` as the table.
#[inline(always)]
pub fn shuffle_u8s(self, control: Self) -> Self {
<Self as Vec128Impl>::shuffle_u8s(self, control)
}
/// Combine the bytes of this [`Vec128`] and either another [`Vec128`] or a `u8`.
///
/// Each byte of the result comes from `other` if the same byte of `mask` is `0xFF`,
/// or from `self` if it's `0x00`.
/// Other values in `mask` give different results on different CPUs.
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn blend(self, other: impl Operand128<u8, C>, mask: Self) -> Self {
<Self as Vec128Impl>::blend(self, other, mask)
}
// ----------------------------------------
// `u64`-wise ops
// ----------------------------------------
/// Get one of the 2 `u64`s of this [`Vec128`].
///
/// `INDEX` is 0 for the low 8 bytes, or 1 for the high 8 bytes.
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn get_u64<const INDEX: i32>(self) -> u64 {
<Self as Vec128Impl>::get_u64::<INDEX>(self)
}
}
// Operators `!`, `&`, `|`, `^`, `&=`, `|=` and `^=`, which call `not`, `and`, `or` and `xor`.
// As with those methods, the right-hand side can be a vector or a `u8`, so `v & 0x0f` works.
impl<C: CpuSimd> Not for Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn not(self) -> Self {
Vec128::not(self)
}
}
impl<C: CpuSimd, T: Operand128<u8, C>> BitAnd<T> for Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn bitand(self, rhs: T) -> Self {
self.and(rhs)
}
}
impl<C: CpuSimd, T: Operand128<u8, C>> BitAndAssign<T> for Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
#[inline(always)]
fn bitand_assign(&mut self, rhs: T) {
*self = self.and(rhs);
}
}
impl<C: CpuSimd, T: Operand128<u8, C>> BitOr<T> for Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn bitor(self, rhs: T) -> Self {
self.or(rhs)
}
}
impl<C: CpuSimd, T: Operand128<u8, C>> BitOrAssign<T> for Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
#[inline(always)]
fn bitor_assign(&mut self, rhs: T) {
*self = self.or(rhs);
}
}
impl<C: CpuSimd, T: Operand128<u8, C>> BitXor<T> for Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn bitxor(self, rhs: T) -> Self {
self.xor(rhs)
}
}
impl<C: CpuSimd, T: Operand128<u8, C>> BitXorAssign<T> for Vec128<C>
where
Self: Vec128Impl<Cpu = C>,
{
#[inline(always)]
fn bitxor_assign(&mut self, rhs: T) {
*self = self.xor(rhs);
}
}
crates/oxc_lexer/src/simd/vec256.rs
+670
View File
@@ -0,0 +1,670 @@
//! [`Vec256`], a 256-bit SIMD vector which is generic over the CPU type.
//!
//! [`Vec256Impl`] holds the methods which every CPU type's [`Vec256`] has.
//! The inherent methods here are generic over the CPU type. Each calls the trait method
//! of the same name, so code can call the methods without the trait in scope.
//! Each CPU type implements the trait in its own module (`avx2` or `neon`),
//! along with the methods which only it has.
//!
//! The `!`, `&`, `|` and `^` operators for [`Vec256`] are here too.
#![expect(clippy::inline_always, reason = "wrappers must inline into the function calling them")]
use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Not};
use super::{CpuSimd, Vec128, VecData256, arrays::IntArray256, operand::Operand256};
/// 256-bit SIMD vector.
///
/// Holds a `C`, so a vector can only exist if the CPU supports that feature set.
/// `C` is zero-sized, so [`Vec256`] is the same size as the type it wraps.
///
/// The methods which every CPU type has are in [`Vec256Impl`], and are also inherent methods.
/// The `avx2` and `neon` modules have the methods which only one CPU type has.
pub struct Vec256<C: CpuSimd> {
pub(super) inner: <C::Vec256 as Vec256Impl>::Inner,
pub(super) cpu: C,
}
// `Clone` and `Copy` are implemented by hand, because `#[derive(Copy)]` would add a bound
// requiring the type of `inner` to be `Copy`. Checking that bound requires
// `Vec256<C>: Vec256Impl`, which requires `Vec256<C>: Copy`, which is a cycle.
// `#[derive(Clone)]` has the same problem.
#[expect(clippy::expl_impl_clone_on_copy)]
impl<C: CpuSimd> Clone for Vec256<C> {
/// Copy this [`Vec256`].
#[inline(always)]
fn clone(&self) -> Self {
*self
}
}
impl<C: CpuSimd> Copy for Vec256<C> {}
/// Methods of [`Vec256`] which code that is generic over the CPU type can call.
///
/// Each CPU type's [`Vec256`] implements this trait.
/// Each method is also an inherent method of [`Vec256`],
/// which can be called without this trait in scope.
pub trait Vec256Impl: Copy {
/// CPU feature set which this vector is for.
type Cpu: CpuSimd;
/// Type which this vector wraps.
type Inner: Copy;
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec256`] with all bits zero.
fn zero(cpu: Self::Cpu) -> Self;
/// Create a [`Vec256`] by repeating `n` 32 times.
fn splat(n: u8, cpu: Self::Cpu) -> Self;
/// Create a [`Vec256`] by repeating `n` 8 times.
fn splat_u32(n: u32, cpu: Self::Cpu) -> Self;
/// Create a [`Vec256`] by repeating `v` twice.
fn splat_v128(v: Vec128<Self::Cpu>) -> Self;
/// Create a [`Vec256`] by zero-extending the low 8 bytes of `v` to 8 `u32`s.
fn widen_u8_to_u32(v: Vec128<Self::Cpu>) -> Self;
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get the [`CpuSimd`] token from this [`Vec256`].
fn cpu(self) -> Self::Cpu;
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec256`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for reading 32 bytes.
/// * `ptr` must be aligned on 32.
unsafe fn load(ptr: *const u8, cpu: Self::Cpu) -> Self;
/// Read a [`Vec256`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 32 bytes.
unsafe fn load_unaligned(ptr: *const u8, cpu: Self::Cpu) -> Self;
/// Read a [`Vec256`] from a [`VecData256`], which is aligned on 32.
fn read(data: &VecData256, cpu: Self::Cpu) -> Self;
/// Read a [`Vec256`] from a slice of integers, with no alignment requirement.
///
/// `slice` can be any slice of integers which is 32 bytes in total
/// (e.g. `&[u8; 32]`, `&[i64; 4]`).
fn read_unaligned(slice: &impl IntArray256, cpu: Self::Cpu) -> Self;
/// Store a [`Vec256`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for writing 32 bytes.
/// * `ptr` must be aligned on 32.
unsafe fn store(self, ptr: *mut u8);
/// Store a [`Vec256`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 32 bytes.
unsafe fn store_unaligned(self, ptr: *mut u8);
/// Write a [`Vec256`] into a [`VecData256`], which is aligned on 32.
fn write(self, dst: &mut VecData256);
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec256`] are zero.
fn is_zero(self) -> bool;
/// Invert all the bits of this [`Vec256`].
fn not(self) -> Self;
/// AND this [`Vec256`] with either another [`Vec256`] or a `u8`.
fn and(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// OR this [`Vec256`] with either another [`Vec256`] or a `u8`.
fn or(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// XOR this [`Vec256`] with either another [`Vec256`] or a `u8`.
fn xor(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// Shift each of the 16 `u16`s of this [`Vec256`] left by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
fn shl_u16<const BITS: i32>(self) -> Self;
/// Shift each of the 16 `u16`s of this [`Vec256`] right by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
fn shr_u16<const BITS: i32>(self) -> Self;
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// wrapping on overflow.
fn add_u8(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// stopping at 255.
fn saturating_add_u8(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// wrapping on overflow.
fn sub_u8(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// stopping at 0.
fn saturating_sub_u8(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// Add either another [`Vec256`] or a `u32` to every `u32` of this [`Vec256`],
/// wrapping on overflow.
fn add_u32(self, other: impl Operand256<u32, Self::Cpu>) -> Self;
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec256`] to either another [`Vec256`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
fn eq_u8(self, other: impl Operand256<u8, Self::Cpu>) -> Self;
/// Check if every byte of this [`Vec256`] equals either another [`Vec256`] or a `u8`.
fn all_eq_u8(self, other: impl Operand256<u8, Self::Cpu>) -> bool;
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
fn lt_i8(self, other: impl Operand256<i8, Self::Cpu>) -> Self;
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
fn gt_i8(self, other: impl Operand256<i8, Self::Cpu>) -> Self;
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Combine the bytes of this [`Vec256`] and either another [`Vec256`] or a `u8`.
///
/// Each byte of the result comes from `other` if the same byte of `mask` is `0xFF`,
/// or from `self` if it's `0x00`.
/// Other values in `mask` give different results on different CPUs.
fn blend(self, other: impl Operand256<u8, Self::Cpu>, mask: Self) -> Self;
// ----------------------------------------
// `u32`-wise ops
// ----------------------------------------
/// Interleave the low 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s0, o0, s1, o1, s4, o4, s5, o5]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
fn unpack_lo_u32(self, other: Self) -> Self;
/// Interleave the high 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s2, o2, s3, o3, s6, o6, s7, o7]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
fn unpack_hi_u32(self, other: Self) -> Self;
// ----------------------------------------
// Operations on 128-bit halves
// ----------------------------------------
/// Create a [`Vec256`] from its low and high 128 bits.
fn from_halves(low: Vec128<Self::Cpu>, high: Vec128<Self::Cpu>) -> Self;
/// Get the low and high 128 bits of this [`Vec256`].
fn into_halves(self) -> (Vec128<Self::Cpu>, Vec128<Self::Cpu>);
/// Get a [`Vec256`] made of the low 128 bits of this [`Vec256`],
/// followed by the low 128 bits of `other`.
fn join_lo_halves(self, other: Self) -> Self;
/// Get a [`Vec256`] made of the high 128 bits of this [`Vec256`],
/// followed by the high 128 bits of `other`.
fn join_hi_halves(self, other: Self) -> Self;
}
/// Methods which are generic over the CPU type.
///
/// Each calls the [`Vec256Impl`] method of the same name, which each CPU type implements.
/// Calling them doesn't need [`Vec256Impl`] in scope.
impl<C: CpuSimd> Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
// ----------------------------------------
// Creation methods
// ----------------------------------------
/// Create a [`Vec256`] with all bits zero.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn zero(cpu: C) -> Self {
<Self as Vec256Impl>::zero(cpu)
}
/// Create a [`Vec256`] by repeating `n` 32 times.
#[inline(always)]
pub fn splat(n: u8, cpu: C) -> Self {
<Self as Vec256Impl>::splat(n, cpu)
}
/// Create a [`Vec256`] by repeating `n` 8 times.
#[inline(always)]
pub fn splat_u32(n: u32, cpu: C) -> Self {
<Self as Vec256Impl>::splat_u32(n, cpu)
}
/// Create a [`Vec256`] by repeating `v` twice.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn splat_v128(v: Vec128<C>) -> Self {
<Self as Vec256Impl>::splat_v128(v)
}
/// Create a [`Vec256`] by zero-extending the low 8 bytes of `v` to 8 `u32`s.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn widen_u8_to_u32(v: Vec128<C>) -> Self {
<Self as Vec256Impl>::widen_u8_to_u32(v)
}
// ----------------------------------------
// Accessors
// ----------------------------------------
/// Get the [`CpuSimd`] token from this [`Vec256`].
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn cpu(self) -> C {
<Self as Vec256Impl>::cpu(self)
}
// ----------------------------------------
// Loads and stores
// ----------------------------------------
/// Read a [`Vec256`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for reading 32 bytes.
/// * `ptr` must be aligned on 32.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub unsafe fn load(ptr: *const u8, cpu: C) -> Self {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec256Impl>::load(ptr, cpu) }
}
/// Read a [`Vec256`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for reading 32 bytes.
#[inline(always)]
pub unsafe fn load_unaligned(ptr: *const u8, cpu: C) -> Self {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec256Impl>::load_unaligned(ptr, cpu) }
}
/// Read a [`Vec256`] from a [`VecData256`], which is aligned on 32.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn read(data: &VecData256, cpu: C) -> Self {
<Self as Vec256Impl>::read(data, cpu)
}
/// Read a [`Vec256`] from a slice of integers, with no alignment requirement.
///
/// `slice` can be any slice of integers which is 32 bytes in total
/// (e.g. `&[u8; 32]`, `&[i64; 4]`).
#[inline(always)]
pub fn read_unaligned(slice: &impl IntArray256, cpu: C) -> Self {
<Self as Vec256Impl>::read_unaligned(slice, cpu)
}
/// Store a [`Vec256`] at pointer `ptr`, which must be aligned.
///
/// # SAFETY
///
/// * `ptr` must be valid for writing 32 bytes.
/// * `ptr` must be aligned on 32.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub unsafe fn store(self, ptr: *mut u8) {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec256Impl>::store(self, ptr) }
}
/// Store a [`Vec256`] at pointer `ptr`, with no alignment requirement.
///
/// # SAFETY
///
/// `ptr` must be valid for writing 32 bytes.
#[inline(always)]
pub unsafe fn store_unaligned(self, ptr: *mut u8) {
// SAFETY: Caller guarantees the same requirements as the trait method has
unsafe { <Self as Vec256Impl>::store_unaligned(self, ptr) }
}
/// Write a [`Vec256`] into a [`VecData256`], which is aligned on 32.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn write(self, dst: &mut VecData256) {
<Self as Vec256Impl>::write(self, dst);
}
// ----------------------------------------
// Bitwise ops
// ----------------------------------------
/// Check if all the bits of this [`Vec256`] are zero.
#[inline(always)]
pub fn is_zero(self) -> bool {
<Self as Vec256Impl>::is_zero(self)
}
/// Invert all the bits of this [`Vec256`].
#[inline(always)]
pub fn not(self) -> Self {
<Self as Vec256Impl>::not(self)
}
/// AND this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn and(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::and(self, other)
}
/// OR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn or(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::or(self, other)
}
/// XOR this [`Vec256`] with either another [`Vec256`] or a `u8`.
#[inline(always)]
pub fn xor(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::xor(self, other)
}
/// Shift each of the 16 `u16`s of this [`Vec256`] left by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn shl_u16<const BITS: i32>(self) -> Self {
<Self as Vec256Impl>::shl_u16::<BITS>(self)
}
/// Shift each of the 16 `u16`s of this [`Vec256`] right by `BITS` bits.
///
/// `BITS` must be between 1 and 15 (inclusive).
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn shr_u16<const BITS: i32>(self) -> Self {
<Self as Vec256Impl>::shr_u16::<BITS>(self)
}
// ----------------------------------------
// Arithmetic ops
// ----------------------------------------
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// wrapping on overflow.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn add_u8(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::add_u8(self, other)
}
/// Add either another [`Vec256`] or a `u8` to every byte of this [`Vec256`],
/// stopping at 255.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn saturating_add_u8(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::saturating_add_u8(self, other)
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// wrapping on overflow.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn sub_u8(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::sub_u8(self, other)
}
/// Subtract either another [`Vec256`] or a `u8` from every byte of this [`Vec256`],
/// stopping at 0.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn saturating_sub_u8(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::saturating_sub_u8(self, other)
}
/// Add either another [`Vec256`] or a `u32` to every `u32` of this [`Vec256`],
/// wrapping on overflow.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn add_u32(self, other: impl Operand256<u32, C>) -> Self {
<Self as Vec256Impl>::add_u32(self, other)
}
// ----------------------------------------
// Comparison
// ----------------------------------------
/// Compare every byte of this [`Vec256`] to either another [`Vec256`] or a `u8`.
///
/// Each byte of the result is `0xFF` if the bytes are equal, otherwise `0x00`.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn eq_u8(self, other: impl Operand256<u8, C>) -> Self {
<Self as Vec256Impl>::eq_u8(self, other)
}
/// Check if every byte of this [`Vec256`] equals either another [`Vec256`] or a `u8`.
#[cfg_attr(not(test), expect(dead_code))]
#[inline(always)]
pub fn all_eq_u8(self, other: impl Operand256<u8, C>) -> bool {
<Self as Vec256Impl>::all_eq_u8(self, other)
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is less, otherwise `0x00`.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn lt_i8(self, other: impl Operand256<i8, C>) -> Self {
<Self as Vec256Impl>::lt_i8(self, other)
}
/// Compare every byte of this [`Vec256`] to `other`, as signed integers.
/// `other` is either an `i8`, or another [`Vec256`] to compare byte by byte.
///
/// Each byte of the result is `0xFF` if the byte in `self` is greater, otherwise `0x00`.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn gt_i8(self, other: impl Operand256<i8, C>) -> Self {
<Self as Vec256Impl>::gt_i8(self, other)
}
// ----------------------------------------
// Bytewise ops
// ----------------------------------------
/// Combine the bytes of this [`Vec256`] and either another [`Vec256`] or a `u8`.
///
/// Each byte of the result comes from `other` if the same byte of `mask` is `0xFF`,
/// or from `self` if it's `0x00`.
/// Other values in `mask` give different results on different CPUs.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn blend(self, other: impl Operand256<u8, C>, mask: Self) -> Self {
<Self as Vec256Impl>::blend(self, other, mask)
}
// ----------------------------------------
// `u32`-wise ops
// ----------------------------------------
/// Interleave the low 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s0, o0, s1, o1, s4, o4, s5, o5]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn unpack_lo_u32(self, other: Self) -> Self {
<Self as Vec256Impl>::unpack_lo_u32(self, other)
}
/// Interleave the high 2 `u32`s of each 128-bit half of this [`Vec256`] and `other`.
///
/// The result is `[s2, o2, s3, o3, s6, o6, s7, o7]`,
/// where `sN` and `oN` are the `u32`s at index `N` in `self` and `other`.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn unpack_hi_u32(self, other: Self) -> Self {
<Self as Vec256Impl>::unpack_hi_u32(self, other)
}
// ----------------------------------------
// Operations on 128-bit halves
// ----------------------------------------
/// Create a [`Vec256`] from its low and high 128 bits.
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn from_halves(low: Vec128<C>, high: Vec128<C>) -> Self {
<Self as Vec256Impl>::from_halves(low, high)
}
/// Get the low and high 128 bits of this [`Vec256`].
#[cfg_attr(all(not(test), target_arch = "x86_64"), expect(dead_code))]
#[inline(always)]
pub fn into_halves(self) -> (Vec128<C>, Vec128<C>) {
<Self as Vec256Impl>::into_halves(self)
}
/// Get a [`Vec256`] made of the low 128 bits of this [`Vec256`],
/// followed by the low 128 bits of `other`.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn join_lo_halves(self, other: Self) -> Self {
<Self as Vec256Impl>::join_lo_halves(self, other)
}
/// Get a [`Vec256`] made of the high 128 bits of this [`Vec256`],
/// followed by the high 128 bits of `other`.
#[cfg_attr(all(not(test), target_arch = "aarch64"), expect(dead_code))]
#[inline(always)]
pub fn join_hi_halves(self, other: Self) -> Self {
<Self as Vec256Impl>::join_hi_halves(self, other)
}
}
// Operators `!`, `&`, `|`, `^`, `&=`, `|=` and `^=`, which call `not`, `and`, `or` and `xor`.
// As with those methods, the right-hand side can be a vector or a `u8`, so `v & 0x0f` works.
impl<C: CpuSimd> Not for Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn not(self) -> Self {
Vec256::not(self)
}
}
impl<C: CpuSimd, T: Operand256<u8, C>> BitAnd<T> for Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn bitand(self, rhs: T) -> Self {
self.and(rhs)
}
}
impl<C: CpuSimd, T: Operand256<u8, C>> BitAndAssign<T> for Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
#[inline(always)]
fn bitand_assign(&mut self, rhs: T) {
*self = self.and(rhs);
}
}
impl<C: CpuSimd, T: Operand256<u8, C>> BitOr<T> for Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn bitor(self, rhs: T) -> Self {
self.or(rhs)
}
}
impl<C: CpuSimd, T: Operand256<u8, C>> BitOrAssign<T> for Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
#[inline(always)]
fn bitor_assign(&mut self, rhs: T) {
*self = self.or(rhs);
}
}
impl<C: CpuSimd, T: Operand256<u8, C>> BitXor<T> for Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
type Output = Self;
#[inline(always)]
fn bitxor(self, rhs: T) -> Self {
self.xor(rhs)
}
}
impl<C: CpuSimd, T: Operand256<u8, C>> BitXorAssign<T> for Vec256<C>
where
Self: Vec256Impl<Cpu = C>,
{
#[inline(always)]
fn bitxor_assign(&mut self, rhs: T) {
*self = self.xor(rhs);
}
}