mirror of
https://github.com/oxc-project/oxc.git
synced 2026-11-04 10:29:58 +00:00
refactor(lexer): make CPU types and all DynCpu variants exist on all platforms (#28769)
Pure refactor - no change to codegen. Preparatory step for compiling all of the lexer's code on all platforms (#28772). Previously, `Avx2` only existed on x86_64, and `Neon` only on aarch64, and the variants of `DynCpu` had `#[cfg]` attributes to match. `Generic` existed on all platforms, but was uninhabited on aarch64 with Neon enabled. Make `Avx2` and `Neon` work the same way as `Generic`. All these types now exists on every platform. Each has a private field which is `()` on its own platform, and `Never` (an enum with no variants) everywhere else: ```rust pub struct Avx2( cfg_select! { target_arch = "x86_64" => (), _ => Never, }, ); ``` On platforms which can't support the CPU feature set, the type is uninhabited, so it can never be created, and the compiler removes code which handles it. `DynCpu` has all 3 variants on every platform. Variants which can't exist on a platform are `#[expect(dead_code)]` there. The field is private, so only code in the `cpu` module knows that a type is uninhabited. A `match` anywhere else needs an arm for every variant, including ones which can't exist. So `cpu_select!` and `#[cpu_multiversion]` add arms for them, e.g. `DynCpu::Neon(_) => unreachable!()` on x86_64.
This commit is contained in:
overlookmotel committed
1 parent
b03fadad8d
commit
6d5be86ea1
6 files changed
+213
-166
No files matched your search
+146
-159
@@ -1,8 +1,11 @@
|
||||
//! Types representing CPU feature sets, used to choose between SIMD and generic implementations.
|
||||
//!
|
||||
//! - `Avx2`: x86_64 with AVX2, BMI2 and POPCNT.
|
||||
//! - `Neon`: aarch64 with Neon.
|
||||
//! - [`Generic`]: No SIMD. Exists on all platforms.
|
||||
//! - [`Avx2`]: x86_64 with AVX2, BMI2 and POPCNT.
|
||||
//! - [`Neon`]: aarch64 with Neon.
|
||||
//! - [`Generic`]: No SIMD.
|
||||
//!
|
||||
//! All 3 types exist on all platforms. On a platform where a feature set is never used,
|
||||
//! its type is uninhabited - it can never be created.
|
||||
//!
|
||||
//! [`DynCpu`] is an enum of these types. [`DynCpu::detect`] returns the best one available.
|
||||
//!
|
||||
@@ -26,22 +29,24 @@
|
||||
//! can do either a compile-time or runtime features check, depending on whether it's called with
|
||||
//! a concrete type (e.g. `Avx2`) or [`DynCpu`].
|
||||
//!
|
||||
//! `Avx2`, `Neon`, and [`Generic`] types are all zero-sized, so can be passed between functions
|
||||
//! [`Avx2`], [`Neon`] and [`Generic`] types are all zero-sized, so can be passed between functions
|
||||
//! without any runtime cost. They operate at the level of the type system only.
|
||||
//!
|
||||
//! # SAFETY
|
||||
//!
|
||||
//! These types are unforgeable.
|
||||
//!
|
||||
//! * Code outside this file can't create them.
|
||||
//! * Each has a private field, and only 3 places create them:
|
||||
//! - `DynCpu::detect` only creates an `Avx2` if that feature set is available.
|
||||
//! - [`BASELINE`] is only a `Neon` when Neon is enabled at compile time, which guarantees it's available.
|
||||
//! * Each has a private field, so code outside this module can't create them.
|
||||
//! * Only 3 places create them:
|
||||
//! - `DynCpu::detect` only creates an [`Avx2`] if that feature set is available.
|
||||
//! - [`BASELINE`] is only a [`Neon`] when Neon is enabled at compile time,
|
||||
//! which guarantees it's available.
|
||||
//! Otherwise it's a [`Generic`], which guarantees nothing.
|
||||
//! - `CpuPrivate::DYN_CPU` holds an instance of the type.
|
||||
//! The trait is private, so only code in this file can read it.
|
||||
//! Only [`to_dyn`] reads it, and it takes an instance of the same type,
|
||||
//! so it only gives one to code which already has one.
|
||||
//! * On a platform where a type is uninhabited, it can't be created at all.
|
||||
//! * A [`DynCpu`] can only be built from one of these types, so it's unforgeable too.
|
||||
//! * [`Cpu`] trait is sealed, so no other type can implement it.
|
||||
//!
|
||||
@@ -55,7 +60,7 @@
|
||||
//! [`#[cpu_version]`]: macro@cpu_version
|
||||
//! [`#[cpu_multiversion]`]: macro@cpu_multiversion
|
||||
|
||||
use std::mem;
|
||||
use std::{convert::Infallible as Never, mem};
|
||||
|
||||
pub(crate) use oxc_lexer_macros::{cpu_multiversion, cpu_version};
|
||||
|
||||
@@ -93,8 +98,8 @@ pub trait Cpu: CpuPrivate + Copy {
|
||||
trait CpuPrivate {
|
||||
/// The [`DynCpu`] which [`to_dyn`] returns for this type.
|
||||
///
|
||||
/// `Avx2`, `Neon` and [`Generic`] each have only one value, so this can hold it.
|
||||
/// `None` for [`DynCpu`] itself, and for [`Generic`] when it's uninhabited.
|
||||
/// [`Avx2`], [`Neon`] and [`Generic`] each have only one value, so this can hold it.
|
||||
/// `None` for [`DynCpu`] itself, and for each of them on platforms where it's uninhabited.
|
||||
const DYN_CPU: Option<DynCpu>;
|
||||
}
|
||||
|
||||
@@ -108,16 +113,22 @@ trait CpuPrivate {
|
||||
/// [`detect`]: Self::detect
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum DynCpu {
|
||||
/// AVX2 feature set. Only available on x86_64.
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
/// AVX2 feature set.
|
||||
/// Exists on all platforms, but can only be created on x86_64. See [`Avx2`].
|
||||
#[cfg_attr(not(target_arch = "x86_64"), expect(dead_code))]
|
||||
Avx2(Avx2),
|
||||
/// Neon feature set. Only available on aarch64.
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
Neon(Neon),
|
||||
/// Generic feature set.
|
||||
/// Exists on all platforms, but can never be created on aarch64
|
||||
/// when Neon is enabled at compile time. See [`Generic`].
|
||||
#[cfg_attr(all(target_arch = "aarch64", target_feature = "neon"), expect(dead_code))]
|
||||
Generic(Generic),
|
||||
// `Neon` is last. rustc numbers variants in the order they're declared, uninhabited ones too.
|
||||
// So on x86_64, where `Neon` is uninhabited, the 2 variants which can exist are 0 and 1,
|
||||
// like a `bool`.
|
||||
/// Neon feature set.
|
||||
/// Exists on all platforms, but can only be created on aarch64. See [`Neon`].
|
||||
#[cfg_attr(not(target_arch = "aarch64"), expect(dead_code))]
|
||||
Neon(Neon),
|
||||
}
|
||||
|
||||
impl Cpu for DynCpu {}
|
||||
@@ -141,158 +152,134 @@ impl CpuPrivate for DynCpu {
|
||||
pub const fn to_dyn<C: Cpu>(cpu: C) -> DynCpu {
|
||||
match C::DYN_CPU {
|
||||
Some(dyn_cpu) => dyn_cpu,
|
||||
// SAFETY: `DYN_CPU` is `None` only for `DynCpu`, and for `Generic` when it's uninhabited.
|
||||
// SAFETY: `DYN_CPU` is `None` only for `DynCpu`, and for CPU types which are uninhabited.
|
||||
// No value of an uninhabited type exists, so `cpu` must be a `DynCpu`.
|
||||
None => unsafe { mem::transmute_copy::<C, DynCpu>(&cpu) },
|
||||
}
|
||||
}
|
||||
|
||||
/// Generic implementation.
|
||||
mod generic {
|
||||
use super::*;
|
||||
|
||||
/// Generic CPU feature set.
|
||||
///
|
||||
/// On aarch64 with Neon enabled at compile time, this type is uninhabited - `Generic` can never be created.
|
||||
/// So [`DynCpu::Generic`] can never exist there, and the compiler removes code which handles it.
|
||||
/// Every aarch64 target with `std` enables Neon by default, so in practice this applies to all aarch64 builds.
|
||||
///
|
||||
/// The type still exists there, so code which uses it is still compiled and type-checked.
|
||||
/// That means generic code is still linted, and is visible to rust-analyzer, when working on an ARM Mac.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Generic(GenericInner);
|
||||
|
||||
#[cfg(not(all(target_arch = "aarch64", target_feature = "neon")))]
|
||||
type GenericInner = ();
|
||||
|
||||
// When Neon is enabled at compile time, Neon is always available, so the generic implementation
|
||||
// is never needed. `Infallible` has no values, which makes `Generic` impossible to create.
|
||||
#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
|
||||
type GenericInner = std::convert::Infallible;
|
||||
|
||||
impl Cpu for Generic {}
|
||||
|
||||
impl CpuPrivate for Generic {
|
||||
const DYN_CPU: Option<DynCpu> = cfg_select! {
|
||||
all(target_arch = "aarch64", target_feature = "neon") => None,
|
||||
_ => Some(DynCpu::Generic(GENERIC)),
|
||||
};
|
||||
}
|
||||
|
||||
/// [`Generic`] feature set.
|
||||
///
|
||||
/// Private within this file. Used only to define [`BASELINE`] and `CpuPrivate::DYN_CPU`.
|
||||
/// Not available on aarch64 with Neon enabled at compile time, where a [`Generic`] can't be created.
|
||||
#[cfg(not(all(target_arch = "aarch64", target_feature = "neon")))]
|
||||
pub(super) const GENERIC: Generic = Generic(());
|
||||
}
|
||||
|
||||
pub use generic::Generic;
|
||||
|
||||
/// x86_64 features.
|
||||
/// AVX2 or generic.
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod x86_64 {
|
||||
use super::*;
|
||||
|
||||
/// x64_64 AVX2 CPU feature set.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Avx2(());
|
||||
|
||||
impl Cpu for Avx2 {}
|
||||
|
||||
impl CpuPrivate for Avx2 {
|
||||
const DYN_CPU: Option<DynCpu> = Some(DynCpu::Avx2(Avx2(())));
|
||||
}
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Always [`Generic`] on x86_64.
|
||||
pub const BASELINE: Generic = generic::GENERIC;
|
||||
|
||||
impl DynCpu {
|
||||
/// Detect CPU feature set for x86_64 platform.
|
||||
pub fn detect() -> Self {
|
||||
// Currently a compile-time check. Later on will be replaced by runtime detection.
|
||||
if cfg!(all(
|
||||
target_feature = "avx2",
|
||||
target_feature = "bmi2",
|
||||
target_feature = "popcnt"
|
||||
)) {
|
||||
Avx2(()).to_dyn()
|
||||
} else {
|
||||
BASELINE.to_dyn()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub use x86_64::{Avx2, BASELINE};
|
||||
|
||||
/// aarch64 features.
|
||||
/// Neon or generic.
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
mod aarch64 {
|
||||
use super::*;
|
||||
|
||||
/// aarch64 Neon CPU feature set.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Neon(());
|
||||
|
||||
impl Cpu for Neon {}
|
||||
|
||||
impl CpuPrivate for Neon {
|
||||
const DYN_CPU: Option<DynCpu> = Some(DynCpu::Neon(Neon(())));
|
||||
}
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Neon is enabled at compile time, which guarantees that it's available.
|
||||
#[cfg(target_feature = "neon")]
|
||||
pub const BASELINE: Neon = Neon(());
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Neon is disabled at compile time, so only the generic feature set is known to be available.
|
||||
#[cfg(not(target_feature = "neon"))]
|
||||
pub const BASELINE: Generic = generic::GENERIC;
|
||||
|
||||
impl DynCpu {
|
||||
/// Detect CPU feature set for aarch64 platform.
|
||||
pub fn detect() -> Self {
|
||||
// Neon is always enabled except on `aarch64*-unknown-none-softfloat` platforms
|
||||
// or when it's specifically disabled by compiling with `-C target-feature=-neon`.
|
||||
// Either way, it can be detected at compile time. No runtime feature check.
|
||||
BASELINE.to_dyn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
pub use aarch64::{BASELINE, Neon};
|
||||
|
||||
/// Fallback implementation for other platforms.
|
||||
/// x86_64 AVX2 CPU feature set.
|
||||
///
|
||||
/// Only generic feature set is available.
|
||||
/// On platforms other than x86_64, this type is uninhabited - `Avx2` can never be created.
|
||||
/// So [`DynCpu::Avx2`] can never exist there, and the compiler removes code which handles it.
|
||||
///
|
||||
/// The type still exists there, so [`DynCpu`] has the same variants on every platform.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Avx2(
|
||||
cfg_select! {
|
||||
target_arch = "x86_64" => (),
|
||||
_ => Never,
|
||||
},
|
||||
);
|
||||
|
||||
impl Cpu for Avx2 {}
|
||||
|
||||
impl CpuPrivate for Avx2 {
|
||||
const DYN_CPU: Option<DynCpu> = cfg_select! {
|
||||
target_arch = "x86_64" => Some(DynCpu::Avx2(Avx2(()))),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
|
||||
/// aarch64 Neon CPU feature set.
|
||||
///
|
||||
/// On platforms other than aarch64, this type is uninhabited - `Neon` can never be created.
|
||||
/// So [`DynCpu::Neon`] can never exist there, and the compiler removes code which handles it.
|
||||
///
|
||||
/// The type still exists there, so [`DynCpu`] has the same variants on every platform.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Neon(
|
||||
cfg_select! {
|
||||
target_arch = "aarch64" => (),
|
||||
_ => Never,
|
||||
},
|
||||
);
|
||||
|
||||
impl Cpu for Neon {}
|
||||
|
||||
impl CpuPrivate for Neon {
|
||||
const DYN_CPU: Option<DynCpu> = cfg_select! {
|
||||
target_arch = "aarch64" => Some(DynCpu::Neon(Neon(()))),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
|
||||
/// Generic CPU feature set.
|
||||
///
|
||||
/// On aarch64 with Neon enabled at compile time, this type is uninhabited - `Generic` can never be created.
|
||||
/// So [`DynCpu::Generic`] can never exist there, and the compiler removes code which handles it.
|
||||
/// Every aarch64 target with `std` enables Neon by default, so in practice this applies to all aarch64 builds.
|
||||
///
|
||||
/// The type still exists there, so code which uses it is still compiled and type-checked.
|
||||
/// That means generic code is still linted, and is visible to rust-analyzer, when working on an ARM Mac.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Generic(
|
||||
cfg_select! {
|
||||
not(all(target_arch = "aarch64", target_feature = "neon")) => (),
|
||||
_ => Never,
|
||||
},
|
||||
);
|
||||
|
||||
impl Cpu for Generic {}
|
||||
|
||||
impl CpuPrivate for Generic {
|
||||
const DYN_CPU: Option<DynCpu> = cfg_select! {
|
||||
not(all(target_arch = "aarch64", target_feature = "neon")) => {
|
||||
Some(DynCpu::Generic(Generic(())))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Always [`Generic`] on x86_64.
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub const BASELINE: Generic = Generic(());
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Neon is enabled at compile time, which guarantees that it's available.
|
||||
#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
|
||||
pub const BASELINE: Neon = Neon(());
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Neon is disabled at compile time, so only the generic feature set is known to be available.
|
||||
#[cfg(all(target_arch = "aarch64", not(target_feature = "neon")))]
|
||||
pub const BASELINE: Generic = Generic(());
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Only the generic feature set is available on platforms other than x86_64 and aarch64.
|
||||
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
|
||||
mod other {
|
||||
use super::*;
|
||||
pub const BASELINE: Generic = Generic(());
|
||||
|
||||
/// Feature set to use without detecting the CPU's features.
|
||||
///
|
||||
/// Only the generic feature set is available on platforms other than x86_64 and aarch64.
|
||||
pub const BASELINE: Generic = generic::GENERIC;
|
||||
|
||||
impl DynCpu {
|
||||
/// Detect CPU feature set for other platform.
|
||||
///
|
||||
/// Only generic feature set is available.
|
||||
pub fn detect() -> Self {
|
||||
impl DynCpu {
|
||||
/// Detect CPU feature set.
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub fn detect() -> Self {
|
||||
// Currently a compile-time check. Later on will be replaced by runtime detection.
|
||||
if cfg!(all(target_feature = "avx2", target_feature = "bmi2", target_feature = "popcnt")) {
|
||||
Avx2(()).to_dyn()
|
||||
} else {
|
||||
BASELINE.to_dyn()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
|
||||
pub use other::BASELINE;
|
||||
/// Detect CPU feature set.
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
pub fn detect() -> Self {
|
||||
// Neon is always enabled except on `aarch64*-unknown-none-softfloat` platforms
|
||||
// or when it's specifically disabled by compiling with `-C target-feature=-neon`.
|
||||
// Either way, it can be detected at compile time. No runtime feature check.
|
||||
BASELINE.to_dyn()
|
||||
}
|
||||
|
||||
/// Detect CPU feature set.
|
||||
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
|
||||
pub fn detect() -> Self {
|
||||
// Only generic feature set is available
|
||||
BASELINE.to_dyn()
|
||||
}
|
||||
}
|
||||
@@ -3,11 +3,18 @@
|
||||
/// Arms use normal `match` syntax, but there must be exactly these 3 arms, in this order:
|
||||
///
|
||||
/// - `Avx2(...)`: Only compiled on x86_64.
|
||||
/// On other platforms, the arm's body is replaced with `unreachable!()`.
|
||||
/// - `Neon(...)`: Only compiled on aarch64.
|
||||
/// On other platforms, the arm's body is replaced with `unreachable!()`.
|
||||
/// - `Generic(...)`: Compiled everywhere.
|
||||
/// On aarch64 with Neon enabled at compile time, it's compiled and type-checked,
|
||||
/// but can never run, so the compiler removes it. See [`Generic`].
|
||||
///
|
||||
/// [`DynCpu`] has all 3 variants on every platform. On a platform which doesn't have
|
||||
/// a SIMD feature set, its type is uninhabited, so its arm can never run.
|
||||
/// The arm is still needed, because the type's field is private, so outside the module which
|
||||
/// defines the type, rustc doesn't know that it's uninhabited.
|
||||
///
|
||||
/// Each arm binds its CPU feature set type (e.g. `Avx2`), so it can be passed on.
|
||||
/// Fallback arms (e.g. `_ => ...`) and match guards are not allowed.
|
||||
///
|
||||
@@ -52,9 +59,13 @@
|
||||
///
|
||||
/// ```ignore
|
||||
/// match to_dyn(cpu) {
|
||||
/// #[cfg(not(target_arch = "x86_64"))]
|
||||
/// DynCpu::Avx2(_) => unreachable!(),
|
||||
/// #[cfg(target_arch = "x86_64")]
|
||||
/// DynCpu::Avx2(cpu) => classify_avx2(self, cpu),
|
||||
///
|
||||
/// #[cfg(not(target_arch = "aarch64"))]
|
||||
/// DynCpu::Neon(_) => unreachable!(),
|
||||
/// #[cfg(target_arch = "aarch64")]
|
||||
/// DynCpu::Neon(cpu) => classify_neon(self, cpu),
|
||||
///
|
||||
@@ -66,7 +77,9 @@
|
||||
///
|
||||
/// Tokens of the arms are consumed one at a time, and copied into `arms`.
|
||||
/// `Avx2(...) =>` is rewritten to `DynCpu::Avx2(...) =>`, with a `#[cfg]` attribute for x86_64.
|
||||
/// `Neon(...) =>` is rewritten the same way, with a `#[cfg]` attribute for aarch64.
|
||||
/// Before it goes an arm `DynCpu::Avx2(_) => unreachable!(),` for other platforms.
|
||||
/// That replaces the body without having to find where it ends.
|
||||
/// `Neon(...) =>` is rewritten the same way, with `#[cfg]` attributes for aarch64.
|
||||
/// `Generic(...) =>` is rewritten to `DynCpu::Generic(...) =>`, with no `#[cfg]` attribute.
|
||||
/// All other tokens are copied unchanged, so the arm bodies are parsed by the `match` itself.
|
||||
///
|
||||
@@ -83,6 +96,7 @@
|
||||
/// Each arm's rule only matches if the arms before it have been seen, and it hasn't.
|
||||
/// So the arms must come in that order, once each. Anything else is a compile error.
|
||||
///
|
||||
/// [`DynCpu`]: super::DynCpu
|
||||
/// [`Generic`]: super::Generic
|
||||
macro_rules! cpu_select {
|
||||
// Entry point
|
||||
@@ -113,6 +127,8 @@ macro_rules! cpu_select {
|
||||
seen = [ avx2 ]
|
||||
arms = [
|
||||
$($arms)*
|
||||
#[cfg(not(target_arch = "x86_64"))]
|
||||
crate::cpu::DynCpu::Avx2(_) => unreachable!(),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
crate::cpu::DynCpu::Avx2($($pat)*) =>
|
||||
]
|
||||
@@ -137,6 +153,8 @@ macro_rules! cpu_select {
|
||||
seen = [ avx2, neon ]
|
||||
arms = [
|
||||
$($arms)*
|
||||
#[cfg(not(target_arch = "aarch64"))]
|
||||
crate::cpu::DynCpu::Neon(_) => unreachable!(),
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
crate::cpu::DynCpu::Neon($($pat)*) =>
|
||||
]
|
||||
|
||||
@@ -669,6 +669,12 @@ macro_rules! wrapped_tests {
|
||||
/// All these calls are safe.
|
||||
#[test]
|
||||
fn wrapped() {
|
||||
// This module can see the CPU types' private fields. So on aarch64 with Neon,
|
||||
// rustc knows that `Avx2` and `Generic` are uninhabited, and this always matches.
|
||||
#[cfg_attr(
|
||||
all(target_arch = "aarch64", target_feature = "neon"),
|
||||
expect(irrefutable_let_patterns)
|
||||
)]
|
||||
let DynCpu::$cpu(cpu) = DynCpu::detect() else { return };
|
||||
|
||||
assert_eq!(generics(&[1u8, 2], &[3, 4], false, cpu), 2);
|
||||
|
||||
@@ -14,14 +14,17 @@ pub enum Cpu {
|
||||
Avx2,
|
||||
/// aarch64 with Neon
|
||||
Neon,
|
||||
/// No SIMD. Exists on all platforms.
|
||||
/// No SIMD
|
||||
Generic,
|
||||
}
|
||||
|
||||
impl Cpu {
|
||||
/// All the feature sets, in the order which `cpu_select!` requires its arms to be in.
|
||||
pub const ALL: [Cpu; 3] = [Cpu::Avx2, Cpu::Neon, Cpu::Generic];
|
||||
|
||||
/// The feature set called `ident`, e.g. the `Avx2` in `#[cpu_version(Avx2)]`.
|
||||
pub fn from_ident(ident: &Ident) -> Option<Cpu> {
|
||||
[Cpu::Avx2, Cpu::Neon, Cpu::Generic].into_iter().find(|cpu| ident == cpu.name())
|
||||
Cpu::ALL.into_iter().find(|cpu| ident == cpu.name())
|
||||
}
|
||||
|
||||
/// Name of the type, and of the `DynCpu` variant.
|
||||
@@ -59,8 +62,8 @@ impl Cpu {
|
||||
quote!(crate::cpu::DynCpu::#name)
|
||||
}
|
||||
|
||||
/// `#[cfg(...)]` attribute for the platform the feature set exists on.
|
||||
/// `None` for `Generic`, which exists everywhere.
|
||||
/// `#[cfg(...)]` attribute for the platform which supports the feature set.
|
||||
/// `None` for `Generic`, which every platform supports.
|
||||
pub fn cfg_attr(self) -> Option<TokenStream> {
|
||||
let arch = match self {
|
||||
Cpu::Avx2 => "x86_64",
|
||||
|
||||
@@ -173,6 +173,8 @@ mod version;
|
||||
/// match to_dyn(cpu) {
|
||||
/// // SAFETY: `Avx2` type guarantees the CPU supports the features `__avx2` uses
|
||||
/// DynCpu::Avx2(cpu) => unsafe { __avx2(src, n, cpu) },
|
||||
/// // `Neon` type is uninhabited on x86_64
|
||||
/// DynCpu::Neon(_) => unreachable!(),
|
||||
/// // SAFETY: `__generic` only has `#[target_feature(enable = "sse2")]`
|
||||
/// // when SSE2 is enabled for the whole crate, so the CPU supports it
|
||||
/// DynCpu::Generic(cpu) => unsafe { __generic(src, n, cpu) },
|
||||
@@ -186,6 +188,8 @@ mod version;
|
||||
/// }
|
||||
///
|
||||
/// match to_dyn(cpu) {
|
||||
/// // `Avx2` type is uninhabited on aarch64
|
||||
/// DynCpu::Avx2(_) => unreachable!(),
|
||||
/// // SAFETY: `Neon` type guarantees the CPU supports the features `__neon` uses
|
||||
/// DynCpu::Neon(cpu) => unsafe { __neon(src, n, cpu) },
|
||||
/// DynCpu::Generic(cpu) => unsafe { __generic(src, n, cpu) },
|
||||
@@ -193,6 +197,9 @@ mod version;
|
||||
/// }
|
||||
/// _ => {
|
||||
/// match to_dyn(cpu) {
|
||||
/// // `Avx2` and `Neon` types are uninhabited on other platforms
|
||||
/// DynCpu::Avx2(_) => unreachable!(),
|
||||
/// DynCpu::Neon(_) => unreachable!(),
|
||||
/// // SAFETY: `__generic` only has `#[target_feature(enable = "sse2")]`
|
||||
/// // when SSE2 is enabled for the whole crate, so the CPU supports it
|
||||
/// DynCpu::Generic(cpu) => unsafe { __generic(src, n, cpu) },
|
||||
@@ -202,6 +209,10 @@ mod version;
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// `DynCpu` has a variant for each feature set on every platform, so each `match` has an arm
|
||||
/// for each. On a platform without a SIMD feature set, that feature set's type is uninhabited,
|
||||
/// so its arm can never run, and is `unreachable!()`.
|
||||
///
|
||||
/// For a method, the `match` is the whole body of the outer method, in a `cfg_select!` of its
|
||||
/// own, and the other functions are methods next to it.
|
||||
#[proc_macro_attribute]
|
||||
|
||||
@@ -83,6 +83,7 @@ impl Multiversion<'_> {
|
||||
///
|
||||
/// match crate::cpu::to_dyn(cpu) {
|
||||
/// crate::cpu::DynCpu::Avx2(cpu) => unsafe { __avx2::<N, T>(arr, cpu) },
|
||||
/// crate::cpu::DynCpu::Neon(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Generic(cpu) => unsafe { __generic::<N, T>(arr, cpu) },
|
||||
/// }
|
||||
/// }
|
||||
@@ -94,12 +95,15 @@ impl Multiversion<'_> {
|
||||
/// }
|
||||
///
|
||||
/// match crate::cpu::to_dyn(cpu) {
|
||||
/// crate::cpu::DynCpu::Avx2(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Neon(cpu) => unsafe { __neon::<N, T>(arr, cpu) },
|
||||
/// crate::cpu::DynCpu::Generic(cpu) => unsafe { __generic::<N, T>(arr, cpu) },
|
||||
/// }
|
||||
/// }
|
||||
/// _ => {
|
||||
/// match crate::cpu::to_dyn(cpu) {
|
||||
/// crate::cpu::DynCpu::Avx2(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Neon(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Generic(cpu) => unsafe { __generic::<N, T>(arr, cpu) },
|
||||
/// }
|
||||
/// }
|
||||
@@ -143,6 +147,7 @@ impl Multiversion<'_> {
|
||||
/// crate::cpu::DynCpu::Avx2(cpu) => unsafe {
|
||||
/// Self::__foo_avx2::<N, T>(self, arr, cpu)
|
||||
/// },
|
||||
/// crate::cpu::DynCpu::Neon(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Generic(cpu) => unsafe {
|
||||
/// Self::__foo_generic::<N, T>(self, arr, cpu)
|
||||
/// },
|
||||
@@ -150,6 +155,7 @@ impl Multiversion<'_> {
|
||||
/// }
|
||||
/// target_arch = "aarch64" => {
|
||||
/// match crate::cpu::to_dyn(cpu) {
|
||||
/// crate::cpu::DynCpu::Avx2(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Neon(cpu) => unsafe {
|
||||
/// Self::__foo_neon::<N, T>(self, arr, cpu)
|
||||
/// },
|
||||
@@ -160,6 +166,8 @@ impl Multiversion<'_> {
|
||||
/// }
|
||||
/// _ => {
|
||||
/// match crate::cpu::to_dyn(cpu) {
|
||||
/// crate::cpu::DynCpu::Avx2(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Neon(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Generic(cpu) => unsafe {
|
||||
/// Self::__foo_generic::<N, T>(self, arr, cpu)
|
||||
/// },
|
||||
@@ -296,18 +304,32 @@ impl Multiversion<'_> {
|
||||
}
|
||||
|
||||
/// `match` which calls the entry point for the CPU's feature set.
|
||||
/// `simd` is the SIMD feature set which exists on the platform, if there is one.
|
||||
/// `simd` is the SIMD feature set which the platform has, if there is one.
|
||||
///
|
||||
/// `DynCpu` has a variant for each feature set on every platform.
|
||||
/// The type of a SIMD feature set which the platform doesn't have is uninhabited,
|
||||
/// so its arm can never run, and is `unreachable!()`.
|
||||
/// The arm is still needed, because the type's field is private, so outside the module which
|
||||
/// defines the type, rustc doesn't know that it's uninhabited.
|
||||
///
|
||||
/// ```ignore
|
||||
/// match crate::cpu::to_dyn(cpu) {
|
||||
/// crate::cpu::DynCpu::Avx2(cpu) => unsafe { __avx2::<N, T>(arr, cpu) },
|
||||
/// crate::cpu::DynCpu::Neon(_) => unreachable!(),
|
||||
/// crate::cpu::DynCpu::Generic(cpu) => unsafe { __generic::<N, T>(arr, cpu) },
|
||||
/// }
|
||||
/// ```
|
||||
fn cpu_match(&self, simd: Option<Cpu>) -> TokenStream {
|
||||
let cpu_name = self.cpu_param.arg();
|
||||
let arms = simd.into_iter().chain([Cpu::Generic]).map(|cpu| {
|
||||
let arms = Cpu::ALL.into_iter().map(|cpu| {
|
||||
let variant = cpu.variant();
|
||||
|
||||
if cpu != Cpu::Generic && simd != Some(cpu) {
|
||||
return quote! {
|
||||
#variant(_) => unreachable!(),
|
||||
};
|
||||
}
|
||||
|
||||
let call = self.call(cpu.lower_name());
|
||||
|
||||
// SAFETY: `Avx2` and `Neon` types guarantee the CPU supports the features their
|
||||
|
||||
Reference in new issue
Block a user