From a76e5fd80d66a0047932041d65a82900ca2cef39 Mon Sep 17 00:00:00 2001 From: overlookmotel <557937+overlookmotel@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:59:01 +0000 Subject: [PATCH] test(lexer): update `just` recipes to cover Neon implementation (#27705) #27642 added an aarch64 Neon implementation of `classify`. So now we have 3 implementations. Update `just ready-lexer` recipe to test and lint all 3. --- .cargo/{lexer-simd.toml => lexer-avx2.toml} | 4 +- .github/workflows/ci.yml | 12 +-- Cargo.toml | 2 +- justfile | 102 +++++++++++------- tasks/benchmark/README.md | 8 +- .../benchmark/benches/lexer_wallclock_new.rs | 10 +- tasks/coverage/Cargo.toml | 9 +- 7 files changed, 89 insertions(+), 58 deletions(-) rename .cargo/{lexer-simd.toml => lexer-avx2.toml} (76%) diff --git a/.cargo/lexer-simd.toml b/.cargo/lexer-avx2.toml similarity index 76% rename from .cargo/lexer-simd.toml rename to .cargo/lexer-avx2.toml index 5fa32769e8..0939f82725 100644 --- a/.cargo/lexer-simd.toml +++ b/.cargo/lexer-avx2.toml @@ -1,5 +1,5 @@ -# Extra rustflags for building `oxc_lexer`'s SIMD core. Passed via `cargo --config` by the -# `just test-lexer-simd` / `just conformance-lexer-simd` recipes - see the justfile. +# Extra rustflags for building `oxc_lexer`'s AVX2 implementation. Passed via `cargo --config` by the +# `just *-lexer-avx2` recipes - see the justfile. # # `avx2`, `bmi2` and `popcnt` are not in the x86_64 baseline on any platform, so they have to be asked for explicitly, # even on an x86_64 host. Cargo joins `rustflags` arrays across config sources, so the per-target flags diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18da203495..d7e0d97dc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -678,7 +678,7 @@ jobs: - variant: generic suffix: "" - variant: AVX2 - suffix: "-simd" + suffix: "-avx2" steps: - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 @@ -687,7 +687,7 @@ jobs: uses: $/.github/actions/check-changes with: packages: oxc_lexer,oxc_coverage - paths: crates/oxc_lexer/,tasks/coverage/,.cargo/lexer-simd.toml,justfile,.github/workflows/ci.yml + paths: crates/oxc_lexer/,tasks/coverage/,.cargo/lexer-avx2.toml,justfile,.github/workflows/ci.yml token: ${{ secrets.GITHUB_TOKEN }} pr-number: ${{ github.event.pull_request.number }} @@ -716,12 +716,12 @@ jobs: - name: Lint # No need to run on generic - already covered by "Lint" job if: ${{ steps.filter.outputs.changed == 'true' && matrix.variant == 'AVX2' }} - run: just lint-lexer-simd + run: just lint-lexer-avx2 - name: Test # No need to run on generic - already covered by "Test Linux" job if: ${{ steps.filter.outputs.changed == 'true' && matrix.variant == 'AVX2' }} - run: just test-lexer-simd + run: just test-lexer-avx2 - wait: lexer-fixtures @@ -756,7 +756,7 @@ jobs: id: filter uses: $/.github/actions/check-changes with: - paths: crates/oxc_lexer/,tasks/benchmark/benches/lexer_wallclock_old.rs,tasks/benchmark/benches/lexer_wallclock_new.rs,tasks/benchmark/benches/lexer_common.rs,.cargo/lexer-simd.toml,justfile,.github/workflows/ci.yml + paths: crates/oxc_lexer/,tasks/benchmark/benches/lexer_wallclock_old.rs,tasks/benchmark/benches/lexer_wallclock_new.rs,tasks/benchmark/benches/lexer_common.rs,.cargo/lexer-avx2.toml,justfile,.github/workflows/ci.yml token: ${{ secrets.GITHUB_TOKEN }} pr-number: ${{ github.event.pull_request.number }} @@ -782,7 +782,7 @@ jobs: CRITERION_HOME: ${{ runner.temp }}/criterion run: | set -o pipefail - just bench-lexer-simd | tee "$RUNNER_TEMP/bench.log" + just bench-lexer-avx2 | tee "$RUNNER_TEMP/bench.log" grep -qxF "Lexer implementation: AVX2" "$RUNNER_TEMP/bench.log" || { echo "::error::Expected AVX2, but the benchmark reported otherwise"; exit 1; } diff --git a/Cargo.toml b/Cargo.toml index 64cb8fecd1..6b64b57370 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,7 +170,7 @@ oxc_formatter_tests = { path = "crates/oxc_formatter_tests" } # Formatter test i oxc_formatter_toml = { path = "crates/oxc_formatter_toml" } # TOML formatter oxc_formatter_yaml = { path = "crates/oxc_formatter_yaml" } # YAML formatter oxc_language_server = { path = "crates/oxc_language_server", default-features = false } # Language server -oxc_lexer = { path = "crates/oxc_lexer" } # JS/TS lexer (incubating; x86_64 SIMD core, scalar fallback) +oxc_lexer = { path = "crates/oxc_lexer" } # JS/TS lexer (incubating; AVX2, Neon, and generic implementations) oxc_linter = { path = "crates/oxc_linter" } # Linting engine oxc_macros = { path = "crates/oxc_macros" } # Proc macros oxc_tasks_common = { path = "tasks/common" } # Task utilities diff --git a/justfile b/justfile index 8da64794ce..87a9283e56 100755 --- a/justfile +++ b/justfile @@ -149,44 +149,63 @@ ast: # ==================== LEXER ==================== -# `oxc_lexer` compiles to one of two implementations, chosen at build time: +# `oxc_lexer` compiles to one of 3 implementations, chosen at build time: # -# * SIMD core, on x86_64 with `avx2` + `bmi2` + `popcnt` enabled -# * Scalar fallback, everywhere else +# * AVX2, on x86_64 with `avx2` + `bmi2` + `popcnt` enabled +# * Neon, on aarch64 (every aarch64 target with `std` enables `neon`) +# * Generic, everywhere else # -# Both need testing, and they should produce identical results. None of `avx2`, `bmi2` and `popcnt` is in the x86_64 -# baseline on any platform, so reaching the SIMD core always means asking for them explicitly - even on an x86_64 host. -# The flags live in `.cargo/lexer-simd.toml`, passed with `cargo --config`, which avoids shell quoting entirely +# Parts of the pipeline which have no AVX2 or Neon implementation use the generic one. +# All 3 need testing, and they should produce identical results. +# +# Each task has 3 recipes: +# +# * `*-lexer` builds for the host - Neon on an ARM host, generic on an x86_64 host. +# * `*-lexer-avx2` builds the AVX2 implementation for x86_64. +# * `*-lexer-generic` builds the generic implementation for x86_64. +# On an x86_64 host, that's the same as `*-lexer`. +# +# None of `avx2`, `bmi2` and `popcnt` is in the x86_64 baseline on any platform, so reaching the AVX2 implementation +# always means asking for them explicitly - even on an x86_64 host. +# The flags live in `.cargo/lexer-avx2.toml`, passed with `cargo --config`, which avoids shell quoting entirely # (this justfile runs PowerShell on Windows). # -# `--target` is always passed, so that the flags land on the triple the config file names, -# and so that the two builds get separate target dirs instead of invalidating each other. +# `--target` is always passed for the x86_64 builds, so that the flags land on the triple the config file names. # -# On an ARM host the SIMD build is cross-compiled and run under emulation - macOS provides that via Rosetta 2. +# On an ARM host the x86_64 builds are cross-compiled and run under emulation - macOS provides that via Rosetta 2. # On Mac, ensure x86_64 target is installed: `rustup target add x86_64-apple-darwin`. # If it's not, you'll get a "can't find crate for std" error. # -# On ARM Linux or ARM Windows it will fail, loudly, when linking or running. -_lexer-simd-target := if os() == "macos" { "x86_64-apple-darwin" } else if os() == "windows" { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" } -_lexer-simd := "--target " + _lexer-simd-target + " --config .cargo/lexer-simd.toml" +# On ARM Linux or ARM Windows they will fail, loudly, when linking or running. +_lexer-x86-target := if os() == "macos" { "x86_64-apple-darwin" } else if os() == "windows" { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" } +_lexer-generic := "--target " + _lexer-x86-target +_lexer-avx2 := _lexer-generic + " --config .cargo/lexer-avx2.toml" -# Run `oxc_lexer`'s tests against the scalar fallback +# Run `oxc_lexer`'s tests, built for the host test-lexer *args='': cargo test -p oxc_lexer {{args}} -# Run `oxc_lexer`'s tests against the SIMD core -test-lexer-simd *args='': - cargo test -p oxc_lexer {{_lexer-simd}} {{args}} +# Run `oxc_lexer`'s tests against the AVX2 implementation +test-lexer-avx2 *args='': + cargo test -p oxc_lexer {{_lexer-avx2}} {{args}} -# Run lexer conformance against the scalar fallback +# Run `oxc_lexer`'s tests against the generic implementation, on x86_64 +test-lexer-generic *args='': + cargo test -p oxc_lexer {{_lexer-generic}} {{args}} + +# Run lexer conformance, built for the host conformance-lexer *args='': cargo run -p oxc_coverage --profile coverage --features lexer -- lexer {{args}} -# Run lexer conformance against the SIMD core -conformance-lexer-simd *args='': - cargo run -p oxc_coverage --profile coverage {{_lexer-simd}} --features lexer -- lexer {{args}} +# Run lexer conformance against the AVX2 implementation +conformance-lexer-avx2 *args='': + cargo run -p oxc_coverage --profile coverage {{_lexer-avx2}} --features lexer -- lexer {{args}} -# Lint `oxc_lexer` and the conformance harness against the scalar fallback +# Run lexer conformance against the generic implementation, on x86_64 +conformance-lexer-generic *args='': + cargo run -p oxc_coverage --profile coverage {{_lexer-generic}} --features lexer -- lexer {{args}} + +# Lint `oxc_lexer` and the conformance harness, built for the host [unix] lint-lexer *args='': CARGO_BUILD_WARNINGS=deny cargo clippy -p oxc_lexer -p oxc_coverage --all-targets --all-features {{args}} @@ -195,13 +214,17 @@ lint-lexer *args='': lint-lexer *args='': $Env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p oxc_lexer -p oxc_coverage --all-targets --all-features {{args}} -# Lint `oxc_lexer` and the conformance harness against the SIMD core -lint-lexer-simd *args='': - just lint-lexer {{_lexer-simd}} {{args}} +# Lint `oxc_lexer` and the conformance harness against the AVX2 implementation +lint-lexer-avx2 *args='': + just lint-lexer {{_lexer-avx2}} {{args}} + +# Lint `oxc_lexer` and the conformance harness against the generic implementation, on x86_64 +lint-lexer-generic *args='': + just lint-lexer {{_lexer-generic}} {{args}} # The lexer benchmarks measure wallclock time. Unlike the other benchmarks, they don't run on CodSpeed. # -# `oxc_parser`'s lexer is always built without the SIMD flags, so it's benchmarked in a separate build +# `oxc_parser`'s lexer is always built without the AVX2 flags, so it's benchmarked in a separate build # from `oxc_lexer`. Its results are saved as criterion baseline `lexer_old`, and then `oxc_lexer` # is compared against that baseline. # The "change" which criterion reports for `oxc_lexer` is its time relative to `oxc_parser`'s. @@ -213,37 +236,44 @@ lint-lexer-simd *args='': # `args` are passed to criterion, e.g. `just bench-lexer App.tsx` to run only that file. # The recipes set the baseline options themselves, so don't pass those. # -# On an ARM host, `bench-lexer-simd` compares `oxc_parser` running natively against `oxc_lexer` +# On an ARM host, `bench-lexer-avx2` compares `oxc_parser` running natively against `oxc_lexer` # running under emulation, so the comparison is meaningless. _bench-lexer := "cargo bench -p oxc_benchmark --no-default-features --features lexer_wallclock --bench" -# Benchmark `oxc_lexer`'s scalar fallback against `oxc_parser`'s lexer +# Benchmark `oxc_lexer`, built for the host, against `oxc_parser`'s lexer bench-lexer *args='': {{_bench-lexer}} lexer_wallclock_old --bench lexer_wallclock_new --no-run {{_bench-lexer}} lexer_wallclock_old -- --save-baseline lexer_old {{args}} {{_bench-lexer}} lexer_wallclock_new -- --baseline lexer_old {{args}} -# Benchmark `oxc_lexer`'s SIMD core against `oxc_parser`'s lexer -bench-lexer-simd *args='': +# Benchmark `oxc_lexer`'s AVX2 implementation against `oxc_parser`'s lexer +bench-lexer-avx2 *args='': {{_bench-lexer}} lexer_wallclock_old --no-run - {{_bench-lexer}} lexer_wallclock_new {{_lexer-simd}} --no-run + {{_bench-lexer}} lexer_wallclock_new {{_lexer-avx2}} --no-run {{_bench-lexer}} lexer_wallclock_old -- --save-baseline lexer_old {{args}} - {{_bench-lexer}} lexer_wallclock_new {{_lexer-simd}} -- --baseline lexer_old {{args}} + {{_bench-lexer}} lexer_wallclock_new {{_lexer-avx2}} -- --baseline lexer_old {{args}} # `ready-lexer` fails if any step fails, or if conformance changes the lexer snapshots. # Only the lexer snapshots are checked for changes, so it can be run with other uncommitted changes. -# Snapshots are checked after each conformance run, because the SIMD run overwrites the snapshots from the scalar run. +# Snapshots are checked after each conformance run, because each run overwrites the snapshots from the one before. +# +# The generic implementation on x86_64 is only run on hosts which aren't x86_64. +# On an x86_64 host, it's the same as the host build, which `ready-lexer` runs anyway. _lexer-snapshots := "tasks/coverage/snapshots/lexer_*" -# Lint, test, and run conformance for `oxc_lexer` against both implementations, and check snapshots are unchanged +# Lint, test, and run conformance for `oxc_lexer` against each implementation, and check snapshots are unchanged ready-lexer: just lint-lexer - just lint-lexer-simd + just lint-lexer-avx2 + {{ if arch() == "x86_64" { "" } else { "just lint-lexer-generic" } }} just test-lexer - just test-lexer-simd + just test-lexer-avx2 + {{ if arch() == "x86_64" { "" } else { "just test-lexer-generic" } }} just conformance-lexer git diff --exit-code HEAD -- '{{_lexer-snapshots}}' - just conformance-lexer-simd + just conformance-lexer-avx2 + git diff --exit-code HEAD -- '{{_lexer-snapshots}}' + {{ if arch() == "x86_64" { "" } else { "just conformance-lexer-generic" } }} git diff --exit-code HEAD -- '{{_lexer-snapshots}}' just doc -p oxc_lexer diff --git a/tasks/benchmark/README.md b/tasks/benchmark/README.md index f12460e597..531ecad5c0 100644 --- a/tasks/benchmark/README.md +++ b/tasks/benchmark/README.md @@ -19,12 +19,12 @@ This benchmark does not run on CodSpeed, because its CPU simulation does not mod execution and branch prediction which `oxc_lexer` is designed to exploit. ```sh -just bench-lexer # Scalar fallback -just bench-lexer-simd # SIMD core (x86_64 only - on ARM Mac, it runs under emulation) +just bench-lexer # Built for the host (Neon on ARM, generic on x86_64) +just bench-lexer-avx2 # AVX2 (x86_64 only - on ARM Mac, it runs under emulation) ``` -`oxc_parser` is always built without the SIMD flags, so the 2 lexers are benchmarked in separate builds. +`oxc_parser` is always built without the AVX2 flags, so the 2 lexers are benchmarked in separate builds. The "change" which criterion reports for `oxc_lexer` is its time relative to `oxc_parser`'s. The "change" reported for `oxc_parser` (if any) is relative to its previous run. -CI runs `just bench-lexer-simd` when `oxc_lexer` changes, and prints the results in the job log. +CI runs `just bench-lexer-avx2` when `oxc_lexer` changes, and prints the results in the job log. diff --git a/tasks/benchmark/benches/lexer_wallclock_new.rs b/tasks/benchmark/benches/lexer_wallclock_new.rs index ae604fc501..2155f93c84 100644 --- a/tasks/benchmark/benches/lexer_wallclock_new.rs +++ b/tasks/benchmark/benches/lexer_wallclock_new.rs @@ -3,12 +3,12 @@ //! Not run on CodSpeed. `oxc_lexer` is designed to exploit out-of-order execution and branch prediction, //! which CodSpeed's CPU simulation does not model. So these benchmarks measure wallclock time instead. //! -//! `oxc_lexer` compiles to its SIMD core or its scalar fallback depending on target features, +//! `oxc_lexer` compiles to its AVX2, Neon or generic implementation depending on target features, //! which apply to the whole build. So the 2 lexers are benchmarked in separate builds: //! -//! 1. `lexer_wallclock_old` benchmarks `oxc_parser`'s lexer, built without the SIMD target features, +//! 1. `lexer_wallclock_old` benchmarks `oxc_parser`'s lexer, built without the AVX2 target features, //! and saves the results as criterion baseline `lexer_old`. -//! 2. This benchmark benchmarks `oxc_lexer` (either implementation), and compares it against that baseline. +//! 2. This benchmark benchmarks `oxc_lexer` (any implementation), and compares it against that baseline. //! The "change" which criterion reports is `oxc_lexer`'s time relative to `oxc_parser`'s. //! //! When saving the baseline, criterion also compares `oxc_parser` against its previous run, if there is one. @@ -17,8 +17,8 @@ //! Both lexers lex the same cleaned source text, so the contest is fair. //! See `cleaned_test_files` for why the source text needs cleaning. //! -//! Run with `just bench-lexer` (scalar fallback) or `just bench-lexer-simd` (SIMD core). -//! CI runs `just bench-lexer-simd` when `oxc_lexer` changes, and prints the results in the job log. +//! Run with `just bench-lexer` (built for the host) or `just bench-lexer-avx2` (AVX2 implementation). +//! CI runs `just bench-lexer-avx2` when `oxc_lexer` changes, and prints the results in the job log. mod lexer_common; diff --git a/tasks/coverage/Cargo.toml b/tasks/coverage/Cargo.toml index b2b89cb161..cc961e3d3d 100644 --- a/tasks/coverage/Cargo.toml +++ b/tasks/coverage/Cargo.toml @@ -46,11 +46,12 @@ walkdir = { workspace = true } [features] # Differential lexer conformance, opt-in because it is slow and `oxc_lexer` is still incubating. -# The harness runs against whichever of the lexer's two implementations the build selects: +# The harness runs against whichever of the lexer's implementations the build selects: # -# * SIMD core (x86_64 only): RUSTFLAGS="-C target-feature=+avx2,+bmi2,+popcnt" cargo coverage --features lexer -- lexer -# * Scalar fallback: cargo coverage --features lexer -- lexer +# * AVX2 (x86_64 only): RUSTFLAGS="-C target-feature=+avx2,+bmi2,+popcnt" cargo coverage --features lexer -- lexer +# * Neon (aarch64) or generic (everywhere else): cargo coverage --features lexer -- lexer # # `avx2`/`bmi2`/`popcnt` are not in the x86_64 baseline on any platform, so the rustflags are required -# to reach the SIMD core even on an x86_64 host. +# to reach the AVX2 implementation even on an x86_64 host. +# The `just *-lexer*` recipes run conformance for each implementation - see the justfile. lexer = ["dep:oxc_lexer", "oxc_lexer/oxc_diagnostics"]