test(lexer): update just recipes to cover Neon implementation (#27705)

#27642 added an aarch64 Neon implementation of `classify`. So now we have 3 implementations.

Update `just ready-lexer` recipe to test and lint all 3.

.cargo/lexer-avx2.toml

@@ -1,5 +1,5 @@
-# Extra rustflags for building `oxc_lexer`'s SIMD core. Passed via `cargo --config` by the
+# Extra rustflags for building `oxc_lexer`'s AVX2 implementation. Passed via `cargo --config` by the
-# `just test-lexer-simd` / `just conformance-lexer-simd` recipes - see the justfile.
+# `just *-lexer-avx2` recipes - see the justfile.
 #
 # `avx2`, `bmi2` and `popcnt` are not in the x86_64 baseline on any platform, so they have to be asked for explicitly,
 # even on an x86_64 host. Cargo joins `rustflags` arrays across config sources, so the per-target flags

.github/workflows/ci.yml

@@ -678,7 +678,7 @@ jobs:
           - variant: generic
             suffix: ""
           - variant: AVX2
-            suffix: "-simd"
+            suffix: "-avx2"
     steps:
       - uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2
 
@@ -687,7 +687,7 @@ jobs:
         uses: $/.github/actions/check-changes
         with:
           packages: oxc_lexer,oxc_coverage
-          paths: crates/oxc_lexer/,tasks/coverage/,.cargo/lexer-simd.toml,justfile,.github/workflows/ci.yml
+          paths: crates/oxc_lexer/,tasks/coverage/,.cargo/lexer-avx2.toml,justfile,.github/workflows/ci.yml
           token: ${{ secrets.GITHUB_TOKEN }}
           pr-number: ${{ github.event.pull_request.number }}
 
@@ -716,12 +716,12 @@ jobs:
       - name: Lint
         # No need to run on generic - already covered by "Lint" job
         if: ${{ steps.filter.outputs.changed == 'true' && matrix.variant == 'AVX2' }}
-        run: just lint-lexer-simd
+        run: just lint-lexer-avx2
 
       - name: Test
         # No need to run on generic - already covered by "Test Linux" job
         if: ${{ steps.filter.outputs.changed == 'true' && matrix.variant == 'AVX2' }}
-        run: just test-lexer-simd
+        run: just test-lexer-avx2
 
       - wait: lexer-fixtures
 
@@ -756,7 +756,7 @@ jobs:
         id: filter
         uses: $/.github/actions/check-changes
         with:
-          paths: crates/oxc_lexer/,tasks/benchmark/benches/lexer_wallclock_old.rs,tasks/benchmark/benches/lexer_wallclock_new.rs,tasks/benchmark/benches/lexer_common.rs,.cargo/lexer-simd.toml,justfile,.github/workflows/ci.yml
+          paths: crates/oxc_lexer/,tasks/benchmark/benches/lexer_wallclock_old.rs,tasks/benchmark/benches/lexer_wallclock_new.rs,tasks/benchmark/benches/lexer_common.rs,.cargo/lexer-avx2.toml,justfile,.github/workflows/ci.yml
           token: ${{ secrets.GITHUB_TOKEN }}
           pr-number: ${{ github.event.pull_request.number }}
 
@@ -782,7 +782,7 @@ jobs:
           CRITERION_HOME: ${{ runner.temp }}/criterion
         run: |
           set -o pipefail
-          just bench-lexer-simd | tee "$RUNNER_TEMP/bench.log"
+          just bench-lexer-avx2 | tee "$RUNNER_TEMP/bench.log"
           grep -qxF "Lexer implementation: AVX2" "$RUNNER_TEMP/bench.log" ||
             { echo "::error::Expected AVX2, but the benchmark reported otherwise"; exit 1; }
 

Cargo.toml

@@ -170,7 +170,7 @@ oxc_formatter_tests = { path = "crates/oxc_formatter_tests" } # Formatter test i
 oxc_formatter_toml = { path = "crates/oxc_formatter_toml" } # TOML formatter
 oxc_formatter_yaml = { path = "crates/oxc_formatter_yaml" } # YAML formatter
 oxc_language_server = { path = "crates/oxc_language_server", default-features = false } # Language server
-oxc_lexer = { path = "crates/oxc_lexer" } # JS/TS lexer (incubating; x86_64 SIMD core, scalar fallback)
+oxc_lexer = { path = "crates/oxc_lexer" } # JS/TS lexer (incubating; AVX2, Neon, and generic implementations)
 oxc_linter = { path = "crates/oxc_linter" } # Linting engine
 oxc_macros = { path = "crates/oxc_macros" } # Proc macros
 oxc_tasks_common = { path = "tasks/common" } # Task utilities

justfile

@@ -149,44 +149,63 @@ ast:
 
 # ==================== LEXER ====================
 
-# `oxc_lexer` compiles to one of two implementations, chosen at build time:
+# `oxc_lexer` compiles to one of 3 implementations, chosen at build time:
 #
-# * SIMD core, on x86_64 with `avx2` + `bmi2` + `popcnt` enabled
+# * AVX2, on x86_64 with `avx2` + `bmi2` + `popcnt` enabled
-# * Scalar fallback, everywhere else
+# * Neon, on aarch64 (every aarch64 target with `std` enables `neon`)
+# * Generic, everywhere else
 #
-# Both need testing, and they should produce identical results. None of `avx2`, `bmi2` and `popcnt` is in the x86_64
+# Parts of the pipeline which have no AVX2 or Neon implementation use the generic one.
-# baseline on any platform, so reaching the SIMD core always means asking for them explicitly - even on an x86_64 host.
+# All 3 need testing, and they should produce identical results.
-# The flags live in `.cargo/lexer-simd.toml`, passed with `cargo --config`, which avoids shell quoting entirely
+#
+# Each task has 3 recipes:
+#
+# * `*-lexer` builds for the host - Neon on an ARM host, generic on an x86_64 host.
+# * `*-lexer-avx2` builds the AVX2 implementation for x86_64.
+# * `*-lexer-generic` builds the generic implementation for x86_64.
+#   On an x86_64 host, that's the same as `*-lexer`.
+#
+# None of `avx2`, `bmi2` and `popcnt` is in the x86_64 baseline on any platform, so reaching the AVX2 implementation
+# always means asking for them explicitly - even on an x86_64 host.
+# The flags live in `.cargo/lexer-avx2.toml`, passed with `cargo --config`, which avoids shell quoting entirely
 # (this justfile runs PowerShell on Windows).
 #
-# `--target` is always passed, so that the flags land on the triple the config file names,
+# `--target` is always passed for the x86_64 builds, so that the flags land on the triple the config file names.
-# and so that the two builds get separate target dirs instead of invalidating each other.
 #
-# On an ARM host the SIMD build is cross-compiled and run under emulation - macOS provides that via Rosetta 2.
+# On an ARM host the x86_64 builds are cross-compiled and run under emulation - macOS provides that via Rosetta 2.
 # On Mac, ensure x86_64 target is installed: `rustup target add x86_64-apple-darwin`.
 # If it's not, you'll get a "can't find crate for std" error.
 #
-# On ARM Linux or ARM Windows it will fail, loudly, when linking or running.
+# On ARM Linux or ARM Windows they will fail, loudly, when linking or running.
-_lexer-simd-target := if os() == "macos" { "x86_64-apple-darwin" } else if os() == "windows" { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" }
+_lexer-x86-target := if os() == "macos" { "x86_64-apple-darwin" } else if os() == "windows" { "x86_64-pc-windows-msvc" } else { "x86_64-unknown-linux-gnu" }
-_lexer-simd := "--target " + _lexer-simd-target + " --config .cargo/lexer-simd.toml"
+_lexer-generic := "--target " + _lexer-x86-target
+_lexer-avx2 := _lexer-generic + " --config .cargo/lexer-avx2.toml"
 
-# Run `oxc_lexer`'s tests against the scalar fallback
+# Run `oxc_lexer`'s tests, built for the host
 test-lexer *args='':
   cargo test -p oxc_lexer {{args}}
 
-# Run `oxc_lexer`'s tests against the SIMD core
+# Run `oxc_lexer`'s tests against the AVX2 implementation
-test-lexer-simd *args='':
+test-lexer-avx2 *args='':
-  cargo test -p oxc_lexer {{_lexer-simd}} {{args}}
+  cargo test -p oxc_lexer {{_lexer-avx2}} {{args}}
 
-# Run lexer conformance against the scalar fallback
+# Run `oxc_lexer`'s tests against the generic implementation, on x86_64
+test-lexer-generic *args='':
+  cargo test -p oxc_lexer {{_lexer-generic}} {{args}}
+
+# Run lexer conformance, built for the host
 conformance-lexer *args='':
   cargo run -p oxc_coverage --profile coverage --features lexer -- lexer {{args}}
 
-# Run lexer conformance against the SIMD core
+# Run lexer conformance against the AVX2 implementation
-conformance-lexer-simd *args='':
+conformance-lexer-avx2 *args='':
-  cargo run -p oxc_coverage --profile coverage {{_lexer-simd}} --features lexer -- lexer {{args}}
+  cargo run -p oxc_coverage --profile coverage {{_lexer-avx2}} --features lexer -- lexer {{args}}
 
-# Lint `oxc_lexer` and the conformance harness against the scalar fallback
+# Run lexer conformance against the generic implementation, on x86_64
+conformance-lexer-generic *args='':
+  cargo run -p oxc_coverage --profile coverage {{_lexer-generic}} --features lexer -- lexer {{args}}
+
+# Lint `oxc_lexer` and the conformance harness, built for the host
 [unix]
 lint-lexer *args='':
   CARGO_BUILD_WARNINGS=deny cargo clippy -p oxc_lexer -p oxc_coverage --all-targets --all-features {{args}}
@@ -195,13 +214,17 @@ lint-lexer *args='':
 lint-lexer *args='':
   $Env:CARGO_BUILD_WARNINGS='deny'; cargo clippy -p oxc_lexer -p oxc_coverage --all-targets --all-features {{args}}
 
-# Lint `oxc_lexer` and the conformance harness against the SIMD core
+# Lint `oxc_lexer` and the conformance harness against the AVX2 implementation
-lint-lexer-simd *args='':
+lint-lexer-avx2 *args='':
-  just lint-lexer {{_lexer-simd}} {{args}}
+  just lint-lexer {{_lexer-avx2}} {{args}}
+
+# Lint `oxc_lexer` and the conformance harness against the generic implementation, on x86_64
+lint-lexer-generic *args='':
+  just lint-lexer {{_lexer-generic}} {{args}}
 
 # The lexer benchmarks measure wallclock time. Unlike the other benchmarks, they don't run on CodSpeed.
 #
-# `oxc_parser`'s lexer is always built without the SIMD flags, so it's benchmarked in a separate build
+# `oxc_parser`'s lexer is always built without the AVX2 flags, so it's benchmarked in a separate build
 # from `oxc_lexer`. Its results are saved as criterion baseline `lexer_old`, and then `oxc_lexer`
 # is compared against that baseline.
 # The "change" which criterion reports for `oxc_lexer` is its time relative to `oxc_parser`'s.
@@ -213,37 +236,44 @@ lint-lexer-simd *args='':
 # `args` are passed to criterion, e.g. `just bench-lexer App.tsx` to run only that file.
 # The recipes set the baseline options themselves, so don't pass those.
 #
-# On an ARM host, `bench-lexer-simd` compares `oxc_parser` running natively against `oxc_lexer`
+# On an ARM host, `bench-lexer-avx2` compares `oxc_parser` running natively against `oxc_lexer`
 # running under emulation, so the comparison is meaningless.
 _bench-lexer := "cargo bench -p oxc_benchmark --no-default-features --features lexer_wallclock --bench"
 
-# Benchmark `oxc_lexer`'s scalar fallback against `oxc_parser`'s lexer
+# Benchmark `oxc_lexer`, built for the host, against `oxc_parser`'s lexer
 bench-lexer *args='':
   {{_bench-lexer}} lexer_wallclock_old --bench lexer_wallclock_new --no-run
   {{_bench-lexer}} lexer_wallclock_old -- --save-baseline lexer_old {{args}}
   {{_bench-lexer}} lexer_wallclock_new -- --baseline lexer_old {{args}}
 
-# Benchmark `oxc_lexer`'s SIMD core against `oxc_parser`'s lexer
+# Benchmark `oxc_lexer`'s AVX2 implementation against `oxc_parser`'s lexer
-bench-lexer-simd *args='':
+bench-lexer-avx2 *args='':
   {{_bench-lexer}} lexer_wallclock_old --no-run
-  {{_bench-lexer}} lexer_wallclock_new {{_lexer-simd}} --no-run
+  {{_bench-lexer}} lexer_wallclock_new {{_lexer-avx2}} --no-run
   {{_bench-lexer}} lexer_wallclock_old -- --save-baseline lexer_old {{args}}
-  {{_bench-lexer}} lexer_wallclock_new {{_lexer-simd}} -- --baseline lexer_old {{args}}
+  {{_bench-lexer}} lexer_wallclock_new {{_lexer-avx2}} -- --baseline lexer_old {{args}}
 
 # `ready-lexer` fails if any step fails, or if conformance changes the lexer snapshots.
 # Only the lexer snapshots are checked for changes, so it can be run with other uncommitted changes.
-# Snapshots are checked after each conformance run, because the SIMD run overwrites the snapshots from the scalar run.
+# Snapshots are checked after each conformance run, because each run overwrites the snapshots from the one before.
+#
+# The generic implementation on x86_64 is only run on hosts which aren't x86_64.
+# On an x86_64 host, it's the same as the host build, which `ready-lexer` runs anyway.
 _lexer-snapshots := "tasks/coverage/snapshots/lexer_*"
 
-# Lint, test, and run conformance for `oxc_lexer` against both implementations, and check snapshots are unchanged
+# Lint, test, and run conformance for `oxc_lexer` against each implementation, and check snapshots are unchanged
 ready-lexer:
   just lint-lexer
-  just lint-lexer-simd
+  just lint-lexer-avx2
+  {{ if arch() == "x86_64" { "" } else { "just lint-lexer-generic" } }}
   just test-lexer
-  just test-lexer-simd
+  just test-lexer-avx2
+  {{ if arch() == "x86_64" { "" } else { "just test-lexer-generic" } }}
   just conformance-lexer
   git diff --exit-code HEAD -- '{{_lexer-snapshots}}'
-  just conformance-lexer-simd
+  just conformance-lexer-avx2
+  git diff --exit-code HEAD -- '{{_lexer-snapshots}}'
+  {{ if arch() == "x86_64" { "" } else { "just conformance-lexer-generic" } }}
   git diff --exit-code HEAD -- '{{_lexer-snapshots}}'
   just doc -p oxc_lexer
 

tasks/benchmark/README.md

@@ -19,12 +19,12 @@ This benchmark does not run on CodSpeed, because its CPU simulation does not mod
 execution and branch prediction which `oxc_lexer` is designed to exploit.
 
 ```sh
-just bench-lexer      # Scalar fallback
+just bench-lexer      # Built for the host (Neon on ARM, generic on x86_64)
-just bench-lexer-simd # SIMD core (x86_64 only - on ARM Mac, it runs under emulation)
+just bench-lexer-avx2 # AVX2 (x86_64 only - on ARM Mac, it runs under emulation)
 ```
 
-`oxc_parser` is always built without the SIMD flags, so the 2 lexers are benchmarked in separate builds.
+`oxc_parser` is always built without the AVX2 flags, so the 2 lexers are benchmarked in separate builds.
 The "change" which criterion reports for `oxc_lexer` is its time relative to `oxc_parser`'s.
 The "change" reported for `oxc_parser` (if any) is relative to its previous run.
 
-CI runs `just bench-lexer-simd` when `oxc_lexer` changes, and prints the results in the job log.
+CI runs `just bench-lexer-avx2` when `oxc_lexer` changes, and prints the results in the job log.

tasks/benchmark/benches/lexer_wallclock_new.rs

@@ -3,12 +3,12 @@
 //! Not run on CodSpeed. `oxc_lexer` is designed to exploit out-of-order execution and branch prediction,
 //! which CodSpeed's CPU simulation does not model. So these benchmarks measure wallclock time instead.
 //!
-//! `oxc_lexer` compiles to its SIMD core or its scalar fallback depending on target features,
+//! `oxc_lexer` compiles to its AVX2, Neon or generic implementation depending on target features,
 //! which apply to the whole build. So the 2 lexers are benchmarked in separate builds:
 //!
-//! 1. `lexer_wallclock_old` benchmarks `oxc_parser`'s lexer, built without the SIMD target features,
+//! 1. `lexer_wallclock_old` benchmarks `oxc_parser`'s lexer, built without the AVX2 target features,
 //!    and saves the results as criterion baseline `lexer_old`.
-//! 2. This benchmark benchmarks `oxc_lexer` (either implementation), and compares it against that baseline.
+//! 2. This benchmark benchmarks `oxc_lexer` (any implementation), and compares it against that baseline.
 //!    The "change" which criterion reports is `oxc_lexer`'s time relative to `oxc_parser`'s.
 //!
 //! When saving the baseline, criterion also compares `oxc_parser` against its previous run, if there is one.
@@ -17,8 +17,8 @@
 //! Both lexers lex the same cleaned source text, so the contest is fair.
 //! See `cleaned_test_files` for why the source text needs cleaning.
 //!
-//! Run with `just bench-lexer` (scalar fallback) or `just bench-lexer-simd` (SIMD core).
+//! Run with `just bench-lexer` (built for the host) or `just bench-lexer-avx2` (AVX2 implementation).
-//! CI runs `just bench-lexer-simd` when `oxc_lexer` changes, and prints the results in the job log.
+//! CI runs `just bench-lexer-avx2` when `oxc_lexer` changes, and prints the results in the job log.
 
 mod lexer_common;
 

tasks/coverage/Cargo.toml

@@ -46,11 +46,12 @@ walkdir = { workspace = true }
 
 [features]
 # Differential lexer conformance, opt-in because it is slow and `oxc_lexer` is still incubating.
-# The harness runs against whichever of the lexer's two implementations the build selects:
+# The harness runs against whichever of the lexer's implementations the build selects:
 #
-# * SIMD core (x86_64 only): RUSTFLAGS="-C target-feature=+avx2,+bmi2,+popcnt" cargo coverage --features lexer -- lexer
+# * AVX2 (x86_64 only): RUSTFLAGS="-C target-feature=+avx2,+bmi2,+popcnt" cargo coverage --features lexer -- lexer
-# * Scalar fallback:         cargo coverage --features lexer -- lexer
+# * Neon (aarch64) or generic (everywhere else): cargo coverage --features lexer -- lexer
 #
 # `avx2`/`bmi2`/`popcnt` are not in the x86_64 baseline on any platform, so the rustflags are required
-# to reach the SIMD core even on an x86_64 host.
+# to reach the AVX2 implementation even on an x86_64 host.
+# The `just *-lexer*` recipes run conformance for each implementation - see the justfile.
 lexer = ["dep:oxc_lexer", "oxc_lexer/oxc_diagnostics"]