mirror of
https://github.com/oxc-project/oxc.git
synced 2026-11-04 10:29:58 +00:00
perf(parser): use peek_token instead of checkpoint/rewind for single-token decisions (#23056)
## What
Two related changes:
1. **perf(parser): use `peek_token` instead of checkpoint/rewind for single-token decisions** — replace the `checkpoint` + `bump` + `rewind` pattern with a single cached lexer-level `peek_token()` in four hot statement-start paths where the parse decision depends only on the one token following the keyword:
- `parse_let` — token after `let`
- `parse_import_statement` — `import.meta` / `import()` vs import declaration
- `parse_async_statement` — `async function` vs other
- `for (let …` — token after `let`
2. **fix(parser): dedup irregular whitespaces recorded during lookahead** — prerequisite for the above (see below).
## Why
In each path the keyword was speculatively consumed only to inspect the next token, then rewound on the cold path. By peeking first and only `bump`-ing once we commit, the keyword is never speculatively consumed, so no rewind is needed. `peek_token()` is a cached lexer-level re-lex and is far cheaper than a parser-level `checkpoint`, which snapshots lexer state, the current token, the error position and the fatal-error slot.
This continues the existing migration of single-token lookaheads from `lookahead`/`checkpoint` to `peek_token` (see the note in `modifiers.rs`).
## The dedup fix
`peek_token`/`lookahead` lex past the current position and record any irregular whitespace they scan into `trivia_builder.irregular_whitespaces`; after rewinding, the committed path re-lexes the same span and records it again, producing duplicate `no-irregular-whitespace` diagnostics (e.g. `let<NBSP>x = 1`).
`add_comment` already handles this exact rewind-duplicate case for comments by skipping re-inserts on an ordered vec (`start <= last.start`). The same guard is now applied to `add_irregular_whitespace`. This also fixes the latent duplicate for the pre-existing `lookahead`-based paths, and correctly keeps genuinely-distinct adjacent whitespaces (`let<NBSP><NBSP>x` still reports two).
## Conformance
No AST change. Verified by panic-isolated, per-suite comparison against `main`:
- estree (full AST + spans + token streams): byte-identical
- transformer: byte-identical
Irregular-whitespace counts verified directly: `let<NBSP>x = 1`, `for (let<NBSP>x of y){}`, `async<NBSP>function f(){}` each report exactly one span (was two); `let<NBSP><NBSP>x` reports two.
This commit is contained in:
3 files changed
+17
-18
No files matched your search
@@ -9,17 +9,14 @@ impl<'a, C: Config> ParserImpl<'a, C> {
|
||||
pub(crate) fn parse_let(&mut self, stmt_ctx: StatementContext) -> Statement<'a> {
|
||||
let span = self.start_span();
|
||||
|
||||
let checkpoint = self.checkpoint();
|
||||
self.bump_any(); // bump `let`
|
||||
let token = self.cur_token();
|
||||
let peeked = token.kind();
|
||||
let peeked = self.lexer.peek_token().kind();
|
||||
|
||||
// Fast path: avoid rewind.
|
||||
if !stmt_ctx.is_single_statement() && peeked.is_after_let() {
|
||||
self.bump_any(); // bump `let`
|
||||
return self.parse_variable_statement(span, VariableDeclarationKind::Let, stmt_ctx);
|
||||
}
|
||||
|
||||
self.rewind(checkpoint);
|
||||
// let = foo, let instanceof x, let + 1
|
||||
if peeked.is_assignment_operator() || peeked.is_binary_operator() {
|
||||
let expr = self.parse_assignment_expression_or_higher();
|
||||
|
||||
@@ -391,10 +391,9 @@ impl<'a, C: Config> ParserImpl<'a, C> {
|
||||
Kind::Let => {
|
||||
// `for (let`
|
||||
let start_span = self.start_span();
|
||||
let checkpoint = self.checkpoint();
|
||||
self.bump_any(); // bump `let`
|
||||
// disallow `for (let in ...`
|
||||
if self.cur_kind().is_after_let() {
|
||||
if self.lexer.peek_token().kind().is_after_let() {
|
||||
self.bump_any(); // bump `let`
|
||||
return self.parse_variable_declaration_for_statement(
|
||||
span,
|
||||
start_span,
|
||||
@@ -404,7 +403,6 @@ impl<'a, C: Config> ParserImpl<'a, C> {
|
||||
);
|
||||
}
|
||||
// Should be a relatively cold branch, since most tokens after `let` will be allowed in most files
|
||||
self.rewind(checkpoint);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
@@ -838,27 +836,23 @@ impl<'a, C: Config> ParserImpl<'a, C> {
|
||||
|
||||
/// Parse import statement or import expression.
|
||||
fn parse_import_statement(&mut self) -> Statement<'a> {
|
||||
let checkpoint = self.checkpoint();
|
||||
let span = self.start_span();
|
||||
self.bump_any();
|
||||
if matches!(self.cur_kind(), Kind::Dot | Kind::LParen) {
|
||||
// Parse the whole expression `import.meta.url` so a rewind is required.
|
||||
self.rewind(checkpoint);
|
||||
if matches!(self.lexer.peek_token().kind(), Kind::Dot | Kind::LParen) {
|
||||
// Parse the whole expression `import.meta.url`.
|
||||
self.parse_expression_or_labeled_statement()
|
||||
} else {
|
||||
self.bump_any(); // bump `import`
|
||||
self.parse_import_declaration(span, self.ctx.has_top_level())
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse statements that start with `sync`.
|
||||
fn parse_async_statement(&mut self, span: u32, stmt_ctx: StatementContext) -> Statement<'a> {
|
||||
let checkpoint = self.checkpoint();
|
||||
self.bump_any(); // bump `async`
|
||||
let token = self.cur_token();
|
||||
let token = self.lexer.peek_token();
|
||||
if token.kind() == Kind::Function && !token.is_on_new_line() {
|
||||
self.bump_any(); // bump `async`
|
||||
return self.parse_function_declaration(span, /* async */ true, stmt_ctx);
|
||||
}
|
||||
self.rewind(checkpoint);
|
||||
if self.is_ts && self.at_start_of_ts_declaration() {
|
||||
return self.parse_ts_declaration_statement(span, stmt_ctx);
|
||||
}
|
||||
|
||||
@@ -72,6 +72,14 @@ impl<'a> TriviaBuilder<'a> {
|
||||
}
|
||||
|
||||
pub fn add_irregular_whitespace(&mut self, start: u32, end: u32) {
|
||||
// The irregular whitespaces array is ordered; only add if not added before, to avoid
|
||||
// duplicates when the parser looks ahead (e.g. `peek_token`) and rewinds, then re-lexes the
|
||||
// same whitespace. Same approach as `add_comment`.
|
||||
if let Some(last) = self.irregular_whitespaces.last()
|
||||
&& start <= last.start
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.irregular_whitespaces.push(Span::new(start, end));
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user