diff --git a/crates/oxc_parser/src/lexer/mod.rs b/crates/oxc_parser/src/lexer/mod.rs index 9503f46fad..a1124b6372 100644 --- a/crates/oxc_parser/src/lexer/mod.rs +++ b/crates/oxc_parser/src/lexer/mod.rs @@ -82,6 +82,8 @@ pub struct Lexer<'a, C: Config> { source_type: SourceType, /// The token currently being built. + /// Only its start and flags are written here; its kind and end remain zero until + /// `Token::finish` constructs the completed token. /// /// Between lexer calls, `token` is always `Token::default()`. Every method which produces a token /// ends in `finish_next_inner` or `finish_re_lex`, and both reset it. @@ -314,9 +316,7 @@ impl<'a, C: Config> Lexer<'a, C> { #[expect(clippy::inline_always)] #[inline(always)] fn finish_next_inner(&mut self, kind: Kind, mode: FinishTokenMode) -> Token { - self.token.set_kind(kind); - self.token.set_end(self.offset()); - let token = self.token; + let token = self.token.finish(kind, self.offset()); if self.config.tokens() { match mode { FinishTokenMode::Push => { @@ -342,9 +342,7 @@ impl<'a, C: Config> Lexer<'a, C> { /// Finish a re-lexed token used only for parser disambiguation. /// This must not mutate the externally collected token stream. fn finish_re_lex(&mut self, kind: Kind) -> Token { - self.token.set_kind(kind); - self.token.set_end(self.offset()); - let token = self.token; + let token = self.token.finish(kind, self.offset()); self.token = Token::default(); token } diff --git a/crates/oxc_parser/src/lexer/token.rs b/crates/oxc_parser/src/lexer/token.rs index 35455e4089..eb084d0ef2 100644 --- a/crates/oxc_parser/src/lexer/token.rs +++ b/crates/oxc_parser/src/lexer/token.rs @@ -82,6 +82,14 @@ impl fmt::Debug for Token { } impl Token { + /// Finish the lexer's scratch token without writing its fields back to memory first. + pub(super) fn finish(self, kind: Kind, end: u32) -> Self { + debug_assert_eq!(self.kind(), Kind::Eof); + debug_assert_eq!(self.end(), 0); + debug_assert!(end >= self.start()); + Self(self.0 | (u128::from(end) << END_SHIFT) | ((kind as u128) << KIND_SHIFT)) + } + #[inline] pub(super) fn new_on_new_line() -> Self { // Start with a default token, then set the flag