perf(parser): construct completed tokens without intermediate writes (#28291)

Build completed tokens directly from the scratch start and flags, avoiding intermediate kind/end writes.

AI-assisted implementation.
This commit is contained in:
Boshen committed 2026-10-29 08:56:48 +00:00
1 parent 0f3abb111a
commit 426c1510da
2 files changed
+12 -6

No files matched your search

+4 -6
View File
@@ -82,6 +82,8 @@ pub struct Lexer<'a, C: Config> {
source_type: SourceType,
/// The token currently being built.
/// Only its start and flags are written here; its kind and end remain zero until
/// `Token::finish` constructs the completed token.
///
/// Between lexer calls, `token` is always `Token::default()`. Every method which produces a token
/// ends in `finish_next_inner` or `finish_re_lex`, and both reset it.
@@ -314,9 +316,7 @@ impl<'a, C: Config> Lexer<'a, C> {
#[expect(clippy::inline_always)]
#[inline(always)]
fn finish_next_inner(&mut self, kind: Kind, mode: FinishTokenMode) -> Token {
self.token.set_kind(kind);
self.token.set_end(self.offset());
let token = self.token;
let token = self.token.finish(kind, self.offset());
if self.config.tokens() {
match mode {
FinishTokenMode::Push => {
@@ -342,9 +342,7 @@ impl<'a, C: Config> Lexer<'a, C> {
/// Finish a re-lexed token used only for parser disambiguation.
/// This must not mutate the externally collected token stream.
fn finish_re_lex(&mut self, kind: Kind) -> Token {
self.token.set_kind(kind);
self.token.set_end(self.offset());
let token = self.token;
let token = self.token.finish(kind, self.offset());
self.token = Token::default();
token
}
+8
View File
@@ -82,6 +82,14 @@ impl fmt::Debug for Token {
}
impl Token {
/// Finish the lexer's scratch token without writing its fields back to memory first.
pub(super) fn finish(self, kind: Kind, end: u32) -> Self {
debug_assert_eq!(self.kind(), Kind::Eof);
debug_assert_eq!(self.end(), 0);
debug_assert!(end >= self.start());
Self(self.0 | (u128::from(end) << END_SHIFT) | ((kind as u128) << KIND_SHIFT))
}
#[inline]
pub(super) fn new_on_new_line() -> Self {
// Start with a default token, then set the flag