mirror of
https://github.com/oxc-project/oxc.git
synced 2026-11-04 10:29:58 +00:00
perf(parser): construct completed tokens without intermediate writes (#28291)
Build completed tokens directly from the scratch start and flags, avoiding intermediate kind/end writes. AI-assisted implementation.
This commit is contained in:
2 files changed
+12
-6
No files matched your search
@@ -82,6 +82,8 @@ pub struct Lexer<'a, C: Config> {
|
||||
source_type: SourceType,
|
||||
|
||||
/// The token currently being built.
|
||||
/// Only its start and flags are written here; its kind and end remain zero until
|
||||
/// `Token::finish` constructs the completed token.
|
||||
///
|
||||
/// Between lexer calls, `token` is always `Token::default()`. Every method which produces a token
|
||||
/// ends in `finish_next_inner` or `finish_re_lex`, and both reset it.
|
||||
@@ -314,9 +316,7 @@ impl<'a, C: Config> Lexer<'a, C> {
|
||||
#[expect(clippy::inline_always)]
|
||||
#[inline(always)]
|
||||
fn finish_next_inner(&mut self, kind: Kind, mode: FinishTokenMode) -> Token {
|
||||
self.token.set_kind(kind);
|
||||
self.token.set_end(self.offset());
|
||||
let token = self.token;
|
||||
let token = self.token.finish(kind, self.offset());
|
||||
if self.config.tokens() {
|
||||
match mode {
|
||||
FinishTokenMode::Push => {
|
||||
@@ -342,9 +342,7 @@ impl<'a, C: Config> Lexer<'a, C> {
|
||||
/// Finish a re-lexed token used only for parser disambiguation.
|
||||
/// This must not mutate the externally collected token stream.
|
||||
fn finish_re_lex(&mut self, kind: Kind) -> Token {
|
||||
self.token.set_kind(kind);
|
||||
self.token.set_end(self.offset());
|
||||
let token = self.token;
|
||||
let token = self.token.finish(kind, self.offset());
|
||||
self.token = Token::default();
|
||||
token
|
||||
}
|
||||
|
||||
@@ -82,6 +82,14 @@ impl fmt::Debug for Token {
|
||||
}
|
||||
|
||||
impl Token {
|
||||
/// Finish the lexer's scratch token without writing its fields back to memory first.
|
||||
pub(super) fn finish(self, kind: Kind, end: u32) -> Self {
|
||||
debug_assert_eq!(self.kind(), Kind::Eof);
|
||||
debug_assert_eq!(self.end(), 0);
|
||||
debug_assert!(end >= self.start());
|
||||
Self(self.0 | (u128::from(end) << END_SHIFT) | ((kind as u128) << KIND_SHIFT))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn new_on_new_line() -> Self {
|
||||
// Start with a default token, then set the flag
|
||||
|
||||
Reference in new issue
Block a user