feat(parser,estree,coverage): collect tokens in parser and convert to ESTree format (#19497)

Implement a `collect_tokens` option for parser. When enabled, the parser stores all tokens produced during parsing in a `Vec`. Implement serializing the tokens to ESTree format.

Currently this is not hooked up to NAPI parser or linter. That happens in a later PR in this stack (#19498).

Serialization of tokens implementation is in new `oxc_estree_tokens` crate. Unfortunately it can't go in existing `oxc_estree` crate because `oxc_ast` depends on `oxc_estree`, and the tokens serialization code requires AST types - which would be a circular dependency. We may be able to move it into another crate (parser?) later on.

Cargo.lock

@@ -1848,6 +1848,7 @@ dependencies = [
  "itertools",
  "lazy-regex",
  "oxc",
+ "oxc_estree_tokens",
  "oxc_formatter",
  "oxc_tasks_common",
  "oxc_tasks_transform_checker",
@@ -1901,6 +1902,20 @@ dependencies = [
  "oxc_data_structures",
 ]
 
+[[package]]
+name = "oxc_estree_tokens"
+version = "0.115.0"
+dependencies = [
+ "oxc_allocator",
+ "oxc_ast",
+ "oxc_ast_visit",
+ "oxc_parser",
+ "oxc_span",
+ "rustc-hash",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "oxc_formatter"
 version = "0.35.0"

Cargo.toml

@@ -117,6 +117,7 @@ oxc_data_structures = { version = "0.115.0", path = "crates/oxc_data_structures"
 oxc_diagnostics = { version = "0.115.0", path = "crates/oxc_diagnostics" } # Error reporting
 oxc_ecmascript = { version = "0.115.0", path = "crates/oxc_ecmascript" } # ECMAScript operations
 oxc_estree = { version = "0.115.0", path = "crates/oxc_estree" } # ESTree format
+oxc_estree_tokens = { version = "0.115.0", path = "crates/oxc_estree_tokens" } # ESTree token conversion
 oxc_isolated_declarations = { version = "0.115.0", path = "crates/oxc_isolated_declarations" } # TS declaration generation
 oxc_mangler = { version = "0.115.0", path = "crates/oxc_mangler" } # Name mangling
 oxc_minifier = { version = "0.115.0", path = "crates/oxc_minifier" } # Code minification

crates/oxc_estree_tokens/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "oxc_estree_tokens"
+version = "0.115.0"
+authors.workspace = true
+categories.workspace = true
+edition.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+license.workspace = true
+readme = "../../README.md"
+publish = true
+repository.workspace = true
+rust-version.workspace = true
+description.workspace = true
+
+[lints]
+workspace = true
+
+[lib]
+doctest = false
+
+[dependencies]
+oxc_allocator = { workspace = true }
+oxc_ast = { workspace = true }
+oxc_ast_visit = { workspace = true, features = ["serialize"] }
+oxc_parser = { workspace = true }
+oxc_span = { workspace = true }
+
+rustc-hash = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true }

crates/oxc_estree_tokens/src/lib.rs

@@ -0,0 +1,421 @@
+use rustc_hash::FxHashSet;
+use serde::Serialize;
+
+use oxc_allocator::{Allocator, Vec as ArenaVec};
+use oxc_ast::ast::{
+    BindingIdentifier, IdentifierName, IdentifierReference, JSXAttribute, JSXAttributeValue,
+    JSXElementName, JSXExpressionContainer, JSXIdentifier, JSXMemberExpressionObject,
+    JSXNamespacedName, JSXSpreadAttribute, JSXSpreadChild, LabelIdentifier, MemberExpression,
+    MetaProperty, ObjectPropertyKind, Program, PropertyKey, TSImportType, TSThisParameter,
+    TSTypeName, TSTypeParameterDeclaration, TSTypeQuery, TSTypeQueryExprName, WithClause,
+    WithClauseKeyword,
+};
+use oxc_ast_visit::{Visit, utf8_to_utf16::Utf8ToUtf16, walk};
+use oxc_parser::{Kind, Token};
+use oxc_span::Span;
+
+#[derive(Serialize)]
+pub struct EstreeRegExpToken<'a> {
+    pub flags: &'a str,
+    pub pattern: &'a str,
+}
+
+#[derive(Serialize)]
+pub struct EstreeToken<'a> {
+    #[serde(rename = "type")]
+    pub token_type: &'static str,
+    pub value: &'a str,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub regex: Option<EstreeRegExpToken<'a>>,
+    pub start: u32,
+    pub end: u32,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct EstreeTokenOptions {
+    pub exclude_legacy_keyword_identifiers: bool,
+    pub decode_identifier_escapes: bool,
+}
+
+impl EstreeTokenOptions {
+    pub const fn test262() -> Self {
+        Self { exclude_legacy_keyword_identifiers: true, decode_identifier_escapes: true }
+    }
+
+    pub const fn typescript() -> Self {
+        Self { exclude_legacy_keyword_identifiers: false, decode_identifier_escapes: false }
+    }
+
+    pub const fn linter() -> Self {
+        Self { exclude_legacy_keyword_identifiers: true, decode_identifier_escapes: false }
+    }
+}
+
+#[derive(Default)]
+pub struct EstreeTokenContext {
+    ast_identifier_spans: FxHashSet<Span>,
+    force_keyword_spans: FxHashSet<Span>,
+    jsx_identifier_spans: FxHashSet<Span>,
+    non_jsx_identifier_spans: FxHashSet<Span>,
+    jsx_text_spans: FxHashSet<Span>,
+    ts_type_parameter_starts: FxHashSet<u32>,
+    jsx_expression_depth: usize,
+    jsx_member_expression_depth: usize,
+    jsx_computed_member_depth: usize,
+}
+
+impl<'a> Visit<'a> for EstreeTokenContext {
+    fn visit_ts_type_query(&mut self, type_query: &TSTypeQuery<'a>) {
+        fn collect_type_query_this(spans: &mut FxHashSet<Span>, type_name: &TSTypeName<'_>) {
+            match type_name {
+                TSTypeName::ThisExpression(this_expression) => {
+                    spans.insert(this_expression.span);
+                }
+                TSTypeName::QualifiedName(qualified_name) => {
+                    collect_type_query_this(spans, &qualified_name.left);
+                }
+                TSTypeName::IdentifierReference(_) => {}
+            }
+        }
+
+        match &type_query.expr_name {
+            TSTypeQueryExprName::ThisExpression(this_expression) => {
+                self.ast_identifier_spans.insert(this_expression.span);
+            }
+            TSTypeQueryExprName::QualifiedName(qualified_name) => {
+                collect_type_query_this(&mut self.ast_identifier_spans, &qualified_name.left);
+            }
+            TSTypeQueryExprName::IdentifierReference(_) | TSTypeQueryExprName::TSImportType(_) => {}
+        }
+
+        walk::walk_ts_type_query(self, type_query);
+    }
+
+    fn visit_ts_import_type(&mut self, import_type: &TSImportType<'a>) {
+        if let Some(options) = &import_type.options {
+            for property in &options.properties {
+                if let ObjectPropertyKind::ObjectProperty(property) = property
+                    && let PropertyKey::StaticIdentifier(identifier) = &property.key
+                    && identifier.name == "with"
+                {
+                    self.force_keyword_spans.insert(identifier.span);
+                }
+            }
+        }
+        walk::walk_ts_import_type(self, import_type);
+    }
+
+    fn visit_ts_type_parameter_declaration(
+        &mut self,
+        declaration: &TSTypeParameterDeclaration<'a>,
+    ) {
+        self.ts_type_parameter_starts.insert(declaration.span.start);
+        walk::walk_ts_type_parameter_declaration(self, declaration);
+    }
+
+    fn visit_identifier_name(&mut self, identifier: &IdentifierName<'a>) {
+        self.ast_identifier_spans.insert(identifier.span);
+        if self.jsx_expression_depth > 0
+            && self.jsx_member_expression_depth > 0
+            && self.jsx_computed_member_depth == 0
+        {
+            self.jsx_identifier_spans.insert(identifier.span);
+        }
+    }
+
+    fn visit_identifier_reference(&mut self, identifier: &IdentifierReference<'a>) {
+        self.ast_identifier_spans.insert(identifier.span);
+        if self.jsx_expression_depth > 0
+            && self.jsx_member_expression_depth > 0
+            && self.jsx_computed_member_depth == 0
+        {
+            self.jsx_identifier_spans.insert(identifier.span);
+        }
+    }
+
+    fn visit_binding_identifier(&mut self, identifier: &BindingIdentifier<'a>) {
+        self.ast_identifier_spans.insert(identifier.span);
+    }
+
+    fn visit_label_identifier(&mut self, identifier: &LabelIdentifier<'a>) {
+        self.ast_identifier_spans.insert(identifier.span);
+    }
+
+    fn visit_ts_this_parameter(&mut self, parameter: &TSThisParameter<'a>) {
+        self.ast_identifier_spans.insert(parameter.this_span);
+        walk::walk_ts_this_parameter(self, parameter);
+    }
+
+    fn visit_meta_property(&mut self, meta_property: &MetaProperty<'a>) {
+        self.force_keyword_spans.insert(meta_property.meta.span);
+        walk::walk_meta_property(self, meta_property);
+    }
+
+    fn visit_with_clause(&mut self, with_clause: &WithClause<'a>) {
+        if matches!(with_clause.keyword, WithClauseKeyword::With) {
+            let span = Span::new(with_clause.span.start, with_clause.span.start + 4);
+            self.force_keyword_spans.insert(span);
+        }
+        walk::walk_with_clause(self, with_clause);
+    }
+
+    fn visit_jsx_identifier(&mut self, identifier: &JSXIdentifier<'a>) {
+        self.jsx_identifier_spans.insert(identifier.span);
+    }
+
+    fn visit_jsx_element_name(&mut self, name: &JSXElementName<'a>) {
+        if let JSXElementName::IdentifierReference(identifier) = name {
+            self.jsx_identifier_spans.insert(identifier.span);
+        }
+        walk::walk_jsx_element_name(self, name);
+    }
+
+    fn visit_jsx_member_expression_object(&mut self, object: &JSXMemberExpressionObject<'a>) {
+        if let JSXMemberExpressionObject::IdentifierReference(identifier) = object {
+            self.jsx_identifier_spans.insert(identifier.span);
+        }
+        walk::walk_jsx_member_expression_object(self, object);
+    }
+
+    fn visit_jsx_namespaced_name(&mut self, name: &JSXNamespacedName<'a>) {
+        self.non_jsx_identifier_spans.insert(name.namespace.span);
+        self.non_jsx_identifier_spans.insert(name.name.span);
+        walk::walk_jsx_namespaced_name(self, name);
+    }
+
+    fn visit_jsx_expression_container(&mut self, container: &JSXExpressionContainer<'a>) {
+        self.jsx_expression_depth += 1;
+        walk::walk_jsx_expression_container(self, container);
+        self.jsx_expression_depth -= 1;
+    }
+
+    fn visit_member_expression(&mut self, expression: &MemberExpression<'a>) {
+        if self.jsx_expression_depth > 0 {
+            self.jsx_member_expression_depth += 1;
+            if matches!(expression, MemberExpression::ComputedMemberExpression(_)) {
+                self.jsx_computed_member_depth += 1;
+            }
+            walk::walk_member_expression(self, expression);
+            if matches!(expression, MemberExpression::ComputedMemberExpression(_)) {
+                self.jsx_computed_member_depth -= 1;
+            }
+            self.jsx_member_expression_depth -= 1;
+        } else {
+            walk::walk_member_expression(self, expression);
+        }
+    }
+
+    fn visit_jsx_spread_attribute(&mut self, attribute: &JSXSpreadAttribute<'a>) {
+        self.jsx_expression_depth += 1;
+        walk::walk_jsx_spread_attribute(self, attribute);
+        self.jsx_expression_depth -= 1;
+    }
+
+    fn visit_jsx_spread_child(&mut self, spread_child: &JSXSpreadChild<'a>) {
+        self.jsx_expression_depth += 1;
+        walk::walk_jsx_spread_child(self, spread_child);
+        self.jsx_expression_depth -= 1;
+    }
+
+    fn visit_jsx_attribute(&mut self, attribute: &JSXAttribute<'a>) {
+        if let Some(JSXAttributeValue::StringLiteral(string_literal)) = &attribute.value {
+            self.jsx_text_spans.insert(string_literal.span);
+        }
+        walk::walk_jsx_attribute(self, attribute);
+    }
+}
+
+pub fn collect_token_context(program: &Program<'_>) -> EstreeTokenContext {
+    let mut context = EstreeTokenContext::default();
+    context.visit_program(program);
+    context
+}
+
+fn regex_parts(raw: &str) -> Option<(&str, &str)> {
+    let bytes = raw.as_bytes();
+    if bytes.first() != Some(&b'/') {
+        return None;
+    }
+
+    let mut escaped = false;
+    let mut in_character_class = false;
+    for index in 1..bytes.len() {
+        let byte = bytes[index];
+        if escaped {
+            escaped = false;
+            continue;
+        }
+        match byte {
+            b'\\' => escaped = true,
+            b'[' if !in_character_class => in_character_class = true,
+            b']' if in_character_class => in_character_class = false,
+            b'/' if !in_character_class => return Some((&raw[1..index], &raw[index + 1..])),
+            _ => {}
+        }
+    }
+
+    None
+}
+
+fn decode_js_unicode_escapes<'a>(allocator: &'a Allocator, raw: &'a str) -> &'a str {
+    if !raw.contains("\\u") {
+        return raw;
+    }
+
+    let bytes = raw.as_bytes();
+    let mut output = String::with_capacity(raw.len());
+    let mut i = 0;
+
+    while i < bytes.len() {
+        if bytes[i] == b'\\' && i + 1 < bytes.len() && bytes[i + 1] == b'u' {
+            // `\u{...}`
+            if i + 2 < bytes.len() && bytes[i + 2] == b'{' {
+                let mut j = i + 3;
+                while j < bytes.len() && bytes[j] != b'}' {
+                    j += 1;
+                }
+                if j < bytes.len()
+                    && j > i + 3
+                    && let Ok(codepoint) = u32::from_str_radix(&raw[i + 3..j], 16)
+                    && let Some(ch) = char::from_u32(codepoint)
+                {
+                    output.push(ch);
+                    i = j + 1;
+                    continue;
+                }
+            // `\uXXXX`
+            } else if i + 6 <= bytes.len()
+                && let Ok(codepoint) = u32::from_str_radix(&raw[i + 2..i + 6], 16)
+                && let Some(ch) = char::from_u32(codepoint)
+            {
+                output.push(ch);
+                i += 6;
+                continue;
+            }
+        }
+
+        let ch = raw[i..].chars().next().unwrap();
+        output.push(ch);
+        i += ch.len_utf8();
+    }
+
+    allocator.alloc_str(&output)
+}
+
+fn token_type(
+    kind: Kind,
+    is_ast_identifier: bool,
+    is_forced_keyword: bool,
+    is_jsx_identifier: bool,
+    is_jsx_text: bool,
+) -> &'static str {
+    if is_jsx_identifier {
+        return "JSXIdentifier";
+    }
+    if is_jsx_text {
+        return "JSXText";
+    }
+    if is_forced_keyword {
+        return "Keyword";
+    }
+    if is_ast_identifier {
+        return "Identifier";
+    }
+
+    match kind {
+        Kind::Ident | Kind::Await => "Identifier",
+        Kind::PrivateIdentifier => "PrivateIdentifier",
+        Kind::JSXText => "JSXText",
+        Kind::Str => "String",
+        Kind::RegExp => "RegularExpression",
+        Kind::NoSubstitutionTemplate
+        | Kind::TemplateHead
+        | Kind::TemplateMiddle
+        | Kind::TemplateTail => "Template",
+        Kind::True | Kind::False => "Boolean",
+        Kind::Null => "Null",
+        _ if kind.is_number() => "Numeric",
+        _ if kind.is_contextual_keyword() => "Identifier",
+        _ if kind.is_any_keyword() => "Keyword",
+        _ => "Punctuator",
+    }
+}
+
+pub fn to_estree_tokens<'a>(
+    allocator: &'a Allocator,
+    source_text: &'a str,
+    tokens: &[Token],
+    context: &EstreeTokenContext,
+    options: EstreeTokenOptions,
+) -> ArenaVec<'a, EstreeToken<'a>> {
+    let utf8_to_utf16 = Utf8ToUtf16::new(source_text);
+    let mut converter = utf8_to_utf16.converter();
+
+    let mut estree_tokens = ArenaVec::with_capacity_in(tokens.len(), allocator);
+    for token in tokens {
+        let kind = token.kind();
+        let source_value = &source_text[token.start() as usize..token.end() as usize];
+
+        let mut start = token.start();
+        let mut end = token.end();
+        if let Some(converter) = converter.as_mut() {
+            converter.convert_offset(&mut start);
+            converter.convert_offset(&mut end);
+        }
+        let span_utf16 = Span::new(start, end);
+
+        // TS estree token streams may already contain the second `<` as a
+        // standalone token here; skip the overlapping `<<` token.
+        if kind == Kind::ShiftLeft
+            && context.ts_type_parameter_starts.contains(&(span_utf16.start + 1))
+        {
+            continue;
+        }
+
+        let is_ast_identifier = context.ast_identifier_spans.contains(&span_utf16);
+        let is_ast_identifier = if options.exclude_legacy_keyword_identifiers {
+            is_ast_identifier && !matches!(kind, Kind::Yield | Kind::Let | Kind::Static)
+        } else {
+            is_ast_identifier
+        };
+        let token_type = token_type(
+            kind,
+            is_ast_identifier,
+            context.force_keyword_spans.contains(&span_utf16),
+            context.jsx_identifier_spans.contains(&span_utf16)
+                && !context.non_jsx_identifier_spans.contains(&span_utf16),
+            context.jsx_text_spans.contains(&span_utf16),
+        );
+
+        let source_value =
+            if kind == Kind::PrivateIdentifier { &source_value[1..] } else { source_value };
+        let value = if options.decode_identifier_escapes
+            && token.escaped()
+            && (kind.is_identifier_name() || kind == Kind::PrivateIdentifier)
+        {
+            decode_js_unicode_escapes(allocator, source_value)
+        } else {
+            source_value
+        };
+        let regex = if kind == Kind::RegExp {
+            regex_parts(source_value).map(|(pattern, flags)| EstreeRegExpToken { flags, pattern })
+        } else {
+            None
+        };
+
+        estree_tokens.push(EstreeToken { token_type, value, regex, start, end });
+    }
+
+    estree_tokens
+}
+
+pub fn to_estree_tokens_json(
+    allocator: &Allocator,
+    source_text: &str,
+    tokens: &[Token],
+    context: &EstreeTokenContext,
+    options: EstreeTokenOptions,
+) -> String {
+    let estree_tokens = to_estree_tokens(allocator, source_text, tokens, context, options);
+    serde_json::to_string_pretty(&estree_tokens).unwrap_or_default()
+}

crates/oxc_formatter/src/service/mod.rs

@@ -11,6 +11,7 @@ pub fn get_parse_options() -> ParseOptions {
         allow_v8_intrinsics: true,
         // `oxc_formatter` expects this to be `false`, otherwise panics
         preserve_parens: false,
+        collect_tokens: false,
     }
 }
 

crates/oxc_parser/src/cursor.rs

@@ -243,7 +243,7 @@ impl<'a> ParserImpl<'a> {
 
     /// Tell lexer to continue reading jsx identifier if the lexer character position is at `-` for `<component-name>`
     pub(crate) fn continue_lex_jsx_identifier(&mut self) {
-        if let Some(token) = self.lexer.continue_lex_jsx_identifier() {
+        if let Some(token) = self.lexer.continue_lex_jsx_identifier(self.token.start()) {
             self.token = token;
         }
     }

crates/oxc_parser/src/lexer/jsx.rs

@@ -110,10 +110,11 @@ impl Lexer<'_> {
     ///   `IdentifierStart`
     ///   `JSXIdentifier` `IdentifierPart`
     ///   `JSXIdentifier` [no `WhiteSpace` or Comment here] -
-    pub(crate) fn continue_lex_jsx_identifier(&mut self) -> Option<Token> {
+    pub(crate) fn continue_lex_jsx_identifier(&mut self, start: u32) -> Option<Token> {
         if self.peek_byte() != Some(b'-') {
             return None;
         }
+        self.token.set_start(start);
         self.consume_char();
 
         // Consume bytes which are part of identifier tail
@@ -121,7 +122,7 @@ impl Lexer<'_> {
             lexer: self,
             table: NOT_ASCII_JSX_ID_CONTINUE_TABLE,
             handle_eof: {
-                return Some(self.finish_next(Kind::Ident));
+                return Some(self.finish_next_retokenized(Kind::Ident));
             },
         };
 
@@ -141,6 +142,6 @@ impl Lexer<'_> {
             });
         }
 
-        Some(self.finish_next(Kind::Ident))
+        Some(self.finish_next_retokenized(Kind::Ident))
     }
 }

crates/oxc_parser/src/lexer/mod.rs

@@ -7,7 +7,7 @@
 
 use rustc_hash::FxHashMap;
 
-use oxc_allocator::Allocator;
+use oxc_allocator::{Allocator, Vec as ArenaVec};
 use oxc_ast::ast::RegExpFlags;
 use oxc_diagnostics::OxcDiagnostic;
 use oxc_span::{SourceType, Span};
@@ -45,6 +45,7 @@ pub struct LexerCheckpoint<'a> {
     source_position: SourcePosition<'a>,
     token: Token,
     errors_snapshot: ErrorSnapshot,
+    tokens_len: usize,
     has_pure_comment: bool,
     has_no_side_effects_comment: bool,
 }
@@ -95,6 +96,12 @@ pub struct Lexer<'a> {
 
     /// `memchr` Finder for end of multi-line comments. Created lazily when first used.
     multi_line_comment_end_finder: Option<memchr::memmem::Finder<'static>>,
+
+    /// Collected tokens in source order.
+    tokens: ArenaVec<'a, Token>,
+
+    /// Whether to collect tokens.
+    collect_tokens: bool,
 }
 
 impl<'a> Lexer<'a> {
@@ -106,6 +113,7 @@ impl<'a> Lexer<'a> {
         allocator: &'a Allocator,
         source_text: &'a str,
         source_type: SourceType,
+        collect_tokens: bool,
         unique: UniquePromise,
     ) -> Self {
         let source = Source::new(source_text, unique);
@@ -124,6 +132,8 @@ impl<'a> Lexer<'a> {
             escaped_strings: FxHashMap::default(),
             escaped_templates: FxHashMap::default(),
             multi_line_comment_end_finder: None,
+            tokens: ArenaVec::new_in(allocator),
+            collect_tokens,
         }
     }
 
@@ -136,7 +146,7 @@ impl<'a> Lexer<'a> {
         source_type: SourceType,
     ) -> Self {
         let unique = UniquePromise::new_for_tests_and_benchmarks();
-        Self::new(allocator, source_text, source_type, unique)
+        Self::new(allocator, source_text, source_type, false, unique)
     }
 
     /// Get errors.
@@ -163,6 +173,7 @@ impl<'a> Lexer<'a> {
             source_position: self.source.position(),
             token: self.token,
             errors_snapshot,
+            tokens_len: self.tokens.len(),
             has_pure_comment: self.trivia_builder.has_pure_comment,
             has_no_side_effects_comment: self.trivia_builder.has_no_side_effects_comment,
         }
@@ -180,6 +191,7 @@ impl<'a> Lexer<'a> {
             source_position: self.source.position(),
             token: self.token,
             errors_snapshot,
+            tokens_len: self.tokens.len(),
             has_pure_comment: self.trivia_builder.has_pure_comment,
             has_no_side_effects_comment: self.trivia_builder.has_no_side_effects_comment,
         }
@@ -192,6 +204,7 @@ impl<'a> Lexer<'a> {
             ErrorSnapshot::Count(len) => self.errors.truncate(len),
             ErrorSnapshot::Full(errors) => self.errors = errors,
         }
+        self.tokens.truncate(checkpoint.tokens_len);
         self.source.set_position(checkpoint.source_position);
         self.token = checkpoint.token;
         self.trivia_builder.has_pure_comment = checkpoint.has_pure_comment;
@@ -240,15 +253,53 @@ impl<'a> Lexer<'a> {
         self.finish_next(kind)
     }
 
+    #[inline]
     fn finish_next(&mut self, kind: Kind) -> Token {
+        self.finish_next_inner::<false>(kind)
+    }
+
+    #[inline]
+    fn finish_next_retokenized(&mut self, kind: Kind) -> Token {
+        self.finish_next_inner::<true>(kind)
+    }
+
+    #[inline]
+    fn finish_next_inner<const REPLACE_SAME_START: bool>(&mut self, kind: Kind) -> Token {
         self.token.set_kind(kind);
         self.token.set_end(self.offset());
         let token = self.token;
+        if self.collect_tokens && !matches!(token.kind(), Kind::Eof | Kind::HashbangComment) {
+            if REPLACE_SAME_START {
+                debug_assert!(self.tokens.last().is_some_and(|last| last.start() == token.start()));
+                let last = self.tokens.last_mut().unwrap();
+                *last = token;
+            } else {
+                self.tokens.push(token);
+            }
+        }
         self.trivia_builder.handle_token(token);
         self.token = Token::default();
         token
     }
 
+    /// Finish a re-lexed token used only for parser disambiguation.
+    /// This must not mutate the externally collected token stream.
+    fn finish_re_lex(&mut self, kind: Kind) -> Token {
+        self.token.set_kind(kind);
+        self.token.set_end(self.offset());
+        let token = self.token;
+        self.token = Token::default();
+        token
+    }
+
+    pub(crate) fn take_tokens(&mut self) -> ArenaVec<'a, Token> {
+        std::mem::replace(&mut self.tokens, ArenaVec::new_in(self.allocator))
+    }
+
+    pub(crate) fn set_tokens(&mut self, tokens: ArenaVec<'a, Token>) {
+        self.tokens = tokens;
+    }
+
     /// Advance source cursor to end of file.
     #[inline]
     pub fn advance_to_end(&mut self) {

crates/oxc_parser/src/lexer/punctuation.rs

@@ -85,9 +85,9 @@ impl Lexer<'_> {
     }
 
     pub(crate) fn re_lex_right_angle(&mut self) -> Token {
-        self.token.set_start(self.offset());
+        self.token.set_start(self.offset() - 1);
         let kind = self.read_right_angle();
-        self.finish_next(kind)
+        self.finish_next_retokenized(kind)
     }
 
     fn read_right_angle(&mut self) -> Kind {

crates/oxc_parser/src/lexer/regex.rs

@@ -21,7 +21,7 @@ impl Lexer<'_> {
                 },
         );
         let (pattern_end, flags, flags_error) = self.read_regex();
-        let token = self.finish_next(Kind::RegExp);
+        let token = self.finish_next_retokenized(Kind::RegExp);
         (token, pattern_end, flags, flags_error)
     }
 

crates/oxc_parser/src/lexer/template.rs

@@ -390,7 +390,7 @@ impl<'a> Lexer<'a> {
     pub(crate) fn next_template_substitution_tail(&mut self) -> Token {
         self.token.set_start(self.offset() - 1);
         let kind = self.read_template_literal(Kind::TemplateMiddle, Kind::TemplateTail);
-        self.finish_next(kind)
+        self.finish_next_retokenized(kind)
     }
 
     /// Save escaped template string
@@ -442,7 +442,8 @@ mod test {
         fn run_test(source_text: String, expected_escaped: String, is_only_part: bool) {
             let allocator = Allocator::default();
             let unique = UniquePromise::new_for_tests_and_benchmarks();
-            let mut lexer = Lexer::new(&allocator, &source_text, SourceType::default(), unique);
+            let mut lexer =
+                Lexer::new(&allocator, &source_text, SourceType::default(), false, unique);
             let token = lexer.next_token();
             assert_eq!(
                 token.kind(),

crates/oxc_parser/src/lexer/typescript.rs

@@ -5,13 +5,13 @@ impl Lexer<'_> {
     pub(crate) fn re_lex_as_typescript_l_angle(&mut self, offset: u32) -> Token {
         self.token.set_start(self.offset() - offset);
         self.source.back(offset as usize - 1);
-        self.finish_next(Kind::LAngle)
+        self.finish_re_lex(Kind::LAngle)
     }
 
     /// Re-tokenize '>>' and '>>>' to '>'
     pub(crate) fn re_lex_as_typescript_r_angle(&mut self, offset: u32) -> Token {
         self.token.set_start(self.offset() - offset);
         self.source.back(offset as usize - 1);
-        self.finish_next(Kind::RAngle)
+        self.finish_re_lex(Kind::RAngle)
     }
 }

crates/oxc_parser/src/lib.rs

@@ -93,10 +93,11 @@ use oxc_diagnostics::OxcDiagnostic;
 use oxc_span::{SourceType, Span};
 use oxc_syntax::module_record::ModuleRecord;
 
+pub use crate::lexer::{Kind, Token};
 use crate::{
     context::{Context, StatementContext},
     error_handler::FatalError,
-    lexer::{Lexer, Token},
+    lexer::Lexer,
     module_record::ModuleRecordBuilder,
     state::ParserState,
 };
@@ -166,6 +167,11 @@ pub struct ParserReturn<'a> {
     /// Irregular whitespaces for `Oxlint`
     pub irregular_whitespaces: Box<[Span]>,
 
+    /// Lexed tokens in source order.
+    ///
+    /// Tokens are only collected when [`ParseOptions::collect_tokens`] is enabled.
+    pub tokens: oxc_allocator::Vec<'a, Token>,
+
     /// Whether the parser panicked and terminated early.
     ///
     /// This will be `false` if parsing was successful, or if parsing was able to recover from a
@@ -219,6 +225,11 @@ pub struct ParseOptions {
     ///
     /// [`V8IntrinsicExpression`]: oxc_ast::ast::V8IntrinsicExpression
     pub allow_v8_intrinsics: bool,
+
+    /// Collect lexer tokens and return them in [`ParserReturn::tokens`].
+    ///
+    /// Default: `false`
+    pub collect_tokens: bool,
 }
 
 impl Default for ParseOptions {
@@ -229,6 +240,7 @@ impl Default for ParseOptions {
             allow_return_outside_function: false,
             preserve_parens: true,
             allow_v8_intrinsics: false,
+            collect_tokens: false,
         }
     }
 }
@@ -413,7 +425,7 @@ impl<'a> ParserImpl<'a> {
     ) -> Self {
         Self {
             options,
-            lexer: Lexer::new(allocator, source_text, source_type, unique),
+            lexer: Lexer::new(allocator, source_text, source_type, options.collect_tokens, unique),
             source_type,
             source_text,
             errors: vec![],
@@ -470,16 +482,16 @@ impl<'a> ParserImpl<'a> {
             is_flow_language = true;
             errors.push(error);
         }
-        let (module_record, module_record_errors) = self.module_record_builder.build();
+        let (module_record, mut module_record_errors) = self.module_record_builder.build();
         if errors.len() != 1 {
             errors
                 .reserve(self.lexer.errors.len() + self.errors.len() + module_record_errors.len());
-            errors.extend(self.lexer.errors);
-            errors.extend(self.errors);
-            errors.extend(module_record_errors);
+            errors.append(&mut self.lexer.errors);
+            errors.append(&mut self.errors);
+            errors.append(&mut module_record_errors);
         }
         let irregular_whitespaces =
-            self.lexer.trivia_builder.irregular_whitespaces.into_boxed_slice();
+            std::mem::take(&mut self.lexer.trivia_builder.irregular_whitespaces).into_boxed_slice();
 
         let source_type = program.source_type;
         if source_type.is_unambiguous() {
@@ -501,6 +513,7 @@ impl<'a> ParserImpl<'a> {
             module_record,
             errors,
             irregular_whitespaces,
+            tokens: self.lexer.take_tokens(),
             panicked,
             is_flow_language,
         }
@@ -564,6 +577,11 @@ impl<'a> ParserImpl<'a> {
         &mut self,
         statements: &mut oxc_allocator::Vec<'a, oxc_ast::ast::Statement<'a>>,
     ) {
+        // Token stream is already complete from the first parse.
+        // Reparsing here is only to patch AST nodes, so keep the original token stream.
+        let original_tokens =
+            if self.options.collect_tokens { Some(self.lexer.take_tokens()) } else { None };
+
         let checkpoints = std::mem::take(&mut self.state.potential_await_reparse);
         for (stmt_index, checkpoint) in checkpoints {
             // Rewind to the checkpoint
@@ -579,6 +597,11 @@ impl<'a> ParserImpl<'a> {
                 statements[stmt_index] = stmt;
             }
         }
+
+        if let Some(tokens) = original_tokens {
+            self.lexer.take_tokens();
+            self.lexer.set_tokens(tokens);
+        }
     }
 
     fn default_context(source_type: SourceType, options: ParseOptions) -> Context {

napi/playground/src/lib.rs

@@ -222,6 +222,7 @@ impl Oxc {
             allow_return_outside_function: parser_options.allow_return_outside_function,
             preserve_parens: parser_options.preserve_parens,
             allow_v8_intrinsics: parser_options.allow_v8_intrinsics,
+            collect_tokens: false,
         };
         let ParserReturn { program, errors, module_record, .. } =
             Parser::new(allocator, source_text, source_type).with_options(parser_options).parse();

tasks/coverage/Cargo.toml

@@ -24,6 +24,7 @@ doctest = false
 [dependencies]
 cow-utils = { workspace = true }
 oxc = { workspace = true, features = ["full", "isolated_declarations", "serialize", "conformance"] }
+oxc_estree_tokens = { workspace = true }
 oxc_formatter = { workspace = true }
 oxc_tasks_common = { workspace = true }
 oxc_tasks_transform_checker = { workspace = true }

tasks/coverage/snapshots/estree_acorn_jsx_tokens.snap

@@ -2,82 +2,18 @@ commit: a53e1b22
 
 estree_acorn_jsx_tokens Summary:
 AST Parsed     : 39/39 (100.00%)
-Positive Passed: 0/39 (0.00%)
+Positive Passed: 32/39 (82.05%)
 Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/003.jsx
 
 Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/004.jsx
 
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/005.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/007.jsx
-
 Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/008.jsx
 
 Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/009.jsx
 
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/010.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/011.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/012.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/013.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/014.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/015.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/016.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/017.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/018.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/019.jsx
-
 Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/020.jsx
 
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/021.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/022.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/023.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/024.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/025.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/026.jsx
-
 Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/027.jsx
 
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/028.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/029.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/030.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/031.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/032.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/033.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/034.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/035.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/041.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/042.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/043.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/044.jsx
-
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/045.jsx
-
 Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/046.jsx
 
-Token mismatch: tasks/coverage/estree-conformance/tests/acorn-jsx/pass/047.jsx
-

tasks/coverage/src/tools.rs

@@ -11,6 +11,7 @@ use oxc::{
     span::{ModuleKind, SourceType, Span},
     transformer::{JsxOptions, JsxRuntime, TransformOptions},
 };
+use oxc_estree_tokens::{EstreeTokenOptions, collect_token_context, to_estree_tokens_json};
 use oxc_formatter::{
     ArrowParentheses, AttributePosition, BracketSameLine, BracketSpacing, Expand, FormatOptions,
     Formatter, IndentStyle, IndentWidth, LineEnding, LineWidth, QuoteProperties, QuoteStyle,
@@ -837,7 +838,9 @@ pub fn run_estree_test262_tokens(files: &[Test262File]) -> Vec<CoverageResult> {
             let is_module = f.meta.flags.contains(&TestFlag::Module);
             let source_type = SourceType::script().with_module(is_module);
             let allocator = Allocator::new();
-            let ret = Parser::new(&allocator, &f.code, source_type).parse();
+            let options = ParseOptions { collect_tokens: true, ..ParseOptions::default() };
+            let ret = Parser::new(&allocator, &f.code, source_type).with_options(options).parse();
+
             if ret.panicked || !ret.errors.is_empty() {
                 let error =
                     ret.errors.first().map_or_else(|| "Panicked".to_string(), ToString::to_string);
@@ -848,12 +851,23 @@ pub fn run_estree_test262_tokens(files: &[Test262File]) -> Vec<CoverageResult> {
                 };
             }
 
+            let ParserReturn { mut program, tokens, .. } = ret;
+            let utf8_to_utf16 = Utf8ToUtf16::new(&f.code);
+            utf8_to_utf16.convert_program_with_ascending_order_checks(&mut program);
+            let token_context = collect_token_context(&program);
+
             let token_path = workspace_root()
                 .join("estree-conformance/tests/test262-tokens")
                 .join(f.path.strip_prefix("test262/").unwrap_or(&f.path))
                 .with_extension("json");
             let expected_tokens_json = fs::read_to_string(&token_path).unwrap_or_default();
-            let oxc_tokens_json = "[]".to_string();
+            let oxc_tokens_json = to_estree_tokens_json(
+                &allocator,
+                &f.code,
+                &tokens,
+                &token_context,
+                EstreeTokenOptions::test262(),
+            );
 
             let result = if oxc_tokens_json == expected_tokens_json {
                 TestResult::Passed
@@ -872,7 +886,9 @@ pub fn run_estree_acorn_jsx_tokens(files: &[AcornJsxFile]) -> Vec<CoverageResult
         .map(|f| {
             let source_type = SourceType::script().with_module(true).with_jsx(true);
             let allocator = Allocator::new();
-            let ret = Parser::new(&allocator, &f.code, source_type).parse();
+            let ret = Parser::new(&allocator, &f.code, source_type)
+                .with_options(ParseOptions { collect_tokens: true, ..Default::default() })
+                .parse();
             if ret.panicked || !ret.errors.is_empty() {
                 let error =
                     ret.errors.first().map_or_else(|| "Panicked".to_string(), ToString::to_string);
@@ -883,12 +899,21 @@ pub fn run_estree_acorn_jsx_tokens(files: &[AcornJsxFile]) -> Vec<CoverageResult
                 };
             }
 
-            let token_path = workspace_root()
-                .join("estree-conformance/tests/test262-tokens")
-                .join(f.path.strip_prefix("test262/").unwrap_or(&f.path))
-                .with_extension("json");
+            let token_path = workspace_root().join(f.path.with_extension("tokens.json"));
+
             let expected_tokens_json = fs::read_to_string(&token_path).unwrap_or_default();
-            let oxc_tokens_json = "[]".to_string();
+            let mut program = ret.program;
+            let utf8_to_utf16 = Utf8ToUtf16::new(&f.code);
+            utf8_to_utf16.convert_program_with_ascending_order_checks(&mut program);
+            let token_context = collect_token_context(&program);
+
+            let oxc_tokens_json = to_estree_tokens_json(
+                &allocator,
+                &f.code,
+                &ret.tokens,
+                &token_context,
+                EstreeTokenOptions::typescript(),
+            );
 
             let result = if oxc_tokens_json == expected_tokens_json {
                 TestResult::Passed
@@ -1038,7 +1063,11 @@ pub fn run_estree_typescript_tokens(files: &[TypeScriptFile]) -> Vec<CoverageRes
 
             for (unit, expected_tokens) in f.units.iter().zip(estree_token_units.iter()) {
                 let allocator = Allocator::new();
-                let options = ParseOptions { preserve_parens: false, ..Default::default() };
+                let options = ParseOptions {
+                    preserve_parens: false,
+                    collect_tokens: true,
+                    ..Default::default()
+                };
                 let ret = Parser::new(&allocator, &unit.content, unit.source_type)
                     .with_options(options)
                     .parse();
@@ -1055,7 +1084,18 @@ pub fn run_estree_typescript_tokens(files: &[TypeScriptFile]) -> Vec<CoverageRes
                     };
                 }
 
-                let oxc_tokens_json = "[]".to_string();
+                let mut program = ret.program;
+                let utf8_to_utf16 = Utf8ToUtf16::new(&unit.content);
+                utf8_to_utf16.convert_program_with_ascending_order_checks(&mut program);
+                let token_context = collect_token_context(&program);
+
+                let oxc_tokens_json = to_estree_tokens_json(
+                    &allocator,
+                    &unit.content,
+                    &ret.tokens,
+                    &token_context,
+                    EstreeTokenOptions::typescript(),
+                );
                 if oxc_tokens_json != *expected_tokens {
                     return CoverageResult {
                         path: f.path.clone(),