feat(estree/tokens): add function to update tokens in place (#19856)

Add a function `update_tokens` which converts tokens to ESTree format in place, mutating the `Token`s in the `Vec<Token>` in place.

This is what will be used for sending tokens to JS side via raw transfer.

Support for serializing tokens to JSON is also retained. The 2 implementations share the same AST visitor, and hook into it via separate `Context` implementations.

crates/oxc_estree_tokens/Cargo.toml

@@ -24,6 +24,6 @@ oxc_ast = { workspace = true }
 oxc_ast_visit = { workspace = true, features = ["serialize"] }
 oxc_data_structures = { workspace = true, features = ["assert_unchecked"] }
 oxc_estree = { workspace = true, features = ["serialize"] }
-oxc_parser = { workspace = true }
+oxc_parser = { workspace = true, features = ["mutate_tokens"] }
 oxc_span = { workspace = true, features = ["serialize"] }
 itoa = { workspace = true }

crates/oxc_estree_tokens/src/lib.rs

@@ -13,6 +13,7 @@ pub use jsx_state::{JSXState, JSXStateJS, JSXStateTS};
 pub use options::{
     ESTreeTokenConfig, ESTreeTokenOptions, ESTreeTokenOptionsJS, ESTreeTokenOptionsTS,
 };
+pub use serialize::update_tokens;
 use serialize::{estimate_json_len, serialize_tokens};
 
 /// Serializer config for tokens.

crates/oxc_parser/src/lexer/kind.rs

@@ -199,6 +199,9 @@ pub enum Kind {
     PrivateIdentifier,
     // JSX
     JSXText,
+    // `JSXIdentifier` tokens are never produced by lexer.
+    // Only used in ESTree conversion.
+    JSXIdentifier,
     // Decorator
     At,
 }
@@ -675,6 +678,7 @@ impl Kind {
             TemplateTail => "}",
             PrivateIdentifier => "#identifier",
             JSXText => "jsx",
+            JSXIdentifier => "jsx_identifier",
             At => "@",
             Assert => "assert",
             Any => "any",

tasks/benchmark/benches/parser.rs

@@ -1,7 +1,7 @@
 use oxc_allocator::Allocator;
 use oxc_ast_visit::utf8_to_utf16::Utf8ToUtf16;
 use oxc_benchmark::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
-use oxc_estree_tokens::{ESTreeTokenOptionsJS, to_estree_tokens_json};
+use oxc_estree_tokens::{ESTreeTokenOptionsJS, to_estree_tokens_json, update_tokens};
 use oxc_parser::{ParseOptions, Parser, ParserReturn, config::RuntimeParserConfig};
 use oxc_tasks_common::TestFiles;
 
@@ -186,23 +186,16 @@ fn bench_estree_tokens_raw(criterion: &mut Criterion) {
                     })
                     .with_config(config)
                     .parse();
-                let ParserReturn { program, tokens, .. } = ret;
+                let ParserReturn { program, mut tokens, .. } = ret;
 
                 // Creating span converter is not performed in measured section, as we only want to measure tokens.
                 // Span converter needs to be created anyway for converting spans in AST.
                 let span_converter = Utf8ToUtf16::new(program.source_text);
 
                 runner.run(|| {
-                    let tokens_json = to_estree_tokens_json(
-                        &tokens,
-                        &program,
-                        program.source_text,
-                        &span_converter,
-                        ESTreeTokenOptionsJS,
-                    );
-                    let tokens_json = black_box(tokens_json);
-                    // Allocate tokens JSON into arena, same as linter and NAPI parser package do
-                    let _tokens_json = allocator.alloc_str(&tokens_json);
+                    update_tokens(&mut tokens, &program, &span_converter, ESTreeTokenOptionsJS);
+
+                    black_box(tokens);
 
                     program
                 });