feat(ast_tools): introduce #[js_only] attr for struct fields and converters (#11160)

Codegen for `ESTree` impls supports a `#[ts]` attribute on struct fields which only appear in TS-ESTree AST. Add a `#[js_only]` attribute which does the same thing in reverse - indicates a field only appears in the JS ESTree AST, and not in TS-ESTree.

This allows removing a custom serializer and hacky workaround for `JSXOpeningFragment`.

crates/oxc_ast/src/ast/jsx.rs

@@ -130,7 +130,7 @@ pub struct JSXFragment<'a> {
 #[ast(visit)]
 #[derive(Debug)]
 #[generate_derive(CloneIn, Dummy, TakeIn, GetSpan, GetSpanMut, ContentEq, ESTree)]
-#[estree(via = JSXOpeningFragmentConverter, add_fields(attributes = TsEmptyArray, selfClosing = TsFalse))]
+#[estree(add_fields(attributes = JsEmptyArray, selfClosing = JsFalse))]
 pub struct JSXOpeningFragment {
     /// Node location in source code
     pub span: Span,

crates/oxc_ast/src/generated/derive_estree.rs

@@ -2049,7 +2049,13 @@ impl ESTree for JSXFragment<'_> {
 
 impl ESTree for JSXOpeningFragment {
     fn serialize<S: Serializer>(&self, serializer: S) {
-        crate::serialize::jsx::JSXOpeningFragmentConverter(self).serialize(serializer)
+        let mut state = serializer.serialize_struct();
+        state.serialize_field("type", &JsonSafeString("JSXOpeningFragment"));
+        state.serialize_field("start", &self.span.start);
+        state.serialize_field("end", &self.span.end);
+        state.serialize_js_field("attributes", &crate::serialize::basic::JsEmptyArray(self));
+        state.serialize_js_field("selfClosing", &crate::serialize::basic::JsFalse(self));
+        state.end();
     }
 }
 

crates/oxc_ast/src/serialize/basic.rs

@@ -46,6 +46,18 @@ impl<T> ESTree for False<T> {
     }
 }
 
+/// Serialized as `false`. Field only present in JS ESTree AST (not TS-ESTree).
+#[ast_meta]
+#[estree(ts_type = "false", raw_deser = "false")]
+#[js_only]
+pub struct JsFalse<T>(pub T);
+
+impl<T> ESTree for JsFalse<T> {
+    fn serialize<S: Serializer>(&self, serializer: S) {
+        false.serialize(serializer);
+    }
+}
+
 /// Serialized as `false`. Field only present in TS-ESTree AST.
 #[ast_meta]
 #[estree(ts_type = "false", raw_deser = "false")]
@@ -114,6 +126,18 @@ impl<T> ESTree for EmptyArray<T> {
     }
 }
 
+/// Serialized as `[]`. Field only present in JS ESTree AST (not TS-ESTree).
+#[ast_meta]
+#[estree(ts_type = "[]", raw_deser = "[]")]
+#[js_only]
+pub struct JsEmptyArray<T>(pub T);
+
+impl<T> ESTree for JsEmptyArray<T> {
+    fn serialize<S: Serializer>(&self, serializer: S) {
+        EmptyArray(()).serialize(serializer);
+    }
+}
+
 /// Serialized as `[]`. Field only present in TS-ESTree AST.
 #[ast_meta]
 #[estree(ts_type = "[]", raw_deser = "[]")]

crates/oxc_ast/src/serialize/jsx.rs

@@ -3,8 +3,6 @@ use oxc_estree::{ESTree, JsonSafeString, Serializer, StructSerializer};
 
 use crate::ast::*;
 
-use super::EmptyArray;
-
 /// Serializer for `opening_element` field of `JSXElement`.
 ///
 /// `selfClosing` field of `JSXOpeningElement` depends on whether `JSXElement` has a `closing_element`.
@@ -88,42 +86,3 @@ impl ESTree for JSXElementThisExpression<'_> {
         JSXIdentifier { span: self.0.span, name: Atom::from("this") }.serialize(serializer);
     }
 }
-
-/// Converter for `JSXOpeningFragment`.
-///
-/// Add `attributes` and `selfClosing` fields in JS AST, but not in TS AST.
-/// Acorn-JSX has these fields, but TS-ESLint parser does not.
-///
-/// The extra fields are added to the type as `TsEmptyArray` and `TsFalse`,
-/// which are incorrect, as these fields appear only in the *JS* AST, not the TS one.
-/// But that results in the fields being optional in TS type definition.
-//
-// TODO: Find a better way to do this.
-#[ast_meta]
-#[estree(raw_deser = "
-    const node = {
-        type: 'JSXOpeningFragment',
-        start: DESER[u32](POS_OFFSET.span.start),
-        end: DESER[u32](POS_OFFSET.span.end),
-        /* IF_JS */
-        attributes: [],
-        selfClosing: false,
-        /* END_IF_JS */
-    };
-    node
-")]
-pub struct JSXOpeningFragmentConverter<'b>(pub &'b JSXOpeningFragment);
-
-impl ESTree for JSXOpeningFragmentConverter<'_> {
-    fn serialize<S: Serializer>(&self, serializer: S) {
-        let mut state = serializer.serialize_struct();
-        state.serialize_field("type", &JsonSafeString("JSXOpeningFragment"));
-        state.serialize_field("start", &self.0.span.start);
-        state.serialize_field("end", &self.0.span.end);
-        if !S::INCLUDE_TS_FIELDS {
-            state.serialize_field("attributes", &EmptyArray(()));
-            state.serialize_field("selfClosing", &false);
-        }
-        state.end();
-    }
-}

crates/oxc_ast_macros/src/lib.rs

@@ -74,6 +74,7 @@ pub fn ast_meta(_args: TokenStream, input: TokenStream) -> TokenStream {
         content_eq,
         estree,
         generate_derive,
+        js_only,
         plural,
         scope,
         span,

crates/oxc_estree/src/serialize/structs.rs

@@ -15,6 +15,17 @@ pub trait StructSerializer {
     /// `key` must not contain any characters which require escaping in JSON.
     fn serialize_field<T: ESTree + ?Sized>(&mut self, key: &'static str, value: &T);
 
+    /// Serialize struct field which is JS syntax only (not in TS AST).
+    ///
+    /// This method behaves differently, depending on the serializer's `Config`:
+    /// * `INCLUDE_TS_FIELDS == false`: Behaves same as `serialize_field`
+    ///   i.e. the field is included in JSON.
+    /// * `INCLUDE_TS_FIELDS == true`: Do nothing.
+    ///   i.e. the field is skipped.
+    ///
+    /// `key` must not contain any characters which require escaping in JSON.
+    fn serialize_js_field<T: ESTree + ?Sized>(&mut self, key: &'static str, value: &T);
+
     /// Serialize struct field which is TypeScript syntax.
     ///
     /// This method behaves differently, depending on the serializer's `Config`:
@@ -85,6 +96,22 @@ impl<C: Config, F: Formatter> StructSerializer for ESTreeStructSerializer<'_, C,
         value.serialize(&mut *self.serializer);
     }
 
+    /// Serialize struct field which is JS syntax only (not in TS AST).
+    ///
+    /// This method behaves differently, depending on the serializer's `Config`:
+    /// * `INCLUDE_TS_FIELDS == false`: Behaves same as `serialize_field`
+    ///   i.e. the field is included in JSON.
+    /// * `INCLUDE_TS_FIELDS == true`: Do nothing.
+    ///   i.e. the field is skipped.
+    ///
+    /// `key` must not contain any characters which require escaping in JSON.
+    #[inline(always)]
+    fn serialize_js_field<T: ESTree + ?Sized>(&mut self, key: &'static str, value: &T) {
+        if !C::INCLUDE_TS_FIELDS {
+            self.serialize_field(key, value);
+        }
+    }
+
     /// Serialize struct field which is TypeScript syntax.
     ///
     /// This method behaves differently, depending on the serializer's `Config`:
@@ -219,6 +246,21 @@ impl<P: StructSerializer> StructSerializer for FlatStructSerializer<'_, P> {
         self.0.serialize_field(key, value);
     }
 
+    /// Serialize struct field which is JS syntax only (not in TS AST).
+    ///
+    /// This method behaves differently, depending on the serializer's `Config`:
+    /// * `INCLUDE_TS_FIELDS == false`: Behaves same as `serialize_field`
+    ///   i.e. the field is included in JSON.
+    /// * `INCLUDE_TS_FIELDS == true`: Do nothing.
+    ///   i.e. the field is skipped.
+    ///
+    /// `key` must not contain any characters which require escaping in JSON.
+    #[inline(always)]
+    fn serialize_js_field<T: ESTree + ?Sized>(&mut self, key: &'static str, value: &T) {
+        // Delegate to parent `StructSerializer`
+        self.0.serialize_js_field(key, value);
+    }
+
     /// Serialize struct field which is TypeScript syntax.
     ///
     /// This method behaves differently, depending on the serializer's `Config`:
@@ -425,7 +467,7 @@ mod tests {
         struct Foo {
             js: u32,
             ts: u32,
-            more_ts: u32,
+            js_only: u32,
             more_js: u32,
         }
 
@@ -434,18 +476,18 @@ mod tests {
                 let mut state = serializer.serialize_struct();
                 state.serialize_field("js", &self.js);
                 state.serialize_ts_field("ts", &self.ts);
-                state.serialize_ts_field("moreTs", &self.more_ts);
+                state.serialize_js_field("jsOnly", &self.js_only);
                 state.serialize_field("moreJs", &self.more_js);
                 state.end();
             }
         }
 
-        let foo = Foo { js: 1, ts: 2, more_ts: 3, more_js: 4 };
+        let foo = Foo { js: 1, ts: 2, js_only: 3, more_js: 4 };
 
         let mut serializer = CompactTSSerializer::new();
         foo.serialize(&mut serializer);
         let s = serializer.into_string();
-        assert_eq!(&s, r#"{"js":1,"ts":2,"moreTs":3,"moreJs":4}"#);
+        assert_eq!(&s, r#"{"js":1,"ts":2,"moreJs":4}"#);
 
         let mut serializer = PrettyTSSerializer::new();
         foo.serialize(&mut serializer);
@@ -455,7 +497,6 @@ mod tests {
             r#"{
   "js": 1,
   "ts": 2,
-  "moreTs": 3,
   "moreJs": 4
 }"#
         );
@@ -463,7 +504,7 @@ mod tests {
         let mut serializer = CompactJSSerializer::new();
         foo.serialize(&mut serializer);
         let s = serializer.into_string();
-        assert_eq!(&s, r#"{"js":1,"moreJs":4}"#);
+        assert_eq!(&s, r#"{"js":1,"jsOnly":3,"moreJs":4}"#);
 
         let mut serializer = PrettyJSSerializer::new();
         foo.serialize(&mut serializer);
@@ -472,6 +513,7 @@ mod tests {
             &s,
             r#"{
   "js": 1,
+  "jsOnly": 3,
   "moreJs": 4
 }"#
         );

napi/parser/generated/deserialize/js.js

@@ -1178,14 +1178,13 @@ function deserializeJSXFragment(pos) {
 }
 
 function deserializeJSXOpeningFragment(pos) {
-  const node = {
+  return {
     type: 'JSXOpeningFragment',
     start: deserializeU32(pos),
     end: deserializeU32(pos + 4),
     attributes: [],
     selfClosing: false,
   };
-  return node;
 }
 
 function deserializeJSXClosingFragment(pos) {

napi/parser/generated/deserialize/ts.js

@@ -1332,12 +1332,11 @@ function deserializeJSXFragment(pos) {
 }
 
 function deserializeJSXOpeningFragment(pos) {
-  const node = {
+  return {
     type: 'JSXOpeningFragment',
     start: deserializeU32(pos),
     end: deserializeU32(pos + 4),
   };
-  return node;
 }
 
 function deserializeJSXClosingFragment(pos) {

tasks/ast_tools/src/derives/estree.rs

@@ -37,7 +37,7 @@ impl Derive for DeriveESTree {
     /// Register that accept `#[estree]` attr on structs, enums, struct fields, enum variants,
     /// or meta types.
     /// Allow attr on structs and enums which don't derive this trait.
-    /// Also accept `#[ts]` attr on struct fields and enum variants.
+    /// Also accept `#[ts]` and `#[js_only]` attrs on struct fields and meta types.
     fn attrs(&self) -> &[(&'static str, AttrPositions)] {
         &[
             (
@@ -47,6 +47,7 @@ impl Derive for DeriveESTree {
                 ),
             ),
             ("ts", attr_positions!(StructField | Meta)),
+            ("js_only", attr_positions!(StructField | Meta)),
         ]
     }
 
@@ -55,6 +56,7 @@ impl Derive for DeriveESTree {
         match attr_name {
             "estree" => parse_estree_attr(location, part),
             "ts" => parse_ts_attr(location, &part),
+            "js_only" => parse_js_only_attr(location, &part),
             _ => unreachable!(),
         }
     }
@@ -227,6 +229,24 @@ fn parse_ts_attr(location: AttrLocation, part: &AttrPart) -> Result<()> {
     Ok(())
 }
 
+/// Parse `#[js_only]` attr on struct field or meta type.
+fn parse_js_only_attr(location: AttrLocation, part: &AttrPart) -> Result<()> {
+    if !matches!(part, AttrPart::None) {
+        return Err(());
+    }
+
+    // Location can only be `StructField` or `Meta`
+    match location {
+        AttrLocation::StructField(struct_def, field_index) => {
+            struct_def.fields[field_index].estree.is_js = true;
+        }
+        AttrLocation::Meta(meta) => meta.estree.is_js = true,
+        _ => unreachable!(),
+    }
+
+    Ok(())
+}
+
 /// Initialize `estree.field_order` on all structs.
 fn prepare_field_orders(schema: &mut Schema, estree_derive_id: DeriveId) {
     // Note: Outside the loop to avoid allocating temporary `Vec`s on each turn of the loop.
@@ -479,7 +499,9 @@ impl<'s> StructSerializerGenerator<'s> {
             quote!( #self_path.#field_name_ident )
         };
 
-        let serialize_method_ident = create_safe_ident(if field.estree.is_ts {
+        let serialize_method_ident = create_safe_ident(if field.estree.is_js {
+            "serialize_js_field"
+        } else if field.estree.is_ts {
             "serialize_ts_field"
         } else {
             "serialize_field"
@@ -498,7 +520,9 @@ impl<'s> StructSerializerGenerator<'s> {
     ) {
         let converter = self.schema.meta_by_name(converter_name);
         let converter_path = converter.import_path_from_crate(self.krate, self.schema);
-        let serialize_method_ident = create_safe_ident(if converter.estree.is_ts {
+        let serialize_method_ident = create_safe_ident(if converter.estree.is_js {
+            "serialize_js_field"
+        } else if converter.estree.is_ts {
             "serialize_ts_field"
         } else {
             "serialize_field"

tasks/ast_tools/src/generators/raw_transfer.rs

@@ -272,7 +272,7 @@ impl<'s> StructDeserializerGenerator<'s> {
         struct_def: &StructDef,
         struct_offset: u32,
     ) {
-        if !self.is_ts && field.estree.is_ts {
+        if (self.is_ts && field.estree.is_js) || (!self.is_ts && field.estree.is_ts) {
             return;
         }
 
@@ -372,7 +372,7 @@ impl<'s> StructDeserializerGenerator<'s> {
         struct_offset: u32,
     ) {
         let converter = self.schema.meta_by_name(converter_name);
-        if !self.is_ts && converter.estree.is_ts {
+        if (self.is_ts && converter.estree.is_js) || (!self.is_ts && converter.estree.is_ts) {
             return;
         }
 

tasks/ast_tools/src/generators/typescript.rs

@@ -271,7 +271,7 @@ fn generate_ts_type_def_for_struct_field_impl<'s>(
     }
 
     let field_camel_name = get_struct_field_name(field);
-    let question_mark = if field.estree.is_ts { "?" } else { "" };
+    let question_mark = if field.estree.is_js || field.estree.is_ts { "?" } else { "" };
     write_it!(fields_str, "\n\t{field_camel_name}{question_mark}: {field_type_name};");
 }
 
@@ -287,7 +287,7 @@ fn generate_ts_type_def_for_added_struct_field(
     let Some(ts_type) = converter.estree.ts_type.as_deref() else {
         panic!("No `ts_type` provided for ESTree converter `{converter_name}`");
     };
-    let question_mark = if converter.estree.is_ts { "?" } else { "" };
+    let question_mark = if converter.estree.is_js || converter.estree.is_ts { "?" } else { "" };
     write_it!(fields_str, "\n\t{field_name}{question_mark}: {ts_type};");
 }
 

tasks/ast_tools/src/schema/extensions/estree.rs

@@ -79,6 +79,8 @@ pub struct ESTreeStructField {
     pub no_flatten: bool,
     /// `true` for fields containing a `&str` or `Atom` which does not need escaping in JSON
     pub json_safe: bool,
+    /// `true` if field is only included in JS ESTree AST (not TS-ESTree AST).
+    pub is_js: bool,
     /// `true` if field is only included in TS-ESTree AST (not JS ESTree AST).
     pub is_ts: bool,
 }
@@ -103,6 +105,8 @@ pub struct ESTreeMeta {
     pub ts_type: Option<String>,
     /// JS code for raw transfer deserializer.
     pub raw_deser: Option<String>,
+    /// `true` if meta type is for a struct field which is present only in JS AST.
+    pub is_js: bool,
     /// `true` if meta type is for a struct field which is present only in TS AST.
     pub is_ts: bool,
 }