fix(regex): track capture groups and disjunction identity (#28297)

Modifier groups are non-capturing, and duplicate names are legal only
across mutually exclusive alternatives of the same disjunction. Track
disjunction identity and Unicode-set class nesting so backreferences and
duplicate names are validated correctly.

AI-assisted implementation.

crates/oxc_regular_expression/src/parser/pattern_parser/state.rs

@@ -51,7 +51,7 @@ impl<'a> State<'a> {
 
         let checkpoint = reader.checkpoint();
         let (num_of_left_capturing_parens, capturing_group_names) =
-            parse_capturing_groups(reader, self.unicode_mode, allocator)?;
+            parse_capturing_groups(reader, self.unicode_mode, self.unicode_sets_mode, allocator)?;
         reader.rewind(checkpoint);
 
         // In Annex B, this is `false` by default.
@@ -74,6 +74,7 @@ impl<'a> State<'a> {
 fn parse_capturing_groups<'a>(
     reader: &mut Reader<'a>,
     unicode_mode: bool,
+    unicode_sets_mode: bool,
     allocator: &'a Allocator,
 ) -> Result<(u32, FxHashSet<Str<'a>>), DuplicatedNamedCapturingGroupOffsets> {
     // Count only normal CapturingGroup(named, unnamed)
@@ -91,21 +92,24 @@ fn parse_capturing_groups<'a>(
     let mut tracker = AlternativeTracker::new();
 
     let mut in_escape = false;
-    let mut in_character_class = false;
+    let mut character_class_depth = 0_u32;
     while let Some(cp) = reader.peek() {
         if in_escape {
             in_escape = false;
         } else if cp == '\\' as u32 {
             in_escape = true;
         } else if cp == '[' as u32 {
-            in_character_class = true;
+            // Unicode sets allow nested classes; their string alternatives are not group alternatives.
+            if unicode_sets_mode || character_class_depth == 0 {
+                character_class_depth += 1;
+            }
         } else if cp == ']' as u32 {
-            in_character_class = false;
-        } else if !in_character_class && cp == '|' as u32 {
+            character_class_depth = character_class_depth.saturating_sub(1);
+        } else if character_class_depth == 0 && cp == '|' as u32 {
             tracker.mark_alternative();
-        } else if !in_character_class && cp == ')' as u32 {
+        } else if character_class_depth == 0 && cp == ')' as u32 {
             tracker.exit_group();
-        } else if !in_character_class && cp == '(' as u32 {
+        } else if character_class_depth == 0 && cp == '(' as u32 {
             reader.advance();
             tracker.enter_group();
 
@@ -116,8 +120,13 @@ fn parse_capturing_groups<'a>(
                 || reader.eat2('?', '!')
                 || reader.eat3('?', '<', '=')
                 || reader.eat3('?', '<', '!')
+                || reader.eat2('?', 'i')
+                || reader.eat2('?', 'm')
+                || reader.eat2('?', 's')
+                || reader.eat2('?', '-')
             {
-                // Non-capturing group or lookaround - depth already incremented
+                // Non-capturing group, modifier group, or lookaround.
+                // Full parsing validates the rest of the modifier prefix.
                 continue;
             }
 
@@ -153,7 +162,7 @@ fn parse_capturing_groups<'a>(
                     for existing in &named_groups {
                         if existing.name == group_name {
                             // Check if they can participate together
-                            if !AlternativeTracker::can_participate(
+                            if AlternativeTracker::can_participate(
                                 &existing.alternative_path,
                                 &alternative_path,
                             ) {
@@ -184,56 +193,54 @@ fn parse_capturing_groups<'a>(
     Ok((num_of_left_capturing_parens, group_names))
 }
 
-/// Tracks which alternatives at each depth level have been seen.
-/// Used to determine if duplicate named groups are in different alternatives.
+/// A position in each enclosing disjunction: (disjunction ID, alternative index).
+type AlternativePath = Vec<(usize, usize)>;
+
+/// Tracks alternatives within distinct disjunctions.
+/// Different alternatives exclude each other only if they belong to the same disjunction.
 #[derive(Debug)]
 struct AlternativeTracker {
-    /// Current nesting depth
-    depth: u32,
-    /// Current alternative index at each depth level (stack-based)
-    /// Each level represents the alternative index at that nesting depth
-    current_alternative: Vec<u32>,
+    next_disjunction_id: usize,
+    current_alternative: AlternativePath,
 }
 
 impl AlternativeTracker {
     fn new() -> Self {
-        Self { depth: 0, current_alternative: vec![0] }
+        Self { next_disjunction_id: 1, current_alternative: vec![(0, 0)] }
     }
 
     fn enter_group(&mut self) {
-        self.depth += 1;
-        while self.current_alternative.len() <= self.depth as usize {
-            self.current_alternative.push(0);
-        }
+        self.current_alternative.push((self.next_disjunction_id, 0));
+        self.next_disjunction_id += 1;
     }
 
     fn exit_group(&mut self) {
-        if let Some(alt) = self.current_alternative.get_mut(self.depth as usize) {
-            *alt = 0;
+        if self.current_alternative.len() > 1 {
+            self.current_alternative.pop();
         }
-        self.depth = self.depth.saturating_sub(1);
     }
 
     fn mark_alternative(&mut self) {
-        if let Some(alt) = self.current_alternative.get_mut(self.depth as usize) {
-            *alt += 1;
-        }
+        self.current_alternative.last_mut().unwrap().1 += 1;
     }
 
-    fn get_alternative_path(&self) -> Vec<u32> {
-        self.current_alternative.iter().take((self.depth + 1) as usize).copied().collect()
+    fn get_alternative_path(&self) -> AlternativePath {
+        // A GroupSpecifier is outside its own group's Disjunction, so only
+        // enclosing groups can put it in a mutually exclusive alternative.
+        self.current_alternative[..self.current_alternative.len() - 1].to_vec()
     }
 
-    fn can_participate(alt1: &[u32], alt2: &[u32]) -> bool {
-        let min_len = alt1.len().min(alt2.len());
-        // Check as prefixes, if they differ at any level,
-        // it means they are in different alternatives, so they can participate together.
-        for i in 0..min_len {
-            if alt1[i] != alt2[i] {
-                return true;
+    fn can_participate(alt1: &[(usize, usize)], alt2: &[(usize, usize)]) -> bool {
+        for ((disjunction1, alternative1), (disjunction2, alternative2)) in alt1.iter().zip(alt2) {
+            if disjunction1 != disjunction2 {
+                // The paths diverged into separate groups, not alternative branches.
+                break;
+            }
+            if alternative1 != alternative2 {
+                return false;
             }
         }
-        false
+        true
     }
 }
 
@@ -242,7 +249,7 @@ impl AlternativeTracker {
 struct NamedGroupInfo<'a> {
     name: Str<'a>,
     span: (u32, u32),
-    alternative_path: Vec<u32>,
+    alternative_path: AlternativePath,
 }
 
 // ---
@@ -277,7 +284,7 @@ mod tests {
 
             let allocator = Allocator::default();
             let (num_of_left_capturing_parens, capturing_group_names) =
-                parse_capturing_groups(&mut reader, true, &allocator).unwrap();
+                parse_capturing_groups(&mut reader, true, false, &allocator).unwrap();
 
             let actual = (num_of_left_capturing_parens, capturing_group_names.len());
             assert_eq!(expected, actual, "{source_text}");
@@ -299,7 +306,7 @@ mod tests {
 
             let allocator = Allocator::default();
             assert!(
-                parse_capturing_groups(&mut reader, true, &allocator).is_err(),
+                parse_capturing_groups(&mut reader, true, false, &allocator).is_err(),
                 "{source_text}"
             );
         }

tasks/coverage/misc/fail/regexp-duplicate-name-in-own-disjunction.js

@@ -0,0 +1 @@
+/(?<x>a|(?<x>b))/u;

tasks/coverage/misc/fail/regexp-duplicate-names-nested-class-strings.js

@@ -0,0 +1 @@
+/(?<x>a)[[a]\q{b|c}](?<x>d)/v;

tasks/coverage/misc/fail/regexp-duplicate-names-separate-disjunctions.js

@@ -0,0 +1 @@
+/(?:(?<x>a)|b)(?:c|(?<x>d))/u;

tasks/coverage/misc/fail/regexp-modifier-group-capture-count.js

@@ -0,0 +1 @@
+/(?i:a)(?m:b)(?s:c)(?-ims:d)\1/u;

tasks/coverage/snapshots/lexer_misc.snap

@@ -1,3 +1,3 @@
 lexer_misc Summary:
-AST Parsed     : 380/380 (100.00%)
-Positive Passed: 380/380 (100.00%)
+AST Parsed     : 384/384 (100.00%)
+Positive Passed: 384/384 (100.00%)

tasks/coverage/snapshots/parser_misc.snap

@@ -1,7 +1,7 @@
 parser_misc Summary:
 AST Parsed     : 121/121 (100.00%)
 Positive Passed: 121/121 (100.00%)
-Negative Passed: 261/261 (100.00%)
+Negative Passed: 265/265 (100.00%)
 
   × TS(1275): 'accessor' modifier can only appear on a property declaration.
    ╭─[misc/fail/accessor-modifier-abstract-method.ts:1:29]
@@ -5109,6 +5109,34 @@ Negative Passed: 261/261 (100.00%)
    ·           ────────
    ╰────
 
+  × Invalid regular expression: Duplicated capturing group names
+   ╭─[misc/fail/regexp-duplicate-name-in-own-disjunction.js:1:5]
+ 1 │ /(?<x>a|(?<x>b))/u;
+   ·     ─      ─
+   ╰────
+  help: Duplicate names must be in different alternatives
+
+  × Invalid regular expression: Duplicated capturing group names
+   ╭─[misc/fail/regexp-duplicate-names-nested-class-strings.js:1:5]
+ 1 │ /(?<x>a)[[a]\q{b|c}](?<x>d)/v;
+   ·     ─                  ─
+   ╰────
+  help: Duplicate names must be in different alternatives
+
+  × Invalid regular expression: Duplicated capturing group names
+   ╭─[misc/fail/regexp-duplicate-names-separate-disjunctions.js:1:8]
+ 1 │ /(?:(?<x>a)|b)(?:c|(?<x>d))/u;
+   ·        ─              ─
+   ╰────
+  help: Duplicate names must be in different alternatives
+
+  × Invalid regular expression: Invalid indexed reference
+   ╭─[misc/fail/regexp-modifier-group-capture-count.js:1:29]
+ 1 │ /(?i:a)(?m:b)(?s:c)(?-ims:d)\1/u;
+   ·                             ──
+   ╰────
+  help: There are no capturing groups defined in this pattern
+
   × Invalid regular expression: Invalid class atom
    ╭─[misc/fail/regexp-unicode-class-control-digit.js:1:3]
  1 │ /[\c1]/u;