fix(grammar): accept a lone '&' in JSX strings and text (#1736)

A plain '&' inside JSX -- a URL query in an attribute
(href="https://fonts.googleapis.com/css2?family=Inter&display=swap"),
or text such as <p>Tom & Jerry</p> -- produced a tree-sitter ERROR node
in .js, .jsx and .tsx files. The file was flagged parse_partial in the
coverage view, and the error recovery turned words from the URL
(create-next-app -> create/next/app) into identifiers that resolved to
unrelated symbols as false USAGE edges.

Root cause is upstream tree-sitter-javascript, which the tsx dialect
builds on: unescaped_{double,single}_jsx_string_fragment is
/([^"&]|&[^#A-Za-z])+/, so '&' is only accepted before a space/digit
or as a complete html_character_reference (&amp; / &#38;). jsx_text is
scanned by the external scanner, which stops at every '&', and nothing
in _jsx_child accepted a '&' that is not a reference. So '&' + letter in
a string, and any lone '&' in JSX text, had no valid token.

Fix: javascript and tsx become self-maintained forks of the pins we
already vendored (javascript @ 58404d8cf191; typescript tsx @
75b3874edb2d on its javascript base @ 3a837b6f3658), following the
arkts model. One patch (tools/tree-sitter-javascript/patches/
jsx-lone-ampersand.patch) is applied to both javascript bases:
- a one-character '&' token at lexical precedence -1, aliased to the
  existing string_fragment (in strings) and jsx_text (in text); the
  lower precedence keeps &amp; / &#38; lexing as html_character_reference
- the string-fragment regexes exclude a '&' directly before the closing
  quote, so "a&" no longer runs past the end of the string
Node types are unchanged (same type/named set). The external scanners
stay upstream bytes. Parsers regenerated with tree-sitter-cli 0.25.10
(ABI 15; tsx moves from ABI 14, and its tree_sitter/ headers now match
arkts). Without the patch both fork dirs regenerate upstream exactly.
The upstream corpus passes unchanged: javascript 116/116; tsx gives the
same results as unpatched upstream.

Proof on vercel/next.js examples/ (canary 725705bd, 1,722 js/jsx/tsx
files): parse_partial 92 -> 47 (tsx 33 -> 0, jsx 2 -> 0, js 12 -> 2; the
two left are Babel do-expressions and Flow types), no newly flagged
file, code-graph nodes 14,739 -> 14,739, edges 24,773 -> 24,657: the
115 USAGE edges removed were all false cross-example edges from URL
fragments, plus a 4 SIMILAR_TO / +3 SEMANTICALLY_RELATED reshuffle
from the corrected ASTs.

Registered like arkts: MANIFEST.md fork rows + notes (ABI tally
recounted), audit FORKS map, THIRD_PARTY.md, vendored checksums.
LICENSEs stay upstream's MIT byte-identical.

Tests: parse_coverage gains a .js/.jsx/.tsx matrix (double/single-quoted
attribute, '&' before the quote, '&' + letter/space/digit in text, plus
&amp;/&#38; controls) and a guard that broken JSX is still flagged.

Signed-off-by: Martin Vogel <martin.vogel.tech@gmail.com>

tools/tree-sitter-javascript/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "tree-sitter-javascript-cbm",
+  "version": "0.25.0-cbm.1",
+  "description": "cbm fork of tree-sitter-javascript @ 58404d8 (JSX lone-ampersand fix, #1736)",
+  "license": "MIT",
+  "devDependencies": {
+    "tree-sitter-cli": "0.25.10"
+  }
+}

tools/tree-sitter-javascript/patches/jsx-lone-ampersand.patch

@@ -0,0 +1,52 @@
+--- a/grammar.js
++++ b/grammar.js
+@@ -684,6 +684,9 @@
+     _jsx_child: $ => choice(
+       $.jsx_text,
+       $.html_character_reference,
++      // cbm fork (#1736): a lone '&' that does not start a complete
++      // html_character_reference is plain text, as in HTML/React.
++      alias($._jsx_lone_ampersand, $.jsx_text),
+       $._jsx_element,
+       $.jsx_expression,
+     ),
+@@ -749,6 +752,7 @@
+         repeat(choice(
+           alias($.unescaped_double_jsx_string_fragment, $.string_fragment),
+           $.html_character_reference,
++          alias($._jsx_string_lone_ampersand, $.string_fragment), // cbm fork (#1736)
+         )),
+         '"',
+       ),
+@@ -757,6 +761,7 @@
+         repeat(choice(
+           alias($.unescaped_single_jsx_string_fragment, $.string_fragment),
+           $.html_character_reference,
++          alias($._jsx_string_lone_ampersand, $.string_fragment), // cbm fork (#1736)
+         )),
+         '\'',
+       ),
+@@ -766,11 +771,21 @@
+     // We give names to the token() constructs containing a regexp
+     // so as to obtain a node in the CST.
+     //
+-    unescaped_double_jsx_string_fragment: _ => token.immediate(prec(1, /([^"&]|&[^#A-Za-z])+/)),
++    // cbm fork (#1736): '&' directly before the closing quote is excluded so the
++    // fragment cannot run past the end of the string (upstream: &[^#A-Za-z]).
++    unescaped_double_jsx_string_fragment: _ => token.immediate(prec(1, /([^"&]|&[^#A-Za-z"])+/)),
+ 
+     // same here
+-    unescaped_single_jsx_string_fragment: _ => token.immediate(prec(1, /([^'&]|&[^#A-Za-z])+/)),
++    unescaped_single_jsx_string_fragment: _ => token.immediate(prec(1, /([^'&]|&[^#A-Za-z'])+/)),
+ 
++    // cbm fork (#1736): a lone '&' that does not begin a complete
++    // html_character_reference (`?a=1&b=2`, `Tom &Jerry`). Precedence -1 keeps
++    // html_character_reference (precedence 0) winning for `&` / `&`; the
++    // lexer falls back to this one-character token only when no reference
++    // matches. Upstream leaves `&` + letter/`#` unmatched, producing an ERROR.
++    _jsx_lone_ampersand: _ => token(prec(-1, '&')),
++    _jsx_string_lone_ampersand: _ => token.immediate(prec(-1, '&')),
++
+     _jsx_attribute_value: $ => choice(
+       alias($._jsx_string, $.string),
+       $.jsx_expression,

tools/tree-sitter-javascript/src/scanner.c

@@ -0,0 +1,364 @@
+#include "tree_sitter/parser.h"
+
+#include <stdio.h>
+#include <wctype.h>
+
+enum TokenType {
+    AUTOMATIC_SEMICOLON,
+    TEMPLATE_CHARS,
+    TERNARY_QMARK,
+    HTML_COMMENT,
+    LOGICAL_OR,
+    ESCAPE_SEQUENCE,
+    REGEX_PATTERN,
+    JSX_TEXT,
+};
+
+void *tree_sitter_javascript_external_scanner_create() { return NULL; }
+
+void tree_sitter_javascript_external_scanner_destroy(void *p) {}
+
+unsigned tree_sitter_javascript_external_scanner_serialize(void *payload, char *buffer) { return 0; }
+
+void tree_sitter_javascript_external_scanner_deserialize(void *p, const char *b, unsigned n) {}
+
+static inline void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
+
+static inline void skip(TSLexer *lexer) { lexer->advance(lexer, true); }
+
+static bool scan_template_chars(TSLexer *lexer) {
+    lexer->result_symbol = TEMPLATE_CHARS;
+    for (bool has_content = false;; has_content = true) {
+        lexer->mark_end(lexer);
+        switch (lexer->lookahead) {
+            case '`':
+                return has_content;
+            case '\0':
+                return false;
+            case '$':
+                advance(lexer);
+                if (lexer->lookahead == '{') {
+                    return has_content;
+                }
+                break;
+            case '\\':
+                return has_content;
+            default:
+                advance(lexer);
+        }
+    }
+}
+
+typedef enum {
+    REJECT,     // Semicolon is illegal, ie a syntax error occurred
+    NO_NEWLINE, // Unclear if semicolon will be legal, continue
+    ACCEPT,     // Semicolon is legal, assuming a comment was encountered
+} WhitespaceResult;
+
+/**
+ * @param consume If false, only consume enough to check if comment indicates semicolon-legality
+ */
+static WhitespaceResult scan_whitespace_and_comments(TSLexer *lexer, bool *scanned_comment, bool consume) {
+    bool saw_block_newline = false;
+
+    for (;;) {
+        while (iswspace(lexer->lookahead)) {
+            skip(lexer);
+        }
+
+        if (lexer->lookahead == '/') {
+            skip(lexer);
+
+            if (lexer->lookahead == '/') {
+                skip(lexer);
+                while (lexer->lookahead != 0 && lexer->lookahead != '\n' && lexer->lookahead != 0x2028 &&
+                       lexer->lookahead != 0x2029) {
+                    skip(lexer);
+                }
+                *scanned_comment = true;
+            } else if (lexer->lookahead == '*') {
+                skip(lexer);
+                while (lexer->lookahead != 0) {
+                    if (lexer->lookahead == '*') {
+                        skip(lexer);
+                        if (lexer->lookahead == '/') {
+                            skip(lexer);
+                            *scanned_comment = true;
+
+                            if (lexer->lookahead != '/' && !consume) {
+                                return saw_block_newline ? ACCEPT : NO_NEWLINE;
+                            }
+
+                            break;
+                        }
+                    } else if (lexer->lookahead == '\n' || lexer->lookahead == 0x2028 || lexer->lookahead == 0x2029) {
+                        saw_block_newline = true;
+                        skip(lexer);
+                    } else {
+                        skip(lexer);
+                    }
+                }
+            } else {
+                return REJECT;
+            }
+        } else {
+            return ACCEPT;
+        }
+    }
+}
+
+static bool scan_automatic_semicolon(TSLexer *lexer, bool comment_condition, bool *scanned_comment) {
+    lexer->result_symbol = AUTOMATIC_SEMICOLON;
+    lexer->mark_end(lexer);
+
+    for (;;) {
+        if (lexer->lookahead == 0) {
+            return true;
+        }
+
+        if (lexer->lookahead == '/') {
+            WhitespaceResult result = scan_whitespace_and_comments(lexer, scanned_comment, false);
+            if (result == REJECT) {
+                return false;
+            }
+
+            if (result == ACCEPT && comment_condition && lexer->lookahead != ',' && lexer->lookahead != '=') {
+                return true;
+            }
+        }
+
+        if (lexer->lookahead == '}') {
+            return true;
+        }
+
+        if (lexer->is_at_included_range_start(lexer)) {
+            return true;
+        }
+
+        if (lexer->lookahead == '\n' || lexer->lookahead == 0x2028 || lexer->lookahead == 0x2029) {
+            break;
+        }
+
+        if (!iswspace(lexer->lookahead)) {
+            return false;
+        }
+
+        skip(lexer);
+    }
+
+    skip(lexer);
+
+    if (scan_whitespace_and_comments(lexer, scanned_comment, true) == REJECT) {
+        return false;
+    }
+
+    switch (lexer->lookahead) {
+        case '`':
+        case ',':
+        case ':':
+        case ';':
+        case '*':
+        case '%':
+        case '>':
+        case '<':
+        case '=':
+        case '[':
+        case '(':
+        case '?':
+        case '^':
+        case '|':
+        case '&':
+        case '/':
+            return false;
+
+        // Insert a semicolon before decimals literals but not otherwise.
+        case '.':
+            skip(lexer);
+            return iswdigit(lexer->lookahead);
+
+        // Insert a semicolon before `--` and `++`, but not before binary `+` or `-`.
+        case '+':
+            skip(lexer);
+            return lexer->lookahead == '+';
+        case '-':
+            skip(lexer);
+            return lexer->lookahead == '-';
+
+        // Don't insert a semicolon before `!=`, but do insert one before a unary `!`.
+        case '!':
+            skip(lexer);
+            return lexer->lookahead != '=';
+
+        // Don't insert a semicolon before `in` or `instanceof`, but do insert one
+        // before an identifier.
+        case 'i':
+            skip(lexer);
+
+            if (lexer->lookahead != 'n') {
+                return true;
+            }
+            skip(lexer);
+
+            if (!iswalpha(lexer->lookahead)) {
+                return false;
+            }
+
+            for (unsigned i = 0; i < 8; i++) {
+                if (lexer->lookahead != "stanceof"[i]) {
+                    return true;
+                }
+                skip(lexer);
+            }
+
+            if (!iswalpha(lexer->lookahead)) {
+                return false;
+            }
+            break;
+
+        default:
+            break;
+    }
+
+    return true;
+}
+
+static bool scan_ternary_qmark(TSLexer *lexer) {
+    for (;;) {
+        if (!iswspace(lexer->lookahead)) {
+            break;
+        }
+        skip(lexer);
+    }
+
+    if (lexer->lookahead == '?') {
+        advance(lexer);
+
+        if (lexer->lookahead == '?') {
+            return false;
+        }
+
+        lexer->mark_end(lexer);
+        lexer->result_symbol = TERNARY_QMARK;
+
+        if (lexer->lookahead == '.') {
+            advance(lexer);
+            if (iswdigit(lexer->lookahead)) {
+                return true;
+            }
+            return false;
+        }
+        return true;
+    }
+    return false;
+}
+
+static bool scan_html_comment(TSLexer *lexer) {
+    while (iswspace(lexer->lookahead) || lexer->lookahead == 0x2028 || lexer->lookahead == 0x2029) {
+        skip(lexer);
+    }
+
+    const char *comment_start = "<!--";
+    const char *comment_end = "-->";
+
+    if (lexer->lookahead == '<') {
+        for (unsigned i = 0; i < 4; i++) {
+            if (lexer->lookahead != comment_start[i]) {
+                return false;
+            }
+            advance(lexer);
+        }
+    } else if (lexer->lookahead == '-') {
+        for (unsigned i = 0; i < 3; i++) {
+            if (lexer->lookahead != comment_end[i]) {
+                return false;
+            }
+            advance(lexer);
+        }
+    } else {
+        return false;
+    }
+
+    while (lexer->lookahead != 0 && lexer->lookahead != '\n' && lexer->lookahead != 0x2028 &&
+           lexer->lookahead != 0x2029) {
+        advance(lexer);
+    }
+
+    lexer->result_symbol = HTML_COMMENT;
+    lexer->mark_end(lexer);
+
+    return true;
+}
+
+static bool scan_jsx_text(TSLexer *lexer) {
+    // saw_text will be true if we see any non-whitespace content, or any whitespace content that is not a newline and
+    // does not immediately follow a newline.
+    bool saw_text = false;
+    // at_newline will be true if we are currently at a newline, or if we are at whitespace that is not a newline but
+    // immediately follows a newline.
+    bool at_newline = false;
+
+    while (lexer->lookahead != 0 && lexer->lookahead != '<' && lexer->lookahead != '>' && lexer->lookahead != '{' &&
+           lexer->lookahead != '}' && lexer->lookahead != '&') {
+        bool is_wspace = iswspace(lexer->lookahead);
+        if (lexer->lookahead == '\n') {
+            at_newline = true;
+        } else {
+            // If at_newline is already true, and we see some whitespace, then it must stay true.
+            // Otherwise, it should be false.
+            //
+            // See the table below to determine the logic for computing `saw_text`.
+            //
+            // |------------------------------------|
+            // | at_newline | is_wspace | saw_text  |
+            // |------------|-----------|-----------|
+            // | false (0)  | false (0) | true  (1) |
+            // | false (0)  | true  (1) | true  (1) |
+            // | true  (1)  | false (0) | true  (1) |
+            // | true  (1)  | true  (1) | false (0) |
+            // |------------------------------------|
+
+            at_newline &= is_wspace;
+            if (!at_newline) {
+                saw_text = true;
+            }
+        }
+
+        advance(lexer);
+    }
+
+    lexer->result_symbol = JSX_TEXT;
+    return saw_text;
+}
+
+bool tree_sitter_javascript_external_scanner_scan(void *payload, TSLexer *lexer, const bool *valid_symbols) {
+    if (valid_symbols[TEMPLATE_CHARS]) {
+        if (valid_symbols[AUTOMATIC_SEMICOLON]) {
+            return false;
+        }
+        return scan_template_chars(lexer);
+    }
+
+    if (valid_symbols[JSX_TEXT] && scan_jsx_text(lexer)) {
+        return true;
+    }
+
+    if (valid_symbols[AUTOMATIC_SEMICOLON]) {
+        bool scanned_comment = false;
+        bool ret = scan_automatic_semicolon(lexer, !valid_symbols[LOGICAL_OR], &scanned_comment);
+        if (!ret && !scanned_comment && valid_symbols[TERNARY_QMARK] && lexer->lookahead == '?') {
+            return scan_ternary_qmark(lexer);
+        }
+        return ret;
+    }
+
+    if (valid_symbols[TERNARY_QMARK]) {
+        return scan_ternary_qmark(lexer);
+    }
+
+    if (valid_symbols[HTML_COMMENT] && !valid_symbols[LOGICAL_OR] && !valid_symbols[ESCAPE_SEQUENCE] &&
+        !valid_symbols[REGEX_PATTERN]) {
+        return scan_html_comment(lexer);
+    }
+
+    return false;
+}

tools/tree-sitter-javascript/src/tree_sitter/alloc.h

@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+// Allow clients to override allocation functions
+#ifdef TREE_SITTER_REUSE_ALLOCATOR
+
+extern void *(*ts_current_malloc)(size_t size);
+extern void *(*ts_current_calloc)(size_t count, size_t size);
+extern void *(*ts_current_realloc)(void *ptr, size_t size);
+extern void (*ts_current_free)(void *ptr);
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

tools/tree-sitter-javascript/src/tree_sitter/array.h

@@ -0,0 +1,291 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self) _array__delete((Array *)(self))
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count) \
+  do { \
+    if ((count) == 0) break; \
+    _array__grow((Array *)(self), count, array_elem_size(self)); \
+    memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
+    (self)->size += (count); \
+  } while (0)
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other)                                       \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((Array *)(self), array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+
+/// Swap one array with another
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  assert(index < self->size);
+  char *contents = (char *)self->contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (self->size - index - 1) * element_size);
+  self->size--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+    } else {
+      self->contents = ts_malloc(new_capacity * element_size);
+    }
+    self->capacity = new_capacity;
+  }
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    _array__reserve(self, element_size, new_capacity);
+  }
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void _array__splice(Array *self, size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  assert(old_end <= self->size);
+
+  _array__reserve(self, element_size, new_size);
+
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (self->size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  self->size += new_count - old_count;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_

tools/tree-sitter-javascript/src/tree_sitter/parser.h

@@ -0,0 +1,286 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+typedef struct TSLanguageMetadata {
+  uint8_t major_version;
+  uint8_t minor_version;
+  uint8_t patch_version;
+} TSLanguageMetadata;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+// Used to index the field and supertype maps.
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+  void (*log)(const TSLexer *, const char *, ...);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+  uint16_t reserved_word_set_id;
+} TSLexerMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
+struct TSLanguage {
+  uint32_t abi_version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexerMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+  const TSStateId *primary_state_ids;
+  const char *name;
+  const TSSymbol *reserved_words;
+  uint16_t max_reserved_word_set_size;
+  uint32_t supertype_count;
+  const TSSymbol *supertype_symbols;
+  const TSMapSlice *supertype_map_slices;
+  const TSSymbol *supertype_map_entries;
+  TSLanguageMetadata metadata;
+};
+
+static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    const TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  const TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
+/*
+ *  Lexer Macros
+ */
+
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  UNUSED                        \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value)          \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value),         \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_name, children, precedence, prod_id) \
+  {{                                                       \
+    .reduce = {                                            \
+      .type = TSParseActionTypeReduce,                     \
+      .symbol = symbol_name,                               \
+      .child_count = children,                             \
+      .dynamic_precedence = precedence,                    \
+      .production_id = prod_id                             \
+    },                                                     \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_

tools/tree-sitter-javascript/test/corpus/jsx_ampersand.txt

@@ -0,0 +1,140 @@
+================================================================================
+JSX attribute: lone ampersand in a double-quoted string (#1736)
+================================================================================
+
+<link href="https://x.com/a?a=1&b=2" />;
+
+--------------------------------------------------------------------------------
+
+(program
+  (expression_statement
+    (jsx_self_closing_element
+      (identifier)
+      (jsx_attribute
+        (property_identifier)
+        (string
+          (string_fragment)
+          (string_fragment)
+          (string_fragment))))))
+
+================================================================================
+JSX attribute: lone ampersand in a single-quoted string (#1736)
+================================================================================
+
+<link href='https://x.com/a?a=1&b=2' />;
+
+--------------------------------------------------------------------------------
+
+(program
+  (expression_statement
+    (jsx_self_closing_element
+      (identifier)
+      (jsx_attribute
+        (property_identifier)
+        (string
+          (string_fragment)
+          (string_fragment)
+          (string_fragment))))))
+
+================================================================================
+JSX attribute: ampersand before the closing quote (#1736)
+================================================================================
+
+<p title="a&" alt='b&' />;
+
+--------------------------------------------------------------------------------
+
+(program
+  (expression_statement
+    (jsx_self_closing_element
+      (identifier)
+      (jsx_attribute
+        (property_identifier)
+        (string
+          (string_fragment)
+          (string_fragment)))
+      (jsx_attribute
+        (property_identifier)
+        (string
+          (string_fragment)
+          (string_fragment))))))
+
+================================================================================
+JSX text: lone ampersand (#1736)
+================================================================================
+
+<p>Tom &Jerry, Tom & Jerry, x &1 y, a&</p>;
+
+--------------------------------------------------------------------------------
+
+(program
+  (expression_statement
+    (jsx_element
+      (jsx_opening_element
+        (identifier))
+      (jsx_text)
+      (jsx_text)
+      (jsx_text)
+      (jsx_text)
+      (jsx_text)
+      (jsx_text)
+      (jsx_text)
+      (jsx_text)
+      (jsx_closing_element
+        (identifier)))))
+
+================================================================================
+JSX: character references still parse as references
+================================================================================
+
+<p title="x&amp;y&#38;z">A&amp;B &#38; C &#x26;</p>;
+
+--------------------------------------------------------------------------------
+
+(program
+  (expression_statement
+    (jsx_element
+      (jsx_opening_element
+        (identifier)
+        (jsx_attribute
+          (property_identifier)
+          (string
+            (string_fragment)
+            (html_character_reference)
+            (string_fragment)
+            (html_character_reference)
+            (string_fragment))))
+      (jsx_text)
+      (html_character_reference)
+      (jsx_text)
+      (html_character_reference)
+      (jsx_text)
+      (html_character_reference)
+      (jsx_closing_element
+        (identifier)))))
+
+================================================================================
+JSX: bitwise and logical and in expressions are unaffected
+================================================================================
+
+<b title={a & b}>{a && b}</b>;
+
+--------------------------------------------------------------------------------
+
+(program
+  (expression_statement
+    (jsx_element
+      (jsx_opening_element
+        (identifier)
+        (jsx_attribute
+          (property_identifier)
+          (jsx_expression
+            (binary_expression
+              (identifier)
+              (identifier)))))
+      (jsx_expression
+        (binary_expression
+          (identifier)
+          (identifier)))
+      (jsx_closing_element
+        (identifier)))))

tools/tree-sitter-javascript/tree-sitter.json

@@ -0,0 +1,16 @@
+{
+  "grammars": [
+    {
+      "name": "javascript",
+      "camelcase": "JavaScript",
+      "scope": "source.js",
+      "path": ".",
+      "file-types": ["js", "mjs", "cjs", "jsx"]
+    }
+  ],
+  "metadata": {
+    "version": "0.25.0-cbm.1",
+    "license": "MIT",
+    "description": "cbm fork of tree-sitter-javascript @ 58404d8 (JSX lone-ampersand fix)"
+  }
+}