fix(extract): parse PHP past inline HTML so symbols after ?> reach the graph (#2000)

Signed-off-by: harshitaajoshi <j.harshitaa06@gmail.com>
This commit is contained in:
harshitaajoshi committed 2026-10-19 23:11:02 -05:00
1 parent 4c80d89edf
commit 09dc36d261
7 files changed
+528

No files matched your search

+1
View File
@@ -398,6 +398,7 @@ EXTRACTION_SRCS = \
$(CBM_DIR)/route_path.c \
$(CBM_DIR)/openapi.c \
$(CBM_DIR)/sql_values.c \
$(CBM_DIR)/php_inline_html.c \
$(CBM_DIR)/helpers.c \
$(CBM_DIR)/cpp_branch_views.c \
$(CBM_DIR)/result_compact.c \
+14
View File
@@ -3313,6 +3313,18 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
return r;
}
/* The bytes the grammar parses. PHP that leaves PHP mode (`?> markup <?php`)
* is parsed with its markup blanked (#2000); the rewrite keeps the length and
* every line break, so offsets and line numbers still refer to the file on
* disk. Every other language parses the file as it is. */
static const char *extract_parse_bytes(CBMArena *a, CBMLanguage language, const char *source,
int source_len) {
if (language == CBM_LANG_PHP) {
return cbm_php_mask_inline_html(a, source, source_len);
}
return source;
}
/* Initial block for the per-file traversal scratch arena, chosen by measuring
* arena_grow on a 14k-file TypeScript tree: it fires on one file in 12,000 at
* both this size and at 1 MB, and on most files at 256 KB, where the two
@@ -3733,6 +3745,8 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
return result;
}
source = extract_parse_bytes(a, language, source, source_len);
// Skip pathologically nested Perl before tree-sitter's recursive GLR stack
// merge overflows a small stack during the parse (see
// cbm_source_nesting_exceeds). Scoped to Perl: its ambiguous call grammar is
+4
View File
@@ -1361,6 +1361,10 @@ void cbm_extract_definitions_without_module(CBMExtractCtx *ctx);
// per ref()/source() call. No-op unless the file parses as SQL and actually
// contains a dbt builtin call. Defined in extract_dbt.c.
void cbm_extract_dbt(CBMExtractCtx *ctx);
// PHP source with its inline-HTML regions blanked so the php_only grammar can
// parse past `?>` (#2000). Same length and line breaks as the input; returns
// `source` itself when nothing needs rewriting. Defined in php_inline_html.c.
const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len);
void cbm_extract_imports(CBMExtractCtx *ctx);
void cbm_extract_usages(CBMExtractCtx *ctx);
void cbm_extract_semantic(CBMExtractCtx *ctx);
+4
View File
@@ -20,6 +20,7 @@
#include "php_lsp.h"
#include "lsp_node_iter.h"
#include "lsp_work.h"
#include "../cbm.h"
#include "../helpers.h"
#include "../../../src/foundation/hash_table.h"
#include <ctype.h>
@@ -4795,6 +4796,9 @@ void cbm_run_php_lsp_cross_with_bindings(CBMArena *arena, const char *source, in
const CBMPHPImportBindings *bindings) {
if (!arena || !source || source_len <= 0 || !out)
return;
/* The retained source is the file on disk; read it the way the per-file
* extract did, so the cached tree and node text agree (#2000). */
source = cbm_php_mask_inline_html(arena, source, source_len);
TSParser *parser = NULL;
TSTree *tree = cached_tree;
+317
View File
@@ -0,0 +1,317 @@
// php_inline_html.c — let the php_only grammar read a file that leaves PHP mode.
//
// A PHP file starts in inline-HTML mode and may switch in and out of PHP any
// number of times: `?> <div>...</div> <?php`. The vendored grammar is the
// tree-sitter-php `php_only` variant, which has no rule for `?>` or for the
// text that follows it, so every byte after the first close tag used to land in
// an ERROR region and the declarations there never reached the graph (#2000).
//
// Instead of parsing the markup, blank it. The rewritten buffer has the same
// length as the original and keeps every line break, so byte offsets, rows and
// columns are unchanged and node text inside PHP code is byte-identical:
//
// - markup before the first open tag and between tags becomes spaces
// - the first open tag stays, so the tree keeps its php_tag
// - each later `<?php`, `<?=` or `<?` becomes spaces; after `<?=` the echoed
// expression is left as an expression statement, so its calls are kept
// - each `?>` becomes `; `, the statement terminator PHP itself treats it as
//
// Only code and `//` / `#` comments can close PHP mode. Strings, heredoc,
// nowdoc and block comments are skipped, `#[` is an attribute, and everything
// after __halt_compiler is left alone, as PHP stops reading there.
#include "cbm.h"
#include "foundation/arena.h"
#include "foundation/constants.h"
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
enum {
PHP_TAG_OPEN_LEN = 5, /* <?php */
PHP_TAG_ECHO_LEN = 3, /* <?= */
PHP_TAG_SHORT_LEN = 2, /* <? */
PHP_HEREDOC_LEN = 3, /* <<< */
PHP_FIRST_NON_ASCII = 0x80,
PHP_NOT_FOUND = -1,
};
static const char PHP_HALT_COMPILER[] = "__halt_compiler";
enum { PHP_HALT_COMPILER_LEN = (int)sizeof(PHP_HALT_COMPILER) - SKIP_ONE };
/* True when the literal `lit` starts at s[i]. */
static bool php_at(const char *s, int len, int i, const char *lit) {
size_t n = strlen(lit);
return i >= 0 && i <= len && (size_t)(len - i) >= n && memcmp(s + i, lit, n) == 0;
}
static bool php_is_ident_char(unsigned char c) {
return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') ||
c >= PHP_FIRST_NON_ASCII;
}
static bool php_is_ident_start(unsigned char c) {
return php_is_ident_char(c) && !(c >= '0' && c <= '9');
}
static bool php_is_space(unsigned char c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v';
}
static bool php_is_blank(char c) {
return c == ' ' || c == '\t';
}
/* Case-insensitive match of the lowercase `word` at s[i]. */
static bool php_ieq(const char *s, int len, int i, const char *word) {
int n = (int)strlen(word);
if (i + n > len) {
return false;
}
for (int k = 0; k < n; k++) {
char c = s[i + k];
if (c >= 'A' && c <= 'Z') {
c = (char)(c - 'A' + 'a');
}
if (c != word[k]) {
return false;
}
}
return true;
}
/* Length of the open tag at i, or 0. `<?php` needs whitespace or EOF after it
* and `<?=` is always a tag. A bare `<?` counts only when whitespace follows,
* so an `<?xml ...?>` prolog in markup stays markup. */
static int php_open_tag_len(const char *s, int len, int i) {
if (!php_at(s, len, i, "<?")) {
return 0;
}
int after = i + PHP_TAG_SHORT_LEN;
if (php_ieq(s, len, after, "php")) {
int end = i + PHP_TAG_OPEN_LEN;
return (end == len || php_is_space((unsigned char)s[end])) ? PHP_TAG_OPEN_LEN : 0;
}
if (after < len && s[after] == '=') {
return PHP_TAG_ECHO_LEN;
}
if (after < len && php_is_space((unsigned char)s[after])) {
return PHP_TAG_SHORT_LEN;
}
return 0;
}
static bool php_has_close_tag(const char *s, int len) {
for (int i = 0; i < len; i++) {
if (php_at(s, len, i, "?>")) {
return true;
}
}
return false;
}
/* Blank [from, to) in place, keeping line breaks. */
static void php_blank(char *out, int from, int to) {
for (int k = from; k < to; k++) {
if (out[k] != '\n' && out[k] != '\r') {
out[k] = ' ';
}
}
}
/* Index just past the quoted string opened at i. */
static int php_skip_quoted(const char *s, int len, int i) {
char quote = s[i];
i += SKIP_ONE;
while (i < len && s[i] != quote) {
i += (s[i] == '\\') ? PAIR_LEN : SKIP_ONE;
}
return i < len ? i + SKIP_ONE : len;
}
/* Index just past the block comment opened at i. */
static int php_skip_block_comment(const char *s, int len, int i) {
for (int k = i + PAIR_LEN; k < len; k++) {
if (php_at(s, len, k, "*/")) {
return k + PAIR_LEN;
}
}
return len;
}
/* Index of the line break or `?>` that ends the one-line comment at i. */
static int php_skip_line_comment(const char *s, int len, int i) {
while (i < len && s[i] != '\n' && !php_at(s, len, i, "?>")) {
i++;
}
return i;
}
/* Parse the heredoc/nowdoc opener `<<<ID`, `<<<"ID"` or `<<<'ID'` at i. On
* success stores the label span and returns the index of the line break that
* ends the opener; returns PHP_NOT_FOUND when the bytes are not an opener. */
static int php_heredoc_opener(const char *s, int len, int i, int *label, int *label_len) {
int p = i + PHP_HEREDOC_LEN;
while (p < len && php_is_blank(s[p])) {
p++;
}
char quote = 0;
if (p < len && (s[p] == '\'' || s[p] == '"')) {
quote = s[p];
p++;
}
int start = p;
while (p < len && php_is_ident_char((unsigned char)s[p])) {
p++;
}
if (p == start || !php_is_ident_start((unsigned char)s[start])) {
return PHP_NOT_FOUND;
}
*label = start;
*label_len = p - start;
if (quote) {
if (p >= len || s[p] != quote) {
return PHP_NOT_FOUND;
}
p++;
}
if (p < len && s[p] == '\r') {
p++;
}
return (p < len && s[p] == '\n') ? p : PHP_NOT_FOUND;
}
/* True when the line starting at p closes the heredoc. PHP 7.3+ allows the
* closing label to be indented and followed by any non-identifier byte. */
static bool php_heredoc_closes(const char *s, int len, int p, int label, int label_len, int *end) {
while (p < len && php_is_blank(s[p])) {
p++;
}
if (len - p < label_len || memcmp(s + p, s + label, (size_t)label_len) != 0) {
return false;
}
int after = p + label_len;
if (after < len && php_is_ident_char((unsigned char)s[after])) {
return false;
}
*end = after;
return true;
}
/* Index just past the heredoc/nowdoc opened by the `<<<` at i. */
static int php_skip_heredoc(const char *s, int len, int i) {
int label = 0;
int label_len = 0;
int nl = php_heredoc_opener(s, len, i, &label, &label_len);
if (nl < 0) {
return i + PHP_HEREDOC_LEN;
}
while (nl < len) {
int line = nl + SKIP_ONE;
int end = 0;
if (php_heredoc_closes(s, len, line, label, label_len, &end)) {
return end;
}
const char *next = memchr(s + line, '\n', (size_t)(len - line));
if (!next) {
return len;
}
nl = (int)(next - s);
}
return len;
}
/* Index just past the identifier at i; sets *halt when it is __halt_compiler
* (a keyword, so not a `$variable` of that name). */
static int php_skip_ident(const char *s, int len, int i, bool *halt) {
int start = i;
while (i < len && php_is_ident_char((unsigned char)s[i])) {
i++;
}
bool is_var = start > 0 && s[start - SKIP_ONE] == '$';
*halt =
!is_var && i - start == PHP_HALT_COMPILER_LEN && php_ieq(s, len, start, PHP_HALT_COMPILER);
return i;
}
/* Scan PHP code from i to the next close tag. Returns the index of that `?>`,
* len when the file ends in PHP mode, or PHP_NOT_FOUND at __halt_compiler. */
static int php_scan_code(const char *s, int len, int i) {
while (i < len) {
unsigned char c = (unsigned char)s[i];
if (php_at(s, len, i, "?>")) {
return i;
}
if (c == '\'' || c == '"' || c == '`') {
i = php_skip_quoted(s, len, i);
} else if (php_at(s, len, i, "/*")) {
i = php_skip_block_comment(s, len, i);
} else if (php_at(s, len, i, "//") || (c == '#' && !php_at(s, len, i, "#["))) {
i = php_skip_line_comment(s, len, i);
} else if (php_at(s, len, i, "<<<")) {
i = php_skip_heredoc(s, len, i);
} else if (php_is_ident_start(c)) {
bool halt = false;
i = php_skip_ident(s, len, i, &halt);
if (halt) {
return PHP_NOT_FOUND;
}
} else {
i++;
}
}
return len;
}
const char *cbm_php_mask_inline_html(CBMArena *arena, const char *source, int source_len) {
if (!arena || !source || source_len <= 0) {
return source;
}
/* Fast path: a file that opens PHP at its first byte and never writes a
* close tag is exactly what the grammar already accepts. */
if (php_open_tag_len(source, source_len, 0) > 0 && !php_has_close_tag(source, source_len)) {
return source;
}
char *out = (char *)cbm_arena_alloc(arena, (size_t)source_len + SKIP_ONE);
if (!out) {
return source;
}
memcpy(out, source, (size_t)source_len);
out[source_len] = '\0';
bool changed = false;
bool seen_open = false;
int i = 0;
while (i < source_len) {
/* Inline-HTML mode: blank up to the next open tag. */
int html_start = i;
int tag = 0;
while (i < source_len && (tag = php_open_tag_len(source, source_len, i)) == 0) {
i++;
}
if (i > html_start) {
php_blank(out, html_start, i);
changed = true;
}
if (i >= source_len) {
break;
}
if (seen_open) {
php_blank(out, i, i + tag);
changed = true;
}
seen_open = true;
/* PHP mode, up to the close tag. */
int close = php_scan_code(source, source_len, i + tag);
if (close < 0 || close >= source_len) {
break;
}
out[close] = ';';
out[close + SKIP_ONE] = ' ';
changed = true;
i = close + PAIR_LEN;
}
return changed ? out : source;
}
+128
View File
@@ -833,6 +833,130 @@ TEST(php_function) {
PASS();
}
/* #2000: inline HTML between `?>` and `<?php` is valid PHP, but the php_only
* grammar has no rule for it, so everything after the first `?>` fell into an
* ERROR region and never reached the graph. */
static const CBMCall *find_call_named(CBMFileResult *r, const char *callee) {
for (int i = 0; i < r->calls.count; i++) {
if (r->calls.items[i].callee_name && strcmp(r->calls.items[i].callee_name, callee) == 0)
return &r->calls.items[i];
}
return NULL;
}
static int php_def_line(CBMFileResult *r, const char *name) {
for (int i = 0; i < r->defs.count; i++) {
if (strcmp(r->defs.items[i].name, name) == 0)
return (int)r->defs.items[i].start_line;
}
return -1;
}
TEST(php_inline_html_tail_reaches_graph_issue2000) {
CBMFileResult *r = extract("<?php\n"
"function gamma() { return 2; }\n"
"?>\n"
"<div class=\"box\">\n"
" <p>plain markup after the closing tag</p>\n"
"</div>\n"
"<?php\n"
"function delta() { return gamma(); }\n",
CBM_LANG_PHP, "t", "inline_html.php");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Function", "gamma"));
ASSERT(has_def(r, "Function", "delta"));
/* Lines are in original-file coordinates. */
ASSERT_EQ(php_def_line(r, "gamma"), 2);
ASSERT_EQ(php_def_line(r, "delta"), 8);
const CBMCall *c = find_call_named(r, "gamma");
ASSERT_NOT_NULL(c);
ASSERT_NOT_NULL(strstr(c->enclosing_func_qn, "delta"));
ASSERT_FALSE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
/* View-file shapes: echo tags keep their calls, and control structures that
* are opened in one PHP block and closed in a later one still parse. */
TEST(php_inline_html_template_shapes_issue2000) {
CBMFileResult *r = extract("<!DOCTYPE html>\n"
"<html><head><title><?= page_title($page) ?></title></head>\n"
"<body>\n"
"<?php if ($user): ?>\n"
" <p>Hello <?=$user->name?></p>\n"
"<?php else: ?>\n"
" <a href=\"/login\">log in</a>\n"
"<?php endif; ?>\n"
"<?php foreach ($items as $item) { ?>\n"
" <li><?php echo render_item($item); ?></li>\n"
"<?php } ?>\n"
"<?php function footer_links() { return nav_links(); } ?>\n"
"</body></html>\n",
CBM_LANG_PHP, "t", "view.php");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_call(r, "page_title"));
ASSERT(has_call(r, "render_item"));
ASSERT(has_def(r, "Function", "footer_links"));
ASSERT_EQ(php_def_line(r, "footer_links"), 12);
ASSERT(has_call(r, "nav_links"));
ASSERT_FALSE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
/* `?>` only leaves PHP mode from code or a one-line comment. Inside strings,
* heredoc, nowdoc and block comments it is text; `#[` is an attribute, not a
* comment; `<?xml` in markup is not an open tag. */
TEST(php_inline_html_close_tag_in_literals_issue2000) {
CBMFileResult *r = extract("<?php\n"
"$a = 'single ?> quoted';\n"
"$b = \"double ?> quoted \\\" still\";\n"
"$c = <<<EOT\n"
" heredoc ?> body\n"
" EOT;\n"
"$d = <<<'NOW'\n"
"nowdoc ?> body\n"
"NOW;\n"
"/* block ?> comment */\n"
"#[Pure]\n"
"function one() { return 1; }\n"
"// line comment ends here ?>\n"
"<?xml version=\"1.0\"?><note/>\n"
"<?php\n"
"function two() { return one(); }\n",
CBM_LANG_PHP, "t", "literals.php");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Function", "one"));
ASSERT(has_def(r, "Function", "two"));
ASSERT_EQ(php_def_line(r, "two"), 16);
ASSERT(has_call(r, "one"));
ASSERT_FALSE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
/* A trailing `?>` (common at the end of older class files) is not a partial
* parse, and CRLF line endings keep their line numbers. */
TEST(php_inline_html_trailing_close_tag_crlf_issue2000) {
CBMFileResult *r = extract("<?php\r\n"
"class Repo {\r\n"
" public function find($id) { return $id; }\r\n"
"}\r\n"
"?>\r\n",
CBM_LANG_PHP, "t", "Repo.php");
ASSERT_NOT_NULL(r);
ASSERT_FALSE(r->has_error);
ASSERT(has_def(r, "Class", "Repo"));
ASSERT(has_def(r, "Method", "find"));
ASSERT_EQ(php_def_line(r, "find"), 3);
ASSERT_FALSE(r->parse_incomplete);
cbm_free_result(r);
PASS();
}
/* --- Ruby --- */
TEST(ruby_class) {
CBMFileResult *r = extract("class Animal\n def initialize(name)\n @name = name\n end\n "
@@ -13125,6 +13249,10 @@ SUITE(extraction) {
RUN_TEST(python_class_base_extracted_bare);
RUN_TEST(php_class);
RUN_TEST(php_function);
RUN_TEST(php_inline_html_tail_reaches_graph_issue2000);
RUN_TEST(php_inline_html_template_shapes_issue2000);
RUN_TEST(php_inline_html_close_tag_in_literals_issue2000);
RUN_TEST(php_inline_html_trailing_close_tag_crlf_issue2000);
RUN_TEST(ruby_class);
RUN_TEST(ruby_module);
RUN_TEST(csharp_class);
+60
View File
@@ -1376,6 +1376,65 @@ TEST(pipeline_nix_scoped_binding_calls_resolve) {
PASS();
}
/* #2000, end to end: functions declared after an inline-HTML block reach the
* store with their CALLS edges, in the same file and across files. */
TEST(pipeline_php_inline_html_calls_resolve_issue2000) {
if (setup_test_repo() != 0) {
FAIL("failed to create temp dir");
}
static const struct {
const char *name;
const char *body;
} files[] = {
{"inline_html.php", "<?php\n"
"function gamma() { return 2; }\n"
"?>\n"
"<div class=\"box\">\n"
" <p>plain markup after the closing tag</p>\n"
"</div>\n"
"<?php\n"
"function delta() { return gamma(); }\n"},
{"price_helpers.php",
"<!-- shared helpers -->\n"
"<?php\n"
"function formatPrice($cents) { return number_format($cents / 100, 2); }\n"},
{"card_view.php", "<?php $title = 'Cards'; ?>\n"
"<h1><?= $title ?></h1>\n"
"<?php\n"
"function renderCard($item) { return formatPrice($item['cents']); }\n"},
};
for (size_t i = 0; i < sizeof(files) / sizeof(files[0]); i++) {
char path[512];
snprintf(path, sizeof(path), "%s/%s", g_tmpdir, files[i].name);
FILE *f = fopen(path, "w");
if (!f) {
teardown_test_repo();
FAIL("failed to write php fixture");
}
fputs(files[i].body, f);
fclose(f);
}
char db[512];
snprintf(db, sizeof(db), "%s/test_php_inline_html.db", g_tmpdir);
cbm_pipeline_t *p = cbm_pipeline_new(g_tmpdir, db, CBM_MODE_FULL);
ASSERT_NOT_NULL(p);
ASSERT_EQ(cbm_pipeline_run(p), 0);
cbm_store_t *s = cbm_store_open_path(db);
ASSERT_NOT_NULL(s);
const char *project = cbm_pipeline_project_name(p);
ASSERT(cross_file_call_exists(s, project, "delta", "gamma"));
ASSERT(cross_file_call_exists(s, project, "renderCard", "formatPrice"));
cbm_store_close(s);
cbm_pipeline_free(p);
teardown_test_repo();
PASS();
}
/* Terraform reference resolution, end to end.
*
* An HCL block names itself with its labels appended -- find_hcl_block_name
@@ -26475,6 +26534,7 @@ SUITE(pipeline) {
/* Calls pass */
RUN_TEST(pipeline_calls_resolution);
RUN_TEST(pipeline_nix_scoped_binding_calls_resolve);
RUN_TEST(pipeline_php_inline_html_calls_resolve_issue2000);
RUN_TEST(pipeline_hcl_block_reference_resolves_to_its_block);
RUN_TEST(pipeline_incremental_preserves_cross_file_calls);
RUN_TEST(pipeline_incremental_drops_terraform_reads_of_a_moved_module);