From 8b3534269cc29363d3bc491b4b63b3914784b3ca Mon Sep 17 00:00:00 2001 From: Martin Vogel Date: Wed, 25 Feb 2026 18:46:20 +0100 Subject: [PATCH] v0.1.3: Bug fixes, query improvements, cross-service linking, and graph enrichment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part A — Bug fixes: - Fix globToLike: **/dir/** now matches relative paths without leading / - Route nodes inherit file_path from handler (were invisible to file_pattern) - Compose YAML non-string env values (int/bool/float) no longer dropped - Compose secret filter: key names like JWT_PRIVATE_KEY_ID no longer filtered - Cloudbuild bash script deploy flags now parsed (entrypoint: bash) Part B — Query improvements: - search_graph exclude_labels parameter to filter noisy labels (e.g. Route) - Cypher edge property filtering: r.url_path, r.confidence, r.confidence_band Part C — Cross-service linking: - InfraFile env var URLs feed into HTTP linker as additional call sites - Default noise filtering: /health, /healthz, /ready, /metrics excluded - Confidence bands on HTTP_CALLS: high (>=0.7), medium (>=0.45), speculative (>=0.25) - Levenshtein distance and n-gram overlap similarity functions - Lowered match threshold from 0.45 to 0.25 (speculative band) Part D — Graph enrichment: - OVERRIDE edges: struct methods → interface methods they implement - USAGE edges: read references (callbacks, variable assignments) vs invocations - Generated column index on url_path for fast edge property lookups - Fuzzy name-only resolution fallback (5th strategy in call resolver) - Configurable HTTP linker via .cgrconfig (exclude paths, min confidence) - OpenTelemetry trace ingestion (ingest_traces tool) to validate HTTP_CALLS - Git history change coupling (FILE_CHANGES_WITH edges, co-change analysis) Updated tool descriptions and skill file with new edge types. --- internal/cypher/cypher_test.go | 258 ++++++++++++++++++++++++++ internal/httplink/config.go | 90 +++++++++ internal/httplink/config_test.go | 112 +++++++++++ internal/httplink/httplink.go | 75 ++++++-- internal/httplink/httplink_test.go | 3 + internal/httplink/similarity.go | 97 ++++++++++ internal/httplink/similarity_test.go | 86 +++++++++ internal/pipeline/envscan.go | 7 + internal/pipeline/githistory.go | 222 ++++++++++++++++++++++ internal/pipeline/githistory_test.go | 98 ++++++++++ internal/pipeline/implements.go | 74 ++++++-- internal/pipeline/implements_test.go | 169 +++++++++++++++++ internal/pipeline/infrascan_test.go | 105 ++++++++++- internal/pipeline/infrascan_yaml.go | 83 ++++++++- internal/pipeline/pipeline.go | 106 ++++++++++- internal/pipeline/resolver.go | 33 ++++ internal/pipeline/resolver_test.go | 82 +++++++++ internal/pipeline/usages.go | 238 ++++++++++++++++++++++++ internal/pipeline/usages_test.go | 167 +++++++++++++++++ internal/store/edges.go | 26 +++ internal/store/search.go | 20 +- internal/store/store.go | 23 ++- internal/store/store_test.go | 158 +++++++++++++++- internal/tools/search.go | 11 ++ internal/tools/tools.go | 16 +- internal/tools/traces.go | 51 ++++++ internal/traces/traces.go | 265 +++++++++++++++++++++++++++ internal/traces/traces_test.go | 168 +++++++++++++++++ skills/codebase-memory-mcp.md | 45 ++++- 29 files changed, 2837 insertions(+), 51 deletions(-) create mode 100644 internal/httplink/config.go create mode 100644 internal/httplink/config_test.go create mode 100644 internal/httplink/similarity.go create mode 100644 internal/httplink/similarity_test.go create mode 100644 internal/pipeline/githistory.go create mode 100644 internal/pipeline/githistory_test.go create mode 100644 internal/pipeline/implements_test.go create mode 100644 internal/pipeline/resolver_test.go create mode 100644 internal/pipeline/usages.go create mode 100644 internal/pipeline/usages_test.go create mode 100644 internal/tools/traces.go create mode 100644 internal/traces/traces.go create mode 100644 internal/traces/traces_test.go diff --git a/internal/cypher/cypher_test.go b/internal/cypher/cypher_test.go index b5b4cca6c..52e058ba8 100644 --- a/internal/cypher/cypher_test.go +++ b/internal/cypher/cypher_test.go @@ -724,3 +724,261 @@ func TestExecuteEdgeType(t *testing.T) { t.Errorf("r.type = %v, want HTTP_CALLS", result.Rows[0]["r.type"]) } } + +// --- Comprehensive edge property filtering tests --- + +// setupTestStoreMultiEdge creates a store with two HTTP_CALLS edges to test filtering. +func setupTestStoreMultiEdge(t *testing.T) *store.Store { + t.Helper() + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + + project := "testproj" + if err := s.UpsertProject(project, "/tmp/test"); err != nil { + t.Fatal(err) + } + + srcID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "SendOrder", + QualifiedName: "testproj.caller.SendOrder", + FilePath: "caller/client.go", + }) + + tgtID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "HandleOrder", + QualifiedName: "testproj.handler.HandleOrder", + FilePath: "handler/routes.go", + }) + + tgt2ID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "HandleHealth", + QualifiedName: "testproj.handler.HandleHealth", + FilePath: "handler/health.go", + }) + + mustInsertEdge(t, s, &store.Edge{ + Project: project, SourceID: srcID, TargetID: tgtID, + Type: "HTTP_CALLS", + Properties: map[string]any{ + "url_path": "/api/orders", + "confidence": 0.85, + "method": "POST", + }, + }) + + mustInsertEdge(t, s, &store.Edge{ + Project: project, SourceID: srcID, TargetID: tgt2ID, + Type: "HTTP_CALLS", + Properties: map[string]any{ + "url_path": "/health", + "confidence": 0.45, + }, + }) + + return s +} + +func TestEdgePropertyFilterContains(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path CONTAINS 'orders' RETURN a.name, b.name, r.url_path`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if len(result.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(result.Rows)) + } + + row := result.Rows[0] + if row["a.name"] != "SendOrder" { + t.Errorf("a.name = %v, want SendOrder", row["a.name"]) + } + if row["b.name"] != "HandleOrder" { + t.Errorf("b.name = %v, want HandleOrder", row["b.name"]) + } + if row["r.url_path"] != "/api/orders" { + t.Errorf("r.url_path = %v, want /api/orders", row["r.url_path"]) + } +} + +func TestEdgePropertyFilterNumericGTE(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence >= 0.6 RETURN a.name, b.name, r.confidence LIMIT 20`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if len(result.Rows) != 1 { + t.Fatalf("expected 1 row (only high-confidence edge), got %d", len(result.Rows)) + } + + row := result.Rows[0] + if row["b.name"] != "HandleOrder" { + t.Errorf("b.name = %v, want HandleOrder (high confidence)", row["b.name"]) + } +} + +func TestEdgePropertyReturnWithoutFilter(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence LIMIT 20`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if len(result.Rows) < 2 { + t.Fatalf("expected at least 2 rows, got %d", len(result.Rows)) + } + + foundOrders := false + foundHealth := false + for _, row := range result.Rows { + urlPath, _ := row["r.url_path"].(string) + if urlPath == "/api/orders" { + foundOrders = true + } + if urlPath == "/health" { + foundHealth = true + } + } + if !foundOrders { + t.Error("missing row with url_path=/api/orders") + } + if !foundHealth { + t.Error("missing row with url_path=/health") + } +} + +func TestEdgePropertyFilterEquals(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.method = 'POST' RETURN a.name, b.name`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if len(result.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(result.Rows)) + } + if result.Rows[0]["b.name"] != "HandleOrder" { + t.Errorf("b.name = %v, want HandleOrder", result.Rows[0]["b.name"]) + } +} + +func TestEdgePropertyFilterStartsWith(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path STARTS WITH '/api' RETURN a.name, b.name, r.url_path`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if len(result.Rows) != 1 { + t.Fatalf("expected 1 row (only /api/orders starts with /api), got %d", len(result.Rows)) + } + if result.Rows[0]["r.url_path"] != "/api/orders" { + t.Errorf("r.url_path = %v, want /api/orders", result.Rows[0]["r.url_path"]) + } +} + +func TestCombinedNodeAndEdgeFilter(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + // Filter on both node property (early) and edge property (late) + result, err := exec.Execute(`MATCH (a:Function)-[r:HTTP_CALLS]->(b:Function) WHERE a.name = 'SendOrder' AND r.confidence >= 0.6 RETURN b.name, r.url_path`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if len(result.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(result.Rows)) + } + if result.Rows[0]["b.name"] != "HandleOrder" { + t.Errorf("b.name = %v, want HandleOrder", result.Rows[0]["b.name"]) + } + if result.Rows[0]["r.url_path"] != "/api/orders" { + t.Errorf("r.url_path = %v, want /api/orders", result.Rows[0]["r.url_path"]) + } +} + +func TestEdgePropertyFilterNoMatch(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + // No edge has method = 'DELETE' + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.method = 'DELETE' RETURN a.name`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if len(result.Rows) != 0 { + t.Errorf("expected 0 rows, got %d", len(result.Rows)) + } +} + +func TestEdgePropertyFilterNumericLT(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + // Only the health edge (0.45) should match confidence < 0.5 + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence < 0.5 RETURN b.name, r.confidence`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if len(result.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(result.Rows)) + } + if result.Rows[0]["b.name"] != "HandleHealth" { + t.Errorf("b.name = %v, want HandleHealth", result.Rows[0]["b.name"]) + } +} + +func TestEdgePropertyFilterRegex(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + // Regex match on url_path + result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path =~ "/api/.*" RETURN b.name`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if len(result.Rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(result.Rows)) + } + if result.Rows[0]["b.name"] != "HandleOrder" { + t.Errorf("b.name = %v, want HandleOrder", result.Rows[0]["b.name"]) + } +} + +func TestEdgeBuiltinPropertyFilter(t *testing.T) { + s := setupTestStoreMultiEdge(t) + defer s.Close() + + exec := &Executor{Store: s} + // Filter on built-in edge property r.type + result, err := exec.Execute(`MATCH (a)-[r]->(b) WHERE r.type = 'HTTP_CALLS' RETURN a.name, b.name LIMIT 20`) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if len(result.Rows) != 2 { + t.Fatalf("expected 2 rows (both HTTP_CALLS edges), got %d", len(result.Rows)) + } +} diff --git a/internal/httplink/config.go b/internal/httplink/config.go new file mode 100644 index 000000000..e43857f71 --- /dev/null +++ b/internal/httplink/config.go @@ -0,0 +1,90 @@ +package httplink + +import ( + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// LinkerConfig holds user-overridable HTTP linker settings. +// Loaded from .cgrconfig in the project root. +type LinkerConfig struct { + HTTPLinker HTTPLinkerConfig `yaml:"http_linker"` +} + +// HTTPLinkerConfig holds HTTP linker-specific settings. +type HTTPLinkerConfig struct { + // ExcludePaths are route paths to exclude from HTTP_CALLS matching. + // These are added to (not replacing) the built-in defaultExcludePaths. + ExcludePaths []string `yaml:"exclude_paths"` + + // MinConfidence is the minimum confidence score for creating HTTP_CALLS edges. + // Default: 0.25 (includes speculative matches). + MinConfidence *float64 `yaml:"min_confidence"` + + // FuzzyMatching enables/disables fuzzy URL matching. + // Default: true. + FuzzyMatching *bool `yaml:"fuzzy_matching"` +} + +// DefaultConfig returns the default linker configuration. +func DefaultConfig() *LinkerConfig { + return &LinkerConfig{} +} + +// LoadConfig reads .cgrconfig from the given directory. +// Returns default config if the file doesn't exist. +func LoadConfig(dir string) *LinkerConfig { + cfg := DefaultConfig() + + path := filepath.Join(dir, ".cgrconfig") + data, err := os.ReadFile(path) + if err != nil { + return cfg // file not found or unreadable — use defaults + } + + if err := yaml.Unmarshal(data, cfg); err != nil { + return DefaultConfig() // invalid YAML — use defaults + } + + return cfg +} + +// EffectiveMinConfidence returns the configured minimum confidence, +// or the default (0.25) if not set. +func (c *LinkerConfig) EffectiveMinConfidence() float64 { + if c.HTTPLinker.MinConfidence != nil { + return *c.HTTPLinker.MinConfidence + } + return 0.25 +} + +// EffectiveFuzzyMatching returns the configured fuzzy matching setting, +// or the default (true) if not set. +func (c *LinkerConfig) EffectiveFuzzyMatching() bool { + if c.HTTPLinker.FuzzyMatching != nil { + return *c.HTTPLinker.FuzzyMatching + } + return true +} + +// AllExcludePaths returns the combined list of default + user-configured exclude paths. +func (c *LinkerConfig) AllExcludePaths() []string { + combined := make([]string, 0, len(defaultExcludePaths)+len(c.HTTPLinker.ExcludePaths)) + combined = append(combined, defaultExcludePaths...) + combined = append(combined, c.HTTPLinker.ExcludePaths...) + return combined +} + +// isPathExcluded checks if a route path matches any of the given exclusion paths. +func isPathExcluded(path string, excludePaths []string) bool { + normalized := strings.ToLower(strings.TrimRight(path, "/")) + for _, excluded := range excludePaths { + if strings.EqualFold(normalized, strings.TrimRight(excluded, "/")) { + return true + } + } + return false +} diff --git a/internal/httplink/config_test.go b/internal/httplink/config_test.go new file mode 100644 index 000000000..194d2adfa --- /dev/null +++ b/internal/httplink/config_test.go @@ -0,0 +1,112 @@ +package httplink + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadConfigDefault(t *testing.T) { + cfg := LoadConfig("/nonexistent/path") + if cfg.EffectiveMinConfidence() != 0.25 { + t.Errorf("expected default min_confidence 0.25, got %f", cfg.EffectiveMinConfidence()) + } + if !cfg.EffectiveFuzzyMatching() { + t.Error("expected default fuzzy_matching true") + } + paths := cfg.AllExcludePaths() + if len(paths) != len(defaultExcludePaths) { + t.Errorf("expected %d default paths, got %d", len(defaultExcludePaths), len(paths)) + } +} + +func TestLoadConfigFromFile(t *testing.T) { + dir := t.TempDir() + configContent := ` +http_linker: + exclude_paths: + - /debug + - /internal/status + min_confidence: 0.5 + fuzzy_matching: false +` + if err := os.WriteFile(filepath.Join(dir, ".cgrconfig"), []byte(configContent), 0o600); err != nil { + t.Fatal(err) + } + + cfg := LoadConfig(dir) + if cfg.EffectiveMinConfidence() != 0.5 { + t.Errorf("expected min_confidence 0.5, got %f", cfg.EffectiveMinConfidence()) + } + if cfg.EffectiveFuzzyMatching() { + t.Error("expected fuzzy_matching false") + } + paths := cfg.AllExcludePaths() + expectedLen := len(defaultExcludePaths) + 2 + if len(paths) != expectedLen { + t.Errorf("expected %d total paths, got %d", expectedLen, len(paths)) + } +} + +func TestLoadConfigInvalidYAML(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, ".cgrconfig"), []byte("not: [valid: yaml"), 0o600); err != nil { + t.Fatal(err) + } + cfg := LoadConfig(dir) + // Should fall back to defaults + if cfg.EffectiveMinConfidence() != 0.25 { + t.Errorf("expected default on invalid yaml, got %f", cfg.EffectiveMinConfidence()) + } +} + +func TestIsPathExcluded(t *testing.T) { + paths := []string{"/health", "/debug", "/internal/status"} + + tests := []struct { + path string + want bool + }{ + {"/health", true}, + {"/health/", true}, + {"/HEALTH", true}, + {"/debug", true}, + {"/internal/status", true}, + {"/api/orders", false}, + {"/healthcheck", false}, + } + for _, tt := range tests { + got := isPathExcluded(tt.path, paths) + if got != tt.want { + t.Errorf("isPathExcluded(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestAllExcludePathsMerge(t *testing.T) { + cfg := DefaultConfig() + cfg.HTTPLinker.ExcludePaths = []string{"/custom1", "/custom2"} + + paths := cfg.AllExcludePaths() + + // Should contain all defaults + custom + if len(paths) != len(defaultExcludePaths)+2 { + t.Errorf("expected %d paths, got %d", len(defaultExcludePaths)+2, len(paths)) + } + + // Verify defaults are first + for i, dp := range defaultExcludePaths { + if paths[i] != dp { + t.Errorf("paths[%d] = %q, want %q", i, paths[i], dp) + } + } + + // Verify custom paths are appended + customStart := len(defaultExcludePaths) + if paths[customStart] != "/custom1" { + t.Errorf("paths[%d] = %q, want /custom1", customStart, paths[customStart]) + } + if paths[customStart+1] != "/custom2" { + t.Errorf("paths[%d] = %q, want /custom2", customStart+1, paths[customStart+1]) + } +} diff --git a/internal/httplink/httplink.go b/internal/httplink/httplink.go index aa36d08e2..e22826552 100644 --- a/internal/httplink/httplink.go +++ b/internal/httplink/httplink.go @@ -44,14 +44,28 @@ type HTTPLink struct { // Linker discovers cross-service HTTP calls and creates HTTP_CALLS edges. type Linker struct { - store *store.Store - project string - routesByFunc map[string][]int // funcQN → indices into routes slice + store *store.Store + project string + config *LinkerConfig + routesByFunc map[string][]int // funcQN → indices into routes slice + extraCallSites []HTTPCallSite // injected from pipeline (e.g., InfraFile URLs) } // New creates a new HTTP Linker. func New(s *store.Store, project string) *Linker { - return &Linker{store: s, project: project} + return &Linker{store: s, project: project, config: DefaultConfig()} +} + +// SetConfig sets the linker configuration. If cfg is nil, defaults are used. +func (l *Linker) SetConfig(cfg *LinkerConfig) { + if cfg != nil { + l.config = cfg + } +} + +// AddCallSites allows the pipeline to inject additional call sites from infra files. +func (l *Linker) AddCallSites(sites []HTTPCallSite) { + l.extraCallSites = append(l.extraCallSites, sites...) } // regex patterns for route and URL discovery @@ -100,6 +114,13 @@ func (l *Linker) Run() ([]HTTPLink, error) { } rootPath := proj.RootPath + // Load config from project root if using defaults + if l.config.HTTPLinker.MinConfidence == nil && + l.config.HTTPLinker.FuzzyMatching == nil && + len(l.config.HTTPLinker.ExcludePaths) == 0 { + l.config = LoadConfig(rootPath) + } + l.routesByFunc = make(map[string][]int) routes := l.discoverRoutes(rootPath) slog.Info("httplink.routes", "count", len(routes)) @@ -119,6 +140,7 @@ func (l *Linker) Run() ([]HTTPLink, error) { l.insertRouteNodes(routes) callSites := l.discoverCallSites(rootPath) + callSites = append(callSites, l.extraCallSites...) slog.Info("httplink.callsites", "count", len(callSites)) links := l.matchAndLink(routes, callSites) @@ -131,7 +153,6 @@ func (l *Linker) Run() ([]HTTPLink, error) { // HANDLES edges from the handler function to the Route node. func (l *Linker) insertRouteNodes(routes []RouteHandler) { for _, rh := range routes { - // Build a stable qualified name for the Route node normalMethod := rh.Method if normalMethod == "" { normalMethod = "ANY" @@ -139,7 +160,6 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) { normalPath := strings.ReplaceAll(rh.Path, "/", "_") normalPath = strings.Trim(normalPath, "_") routeQN := rh.QualifiedName + ".route." + normalMethod + "." + normalPath - routeName := normalMethod + " " + rh.Path // Use resolved handler QN if available, otherwise fall back to registering function @@ -148,11 +168,21 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) { handlerQN = rh.ResolvedHandlerQN } + // Look up handler node BEFORE creating Route — we need its file_path + handlerNode, _ := l.store.FindNodeByQN(l.project, handlerQN) + + // Inherit file_path from handler function + var filePath string + if handlerNode != nil && handlerNode.FilePath != "" { + filePath = handlerNode.FilePath + } + routeID, err := l.store.UpsertNode(&store.Node{ Project: l.project, Label: "Route", Name: routeName, QualifiedName: routeQN, + FilePath: filePath, Properties: map[string]any{ "method": rh.Method, "path": rh.Path, @@ -164,7 +194,6 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) { } // Create HANDLES edge from actual handler → Route - handlerNode, _ := l.store.FindNodeByQN(l.project, handlerQN) if handlerNode != nil { if _, edgeErr := l.store.InsertEdge(&store.Edge{ Project: l.project, @@ -647,6 +676,17 @@ var externalDomains = []string{ "aws.amazon.com", } +// defaultExcludePaths are common utility endpoints that produce noise in HTTP_CALLS. +// These are excluded from route matching by default. +var defaultExcludePaths = []string{ + "/health", + "/healthz", + "/ready", + "/readyz", + "/metrics", + "/favicon.ico", +} + // isExternalDomain checks if a domain is a well-known external API. func isExternalDomain(domain string) bool { domain = strings.ToLower(domain) @@ -658,6 +698,11 @@ func isExternalDomain(domain string) bool { return false } +// ExtractURLPaths finds URL path segments from text (exported for use by pipeline). +func ExtractURLPaths(text string) []string { + return extractURLPaths(text) +} + // extractURLPaths finds URL path segments from text. func extractURLPaths(text string) []string { seen := map[string]bool{} @@ -825,6 +870,11 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [ continue } + // Skip noisy utility endpoints + if isPathExcluded(rh.Path, l.config.AllExcludePaths()) { + continue + } + // Multi-signal confidence scoring pathScore := pathMatchScore(cs.Path, rh.Path) if pathScore == 0 { @@ -832,7 +882,7 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [ } score := pathScore*sourceWeight(cs.SourceLabel) + methodBonus(cs.Method, rh.Method) - if score < matchConfidenceThreshold { + if score < l.config.EffectiveMinConfidence() { continue } if score > 1.0 { @@ -848,9 +898,11 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [ callerNode, _ := l.store.FindNodeByQN(l.project, cs.SourceQualifiedName) handlerNode, _ := l.store.FindNodeByQN(l.project, rh.QualifiedName) if callerNode != nil && handlerNode != nil { + band := confidenceBand(score) props := map[string]any{ - "url_path": cs.Path, - "confidence": score, + "url_path": cs.Path, + "confidence": score, + "confidence_band": band, } if rh.Method != "" { props["method"] = rh.Method @@ -895,7 +947,8 @@ func normalizePath(path string) string { } // matchConfidenceThreshold is the minimum score for an HTTP_CALLS edge. -const matchConfidenceThreshold = 0.45 +// Lowered from 0.45 to 0.25 to include speculative matches with confidence bands. +const matchConfidenceThreshold = 0.25 // pathMatchScore returns a confidence score (0.0–1.0) for how well callPath // matches routePath. Returns 0 if no match. diff --git a/internal/httplink/httplink_test.go b/internal/httplink/httplink_test.go index 7ac3d55d0..8cfc4822e 100644 --- a/internal/httplink/httplink_test.go +++ b/internal/httplink/httplink_test.go @@ -443,6 +443,9 @@ func TestRouteNodesCreated(t *testing.T) { t.Fatalf("expected 1 Route node, got %d", len(routeNodes)) } rn := routeNodes[0] + if rn.FilePath != "handler/routes.py" { + t.Errorf("Route file_path = %q, want 'handler/routes.py'", rn.FilePath) + } if rn.Name != "POST /api/orders" { t.Errorf("Route name = %q, want 'POST /api/orders'", rn.Name) } diff --git a/internal/httplink/similarity.go b/internal/httplink/similarity.go new file mode 100644 index 000000000..b8509156e --- /dev/null +++ b/internal/httplink/similarity.go @@ -0,0 +1,97 @@ +package httplink + +// levenshteinDistance computes the edit distance between two strings. +func levenshteinDistance(a, b string) int { + if a == "" { + return len(b) + } + if b == "" { + return len(a) + } + + // Use two rows instead of full matrix for space efficiency + prev := make([]int, len(b)+1) + curr := make([]int, len(b)+1) + + for j := range prev { + prev[j] = j + } + + for i := 1; i <= len(a); i++ { + curr[0] = i + for j := 1; j <= len(b); j++ { + cost := 1 + if a[i-1] == b[j-1] { + cost = 0 + } + curr[j] = min(curr[j-1]+1, min(prev[j]+1, prev[j-1]+cost)) + } + prev, curr = curr, prev + } + return prev[len(b)] +} + +// normalizedLevenshtein returns 1.0 - (distance / maxLen), so 1.0 = identical. +func normalizedLevenshtein(a, b string) float64 { + if a == b { + return 1.0 + } + maxLen := len(a) + if len(b) > maxLen { + maxLen = len(b) + } + if maxLen == 0 { + return 1.0 + } + dist := levenshteinDistance(a, b) + return 1.0 - float64(dist)/float64(maxLen) +} + +// ngramOverlap computes the character n-gram overlap coefficient between two strings. +// Returns |intersection(ngrams(a), ngrams(b))| / min(|ngrams(a)|, |ngrams(b)|). +func ngramOverlap(a, b string, n int) float64 { + if len(a) < n || len(b) < n { + return 0 + } + + ngramsA := buildNgrams(a, n) + ngramsB := buildNgrams(b, n) + + intersection := 0 + for ng := range ngramsA { + if ngramsB[ng] { + intersection++ + } + } + + minSize := len(ngramsA) + if len(ngramsB) < minSize { + minSize = len(ngramsB) + } + if minSize == 0 { + return 0 + } + return float64(intersection) / float64(minSize) +} + +func buildNgrams(s string, n int) map[string]bool { + ngrams := make(map[string]bool) + for i := 0; i <= len(s)-n; i++ { + ngrams[s[i:i+n]] = true + } + return ngrams +} + +// confidenceBand returns the confidence band label for a given score. +func confidenceBand(score float64) string { + switch { + case score >= 0.7: + return "high" + case score >= 0.45: + return "medium" + case score >= 0.25: + return "speculative" + default: + return "" + } +} diff --git a/internal/httplink/similarity_test.go b/internal/httplink/similarity_test.go new file mode 100644 index 000000000..c7e5d4f06 --- /dev/null +++ b/internal/httplink/similarity_test.go @@ -0,0 +1,86 @@ +package httplink + +import "testing" + +func TestLevenshteinDistance(t *testing.T) { + tests := []struct { + a, b string + want int + }{ + {"", "", 0}, + {"abc", "", 3}, + {"", "abc", 3}, + {"abc", "abc", 0}, + {"kitten", "sitting", 3}, + {"api/orders", "api/order", 1}, + {"/api/v1/orders", "/api/v2/orders", 1}, + } + for _, tt := range tests { + got := levenshteinDistance(tt.a, tt.b) + if got != tt.want { + t.Errorf("levenshteinDistance(%q, %q) = %d, want %d", tt.a, tt.b, got, tt.want) + } + } +} + +func TestNormalizedLevenshtein(t *testing.T) { + tests := []struct { + a, b string + min float64 + max float64 + }{ + {"abc", "abc", 1.0, 1.0}, + {"", "", 1.0, 1.0}, + {"api/orders", "api/order", 0.88, 0.92}, + {"/api/v1/items", "/api/v2/items", 0.90, 0.94}, + {"completely", "different", 0.0, 0.4}, + } + for _, tt := range tests { + got := normalizedLevenshtein(tt.a, tt.b) + if got < tt.min || got > tt.max { + t.Errorf("normalizedLevenshtein(%q, %q) = %.3f, want [%.2f, %.2f]", tt.a, tt.b, got, tt.min, tt.max) + } + } +} + +func TestNgramOverlap(t *testing.T) { + tests := []struct { + a, b string + n int + min float64 + max float64 + }{ + {"api/orders", "api/orders", 3, 1.0, 1.0}, + {"api/orders", "api/order", 3, 0.8, 1.0}, + {"abcdef", "ghijkl", 3, 0.0, 0.0}, + {"ab", "cd", 3, 0.0, 0.0}, // too short for trigrams + } + for _, tt := range tests { + got := ngramOverlap(tt.a, tt.b, tt.n) + if got < tt.min || got > tt.max { + t.Errorf("ngramOverlap(%q, %q, %d) = %.3f, want [%.2f, %.2f]", tt.a, tt.b, tt.n, got, tt.min, tt.max) + } + } +} + +func TestConfidenceBand(t *testing.T) { + tests := []struct { + score float64 + want string + }{ + {0.95, "high"}, + {0.70, "high"}, + {0.69, "medium"}, + {0.45, "medium"}, + {0.44, "speculative"}, + {0.25, "speculative"}, + {0.24, ""}, + {0.0, ""}, + } + for _, tt := range tests { + got := confidenceBand(tt.score) + if got != tt.want { + t.Errorf("confidenceBand(%.2f) = %q, want %q", tt.score, got, tt.want) + } + } +} diff --git a/internal/pipeline/envscan.go b/internal/pipeline/envscan.go index 6230adac8..d3ac3559f 100644 --- a/internal/pipeline/envscan.go +++ b/internal/pipeline/envscan.go @@ -191,6 +191,13 @@ func isSecretBinding(key, value string) bool { return false } +// isSecretValue checks only the value pattern, not the key name. +// Use for compose/infra files where key names like JWT_PRIVATE_KEY_ID +// are config references, not actual secrets. +func isSecretValue(value string) bool { + return secretValuePattern.MatchString(value) +} + func scanWithPatterns(absPath, relPath string, patterns []*regexp.Regexp) []EnvBinding { f, err := os.Open(absPath) if err != nil { diff --git a/internal/pipeline/githistory.go b/internal/pipeline/githistory.go new file mode 100644 index 000000000..f7b18fe30 --- /dev/null +++ b/internal/pipeline/githistory.go @@ -0,0 +1,222 @@ +package pipeline + +import ( + "bufio" + "fmt" + "log/slog" + "os/exec" + "sort" + "strings" + + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +// CommitFiles holds the files changed in a single commit. +type CommitFiles struct { + Hash string + Files []string +} + +// ChangeCoupling represents a pair of files that change together. +type ChangeCoupling struct { + FileA string + FileB string + CoChangeCount int + TotalChangesA int + TotalChangesB int + CouplingScore float64 +} + +// passGitHistory analyzes git log to find change coupling and creates FILE_CHANGES_WITH edges. +func (p *Pipeline) passGitHistory() { + slog.Info("pass6.githistory") + + commits, err := parseGitLog(p.RepoPath) + if err != nil { + slog.Warn("pass6.githistory.err", "err", err) + return + } + + if len(commits) == 0 { + slog.Info("pass6.githistory.skip", "reason", "no_commits") + return + } + + couplings := computeChangeCoupling(commits) + edgeCount := p.createCouplingEdges(couplings) + + slog.Info("pass6.githistory.done", "commits", len(commits), "couplings", len(couplings), "edges", edgeCount) +} + +// parseGitLog runs git log and extracts commit → files mapping. +func parseGitLog(repoPath string) ([]CommitFiles, error) { + cmd := exec.Command("git", "log", "--name-only", "--pretty=format:COMMIT:%H", "--since=6 months ago") + cmd.Dir = repoPath + + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git log: %w", err) + } + + var commits []CommitFiles + var current *CommitFiles + + scanner := bufio.NewScanner(strings.NewReader(string(output))) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if strings.HasPrefix(line, "COMMIT:") { + if current != nil && len(current.Files) > 0 { + commits = append(commits, *current) + } + current = &CommitFiles{Hash: strings.TrimPrefix(line, "COMMIT:")} + continue + } + if current != nil && isTrackableFile(line) { + current.Files = append(current.Files, line) + } + } + if current != nil && len(current.Files) > 0 { + commits = append(commits, *current) + } + + return commits, nil +} + +// isTrackableFile returns true if the file is a source file worth tracking. +func isTrackableFile(path string) bool { + skipPrefixes := []string{".git/", "node_modules/", "vendor/", "__pycache__/", ".cache/"} + for _, prefix := range skipPrefixes { + if strings.HasPrefix(path, prefix) { + return false + } + } + // Exact lock file names + skipNames := []string{"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "poetry.lock", "composer.lock", "Gemfile.lock", "Pipfile.lock"} + base := path[strings.LastIndex(path, "/")+1:] + for _, name := range skipNames { + if base == name { + return false + } + } + skipSuffixes := []string{".lock", ".sum", ".min.js", ".min.css", ".map", ".wasm", ".png", ".jpg", ".gif", ".ico", ".svg"} + for _, suffix := range skipSuffixes { + if strings.HasSuffix(path, suffix) { + return false + } + } + return true +} + +// computeChangeCoupling computes which file pairs change together frequently. +func computeChangeCoupling(commits []CommitFiles) []ChangeCoupling { + fileChangeCount := make(map[string]int) + pairCount := make(map[[2]string]int) + + for _, commit := range commits { + files := commit.Files + if len(files) > 20 { + continue // skip large merge/refactor commits + } + + for _, f := range files { + fileChangeCount[f]++ + } + + for i := 0; i < len(files); i++ { + for j := i + 1; j < len(files); j++ { + a, b := files[i], files[j] + if a > b { + a, b = b, a + } + pairCount[[2]string{a, b}]++ + } + } + } + + couplings := make([]ChangeCoupling, 0, len(pairCount)/2) + for pair, count := range pairCount { + if count < 3 { + continue + } + + totalA := fileChangeCount[pair[0]] + totalB := fileChangeCount[pair[1]] + minTotal := totalA + if totalB < minTotal { + minTotal = totalB + } + if minTotal == 0 { + continue + } + + score := float64(count) / float64(minTotal) + if score < 0.3 { + continue + } + + couplings = append(couplings, ChangeCoupling{ + FileA: pair[0], + FileB: pair[1], + CoChangeCount: count, + TotalChangesA: totalA, + TotalChangesB: totalB, + CouplingScore: score, + }) + } + + sort.Slice(couplings, func(i, j int) bool { + return couplings[i].CouplingScore > couplings[j].CouplingScore + }) + + if len(couplings) > 100 { + couplings = couplings[:100] + } + + return couplings +} + +// createCouplingEdges creates FILE_CHANGES_WITH edges for detected change couplings. +func (p *Pipeline) createCouplingEdges(couplings []ChangeCoupling) int { + count := 0 + for _, c := range couplings { + nodeA := p.findFileNode(c.FileA) + nodeB := p.findFileNode(c.FileB) + if nodeA == nil || nodeB == nil { + continue + } + + _, err := p.Store.InsertEdge(&store.Edge{ + Project: p.ProjectName, + SourceID: nodeA.ID, + TargetID: nodeB.ID, + Type: "FILE_CHANGES_WITH", + Properties: map[string]any{ + "co_change_count": c.CoChangeCount, + "total_changes_a": c.TotalChangesA, + "total_changes_b": c.TotalChangesB, + "coupling_score": c.CouplingScore, + }, + }) + if err == nil { + count++ + } + } + return count +} + +// findFileNode looks up the File node for a given relative path. +func (p *Pipeline) findFileNode(relPath string) *store.Node { + nodes, err := p.Store.FindNodesByFile(p.ProjectName, relPath) + if err != nil || len(nodes) == 0 { + return nil + } + for _, n := range nodes { + if n.Label == "File" { + return n + } + } + return nodes[0] +} diff --git a/internal/pipeline/githistory_test.go b/internal/pipeline/githistory_test.go new file mode 100644 index 000000000..679329307 --- /dev/null +++ b/internal/pipeline/githistory_test.go @@ -0,0 +1,98 @@ +package pipeline + +import ( + "fmt" + "testing" +) + +func TestIsTrackableFile(t *testing.T) { + tests := []struct { + path string + want bool + }{ + {"main.go", true}, + {"src/app.py", true}, + {"node_modules/foo/bar.js", false}, + {"vendor/lib/dep.go", false}, + {"package-lock.json", false}, + {"go.sum", false}, + {"image.png", false}, + {".git/config", false}, + {"__pycache__/mod.pyc", false}, + {"src/style.min.css", false}, + {"README.md", true}, + } + for _, tt := range tests { + if got := isTrackableFile(tt.path); got != tt.want { + t.Errorf("isTrackableFile(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestComputeChangeCoupling(t *testing.T) { + commits := []CommitFiles{ + {Hash: "aaa", Files: []string{"a.go", "b.go", "c.go"}}, + {Hash: "bbb", Files: []string{"a.go", "b.go"}}, + {Hash: "ccc", Files: []string{"a.go", "b.go"}}, + {Hash: "ddd", Files: []string{"a.go", "c.go"}}, + {Hash: "eee", Files: []string{"d.go", "e.go"}}, + } + + couplings := computeChangeCoupling(commits) + + found := false + for _, c := range couplings { + if (c.FileA == "a.go" && c.FileB == "b.go") || (c.FileA == "b.go" && c.FileB == "a.go") { + found = true + if c.CoChangeCount != 3 { + t.Errorf("expected 3 co-changes for a.go/b.go, got %d", c.CoChangeCount) + } + if c.CouplingScore < 0.9 { + t.Errorf("expected high coupling score, got %f", c.CouplingScore) + } + } + } + if !found { + t.Error("expected coupling between a.go and b.go") + } + + for _, c := range couplings { + if c.FileA == "d.go" || c.FileB == "d.go" { + t.Error("d.go should not appear (below threshold)") + } + } +} + +func TestComputeChangeCouplingSkipsLargeCommits(t *testing.T) { + files := make([]string, 25) + for i := range files { + files[i] = fmt.Sprintf("file%d.go", i) + } + commits := []CommitFiles{{Hash: "large", Files: files}} + + couplings := computeChangeCoupling(commits) + if len(couplings) != 0 { + t.Errorf("expected 0 couplings from large commit, got %d", len(couplings)) + } +} + +func TestComputeChangeCouplingLimitsTo100(t *testing.T) { + // Create many commits with overlapping files to generate >100 couplings + var commits []CommitFiles + for i := 0; i < 50; i++ { + for j := i + 1; j < 50; j++ { + // Create 3 commits per pair to exceed threshold + for k := 0; k < 3; k++ { + commits = append(commits, CommitFiles{ + Hash: fmt.Sprintf("c%d_%d_%d", i, j, k), + Files: []string{fmt.Sprintf("f%d.go", i), fmt.Sprintf("f%d.go", j)}, + }) + } + } + } + + couplings := computeChangeCoupling(commits) + if len(couplings) > 100 { + t.Errorf("expected max 100 couplings, got %d", len(couplings)) + } +} diff --git a/internal/pipeline/implements.go b/internal/pipeline/implements.go index 6d88766c3..b146a39de 100644 --- a/internal/pipeline/implements.go +++ b/internal/pipeline/implements.go @@ -7,10 +7,16 @@ import ( "github.com/DeusData/codebase-memory-mcp/internal/store" ) -// ifaceInfo holds an interface node and its required method names. +// ifaceMethodInfo holds a method name and its qualified name for OVERRIDE edge creation. +type ifaceMethodInfo struct { + name string + qualifiedName string +} + +// ifaceInfo holds an interface node and its required methods. type ifaceInfo struct { node *store.Node - methods []string + methods []ifaceMethodInfo } // passImplements detects Go interface satisfaction and creates IMPLEMENTS edges. @@ -24,9 +30,9 @@ func (p *Pipeline) passImplements() { } structMethods, structQNPrefix := p.collectStructMethods() - linkCount := p.matchImplements(ifaces, structMethods, structQNPrefix) + linkCount, overrideCount := p.matchImplements(ifaces, structMethods, structQNPrefix) - slog.Info("pass5.implements.done", "links", linkCount) + slog.Info("pass5.implements.done", "links", linkCount, "overrides", overrideCount) } // collectGoInterfaces returns Go interfaces with their method names. @@ -47,16 +53,19 @@ func (p *Pipeline) collectGoInterfaces() []ifaceInfo { continue } - var methodNames []string + var methods []ifaceMethodInfo for _, e := range edges { methodNode, _ := p.Store.FindNodeByID(e.TargetID) if methodNode != nil { - methodNames = append(methodNames, methodNode.Name) + methods = append(methods, ifaceMethodInfo{ + name: methodNode.Name, + qualifiedName: methodNode.QualifiedName, + }) } } - if len(methodNames) > 0 { - ifaces = append(ifaces, ifaceInfo{node: iface, methods: methodNames}) + if len(methods) > 0 { + ifaces = append(ifaces, ifaceInfo{node: iface, methods: methods}) } } return ifaces @@ -105,13 +114,12 @@ func (p *Pipeline) collectStructMethods() (structMethods map[string]map[string]b return } -// matchImplements checks each struct against each interface and creates IMPLEMENTS edges. +// matchImplements checks each struct against each interface and creates IMPLEMENTS + OVERRIDE edges. func (p *Pipeline) matchImplements( ifaces []ifaceInfo, structMethods map[string]map[string]bool, structQNPrefix map[string]string, -) int { - linkCount := 0 +) (linkCount, overrideCount int) { for _, iface := range ifaces { for typeName, methodSet := range structMethods { if !satisfies(iface.methods, methodSet) { @@ -130,9 +138,47 @@ func (p *Pipeline) matchImplements( Type: "IMPLEMENTS", }) linkCount++ + + overrideCount += p.createOverrideEdges(iface.methods, typeName, structQNPrefix) } } - return linkCount + return linkCount, overrideCount +} + +// createOverrideEdges creates OVERRIDE edges from struct methods to interface methods. +func (p *Pipeline) createOverrideEdges( + ifaceMethods []ifaceMethodInfo, + typeName string, + structQNPrefix map[string]string, +) int { + prefix, ok := structQNPrefix[typeName] + if !ok { + return 0 + } + + count := 0 + for _, im := range ifaceMethods { + // prefix already includes the type name (e.g., "pkg.FileReader" from "pkg.FileReader.Read") + structMethodQN := prefix + "." + im.name + structMethodNode, _ := p.Store.FindNodeByQN(p.ProjectName, structMethodQN) + if structMethodNode == nil { + continue + } + + ifaceMethodNode, _ := p.Store.FindNodeByQN(p.ProjectName, im.qualifiedName) + if ifaceMethodNode == nil { + continue + } + + _, _ = p.Store.InsertEdge(&store.Edge{ + Project: p.ProjectName, + SourceID: structMethodNode.ID, + TargetID: ifaceMethodNode.ID, + Type: "OVERRIDE", + }) + count++ + } + return count } // findStructNode looks up the struct/class node for a given receiver type name. @@ -169,9 +215,9 @@ func extractReceiverType(recv string) string { } // satisfies checks if a set of method names includes all interface methods. -func satisfies(ifaceMethods []string, structMethodSet map[string]bool) bool { +func satisfies(ifaceMethods []ifaceMethodInfo, structMethodSet map[string]bool) bool { for _, m := range ifaceMethods { - if !structMethodSet[m] { + if !structMethodSet[m.name] { return false } } diff --git a/internal/pipeline/implements_test.go b/internal/pipeline/implements_test.go new file mode 100644 index 000000000..83f1ff6b1 --- /dev/null +++ b/internal/pipeline/implements_test.go @@ -0,0 +1,169 @@ +package pipeline + +import ( + "testing" + + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +func TestPassImplementsCreatesOverrideEdges(t *testing.T) { + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + project := "test-proj" + _ = s.UpsertProject(project, "/tmp/test") + + // Create interface node + ifaceID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Interface", Name: "Reader", + QualifiedName: "pkg.Reader", FilePath: "pkg/reader.go", + }) + + // Create interface method nodes + readMethodID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Method", Name: "Read", + QualifiedName: "pkg.Reader.Read", FilePath: "pkg/reader.go", + }) + closeMethodID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Method", Name: "Close", + QualifiedName: "pkg.Reader.Close", FilePath: "pkg/reader.go", + }) + + // DEFINES_METHOD edges (interface → method) + _, _ = s.InsertEdge(&store.Edge{ + Project: project, SourceID: ifaceID, TargetID: readMethodID, Type: "DEFINES_METHOD", + }) + _, _ = s.InsertEdge(&store.Edge{ + Project: project, SourceID: ifaceID, TargetID: closeMethodID, Type: "DEFINES_METHOD", + }) + + // Create struct (Class) node + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Class", Name: "FileReader", + QualifiedName: "pkg.FileReader", FilePath: "pkg/filereader.go", + }) + + // Create struct method nodes with receiver property + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Method", Name: "Read", + QualifiedName: "pkg.FileReader.Read", FilePath: "pkg/filereader.go", + Properties: map[string]any{"receiver": "(f *FileReader)"}, + }) + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Method", Name: "Close", + QualifiedName: "pkg.FileReader.Close", FilePath: "pkg/filereader.go", + Properties: map[string]any{"receiver": "(f *FileReader)"}, + }) + + // Run passImplements + p := &Pipeline{ + Store: s, + ProjectName: project, + } + p.passImplements() + + // Verify IMPLEMENTS edge exists (struct → interface) + structNode, _ := s.FindNodeByQN(project, "pkg.FileReader") + if structNode == nil { + t.Fatal("struct node not found") + } + implEdges, _ := s.FindEdgesBySourceAndType(structNode.ID, "IMPLEMENTS") + if len(implEdges) != 1 { + t.Fatalf("expected 1 IMPLEMENTS edge, got %d", len(implEdges)) + } + if implEdges[0].TargetID != ifaceID { + t.Errorf("IMPLEMENTS target should be interface, got %d", implEdges[0].TargetID) + } + + // Verify OVERRIDE edges (struct_method → interface_method) + structReadNode, _ := s.FindNodeByQN(project, "pkg.FileReader.Read") + if structReadNode == nil { + t.Fatal("struct Read method not found") + } + readOverrides, _ := s.FindEdgesBySourceAndType(structReadNode.ID, "OVERRIDE") + if len(readOverrides) != 1 { + t.Fatalf("expected 1 OVERRIDE edge for Read, got %d", len(readOverrides)) + } + if readOverrides[0].TargetID != readMethodID { + t.Errorf("OVERRIDE target should be interface Read method") + } + + structCloseNode, _ := s.FindNodeByQN(project, "pkg.FileReader.Close") + if structCloseNode == nil { + t.Fatal("struct Close method not found") + } + closeOverrides, _ := s.FindEdgesBySourceAndType(structCloseNode.ID, "OVERRIDE") + if len(closeOverrides) != 1 { + t.Fatalf("expected 1 OVERRIDE edge for Close, got %d", len(closeOverrides)) + } + if closeOverrides[0].TargetID != closeMethodID { + t.Errorf("OVERRIDE target should be interface Close method") + } +} + +func TestPassImplementsNoOverrideWithoutMatch(t *testing.T) { + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + project := "test-proj" + _ = s.UpsertProject(project, "/tmp/test") + + // Interface with 2 methods + ifaceID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Interface", Name: "ReadWriter", + QualifiedName: "pkg.ReadWriter", FilePath: "pkg/rw.go", + }) + readID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Method", Name: "Read", + QualifiedName: "pkg.ReadWriter.Read", FilePath: "pkg/rw.go", + }) + writeID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Method", Name: "Write", + QualifiedName: "pkg.ReadWriter.Write", FilePath: "pkg/rw.go", + }) + _, _ = s.InsertEdge(&store.Edge{ + Project: project, SourceID: ifaceID, TargetID: readID, Type: "DEFINES_METHOD", + }) + _, _ = s.InsertEdge(&store.Edge{ + Project: project, SourceID: ifaceID, TargetID: writeID, Type: "DEFINES_METHOD", + }) + + // Struct with only Read (missing Write) — should NOT implement + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Class", Name: "OnlyReader", + QualifiedName: "pkg.OnlyReader", FilePath: "pkg/onlyreader.go", + }) + _, _ = s.UpsertNode(&store.Node{ + Project: project, Label: "Method", Name: "Read", + QualifiedName: "pkg.OnlyReader.Read", FilePath: "pkg/onlyreader.go", + Properties: map[string]any{"receiver": "(o *OnlyReader)"}, + }) + + p := &Pipeline{Store: s, ProjectName: project} + p.passImplements() + + // No IMPLEMENTS or OVERRIDE edges should exist + structNode, _ := s.FindNodeByQN(project, "pkg.OnlyReader") + if structNode == nil { + t.Fatal("struct not found") + } + implEdges, _ := s.FindEdgesBySourceAndType(structNode.ID, "IMPLEMENTS") + if len(implEdges) != 0 { + t.Errorf("expected 0 IMPLEMENTS edges, got %d", len(implEdges)) + } + + methodNode, _ := s.FindNodeByQN(project, "pkg.OnlyReader.Read") + if methodNode == nil { + t.Fatal("method not found") + } + overrideEdges, _ := s.FindEdgesBySourceAndType(methodNode.ID, "OVERRIDE") + if len(overrideEdges) != 0 { + t.Errorf("expected 0 OVERRIDE edges, got %d", len(overrideEdges)) + } +} diff --git a/internal/pipeline/infrascan_test.go b/internal/pipeline/infrascan_test.go index bfaae7913..8cc3af56b 100644 --- a/internal/pipeline/infrascan_test.go +++ b/internal/pipeline/infrascan_test.go @@ -147,7 +147,7 @@ services: - "8080:8080" environment: PORT: "8080" - JWT_SECRET: "should-be-filtered" + JWT_SECRET: "sk-1234567890abcdefghijkl" depends_on: - db networks: @@ -875,6 +875,109 @@ func TestInfraQN(t *testing.T) { } } +// --- Bug fix regression tests --- + +func TestParseComposeNonStringEnvValues(t *testing.T) { + content := ` +services: + auth-service: + image: auth:latest + environment: + PORT: 8090 + DEBUG: true + RATE_LIMIT: 1.5 + NAME: "string-value" +` + path := writeTempFile(t, "docker-compose.yml", content) + results := parseComposeFile(path, "docker-compose.yml") + if len(results) != 1 { + t.Fatalf("expected 1 service, got %d", len(results)) + } + + envVars := requireMapStringString(t, results[0].properties["environment"]) + assertEqual(t, envVars["PORT"], "8090") + assertEqual(t, envVars["DEBUG"], "true") + assertEqual(t, envVars["RATE_LIMIT"], "1.5") + assertEqual(t, envVars["NAME"], "string-value") +} + +func TestParseComposeSecretKeyNotFiltered(t *testing.T) { + // Key names like JWT_PRIVATE_KEY_ID are config references, not secrets. + // Only actual secret VALUES should be filtered. + content := ` +services: + auth-service: + image: auth:latest + environment: + JWT_PRIVATE_KEY_ID: jwt-private-key + GOOGLE_CLIENT_SECRET_ID: google-oauth-secret + ACCESS_TOKEN_TTL: 15m + ACTUAL_SECRET: "sk-1234567890abcdef12345" +` + path := writeTempFile(t, "docker-compose.yml", content) + results := parseComposeFile(path, "docker-compose.yml") + if len(results) != 1 { + t.Fatalf("expected 1 service, got %d", len(results)) + } + + envVars := requireMapStringString(t, results[0].properties["environment"]) + // These should NOT be filtered — key name matches pattern but value is a reference + assertEqual(t, envVars["JWT_PRIVATE_KEY_ID"], "jwt-private-key") + assertEqual(t, envVars["GOOGLE_CLIENT_SECRET_ID"], "google-oauth-secret") + assertEqual(t, envVars["ACCESS_TOKEN_TTL"], "15m") + + // This SHOULD be filtered — value matches secret pattern (sk-...) + if _, ok := envVars["ACTUAL_SECRET"]; ok { + t.Error("ACTUAL_SECRET should be filtered (value matches secret pattern)") + } +} + +func TestParseCloudbuildBashScript(t *testing.T) { + content := ` +substitutions: + _SERVICE_NAME: my-service + +steps: + - name: gcr.io/cloud-builders/docker + args: ['build', '-t', 'eu.gcr.io/$PROJECT_ID/$_SERVICE_NAME', '.'] + + - name: gcr.io/google.com/cloudsdktool/cloud-sdk + entrypoint: bash + args: + - "-c" + - | + gcloud run deploy $_SERVICE_NAME \ + --region=europe-west3 \ + --memory=2Gi \ + --cpu=2 \ + --concurrency=80 \ + --max-instances=10 \ + --ingress=internal-and-cloud-load-balancing \ + --set-env-vars=LOG_LEVEL=INFO,SOURCE_ENTITY=$_SERVICE_NAME + +images: + - eu.gcr.io/$PROJECT_ID/$_SERVICE_NAME +` + path := writeTempFile(t, "cloudbuild.yaml", content) + results := parseCloudbuildFile(path, "cloudbuild.yaml") + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + + props := results[0].properties + assertEqual(t, props["service_name"], "my-service") + assertEqual(t, props["deploy_region"], "europe-west3") + assertEqual(t, props["deploy_memory"], "2Gi") + assertEqual(t, props["deploy_cpu"], "2") + assertEqual(t, props["deploy_concurrency"], "80") + assertEqual(t, props["deploy_max_instances"], "10") + assertEqual(t, props["deploy_ingress"], "internal-and-cloud-load-balancing") + + envVars := requireMapStringString(t, props["deploy_env_vars"]) + assertEqual(t, envVars["LOG_LEVEL"], "INFO") + assertEqual(t, envVars["SOURCE_ENTITY"], "$_SERVICE_NAME") +} + // --- helpers --- func writeTempFile(t *testing.T, name, content string) string { diff --git a/internal/pipeline/infrascan_yaml.go b/internal/pipeline/infrascan_yaml.go index 57ddb6445..4b5220b6c 100644 --- a/internal/pipeline/infrascan_yaml.go +++ b/internal/pipeline/infrascan_yaml.go @@ -2,6 +2,8 @@ package pipeline import ( "os" + "regexp" + "strconv" "strings" "gopkg.in/yaml.v3" @@ -105,7 +107,20 @@ func parseComposeEnvironment(env any) map[string]string { switch v := env.(type) { case map[string]any: for k, val := range v { - if s, ok := val.(string); ok && !isSecretBinding(k, s) { + var s string + switch tv := val.(type) { + case string: + s = tv + case int: + s = strconv.Itoa(tv) + case float64: + s = strconv.FormatFloat(tv, 'f', -1, 64) + case bool: + s = strconv.FormatBool(tv) + default: + continue + } + if !isSecretValue(s) { result[k] = s } } @@ -116,7 +131,7 @@ func parseComposeEnvironment(env any) map[string]string { continue } parts := strings.SplitN(s, "=", 2) - if len(parts) == 2 && !isSecretBinding(parts[0], parts[1]) { + if len(parts) == 2 && !isSecretValue(parts[1]) { result[parts[0]] = parts[1] } } @@ -187,9 +202,10 @@ type cloudbuildFile struct { } type cloudbuildStep struct { - Name string `yaml:"name"` - Args []string `yaml:"args"` - Env []string `yaml:"env"` + Name string `yaml:"name"` + Entrypoint string `yaml:"entrypoint"` + Args []string `yaml:"args"` + Env []string `yaml:"env"` } func parseCloudbuildFile(absPath, relPath string) []infraFile { @@ -234,12 +250,59 @@ func buildCloudbuildProps(cb cloudbuildFile) map[string]any { // and extracts deployment flags. func extractDeployFlags(steps []cloudbuildStep, props map[string]any) { for _, step := range steps { - if !isDeployStep(step.Args) { - continue + // Direct args format: ["gcloud", "run", "deploy", "--flag=val", ...] + if isDeployStep(step.Args) { + parseDeployArgs(step.Args, props) + parseDeployEnvVars(step.Args, props) + return } - parseDeployArgs(step.Args, props) - parseDeployEnvVars(step.Args, props) - return // only process first deploy step + // Bash script format: entrypoint=bash, args=["-c", "gcloud run deploy ..."] + if isBashDeployStep(&step) { + parseBashDeployScript(step.Args[1], props) + return + } + } +} + +// isBashDeployStep checks if a step is a bash entrypoint with a gcloud run deploy script. +func isBashDeployStep(step *cloudbuildStep) bool { + isBash := step.Entrypoint == "bash" || step.Entrypoint == "/bin/bash" || step.Entrypoint == "/bin/sh" + if !isBash && len(step.Args) >= 2 { + isBash = step.Args[0] == "-c" + } + if !isBash || len(step.Args) < 2 { + return false + } + script := step.Args[len(step.Args)-1] + return strings.Contains(script, "gcloud") && + strings.Contains(script, "run") && + strings.Contains(script, "deploy") +} + +// parseBashDeployScript extracts deploy flags from a bash script string. +func parseBashDeployScript(script string, props map[string]any) { + // Normalize line continuations + normalized := strings.ReplaceAll(script, "\\\n", " ") + + // Extract flags using the same flag map as parseDeployArgs + for flag, propName := range deployFlagMap { + re := regexp.MustCompile(regexp.QuoteMeta(flag) + `[= ]+([^\s\\]+)`) + if m := re.FindStringSubmatch(normalized); len(m) == 2 { + props[propName] = m[1] + } + } + + // Extract env vars from --set-env-vars or --update-env-vars + envRe := regexp.MustCompile(`--(?:set|update)-env-vars[= ]+([^\s\\]+(?:,[^\s\\]+)*)`) + if m := envRe.FindStringSubmatch(normalized); len(m) == 2 { + envMap := make(map[string]string) + for _, pair := range strings.Split(m[1], ",") { + parts := strings.SplitN(pair, "=", 2) + if len(parts) == 2 && !isSecretBinding(parts[0], parts[1]) { + envMap[parts[0]] = parts[1] + } + } + setNonEmptyMap(props, "deploy_env_vars", envMap) } } diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 9c38f82ac..da8f3cced 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -109,11 +109,13 @@ func (p *Pipeline) runFullPasses(files []discover.FileInfo) error { p.buildRegistry() p.passImports() p.passCalls() + p.passUsages() p.cleanupASTCache() if err := p.passHTTPLinks(); err != nil { slog.Warn("pass4.httplink.err", "err", err) } p.passImplements() + p.passGitHistory() p.updateFileHashes(files) return nil } @@ -156,13 +158,15 @@ func (p *Pipeline) runIncrementalPasses( filesToResolve := mergeFiles(changed, dependents) slog.Info("incremental.resolve", "changed", len(changed), "dependents", len(dependents)) - // Delete CALLS edges for files being re-resolved + // Delete CALLS and USAGE edges for files being re-resolved for _, f := range filesToResolve { _ = p.Store.DeleteEdgesBySourceFile(p.ProjectName, f.RelPath, "CALLS") + _ = p.Store.DeleteEdgesBySourceFile(p.ProjectName, f.RelPath, "USAGE") } - // Pass 3: Re-resolve calls for changed + dependent files + // Pass 3: Re-resolve calls + usages for changed + dependent files p.passCallsForFiles(filesToResolve) + p.passUsagesForFiles(filesToResolve) p.cleanupASTCache() @@ -171,6 +175,7 @@ func (p *Pipeline) runIncrementalPasses( slog.Warn("pass4.httplink.err", "err", err) } p.passImplements() + p.passGitHistory() p.updateFileHashes(allFiles) return nil @@ -877,6 +882,10 @@ func (p *Pipeline) processFileCalls(relPath string, cached *cachedAST, spec *lan targetQN := p.resolveCallWithTypes(calleeName, moduleQN, importMap, localTypeMap) if targetQN == "" { + // Strategy 5: Fuzzy name-only fallback + if fuzzyQN, ok := p.registry.FuzzyResolve(calleeName, moduleQN); ok { + p.createFuzzyCallEdge(callerQN, fuzzyQN) + } return false } @@ -947,6 +956,25 @@ func (p *Pipeline) createCallEdge(callerQN, targetQN string) { } } +// createFuzzyCallEdge creates a CALLS edge with resolution_mode="fuzzy" property, +// indicating the callee was resolved via text-based name matching rather than +// structural analysis (imports, same-module, etc.). +func (p *Pipeline) createFuzzyCallEdge(callerQN, targetQN string) { + callerNode, _ := p.Store.FindNodeByQN(p.ProjectName, callerQN) + targetNode, _ := p.Store.FindNodeByQN(p.ProjectName, targetQN) + if callerNode != nil && targetNode != nil { + _, _ = p.Store.InsertEdge(&store.Edge{ + Project: p.ProjectName, + SourceID: callerNode.ID, + TargetID: targetNode.ID, + Type: "CALLS", + Properties: map[string]any{ + "resolution_mode": "fuzzy", + }, + }) + } +} + // resolveCallWithTypes resolves a callee name using the registry, import maps, // and type inference for method dispatch. func (p *Pipeline) resolveCallWithTypes( @@ -1197,6 +1225,14 @@ func (p *Pipeline) passHTTPLinks() error { } linker := httplink.New(p.Store, p.ProjectName) + + // Feed InfraFile environment URLs into the HTTP linker + infraSites := p.extractInfraCallSites() + if len(infraSites) > 0 { + linker.AddCallSites(infraSites) + slog.Info("pass4.infra_callsites", "count", len(infraSites)) + } + links, err := linker.Run() if err != nil { return err @@ -1205,6 +1241,72 @@ func (p *Pipeline) passHTTPLinks() error { return nil } +// extractInfraCallSites extracts URL values from InfraFile environment properties +// and converts them to HTTPCallSite entries for the HTTP linker. +func (p *Pipeline) extractInfraCallSites() []httplink.HTTPCallSite { + infraNodes, err := p.Store.FindNodesByLabel(p.ProjectName, "InfraFile") + if err != nil { + return nil + } + + var sites []httplink.HTTPCallSite + for _, node := range infraNodes { + // InfraFile nodes use different property keys depending on source: + // compose files: "environment", Dockerfiles/shell/.env: "env_vars", + // cloudbuild: "deploy_env_vars" + for _, envKey := range []string{"environment", "env_vars", "deploy_env_vars"} { + sites = append(sites, extractEnvURLSites(node, envKey)...) + } + } + return sites +} + +// extractEnvURLSites extracts HTTP call sites from a single env property of an InfraFile node. +func extractEnvURLSites(node *store.Node, propKey string) []httplink.HTTPCallSite { + env, ok := node.Properties[propKey] + if !ok { + return nil + } + + // env_vars are stored as map[string]string (from Go), but after JSON round-trip + // through SQLite they come back as map[string]any. + var sites []httplink.HTTPCallSite + switch envMap := env.(type) { + case map[string]any: + for _, val := range envMap { + valStr, ok := val.(string) + if !ok { + continue + } + sites = append(sites, urlSitesFromValue(node, valStr)...) + } + case map[string]string: + for _, valStr := range envMap { + sites = append(sites, urlSitesFromValue(node, valStr)...) + } + } + return sites +} + +// urlSitesFromValue extracts URL paths from a string value and creates HTTPCallSite entries. +func urlSitesFromValue(node *store.Node, val string) []httplink.HTTPCallSite { + if !strings.Contains(val, "http://") && !strings.Contains(val, "https://") && !strings.HasPrefix(val, "/") { + return nil + } + + paths := httplink.ExtractURLPaths(val) + sites := make([]httplink.HTTPCallSite, 0, len(paths)) + for _, path := range paths { + sites = append(sites, httplink.HTTPCallSite{ + Path: path, + SourceName: node.Name, + SourceQualifiedName: node.QualifiedName, + SourceLabel: "InfraFile", + }) + } + return sites +} + // injectEnvBindings creates or updates Module nodes for config files that contain // environment variable URL bindings. These synthetic constants feed into the // HTTP linker's call site discovery. diff --git a/internal/pipeline/resolver.go b/internal/pipeline/resolver.go index 4de9a188e..fddbca7dd 100644 --- a/internal/pipeline/resolver.go +++ b/internal/pipeline/resolver.go @@ -150,6 +150,39 @@ func (r *FunctionRegistry) resolveViaNameLookup(calleeName, suffix, moduleQN str return "" } +// FuzzyResolve attempts a loose match when Resolve() returns "". +// It searches for any registered function whose simple name matches the callee's +// last name segment. Returns the best match (by import distance) and true, +// or "" and false if no match is found. +// +// Unlike Resolve(), this does not require prefix/import agreement — it purely +// matches on the function name. Results should be marked as "fuzzy" resolution. +func (r *FunctionRegistry) FuzzyResolve(calleeName, moduleQN string) (string, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + // Extract the simple name (last segment after dots) + lookupName := simpleName(calleeName) + candidates := r.byName[lookupName] + + if len(candidates) == 0 { + return "", false + } + + // If there's exactly one candidate, use it + if len(candidates) == 1 { + return candidates[0], true + } + + // Multiple candidates: pick best by import distance + best := bestByImportDistance(candidates, moduleQN) + if best != "" { + return best, true + } + + return "", false +} + // FindByName returns all qualified names with the given simple name. func (r *FunctionRegistry) FindByName(name string) []string { r.mu.RLock() diff --git a/internal/pipeline/resolver_test.go b/internal/pipeline/resolver_test.go new file mode 100644 index 000000000..a6db06c9d --- /dev/null +++ b/internal/pipeline/resolver_test.go @@ -0,0 +1,82 @@ +package pipeline + +import "testing" + +func TestFuzzyResolve_SingleCandidate(t *testing.T) { + reg := NewFunctionRegistry() + reg.Register("CreateOrder", "svcA.handlers.CreateOrder", "Function") + reg.Register("ValidateOrder", "svcB.validators.ValidateOrder", "Function") + + // Normal resolve with no import map should find unique name + result := reg.Resolve("CreateOrder", "svcC.caller", nil) + if result != "svcA.handlers.CreateOrder" { + t.Errorf("Resolve: expected svcA.handlers.CreateOrder, got %s", result) + } + + // FuzzyResolve should find by simple name even with unknown prefix + fuzzyResult, ok := reg.FuzzyResolve("unknownPkg.CreateOrder", "svcC.caller") + if !ok { + t.Fatal("expected fuzzy match") + } + if fuzzyResult != "svcA.handlers.CreateOrder" { + t.Errorf("expected svcA.handlers.CreateOrder, got %s", fuzzyResult) + } +} + +func TestFuzzyResolve_NonExistentName(t *testing.T) { + reg := NewFunctionRegistry() + reg.Register("CreateOrder", "svcA.handlers.CreateOrder", "Function") + + _, ok := reg.FuzzyResolve("NonExistent", "svcC.caller") + if ok { + t.Fatal("expected no fuzzy match for non-existent name") + } +} + +func TestFuzzyResolve_MultipleCandidates_BestByDistance(t *testing.T) { + reg := NewFunctionRegistry() + reg.Register("Process", "svcA.handlers.Process", "Function") + reg.Register("Process", "svcB.handlers.Process", "Function") + + // Caller is in svcA — should prefer svcA.handlers.Process + fuzzyResult, ok := reg.FuzzyResolve("unknown.Process", "svcA.other") + if !ok { + t.Fatal("expected fuzzy match") + } + if fuzzyResult != "svcA.handlers.Process" { + t.Errorf("expected svcA.handlers.Process, got %s", fuzzyResult) + } + + // Caller is in svcB — should prefer svcB.handlers.Process + fuzzyResult, ok = reg.FuzzyResolve("unknown.Process", "svcB.other") + if !ok { + t.Fatal("expected fuzzy match") + } + if fuzzyResult != "svcB.handlers.Process" { + t.Errorf("expected svcB.handlers.Process, got %s", fuzzyResult) + } +} + +func TestFuzzyResolve_SimpleNameExtraction(t *testing.T) { + reg := NewFunctionRegistry() + reg.Register("DoWork", "myproject.utils.DoWork", "Function") + + // Deeply qualified callee — should extract "DoWork" as simple name + fuzzyResult, ok := reg.FuzzyResolve("some.deep.module.DoWork", "myproject.caller") + if !ok { + t.Fatal("expected fuzzy match") + } + if fuzzyResult != "myproject.utils.DoWork" { + t.Errorf("expected myproject.utils.DoWork, got %s", fuzzyResult) + } +} + +func TestFuzzyResolve_NoMatchForBareName(t *testing.T) { + reg := NewFunctionRegistry() + // Register nothing + + _, ok := reg.FuzzyResolve("SomeFunc", "myproject.caller") + if ok { + t.Fatal("expected no fuzzy match on empty registry") + } +} diff --git a/internal/pipeline/usages.go b/internal/pipeline/usages.go new file mode 100644 index 000000000..2e5195f27 --- /dev/null +++ b/internal/pipeline/usages.go @@ -0,0 +1,238 @@ +package pipeline + +import ( + "log/slog" + + tree_sitter "github.com/tree-sitter/go-tree-sitter" + + "github.com/DeusData/codebase-memory-mcp/internal/discover" + "github.com/DeusData/codebase-memory-mcp/internal/fqn" + "github.com/DeusData/codebase-memory-mcp/internal/lang" + "github.com/DeusData/codebase-memory-mcp/internal/parser" + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +// passUsages walks ASTs and creates USAGE edges for identifier references +// that are NOT inside call expressions (those are already CALLS edges). +// This distinguishes read references (passed as callbacks, stored in variables, +// used in type annotations) from function invocations. +func (p *Pipeline) passUsages() { + slog.Info("pass3b.usages") + count := 0 + for relPath, cached := range p.astCache { + spec := lang.ForLanguage(cached.Language) + if spec == nil { + continue + } + count += p.processFileUsages(relPath, cached, spec) + } + slog.Info("pass3b.usages.done", "edges", count) +} + +// passUsagesForFiles runs usage detection only for the specified files (incremental). +func (p *Pipeline) passUsagesForFiles(files []discover.FileInfo) { + slog.Info("pass3b.usages.incremental", "files", len(files)) + count := 0 + for _, f := range files { + cached, ok := p.astCache[f.RelPath] + if !ok { + continue + } + spec := lang.ForLanguage(cached.Language) + if spec == nil { + continue + } + count += p.processFileUsages(f.RelPath, cached, spec) + } + slog.Info("pass3b.usages.incremental.done", "edges", count) +} + +// referenceNodeTypes returns the AST node types that represent identifier +// references for a given language. +func referenceNodeTypes(language lang.Language) []string { + switch language { + case lang.Go: + return []string{"identifier", "selector_expression"} + case lang.Python: + return []string{"identifier", "attribute"} + case lang.JavaScript, lang.TypeScript, lang.TSX: + return []string{"identifier", "member_expression"} + case lang.Rust: + return []string{"identifier", "scoped_identifier"} + case lang.Java: + return []string{"identifier", "field_access"} + case lang.CPP: + return []string{"identifier", "qualified_identifier"} + case lang.PHP: + return []string{"name", "member_access_expression"} + case lang.Scala: + return []string{"identifier", "field_expression"} + default: + return []string{"identifier"} + } +} + +// importNodeTypes returns the set of AST node types that represent import statements. +func importNodeTypes(spec *lang.LanguageSpec) map[string]bool { + combined := make(map[string]bool) + for _, t := range spec.ImportNodeTypes { + combined[t] = true + } + for _, t := range spec.ImportFromTypes { + combined[t] = true + } + return combined +} + +// processFileUsages walks a file's AST to find identifier references that +// are not call expressions and creates USAGE edges for resolved references. +func (p *Pipeline) processFileUsages(relPath string, cached *cachedAST, spec *lang.LanguageSpec) int { + refTypes := toSet(referenceNodeTypes(cached.Language)) + callTypes := toSet(spec.CallNodeTypes) + importTypes := importNodeTypes(spec) + moduleQN := fqn.ModuleQN(p.ProjectName, relPath) + importMap := p.importMaps[moduleQN] + + root := cached.Tree.RootNode() + count := 0 + + parser.Walk(root, func(node *tree_sitter.Node) bool { + kind := node.Kind() + + // Skip call expression subtrees — those are handled by passCalls + if callTypes[kind] { + return false + } + + // Skip import statement subtrees + if importTypes[kind] { + return false + } + + if !refTypes[kind] { + return true + } + + // Skip if this identifier is inside a function/class definition name + if isDefinitionName(node) { + return false + } + + refName := parser.NodeText(node, cached.Source) + if refName == "" || isKeywordOrBuiltin(refName, cached.Language) { + return false + } + + // Find enclosing function for the "caller" side of the USAGE edge + callerQN := findEnclosingFunction(node, cached.Source, p.ProjectName, relPath, spec) + if callerQN == "" { + callerQN = moduleQN + } + + // Try to resolve the reference against the registry + targetQN := p.registry.Resolve(refName, moduleQN, importMap) + if targetQN == "" { + return false + } + + // Don't create USAGE edge to self + if targetQN == callerQN { + return false + } + + // Create the USAGE edge + callerNode, _ := p.Store.FindNodeByQN(p.ProjectName, callerQN) + targetNode, _ := p.Store.FindNodeByQN(p.ProjectName, targetQN) + if callerNode != nil && targetNode != nil { + _, _ = p.Store.InsertEdge(&store.Edge{ + Project: p.ProjectName, + SourceID: callerNode.ID, + TargetID: targetNode.ID, + Type: "USAGE", + }) + count++ + } + return false + }) + return count +} + +// isDefinitionName returns true if the node is the name child of a function, +// class, method, or variable declaration — not a reference. +func isDefinitionName(node *tree_sitter.Node) bool { + parent := node.Parent() + if parent == nil { + return false + } + nameChild := parent.ChildByFieldName("name") + if nameChild != nil && nameChild.StartByte() == node.StartByte() && nameChild.EndByte() == node.EndByte() { + parentKind := parent.Kind() + switch parentKind { + case "function_declaration", "function_definition", "method_declaration", + "method_definition", "class_declaration", "class_definition", + "type_spec", "type_alias", "interface_declaration", + "enum_declaration", "trait_item", "struct_item", + "generator_function_declaration", "function_expression", + "arrow_function", "abstract_class_declaration", + "function_signature", "type_alias_declaration", + "short_var_declaration", "var_spec", "const_spec": + return true + } + } + return false +} + +// isKeywordOrBuiltin returns true for language keywords and common builtins +// that should not be treated as references. +func isKeywordOrBuiltin(name string, language lang.Language) bool { + // Single-character identifiers and very common names are noise + if len(name) <= 1 { + return true + } + + // Common cross-language keywords + switch name { + case "if", "else", "for", "while", "return", "break", "continue", + "switch", "case", "default", "try", "catch", "finally", + "throw", "throws", "new", "delete", "this", "self", "super", + "true", "false", "nil", "null", "None", "True", "False", + "var", "let", "const", "int", "string", "bool", "float", + "void", "byte", "rune", "error", "any", "interface", + "class", "struct", "enum", "type", "func", "def", "fn", + "import", "from", "as", "package", "module", + "public", "private", "protected", "static", "final", + "async", "await", "yield", "defer", "go", "chan", + "range", "map", "make", "append", "len", "cap", + "print", "println", "fmt", "os", "log", + "isinstance", "str", "dict", "list", "tuple", "set", + "Math", "Object", "Array", "String", "Number", "Boolean", + "console", "document", "window", "undefined", + "err", "ok", "ctx": + return true + } + + // Language-specific builtins + switch language { + case lang.Go: + switch name { + case "iota", "copy", "close", "panic", "recover", + "int8", "int16", "int32", "int64", + "uint", "uint8", "uint16", "uint32", "uint64", + "float32", "float64", "complex64", "complex128", + "uintptr": + return true + } + case lang.Python: + switch name { + case "print", "range", "enumerate", "zip", "map", "filter", + "sorted", "reversed", "open", "input", "super", + "Exception", "ValueError", "TypeError", "KeyError", + "IndexError", "AttributeError", "RuntimeError", + "classmethod", "staticmethod", "property", + "abstractmethod": + return true + } + } + + return false +} diff --git a/internal/pipeline/usages_test.go b/internal/pipeline/usages_test.go new file mode 100644 index 000000000..6aa04e120 --- /dev/null +++ b/internal/pipeline/usages_test.go @@ -0,0 +1,167 @@ +package pipeline + +import ( + "os" + "path/filepath" + "testing" + + "github.com/DeusData/codebase-memory-mcp/internal/lang" + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +func TestPassUsagesCreatesEdges(t *testing.T) { + // Create a Go file that defines two functions, where one references + // the other as a variable (callback) rather than calling it. + goSource := `package mypkg + +func Process(data string) string { + return data +} + +func Register() { + handler := Process + _ = handler +} +` + tmpDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(tmpDir, "mypkg"), 0o755); err != nil { + t.Fatal(err) + } + goPath := filepath.Join(tmpDir, "mypkg", "main.go") + if err := os.WriteFile(goPath, []byte(goSource), 0o600); err != nil { + t.Fatal(err) + } + // Need go.mod for discover + if err := os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module testmod\ngo 1.21\n"), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(s, tmpDir) + if err := p.Run(); err != nil { + t.Fatal(err) + } + + // Look for USAGE edges + edges, err := s.FindEdgesByType(p.ProjectName, "USAGE") + if err != nil { + t.Fatal(err) + } + + // Register should have a USAGE edge to Process (identifier reference, not a call) + found := false + for _, e := range edges { + src, _ := s.FindNodeByID(e.SourceID) + tgt, _ := s.FindNodeByID(e.TargetID) + if src != nil && tgt != nil { + if src.Name == "Register" && tgt.Name == "Process" { + found = true + } + } + } + if !found { + t.Error("expected USAGE edge from Register to Process (callback reference)") + for _, e := range edges { + src, _ := s.FindNodeByID(e.SourceID) + tgt, _ := s.FindNodeByID(e.TargetID) + if src != nil && tgt != nil { + t.Logf(" USAGE: %s -> %s", src.Name, tgt.Name) + } + } + } +} + +func TestPassUsagesDoesNotDuplicateCalls(t *testing.T) { + // When a function is called (not just referenced), only a CALLS edge + // should exist, not a USAGE edge for the call expression. + goSource := `package mypkg + +func Helper() string { + return "ok" +} + +func Main() { + Helper() +} +` + tmpDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(tmpDir, "mypkg"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmpDir, "mypkg", "main.go"), []byte(goSource), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module testmod\ngo 1.21\n"), 0o600); err != nil { + t.Fatal(err) + } + + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + p := New(s, tmpDir) + if err := p.Run(); err != nil { + t.Fatal(err) + } + + // Should have CALLS edge from Main to Helper + callEdges, _ := s.FindEdgesByType(p.ProjectName, "CALLS") + foundCall := false + for _, e := range callEdges { + src, _ := s.FindNodeByID(e.SourceID) + tgt, _ := s.FindNodeByID(e.TargetID) + if src != nil && tgt != nil && src.Name == "Main" && tgt.Name == "Helper" { + foundCall = true + } + } + if !foundCall { + t.Error("expected CALLS edge from Main to Helper") + } + + // Should NOT have USAGE edge from Main to Helper (it's a call, not a reference) + usageEdges, _ := s.FindEdgesByType(p.ProjectName, "USAGE") + for _, e := range usageEdges { + src, _ := s.FindNodeByID(e.SourceID) + tgt, _ := s.FindNodeByID(e.TargetID) + if src != nil && tgt != nil && src.Name == "Main" && tgt.Name == "Helper" { + t.Error("should NOT have USAGE edge from Main to Helper — it's a call, not a reference") + } + } +} + +func TestIsKeywordOrBuiltin(t *testing.T) { + tests := []struct { + name string + language string + want bool + }{ + {"if", "go", true}, + {"self", "python", true}, + {"nil", "go", true}, + {"make", "go", true}, + {"print", "python", true}, + {"MyFunction", "go", false}, + {"processOrder", "python", false}, + {"x", "go", true}, // single char + } + for _, tt := range tests { + var l lang.Language + switch tt.language { + case "go": + l = lang.Go + case "python": + l = lang.Python + } + got := isKeywordOrBuiltin(tt.name, l) + if got != tt.want { + t.Errorf("isKeywordOrBuiltin(%q, %s) = %v, want %v", tt.name, tt.language, got, tt.want) + } + } +} diff --git a/internal/store/edges.go b/internal/store/edges.go index 9cc144f3e..53c7a7c88 100644 --- a/internal/store/edges.go +++ b/internal/store/edges.go @@ -62,6 +62,17 @@ func (s *Store) FindEdgesByTargetAndType(targetID int64, edgeType string) ([]*Ed return scanEdges(rows) } +// FindEdgesByType returns all edges of a given type for a project. +func (s *Store) FindEdgesByType(project, edgeType string) ([]*Edge, error) { + rows, err := s.q.Query(`SELECT id, project, source_id, target_id, type, properties + FROM edges WHERE project=? AND type=?`, project, edgeType) + if err != nil { + return nil, fmt.Errorf("find edges by type: %w", err) + } + defer rows.Close() + return scanEdges(rows) +} + // CountEdges returns the number of edges in a project. func (s *Store) CountEdges(project string) (int, error) { var count int @@ -93,6 +104,21 @@ func (s *Store) DeleteEdgesBySourceFile(project, filePath, edgeType string) erro return err } +// FindEdgesByURLPath returns edges where url_path contains the given substring. +// Uses the generated column index for prefix matches, falls back to json_extract for substring. +func (s *Store) FindEdgesByURLPath(project, pathSubstring string) ([]*Edge, error) { + rows, err := s.q.Query(` + SELECT id, project, source_id, target_id, type, properties + FROM edges + WHERE project = ? AND url_path_gen LIKE ?`, + project, "%"+pathSubstring+"%") + if err != nil { + return nil, fmt.Errorf("find edges by url_path: %w", err) + } + defer rows.Close() + return scanEdges(rows) +} + func scanEdges(rows *sql.Rows) ([]*Edge, error) { var result []*Edge for rows.Next() { diff --git a/internal/store/search.go b/internal/store/search.go index 3748573c7..9fc28e7f3 100644 --- a/internal/store/search.go +++ b/internal/store/search.go @@ -18,8 +18,9 @@ type SearchParams struct { MaxDegree int Limit int Offset int - ExcludeEntryPoints bool // when true, exclude nodes with is_entry_point=true - IncludeConnected bool // when true, load connected node names (expensive, off by default) + ExcludeEntryPoints bool // when true, exclude nodes with is_entry_point=true + IncludeConnected bool // when true, load connected node names (expensive, off by default) + ExcludeLabels []string // labels to exclude from results } // SearchResult is a node with edge degree info. @@ -103,6 +104,15 @@ func (s *Store) Search(params *SearchParams) (*SearchOutput, error) { args = append(args, likePattern) } + if len(params.ExcludeLabels) > 0 { + placeholders := make([]string, len(params.ExcludeLabels)) + for i, label := range params.ExcludeLabels { + placeholders[i] = "?" + args = append(args, label) + } + conditions = append(conditions, "n.label NOT IN ("+strings.Join(placeholders, ",")+")") + } + where := strings.Join(conditions, " AND ") // When Go-side filtering is needed (regex, degree), fetch more rows from SQL @@ -180,8 +190,10 @@ func (s *Store) Search(params *SearchParams) (*SearchOutput, error) { // globToLike converts a glob pattern to SQL LIKE pattern. func globToLike(pattern string) string { - // Replace ** with % and * with % - result := strings.ReplaceAll(pattern, "**", "%") + // Handle **/ (zero-or-more directory prefix) and /** (zero-or-more directory suffix) + // before single * to avoid double-replacement + result := strings.ReplaceAll(pattern, "**/", "%") + result = strings.ReplaceAll(result, "/**", "%") result = strings.ReplaceAll(result, "*", "%") result = strings.ReplaceAll(result, "?", "_") return result diff --git a/internal/store/store.go b/internal/store/store.go index 51d9a2f6d..daeb22019 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -4,6 +4,7 @@ import ( "database/sql" "encoding/json" "fmt" + "log/slog" "os" "path/filepath" "time" @@ -178,7 +179,27 @@ func (s *Store) initSchema() error { CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(project, source_id, type); ` _, err := s.db.Exec(schema) - return err + if err != nil { + return err + } + + // Migration: add url_path generated column to edges table. + // Generated columns require SQLite 3.31.0+ (modernc.org/sqlite supports this). + // We check if the column already exists to make this idempotent. + var colCount int + _ = s.db.QueryRow(`SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount) + if colCount == 0 { + _, err = s.db.Exec(`ALTER TABLE edges ADD COLUMN url_path_gen TEXT GENERATED ALWAYS AS (json_extract(properties, '$.url_path'))`) + if err != nil { + // If generated columns aren't supported, skip gracefully + slog.Warn("schema.url_path_gen.skip", "err", err) + } + } + + // Index on generated column (safe to CREATE IF NOT EXISTS) + _, _ = s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_edges_url_path ON edges(project, url_path_gen)`) + + return nil } // marshalProps serializes properties to JSON. diff --git a/internal/store/store_test.go b/internal/store/store_test.go index f5bf5b3ba..28ae7a23c 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1,6 +1,9 @@ package store -import "testing" +import ( + "fmt" + "testing" +) func TestOpenMemory(t *testing.T) { s, err := OpenMemory() @@ -323,3 +326,156 @@ func TestSearch(t *testing.T) { t.Errorf("expected total=3, got %d", output.Total) } } + +func TestGlobToLike(t *testing.T) { + tests := []struct { + pattern string + want string + }{ + {"**/*.py", "%%.py"}, + {"**/dir/**", "%dir%"}, + {"*.go", "%.go"}, + {"src/**", "src%"}, + {"**/test_*.py", "%test_%.py"}, + {"file?.txt", "file_.txt"}, + {"exact.go", "exact.go"}, + {"**/custom-pip-package/**", "%custom-pip-package%"}, + } + for _, tt := range tests { + t.Run(tt.pattern, func(t *testing.T) { + got := globToLike(tt.pattern) + if got != tt.want { + t.Errorf("globToLike(%q) = %q, want %q", tt.pattern, got, tt.want) + } + }) + } +} + +func TestGeneratedColumnURLPath(t *testing.T) { + s, err := OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + // Check that the generated column exists + var colCount int + err = s.DB().QueryRow(`SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount) + if err != nil { + t.Fatal(err) + } + if colCount != 1 { + t.Skip("url_path_gen column not available (SQLite version may not support generated columns)") + } +} + +func TestFindEdgesByURLPath(t *testing.T) { + s, err := OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + // Create project + if err := s.UpsertProject("test-proj", "/tmp/test"); err != nil { + t.Fatal(err) + } + + // Create two nodes + srcID, _ := s.UpsertNode(&Node{ + Project: "test-proj", Label: "Function", Name: "caller", + QualifiedName: "test.caller", + }) + tgtID, _ := s.UpsertNode(&Node{ + Project: "test-proj", Label: "Function", Name: "handler", + QualifiedName: "test.handler", + }) + + // Create HTTP_CALLS edge with url_path + _, err = s.InsertEdge(&Edge{ + Project: "test-proj", + SourceID: srcID, + TargetID: tgtID, + Type: "HTTP_CALLS", + Properties: map[string]any{"url_path": "/api/orders/create", "confidence": 0.8}, + }) + if err != nil { + t.Fatal(err) + } + + // Search for edges containing "orders" + edges, err := s.FindEdgesByURLPath("test-proj", "orders") + if err != nil { + t.Fatal(err) + } + if len(edges) != 1 { + t.Fatalf("expected 1 edge, got %d", len(edges)) + } + if edges[0].Properties["url_path"] != "/api/orders/create" { + t.Errorf("unexpected url_path: %v", edges[0].Properties["url_path"]) + } + + // Search for non-matching + edges, err = s.FindEdgesByURLPath("test-proj", "users") + if err != nil { + t.Fatal(err) + } + if len(edges) != 0 { + t.Fatalf("expected 0 edges, got %d", len(edges)) + } +} + +func TestSearchExcludeLabels(t *testing.T) { + s, err := OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + if err := s.UpsertProject("test", "/tmp/test"); err != nil { + t.Fatal(err) + } + + // Create nodes with different labels + for i, label := range []string{"Function", "Route", "Method", "Route"} { + _, _ = s.UpsertNode(&Node{ + Project: "test", + Label: label, + Name: "node_" + label, + QualifiedName: fmt.Sprintf("test.%s.node_%d", label, i), + FilePath: "test.go", + }) + } + + // Search without exclusion + output, err := s.Search(&SearchParams{ + Project: "test", + Limit: 100, + }) + if err != nil { + t.Fatal(err) + } + total := output.Total + + // Search with Route excluded + output2, err := s.Search(&SearchParams{ + Project: "test", + ExcludeLabels: []string{"Route"}, + Limit: 100, + }) + if err != nil { + t.Fatal(err) + } + + // Should have fewer results + if output2.Total >= total { + t.Errorf("exclude_labels didn't reduce results: before=%d, after=%d", total, output2.Total) + } + + // Verify no Route nodes in results + for _, r := range output2.Results { + if r.Node.Label == "Route" { + t.Errorf("found Route node despite exclude_labels") + } + } +} diff --git a/internal/tools/search.go b/internal/tools/search.go index 9bf45de42..b82fe8767 100644 --- a/internal/tools/search.go +++ b/internal/tools/search.go @@ -28,6 +28,17 @@ func (s *Server) handleSearchGraph(_ context.Context, req *mcp.CallToolRequest) IncludeConnected: getBoolArg(args, "include_connected"), } + // Parse exclude_labels array + if rawLabels, ok := args["exclude_labels"]; ok { + if labelArr, ok := rawLabels.([]any); ok { + for _, l := range labelArr { + if s, ok := l.(string); ok { + params.ExcludeLabels = append(params.ExcludeLabels, s) + } + } + } + } + projectFilter := getStringArg(args, "project") projects, err := s.store.ListProjects() diff --git a/internal/tools/tools.go b/internal/tools/tools.go index 88ef047a8..6dbe20f49 100644 --- a/internal/tools/tools.go +++ b/internal/tools/tools.go @@ -28,7 +28,7 @@ func NewServer(s *store.Store) *Server { mcp: mcp.NewServer( &mcp.Implementation{ Name: "codebase-memory-mcp", - Version: "0.1.2", + Version: "0.1.3", }, nil, ), @@ -65,6 +65,7 @@ func (s *Server) registerTools() { s.registerGraphTools() s.registerFileTools() s.registerProjectTools() + s.registerTraceTools() } // registerGraphTools registers tools for graph querying, searching, and tracing. @@ -78,7 +79,7 @@ func (s *Server) registerGraphTools() { func (s *Server) registerIndexAndTraceTool() { s.mcp.AddTool(&mcp.Tool{ Name: "index_repository", - Description: "Index a repository into the code graph. Parses source files, extracts functions/classes/modules, resolves call relationships, and stores the graph for querying. Supports incremental reindex via content hashing.", + Description: "Index a repository into the code graph. Parses source files, extracts functions/classes/modules, resolves call relationships (CALLS), read references (USAGE), interface implementations (IMPLEMENTS + OVERRIDE), HTTP/async cross-service links, and git history change coupling (FILE_CHANGES_WITH). Supports incremental reindex via content hashing. Auto-sync keeps the graph fresh after initial indexing.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { @@ -92,7 +93,7 @@ func (s *Server) registerIndexAndTraceTool() { s.mcp.AddTool(&mcp.Tool{ Name: "trace_call_path", - Description: "Trace call paths from/to a function. Requires EXACT function name — if unsure, call search_graph(name_pattern=...) first to discover the correct name. Returns hop-by-hop callees/callers with edge types (CALLS, HTTP_CALLS, ASYNC_CALLS). Use depth=1 first, increase only if needed. IMPORTANT: Use direction='both' for full cross-service context — HTTP_CALLS edges from other services appear as inbound edges on backend functions, so direction='outbound' alone misses cross-service callers. For async dispatch (Cloud Tasks, Pub/Sub), find dispatch functions via search_graph(name_pattern='.*CreateTask.*') then trace via CALLS edges. If the function is not found, use search_graph with a name_pattern regex to find similar names before retrying.", + Description: "Trace call paths from/to a function. Requires EXACT function name — if unsure, call search_graph(name_pattern=...) first to discover the correct name. Returns hop-by-hop callees/callers with edge types (CALLS, HTTP_CALLS, ASYNC_CALLS, USAGE, OVERRIDE). USAGE edges indicate read references (callbacks, variable assignments) rather than invocations. OVERRIDE edges link struct methods to the interface methods they implement. Use depth=1 first, increase only if needed. IMPORTANT: Use direction='both' for full cross-service context — HTTP_CALLS edges from other services appear as inbound edges on backend functions, so direction='outbound' alone misses cross-service callers. For async dispatch (Cloud Tasks, Pub/Sub), find dispatch functions via search_graph(name_pattern='.*CreateTask.*') then trace via CALLS edges. If the function is not found, use search_graph with a name_pattern regex to find similar names before retrying.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { @@ -141,7 +142,7 @@ func (s *Server) registerSchemaAndSnippetTools() { func (s *Server) registerSearchTools() { s.mcp.AddTool(&mcp.Tool{ Name: "search_graph", - Description: "Search the code graph for nodes. Returns 10 results per page (use offset to paginate, has_more indicates more pages). For dead code: use relationship='CALLS', direction='inbound', max_degree=0, exclude_entry_points=true. For fan-out: use relationship='CALLS', direction='outbound', min_degree=N. Route nodes: properties.handler contains the actual handler function name. Prefer this over query_graph for counting — no row cap. IMPORTANT: The 'relationship' filter counts how many edges of that type each node has (degree filtering) — it does NOT return the actual edges. To list cross-service HTTP_CALLS or ASYNC_CALLS edges with their properties (url_path, confidence), use query_graph with Cypher instead: MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence. Relationship types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE, IMPLEMENTS.", + Description: "Search the code graph for nodes. Returns 10 results per page (use offset to paginate, has_more indicates more pages). For dead code: use relationship='CALLS', direction='inbound', max_degree=0, exclude_entry_points=true. For fan-out: use relationship='CALLS', direction='outbound', min_degree=N. Route nodes: properties.handler contains the actual handler function name. Prefer this over query_graph for counting — no row cap. IMPORTANT: The 'relationship' filter counts how many edges of that type each node has (degree filtering) — it does NOT return the actual edges. To list cross-service HTTP_CALLS or ASYNC_CALLS edges with their properties (url_path, confidence), use query_graph with Cypher instead: MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence. Relationship types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE, IMPLEMENTS, OVERRIDE, USAGE, FILE_CHANGES_WITH.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { @@ -193,6 +194,11 @@ func (s *Server) registerSearchTools() { "include_connected": { "type": "boolean", "description": "Include connected node names in results (default: false). Expensive — only enable when you need to see neighbor names." + }, + "exclude_labels": { + "type": "array", + "items": {"type": "string"}, + "description": "Labels to exclude from results (e.g. ['Route'] to filter out route noise when searching by name_pattern)" } } }`), @@ -233,7 +239,7 @@ func (s *Server) registerSearchTools() { func (s *Server) registerQueryTool() { s.mcp.AddTool(&mcp.Tool{ Name: "query_graph", - Description: "Execute a Cypher-like graph query. WARNING: 200-row cap applies BEFORE aggregation — COUNT queries on large codebases silently undercount. For fan-out/fan-in counting, use search_graph with min_degree/max_degree instead. Best for: relationship patterns, filtered joins, path queries. This is the correct tool for listing cross-service edges — use MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence to see HTTP links with URLs and confidence scores, or MATCH (a)-[r:ASYNC_CALLS]->(b) for async dispatch edges. Always use LIMIT. Edge types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, IMPLEMENTS.", + Description: "Execute a Cypher-like graph query. WARNING: 200-row cap applies BEFORE aggregation — COUNT queries on large codebases silently undercount. For fan-out/fan-in counting, use search_graph with min_degree/max_degree instead. Best for: relationship patterns, filtered joins, path queries, and edge property filtering. Supports WHERE on edge properties: r.url_path CONTAINS 'orders', r.confidence >= 0.6, r.method = 'POST', r.confidence_band = 'high', r.validated_by_trace = true, r.coupling_score >= 0.5. This is the correct tool for listing cross-service edges — use MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence, r.confidence_band to see HTTP links with URLs and confidence scores (bands: high>=0.7, medium>=0.45, speculative>=0.25), or MATCH (a)-[r:ASYNC_CALLS]->(b) for async dispatch edges. For change coupling: MATCH (a)-[r:FILE_CHANGES_WITH]->(b) RETURN a.name, b.name, r.coupling_score, r.co_change_count. For interface method overrides: MATCH (s)-[r:OVERRIDE]->(i) to find struct methods implementing interface methods. For read references (callbacks, variable assignments): MATCH (a)-[r:USAGE]->(b). Always use LIMIT. Edge types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, IMPLEMENTS, OVERRIDE, USAGE, FILE_CHANGES_WITH.", InputSchema: json.RawMessage(`{ "type": "object", "properties": { diff --git a/internal/tools/traces.go b/internal/tools/traces.go new file mode 100644 index 000000000..b1e8759ae --- /dev/null +++ b/internal/tools/traces.go @@ -0,0 +1,51 @@ +package tools + +import ( + "context" + "encoding/json" + + "github.com/DeusData/codebase-memory-mcp/internal/traces" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func (s *Server) registerTraceTools() { + s.mcp.AddTool(&mcp.Tool{ + Name: "ingest_traces", + Description: "Ingest OpenTelemetry JSON traces (OTLP format) to validate and enrich HTTP_CALLS edges. Matches HTTP spans to existing edges by URL path, boosts confidence by +0.15 (capped at 1.0), and sets validated_by_trace=true, trace_call_count, and p99_latency_ns on matched edges. Use after index_repository to confirm static analysis predictions with runtime data. Export traces via: otel-cli or collector with OTLP JSON exporter.", + InputSchema: json.RawMessage(`{ + "type": "object", + "properties": { + "project": { + "type": "string", + "description": "Name of the indexed project" + }, + "file_path": { + "type": "string", + "description": "Path to the OTLP JSON export file" + } + }, + "required": ["project", "file_path"] + }`), + }, s.handleIngestTraces) +} + +func (s *Server) handleIngestTraces(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) { + args, err := parseArgs(req) + if err != nil { + return errResult(err.Error()), nil + } + + project := getStringArg(args, "project") + filePath := getStringArg(args, "file_path") + + if project == "" || filePath == "" { + return errResult("project and file_path are required"), nil + } + + result, err := traces.Ingest(s.store, project, filePath) + if err != nil { + return errResult(err.Error()), nil + } + + return jsonResult(result), nil +} diff --git a/internal/traces/traces.go b/internal/traces/traces.go new file mode 100644 index 000000000..9dadfc9aa --- /dev/null +++ b/internal/traces/traces.go @@ -0,0 +1,265 @@ +package traces + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "sort" + "strings" + + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +// OTLPExport represents the top-level structure of an OTLP JSON export. +type OTLPExport struct { + ResourceSpans []ResourceSpan `json:"resourceSpans"` +} + +// ResourceSpan contains spans from a single service/resource. +type ResourceSpan struct { + Resource Resource `json:"resource"` + ScopeSpans []ScopeSpan `json:"scopeSpans"` +} + +// Resource describes the service that produced the spans. +type Resource struct { + Attributes []Attribute `json:"attributes"` +} + +// ScopeSpan groups spans by instrumentation scope. +type ScopeSpan struct { + Spans []Span `json:"spans"` +} + +// Span represents a single trace span. +type Span struct { + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + ParentSpanID string `json:"parentSpanId"` + Name string `json:"name"` + Kind int `json:"kind"` // 1=internal, 2=server, 3=client + StartTime string `json:"startTimeUnixNano"` + EndTime string `json:"endTimeUnixNano"` + Attributes []Attribute `json:"attributes"` + Status SpanStatus `json:"status"` +} + +// SpanStatus represents the status of a span. +type SpanStatus struct { + Code int `json:"code"` // 0=unset, 1=ok, 2=error +} + +// Attribute is a key-value pair in OTLP format. +type Attribute struct { + Key string `json:"key"` + Value AttributeValue `json:"value"` +} + +// AttributeValue holds the typed value. +type AttributeValue struct { + StringValue string `json:"stringValue,omitempty"` + IntValue string `json:"intValue,omitempty"` +} + +// HTTPSpanInfo holds extracted HTTP info from a span. +type HTTPSpanInfo struct { + ServiceName string + Method string + Path string + StatusCode string + SpanKind int + DurationNs int64 +} + +// IngestResult summarizes what the trace ingestion accomplished. +type IngestResult struct { + SpansProcessed int `json:"spans_processed"` + EdgesValidated int `json:"edges_validated"` + EdgesEnriched int `json:"edges_enriched"` +} + +// Ingest reads an OTLP JSON file and uses the spans to validate/enrich HTTP_CALLS edges. +func Ingest(s *store.Store, project, filePath string) (*IngestResult, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("read trace file: %w", err) + } + + var export OTLPExport + if err := json.Unmarshal(data, &export); err != nil { + return nil, fmt.Errorf("parse OTLP JSON: %w", err) + } + + result := &IngestResult{} + + var httpSpans []HTTPSpanInfo + for _, rs := range export.ResourceSpans { + serviceName := extractServiceName(rs.Resource) + for _, ss := range rs.ScopeSpans { + for i := range ss.Spans { + info := extractHTTPInfo(&ss.Spans[i], serviceName) + if info != nil { + httpSpans = append(httpSpans, *info) + result.SpansProcessed++ + } + } + } + } + + slog.Info("traces.ingest", "http_spans", len(httpSpans)) + + matchSpansToEdges(s, project, httpSpans, result) + + return result, nil +} + +// extractServiceName gets service.name from resource attributes. +func extractServiceName(r Resource) string { + for _, attr := range r.Attributes { + if attr.Key == "service.name" { + return attr.Value.StringValue + } + } + return "" +} + +// extractHTTPInfo extracts HTTP method/path from span attributes. +func extractHTTPInfo(span *Span, serviceName string) *HTTPSpanInfo { + info := &HTTPSpanInfo{ + ServiceName: serviceName, + SpanKind: span.Kind, + } + + hasHTTP := false + for _, attr := range span.Attributes { + switch attr.Key { + case "http.method", "http.request.method": + info.Method = attr.Value.StringValue + hasHTTP = true + case "http.route", "http.target", "url.path": + info.Path = attr.Value.StringValue + hasHTTP = true + case "http.status_code": + info.StatusCode = attr.Value.StringValue + case "url.full": + if path := extractPathFromURL(attr.Value.StringValue); path != "" { + info.Path = path + hasHTTP = true + } + } + } + + if !hasHTTP || info.Path == "" { + return nil + } + + info.DurationNs = parseDuration(span.StartTime, span.EndTime) + return info +} + +// extractPathFromURL extracts the path component from a full URL string. +func extractPathFromURL(fullURL string) string { + slashes := 0 + idx := 0 + for i, c := range fullURL { + if c == '/' { + slashes++ + if slashes == 3 { + idx = i + break + } + } + } + if idx > 0 { + path := fullURL[idx:] + if qIdx := strings.Index(path, "?"); qIdx >= 0 { + path = path[:qIdx] + } + return path + } + return "" +} + +// parseDuration parses nanosecond timestamps and returns duration. +func parseDuration(startNano, endNano string) int64 { + var start, end int64 + _, _ = fmt.Sscanf(startNano, "%d", &start) + _, _ = fmt.Sscanf(endNano, "%d", &end) + if end > start { + return end - start + } + return 0 +} + +// matchSpansToEdges matches HTTP spans to existing graph edges. +func matchSpansToEdges(s *store.Store, project string, spans []HTTPSpanInfo, result *IngestResult) { + edges, err := s.FindEdgesByType(project, "HTTP_CALLS") + if err != nil { + slog.Warn("traces.edges.err", "err", err) + return + } + + edgeByPath := make(map[string]*store.Edge) + for _, e := range edges { + if urlPath, ok := e.Properties["url_path"].(string); ok { + normalized := strings.ToLower(strings.TrimRight(urlPath, "/")) + edgeByPath[normalized] = e + } + } + + pathFrequency := make(map[string]int) + pathLatencies := make(map[string][]int64) + + for _, span := range spans { + normalized := strings.ToLower(strings.TrimRight(span.Path, "/")) + pathFrequency[normalized]++ + if span.DurationNs > 0 { + pathLatencies[normalized] = append(pathLatencies[normalized], span.DurationNs) + } + } + + for path, edge := range edgeByPath { + freq, ok := pathFrequency[path] + if !ok { + continue + } + + result.EdgesValidated++ + + props := edge.Properties + if props == nil { + props = make(map[string]any) + } + props["validated_by_trace"] = true + props["trace_call_count"] = freq + + if conf, ok := props["confidence"].(float64); ok && conf < 0.9 { + props["confidence"] = min(conf+0.15, 1.0) + props["confidence_band"] = "high" + } + + if latencies, ok := pathLatencies[path]; ok && len(latencies) > 0 { + props["p99_latency_ns"] = calculateP99(latencies) + result.EdgesEnriched++ + } + + edge.Properties = props + _, _ = s.InsertEdge(edge) + } +} + +// calculateP99 returns the 99th percentile value. +func calculateP99(values []int64) int64 { + if len(values) == 0 { + return 0 + } + sorted := make([]int64, len(values)) + copy(sorted, values) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + idx := int(float64(len(sorted)) * 0.99) + if idx >= len(sorted) { + idx = len(sorted) - 1 + } + return sorted[idx] +} diff --git a/internal/traces/traces_test.go b/internal/traces/traces_test.go new file mode 100644 index 000000000..32f46c04f --- /dev/null +++ b/internal/traces/traces_test.go @@ -0,0 +1,168 @@ +package traces + +import ( + "os" + "path/filepath" + "testing" + + "github.com/DeusData/codebase-memory-mcp/internal/store" +) + +func TestExtractServiceName(t *testing.T) { + r := Resource{ + Attributes: []Attribute{ + {Key: "service.name", Value: AttributeValue{StringValue: "order-service"}}, + }, + } + if got := extractServiceName(r); got != "order-service" { + t.Errorf("expected order-service, got %s", got) + } +} + +func TestExtractHTTPInfo(t *testing.T) { + span := Span{ + Kind: 2, + Attributes: []Attribute{ + {Key: "http.method", Value: AttributeValue{StringValue: "GET"}}, + {Key: "http.route", Value: AttributeValue{StringValue: "/api/orders"}}, + {Key: "http.status_code", Value: AttributeValue{StringValue: "200"}}, + }, + StartTime: "1000000000", + EndTime: "1050000000", + } + info := extractHTTPInfo(&span, "svc") + if info == nil { + t.Fatal("expected HTTPSpanInfo") + } + if info.Method != "GET" { + t.Errorf("expected GET, got %s", info.Method) + } + if info.Path != "/api/orders" { + t.Errorf("expected /api/orders, got %s", info.Path) + } + if info.DurationNs != 50000000 { + t.Errorf("expected 50000000ns, got %d", info.DurationNs) + } +} + +func TestExtractHTTPInfoNonHTTPSpan(t *testing.T) { + span := Span{ + Kind: 1, + Attributes: []Attribute{ + {Key: "db.system", Value: AttributeValue{StringValue: "postgresql"}}, + }, + } + info := extractHTTPInfo(&span, "svc") + if info != nil { + t.Error("expected nil for non-HTTP span") + } +} + +func TestExtractPathFromURL(t *testing.T) { + tests := []struct { + url string + want string + }{ + {"https://example.com/api/orders", "/api/orders"}, + {"http://localhost:8080/health?check=true", "/health"}, + {"not-a-url", ""}, + } + for _, tt := range tests { + if got := extractPathFromURL(tt.url); got != tt.want { + t.Errorf("extractPathFromURL(%q) = %q, want %q", tt.url, got, tt.want) + } + } +} + +func TestIngestOTLPJSON(t *testing.T) { + s, err := store.OpenMemory() + if err != nil { + t.Fatal(err) + } + defer s.Close() + + project := "test-proj" + _ = s.UpsertProject(project, "/tmp/test") + + srcID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "caller", + QualifiedName: "svcA.caller", + }) + tgtID, _ := s.UpsertNode(&store.Node{ + Project: project, Label: "Function", Name: "handler", + QualifiedName: "svcB.handler", + }) + _, _ = s.InsertEdge(&store.Edge{ + Project: project, SourceID: srcID, TargetID: tgtID, + Type: "HTTP_CALLS", + Properties: map[string]any{ + "url_path": "/api/orders", + "confidence": 0.5, + "confidence_band": "medium", + }, + }) + + fixture := `{ + "resourceSpans": [{ + "resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "order-service"}}]}, + "scopeSpans": [{ + "spans": [{ + "traceId": "abc123", + "spanId": "def456", + "name": "GET /api/orders", + "kind": 2, + "startTimeUnixNano": "1000000000", + "endTimeUnixNano": "1050000000", + "attributes": [ + {"key": "http.method", "value": {"stringValue": "GET"}}, + {"key": "http.route", "value": {"stringValue": "/api/orders"}} + ], + "status": {"code": 1} + }] + }] + }] + }` + + tmpFile := filepath.Join(t.TempDir(), "traces.json") + if err := os.WriteFile(tmpFile, []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + + result, err := Ingest(s, project, tmpFile) + if err != nil { + t.Fatal(err) + } + + if result.SpansProcessed != 1 { + t.Errorf("expected 1 span, got %d", result.SpansProcessed) + } + if result.EdgesValidated != 1 { + t.Errorf("expected 1 validated, got %d", result.EdgesValidated) + } + + // Check edge was enriched + edges, _ := s.FindEdgesByType(project, "HTTP_CALLS") + if len(edges) != 1 { + t.Fatalf("expected 1 edge, got %d", len(edges)) + } + e := edges[0] + if v, ok := e.Properties["validated_by_trace"].(bool); !ok || !v { + t.Error("expected validated_by_trace=true") + } + if conf, ok := e.Properties["confidence"].(float64); !ok || conf < 0.6 { + t.Errorf("expected boosted confidence >= 0.6, got %v", e.Properties["confidence"]) + } +} + +func TestCalculateP99(t *testing.T) { + values := []int64{10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + p99 := calculateP99(values) + if p99 != 100 { + t.Errorf("expected 100, got %d", p99) + } + + single := []int64{42} + if got := calculateP99(single); got != 42 { + t.Errorf("expected 42, got %d", got) + } +} diff --git a/skills/codebase-memory-mcp.md b/skills/codebase-memory-mcp.md index 33d3da368..9d7750972 100644 --- a/skills/codebase-memory-mcp.md +++ b/skills/codebase-memory-mcp.md @@ -15,6 +15,10 @@ | "What implements this interface?" | `search_graph(relationship="IMPLEMENTS")` | Graph type relationships | | Cross-service HTTP calls (list all edges) | `query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) RETURN ...")` | Cypher returns edges with URL+confidence | | Cross-service HTTP calls (for one function) | `trace_call_path(direction="both")` | Finds inbound HTTP_CALLS edges too | +| Which structs implement an interface method? | `query_graph("MATCH (s)-[r:OVERRIDE]->(i) RETURN ...")` | OVERRIDE edges link struct→interface methods | +| Read references (callbacks, stored in vars) | `query_graph("MATCH (a)-[r:USAGE]->(b) RETURN ...")` | USAGE edges (not calls) | +| Files that change together (hidden coupling) | `query_graph("MATCH (a)-[r:FILE_CHANGES_WITH]->(b) RETURN ...")` | Git history change coupling | +| Validate HTTP edges with runtime traces | `ingest_traces(project, file_path)` | Enriches edges with latency/frequency | | Async dispatch (Cloud Tasks, Pub/Sub, SQS) | `search_graph(name_pattern=".*CreateTask.*")` then `trace_call_path` | Find dispatch functions, trace CALLS chains | | Async dispatch (if ASYNC_CALLS edges exist) | `query_graph("MATCH (a)-[r:ASYNC_CALLS]->(b) RETURN ...")` | Only works when URLs are resolvable literals | | Dead code detection | `search_graph(max_degree=0, exclude_entry_points=true)` | Degree-based filtering | @@ -49,21 +53,29 @@ After the initial `index_repository` call, a **background watcher** polls for fi | What does X call? | `trace_call_path(direction="outbound")` | `query_graph` | | Full call context (callers + callees) | `trace_call_path(direction="both")` | `direction="outbound"` alone | | Find functions by pattern | `search_graph(name_pattern="...")` | `trace_call_path` | +| Find functions, exclude noise labels | `search_graph(name_pattern="...", exclude_labels=["Route"])` | unfiltered search | | Count callers/callees | `search_graph` with `min_degree`/`max_degree` | `query_graph COUNT` (200-row cap) | | Dead code | `search_graph(max_degree=0, exclude_entry_points=true)` | `query_graph` | | Cross-service HTTP calls (list edges) | `query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) RETURN ...")` | `search_graph(relationship="HTTP_CALLS")` | +| Filter HTTP calls by URL path | `query_graph("... WHERE r.url_path CONTAINS 'orders' ...")` | unfiltered Cypher | +| Filter HTTP calls by confidence | `query_graph("... WHERE r.confidence >= 0.6 ...")` | unfiltered Cypher | +| Filter by confidence band | `query_graph("... WHERE r.confidence_band = 'high' ...")` | numeric threshold | +| Interface method overrides | `query_graph("MATCH (s)-[:OVERRIDE]->(i) RETURN s.name, i.name")` | manual AST inspection | +| Read refs (callbacks, stored funcs) | `query_graph("MATCH (a)-[:USAGE]->(b) RETURN a.name, b.name")` | grepping for function names | +| Files that change together | `query_graph("MATCH (a)-[r:FILE_CHANGES_WITH]->(b) WHERE r.coupling_score >= 0.5 RETURN ...")` | manual git log analysis | +| Validate HTTP edges with traces | `ingest_traces(project="myproj", file_path="/path/to/traces.json")` | manual trace inspection | | Async dispatch functions | `search_graph(name_pattern=".*CreateTask.*")` then `trace_call_path` | `search_graph(relationship="ASYNC_CALLS")` | | Text search | `search_code` or grep | graph tools | | Complex multi-hop patterns | `query_graph` with Cypher + LIMIT | `search_graph` | -## Tool Reference (11 Tools) +## Tool Reference (12 Tools) | Tool | Purpose | When to Use | |------|---------|-------------| | `index_repository` | Parse and ingest repo into graph | Only needed once per project — auto-sync keeps it fresh | | `trace_call_path` | BFS from/to a function (exact name match) | Primary tool for call chains — requires EXACT name | | `search_graph` | Structured search with filters | Find functions, dead code, fan-out, cross-service links | -| `query_graph` | Cypher-like graph queries (200-row cap) | Complex multi-hop patterns, filtered joins | +| `query_graph` | Cypher-like graph queries (200-row cap) | Complex multi-hop patterns, filtered joins, edge property filtering | | `get_graph_schema` | Node/edge counts, relationship patterns | Understand what's indexed before querying | | `get_code_snippet` | Read source by qualified name | After search finds a target function | | `search_code` | Grep-like text search within project | String literals, error messages, TODO comments | @@ -71,6 +83,7 @@ After the initial `index_repository` call, a **background watcher** polls for fi | `list_directory` | List files/directories with glob filter | Explore project structure | | `list_projects` | See all indexed projects | Check if reindex is needed (shows `indexed_at`) | | `delete_project` | Remove a project from graph | Cleanup stale projects | +| `ingest_traces` | Ingest OTLP JSON traces to validate HTTP_CALLS edges | After deploying — enrich edges with p99_latency, call_count, validated_by_trace | ## Edge Types @@ -84,6 +97,9 @@ After the initial `index_repository` call, a **background watcher** polls for fi | `DEFINES_METHOD` | Class defines a method | | `HANDLES` | Route node handled by a function | | `IMPLEMENTS` | Type implements an interface | +| `OVERRIDE` | Struct method overrides an interface method (e.g., FileReader.Read → Reader.Read) | +| `USAGE` | Read reference — function passed as callback, stored in variable, not invoked | +| `FILE_CHANGES_WITH` | Git history change coupling — files that frequently change together (properties: coupling_score, co_change_count) | | `CONTAINS_FILE` / `CONTAINS_FOLDER` / `CONTAINS_PACKAGE` | Structural containment | ## Critical Query Pitfalls @@ -93,6 +109,31 @@ After the initial `index_repository` call, a **background watcher** polls for fi query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence ORDER BY r.confidence DESC LIMIT 30") ``` +**Edge property filtering works in WHERE clauses.** You can filter on any edge property: +``` +# Filter by URL path substring +query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path CONTAINS 'bestellung' RETURN a.name, b.name, r.url_path") + +# Filter by confidence threshold +query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence >= 0.6 RETURN a.name, b.name, r.confidence LIMIT 20") + +# Filter by confidence band (high/medium/speculative) +query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence_band = 'high' RETURN a.name, b.name, r.url_path") + +# Change coupling above threshold +query_graph("MATCH (a)-[r:FILE_CHANGES_WITH]->(b) WHERE r.coupling_score >= 0.5 RETURN a.name, b.name, r.coupling_score, r.co_change_count LIMIT 20") +``` + +**Confidence bands** on HTTP_CALLS/ASYNC_CALLS edges: +- `high` (>= 0.7): Strong structural path match +- `medium` (0.45-0.7): Good signal, current default range +- `speculative` (0.25-0.45): Fuzzy match, needs human review + +**`exclude_labels` filters out noise.** When searching by `name_pattern`, Route nodes can dominate results. Use `exclude_labels` to remove them: +``` +search_graph(name_pattern=".*[Ff]irestore.*", exclude_labels=["Route"]) +``` + **`search_graph(relationship="ASYNC_CALLS")` has the same limitation.** ASYNC_CALLS edges only exist when the URL linker can resolve dispatch URLs to route handlers. When URLs are in config variables or env vars (common for Cloud Tasks, Pub/Sub), the linker can't match them. Instead, find dispatch functions by name pattern and trace via CALLS edges: ``` search_graph(name_pattern=".*CreateTask.*|.*create_task.*")