v0.1.3: Bug fixes, query improvements, cross-service linking, and graph enrichment

Part A — Bug fixes:
- Fix globToLike: **/dir/** now matches relative paths without leading /
- Route nodes inherit file_path from handler (were invisible to file_pattern)
- Compose YAML non-string env values (int/bool/float) no longer dropped
- Compose secret filter: key names like JWT_PRIVATE_KEY_ID no longer filtered
- Cloudbuild bash script deploy flags now parsed (entrypoint: bash)

Part B — Query improvements:
- search_graph exclude_labels parameter to filter noisy labels (e.g. Route)
- Cypher edge property filtering: r.url_path, r.confidence, r.confidence_band

Part C — Cross-service linking:
- InfraFile env var URLs feed into HTTP linker as additional call sites
- Default noise filtering: /health, /healthz, /ready, /metrics excluded
- Confidence bands on HTTP_CALLS: high (>=0.7), medium (>=0.45), speculative (>=0.25)
- Levenshtein distance and n-gram overlap similarity functions
- Lowered match threshold from 0.45 to 0.25 (speculative band)

Part D — Graph enrichment:
- OVERRIDE edges: struct methods → interface methods they implement
- USAGE edges: read references (callbacks, variable assignments) vs invocations
- Generated column index on url_path for fast edge property lookups
- Fuzzy name-only resolution fallback (5th strategy in call resolver)
- Configurable HTTP linker via .cgrconfig (exclude paths, min confidence)
- OpenTelemetry trace ingestion (ingest_traces tool) to validate HTTP_CALLS
- Git history change coupling (FILE_CHANGES_WITH edges, co-change analysis)

Updated tool descriptions and skill file with new edge types.
This commit is contained in:
Martin Vogel committed 2026-02-25 18:46:20 +01:00
1 parent 6b51d3e75e
commit 8b3534269c
29 files changed
+2837 -51

No files matched your search

+258
View File
@@ -724,3 +724,261 @@ func TestExecuteEdgeType(t *testing.T) {
t.Errorf("r.type = %v, want HTTP_CALLS", result.Rows[0]["r.type"])
}
}
// --- Comprehensive edge property filtering tests ---
// setupTestStoreMultiEdge creates a store with two HTTP_CALLS edges to test filtering.
func setupTestStoreMultiEdge(t *testing.T) *store.Store {
t.Helper()
s, err := store.OpenMemory()
if err != nil {
t.Fatal(err)
}
project := "testproj"
if err := s.UpsertProject(project, "/tmp/test"); err != nil {
t.Fatal(err)
}
srcID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Function", Name: "SendOrder",
QualifiedName: "testproj.caller.SendOrder",
FilePath: "caller/client.go",
})
tgtID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Function", Name: "HandleOrder",
QualifiedName: "testproj.handler.HandleOrder",
FilePath: "handler/routes.go",
})
tgt2ID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Function", Name: "HandleHealth",
QualifiedName: "testproj.handler.HandleHealth",
FilePath: "handler/health.go",
})
mustInsertEdge(t, s, &store.Edge{
Project: project, SourceID: srcID, TargetID: tgtID,
Type: "HTTP_CALLS",
Properties: map[string]any{
"url_path": "/api/orders",
"confidence": 0.85,
"method": "POST",
},
})
mustInsertEdge(t, s, &store.Edge{
Project: project, SourceID: srcID, TargetID: tgt2ID,
Type: "HTTP_CALLS",
Properties: map[string]any{
"url_path": "/health",
"confidence": 0.45,
},
})
return s
}
func TestEdgePropertyFilterContains(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path CONTAINS 'orders' RETURN a.name, b.name, r.url_path`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(result.Rows))
}
row := result.Rows[0]
if row["a.name"] != "SendOrder" {
t.Errorf("a.name = %v, want SendOrder", row["a.name"])
}
if row["b.name"] != "HandleOrder" {
t.Errorf("b.name = %v, want HandleOrder", row["b.name"])
}
if row["r.url_path"] != "/api/orders" {
t.Errorf("r.url_path = %v, want /api/orders", row["r.url_path"])
}
}
func TestEdgePropertyFilterNumericGTE(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence >= 0.6 RETURN a.name, b.name, r.confidence LIMIT 20`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 1 {
t.Fatalf("expected 1 row (only high-confidence edge), got %d", len(result.Rows))
}
row := result.Rows[0]
if row["b.name"] != "HandleOrder" {
t.Errorf("b.name = %v, want HandleOrder (high confidence)", row["b.name"])
}
}
func TestEdgePropertyReturnWithoutFilter(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence LIMIT 20`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) < 2 {
t.Fatalf("expected at least 2 rows, got %d", len(result.Rows))
}
foundOrders := false
foundHealth := false
for _, row := range result.Rows {
urlPath, _ := row["r.url_path"].(string)
if urlPath == "/api/orders" {
foundOrders = true
}
if urlPath == "/health" {
foundHealth = true
}
}
if !foundOrders {
t.Error("missing row with url_path=/api/orders")
}
if !foundHealth {
t.Error("missing row with url_path=/health")
}
}
func TestEdgePropertyFilterEquals(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.method = 'POST' RETURN a.name, b.name`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(result.Rows))
}
if result.Rows[0]["b.name"] != "HandleOrder" {
t.Errorf("b.name = %v, want HandleOrder", result.Rows[0]["b.name"])
}
}
func TestEdgePropertyFilterStartsWith(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path STARTS WITH '/api' RETURN a.name, b.name, r.url_path`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 1 {
t.Fatalf("expected 1 row (only /api/orders starts with /api), got %d", len(result.Rows))
}
if result.Rows[0]["r.url_path"] != "/api/orders" {
t.Errorf("r.url_path = %v, want /api/orders", result.Rows[0]["r.url_path"])
}
}
func TestCombinedNodeAndEdgeFilter(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
// Filter on both node property (early) and edge property (late)
result, err := exec.Execute(`MATCH (a:Function)-[r:HTTP_CALLS]->(b:Function) WHERE a.name = 'SendOrder' AND r.confidence >= 0.6 RETURN b.name, r.url_path`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(result.Rows))
}
if result.Rows[0]["b.name"] != "HandleOrder" {
t.Errorf("b.name = %v, want HandleOrder", result.Rows[0]["b.name"])
}
if result.Rows[0]["r.url_path"] != "/api/orders" {
t.Errorf("r.url_path = %v, want /api/orders", result.Rows[0]["r.url_path"])
}
}
func TestEdgePropertyFilterNoMatch(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
// No edge has method = 'DELETE'
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.method = 'DELETE' RETURN a.name`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 0 {
t.Errorf("expected 0 rows, got %d", len(result.Rows))
}
}
func TestEdgePropertyFilterNumericLT(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
// Only the health edge (0.45) should match confidence < 0.5
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence < 0.5 RETURN b.name, r.confidence`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(result.Rows))
}
if result.Rows[0]["b.name"] != "HandleHealth" {
t.Errorf("b.name = %v, want HandleHealth", result.Rows[0]["b.name"])
}
}
func TestEdgePropertyFilterRegex(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
// Regex match on url_path
result, err := exec.Execute(`MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path =~ "/api/.*" RETURN b.name`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(result.Rows))
}
if result.Rows[0]["b.name"] != "HandleOrder" {
t.Errorf("b.name = %v, want HandleOrder", result.Rows[0]["b.name"])
}
}
func TestEdgeBuiltinPropertyFilter(t *testing.T) {
s := setupTestStoreMultiEdge(t)
defer s.Close()
exec := &Executor{Store: s}
// Filter on built-in edge property r.type
result, err := exec.Execute(`MATCH (a)-[r]->(b) WHERE r.type = 'HTTP_CALLS' RETURN a.name, b.name LIMIT 20`)
if err != nil {
t.Fatalf("Execute: %v", err)
}
if len(result.Rows) != 2 {
t.Fatalf("expected 2 rows (both HTTP_CALLS edges), got %d", len(result.Rows))
}
}
+90
View File
@@ -0,0 +1,90 @@
package httplink
import (
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
// LinkerConfig holds user-overridable HTTP linker settings.
// Loaded from .cgrconfig in the project root.
type LinkerConfig struct {
HTTPLinker HTTPLinkerConfig `yaml:"http_linker"`
}
// HTTPLinkerConfig holds HTTP linker-specific settings.
type HTTPLinkerConfig struct {
// ExcludePaths are route paths to exclude from HTTP_CALLS matching.
// These are added to (not replacing) the built-in defaultExcludePaths.
ExcludePaths []string `yaml:"exclude_paths"`
// MinConfidence is the minimum confidence score for creating HTTP_CALLS edges.
// Default: 0.25 (includes speculative matches).
MinConfidence *float64 `yaml:"min_confidence"`
// FuzzyMatching enables/disables fuzzy URL matching.
// Default: true.
FuzzyMatching *bool `yaml:"fuzzy_matching"`
}
// DefaultConfig returns the default linker configuration.
func DefaultConfig() *LinkerConfig {
return &LinkerConfig{}
}
// LoadConfig reads .cgrconfig from the given directory.
// Returns default config if the file doesn't exist.
func LoadConfig(dir string) *LinkerConfig {
cfg := DefaultConfig()
path := filepath.Join(dir, ".cgrconfig")
data, err := os.ReadFile(path)
if err != nil {
return cfg // file not found or unreadable — use defaults
}
if err := yaml.Unmarshal(data, cfg); err != nil {
return DefaultConfig() // invalid YAML — use defaults
}
return cfg
}
// EffectiveMinConfidence returns the configured minimum confidence,
// or the default (0.25) if not set.
func (c *LinkerConfig) EffectiveMinConfidence() float64 {
if c.HTTPLinker.MinConfidence != nil {
return *c.HTTPLinker.MinConfidence
}
return 0.25
}
// EffectiveFuzzyMatching returns the configured fuzzy matching setting,
// or the default (true) if not set.
func (c *LinkerConfig) EffectiveFuzzyMatching() bool {
if c.HTTPLinker.FuzzyMatching != nil {
return *c.HTTPLinker.FuzzyMatching
}
return true
}
// AllExcludePaths returns the combined list of default + user-configured exclude paths.
func (c *LinkerConfig) AllExcludePaths() []string {
combined := make([]string, 0, len(defaultExcludePaths)+len(c.HTTPLinker.ExcludePaths))
combined = append(combined, defaultExcludePaths...)
combined = append(combined, c.HTTPLinker.ExcludePaths...)
return combined
}
// isPathExcluded checks if a route path matches any of the given exclusion paths.
func isPathExcluded(path string, excludePaths []string) bool {
normalized := strings.ToLower(strings.TrimRight(path, "/"))
for _, excluded := range excludePaths {
if strings.EqualFold(normalized, strings.TrimRight(excluded, "/")) {
return true
}
}
return false
}
+112
View File
@@ -0,0 +1,112 @@
package httplink
import (
"os"
"path/filepath"
"testing"
)
func TestLoadConfigDefault(t *testing.T) {
cfg := LoadConfig("/nonexistent/path")
if cfg.EffectiveMinConfidence() != 0.25 {
t.Errorf("expected default min_confidence 0.25, got %f", cfg.EffectiveMinConfidence())
}
if !cfg.EffectiveFuzzyMatching() {
t.Error("expected default fuzzy_matching true")
}
paths := cfg.AllExcludePaths()
if len(paths) != len(defaultExcludePaths) {
t.Errorf("expected %d default paths, got %d", len(defaultExcludePaths), len(paths))
}
}
func TestLoadConfigFromFile(t *testing.T) {
dir := t.TempDir()
configContent := `
http_linker:
exclude_paths:
- /debug
- /internal/status
min_confidence: 0.5
fuzzy_matching: false
`
if err := os.WriteFile(filepath.Join(dir, ".cgrconfig"), []byte(configContent), 0o600); err != nil {
t.Fatal(err)
}
cfg := LoadConfig(dir)
if cfg.EffectiveMinConfidence() != 0.5 {
t.Errorf("expected min_confidence 0.5, got %f", cfg.EffectiveMinConfidence())
}
if cfg.EffectiveFuzzyMatching() {
t.Error("expected fuzzy_matching false")
}
paths := cfg.AllExcludePaths()
expectedLen := len(defaultExcludePaths) + 2
if len(paths) != expectedLen {
t.Errorf("expected %d total paths, got %d", expectedLen, len(paths))
}
}
func TestLoadConfigInvalidYAML(t *testing.T) {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, ".cgrconfig"), []byte("not: [valid: yaml"), 0o600); err != nil {
t.Fatal(err)
}
cfg := LoadConfig(dir)
// Should fall back to defaults
if cfg.EffectiveMinConfidence() != 0.25 {
t.Errorf("expected default on invalid yaml, got %f", cfg.EffectiveMinConfidence())
}
}
func TestIsPathExcluded(t *testing.T) {
paths := []string{"/health", "/debug", "/internal/status"}
tests := []struct {
path string
want bool
}{
{"/health", true},
{"/health/", true},
{"/HEALTH", true},
{"/debug", true},
{"/internal/status", true},
{"/api/orders", false},
{"/healthcheck", false},
}
for _, tt := range tests {
got := isPathExcluded(tt.path, paths)
if got != tt.want {
t.Errorf("isPathExcluded(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestAllExcludePathsMerge(t *testing.T) {
cfg := DefaultConfig()
cfg.HTTPLinker.ExcludePaths = []string{"/custom1", "/custom2"}
paths := cfg.AllExcludePaths()
// Should contain all defaults + custom
if len(paths) != len(defaultExcludePaths)+2 {
t.Errorf("expected %d paths, got %d", len(defaultExcludePaths)+2, len(paths))
}
// Verify defaults are first
for i, dp := range defaultExcludePaths {
if paths[i] != dp {
t.Errorf("paths[%d] = %q, want %q", i, paths[i], dp)
}
}
// Verify custom paths are appended
customStart := len(defaultExcludePaths)
if paths[customStart] != "/custom1" {
t.Errorf("paths[%d] = %q, want /custom1", customStart, paths[customStart])
}
if paths[customStart+1] != "/custom2" {
t.Errorf("paths[%d] = %q, want /custom2", customStart+1, paths[customStart+1])
}
}
+64 -11
View File
@@ -44,14 +44,28 @@ type HTTPLink struct {
// Linker discovers cross-service HTTP calls and creates HTTP_CALLS edges.
type Linker struct {
store *store.Store
project string
routesByFunc map[string][]int // funcQN → indices into routes slice
store *store.Store
project string
config *LinkerConfig
routesByFunc map[string][]int // funcQN → indices into routes slice
extraCallSites []HTTPCallSite // injected from pipeline (e.g., InfraFile URLs)
}
// New creates a new HTTP Linker.
func New(s *store.Store, project string) *Linker {
return &Linker{store: s, project: project}
return &Linker{store: s, project: project, config: DefaultConfig()}
}
// SetConfig sets the linker configuration. If cfg is nil, defaults are used.
func (l *Linker) SetConfig(cfg *LinkerConfig) {
if cfg != nil {
l.config = cfg
}
}
// AddCallSites allows the pipeline to inject additional call sites from infra files.
func (l *Linker) AddCallSites(sites []HTTPCallSite) {
l.extraCallSites = append(l.extraCallSites, sites...)
}
// regex patterns for route and URL discovery
@@ -100,6 +114,13 @@ func (l *Linker) Run() ([]HTTPLink, error) {
}
rootPath := proj.RootPath
// Load config from project root if using defaults
if l.config.HTTPLinker.MinConfidence == nil &&
l.config.HTTPLinker.FuzzyMatching == nil &&
len(l.config.HTTPLinker.ExcludePaths) == 0 {
l.config = LoadConfig(rootPath)
}
l.routesByFunc = make(map[string][]int)
routes := l.discoverRoutes(rootPath)
slog.Info("httplink.routes", "count", len(routes))
@@ -119,6 +140,7 @@ func (l *Linker) Run() ([]HTTPLink, error) {
l.insertRouteNodes(routes)
callSites := l.discoverCallSites(rootPath)
callSites = append(callSites, l.extraCallSites...)
slog.Info("httplink.callsites", "count", len(callSites))
links := l.matchAndLink(routes, callSites)
@@ -131,7 +153,6 @@ func (l *Linker) Run() ([]HTTPLink, error) {
// HANDLES edges from the handler function to the Route node.
func (l *Linker) insertRouteNodes(routes []RouteHandler) {
for _, rh := range routes {
// Build a stable qualified name for the Route node
normalMethod := rh.Method
if normalMethod == "" {
normalMethod = "ANY"
@@ -139,7 +160,6 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) {
normalPath := strings.ReplaceAll(rh.Path, "/", "_")
normalPath = strings.Trim(normalPath, "_")
routeQN := rh.QualifiedName + ".route." + normalMethod + "." + normalPath
routeName := normalMethod + " " + rh.Path
// Use resolved handler QN if available, otherwise fall back to registering function
@@ -148,11 +168,21 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) {
handlerQN = rh.ResolvedHandlerQN
}
// Look up handler node BEFORE creating Route — we need its file_path
handlerNode, _ := l.store.FindNodeByQN(l.project, handlerQN)
// Inherit file_path from handler function
var filePath string
if handlerNode != nil && handlerNode.FilePath != "" {
filePath = handlerNode.FilePath
}
routeID, err := l.store.UpsertNode(&store.Node{
Project: l.project,
Label: "Route",
Name: routeName,
QualifiedName: routeQN,
FilePath: filePath,
Properties: map[string]any{
"method": rh.Method,
"path": rh.Path,
@@ -164,7 +194,6 @@ func (l *Linker) insertRouteNodes(routes []RouteHandler) {
}
// Create HANDLES edge from actual handler → Route
handlerNode, _ := l.store.FindNodeByQN(l.project, handlerQN)
if handlerNode != nil {
if _, edgeErr := l.store.InsertEdge(&store.Edge{
Project: l.project,
@@ -647,6 +676,17 @@ var externalDomains = []string{
"aws.amazon.com",
}
// defaultExcludePaths are common utility endpoints that produce noise in HTTP_CALLS.
// These are excluded from route matching by default.
var defaultExcludePaths = []string{
"/health",
"/healthz",
"/ready",
"/readyz",
"/metrics",
"/favicon.ico",
}
// isExternalDomain checks if a domain is a well-known external API.
func isExternalDomain(domain string) bool {
domain = strings.ToLower(domain)
@@ -658,6 +698,11 @@ func isExternalDomain(domain string) bool {
return false
}
// ExtractURLPaths finds URL path segments from text (exported for use by pipeline).
func ExtractURLPaths(text string) []string {
return extractURLPaths(text)
}
// extractURLPaths finds URL path segments from text.
func extractURLPaths(text string) []string {
seen := map[string]bool{}
@@ -825,6 +870,11 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [
continue
}
// Skip noisy utility endpoints
if isPathExcluded(rh.Path, l.config.AllExcludePaths()) {
continue
}
// Multi-signal confidence scoring
pathScore := pathMatchScore(cs.Path, rh.Path)
if pathScore == 0 {
@@ -832,7 +882,7 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [
}
score := pathScore*sourceWeight(cs.SourceLabel) + methodBonus(cs.Method, rh.Method)
if score < matchConfidenceThreshold {
if score < l.config.EffectiveMinConfidence() {
continue
}
if score > 1.0 {
@@ -848,9 +898,11 @@ func (l *Linker) matchAndLink(routes []RouteHandler, callSites []HTTPCallSite) [
callerNode, _ := l.store.FindNodeByQN(l.project, cs.SourceQualifiedName)
handlerNode, _ := l.store.FindNodeByQN(l.project, rh.QualifiedName)
if callerNode != nil && handlerNode != nil {
band := confidenceBand(score)
props := map[string]any{
"url_path": cs.Path,
"confidence": score,
"url_path": cs.Path,
"confidence": score,
"confidence_band": band,
}
if rh.Method != "" {
props["method"] = rh.Method
@@ -895,7 +947,8 @@ func normalizePath(path string) string {
}
// matchConfidenceThreshold is the minimum score for an HTTP_CALLS edge.
const matchConfidenceThreshold = 0.45
// Lowered from 0.45 to 0.25 to include speculative matches with confidence bands.
const matchConfidenceThreshold = 0.25
// pathMatchScore returns a confidence score (0.0–1.0) for how well callPath
// matches routePath. Returns 0 if no match.
+3
View File
@@ -443,6 +443,9 @@ func TestRouteNodesCreated(t *testing.T) {
t.Fatalf("expected 1 Route node, got %d", len(routeNodes))
}
rn := routeNodes[0]
if rn.FilePath != "handler/routes.py" {
t.Errorf("Route file_path = %q, want 'handler/routes.py'", rn.FilePath)
}
if rn.Name != "POST /api/orders" {
t.Errorf("Route name = %q, want 'POST /api/orders'", rn.Name)
}
+97
View File
@@ -0,0 +1,97 @@
package httplink
// levenshteinDistance computes the edit distance between two strings.
func levenshteinDistance(a, b string) int {
if a == "" {
return len(b)
}
if b == "" {
return len(a)
}
// Use two rows instead of full matrix for space efficiency
prev := make([]int, len(b)+1)
curr := make([]int, len(b)+1)
for j := range prev {
prev[j] = j
}
for i := 1; i <= len(a); i++ {
curr[0] = i
for j := 1; j <= len(b); j++ {
cost := 1
if a[i-1] == b[j-1] {
cost = 0
}
curr[j] = min(curr[j-1]+1, min(prev[j]+1, prev[j-1]+cost))
}
prev, curr = curr, prev
}
return prev[len(b)]
}
// normalizedLevenshtein returns 1.0 - (distance / maxLen), so 1.0 = identical.
func normalizedLevenshtein(a, b string) float64 {
if a == b {
return 1.0
}
maxLen := len(a)
if len(b) > maxLen {
maxLen = len(b)
}
if maxLen == 0 {
return 1.0
}
dist := levenshteinDistance(a, b)
return 1.0 - float64(dist)/float64(maxLen)
}
// ngramOverlap computes the character n-gram overlap coefficient between two strings.
// Returns |intersection(ngrams(a), ngrams(b))| / min(|ngrams(a)|, |ngrams(b)|).
func ngramOverlap(a, b string, n int) float64 {
if len(a) < n || len(b) < n {
return 0
}
ngramsA := buildNgrams(a, n)
ngramsB := buildNgrams(b, n)
intersection := 0
for ng := range ngramsA {
if ngramsB[ng] {
intersection++
}
}
minSize := len(ngramsA)
if len(ngramsB) < minSize {
minSize = len(ngramsB)
}
if minSize == 0 {
return 0
}
return float64(intersection) / float64(minSize)
}
func buildNgrams(s string, n int) map[string]bool {
ngrams := make(map[string]bool)
for i := 0; i <= len(s)-n; i++ {
ngrams[s[i:i+n]] = true
}
return ngrams
}
// confidenceBand returns the confidence band label for a given score.
func confidenceBand(score float64) string {
switch {
case score >= 0.7:
return "high"
case score >= 0.45:
return "medium"
case score >= 0.25:
return "speculative"
default:
return ""
}
}
+86
View File
@@ -0,0 +1,86 @@
package httplink
import "testing"
func TestLevenshteinDistance(t *testing.T) {
tests := []struct {
a, b string
want int
}{
{"", "", 0},
{"abc", "", 3},
{"", "abc", 3},
{"abc", "abc", 0},
{"kitten", "sitting", 3},
{"api/orders", "api/order", 1},
{"/api/v1/orders", "/api/v2/orders", 1},
}
for _, tt := range tests {
got := levenshteinDistance(tt.a, tt.b)
if got != tt.want {
t.Errorf("levenshteinDistance(%q, %q) = %d, want %d", tt.a, tt.b, got, tt.want)
}
}
}
func TestNormalizedLevenshtein(t *testing.T) {
tests := []struct {
a, b string
min float64
max float64
}{
{"abc", "abc", 1.0, 1.0},
{"", "", 1.0, 1.0},
{"api/orders", "api/order", 0.88, 0.92},
{"/api/v1/items", "/api/v2/items", 0.90, 0.94},
{"completely", "different", 0.0, 0.4},
}
for _, tt := range tests {
got := normalizedLevenshtein(tt.a, tt.b)
if got < tt.min || got > tt.max {
t.Errorf("normalizedLevenshtein(%q, %q) = %.3f, want [%.2f, %.2f]", tt.a, tt.b, got, tt.min, tt.max)
}
}
}
func TestNgramOverlap(t *testing.T) {
tests := []struct {
a, b string
n int
min float64
max float64
}{
{"api/orders", "api/orders", 3, 1.0, 1.0},
{"api/orders", "api/order", 3, 0.8, 1.0},
{"abcdef", "ghijkl", 3, 0.0, 0.0},
{"ab", "cd", 3, 0.0, 0.0}, // too short for trigrams
}
for _, tt := range tests {
got := ngramOverlap(tt.a, tt.b, tt.n)
if got < tt.min || got > tt.max {
t.Errorf("ngramOverlap(%q, %q, %d) = %.3f, want [%.2f, %.2f]", tt.a, tt.b, tt.n, got, tt.min, tt.max)
}
}
}
func TestConfidenceBand(t *testing.T) {
tests := []struct {
score float64
want string
}{
{0.95, "high"},
{0.70, "high"},
{0.69, "medium"},
{0.45, "medium"},
{0.44, "speculative"},
{0.25, "speculative"},
{0.24, ""},
{0.0, ""},
}
for _, tt := range tests {
got := confidenceBand(tt.score)
if got != tt.want {
t.Errorf("confidenceBand(%.2f) = %q, want %q", tt.score, got, tt.want)
}
}
}
+7
View File
@@ -191,6 +191,13 @@ func isSecretBinding(key, value string) bool {
return false
}
// isSecretValue checks only the value pattern, not the key name.
// Use for compose/infra files where key names like JWT_PRIVATE_KEY_ID
// are config references, not actual secrets.
func isSecretValue(value string) bool {
return secretValuePattern.MatchString(value)
}
func scanWithPatterns(absPath, relPath string, patterns []*regexp.Regexp) []EnvBinding {
f, err := os.Open(absPath)
if err != nil {
+222
View File
@@ -0,0 +1,222 @@
package pipeline
import (
"bufio"
"fmt"
"log/slog"
"os/exec"
"sort"
"strings"
"github.com/DeusData/codebase-memory-mcp/internal/store"
)
// CommitFiles holds the files changed in a single commit.
type CommitFiles struct {
Hash string
Files []string
}
// ChangeCoupling represents a pair of files that change together.
type ChangeCoupling struct {
FileA string
FileB string
CoChangeCount int
TotalChangesA int
TotalChangesB int
CouplingScore float64
}
// passGitHistory analyzes git log to find change coupling and creates FILE_CHANGES_WITH edges.
func (p *Pipeline) passGitHistory() {
slog.Info("pass6.githistory")
commits, err := parseGitLog(p.RepoPath)
if err != nil {
slog.Warn("pass6.githistory.err", "err", err)
return
}
if len(commits) == 0 {
slog.Info("pass6.githistory.skip", "reason", "no_commits")
return
}
couplings := computeChangeCoupling(commits)
edgeCount := p.createCouplingEdges(couplings)
slog.Info("pass6.githistory.done", "commits", len(commits), "couplings", len(couplings), "edges", edgeCount)
}
// parseGitLog runs git log and extracts commit → files mapping.
func parseGitLog(repoPath string) ([]CommitFiles, error) {
cmd := exec.Command("git", "log", "--name-only", "--pretty=format:COMMIT:%H", "--since=6 months ago")
cmd.Dir = repoPath
output, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("git log: %w", err)
}
var commits []CommitFiles
var current *CommitFiles
scanner := bufio.NewScanner(strings.NewReader(string(output)))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
if strings.HasPrefix(line, "COMMIT:") {
if current != nil && len(current.Files) > 0 {
commits = append(commits, *current)
}
current = &CommitFiles{Hash: strings.TrimPrefix(line, "COMMIT:")}
continue
}
if current != nil && isTrackableFile(line) {
current.Files = append(current.Files, line)
}
}
if current != nil && len(current.Files) > 0 {
commits = append(commits, *current)
}
return commits, nil
}
// isTrackableFile returns true if the file is a source file worth tracking.
func isTrackableFile(path string) bool {
skipPrefixes := []string{".git/", "node_modules/", "vendor/", "__pycache__/", ".cache/"}
for _, prefix := range skipPrefixes {
if strings.HasPrefix(path, prefix) {
return false
}
}
// Exact lock file names
skipNames := []string{"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "poetry.lock", "composer.lock", "Gemfile.lock", "Pipfile.lock"}
base := path[strings.LastIndex(path, "/")+1:]
for _, name := range skipNames {
if base == name {
return false
}
}
skipSuffixes := []string{".lock", ".sum", ".min.js", ".min.css", ".map", ".wasm", ".png", ".jpg", ".gif", ".ico", ".svg"}
for _, suffix := range skipSuffixes {
if strings.HasSuffix(path, suffix) {
return false
}
}
return true
}
// computeChangeCoupling computes which file pairs change together frequently.
func computeChangeCoupling(commits []CommitFiles) []ChangeCoupling {
fileChangeCount := make(map[string]int)
pairCount := make(map[[2]string]int)
for _, commit := range commits {
files := commit.Files
if len(files) > 20 {
continue // skip large merge/refactor commits
}
for _, f := range files {
fileChangeCount[f]++
}
for i := 0; i < len(files); i++ {
for j := i + 1; j < len(files); j++ {
a, b := files[i], files[j]
if a > b {
a, b = b, a
}
pairCount[[2]string{a, b}]++
}
}
}
couplings := make([]ChangeCoupling, 0, len(pairCount)/2)
for pair, count := range pairCount {
if count < 3 {
continue
}
totalA := fileChangeCount[pair[0]]
totalB := fileChangeCount[pair[1]]
minTotal := totalA
if totalB < minTotal {
minTotal = totalB
}
if minTotal == 0 {
continue
}
score := float64(count) / float64(minTotal)
if score < 0.3 {
continue
}
couplings = append(couplings, ChangeCoupling{
FileA: pair[0],
FileB: pair[1],
CoChangeCount: count,
TotalChangesA: totalA,
TotalChangesB: totalB,
CouplingScore: score,
})
}
sort.Slice(couplings, func(i, j int) bool {
return couplings[i].CouplingScore > couplings[j].CouplingScore
})
if len(couplings) > 100 {
couplings = couplings[:100]
}
return couplings
}
// createCouplingEdges creates FILE_CHANGES_WITH edges for detected change couplings.
func (p *Pipeline) createCouplingEdges(couplings []ChangeCoupling) int {
count := 0
for _, c := range couplings {
nodeA := p.findFileNode(c.FileA)
nodeB := p.findFileNode(c.FileB)
if nodeA == nil || nodeB == nil {
continue
}
_, err := p.Store.InsertEdge(&store.Edge{
Project: p.ProjectName,
SourceID: nodeA.ID,
TargetID: nodeB.ID,
Type: "FILE_CHANGES_WITH",
Properties: map[string]any{
"co_change_count": c.CoChangeCount,
"total_changes_a": c.TotalChangesA,
"total_changes_b": c.TotalChangesB,
"coupling_score": c.CouplingScore,
},
})
if err == nil {
count++
}
}
return count
}
// findFileNode looks up the File node for a given relative path.
func (p *Pipeline) findFileNode(relPath string) *store.Node {
nodes, err := p.Store.FindNodesByFile(p.ProjectName, relPath)
if err != nil || len(nodes) == 0 {
return nil
}
for _, n := range nodes {
if n.Label == "File" {
return n
}
}
return nodes[0]
}
+98
View File
@@ -0,0 +1,98 @@
package pipeline
import (
"fmt"
"testing"
)
func TestIsTrackableFile(t *testing.T) {
tests := []struct {
path string
want bool
}{
{"main.go", true},
{"src/app.py", true},
{"node_modules/foo/bar.js", false},
{"vendor/lib/dep.go", false},
{"package-lock.json", false},
{"go.sum", false},
{"image.png", false},
{".git/config", false},
{"__pycache__/mod.pyc", false},
{"src/style.min.css", false},
{"README.md", true},
}
for _, tt := range tests {
if got := isTrackableFile(tt.path); got != tt.want {
t.Errorf("isTrackableFile(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestComputeChangeCoupling(t *testing.T) {
commits := []CommitFiles{
{Hash: "aaa", Files: []string{"a.go", "b.go", "c.go"}},
{Hash: "bbb", Files: []string{"a.go", "b.go"}},
{Hash: "ccc", Files: []string{"a.go", "b.go"}},
{Hash: "ddd", Files: []string{"a.go", "c.go"}},
{Hash: "eee", Files: []string{"d.go", "e.go"}},
}
couplings := computeChangeCoupling(commits)
found := false
for _, c := range couplings {
if (c.FileA == "a.go" && c.FileB == "b.go") || (c.FileA == "b.go" && c.FileB == "a.go") {
found = true
if c.CoChangeCount != 3 {
t.Errorf("expected 3 co-changes for a.go/b.go, got %d", c.CoChangeCount)
}
if c.CouplingScore < 0.9 {
t.Errorf("expected high coupling score, got %f", c.CouplingScore)
}
}
}
if !found {
t.Error("expected coupling between a.go and b.go")
}
for _, c := range couplings {
if c.FileA == "d.go" || c.FileB == "d.go" {
t.Error("d.go should not appear (below threshold)")
}
}
}
func TestComputeChangeCouplingSkipsLargeCommits(t *testing.T) {
files := make([]string, 25)
for i := range files {
files[i] = fmt.Sprintf("file%d.go", i)
}
commits := []CommitFiles{{Hash: "large", Files: files}}
couplings := computeChangeCoupling(commits)
if len(couplings) != 0 {
t.Errorf("expected 0 couplings from large commit, got %d", len(couplings))
}
}
func TestComputeChangeCouplingLimitsTo100(t *testing.T) {
// Create many commits with overlapping files to generate >100 couplings
var commits []CommitFiles
for i := 0; i < 50; i++ {
for j := i + 1; j < 50; j++ {
// Create 3 commits per pair to exceed threshold
for k := 0; k < 3; k++ {
commits = append(commits, CommitFiles{
Hash: fmt.Sprintf("c%d_%d_%d", i, j, k),
Files: []string{fmt.Sprintf("f%d.go", i), fmt.Sprintf("f%d.go", j)},
})
}
}
}
couplings := computeChangeCoupling(commits)
if len(couplings) > 100 {
t.Errorf("expected max 100 couplings, got %d", len(couplings))
}
}
+60 -14
View File
@@ -7,10 +7,16 @@ import (
"github.com/DeusData/codebase-memory-mcp/internal/store"
)
// ifaceInfo holds an interface node and its required method names.
// ifaceMethodInfo holds a method name and its qualified name for OVERRIDE edge creation.
type ifaceMethodInfo struct {
name string
qualifiedName string
}
// ifaceInfo holds an interface node and its required methods.
type ifaceInfo struct {
node *store.Node
methods []string
methods []ifaceMethodInfo
}
// passImplements detects Go interface satisfaction and creates IMPLEMENTS edges.
@@ -24,9 +30,9 @@ func (p *Pipeline) passImplements() {
}
structMethods, structQNPrefix := p.collectStructMethods()
linkCount := p.matchImplements(ifaces, structMethods, structQNPrefix)
linkCount, overrideCount := p.matchImplements(ifaces, structMethods, structQNPrefix)
slog.Info("pass5.implements.done", "links", linkCount)
slog.Info("pass5.implements.done", "links", linkCount, "overrides", overrideCount)
}
// collectGoInterfaces returns Go interfaces with their method names.
@@ -47,16 +53,19 @@ func (p *Pipeline) collectGoInterfaces() []ifaceInfo {
continue
}
var methodNames []string
var methods []ifaceMethodInfo
for _, e := range edges {
methodNode, _ := p.Store.FindNodeByID(e.TargetID)
if methodNode != nil {
methodNames = append(methodNames, methodNode.Name)
methods = append(methods, ifaceMethodInfo{
name: methodNode.Name,
qualifiedName: methodNode.QualifiedName,
})
}
}
if len(methodNames) > 0 {
ifaces = append(ifaces, ifaceInfo{node: iface, methods: methodNames})
if len(methods) > 0 {
ifaces = append(ifaces, ifaceInfo{node: iface, methods: methods})
}
}
return ifaces
@@ -105,13 +114,12 @@ func (p *Pipeline) collectStructMethods() (structMethods map[string]map[string]b
return
}
// matchImplements checks each struct against each interface and creates IMPLEMENTS edges.
// matchImplements checks each struct against each interface and creates IMPLEMENTS + OVERRIDE edges.
func (p *Pipeline) matchImplements(
ifaces []ifaceInfo,
structMethods map[string]map[string]bool,
structQNPrefix map[string]string,
) int {
linkCount := 0
) (linkCount, overrideCount int) {
for _, iface := range ifaces {
for typeName, methodSet := range structMethods {
if !satisfies(iface.methods, methodSet) {
@@ -130,9 +138,47 @@ func (p *Pipeline) matchImplements(
Type: "IMPLEMENTS",
})
linkCount++
overrideCount += p.createOverrideEdges(iface.methods, typeName, structQNPrefix)
}
}
return linkCount
return linkCount, overrideCount
}
// createOverrideEdges creates OVERRIDE edges from struct methods to interface methods.
func (p *Pipeline) createOverrideEdges(
ifaceMethods []ifaceMethodInfo,
typeName string,
structQNPrefix map[string]string,
) int {
prefix, ok := structQNPrefix[typeName]
if !ok {
return 0
}
count := 0
for _, im := range ifaceMethods {
// prefix already includes the type name (e.g., "pkg.FileReader" from "pkg.FileReader.Read")
structMethodQN := prefix + "." + im.name
structMethodNode, _ := p.Store.FindNodeByQN(p.ProjectName, structMethodQN)
if structMethodNode == nil {
continue
}
ifaceMethodNode, _ := p.Store.FindNodeByQN(p.ProjectName, im.qualifiedName)
if ifaceMethodNode == nil {
continue
}
_, _ = p.Store.InsertEdge(&store.Edge{
Project: p.ProjectName,
SourceID: structMethodNode.ID,
TargetID: ifaceMethodNode.ID,
Type: "OVERRIDE",
})
count++
}
return count
}
// findStructNode looks up the struct/class node for a given receiver type name.
@@ -169,9 +215,9 @@ func extractReceiverType(recv string) string {
}
// satisfies checks if a set of method names includes all interface methods.
func satisfies(ifaceMethods []string, structMethodSet map[string]bool) bool {
func satisfies(ifaceMethods []ifaceMethodInfo, structMethodSet map[string]bool) bool {
for _, m := range ifaceMethods {
if !structMethodSet[m] {
if !structMethodSet[m.name] {
return false
}
}
+169
View File
@@ -0,0 +1,169 @@
package pipeline
import (
"testing"
"github.com/DeusData/codebase-memory-mcp/internal/store"
)
func TestPassImplementsCreatesOverrideEdges(t *testing.T) {
s, err := store.OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
project := "test-proj"
_ = s.UpsertProject(project, "/tmp/test")
// Create interface node
ifaceID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Interface", Name: "Reader",
QualifiedName: "pkg.Reader", FilePath: "pkg/reader.go",
})
// Create interface method nodes
readMethodID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Method", Name: "Read",
QualifiedName: "pkg.Reader.Read", FilePath: "pkg/reader.go",
})
closeMethodID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Method", Name: "Close",
QualifiedName: "pkg.Reader.Close", FilePath: "pkg/reader.go",
})
// DEFINES_METHOD edges (interface → method)
_, _ = s.InsertEdge(&store.Edge{
Project: project, SourceID: ifaceID, TargetID: readMethodID, Type: "DEFINES_METHOD",
})
_, _ = s.InsertEdge(&store.Edge{
Project: project, SourceID: ifaceID, TargetID: closeMethodID, Type: "DEFINES_METHOD",
})
// Create struct (Class) node
_, _ = s.UpsertNode(&store.Node{
Project: project, Label: "Class", Name: "FileReader",
QualifiedName: "pkg.FileReader", FilePath: "pkg/filereader.go",
})
// Create struct method nodes with receiver property
_, _ = s.UpsertNode(&store.Node{
Project: project, Label: "Method", Name: "Read",
QualifiedName: "pkg.FileReader.Read", FilePath: "pkg/filereader.go",
Properties: map[string]any{"receiver": "(f *FileReader)"},
})
_, _ = s.UpsertNode(&store.Node{
Project: project, Label: "Method", Name: "Close",
QualifiedName: "pkg.FileReader.Close", FilePath: "pkg/filereader.go",
Properties: map[string]any{"receiver": "(f *FileReader)"},
})
// Run passImplements
p := &Pipeline{
Store: s,
ProjectName: project,
}
p.passImplements()
// Verify IMPLEMENTS edge exists (struct → interface)
structNode, _ := s.FindNodeByQN(project, "pkg.FileReader")
if structNode == nil {
t.Fatal("struct node not found")
}
implEdges, _ := s.FindEdgesBySourceAndType(structNode.ID, "IMPLEMENTS")
if len(implEdges) != 1 {
t.Fatalf("expected 1 IMPLEMENTS edge, got %d", len(implEdges))
}
if implEdges[0].TargetID != ifaceID {
t.Errorf("IMPLEMENTS target should be interface, got %d", implEdges[0].TargetID)
}
// Verify OVERRIDE edges (struct_method → interface_method)
structReadNode, _ := s.FindNodeByQN(project, "pkg.FileReader.Read")
if structReadNode == nil {
t.Fatal("struct Read method not found")
}
readOverrides, _ := s.FindEdgesBySourceAndType(structReadNode.ID, "OVERRIDE")
if len(readOverrides) != 1 {
t.Fatalf("expected 1 OVERRIDE edge for Read, got %d", len(readOverrides))
}
if readOverrides[0].TargetID != readMethodID {
t.Errorf("OVERRIDE target should be interface Read method")
}
structCloseNode, _ := s.FindNodeByQN(project, "pkg.FileReader.Close")
if structCloseNode == nil {
t.Fatal("struct Close method not found")
}
closeOverrides, _ := s.FindEdgesBySourceAndType(structCloseNode.ID, "OVERRIDE")
if len(closeOverrides) != 1 {
t.Fatalf("expected 1 OVERRIDE edge for Close, got %d", len(closeOverrides))
}
if closeOverrides[0].TargetID != closeMethodID {
t.Errorf("OVERRIDE target should be interface Close method")
}
}
func TestPassImplementsNoOverrideWithoutMatch(t *testing.T) {
s, err := store.OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
project := "test-proj"
_ = s.UpsertProject(project, "/tmp/test")
// Interface with 2 methods
ifaceID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Interface", Name: "ReadWriter",
QualifiedName: "pkg.ReadWriter", FilePath: "pkg/rw.go",
})
readID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Method", Name: "Read",
QualifiedName: "pkg.ReadWriter.Read", FilePath: "pkg/rw.go",
})
writeID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Method", Name: "Write",
QualifiedName: "pkg.ReadWriter.Write", FilePath: "pkg/rw.go",
})
_, _ = s.InsertEdge(&store.Edge{
Project: project, SourceID: ifaceID, TargetID: readID, Type: "DEFINES_METHOD",
})
_, _ = s.InsertEdge(&store.Edge{
Project: project, SourceID: ifaceID, TargetID: writeID, Type: "DEFINES_METHOD",
})
// Struct with only Read (missing Write) — should NOT implement
_, _ = s.UpsertNode(&store.Node{
Project: project, Label: "Class", Name: "OnlyReader",
QualifiedName: "pkg.OnlyReader", FilePath: "pkg/onlyreader.go",
})
_, _ = s.UpsertNode(&store.Node{
Project: project, Label: "Method", Name: "Read",
QualifiedName: "pkg.OnlyReader.Read", FilePath: "pkg/onlyreader.go",
Properties: map[string]any{"receiver": "(o *OnlyReader)"},
})
p := &Pipeline{Store: s, ProjectName: project}
p.passImplements()
// No IMPLEMENTS or OVERRIDE edges should exist
structNode, _ := s.FindNodeByQN(project, "pkg.OnlyReader")
if structNode == nil {
t.Fatal("struct not found")
}
implEdges, _ := s.FindEdgesBySourceAndType(structNode.ID, "IMPLEMENTS")
if len(implEdges) != 0 {
t.Errorf("expected 0 IMPLEMENTS edges, got %d", len(implEdges))
}
methodNode, _ := s.FindNodeByQN(project, "pkg.OnlyReader.Read")
if methodNode == nil {
t.Fatal("method not found")
}
overrideEdges, _ := s.FindEdgesBySourceAndType(methodNode.ID, "OVERRIDE")
if len(overrideEdges) != 0 {
t.Errorf("expected 0 OVERRIDE edges, got %d", len(overrideEdges))
}
}
+104 -1
View File
@@ -147,7 +147,7 @@ services:
- "8080:8080"
environment:
PORT: "8080"
JWT_SECRET: "should-be-filtered"
JWT_SECRET: "sk-1234567890abcdefghijkl"
depends_on:
- db
networks:
@@ -875,6 +875,109 @@ func TestInfraQN(t *testing.T) {
}
}
// --- Bug fix regression tests ---
func TestParseComposeNonStringEnvValues(t *testing.T) {
content := `
services:
auth-service:
image: auth:latest
environment:
PORT: 8090
DEBUG: true
RATE_LIMIT: 1.5
NAME: "string-value"
`
path := writeTempFile(t, "docker-compose.yml", content)
results := parseComposeFile(path, "docker-compose.yml")
if len(results) != 1 {
t.Fatalf("expected 1 service, got %d", len(results))
}
envVars := requireMapStringString(t, results[0].properties["environment"])
assertEqual(t, envVars["PORT"], "8090")
assertEqual(t, envVars["DEBUG"], "true")
assertEqual(t, envVars["RATE_LIMIT"], "1.5")
assertEqual(t, envVars["NAME"], "string-value")
}
func TestParseComposeSecretKeyNotFiltered(t *testing.T) {
// Key names like JWT_PRIVATE_KEY_ID are config references, not secrets.
// Only actual secret VALUES should be filtered.
content := `
services:
auth-service:
image: auth:latest
environment:
JWT_PRIVATE_KEY_ID: jwt-private-key
GOOGLE_CLIENT_SECRET_ID: google-oauth-secret
ACCESS_TOKEN_TTL: 15m
ACTUAL_SECRET: "sk-1234567890abcdef12345"
`
path := writeTempFile(t, "docker-compose.yml", content)
results := parseComposeFile(path, "docker-compose.yml")
if len(results) != 1 {
t.Fatalf("expected 1 service, got %d", len(results))
}
envVars := requireMapStringString(t, results[0].properties["environment"])
// These should NOT be filtered — key name matches pattern but value is a reference
assertEqual(t, envVars["JWT_PRIVATE_KEY_ID"], "jwt-private-key")
assertEqual(t, envVars["GOOGLE_CLIENT_SECRET_ID"], "google-oauth-secret")
assertEqual(t, envVars["ACCESS_TOKEN_TTL"], "15m")
// This SHOULD be filtered — value matches secret pattern (sk-...)
if _, ok := envVars["ACTUAL_SECRET"]; ok {
t.Error("ACTUAL_SECRET should be filtered (value matches secret pattern)")
}
}
func TestParseCloudbuildBashScript(t *testing.T) {
content := `
substitutions:
_SERVICE_NAME: my-service
steps:
- name: gcr.io/cloud-builders/docker
args: ['build', '-t', 'eu.gcr.io/$PROJECT_ID/$_SERVICE_NAME', '.']
- name: gcr.io/google.com/cloudsdktool/cloud-sdk
entrypoint: bash
args:
- "-c"
- |
gcloud run deploy $_SERVICE_NAME \
--region=europe-west3 \
--memory=2Gi \
--cpu=2 \
--concurrency=80 \
--max-instances=10 \
--ingress=internal-and-cloud-load-balancing \
--set-env-vars=LOG_LEVEL=INFO,SOURCE_ENTITY=$_SERVICE_NAME
images:
- eu.gcr.io/$PROJECT_ID/$_SERVICE_NAME
`
path := writeTempFile(t, "cloudbuild.yaml", content)
results := parseCloudbuildFile(path, "cloudbuild.yaml")
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
props := results[0].properties
assertEqual(t, props["service_name"], "my-service")
assertEqual(t, props["deploy_region"], "europe-west3")
assertEqual(t, props["deploy_memory"], "2Gi")
assertEqual(t, props["deploy_cpu"], "2")
assertEqual(t, props["deploy_concurrency"], "80")
assertEqual(t, props["deploy_max_instances"], "10")
assertEqual(t, props["deploy_ingress"], "internal-and-cloud-load-balancing")
envVars := requireMapStringString(t, props["deploy_env_vars"])
assertEqual(t, envVars["LOG_LEVEL"], "INFO")
assertEqual(t, envVars["SOURCE_ENTITY"], "$_SERVICE_NAME")
}
// --- helpers ---
func writeTempFile(t *testing.T, name, content string) string {
+73 -10
View File
@@ -2,6 +2,8 @@ package pipeline
import (
"os"
"regexp"
"strconv"
"strings"
"gopkg.in/yaml.v3"
@@ -105,7 +107,20 @@ func parseComposeEnvironment(env any) map[string]string {
switch v := env.(type) {
case map[string]any:
for k, val := range v {
if s, ok := val.(string); ok && !isSecretBinding(k, s) {
var s string
switch tv := val.(type) {
case string:
s = tv
case int:
s = strconv.Itoa(tv)
case float64:
s = strconv.FormatFloat(tv, 'f', -1, 64)
case bool:
s = strconv.FormatBool(tv)
default:
continue
}
if !isSecretValue(s) {
result[k] = s
}
}
@@ -116,7 +131,7 @@ func parseComposeEnvironment(env any) map[string]string {
continue
}
parts := strings.SplitN(s, "=", 2)
if len(parts) == 2 && !isSecretBinding(parts[0], parts[1]) {
if len(parts) == 2 && !isSecretValue(parts[1]) {
result[parts[0]] = parts[1]
}
}
@@ -187,9 +202,10 @@ type cloudbuildFile struct {
}
type cloudbuildStep struct {
Name string `yaml:"name"`
Args []string `yaml:"args"`
Env []string `yaml:"env"`
Name string `yaml:"name"`
Entrypoint string `yaml:"entrypoint"`
Args []string `yaml:"args"`
Env []string `yaml:"env"`
}
func parseCloudbuildFile(absPath, relPath string) []infraFile {
@@ -234,12 +250,59 @@ func buildCloudbuildProps(cb cloudbuildFile) map[string]any {
// and extracts deployment flags.
func extractDeployFlags(steps []cloudbuildStep, props map[string]any) {
for _, step := range steps {
if !isDeployStep(step.Args) {
continue
// Direct args format: ["gcloud", "run", "deploy", "--flag=val", ...]
if isDeployStep(step.Args) {
parseDeployArgs(step.Args, props)
parseDeployEnvVars(step.Args, props)
return
}
parseDeployArgs(step.Args, props)
parseDeployEnvVars(step.Args, props)
return // only process first deploy step
// Bash script format: entrypoint=bash, args=["-c", "gcloud run deploy ..."]
if isBashDeployStep(&step) {
parseBashDeployScript(step.Args[1], props)
return
}
}
}
// isBashDeployStep checks if a step is a bash entrypoint with a gcloud run deploy script.
func isBashDeployStep(step *cloudbuildStep) bool {
isBash := step.Entrypoint == "bash" || step.Entrypoint == "/bin/bash" || step.Entrypoint == "/bin/sh"
if !isBash && len(step.Args) >= 2 {
isBash = step.Args[0] == "-c"
}
if !isBash || len(step.Args) < 2 {
return false
}
script := step.Args[len(step.Args)-1]
return strings.Contains(script, "gcloud") &&
strings.Contains(script, "run") &&
strings.Contains(script, "deploy")
}
// parseBashDeployScript extracts deploy flags from a bash script string.
func parseBashDeployScript(script string, props map[string]any) {
// Normalize line continuations
normalized := strings.ReplaceAll(script, "\\\n", " ")
// Extract flags using the same flag map as parseDeployArgs
for flag, propName := range deployFlagMap {
re := regexp.MustCompile(regexp.QuoteMeta(flag) + `[= ]+([^\s\\]+)`)
if m := re.FindStringSubmatch(normalized); len(m) == 2 {
props[propName] = m[1]
}
}
// Extract env vars from --set-env-vars or --update-env-vars
envRe := regexp.MustCompile(`--(?:set|update)-env-vars[= ]+([^\s\\]+(?:,[^\s\\]+)*)`)
if m := envRe.FindStringSubmatch(normalized); len(m) == 2 {
envMap := make(map[string]string)
for _, pair := range strings.Split(m[1], ",") {
parts := strings.SplitN(pair, "=", 2)
if len(parts) == 2 && !isSecretBinding(parts[0], parts[1]) {
envMap[parts[0]] = parts[1]
}
}
setNonEmptyMap(props, "deploy_env_vars", envMap)
}
}
+104 -2
View File
@@ -109,11 +109,13 @@ func (p *Pipeline) runFullPasses(files []discover.FileInfo) error {
p.buildRegistry()
p.passImports()
p.passCalls()
p.passUsages()
p.cleanupASTCache()
if err := p.passHTTPLinks(); err != nil {
slog.Warn("pass4.httplink.err", "err", err)
}
p.passImplements()
p.passGitHistory()
p.updateFileHashes(files)
return nil
}
@@ -156,13 +158,15 @@ func (p *Pipeline) runIncrementalPasses(
filesToResolve := mergeFiles(changed, dependents)
slog.Info("incremental.resolve", "changed", len(changed), "dependents", len(dependents))
// Delete CALLS edges for files being re-resolved
// Delete CALLS and USAGE edges for files being re-resolved
for _, f := range filesToResolve {
_ = p.Store.DeleteEdgesBySourceFile(p.ProjectName, f.RelPath, "CALLS")
_ = p.Store.DeleteEdgesBySourceFile(p.ProjectName, f.RelPath, "USAGE")
}
// Pass 3: Re-resolve calls for changed + dependent files
// Pass 3: Re-resolve calls + usages for changed + dependent files
p.passCallsForFiles(filesToResolve)
p.passUsagesForFiles(filesToResolve)
p.cleanupASTCache()
@@ -171,6 +175,7 @@ func (p *Pipeline) runIncrementalPasses(
slog.Warn("pass4.httplink.err", "err", err)
}
p.passImplements()
p.passGitHistory()
p.updateFileHashes(allFiles)
return nil
@@ -877,6 +882,10 @@ func (p *Pipeline) processFileCalls(relPath string, cached *cachedAST, spec *lan
targetQN := p.resolveCallWithTypes(calleeName, moduleQN, importMap, localTypeMap)
if targetQN == "" {
// Strategy 5: Fuzzy name-only fallback
if fuzzyQN, ok := p.registry.FuzzyResolve(calleeName, moduleQN); ok {
p.createFuzzyCallEdge(callerQN, fuzzyQN)
}
return false
}
@@ -947,6 +956,25 @@ func (p *Pipeline) createCallEdge(callerQN, targetQN string) {
}
}
// createFuzzyCallEdge creates a CALLS edge with resolution_mode="fuzzy" property,
// indicating the callee was resolved via text-based name matching rather than
// structural analysis (imports, same-module, etc.).
func (p *Pipeline) createFuzzyCallEdge(callerQN, targetQN string) {
callerNode, _ := p.Store.FindNodeByQN(p.ProjectName, callerQN)
targetNode, _ := p.Store.FindNodeByQN(p.ProjectName, targetQN)
if callerNode != nil && targetNode != nil {
_, _ = p.Store.InsertEdge(&store.Edge{
Project: p.ProjectName,
SourceID: callerNode.ID,
TargetID: targetNode.ID,
Type: "CALLS",
Properties: map[string]any{
"resolution_mode": "fuzzy",
},
})
}
}
// resolveCallWithTypes resolves a callee name using the registry, import maps,
// and type inference for method dispatch.
func (p *Pipeline) resolveCallWithTypes(
@@ -1197,6 +1225,14 @@ func (p *Pipeline) passHTTPLinks() error {
}
linker := httplink.New(p.Store, p.ProjectName)
// Feed InfraFile environment URLs into the HTTP linker
infraSites := p.extractInfraCallSites()
if len(infraSites) > 0 {
linker.AddCallSites(infraSites)
slog.Info("pass4.infra_callsites", "count", len(infraSites))
}
links, err := linker.Run()
if err != nil {
return err
@@ -1205,6 +1241,72 @@ func (p *Pipeline) passHTTPLinks() error {
return nil
}
// extractInfraCallSites extracts URL values from InfraFile environment properties
// and converts them to HTTPCallSite entries for the HTTP linker.
func (p *Pipeline) extractInfraCallSites() []httplink.HTTPCallSite {
infraNodes, err := p.Store.FindNodesByLabel(p.ProjectName, "InfraFile")
if err != nil {
return nil
}
var sites []httplink.HTTPCallSite
for _, node := range infraNodes {
// InfraFile nodes use different property keys depending on source:
// compose files: "environment", Dockerfiles/shell/.env: "env_vars",
// cloudbuild: "deploy_env_vars"
for _, envKey := range []string{"environment", "env_vars", "deploy_env_vars"} {
sites = append(sites, extractEnvURLSites(node, envKey)...)
}
}
return sites
}
// extractEnvURLSites extracts HTTP call sites from a single env property of an InfraFile node.
func extractEnvURLSites(node *store.Node, propKey string) []httplink.HTTPCallSite {
env, ok := node.Properties[propKey]
if !ok {
return nil
}
// env_vars are stored as map[string]string (from Go), but after JSON round-trip
// through SQLite they come back as map[string]any.
var sites []httplink.HTTPCallSite
switch envMap := env.(type) {
case map[string]any:
for _, val := range envMap {
valStr, ok := val.(string)
if !ok {
continue
}
sites = append(sites, urlSitesFromValue(node, valStr)...)
}
case map[string]string:
for _, valStr := range envMap {
sites = append(sites, urlSitesFromValue(node, valStr)...)
}
}
return sites
}
// urlSitesFromValue extracts URL paths from a string value and creates HTTPCallSite entries.
func urlSitesFromValue(node *store.Node, val string) []httplink.HTTPCallSite {
if !strings.Contains(val, "http://") && !strings.Contains(val, "https://") && !strings.HasPrefix(val, "/") {
return nil
}
paths := httplink.ExtractURLPaths(val)
sites := make([]httplink.HTTPCallSite, 0, len(paths))
for _, path := range paths {
sites = append(sites, httplink.HTTPCallSite{
Path: path,
SourceName: node.Name,
SourceQualifiedName: node.QualifiedName,
SourceLabel: "InfraFile",
})
}
return sites
}
// injectEnvBindings creates or updates Module nodes for config files that contain
// environment variable URL bindings. These synthetic constants feed into the
// HTTP linker's call site discovery.
+33
View File
@@ -150,6 +150,39 @@ func (r *FunctionRegistry) resolveViaNameLookup(calleeName, suffix, moduleQN str
return ""
}
// FuzzyResolve attempts a loose match when Resolve() returns "".
// It searches for any registered function whose simple name matches the callee's
// last name segment. Returns the best match (by import distance) and true,
// or "" and false if no match is found.
//
// Unlike Resolve(), this does not require prefix/import agreement — it purely
// matches on the function name. Results should be marked as "fuzzy" resolution.
func (r *FunctionRegistry) FuzzyResolve(calleeName, moduleQN string) (string, bool) {
r.mu.RLock()
defer r.mu.RUnlock()
// Extract the simple name (last segment after dots)
lookupName := simpleName(calleeName)
candidates := r.byName[lookupName]
if len(candidates) == 0 {
return "", false
}
// If there's exactly one candidate, use it
if len(candidates) == 1 {
return candidates[0], true
}
// Multiple candidates: pick best by import distance
best := bestByImportDistance(candidates, moduleQN)
if best != "" {
return best, true
}
return "", false
}
// FindByName returns all qualified names with the given simple name.
func (r *FunctionRegistry) FindByName(name string) []string {
r.mu.RLock()
+82
View File
@@ -0,0 +1,82 @@
package pipeline
import "testing"
func TestFuzzyResolve_SingleCandidate(t *testing.T) {
reg := NewFunctionRegistry()
reg.Register("CreateOrder", "svcA.handlers.CreateOrder", "Function")
reg.Register("ValidateOrder", "svcB.validators.ValidateOrder", "Function")
// Normal resolve with no import map should find unique name
result := reg.Resolve("CreateOrder", "svcC.caller", nil)
if result != "svcA.handlers.CreateOrder" {
t.Errorf("Resolve: expected svcA.handlers.CreateOrder, got %s", result)
}
// FuzzyResolve should find by simple name even with unknown prefix
fuzzyResult, ok := reg.FuzzyResolve("unknownPkg.CreateOrder", "svcC.caller")
if !ok {
t.Fatal("expected fuzzy match")
}
if fuzzyResult != "svcA.handlers.CreateOrder" {
t.Errorf("expected svcA.handlers.CreateOrder, got %s", fuzzyResult)
}
}
func TestFuzzyResolve_NonExistentName(t *testing.T) {
reg := NewFunctionRegistry()
reg.Register("CreateOrder", "svcA.handlers.CreateOrder", "Function")
_, ok := reg.FuzzyResolve("NonExistent", "svcC.caller")
if ok {
t.Fatal("expected no fuzzy match for non-existent name")
}
}
func TestFuzzyResolve_MultipleCandidates_BestByDistance(t *testing.T) {
reg := NewFunctionRegistry()
reg.Register("Process", "svcA.handlers.Process", "Function")
reg.Register("Process", "svcB.handlers.Process", "Function")
// Caller is in svcA — should prefer svcA.handlers.Process
fuzzyResult, ok := reg.FuzzyResolve("unknown.Process", "svcA.other")
if !ok {
t.Fatal("expected fuzzy match")
}
if fuzzyResult != "svcA.handlers.Process" {
t.Errorf("expected svcA.handlers.Process, got %s", fuzzyResult)
}
// Caller is in svcB — should prefer svcB.handlers.Process
fuzzyResult, ok = reg.FuzzyResolve("unknown.Process", "svcB.other")
if !ok {
t.Fatal("expected fuzzy match")
}
if fuzzyResult != "svcB.handlers.Process" {
t.Errorf("expected svcB.handlers.Process, got %s", fuzzyResult)
}
}
func TestFuzzyResolve_SimpleNameExtraction(t *testing.T) {
reg := NewFunctionRegistry()
reg.Register("DoWork", "myproject.utils.DoWork", "Function")
// Deeply qualified callee — should extract "DoWork" as simple name
fuzzyResult, ok := reg.FuzzyResolve("some.deep.module.DoWork", "myproject.caller")
if !ok {
t.Fatal("expected fuzzy match")
}
if fuzzyResult != "myproject.utils.DoWork" {
t.Errorf("expected myproject.utils.DoWork, got %s", fuzzyResult)
}
}
func TestFuzzyResolve_NoMatchForBareName(t *testing.T) {
reg := NewFunctionRegistry()
// Register nothing
_, ok := reg.FuzzyResolve("SomeFunc", "myproject.caller")
if ok {
t.Fatal("expected no fuzzy match on empty registry")
}
}
+238
View File
@@ -0,0 +1,238 @@
package pipeline
import (
"log/slog"
tree_sitter "github.com/tree-sitter/go-tree-sitter"
"github.com/DeusData/codebase-memory-mcp/internal/discover"
"github.com/DeusData/codebase-memory-mcp/internal/fqn"
"github.com/DeusData/codebase-memory-mcp/internal/lang"
"github.com/DeusData/codebase-memory-mcp/internal/parser"
"github.com/DeusData/codebase-memory-mcp/internal/store"
)
// passUsages walks ASTs and creates USAGE edges for identifier references
// that are NOT inside call expressions (those are already CALLS edges).
// This distinguishes read references (passed as callbacks, stored in variables,
// used in type annotations) from function invocations.
func (p *Pipeline) passUsages() {
slog.Info("pass3b.usages")
count := 0
for relPath, cached := range p.astCache {
spec := lang.ForLanguage(cached.Language)
if spec == nil {
continue
}
count += p.processFileUsages(relPath, cached, spec)
}
slog.Info("pass3b.usages.done", "edges", count)
}
// passUsagesForFiles runs usage detection only for the specified files (incremental).
func (p *Pipeline) passUsagesForFiles(files []discover.FileInfo) {
slog.Info("pass3b.usages.incremental", "files", len(files))
count := 0
for _, f := range files {
cached, ok := p.astCache[f.RelPath]
if !ok {
continue
}
spec := lang.ForLanguage(cached.Language)
if spec == nil {
continue
}
count += p.processFileUsages(f.RelPath, cached, spec)
}
slog.Info("pass3b.usages.incremental.done", "edges", count)
}
// referenceNodeTypes returns the AST node types that represent identifier
// references for a given language.
func referenceNodeTypes(language lang.Language) []string {
switch language {
case lang.Go:
return []string{"identifier", "selector_expression"}
case lang.Python:
return []string{"identifier", "attribute"}
case lang.JavaScript, lang.TypeScript, lang.TSX:
return []string{"identifier", "member_expression"}
case lang.Rust:
return []string{"identifier", "scoped_identifier"}
case lang.Java:
return []string{"identifier", "field_access"}
case lang.CPP:
return []string{"identifier", "qualified_identifier"}
case lang.PHP:
return []string{"name", "member_access_expression"}
case lang.Scala:
return []string{"identifier", "field_expression"}
default:
return []string{"identifier"}
}
}
// importNodeTypes returns the set of AST node types that represent import statements.
func importNodeTypes(spec *lang.LanguageSpec) map[string]bool {
combined := make(map[string]bool)
for _, t := range spec.ImportNodeTypes {
combined[t] = true
}
for _, t := range spec.ImportFromTypes {
combined[t] = true
}
return combined
}
// processFileUsages walks a file's AST to find identifier references that
// are not call expressions and creates USAGE edges for resolved references.
func (p *Pipeline) processFileUsages(relPath string, cached *cachedAST, spec *lang.LanguageSpec) int {
refTypes := toSet(referenceNodeTypes(cached.Language))
callTypes := toSet(spec.CallNodeTypes)
importTypes := importNodeTypes(spec)
moduleQN := fqn.ModuleQN(p.ProjectName, relPath)
importMap := p.importMaps[moduleQN]
root := cached.Tree.RootNode()
count := 0
parser.Walk(root, func(node *tree_sitter.Node) bool {
kind := node.Kind()
// Skip call expression subtrees — those are handled by passCalls
if callTypes[kind] {
return false
}
// Skip import statement subtrees
if importTypes[kind] {
return false
}
if !refTypes[kind] {
return true
}
// Skip if this identifier is inside a function/class definition name
if isDefinitionName(node) {
return false
}
refName := parser.NodeText(node, cached.Source)
if refName == "" || isKeywordOrBuiltin(refName, cached.Language) {
return false
}
// Find enclosing function for the "caller" side of the USAGE edge
callerQN := findEnclosingFunction(node, cached.Source, p.ProjectName, relPath, spec)
if callerQN == "" {
callerQN = moduleQN
}
// Try to resolve the reference against the registry
targetQN := p.registry.Resolve(refName, moduleQN, importMap)
if targetQN == "" {
return false
}
// Don't create USAGE edge to self
if targetQN == callerQN {
return false
}
// Create the USAGE edge
callerNode, _ := p.Store.FindNodeByQN(p.ProjectName, callerQN)
targetNode, _ := p.Store.FindNodeByQN(p.ProjectName, targetQN)
if callerNode != nil && targetNode != nil {
_, _ = p.Store.InsertEdge(&store.Edge{
Project: p.ProjectName,
SourceID: callerNode.ID,
TargetID: targetNode.ID,
Type: "USAGE",
})
count++
}
return false
})
return count
}
// isDefinitionName returns true if the node is the name child of a function,
// class, method, or variable declaration — not a reference.
func isDefinitionName(node *tree_sitter.Node) bool {
parent := node.Parent()
if parent == nil {
return false
}
nameChild := parent.ChildByFieldName("name")
if nameChild != nil && nameChild.StartByte() == node.StartByte() && nameChild.EndByte() == node.EndByte() {
parentKind := parent.Kind()
switch parentKind {
case "function_declaration", "function_definition", "method_declaration",
"method_definition", "class_declaration", "class_definition",
"type_spec", "type_alias", "interface_declaration",
"enum_declaration", "trait_item", "struct_item",
"generator_function_declaration", "function_expression",
"arrow_function", "abstract_class_declaration",
"function_signature", "type_alias_declaration",
"short_var_declaration", "var_spec", "const_spec":
return true
}
}
return false
}
// isKeywordOrBuiltin returns true for language keywords and common builtins
// that should not be treated as references.
func isKeywordOrBuiltin(name string, language lang.Language) bool {
// Single-character identifiers and very common names are noise
if len(name) <= 1 {
return true
}
// Common cross-language keywords
switch name {
case "if", "else", "for", "while", "return", "break", "continue",
"switch", "case", "default", "try", "catch", "finally",
"throw", "throws", "new", "delete", "this", "self", "super",
"true", "false", "nil", "null", "None", "True", "False",
"var", "let", "const", "int", "string", "bool", "float",
"void", "byte", "rune", "error", "any", "interface",
"class", "struct", "enum", "type", "func", "def", "fn",
"import", "from", "as", "package", "module",
"public", "private", "protected", "static", "final",
"async", "await", "yield", "defer", "go", "chan",
"range", "map", "make", "append", "len", "cap",
"print", "println", "fmt", "os", "log",
"isinstance", "str", "dict", "list", "tuple", "set",
"Math", "Object", "Array", "String", "Number", "Boolean",
"console", "document", "window", "undefined",
"err", "ok", "ctx":
return true
}
// Language-specific builtins
switch language {
case lang.Go:
switch name {
case "iota", "copy", "close", "panic", "recover",
"int8", "int16", "int32", "int64",
"uint", "uint8", "uint16", "uint32", "uint64",
"float32", "float64", "complex64", "complex128",
"uintptr":
return true
}
case lang.Python:
switch name {
case "print", "range", "enumerate", "zip", "map", "filter",
"sorted", "reversed", "open", "input", "super",
"Exception", "ValueError", "TypeError", "KeyError",
"IndexError", "AttributeError", "RuntimeError",
"classmethod", "staticmethod", "property",
"abstractmethod":
return true
}
}
return false
}
+167
View File
@@ -0,0 +1,167 @@
package pipeline
import (
"os"
"path/filepath"
"testing"
"github.com/DeusData/codebase-memory-mcp/internal/lang"
"github.com/DeusData/codebase-memory-mcp/internal/store"
)
func TestPassUsagesCreatesEdges(t *testing.T) {
// Create a Go file that defines two functions, where one references
// the other as a variable (callback) rather than calling it.
goSource := `package mypkg
func Process(data string) string {
return data
}
func Register() {
handler := Process
_ = handler
}
`
tmpDir := t.TempDir()
if err := os.MkdirAll(filepath.Join(tmpDir, "mypkg"), 0o755); err != nil {
t.Fatal(err)
}
goPath := filepath.Join(tmpDir, "mypkg", "main.go")
if err := os.WriteFile(goPath, []byte(goSource), 0o600); err != nil {
t.Fatal(err)
}
// Need go.mod for discover
if err := os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module testmod\ngo 1.21\n"), 0o600); err != nil {
t.Fatal(err)
}
s, err := store.OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
p := New(s, tmpDir)
if err := p.Run(); err != nil {
t.Fatal(err)
}
// Look for USAGE edges
edges, err := s.FindEdgesByType(p.ProjectName, "USAGE")
if err != nil {
t.Fatal(err)
}
// Register should have a USAGE edge to Process (identifier reference, not a call)
found := false
for _, e := range edges {
src, _ := s.FindNodeByID(e.SourceID)
tgt, _ := s.FindNodeByID(e.TargetID)
if src != nil && tgt != nil {
if src.Name == "Register" && tgt.Name == "Process" {
found = true
}
}
}
if !found {
t.Error("expected USAGE edge from Register to Process (callback reference)")
for _, e := range edges {
src, _ := s.FindNodeByID(e.SourceID)
tgt, _ := s.FindNodeByID(e.TargetID)
if src != nil && tgt != nil {
t.Logf(" USAGE: %s -> %s", src.Name, tgt.Name)
}
}
}
}
func TestPassUsagesDoesNotDuplicateCalls(t *testing.T) {
// When a function is called (not just referenced), only a CALLS edge
// should exist, not a USAGE edge for the call expression.
goSource := `package mypkg
func Helper() string {
return "ok"
}
func Main() {
Helper()
}
`
tmpDir := t.TempDir()
if err := os.MkdirAll(filepath.Join(tmpDir, "mypkg"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(tmpDir, "mypkg", "main.go"), []byte(goSource), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module testmod\ngo 1.21\n"), 0o600); err != nil {
t.Fatal(err)
}
s, err := store.OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
p := New(s, tmpDir)
if err := p.Run(); err != nil {
t.Fatal(err)
}
// Should have CALLS edge from Main to Helper
callEdges, _ := s.FindEdgesByType(p.ProjectName, "CALLS")
foundCall := false
for _, e := range callEdges {
src, _ := s.FindNodeByID(e.SourceID)
tgt, _ := s.FindNodeByID(e.TargetID)
if src != nil && tgt != nil && src.Name == "Main" && tgt.Name == "Helper" {
foundCall = true
}
}
if !foundCall {
t.Error("expected CALLS edge from Main to Helper")
}
// Should NOT have USAGE edge from Main to Helper (it's a call, not a reference)
usageEdges, _ := s.FindEdgesByType(p.ProjectName, "USAGE")
for _, e := range usageEdges {
src, _ := s.FindNodeByID(e.SourceID)
tgt, _ := s.FindNodeByID(e.TargetID)
if src != nil && tgt != nil && src.Name == "Main" && tgt.Name == "Helper" {
t.Error("should NOT have USAGE edge from Main to Helper — it's a call, not a reference")
}
}
}
func TestIsKeywordOrBuiltin(t *testing.T) {
tests := []struct {
name string
language string
want bool
}{
{"if", "go", true},
{"self", "python", true},
{"nil", "go", true},
{"make", "go", true},
{"print", "python", true},
{"MyFunction", "go", false},
{"processOrder", "python", false},
{"x", "go", true}, // single char
}
for _, tt := range tests {
var l lang.Language
switch tt.language {
case "go":
l = lang.Go
case "python":
l = lang.Python
}
got := isKeywordOrBuiltin(tt.name, l)
if got != tt.want {
t.Errorf("isKeywordOrBuiltin(%q, %s) = %v, want %v", tt.name, tt.language, got, tt.want)
}
}
}
+26
View File
@@ -62,6 +62,17 @@ func (s *Store) FindEdgesByTargetAndType(targetID int64, edgeType string) ([]*Ed
return scanEdges(rows)
}
// FindEdgesByType returns all edges of a given type for a project.
func (s *Store) FindEdgesByType(project, edgeType string) ([]*Edge, error) {
rows, err := s.q.Query(`SELECT id, project, source_id, target_id, type, properties
FROM edges WHERE project=? AND type=?`, project, edgeType)
if err != nil {
return nil, fmt.Errorf("find edges by type: %w", err)
}
defer rows.Close()
return scanEdges(rows)
}
// CountEdges returns the number of edges in a project.
func (s *Store) CountEdges(project string) (int, error) {
var count int
@@ -93,6 +104,21 @@ func (s *Store) DeleteEdgesBySourceFile(project, filePath, edgeType string) erro
return err
}
// FindEdgesByURLPath returns edges where url_path contains the given substring.
// Uses the generated column index for prefix matches, falls back to json_extract for substring.
func (s *Store) FindEdgesByURLPath(project, pathSubstring string) ([]*Edge, error) {
rows, err := s.q.Query(`
SELECT id, project, source_id, target_id, type, properties
FROM edges
WHERE project = ? AND url_path_gen LIKE ?`,
project, "%"+pathSubstring+"%")
if err != nil {
return nil, fmt.Errorf("find edges by url_path: %w", err)
}
defer rows.Close()
return scanEdges(rows)
}
func scanEdges(rows *sql.Rows) ([]*Edge, error) {
var result []*Edge
for rows.Next() {
+16 -4
View File
@@ -18,8 +18,9 @@ type SearchParams struct {
MaxDegree int
Limit int
Offset int
ExcludeEntryPoints bool // when true, exclude nodes with is_entry_point=true
IncludeConnected bool // when true, load connected node names (expensive, off by default)
ExcludeEntryPoints bool // when true, exclude nodes with is_entry_point=true
IncludeConnected bool // when true, load connected node names (expensive, off by default)
ExcludeLabels []string // labels to exclude from results
}
// SearchResult is a node with edge degree info.
@@ -103,6 +104,15 @@ func (s *Store) Search(params *SearchParams) (*SearchOutput, error) {
args = append(args, likePattern)
}
if len(params.ExcludeLabels) > 0 {
placeholders := make([]string, len(params.ExcludeLabels))
for i, label := range params.ExcludeLabels {
placeholders[i] = "?"
args = append(args, label)
}
conditions = append(conditions, "n.label NOT IN ("+strings.Join(placeholders, ",")+")")
}
where := strings.Join(conditions, " AND ")
// When Go-side filtering is needed (regex, degree), fetch more rows from SQL
@@ -180,8 +190,10 @@ func (s *Store) Search(params *SearchParams) (*SearchOutput, error) {
// globToLike converts a glob pattern to SQL LIKE pattern.
func globToLike(pattern string) string {
// Replace ** with % and * with %
result := strings.ReplaceAll(pattern, "**", "%")
// Handle **/ (zero-or-more directory prefix) and /** (zero-or-more directory suffix)
// before single * to avoid double-replacement
result := strings.ReplaceAll(pattern, "**/", "%")
result = strings.ReplaceAll(result, "/**", "%")
result = strings.ReplaceAll(result, "*", "%")
result = strings.ReplaceAll(result, "?", "_")
return result
+22 -1
View File
@@ -4,6 +4,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"log/slog"
"os"
"path/filepath"
"time"
@@ -178,7 +179,27 @@ func (s *Store) initSchema() error {
CREATE INDEX IF NOT EXISTS idx_edges_source_type ON edges(project, source_id, type);
`
_, err := s.db.Exec(schema)
return err
if err != nil {
return err
}
// Migration: add url_path generated column to edges table.
// Generated columns require SQLite 3.31.0+ (modernc.org/sqlite supports this).
// We check if the column already exists to make this idempotent.
var colCount int
_ = s.db.QueryRow(`SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount)
if colCount == 0 {
_, err = s.db.Exec(`ALTER TABLE edges ADD COLUMN url_path_gen TEXT GENERATED ALWAYS AS (json_extract(properties, '$.url_path'))`)
if err != nil {
// If generated columns aren't supported, skip gracefully
slog.Warn("schema.url_path_gen.skip", "err", err)
}
}
// Index on generated column (safe to CREATE IF NOT EXISTS)
_, _ = s.db.Exec(`CREATE INDEX IF NOT EXISTS idx_edges_url_path ON edges(project, url_path_gen)`)
return nil
}
// marshalProps serializes properties to JSON.
+157 -1
View File
@@ -1,6 +1,9 @@
package store
import "testing"
import (
"fmt"
"testing"
)
func TestOpenMemory(t *testing.T) {
s, err := OpenMemory()
@@ -323,3 +326,156 @@ func TestSearch(t *testing.T) {
t.Errorf("expected total=3, got %d", output.Total)
}
}
func TestGlobToLike(t *testing.T) {
tests := []struct {
pattern string
want string
}{
{"**/*.py", "%%.py"},
{"**/dir/**", "%dir%"},
{"*.go", "%.go"},
{"src/**", "src%"},
{"**/test_*.py", "%test_%.py"},
{"file?.txt", "file_.txt"},
{"exact.go", "exact.go"},
{"**/custom-pip-package/**", "%custom-pip-package%"},
}
for _, tt := range tests {
t.Run(tt.pattern, func(t *testing.T) {
got := globToLike(tt.pattern)
if got != tt.want {
t.Errorf("globToLike(%q) = %q, want %q", tt.pattern, got, tt.want)
}
})
}
}
func TestGeneratedColumnURLPath(t *testing.T) {
s, err := OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
// Check that the generated column exists
var colCount int
err = s.DB().QueryRow(`SELECT COUNT(*) FROM pragma_table_xinfo('edges') WHERE name='url_path_gen'`).Scan(&colCount)
if err != nil {
t.Fatal(err)
}
if colCount != 1 {
t.Skip("url_path_gen column not available (SQLite version may not support generated columns)")
}
}
func TestFindEdgesByURLPath(t *testing.T) {
s, err := OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
// Create project
if err := s.UpsertProject("test-proj", "/tmp/test"); err != nil {
t.Fatal(err)
}
// Create two nodes
srcID, _ := s.UpsertNode(&Node{
Project: "test-proj", Label: "Function", Name: "caller",
QualifiedName: "test.caller",
})
tgtID, _ := s.UpsertNode(&Node{
Project: "test-proj", Label: "Function", Name: "handler",
QualifiedName: "test.handler",
})
// Create HTTP_CALLS edge with url_path
_, err = s.InsertEdge(&Edge{
Project: "test-proj",
SourceID: srcID,
TargetID: tgtID,
Type: "HTTP_CALLS",
Properties: map[string]any{"url_path": "/api/orders/create", "confidence": 0.8},
})
if err != nil {
t.Fatal(err)
}
// Search for edges containing "orders"
edges, err := s.FindEdgesByURLPath("test-proj", "orders")
if err != nil {
t.Fatal(err)
}
if len(edges) != 1 {
t.Fatalf("expected 1 edge, got %d", len(edges))
}
if edges[0].Properties["url_path"] != "/api/orders/create" {
t.Errorf("unexpected url_path: %v", edges[0].Properties["url_path"])
}
// Search for non-matching
edges, err = s.FindEdgesByURLPath("test-proj", "users")
if err != nil {
t.Fatal(err)
}
if len(edges) != 0 {
t.Fatalf("expected 0 edges, got %d", len(edges))
}
}
func TestSearchExcludeLabels(t *testing.T) {
s, err := OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
if err := s.UpsertProject("test", "/tmp/test"); err != nil {
t.Fatal(err)
}
// Create nodes with different labels
for i, label := range []string{"Function", "Route", "Method", "Route"} {
_, _ = s.UpsertNode(&Node{
Project: "test",
Label: label,
Name: "node_" + label,
QualifiedName: fmt.Sprintf("test.%s.node_%d", label, i),
FilePath: "test.go",
})
}
// Search without exclusion
output, err := s.Search(&SearchParams{
Project: "test",
Limit: 100,
})
if err != nil {
t.Fatal(err)
}
total := output.Total
// Search with Route excluded
output2, err := s.Search(&SearchParams{
Project: "test",
ExcludeLabels: []string{"Route"},
Limit: 100,
})
if err != nil {
t.Fatal(err)
}
// Should have fewer results
if output2.Total >= total {
t.Errorf("exclude_labels didn't reduce results: before=%d, after=%d", total, output2.Total)
}
// Verify no Route nodes in results
for _, r := range output2.Results {
if r.Node.Label == "Route" {
t.Errorf("found Route node despite exclude_labels")
}
}
}
+11
View File
@@ -28,6 +28,17 @@ func (s *Server) handleSearchGraph(_ context.Context, req *mcp.CallToolRequest)
IncludeConnected: getBoolArg(args, "include_connected"),
}
// Parse exclude_labels array
if rawLabels, ok := args["exclude_labels"]; ok {
if labelArr, ok := rawLabels.([]any); ok {
for _, l := range labelArr {
if s, ok := l.(string); ok {
params.ExcludeLabels = append(params.ExcludeLabels, s)
}
}
}
}
projectFilter := getStringArg(args, "project")
projects, err := s.store.ListProjects()
+11 -5
View File
@@ -28,7 +28,7 @@ func NewServer(s *store.Store) *Server {
mcp: mcp.NewServer(
&mcp.Implementation{
Name: "codebase-memory-mcp",
Version: "0.1.2",
Version: "0.1.3",
},
nil,
),
@@ -65,6 +65,7 @@ func (s *Server) registerTools() {
s.registerGraphTools()
s.registerFileTools()
s.registerProjectTools()
s.registerTraceTools()
}
// registerGraphTools registers tools for graph querying, searching, and tracing.
@@ -78,7 +79,7 @@ func (s *Server) registerGraphTools() {
func (s *Server) registerIndexAndTraceTool() {
s.mcp.AddTool(&mcp.Tool{
Name: "index_repository",
Description: "Index a repository into the code graph. Parses source files, extracts functions/classes/modules, resolves call relationships, and stores the graph for querying. Supports incremental reindex via content hashing.",
Description: "Index a repository into the code graph. Parses source files, extracts functions/classes/modules, resolves call relationships (CALLS), read references (USAGE), interface implementations (IMPLEMENTS + OVERRIDE), HTTP/async cross-service links, and git history change coupling (FILE_CHANGES_WITH). Supports incremental reindex via content hashing. Auto-sync keeps the graph fresh after initial indexing.",
InputSchema: json.RawMessage(`{
"type": "object",
"properties": {
@@ -92,7 +93,7 @@ func (s *Server) registerIndexAndTraceTool() {
s.mcp.AddTool(&mcp.Tool{
Name: "trace_call_path",
Description: "Trace call paths from/to a function. Requires EXACT function name — if unsure, call search_graph(name_pattern=...) first to discover the correct name. Returns hop-by-hop callees/callers with edge types (CALLS, HTTP_CALLS, ASYNC_CALLS). Use depth=1 first, increase only if needed. IMPORTANT: Use direction='both' for full cross-service context — HTTP_CALLS edges from other services appear as inbound edges on backend functions, so direction='outbound' alone misses cross-service callers. For async dispatch (Cloud Tasks, Pub/Sub), find dispatch functions via search_graph(name_pattern='.*CreateTask.*') then trace via CALLS edges. If the function is not found, use search_graph with a name_pattern regex to find similar names before retrying.",
Description: "Trace call paths from/to a function. Requires EXACT function name — if unsure, call search_graph(name_pattern=...) first to discover the correct name. Returns hop-by-hop callees/callers with edge types (CALLS, HTTP_CALLS, ASYNC_CALLS, USAGE, OVERRIDE). USAGE edges indicate read references (callbacks, variable assignments) rather than invocations. OVERRIDE edges link struct methods to the interface methods they implement. Use depth=1 first, increase only if needed. IMPORTANT: Use direction='both' for full cross-service context — HTTP_CALLS edges from other services appear as inbound edges on backend functions, so direction='outbound' alone misses cross-service callers. For async dispatch (Cloud Tasks, Pub/Sub), find dispatch functions via search_graph(name_pattern='.*CreateTask.*') then trace via CALLS edges. If the function is not found, use search_graph with a name_pattern regex to find similar names before retrying.",
InputSchema: json.RawMessage(`{
"type": "object",
"properties": {
@@ -141,7 +142,7 @@ func (s *Server) registerSchemaAndSnippetTools() {
func (s *Server) registerSearchTools() {
s.mcp.AddTool(&mcp.Tool{
Name: "search_graph",
Description: "Search the code graph for nodes. Returns 10 results per page (use offset to paginate, has_more indicates more pages). For dead code: use relationship='CALLS', direction='inbound', max_degree=0, exclude_entry_points=true. For fan-out: use relationship='CALLS', direction='outbound', min_degree=N. Route nodes: properties.handler contains the actual handler function name. Prefer this over query_graph for counting — no row cap. IMPORTANT: The 'relationship' filter counts how many edges of that type each node has (degree filtering) — it does NOT return the actual edges. To list cross-service HTTP_CALLS or ASYNC_CALLS edges with their properties (url_path, confidence), use query_graph with Cypher instead: MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence. Relationship types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE, IMPLEMENTS.",
Description: "Search the code graph for nodes. Returns 10 results per page (use offset to paginate, has_more indicates more pages). For dead code: use relationship='CALLS', direction='inbound', max_degree=0, exclude_entry_points=true. For fan-out: use relationship='CALLS', direction='outbound', min_degree=N. Route nodes: properties.handler contains the actual handler function name. Prefer this over query_graph for counting — no row cap. IMPORTANT: The 'relationship' filter counts how many edges of that type each node has (degree filtering) — it does NOT return the actual edges. To list cross-service HTTP_CALLS or ASYNC_CALLS edges with their properties (url_path, confidence), use query_graph with Cypher instead: MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence. Relationship types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, CONTAINS_FILE, CONTAINS_FOLDER, CONTAINS_PACKAGE, IMPLEMENTS, OVERRIDE, USAGE, FILE_CHANGES_WITH.",
InputSchema: json.RawMessage(`{
"type": "object",
"properties": {
@@ -193,6 +194,11 @@ func (s *Server) registerSearchTools() {
"include_connected": {
"type": "boolean",
"description": "Include connected node names in results (default: false). Expensive — only enable when you need to see neighbor names."
},
"exclude_labels": {
"type": "array",
"items": {"type": "string"},
"description": "Labels to exclude from results (e.g. ['Route'] to filter out route noise when searching by name_pattern)"
}
}
}`),
@@ -233,7 +239,7 @@ func (s *Server) registerSearchTools() {
func (s *Server) registerQueryTool() {
s.mcp.AddTool(&mcp.Tool{
Name: "query_graph",
Description: "Execute a Cypher-like graph query. WARNING: 200-row cap applies BEFORE aggregation — COUNT queries on large codebases silently undercount. For fan-out/fan-in counting, use search_graph with min_degree/max_degree instead. Best for: relationship patterns, filtered joins, path queries. This is the correct tool for listing cross-service edges — use MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence to see HTTP links with URLs and confidence scores, or MATCH (a)-[r:ASYNC_CALLS]->(b) for async dispatch edges. Always use LIMIT. Edge types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, IMPLEMENTS.",
Description: "Execute a Cypher-like graph query. WARNING: 200-row cap applies BEFORE aggregation — COUNT queries on large codebases silently undercount. For fan-out/fan-in counting, use search_graph with min_degree/max_degree instead. Best for: relationship patterns, filtered joins, path queries, and edge property filtering. Supports WHERE on edge properties: r.url_path CONTAINS 'orders', r.confidence >= 0.6, r.method = 'POST', r.confidence_band = 'high', r.validated_by_trace = true, r.coupling_score >= 0.5. This is the correct tool for listing cross-service edges — use MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence, r.confidence_band to see HTTP links with URLs and confidence scores (bands: high>=0.7, medium>=0.45, speculative>=0.25), or MATCH (a)-[r:ASYNC_CALLS]->(b) for async dispatch edges. For change coupling: MATCH (a)-[r:FILE_CHANGES_WITH]->(b) RETURN a.name, b.name, r.coupling_score, r.co_change_count. For interface method overrides: MATCH (s)-[r:OVERRIDE]->(i) to find struct methods implementing interface methods. For read references (callbacks, variable assignments): MATCH (a)-[r:USAGE]->(b). Always use LIMIT. Edge types: CALLS, HTTP_CALLS, ASYNC_CALLS, IMPORTS, DEFINES, DEFINES_METHOD, HANDLES, IMPLEMENTS, OVERRIDE, USAGE, FILE_CHANGES_WITH.",
InputSchema: json.RawMessage(`{
"type": "object",
"properties": {
+51
View File
@@ -0,0 +1,51 @@
package tools
import (
"context"
"encoding/json"
"github.com/DeusData/codebase-memory-mcp/internal/traces"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
func (s *Server) registerTraceTools() {
s.mcp.AddTool(&mcp.Tool{
Name: "ingest_traces",
Description: "Ingest OpenTelemetry JSON traces (OTLP format) to validate and enrich HTTP_CALLS edges. Matches HTTP spans to existing edges by URL path, boosts confidence by +0.15 (capped at 1.0), and sets validated_by_trace=true, trace_call_count, and p99_latency_ns on matched edges. Use after index_repository to confirm static analysis predictions with runtime data. Export traces via: otel-cli or collector with OTLP JSON exporter.",
InputSchema: json.RawMessage(`{
"type": "object",
"properties": {
"project": {
"type": "string",
"description": "Name of the indexed project"
},
"file_path": {
"type": "string",
"description": "Path to the OTLP JSON export file"
}
},
"required": ["project", "file_path"]
}`),
}, s.handleIngestTraces)
}
func (s *Server) handleIngestTraces(_ context.Context, req *mcp.CallToolRequest) (*mcp.CallToolResult, error) {
args, err := parseArgs(req)
if err != nil {
return errResult(err.Error()), nil
}
project := getStringArg(args, "project")
filePath := getStringArg(args, "file_path")
if project == "" || filePath == "" {
return errResult("project and file_path are required"), nil
}
result, err := traces.Ingest(s.store, project, filePath)
if err != nil {
return errResult(err.Error()), nil
}
return jsonResult(result), nil
}
+265
View File
@@ -0,0 +1,265 @@
package traces
import (
"encoding/json"
"fmt"
"log/slog"
"os"
"sort"
"strings"
"github.com/DeusData/codebase-memory-mcp/internal/store"
)
// OTLPExport represents the top-level structure of an OTLP JSON export.
type OTLPExport struct {
ResourceSpans []ResourceSpan `json:"resourceSpans"`
}
// ResourceSpan contains spans from a single service/resource.
type ResourceSpan struct {
Resource Resource `json:"resource"`
ScopeSpans []ScopeSpan `json:"scopeSpans"`
}
// Resource describes the service that produced the spans.
type Resource struct {
Attributes []Attribute `json:"attributes"`
}
// ScopeSpan groups spans by instrumentation scope.
type ScopeSpan struct {
Spans []Span `json:"spans"`
}
// Span represents a single trace span.
type Span struct {
TraceID string `json:"traceId"`
SpanID string `json:"spanId"`
ParentSpanID string `json:"parentSpanId"`
Name string `json:"name"`
Kind int `json:"kind"` // 1=internal, 2=server, 3=client
StartTime string `json:"startTimeUnixNano"`
EndTime string `json:"endTimeUnixNano"`
Attributes []Attribute `json:"attributes"`
Status SpanStatus `json:"status"`
}
// SpanStatus represents the status of a span.
type SpanStatus struct {
Code int `json:"code"` // 0=unset, 1=ok, 2=error
}
// Attribute is a key-value pair in OTLP format.
type Attribute struct {
Key string `json:"key"`
Value AttributeValue `json:"value"`
}
// AttributeValue holds the typed value.
type AttributeValue struct {
StringValue string `json:"stringValue,omitempty"`
IntValue string `json:"intValue,omitempty"`
}
// HTTPSpanInfo holds extracted HTTP info from a span.
type HTTPSpanInfo struct {
ServiceName string
Method string
Path string
StatusCode string
SpanKind int
DurationNs int64
}
// IngestResult summarizes what the trace ingestion accomplished.
type IngestResult struct {
SpansProcessed int `json:"spans_processed"`
EdgesValidated int `json:"edges_validated"`
EdgesEnriched int `json:"edges_enriched"`
}
// Ingest reads an OTLP JSON file and uses the spans to validate/enrich HTTP_CALLS edges.
func Ingest(s *store.Store, project, filePath string) (*IngestResult, error) {
data, err := os.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("read trace file: %w", err)
}
var export OTLPExport
if err := json.Unmarshal(data, &export); err != nil {
return nil, fmt.Errorf("parse OTLP JSON: %w", err)
}
result := &IngestResult{}
var httpSpans []HTTPSpanInfo
for _, rs := range export.ResourceSpans {
serviceName := extractServiceName(rs.Resource)
for _, ss := range rs.ScopeSpans {
for i := range ss.Spans {
info := extractHTTPInfo(&ss.Spans[i], serviceName)
if info != nil {
httpSpans = append(httpSpans, *info)
result.SpansProcessed++
}
}
}
}
slog.Info("traces.ingest", "http_spans", len(httpSpans))
matchSpansToEdges(s, project, httpSpans, result)
return result, nil
}
// extractServiceName gets service.name from resource attributes.
func extractServiceName(r Resource) string {
for _, attr := range r.Attributes {
if attr.Key == "service.name" {
return attr.Value.StringValue
}
}
return ""
}
// extractHTTPInfo extracts HTTP method/path from span attributes.
func extractHTTPInfo(span *Span, serviceName string) *HTTPSpanInfo {
info := &HTTPSpanInfo{
ServiceName: serviceName,
SpanKind: span.Kind,
}
hasHTTP := false
for _, attr := range span.Attributes {
switch attr.Key {
case "http.method", "http.request.method":
info.Method = attr.Value.StringValue
hasHTTP = true
case "http.route", "http.target", "url.path":
info.Path = attr.Value.StringValue
hasHTTP = true
case "http.status_code":
info.StatusCode = attr.Value.StringValue
case "url.full":
if path := extractPathFromURL(attr.Value.StringValue); path != "" {
info.Path = path
hasHTTP = true
}
}
}
if !hasHTTP || info.Path == "" {
return nil
}
info.DurationNs = parseDuration(span.StartTime, span.EndTime)
return info
}
// extractPathFromURL extracts the path component from a full URL string.
func extractPathFromURL(fullURL string) string {
slashes := 0
idx := 0
for i, c := range fullURL {
if c == '/' {
slashes++
if slashes == 3 {
idx = i
break
}
}
}
if idx > 0 {
path := fullURL[idx:]
if qIdx := strings.Index(path, "?"); qIdx >= 0 {
path = path[:qIdx]
}
return path
}
return ""
}
// parseDuration parses nanosecond timestamps and returns duration.
func parseDuration(startNano, endNano string) int64 {
var start, end int64
_, _ = fmt.Sscanf(startNano, "%d", &start)
_, _ = fmt.Sscanf(endNano, "%d", &end)
if end > start {
return end - start
}
return 0
}
// matchSpansToEdges matches HTTP spans to existing graph edges.
func matchSpansToEdges(s *store.Store, project string, spans []HTTPSpanInfo, result *IngestResult) {
edges, err := s.FindEdgesByType(project, "HTTP_CALLS")
if err != nil {
slog.Warn("traces.edges.err", "err", err)
return
}
edgeByPath := make(map[string]*store.Edge)
for _, e := range edges {
if urlPath, ok := e.Properties["url_path"].(string); ok {
normalized := strings.ToLower(strings.TrimRight(urlPath, "/"))
edgeByPath[normalized] = e
}
}
pathFrequency := make(map[string]int)
pathLatencies := make(map[string][]int64)
for _, span := range spans {
normalized := strings.ToLower(strings.TrimRight(span.Path, "/"))
pathFrequency[normalized]++
if span.DurationNs > 0 {
pathLatencies[normalized] = append(pathLatencies[normalized], span.DurationNs)
}
}
for path, edge := range edgeByPath {
freq, ok := pathFrequency[path]
if !ok {
continue
}
result.EdgesValidated++
props := edge.Properties
if props == nil {
props = make(map[string]any)
}
props["validated_by_trace"] = true
props["trace_call_count"] = freq
if conf, ok := props["confidence"].(float64); ok && conf < 0.9 {
props["confidence"] = min(conf+0.15, 1.0)
props["confidence_band"] = "high"
}
if latencies, ok := pathLatencies[path]; ok && len(latencies) > 0 {
props["p99_latency_ns"] = calculateP99(latencies)
result.EdgesEnriched++
}
edge.Properties = props
_, _ = s.InsertEdge(edge)
}
}
// calculateP99 returns the 99th percentile value.
func calculateP99(values []int64) int64 {
if len(values) == 0 {
return 0
}
sorted := make([]int64, len(values))
copy(sorted, values)
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
idx := int(float64(len(sorted)) * 0.99)
if idx >= len(sorted) {
idx = len(sorted) - 1
}
return sorted[idx]
}
+168
View File
@@ -0,0 +1,168 @@
package traces
import (
"os"
"path/filepath"
"testing"
"github.com/DeusData/codebase-memory-mcp/internal/store"
)
func TestExtractServiceName(t *testing.T) {
r := Resource{
Attributes: []Attribute{
{Key: "service.name", Value: AttributeValue{StringValue: "order-service"}},
},
}
if got := extractServiceName(r); got != "order-service" {
t.Errorf("expected order-service, got %s", got)
}
}
func TestExtractHTTPInfo(t *testing.T) {
span := Span{
Kind: 2,
Attributes: []Attribute{
{Key: "http.method", Value: AttributeValue{StringValue: "GET"}},
{Key: "http.route", Value: AttributeValue{StringValue: "/api/orders"}},
{Key: "http.status_code", Value: AttributeValue{StringValue: "200"}},
},
StartTime: "1000000000",
EndTime: "1050000000",
}
info := extractHTTPInfo(&span, "svc")
if info == nil {
t.Fatal("expected HTTPSpanInfo")
}
if info.Method != "GET" {
t.Errorf("expected GET, got %s", info.Method)
}
if info.Path != "/api/orders" {
t.Errorf("expected /api/orders, got %s", info.Path)
}
if info.DurationNs != 50000000 {
t.Errorf("expected 50000000ns, got %d", info.DurationNs)
}
}
func TestExtractHTTPInfoNonHTTPSpan(t *testing.T) {
span := Span{
Kind: 1,
Attributes: []Attribute{
{Key: "db.system", Value: AttributeValue{StringValue: "postgresql"}},
},
}
info := extractHTTPInfo(&span, "svc")
if info != nil {
t.Error("expected nil for non-HTTP span")
}
}
func TestExtractPathFromURL(t *testing.T) {
tests := []struct {
url string
want string
}{
{"https://example.com/api/orders", "/api/orders"},
{"http://localhost:8080/health?check=true", "/health"},
{"not-a-url", ""},
}
for _, tt := range tests {
if got := extractPathFromURL(tt.url); got != tt.want {
t.Errorf("extractPathFromURL(%q) = %q, want %q", tt.url, got, tt.want)
}
}
}
func TestIngestOTLPJSON(t *testing.T) {
s, err := store.OpenMemory()
if err != nil {
t.Fatal(err)
}
defer s.Close()
project := "test-proj"
_ = s.UpsertProject(project, "/tmp/test")
srcID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Function", Name: "caller",
QualifiedName: "svcA.caller",
})
tgtID, _ := s.UpsertNode(&store.Node{
Project: project, Label: "Function", Name: "handler",
QualifiedName: "svcB.handler",
})
_, _ = s.InsertEdge(&store.Edge{
Project: project, SourceID: srcID, TargetID: tgtID,
Type: "HTTP_CALLS",
Properties: map[string]any{
"url_path": "/api/orders",
"confidence": 0.5,
"confidence_band": "medium",
},
})
fixture := `{
"resourceSpans": [{
"resource": {"attributes": [{"key": "service.name", "value": {"stringValue": "order-service"}}]},
"scopeSpans": [{
"spans": [{
"traceId": "abc123",
"spanId": "def456",
"name": "GET /api/orders",
"kind": 2,
"startTimeUnixNano": "1000000000",
"endTimeUnixNano": "1050000000",
"attributes": [
{"key": "http.method", "value": {"stringValue": "GET"}},
{"key": "http.route", "value": {"stringValue": "/api/orders"}}
],
"status": {"code": 1}
}]
}]
}]
}`
tmpFile := filepath.Join(t.TempDir(), "traces.json")
if err := os.WriteFile(tmpFile, []byte(fixture), 0o600); err != nil {
t.Fatal(err)
}
result, err := Ingest(s, project, tmpFile)
if err != nil {
t.Fatal(err)
}
if result.SpansProcessed != 1 {
t.Errorf("expected 1 span, got %d", result.SpansProcessed)
}
if result.EdgesValidated != 1 {
t.Errorf("expected 1 validated, got %d", result.EdgesValidated)
}
// Check edge was enriched
edges, _ := s.FindEdgesByType(project, "HTTP_CALLS")
if len(edges) != 1 {
t.Fatalf("expected 1 edge, got %d", len(edges))
}
e := edges[0]
if v, ok := e.Properties["validated_by_trace"].(bool); !ok || !v {
t.Error("expected validated_by_trace=true")
}
if conf, ok := e.Properties["confidence"].(float64); !ok || conf < 0.6 {
t.Errorf("expected boosted confidence >= 0.6, got %v", e.Properties["confidence"])
}
}
func TestCalculateP99(t *testing.T) {
values := []int64{10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
p99 := calculateP99(values)
if p99 != 100 {
t.Errorf("expected 100, got %d", p99)
}
single := []int64{42}
if got := calculateP99(single); got != 42 {
t.Errorf("expected 42, got %d", got)
}
}
+43 -2
View File
@@ -15,6 +15,10 @@
| "What implements this interface?" | `search_graph(relationship="IMPLEMENTS")` | Graph type relationships |
| Cross-service HTTP calls (list all edges) | `query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) RETURN ...")` | Cypher returns edges with URL+confidence |
| Cross-service HTTP calls (for one function) | `trace_call_path(direction="both")` | Finds inbound HTTP_CALLS edges too |
| Which structs implement an interface method? | `query_graph("MATCH (s)-[r:OVERRIDE]->(i) RETURN ...")` | OVERRIDE edges link struct→interface methods |
| Read references (callbacks, stored in vars) | `query_graph("MATCH (a)-[r:USAGE]->(b) RETURN ...")` | USAGE edges (not calls) |
| Files that change together (hidden coupling) | `query_graph("MATCH (a)-[r:FILE_CHANGES_WITH]->(b) RETURN ...")` | Git history change coupling |
| Validate HTTP edges with runtime traces | `ingest_traces(project, file_path)` | Enriches edges with latency/frequency |
| Async dispatch (Cloud Tasks, Pub/Sub, SQS) | `search_graph(name_pattern=".*CreateTask.*")` then `trace_call_path` | Find dispatch functions, trace CALLS chains |
| Async dispatch (if ASYNC_CALLS edges exist) | `query_graph("MATCH (a)-[r:ASYNC_CALLS]->(b) RETURN ...")` | Only works when URLs are resolvable literals |
| Dead code detection | `search_graph(max_degree=0, exclude_entry_points=true)` | Degree-based filtering |
@@ -49,21 +53,29 @@ After the initial `index_repository` call, a **background watcher** polls for fi
| What does X call? | `trace_call_path(direction="outbound")` | `query_graph` |
| Full call context (callers + callees) | `trace_call_path(direction="both")` | `direction="outbound"` alone |
| Find functions by pattern | `search_graph(name_pattern="...")` | `trace_call_path` |
| Find functions, exclude noise labels | `search_graph(name_pattern="...", exclude_labels=["Route"])` | unfiltered search |
| Count callers/callees | `search_graph` with `min_degree`/`max_degree` | `query_graph COUNT` (200-row cap) |
| Dead code | `search_graph(max_degree=0, exclude_entry_points=true)` | `query_graph` |
| Cross-service HTTP calls (list edges) | `query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) RETURN ...")` | `search_graph(relationship="HTTP_CALLS")` |
| Filter HTTP calls by URL path | `query_graph("... WHERE r.url_path CONTAINS 'orders' ...")` | unfiltered Cypher |
| Filter HTTP calls by confidence | `query_graph("... WHERE r.confidence >= 0.6 ...")` | unfiltered Cypher |
| Filter by confidence band | `query_graph("... WHERE r.confidence_band = 'high' ...")` | numeric threshold |
| Interface method overrides | `query_graph("MATCH (s)-[:OVERRIDE]->(i) RETURN s.name, i.name")` | manual AST inspection |
| Read refs (callbacks, stored funcs) | `query_graph("MATCH (a)-[:USAGE]->(b) RETURN a.name, b.name")` | grepping for function names |
| Files that change together | `query_graph("MATCH (a)-[r:FILE_CHANGES_WITH]->(b) WHERE r.coupling_score >= 0.5 RETURN ...")` | manual git log analysis |
| Validate HTTP edges with traces | `ingest_traces(project="myproj", file_path="/path/to/traces.json")` | manual trace inspection |
| Async dispatch functions | `search_graph(name_pattern=".*CreateTask.*")` then `trace_call_path` | `search_graph(relationship="ASYNC_CALLS")` |
| Text search | `search_code` or grep | graph tools |
| Complex multi-hop patterns | `query_graph` with Cypher + LIMIT | `search_graph` |
## Tool Reference (11 Tools)
## Tool Reference (12 Tools)
| Tool | Purpose | When to Use |
|------|---------|-------------|
| `index_repository` | Parse and ingest repo into graph | Only needed once per project — auto-sync keeps it fresh |
| `trace_call_path` | BFS from/to a function (exact name match) | Primary tool for call chains — requires EXACT name |
| `search_graph` | Structured search with filters | Find functions, dead code, fan-out, cross-service links |
| `query_graph` | Cypher-like graph queries (200-row cap) | Complex multi-hop patterns, filtered joins |
| `query_graph` | Cypher-like graph queries (200-row cap) | Complex multi-hop patterns, filtered joins, edge property filtering |
| `get_graph_schema` | Node/edge counts, relationship patterns | Understand what's indexed before querying |
| `get_code_snippet` | Read source by qualified name | After search finds a target function |
| `search_code` | Grep-like text search within project | String literals, error messages, TODO comments |
@@ -71,6 +83,7 @@ After the initial `index_repository` call, a **background watcher** polls for fi
| `list_directory` | List files/directories with glob filter | Explore project structure |
| `list_projects` | See all indexed projects | Check if reindex is needed (shows `indexed_at`) |
| `delete_project` | Remove a project from graph | Cleanup stale projects |
| `ingest_traces` | Ingest OTLP JSON traces to validate HTTP_CALLS edges | After deploying — enrich edges with p99_latency, call_count, validated_by_trace |
## Edge Types
@@ -84,6 +97,9 @@ After the initial `index_repository` call, a **background watcher** polls for fi
| `DEFINES_METHOD` | Class defines a method |
| `HANDLES` | Route node handled by a function |
| `IMPLEMENTS` | Type implements an interface |
| `OVERRIDE` | Struct method overrides an interface method (e.g., FileReader.Read → Reader.Read) |
| `USAGE` | Read reference — function passed as callback, stored in variable, not invoked |
| `FILE_CHANGES_WITH` | Git history change coupling — files that frequently change together (properties: coupling_score, co_change_count) |
| `CONTAINS_FILE` / `CONTAINS_FOLDER` / `CONTAINS_PACKAGE` | Structural containment |
## Critical Query Pitfalls
@@ -93,6 +109,31 @@ After the initial `index_repository` call, a **background watcher** polls for fi
query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) RETURN a.name, b.name, r.url_path, r.confidence ORDER BY r.confidence DESC LIMIT 30")
```
**Edge property filtering works in WHERE clauses.** You can filter on any edge property:
```
# Filter by URL path substring
query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.url_path CONTAINS 'bestellung' RETURN a.name, b.name, r.url_path")
# Filter by confidence threshold
query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence >= 0.6 RETURN a.name, b.name, r.confidence LIMIT 20")
# Filter by confidence band (high/medium/speculative)
query_graph("MATCH (a)-[r:HTTP_CALLS]->(b) WHERE r.confidence_band = 'high' RETURN a.name, b.name, r.url_path")
# Change coupling above threshold
query_graph("MATCH (a)-[r:FILE_CHANGES_WITH]->(b) WHERE r.coupling_score >= 0.5 RETURN a.name, b.name, r.coupling_score, r.co_change_count LIMIT 20")
```
**Confidence bands** on HTTP_CALLS/ASYNC_CALLS edges:
- `high` (>= 0.7): Strong structural path match
- `medium` (0.45-0.7): Good signal, current default range
- `speculative` (0.25-0.45): Fuzzy match, needs human review
**`exclude_labels` filters out noise.** When searching by `name_pattern`, Route nodes can dominate results. Use `exclude_labels` to remove them:
```
search_graph(name_pattern=".*[Ff]irestore.*", exclude_labels=["Route"])
```
**`search_graph(relationship="ASYNC_CALLS")` has the same limitation.** ASYNC_CALLS edges only exist when the URL linker can resolve dispatch URLs to route handlers. When URLs are in config variables or env vars (common for Cloud Tasks, Pub/Sub), the linker can't match them. Instead, find dispatch functions by name pattern and trace via CALLS edges:
```
search_graph(name_pattern=".*CreateTask.*|.*create_task.*")