Code aware chunking in RAG strategies using treesitter

Signed-off-by: Christopher Petito <chrisjpetito@gmail.com>

Dockerfile

@@ -6,6 +6,9 @@ ARG ALPINE_VERSION="3.22"
 # xx is a helper for cross-compilation
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.7.0 AS xx
 
+# osxcross contains the MacOSX cross toolchain for xx
+FROM crazymax/osxcross:15.5-debian AS osxcross
+
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS builder-base
 COPY --from=xx / /
 WORKDIR /src
@@ -13,24 +16,50 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=bind,source=go.mod,target=go.mod \
     --mount=type=bind,source=go.sum,target=go.sum \
     go mod download
+ENV CGO_ENABLED=1
+
 
-FROM builder-base AS builder
-COPY . ./
 ARG GIT_TAG
 ARG GIT_COMMIT
 ARG TARGETPLATFORM
 ARG TARGETOS
 ARG TARGETARCH
-RUN --mount=type=cache,target=/root/.cache,id=docker-ai-$TARGETPLATFORM \
+
+FROM builder-base AS builder-darwin
+RUN apk add clang
+COPY . ./
+RUN --mount=type=bind,from=osxcross,src=/osxsdk,target=/xx-sdk \
+    --mount=type=cache,target=/root/.cache,id=docker-ai-$TARGETPLATFORM \
     --mount=type=cache,target=/go/pkg/mod <<EOT
     set -ex
     xx-go build -trimpath -ldflags "-s -w -X 'github.com/docker/cagent/pkg/version.Version=$GIT_TAG' -X 'github.com/docker/cagent/pkg/version.Commit=$GIT_COMMIT'" -o /binaries/cagent-$TARGETOS-$TARGETARCH .
-    xx-verify --static /binaries/cagent-$TARGETOS-$TARGETARCH
-    if [ "$TARGETOS" = "windows" ]; then
-      mv /binaries/cagent-$TARGETOS-$TARGETARCH /binaries/cagent-$TARGETOS-$TARGETARCH.exe
-    fi
+    xx-verify --static /binaries/cagent-darwin-$TARGETARCH
 EOT
 
+FROM builder-base AS builder-linux
+RUN apk add clang
+RUN xx-apk add musl-dev gcc
+COPY . ./
+RUN --mount=type=cache,target=/root/.cache,id=docker-ai-$TARGETPLATFORM \
+    --mount=type=cache,target=/go/pkg/mod <<EOT
+    set -ex
+    xx-go build -trimpath -ldflags "-s -w -linkmode=external -extldflags '-static' -X 'github.com/docker/cagent/pkg/version.Version=$GIT_TAG' -X 'github.com/docker/cagent/pkg/version.Commit=$GIT_COMMIT'" -o /binaries/cagent-$TARGETOS-$TARGETARCH .
+    xx-verify --static /binaries/cagent-linux-$TARGETARCH
+EOT
+
+FROM builder-base AS builder-windows
+RUN apk add zig build-base
+COPY . ./
+RUN --mount=type=cache,target=/root/.cache,id=docker-ai-$TARGETPLATFORM \
+    --mount=type=cache,target=/go/pkg/mod <<EOT
+    set -ex
+    CC="zig cc -target x86_64-windows-gnu" CXX="zig c++ -target x86_64-windows-gnu" xx-go build -trimpath -ldflags "-s -w -X 'github.com/docker/cagent/pkg/version.Version=$GIT_TAG' -X 'github.com/docker/cagent/pkg/version.Commit=$GIT_COMMIT'" -o /binaries/cagent-$TARGETOS-$TARGETARCH .
+    mv /binaries/cagent-$TARGETOS-$TARGETARCH /binaries/cagent-$TARGETOS-$TARGETARCH.exe
+    xx-verify --static /binaries/cagent-windows-$TARGETARCH.exe
+EOT
+
+FROM builder-$TARGETOS AS builder
+
 FROM scratch AS local
 ARG TARGETOS TARGETARCH
 COPY --from=builder /binaries/cagent-$TARGETOS-$TARGETARCH cagent

cagent-schema.json

@@ -728,7 +728,7 @@
                   "chunked-embeddings"
                 ]
               },
-              "model": {
+              "embedding_model": {
                 "type": "string",
                 "description": "Embedding model reference for chunked-embeddings strategies (looked up in models map, or 'auto' for automatic selection)",
                 "examples": [
@@ -804,6 +804,10 @@
                   "respect_word_boundaries": {
                     "type": "boolean",
                     "description": "When true, chunks will split on the nearest whitespace boundary instead of at the exact character limit, preventing words from being truncated."
+                  },
+                  "code_aware": {
+                    "type": "boolean",
+                    "description": "Enable code-aware chunking for source files. When true, the chunking strategy will prefer AST-based or language-aware processors when available (tree-sitter based), and fall back to plain text chunking for unsupported languages."
                   }
                 },
                 "additionalProperties": false

docs/USAGE.md

@@ -972,6 +972,30 @@ models:
 - `chunking.size`: Chunk size in characters (default: `1000`)
 - `chunking.overlap`: Overlap between chunks (default: `75`)
 
+**Code-Aware Chunking:**
+
+When indexing source code, you can enable code-aware chunking to produce semantically aligned chunks based on the code's AST (Abstract Syntax Tree). This keeps functions and methods intact rather than splitting them arbitrarily:
+
+```yaml
+rag:
+  codebase:
+    docs: [./src]
+    strategies:
+      - type: bm25
+        database: ./code.db
+        chunking:
+          size: 2000
+          code_aware: true  # Enable AST-based chunking
+```
+
+- `chunking.code_aware`: When `true`, uses tree-sitter for AST-based chunking (default: `false`), and `size` becomes indicative
+
+**Notes:**
+- Currently supports **Go** source files (`.go`). More languages will be added incrementally.
+- Falls back to plain text chunking for unsupported file types.
+- Produces chunks that align with code structure (functions, methods, type declarations).
+- Particularly useful for code search and retrieval tasks.
+
 **Results:**
 - `limit`: Final number of results (default: `15`)
 - `deduplicate`: Remove duplicates (default: `true`)

go.mod

@@ -33,6 +33,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.19
 	github.com/modelcontextprotocol/go-sdk v1.1.0
 	github.com/openai/openai-go/v3 v3.8.1
+	github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82
 	github.com/spf13/cobra v1.10.1
 	github.com/stretchr/testify v1.11.1
 	github.com/temoto/robotstxt v1.1.2

go.sum

@@ -240,6 +240,8 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8=
 github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY=
+github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82 h1:6C8qej6f1bStuePVkLSFxoU22XBS165D3klxlzRg8F4=
+github.com/smacker/go-tree-sitter v0.0.0-20240827094217-dd81d9e9be82/go.mod h1:xe4pgH49k4SsmkQq5OT8abwhWmnzkhpgnXeekbx2efw=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=

pkg/config/latest/types.go

@@ -518,6 +518,13 @@ func unmarshalChunkingConfig(src any, dst *RAGChunkingConfig) {
 			dst.RespectWordBoundaries = val
 		}
 	}
+
+	// Handle code_aware - YAML should give us a bool
+	if ca, ok := m["code_aware"]; ok {
+		if val, ok := ca.(bool); ok {
+			dst.CodeAware = val
+		}
+	}
 }
 
 // coerceToInt converts various numeric types to int
@@ -579,6 +586,11 @@ type RAGChunkingConfig struct {
 	Size                  int  `json:"size,omitempty"`
 	Overlap               int  `json:"overlap,omitempty"`
 	RespectWordBoundaries bool `json:"respect_word_boundaries,omitempty"`
+	// CodeAware enables code-aware chunking for source files. When true, the
+	// chunking strategy uses tree-sitter for AST-based chunking, producing
+	// semantically aligned chunks (e.g., whole functions). Falls back to
+	// plain text chunking for unsupported languages.
+	CodeAware bool `json:"code_aware,omitempty"`
 }
 
 // UnmarshalYAML implements custom unmarshaling to apply sensible defaults for chunking

pkg/rag/chunk/chunk.go

@@ -19,26 +19,21 @@ type Chunk struct {
 	Metadata map[string]string
 }
 
-// Processor handles document processing
-type Processor struct{}
-
-// New creates a new document processor
-func New() *Processor {
-	return &Processor{}
+// DocumentProcessor takes file content and returns chunks.
+// Config (size, overlap, etc.) is set at construction time.
+type DocumentProcessor interface {
+	Process(path string, content []byte) ([]Chunk, error)
 }
 
-// ProcessFile reads a file and splits it into chunks
-func (p *Processor) ProcessFile(path string, chunkSize, overlap int, respectWordBoundaries bool) ([]Chunk, error) {
-	content, err := os.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read file: %w", err)
-	}
-
-	return p.ChunkText(string(content), chunkSize, overlap, respectWordBoundaries), nil
+// TextDocumentProcessor is the default text-based chunker
+type TextDocumentProcessor struct {
+	size                  int
+	overlap               int
+	respectWordBoundaries bool
 }
 
-// ChunkText splits text into overlapping chunks
-func (p *Processor) ChunkText(text string, size, overlap int, respectWordBoundaries bool) []Chunk {
+// NewTextDocumentProcessor creates a text-based document processor
+func NewTextDocumentProcessor(size, overlap int, respectWordBoundaries bool) *TextDocumentProcessor {
 	if size <= 0 {
 		size = 1000
 	}
@@ -48,7 +43,20 @@ func (p *Processor) ChunkText(text string, size, overlap int, respectWordBoundar
 	if overlap >= size {
 		overlap = size / 2
 	}
+	return &TextDocumentProcessor{
+		size:                  size,
+		overlap:               overlap,
+		respectWordBoundaries: respectWordBoundaries,
+	}
+}
 
+// Process implements DocumentProcessor for text-based chunking
+func (t *TextDocumentProcessor) Process(_ string, content []byte) ([]Chunk, error) {
+	return t.chunkText(string(content)), nil
+}
+
+// chunkText splits text into overlapping chunks
+func (t *TextDocumentProcessor) chunkText(text string) []Chunk {
 	var chunks []Chunk
 	runes := []rune(text)
 	totalLen := len(runes)
@@ -62,7 +70,7 @@ func (p *Processor) ChunkText(text string, size, overlap int, respectWordBoundar
 
 	for start < totalLen {
 		// Calculate end position (start + size, but not beyond document end)
-		end := start + size
+		end := start + t.size
 		if end > totalLen {
 			end = totalLen
 		}
@@ -72,14 +80,14 @@ func (p *Processor) ChunkText(text string, size, overlap int, respectWordBoundar
 		// For the final chunk (end == totalLen) we always take the remainder
 		// of the document as-is to avoid generating progressively smaller
 		// tail chunks.
-		if respectWordBoundaries && end > start && end < totalLen {
+		if t.respectWordBoundaries && end > start && end < totalLen {
 			// Limit search to the current chunk window.
 			target := end
 
 			// Backtrack from target to find whitespace; if none is found
 			// in a reasonable range, keep the original end so that we
 			// still make progress even for very long "words".
-			searchEnd := p.findNearestWhitespace(runes[start:target+1], target-start) + start
+			searchEnd := t.findNearestWhitespace(runes[start:target+1], target-start) + start
 			if searchEnd > start && searchEnd < end {
 				end = searchEnd
 			}
@@ -100,7 +108,7 @@ func (p *Processor) ChunkText(text string, size, overlap int, respectWordBoundar
 		}
 
 		// Next chunk starts at the end of the previous chunk minus overlap
-		nextStart := end - overlap
+		nextStart := end - t.overlap
 
 		// CRITICAL: Ensure we always make forward progress
 		// If nextStart would move us backward or keep us in place, advance by at least 1
@@ -111,14 +119,14 @@ func (p *Processor) ChunkText(text string, size, overlap int, respectWordBoundar
 		// When respecting word boundaries, make sure the next chunk
 		// does not start in the middle of a word. Move the start
 		// forward to the next whitespace, then to the next non-whitespace.
-		if respectWordBoundaries {
+		if t.respectWordBoundaries {
 			// Move forward until we hit whitespace or end-of-text
-			for nextStart < totalLen && !p.isWhitespace(runes[nextStart]) {
+			for nextStart < totalLen && !t.isWhitespace(runes[nextStart]) {
 				nextStart++
 			}
 			// Skip the whitespace itself so we start at the first character
 			// of the next word (if any).
-			for nextStart < totalLen && p.isWhitespace(runes[nextStart]) {
+			for nextStart < totalLen && t.isWhitespace(runes[nextStart]) {
 				nextStart++
 			}
 		}
@@ -131,7 +139,7 @@ func (p *Processor) ChunkText(text string, size, overlap int, respectWordBoundar
 
 // findNearestWhitespace finds the nearest whitespace boundary to the target position
 // It searches backward first (within a reasonable distance), then forward if needed
-func (p *Processor) findNearestWhitespace(runes []rune, target int) int {
+func (t *TextDocumentProcessor) findNearestWhitespace(runes []rune, target int) int {
 	// Don't search beyond 20% of the total length in either direction
 	maxSearchDistance := len(runes) / 5
 	if maxSearchDistance < 50 {
@@ -144,9 +152,9 @@ func (p *Processor) findNearestWhitespace(runes []rune, target int) int {
 	// Search backward first (prefer to keep chunks slightly smaller)
 	for i := 0; i < maxSearchDistance && target-i > 0; i++ {
 		pos := target - i
-		if p.isWhitespace(runes[pos]) {
+		if t.isWhitespace(runes[pos]) {
 			// Skip consecutive whitespace
-			for pos > 0 && p.isWhitespace(runes[pos-1]) {
+			for pos > 0 && t.isWhitespace(runes[pos-1]) {
 				pos--
 			}
 			return pos
@@ -156,7 +164,7 @@ func (p *Processor) findNearestWhitespace(runes []rune, target int) int {
 	// Search forward if no whitespace found backward
 	for i := 1; i < maxSearchDistance && target+i < len(runes); i++ {
 		pos := target + i
-		if p.isWhitespace(runes[pos]) {
+		if t.isWhitespace(runes[pos]) {
 			return pos
 		}
 	}
@@ -166,12 +174,23 @@ func (p *Processor) findNearestWhitespace(runes []rune, target int) int {
 }
 
 // isWhitespace checks if a rune is whitespace
-func (p *Processor) isWhitespace(r rune) bool {
+func (t *TextDocumentProcessor) isWhitespace(r rune) bool {
 	return r == ' ' || r == '\t' || r == '\n' || r == '\r'
 }
 
+// --- File utility functions (standalone, not tied to any processor) ---
+
+// ProcessFile reads a file and processes it using the given document processor
+func ProcessFile(dp DocumentProcessor, path string) ([]Chunk, error) {
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read file: %w", err)
+	}
+	return dp.Process(path, content)
+}
+
 // FileHash calculates SHA256 hash of a file
-func (p *Processor) FileHash(path string) (string, error) {
+func FileHash(path string) (string, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return "", fmt.Errorf("failed to open file: %w", err)
@@ -188,12 +207,12 @@ func (p *Processor) FileHash(path string) (string, error) {
 
 // CollectFiles recursively collects all files from given paths
 // Skips paths that don't exist instead of returning an error
-func (p *Processor) CollectFiles(paths []string) ([]string, error) {
+func CollectFiles(paths []string) ([]string, error) {
 	var files []string
 	seen := make(map[string]bool)
 
 	for _, pattern := range paths {
-		expanded, err := p.expandPattern(pattern)
+		expanded, err := expandPattern(pattern)
 		if err != nil {
 			return nil, err
 		}
@@ -245,7 +264,7 @@ func (p *Processor) CollectFiles(paths []string) ([]string, error) {
 
 // Matches reports whether the given path matches any configured document path or glob pattern.
 // To be used in file watchers to determine if a new/changed file matches the glob patterns or not.
-func (p *Processor) Matches(path string, patterns []string) (bool, error) {
+func Matches(path string, patterns []string) (bool, error) {
 	if len(patterns) == 0 {
 		return false, nil
 	}
@@ -293,7 +312,7 @@ func (p *Processor) Matches(path string, patterns []string) (bool, error) {
 	return false, nil
 }
 
-func (p *Processor) expandPattern(pattern string) ([]string, error) {
+func expandPattern(pattern string) ([]string, error) {
 	if !hasGlob(pattern) {
 		return []string{normalizePath(pattern)}, nil
 	}

pkg/rag/chunk/chunk_test.go

@@ -90,10 +90,9 @@ func TestCollectFiles_Globs(t *testing.T) {
 		},
 	}
 
-	p := New()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := p.CollectFiles(tt.paths)
+			got, err := CollectFiles(tt.paths)
 			require.NoError(t, err)
 
 			sort.Strings(got)
@@ -106,8 +105,6 @@ func TestCollectFiles_Globs(t *testing.T) {
 func TestChunkText_RespectWordBoundaries(t *testing.T) {
 	t.Parallel()
 
-	p := New()
-
 	tests := []struct {
 		name                  string
 		text                  string
@@ -191,7 +188,9 @@ func TestChunkText_RespectWordBoundaries(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			chunks := p.ChunkText(tt.text, tt.size, tt.overlap, tt.respectWordBoundaries)
+			pp := NewTextDocumentProcessor(tt.size, tt.overlap, tt.respectWordBoundaries)
+			chunks, err := pp.Process("test.txt", []byte(tt.text))
+			require.NoError(t, err)
 
 			assert.NotEmpty(t, chunks)
 			if tt.validateFunc != nil {
@@ -204,11 +203,12 @@ func TestChunkText_RespectWordBoundaries(t *testing.T) {
 func TestChunkText_BackwardCompatibility(t *testing.T) {
 	t.Parallel()
 
-	p := New()
+	pp := NewTextDocumentProcessor(10, 2, false)
 
 	// Test that default behavior (respectWordBoundaries=false) still works as before
 	text := "This is a test text for chunking"
-	chunks := p.ChunkText(text, 10, 2, false)
+	chunks, err := pp.Process("test.txt", []byte(text))
+	require.NoError(t, err)
 
 	assert.NotEmpty(t, chunks)
 	assert.Greater(t, len(chunks), 1) // Should create multiple chunks
@@ -217,7 +217,7 @@ func TestChunkText_BackwardCompatibility(t *testing.T) {
 func TestFindNearestWhitespace(t *testing.T) {
 	t.Parallel()
 
-	p := New()
+	pp := NewTextDocumentProcessor(1000, 0, true)
 
 	tests := []struct {
 		name    string
@@ -250,7 +250,7 @@ func TestFindNearestWhitespace(t *testing.T) {
 			t.Parallel()
 
 			runes := []rune(tt.text)
-			pos := p.findNearestWhitespace(runes, tt.target)
+			pos := pp.findNearestWhitespace(runes, tt.target)
 
 			assert.LessOrEqual(t, pos, len(runes))
 			assert.GreaterOrEqual(t, pos, 0)

pkg/rag/manager.go

@@ -161,11 +161,13 @@ func (m *Manager) Initialize(ctx context.Context) error {
 				"rag_name", m.name,
 				"strategy", strategyName,
 				"num_docs", len(strategyCfg.Docs),
-				"chunk_size", strategyCfg.ChunkSize,
-				"chunk_overlap", strategyCfg.ChunkOverlap)
+				"chunk_size", strategyCfg.Chunking.Size,
+				"chunk_overlap", strategyCfg.Chunking.Overlap,
+				"respect_word_boundaries", strategyCfg.Chunking.RespectWordBoundaries,
+				"code_aware", strategyCfg.Chunking.CodeAware)
 
 			start := time.Now()
-			err := strategyImpl.Initialize(ctx, strategyCfg.Docs, strategyCfg.ChunkSize, strategyCfg.ChunkOverlap, strategyCfg.RespectWordBoundaries)
+			err := strategyImpl.Initialize(ctx, strategyCfg.Docs, strategyCfg.Chunking)
 			indexDuration := time.Since(start)
 			slog.Debug("[RAG Manager] Strategy indexing duration",
 				"rag_name", m.name,
@@ -433,7 +435,7 @@ func getStrategyNames(stratMap map[string]strategy.Strategy) []string {
 func (m *Manager) CheckAndReindexChangedFiles(ctx context.Context) error {
 	for strategyName, strategyImpl := range m.strategies {
 		strategyCfg := m.strategyConfigs[strategyName]
-		if err := strategyImpl.CheckAndReindexChangedFiles(ctx, strategyCfg.Docs, strategyCfg.ChunkSize, strategyCfg.ChunkOverlap, strategyCfg.RespectWordBoundaries); err != nil {
+		if err := strategyImpl.CheckAndReindexChangedFiles(ctx, strategyCfg.Docs, strategyCfg.Chunking); err != nil {
 			return fmt.Errorf("strategy %s failed: %w", strategyName, err)
 		}
 	}
@@ -444,7 +446,7 @@ func (m *Manager) CheckAndReindexChangedFiles(ctx context.Context) error {
 func (m *Manager) StartFileWatcher(ctx context.Context) error {
 	for strategyName, strategyImpl := range m.strategies {
 		strategyCfg := m.strategyConfigs[strategyName]
-		if err := strategyImpl.StartFileWatcher(ctx, strategyCfg.Docs, strategyCfg.ChunkSize, strategyCfg.ChunkOverlap, strategyCfg.RespectWordBoundaries); err != nil {
+		if err := strategyImpl.StartFileWatcher(ctx, strategyCfg.Docs, strategyCfg.Chunking); err != nil {
 			return fmt.Errorf("strategy %s failed: %w", strategyName, err)
 		}
 	}

pkg/rag/strategy/bm25.go

@@ -16,6 +16,7 @@ import (
 	"github.com/docker/cagent/pkg/config/latest"
 	"github.com/docker/cagent/pkg/rag/chunk"
 	"github.com/docker/cagent/pkg/rag/database"
+	"github.com/docker/cagent/pkg/rag/treesitter"
 	"github.com/docker/cagent/pkg/rag/types"
 )
 
@@ -57,21 +58,8 @@ func NewBM25FromConfig(_ context.Context, cfg latest.RAGStrategyConfig, buildCtx
 		limit = 5
 	}
 
-	// Extract chunking configuration with defaults
-	chunkSize := cfg.Chunking.Size
-	chunkOverlap := cfg.Chunking.Overlap
-	respectWordBoundaries := cfg.Chunking.RespectWordBoundaries
-	slog.Debug("Chunking config from YAML",
-		"strategy", "bm25",
-		"chunk_size", chunkSize,
-		"chunk_overlap", chunkOverlap,
-		"respect_word_boundaries", respectWordBoundaries)
-	if chunkSize == 0 {
-		chunkSize = 1500 // General text: good paragraph/section size
-	}
-	if chunkOverlap == 0 {
-		chunkOverlap = 75
-	}
+	// Parse chunking configuration
+	chunkingCfg := ParseChunkingConfig(cfg)
 
 	// Create strategy
 	strategy := newBM25Strategy(
@@ -80,30 +68,29 @@ func NewBM25FromConfig(_ context.Context, cfg latest.RAGStrategyConfig, buildCtx
 		events,
 		k1,
 		bParam,
+		chunkingCfg,
 	)
 
 	return &Config{
-		Name:                  "bm25",
-		Strategy:              strategy,
-		Docs:                  docs,
-		Limit:                 limit,
-		Threshold:             thresholdVal,
-		ChunkSize:             chunkSize,
-		ChunkOverlap:          chunkOverlap,
-		RespectWordBoundaries: respectWordBoundaries,
+		Name:      "bm25",
+		Strategy:  strategy,
+		Docs:      docs,
+		Limit:     limit,
+		Threshold: thresholdVal,
+		Chunking:  chunkingCfg,
 	}, nil
 }
 
 // BM25Strategy implements BM25 keyword-based retrieval
 // BM25 is a ranking function that uses term frequency and inverse document frequency
 type BM25Strategy struct {
-	name       string
-	db         *bm25DB
-	processor  *chunk.Processor
-	fileHashes map[string]string
-	watcher    *fsnotify.Watcher
-	watcherMu  sync.Mutex
-	events     chan<- types.Event
+	name         string
+	db           *bm25DB
+	docProcessor chunk.DocumentProcessor
+	fileHashes   map[string]string
+	watcher      *fsnotify.Watcher
+	watcherMu    sync.Mutex
+	events       chan<- types.Event
 
 	// BM25 parameters
 	k1           float64 // term frequency saturation parameter (typically 1.2 to 2.0)
@@ -113,26 +100,34 @@ type BM25Strategy struct {
 }
 
 // newBM25Strategy creates a new BM25-based retrieval strategy
-func newBM25Strategy(name string, db *bm25DB, events chan<- types.Event, k1, b float64) *BM25Strategy {
+func newBM25Strategy(name string, db *bm25DB, events chan<- types.Event, k1, b float64, chunking ChunkingConfig) *BM25Strategy {
+	// Create the appropriate document processor based on config
+	var dp chunk.DocumentProcessor
+	if chunking.CodeAware {
+		dp = treesitter.NewDocumentProcessor(chunking.Size, chunking.Overlap, chunking.RespectWordBoundaries)
+	} else {
+		dp = chunk.NewTextDocumentProcessor(chunking.Size, chunking.Overlap, chunking.RespectWordBoundaries)
+	}
+
 	return &BM25Strategy{
-		name:       name,
-		db:         db,
-		processor:  chunk.New(),
-		fileHashes: make(map[string]string),
-		events:     events,
-		k1:         k1,
-		b:          b,
+		name:         name,
+		db:           db,
+		docProcessor: dp,
+		fileHashes:   make(map[string]string),
+		events:       events,
+		k1:           k1,
+		b:            b,
 	}
 }
 
 // Initialize indexes all documents for BM25 retrieval
-func (s *BM25Strategy) Initialize(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
+func (s *BM25Strategy) Initialize(ctx context.Context, docPaths []string, chunking ChunkingConfig) error {
 	slog.Info("Starting BM25 strategy initialization",
 		"name", s.name,
 		"doc_paths", docPaths,
-		"chunk_size", chunkSize,
-		"chunk_overlap", chunkOverlap,
-		"respect_word_boundaries", respectWordBoundaries)
+		"chunk_size", chunking.Size,
+		"chunk_overlap", chunking.Overlap,
+		"respect_word_boundaries", chunking.RespectWordBoundaries)
 
 	// Load existing file hashes
 	slog.Debug("Loading existing file hashes", "strategy", s.name)
@@ -142,7 +137,7 @@ func (s *BM25Strategy) Initialize(ctx context.Context, docPaths []string, chunkS
 
 	// Collect all files
 	slog.Debug("Collecting files", "strategy", s.name, "paths", docPaths)
-	files, err := s.processor.CollectFiles(docPaths)
+	files, err := chunk.CollectFiles(docPaths)
 	if err != nil {
 		s.emitEvent(types.Event{Type: "error", Error: err})
 		return fmt.Errorf("failed to collect files: %w", err)
@@ -217,7 +212,7 @@ func (s *BM25Strategy) Initialize(ctx context.Context, docPaths []string, chunkS
 			},
 		})
 
-		if err := s.indexFile(ctx, status.path, chunkSize, chunkOverlap, respectWordBoundaries); err != nil {
+		if err := s.indexFile(ctx, status.path); err != nil {
 			slog.Error("Failed to index file", "path", status.path, "error", err)
 			continue
 		}
@@ -293,8 +288,8 @@ func (s *BM25Strategy) Query(ctx context.Context, query string, numResults int,
 }
 
 // CheckAndReindexChangedFiles checks for file changes and re-indexes if needed
-func (s *BM25Strategy) CheckAndReindexChangedFiles(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
-	files, err := s.processor.CollectFiles(docPaths)
+func (s *BM25Strategy) CheckAndReindexChangedFiles(ctx context.Context, docPaths []string, chunking ChunkingConfig) error {
+	files, err := chunk.CollectFiles(docPaths)
 	if err != nil {
 		return fmt.Errorf("failed to collect files: %w", err)
 	}
@@ -312,7 +307,7 @@ func (s *BM25Strategy) CheckAndReindexChangedFiles(ctx context.Context, docPaths
 
 		if needsIndexing {
 			slog.Info("File changed, re-indexing", "path", filePath)
-			if err := s.indexFile(ctx, filePath, chunkSize, chunkOverlap, respectWordBoundaries); err != nil {
+			if err := s.indexFile(ctx, filePath); err != nil {
 				slog.Error("Failed to re-index file", "path", filePath, "error", err)
 			}
 		}
@@ -331,7 +326,7 @@ func (s *BM25Strategy) CheckAndReindexChangedFiles(ctx context.Context, docPaths
 }
 
 // StartFileWatcher starts monitoring files for changes
-func (s *BM25Strategy) StartFileWatcher(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
+func (s *BM25Strategy) StartFileWatcher(ctx context.Context, docPaths []string, chunking ChunkingConfig) error {
 	s.watcherMu.Lock()
 	defer s.watcherMu.Unlock()
 
@@ -348,7 +343,7 @@ func (s *BM25Strategy) StartFileWatcher(ctx context.Context, docPaths []string,
 		}
 	}
 
-	go s.watchLoop(ctx, docPaths, chunkSize, chunkOverlap, respectWordBoundaries)
+	go s.watchLoop(ctx, docPaths)
 
 	slog.Info("File watcher started", "strategy", s.name)
 	return nil
@@ -508,7 +503,7 @@ func (s *BM25Strategy) loadExistingHashes(ctx context.Context) error {
 }
 
 func (s *BM25Strategy) needsIndexing(_ context.Context, filePath string) (bool, error) {
-	currentHash, err := s.processor.FileHash(filePath)
+	currentHash, err := chunk.FileHash(filePath)
 	if err != nil {
 		return false, fmt.Errorf("failed to hash file: %w", err)
 	}
@@ -521,8 +516,8 @@ func (s *BM25Strategy) needsIndexing(_ context.Context, filePath string) (bool,
 	return storedHash != currentHash, nil
 }
 
-func (s *BM25Strategy) indexFile(ctx context.Context, filePath string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
-	fileHash, err := s.processor.FileHash(filePath)
+func (s *BM25Strategy) indexFile(ctx context.Context, filePath string) error {
+	fileHash, err := chunk.FileHash(filePath)
 	if err != nil {
 		return fmt.Errorf("failed to hash file: %w", err)
 	}
@@ -531,7 +526,7 @@ func (s *BM25Strategy) indexFile(ctx context.Context, filePath string, chunkSize
 		return fmt.Errorf("failed to delete old documents: %w", err)
 	}
 
-	chunks, err := s.processor.ProcessFile(filePath, chunkSize, chunkOverlap, respectWordBoundaries)
+	chunks, err := chunk.ProcessFile(s.docProcessor, filePath)
 	if err != nil {
 		return fmt.Errorf("failed to process file: %w", err)
 	}
@@ -615,7 +610,7 @@ func (s *BM25Strategy) addPathToWatcher(path string) error {
 	}
 
 	if stat.IsDir() {
-		files, err := s.processor.CollectFiles([]string{absPath})
+		files, err := chunk.CollectFiles([]string{absPath})
 		if err != nil {
 			return fmt.Errorf("failed to collect files: %w", err)
 		}
@@ -633,7 +628,7 @@ func (s *BM25Strategy) addPathToWatcher(path string) error {
 	return nil
 }
 
-func (s *BM25Strategy) watchLoop(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) {
+func (s *BM25Strategy) watchLoop(ctx context.Context, docPaths []string) {
 	var debounceTimer *time.Timer
 	debounceDuration := 2 * time.Second
 	pendingChanges := make(map[string]bool)
@@ -654,7 +649,7 @@ func (s *BM25Strategy) watchLoop(ctx context.Context, docPaths []string, chunkSi
 
 		for _, file := range changedFiles {
 			// Check if the file matches any of the configured document paths/patterns
-			matches, matchErr := s.processor.Matches(file, docPaths)
+			matches, matchErr := chunk.Matches(file, docPaths)
 			if matchErr != nil {
 				slog.Error("Failed to match path", "file", file, "error", matchErr)
 				continue
@@ -669,7 +664,7 @@ func (s *BM25Strategy) watchLoop(ctx context.Context, docPaths []string, chunkSi
 			}
 
 			slog.Debug("Indexing file", "path", file, "strategy", s.name)
-			if err := s.indexFile(ctx, file, chunkSize, chunkOverlap, respectWordBoundaries); err != nil {
+			if err := s.indexFile(ctx, file); err != nil {
 				slog.Error("Failed to re-index file", "path", file, "error", err)
 			}
 		}
@@ -701,7 +696,7 @@ func (s *BM25Strategy) watchLoop(ctx context.Context, docPaths []string, chunkSi
 			}
 
 			// Early filter: only track changes for files that match configured doc patterns
-			matches, err := s.processor.Matches(event.Name, docPaths)
+			matches, err := chunk.Matches(event.Name, docPaths)
 			if err != nil || !matches {
 				continue
 			}

pkg/rag/strategy/chunked_embeddings.go

@@ -85,16 +85,15 @@ func NewChunkedEmbeddingsFromConfig(ctx context.Context, cfg latest.RAGStrategyC
 		ModelsStore:          embeddingCfg.ModelsStore,
 		EmbeddingConcurrency: maxConcurrency,
 		FileIndexConcurrency: fileIndexConcurrency,
+		Chunking:             chunkingCfg,
 	})
 
 	return &Config{
-		Name:                  strategyName,
-		Strategy:              store,
-		Docs:                  docs,
-		Limit:                 limit,
-		Threshold:             threshold,
-		ChunkSize:             chunkingCfg.Size,
-		ChunkOverlap:          chunkingCfg.Overlap,
-		RespectWordBoundaries: chunkingCfg.RespectWordBoundaries,
+		Name:      strategyName,
+		Strategy:  store,
+		Docs:      docs,
+		Limit:     limit,
+		Threshold: threshold,
+		Chunking:  chunkingCfg,
 	}, nil
 }

pkg/rag/strategy/helpers.go

@@ -207,18 +207,17 @@ func ResolveModelConfig(ref string, models map[string]latest.ModelConfig) (lates
 	return latest.ModelConfig{}, fmt.Errorf("model %q not found", ref)
 }
 
-// ChunkingConfig holds chunking parameters.
-type ChunkingConfig struct {
-	Size                  int
-	Overlap               int
-	RespectWordBoundaries bool
-}
-
 // ParseChunkingConfig extracts chunking configuration from RAGStrategyConfig.
 func ParseChunkingConfig(cfg latest.RAGStrategyConfig) ChunkingConfig {
 	chunkSize := cfg.Chunking.Size
 	if chunkSize == 0 {
-		chunkSize = 1500
+		// Use larger default for code-aware chunking since TreeSitter groups
+		// complete functions and benefits from more context
+		if cfg.Chunking.CodeAware {
+			chunkSize = 4000 // Code-aware: captures 2-4 medium functions per chunk
+		} else {
+			chunkSize = 1500 // General text: good paragraph/section size
+		}
 	}
 
 	chunkOverlap := cfg.Chunking.Overlap
@@ -230,5 +229,6 @@ func ParseChunkingConfig(cfg latest.RAGStrategyConfig) ChunkingConfig {
 		Size:                  chunkSize,
 		Overlap:               chunkOverlap,
 		RespectWordBoundaries: cfg.Chunking.RespectWordBoundaries,
+		CodeAware:             cfg.Chunking.CodeAware,
 	}
 }

pkg/rag/strategy/types.go

@@ -10,17 +10,17 @@ import (
 // This is the canonical definition used by both the strategies and rag packages.
 type Strategy interface {
 	// Initialize indexes all documents from the given paths.
-	Initialize(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error
+	Initialize(ctx context.Context, docPaths []string, chunking ChunkingConfig) error
 
 	// Query searches for relevant documents using the strategy's retrieval method.
 	// numResults is the maximum number of candidates to retrieve (before fusion).
 	Query(ctx context.Context, query string, numResults int, threshold float64) ([]database.SearchResult, error)
 
 	// CheckAndReindexChangedFiles checks for file changes and re-indexes if needed.
-	CheckAndReindexChangedFiles(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error
+	CheckAndReindexChangedFiles(ctx context.Context, docPaths []string, chunking ChunkingConfig) error
 
 	// StartFileWatcher starts monitoring files for changes.
-	StartFileWatcher(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error
+	StartFileWatcher(ctx context.Context, docPaths []string, chunking ChunkingConfig) error
 
 	// Close releases resources held by the strategy.
 	Close() error
@@ -28,12 +28,18 @@ type Strategy interface {
 
 // Config contains a strategy and its runtime configuration.
 type Config struct {
-	Name                  string
-	Strategy              Strategy
-	Docs                  []string // Merged document paths (shared + strategy-specific)
-	Limit                 int      // Max results for this strategy
-	Threshold             float64  // Score threshold
-	ChunkSize             int      // Chunk size for this strategy
-	ChunkOverlap          int      // Chunk overlap for this strategy
-	RespectWordBoundaries bool     // Whether to chunk on word boundaries
+	Name      string
+	Strategy  Strategy
+	Docs      []string // Merged document paths (shared + strategy-specific)
+	Limit     int      // Max results for this strategy
+	Threshold float64  // Score threshold
+	Chunking  ChunkingConfig
+}
+
+// ChunkingConfig holds chunking parameters.
+type ChunkingConfig struct {
+	Size                  int
+	Overlap               int
+	RespectWordBoundaries bool
+	CodeAware             bool
 }

pkg/rag/strategy/vector_store.go

@@ -17,6 +17,7 @@ import (
 	"github.com/docker/cagent/pkg/rag/chunk"
 	"github.com/docker/cagent/pkg/rag/database"
 	"github.com/docker/cagent/pkg/rag/embed"
+	"github.com/docker/cagent/pkg/rag/treesitter"
 	"github.com/docker/cagent/pkg/rag/types"
 )
 
@@ -47,7 +48,7 @@ type VectorStore struct {
 	name         string
 	db           vectorStoreDB
 	embedder     *embed.Embedder
-	processor    *chunk.Processor
+	docProcessor chunk.DocumentProcessor
 	fileHashes   map[string]string
 	fileHashesMu sync.Mutex // Protects fileHashes map for concurrent access
 	watcher      *fsnotify.Watcher
@@ -114,15 +115,24 @@ type VectorStoreConfig struct {
 	ModelsStore          modelStore
 	EmbeddingConcurrency int
 	FileIndexConcurrency int
+	Chunking             ChunkingConfig
 }
 
 // NewVectorStore creates a new vector store with the given configuration.
 func NewVectorStore(cfg VectorStoreConfig) *VectorStore {
+	// Create the appropriate document processor based on config
+	var dp chunk.DocumentProcessor
+	if cfg.Chunking.CodeAware {
+		dp = treesitter.NewDocumentProcessor(cfg.Chunking.Size, cfg.Chunking.Overlap, cfg.Chunking.RespectWordBoundaries)
+	} else {
+		dp = chunk.NewTextDocumentProcessor(cfg.Chunking.Size, cfg.Chunking.Overlap, cfg.Chunking.RespectWordBoundaries)
+	}
+
 	s := &VectorStore{
 		name:                  cfg.Name,
 		db:                    cfg.Database,
 		embedder:              cfg.Embedder,
-		processor:             chunk.New(),
+		docProcessor:          dp,
 		fileHashes:            make(map[string]string),
 		events:                cfg.Events,
 		similarityMetric:      cfg.SimilarityMetric,
@@ -200,13 +210,14 @@ func (s *VectorStore) recordUsage(tokens int, cost float64) {
 }
 
 // Initialize indexes all documents
-func (s *VectorStore) Initialize(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
+func (s *VectorStore) Initialize(ctx context.Context, docPaths []string, chunking ChunkingConfig) error {
 	slog.Info("Starting vector store initialization",
 		"name", s.name,
 		"doc_paths", docPaths,
-		"chunk_size", chunkSize,
-		"chunk_overlap", chunkOverlap,
-		"respect_word_boundaries", respectWordBoundaries)
+		"chunk_size", chunking.Size,
+		"chunk_overlap", chunking.Overlap,
+		"respect_word_boundaries", chunking.RespectWordBoundaries,
+		"code_aware", chunking.CodeAware)
 
 	// Load existing file hashes from metadata
 	slog.Debug("Loading existing file hashes", "strategy", s.name)
@@ -216,7 +227,7 @@ func (s *VectorStore) Initialize(ctx context.Context, docPaths []string, chunkSi
 
 	// Collect all files
 	slog.Debug("Collecting files", "strategy", s.name, "paths", docPaths)
-	files, err := s.processor.CollectFiles(docPaths)
+	files, err := chunk.CollectFiles(docPaths)
 	if err != nil {
 		s.emitEvent(types.Event{Type: "error", Error: err})
 		return fmt.Errorf("failed to collect files: %w", err)
@@ -300,7 +311,7 @@ func (s *VectorStore) Initialize(ctx context.Context, docPaths []string, chunkSi
 			}
 
 			// Index the file
-			if err := s.indexFile(gctx, status.path, chunkSize, chunkOverlap, respectWordBoundaries); err != nil {
+			if err := s.indexFile(gctx, status.path); err != nil {
 				slog.Error("Failed to index file", "path", status.path, "error", err)
 				// Don't return error - continue indexing other files
 				return nil
@@ -373,8 +384,8 @@ func (s *VectorStore) Query(ctx context.Context, query string, numResults int, t
 }
 
 // CheckAndReindexChangedFiles checks for file changes and re-indexes if needed
-func (s *VectorStore) CheckAndReindexChangedFiles(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
-	files, err := s.processor.CollectFiles(docPaths)
+func (s *VectorStore) CheckAndReindexChangedFiles(ctx context.Context, docPaths []string, chunking ChunkingConfig) error {
+	files, err := chunk.CollectFiles(docPaths)
 	if err != nil {
 		return fmt.Errorf("failed to collect files: %w", err)
 	}
@@ -392,7 +403,7 @@ func (s *VectorStore) CheckAndReindexChangedFiles(ctx context.Context, docPaths
 
 		if needsIndexing {
 			slog.Info("File changed, re-indexing", "path", filePath)
-			if err := s.indexFile(ctx, filePath, chunkSize, chunkOverlap, respectWordBoundaries); err != nil {
+			if err := s.indexFile(ctx, filePath); err != nil {
 				slog.Error("Failed to re-index file", "path", filePath, "error", err)
 			}
 		}
@@ -406,7 +417,7 @@ func (s *VectorStore) CheckAndReindexChangedFiles(ctx context.Context, docPaths
 }
 
 // StartFileWatcher starts monitoring files for changes
-func (s *VectorStore) StartFileWatcher(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
+func (s *VectorStore) StartFileWatcher(ctx context.Context, docPaths []string, chunking ChunkingConfig) error {
 	s.watcherMu.Lock()
 	defer s.watcherMu.Unlock()
 
@@ -424,7 +435,7 @@ func (s *VectorStore) StartFileWatcher(ctx context.Context, docPaths []string, c
 		slog.Debug("Watching path for changes", "strategy", s.name, "path", docPath)
 	}
 
-	go s.watchLoop(ctx, docPaths, chunkSize, chunkOverlap, respectWordBoundaries)
+	go s.watchLoop(ctx, docPaths)
 
 	slog.Info("File watcher started", "strategy", s.name, "paths", docPaths)
 	return nil
@@ -490,7 +501,7 @@ func (s *VectorStore) loadExistingHashes(ctx context.Context) error {
 }
 
 func (s *VectorStore) needsIndexing(_ context.Context, filePath string) (bool, error) {
-	currentHash, err := s.processor.FileHash(filePath)
+	currentHash, err := chunk.FileHash(filePath)
 	if err != nil {
 		return false, fmt.Errorf("failed to hash file: %w", err)
 	}
@@ -512,8 +523,8 @@ func (s *VectorStore) needsIndexing(_ context.Context, filePath string) (bool, e
 	return needsIndexing, nil
 }
 
-func (s *VectorStore) indexFile(ctx context.Context, filePath string, chunkSize, chunkOverlap int, respectWordBoundaries bool) error {
-	fileHash, err := s.processor.FileHash(filePath)
+func (s *VectorStore) indexFile(ctx context.Context, filePath string) error {
+	fileHash, err := chunk.FileHash(filePath)
 	if err != nil {
 		return fmt.Errorf("failed to hash file: %w", err)
 	}
@@ -522,7 +533,7 @@ func (s *VectorStore) indexFile(ctx context.Context, filePath string, chunkSize,
 		return fmt.Errorf("failed to delete old documents: %w", err)
 	}
 
-	chunks, err := s.processor.ProcessFile(filePath, chunkSize, chunkOverlap, respectWordBoundaries)
+	chunks, err := chunk.ProcessFile(s.docProcessor, filePath)
 	if err != nil {
 		return fmt.Errorf("failed to process file: %w", err)
 	}
@@ -687,7 +698,7 @@ func (s *VectorStore) cleanupOrphanedDocuments(ctx context.Context, seenFiles ma
 
 func (s *VectorStore) addPathToWatcher(path string) error {
 	// Resolve path(s) using Processor (handles globs, directories, files)
-	files, err := s.processor.CollectFiles([]string{path})
+	files, err := chunk.CollectFiles([]string{path})
 	if err != nil {
 		return fmt.Errorf("failed to collect files for watching: %w", err)
 	}
@@ -734,7 +745,7 @@ func (s *VectorStore) addPathToWatcher(path string) error {
 	return nil
 }
 
-func (s *VectorStore) watchLoop(ctx context.Context, docPaths []string, chunkSize, chunkOverlap int, respectWordBoundaries bool) {
+func (s *VectorStore) watchLoop(ctx context.Context, docPaths []string) {
 	var debounceTimer *time.Timer
 	debounceDuration := 2 * time.Second
 	pendingChanges := make(map[string]bool)
@@ -765,7 +776,7 @@ func (s *VectorStore) watchLoop(ctx context.Context, docPaths []string, chunkSiz
 		filesToReindex := make([]string, 0)
 		for _, file := range changedFiles {
 			// Check if the file matches any of the configured document paths/patterns
-			matches, matchErr := s.processor.Matches(file, docPaths)
+			matches, matchErr := chunk.Matches(file, docPaths)
 			if matchErr != nil {
 				slog.Error("Failed to match path", "file", file, "error", matchErr)
 				continue
@@ -801,7 +812,7 @@ func (s *VectorStore) watchLoop(ctx context.Context, docPaths []string, chunkSiz
 					},
 				})
 
-				if err := s.indexFile(ctx, file, chunkSize, chunkOverlap, respectWordBoundaries); err != nil {
+				if err := s.indexFile(ctx, file); err != nil {
 					slog.Error("Failed to re-index file", "path", file, "error", err)
 					s.emitEvent(types.Event{
 						Type:    "error",
@@ -849,7 +860,7 @@ func (s *VectorStore) watchLoop(ctx context.Context, docPaths []string, chunkSiz
 			}
 
 			// Early filter: only track changes for files that match configured doc patterns
-			matches, err := s.processor.Matches(event.Name, docPaths)
+			matches, err := chunk.Matches(event.Name, docPaths)
 			if err != nil {
 				slog.Debug("Could not match path against doc patterns", "path", event.Name, "error", err)
 				continue
@@ -882,7 +893,7 @@ func (s *VectorStore) watchLoop(ctx context.Context, docPaths []string, chunkSiz
 }
 
 func (s *VectorStore) cleanupOrphanedDocumentsFromDisk(ctx context.Context, docPaths []string) error {
-	files, err := s.processor.CollectFiles(docPaths)
+	files, err := chunk.CollectFiles(docPaths)
 	if err != nil {
 		return fmt.Errorf("failed to collect files: %w", err)
 	}

pkg/rag/treesitter/treesitter.go

@@ -0,0 +1,539 @@
+package treesitter
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"log/slog"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	sitter "github.com/smacker/go-tree-sitter"
+	golang "github.com/smacker/go-tree-sitter/golang"
+
+	"github.com/docker/cagent/pkg/rag/chunk"
+)
+
+// DocumentProcessor uses tree-sitter to build syntax trees for source
+// files and produce semantically aligned chunks (e.g., whole functions) while
+// still respecting a maximum chunk size where possible.
+//
+// NOTE: To keep the initial implementation minimal, this currently supports
+// Go source files via the golang grammar. The design is intentionally generic
+// so we can add more languages incrementally.
+//
+// The processor is thread-safe: it creates a new parser for each Process()
+// call since the underlying tree-sitter C library is not thread-safe.
+type DocumentProcessor struct {
+	chunkSize    int
+	chunkOverlap int
+	langByExt    map[string]*sitter.Language
+	functionNode map[string]func(*sitter.Node) bool
+	textFallback *chunk.TextDocumentProcessor
+}
+
+// NewDocumentProcessor creates a new document processor instance with a
+// language mapping that can be expanded over time. Falls back to text chunking
+// for unsupported file types.
+func NewDocumentProcessor(chunkSize, chunkOverlap int, respectWordBoundaries bool) *DocumentProcessor {
+	// Currently only Go is wired; more languages can be added later.
+	langByExt := map[string]*sitter.Language{
+		".go": golang.GetLanguage(),
+	}
+
+	functionNode := map[string]func(*sitter.Node) bool{
+		".go": isGoFunctionLike,
+	}
+
+	return &DocumentProcessor{
+		chunkSize:    chunkSize,
+		chunkOverlap: chunkOverlap,
+		langByExt:    langByExt,
+		functionNode: functionNode,
+		textFallback: chunk.NewTextDocumentProcessor(chunkSize, chunkOverlap, respectWordBoundaries),
+	}
+}
+
+// Process implements chunk.DocumentProcessor.
+func (p *DocumentProcessor) Process(path string, content []byte) ([]chunk.Chunk, error) {
+	slog.Debug("[TreeSitter] Starting to process file",
+		"path", path,
+		"content_size", len(content),
+		"chunk_size", p.chunkSize,
+		"chunk_overlap", p.chunkOverlap)
+
+	ext := strings.ToLower(filepath.Ext(path))
+	lang, ok := p.langByExt[ext]
+	if !ok {
+		slog.Debug("[TreeSitter] Unsupported file extension, falling back to text chunking",
+			"path", path,
+			"extension", ext)
+		return p.textFallback.Process(path, content)
+	}
+
+	slog.Debug("[TreeSitter] Language detected",
+		"path", path,
+		"extension", ext)
+
+	// Create a new parser for each call to ensure thread-safety
+	parser := sitter.NewParser()
+	parser.SetLanguage(lang)
+
+	slog.Debug("[TreeSitter] Parsing source code with tree-sitter",
+		"path", path)
+
+	// Use ParseCtx instead of deprecated Parse
+	tree, err := parser.ParseCtx(context.Background(), nil, content)
+	if err != nil || tree == nil || tree.RootNode() == nil {
+		slog.Debug("[TreeSitter] Parsing failed, falling back to text chunking",
+			"path", path,
+			"error", err)
+		return p.textFallback.Process(path, content)
+	}
+
+	slog.Debug("[TreeSitter] Successfully parsed source code",
+		"path", path)
+
+	root := tree.RootNode()
+	packageName := extractPackageName(root, content)
+	fnFilter, ok := p.functionNode[ext]
+	if !ok {
+		slog.Debug("[TreeSitter] No function filter defined for extension, falling back to text chunking",
+			"path", path,
+			"extension", ext)
+		return p.textFallback.Process(path, content)
+	}
+
+	// Extract function-like nodes.
+	var funcNodes []*sitter.Node
+	var walk func(*sitter.Node)
+	walk = func(n *sitter.Node) {
+		if fnFilter(n) {
+			funcNodes = append(funcNodes, n)
+			return
+		}
+		for i := range int(n.ChildCount()) {
+			child := n.Child(i)
+			if child == nil {
+				continue
+			}
+			walk(child)
+		}
+	}
+	walk(root)
+
+	slog.Debug("[TreeSitter] Extracted function nodes from syntax tree",
+		"path", path,
+		"function_count", len(funcNodes))
+
+	// If we didn't find any function-like nodes, fall back to text chunking.
+	if len(funcNodes) == 0 {
+		slog.Debug("[TreeSitter] No function nodes found, falling back to text chunking",
+			"path", path)
+		return p.textFallback.Process(path, content)
+	}
+
+	// Group functions into chunks under the size budget where possible, without
+	// ever splitting a single function across chunks.
+	text := string(content)
+	var chunksOut []chunk.Chunk
+	index := 0
+
+	var buf strings.Builder
+	currentLen := 0
+	var chunkFunctions []functionMetadata
+
+	flush := func() {
+		if buf.Len() == 0 {
+			return
+		}
+		c := strings.TrimSpace(buf.String())
+		if c == "" {
+			buf.Reset()
+			currentLen = 0
+			return
+		}
+		chunksOut = append(chunksOut, chunk.Chunk{
+			Index:    index,
+			Content:  c,
+			Metadata: buildChunkMetadata(chunkFunctions),
+		})
+		slog.Debug("[TreeSitter] Created code-aware chunk",
+			"chunk_index", index,
+			"chunk_content", c)
+		index++
+		buf.Reset()
+		currentLen = 0
+		chunkFunctions = nil
+	}
+
+	for funcIdx, fn := range funcNodes {
+		// Find any comments that precede this function
+		start := int(findPrecedingComments(fn, content))
+		end := int(fn.EndByte())
+		if start < 0 || end <= start || end > len(text) {
+			slog.Debug("[TreeSitter] Skipping function node with invalid byte range",
+				"path", path,
+				"function_index", funcIdx,
+				"start_byte", start,
+				"end_byte", end)
+			continue
+		}
+
+		fnText := strings.TrimSpace(text[start:end])
+		if fnText == "" {
+			slog.Debug("[TreeSitter] Skipping empty function node",
+				"path", path,
+				"function_index", funcIdx)
+			continue
+		}
+
+		fnLen := utf8.RuneCountInString(fnText)
+		fnType := fn.Type()
+
+		docText := ""
+		funcStart := int(fn.StartByte())
+		if start >= 0 && funcStart <= len(content) && start < funcStart {
+			docText = string(content[start:funcStart])
+		}
+
+		slog.Debug("[TreeSitter] Processing function node",
+			"path", path,
+			"function_index", funcIdx,
+			"function_type", fnType,
+			"function_length", fnLen,
+			"current_chunk_length", currentLen,
+			"chunk_size_limit", p.chunkSize)
+
+		// If the function alone is larger than chunkSize, emit it as its own
+		// chunk to avoid splitting function bodies.
+		if p.chunkSize > 0 && fnLen > p.chunkSize {
+			slog.Debug("[TreeSitter] Function exceeds chunk size, creating dedicated chunk",
+				"path", path,
+				"function_index", funcIdx,
+				"function_length", fnLen,
+				"chunk_size_limit", p.chunkSize,
+				"chunk_index", index)
+			flush()
+			meta := buildFunctionMetadata(fn, content, packageName, docText)
+			chunksOut = append(chunksOut, chunk.Chunk{
+				Index:    index,
+				Content:  fnText,
+				Metadata: buildChunkMetadata([]functionMetadata{meta}),
+			})
+			slog.Debug("[TreeSitter] Created code-aware chunk for large function",
+				"chunk_index", index,
+				"chunk_content", fnText)
+			index++
+			continue
+		}
+
+		// If adding this function would exceed the budget, flush and start new.
+		if p.chunkSize > 0 && currentLen > 0 && currentLen+fnLen > p.chunkSize {
+			slog.Debug("[TreeSitter] Adding function would exceed chunk size, flushing current chunk",
+				"path", path,
+				"function_index", funcIdx,
+				"current_chunk_length", currentLen,
+				"function_length", fnLen,
+				"total_would_be", currentLen+fnLen,
+				"chunk_size_limit", p.chunkSize,
+				"chunk_index", index)
+			flush()
+		}
+
+		slog.Debug("[TreeSitter] Adding function to current chunk",
+			"path", path,
+			"function_index", funcIdx,
+			"function_length", fnLen,
+			"new_chunk_length", currentLen+fnLen)
+
+		if buf.Len() > 0 {
+			buf.WriteString("\n\n")
+		}
+		buf.WriteString(fnText)
+		currentLen += fnLen
+		chunkFunctions = append(chunkFunctions, buildFunctionMetadata(fn, content, packageName, docText))
+	}
+
+	flush()
+
+	if len(chunksOut) == 0 {
+		slog.Debug("[TreeSitter] No chunks produced after processing, falling back to text chunking",
+			"path", path)
+		return p.textFallback.Process(path, content)
+	}
+
+	// Calculate statistics
+	totalChars := 0
+	minChunkSize := int(^uint(0) >> 1) // max int
+	maxChunkSize := 0
+	for _, c := range chunksOut {
+		chunkLen := utf8.RuneCountInString(c.Content)
+		totalChars += chunkLen
+		if chunkLen < minChunkSize {
+			minChunkSize = chunkLen
+		}
+		if chunkLen > maxChunkSize {
+			maxChunkSize = chunkLen
+		}
+	}
+	avgChunkSize := 0
+	if len(chunksOut) > 0 {
+		avgChunkSize = totalChars / len(chunksOut)
+	}
+
+	slog.Debug("[TreeSitter] Successfully chunked file using syntax tree",
+		"path", path,
+		"total_functions", len(funcNodes),
+		"total_chunks", len(chunksOut),
+		"avg_chunk_size", avgChunkSize,
+		"min_chunk_size", minChunkSize,
+		"max_chunk_size", maxChunkSize,
+		"total_content_size", totalChars)
+
+	return chunksOut, nil
+}
+
+// isGoFunctionLike returns true for nodes that represent top-level functions
+// or methods in Go. The exact node types are determined by the golang grammar.
+func isGoFunctionLike(n *sitter.Node) bool {
+	switch n.Type() {
+	case "function_declaration", "method_declaration":
+		return true
+	default:
+		return false
+	}
+}
+
+// findPrecedingComments finds all comment nodes that immediately precede a function
+// in the source code. This includes godoc-style comments and any other comments
+// that are part of the function's documentation.
+func findPrecedingComments(fn *sitter.Node, content []byte) (startByte uint32) {
+	startByte = fn.StartByte()
+	parent := fn.Parent()
+	if parent == nil {
+		return startByte
+	}
+
+	// Find the index of our function node among its siblings
+	fnIndex := -1
+	for i := range int(parent.ChildCount()) {
+		if parent.Child(i) == fn {
+			fnIndex = i
+			break
+		}
+	}
+
+	if fnIndex <= 0 {
+		// No siblings before this function
+		return startByte
+	}
+
+	// Walk backwards through siblings to find comments
+	var commentNodes []*sitter.Node
+	for i := fnIndex - 1; i >= 0; i-- {
+		sibling := parent.Child(i)
+		if sibling == nil {
+			break
+		}
+
+		// Check if this is a comment node
+		if sibling.Type() == "comment" {
+			commentNodes = append([]*sitter.Node{sibling}, commentNodes...)
+			continue
+		}
+
+		// If we hit a non-comment node that's not just whitespace, stop
+		// Check if the node is empty or only contains whitespace
+		nodeStart := int(sibling.StartByte())
+		nodeEnd := int(sibling.EndByte())
+		if nodeStart >= 0 && nodeEnd <= len(content) && nodeEnd > nodeStart {
+			nodeText := strings.TrimSpace(string(content[nodeStart:nodeEnd]))
+			if nodeText != "" {
+				// Hit a non-comment, non-whitespace node
+				break
+			}
+		}
+	}
+
+	// If we found comments, use the start of the first one
+	if len(commentNodes) > 0 {
+		// Check if there are blank lines between comments and function
+		// We want to include comments that are directly adjacent to the function
+		lastComment := commentNodes[len(commentNodes)-1]
+		commentEnd := int(lastComment.EndByte())
+		functionStart := int(fn.StartByte())
+
+		if commentEnd < functionStart && functionStart <= len(content) {
+			// Check the gap between comment and function
+			gap := string(content[commentEnd:functionStart])
+			// Count newlines in the gap
+			newlines := strings.Count(gap, "\n")
+			// If there's more than one blank line, don't include comments
+			// (1 newline = same line, 2 newlines = one blank line)
+			if newlines > 2 {
+				return startByte
+			}
+		}
+
+		return commentNodes[0].StartByte()
+	}
+
+	return startByte
+}
+
+type functionMetadata struct {
+	Name      string
+	Kind      string
+	Receiver  string
+	Signature string
+	Doc       string
+	Package   string
+	StartLine int
+	EndLine   int
+}
+
+func buildChunkMetadata(functions []functionMetadata) map[string]string {
+	if len(functions) == 0 {
+		return nil
+	}
+
+	meta := make(map[string]string, 10)
+	meta["symbol_count"] = strconv.Itoa(len(functions))
+
+	primary := functions[0]
+	if primary.Name != "" {
+		meta["symbol_name"] = primary.Name
+	}
+	if primary.Kind != "" {
+		meta["symbol_kind"] = primary.Kind
+	}
+	if primary.Receiver != "" {
+		meta["receiver"] = primary.Receiver
+	}
+	if primary.Signature != "" {
+		meta["signature"] = primary.Signature
+	}
+	if primary.Doc != "" {
+		meta["doc"] = primary.Doc
+	}
+	if primary.Package != "" {
+		meta["package"] = primary.Package
+	}
+	if primary.StartLine > 0 {
+		meta["start_line"] = strconv.Itoa(primary.StartLine)
+	}
+	if primary.EndLine > 0 {
+		meta["end_line"] = strconv.Itoa(primary.EndLine)
+	}
+
+	if len(functions) > 1 {
+		names := make([]string, 0, len(functions)-1)
+		for _, fn := range functions[1:] {
+			if fn.Name != "" {
+				names = append(names, fn.Name)
+			}
+		}
+		if len(names) > 0 {
+			meta["additional_symbols"] = strings.Join(names, ", ")
+		}
+	}
+
+	return meta
+}
+
+func buildFunctionMetadata(fn *sitter.Node, content []byte, pkgName, docText string) functionMetadata {
+	meta := functionMetadata{
+		Name:      strings.TrimSpace(nodeText(content, fn.ChildByFieldName("name"))),
+		Kind:      mapFunctionKind(fn.Type()),
+		Receiver:  strings.TrimSpace(nodeText(content, fn.ChildByFieldName("receiver"))),
+		Signature: buildGoSignature(content, fn),
+		Doc:       truncateMetadataValue(strings.TrimSpace(docText), 400),
+		Package:   pkgName,
+		StartLine: int(fn.StartPoint().Row) + 1,
+		EndLine:   int(fn.EndPoint().Row) + 1,
+	}
+
+	return meta
+}
+
+func mapFunctionKind(nodeType string) string {
+	if nodeType == "method_declaration" {
+		return "method"
+	}
+	return "function"
+}
+
+func buildGoSignature(content []byte, fn *sitter.Node) string {
+	if fn == nil {
+		return ""
+	}
+
+	text := strings.TrimSpace(string(content[fn.StartByte():fn.EndByte()]))
+	if text == "" {
+		return ""
+	}
+
+	if braceIdx := strings.Index(text, "{"); braceIdx != -1 {
+		text = strings.TrimSpace(text[:braceIdx])
+	}
+	if newlineIdx := strings.Index(text, "\n"); newlineIdx != -1 {
+		text = strings.TrimSpace(text[:newlineIdx])
+	}
+
+	return truncateMetadataValue(text, 240)
+}
+
+func truncateMetadataValue(value string, limit int) string {
+	if limit <= 0 {
+		return value
+	}
+	runes := []rune(value)
+	if len(runes) <= limit {
+		return value
+	}
+	return string(runes[:limit]) + "…"
+}
+
+func extractPackageName(root *sitter.Node, content []byte) string {
+	if root == nil {
+		return ""
+	}
+
+	for i := range int(root.ChildCount()) {
+		child := root.Child(i)
+		if child == nil {
+			continue
+		}
+		if child.Type() != "package_clause" {
+			continue
+		}
+		if name := child.ChildByFieldName("name"); name != nil {
+			return strings.TrimSpace(nodeText(content, name))
+		}
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(content))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if strings.HasPrefix(line, "package ") {
+			return strings.TrimSpace(strings.TrimPrefix(line, "package "))
+		}
+	}
+
+	return ""
+}
+
+func nodeText(content []byte, node *sitter.Node) string {
+	if node == nil {
+		return ""
+	}
+	start := node.StartByte()
+	end := node.EndByte()
+	if end <= start || int(end) > len(content) {
+		return ""
+	}
+	return string(content[start:end])
+}

pkg/rag/treesitter/treesitter_test.go

@@ -0,0 +1,362 @@
+package treesitter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTreeSitterPreProcessor_MetadataCaptured(t *testing.T) {
+	t.Parallel()
+
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	content := []byte(`package main
+
+// Add adds two numbers.
+func (c *Calculator) Add(a, b int) int {
+	return a + b
+}
+`)
+
+	chunks, err := processor.Process("calc.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1)
+
+	meta := chunks[0].Metadata
+	require.NotNil(t, meta, "metadata should be present when AST processing succeeds")
+	assert.Equal(t, "Add", meta["symbol_name"])
+	assert.Equal(t, "method", meta["symbol_kind"])
+	assert.Equal(t, "main", meta["package"])
+	assert.Contains(t, meta["signature"], "func (c *Calculator) Add(a, b int) int")
+	assert.Contains(t, meta["doc"], "Add adds two numbers.")
+	assert.NotEmpty(t, meta["start_line"])
+	assert.NotEmpty(t, meta["end_line"])
+}
+
+func TestTreeSitterPreProcessor_IncludesGodocComments(t *testing.T) {
+	processor := NewDocumentProcessor(80, 0, false)
+
+	// Test case with godoc-style comments
+	content := []byte(`package main
+
+// Add returns the sum of two integers.
+// This is a godoc comment.
+func Add(a, b int) int {
+	return a + b
+}
+
+// Subtract returns the difference between two integers.
+func Subtract(a, b int) int {
+	return a - b
+}
+`)
+
+	// Use small chunk size to force separate chunks
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 2, "Expected 2 chunks (one per function)")
+
+	// Check that the first chunk includes the godoc comment
+	assert.Contains(t, chunks[0].Content, "// Add returns the sum of two integers.")
+	assert.Contains(t, chunks[0].Content, "// This is a godoc comment.")
+	assert.Contains(t, chunks[0].Content, "func Add(a, b int) int {")
+
+	// Check that the second chunk includes its godoc comment
+	assert.Contains(t, chunks[1].Content, "// Subtract returns the difference between two integers.")
+	assert.Contains(t, chunks[1].Content, "func Subtract(a, b int) int {")
+}
+
+func TestTreeSitterPreProcessor_FunctionWithoutComment(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	content := []byte(`package main
+
+func Multiply(a, b int) int {
+	return a * b
+}
+`)
+
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1)
+
+	// Should only contain the function, no extra comments
+	assert.Contains(t, chunks[0].Content, "func Multiply(a, b int) int {")
+	assert.NotContains(t, chunks[0].Content, "//")
+}
+
+func TestTreeSitterPreProcessor_MethodWithComment(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	content := []byte(`package main
+
+type Calculator struct{}
+
+// Calculate performs a calculation.
+func (c Calculator) Calculate(a, b int) int {
+	return a + b
+}
+`)
+
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1)
+
+	// Should include the comment and the method
+	assert.Contains(t, chunks[0].Content, "// Calculate performs a calculation.")
+	assert.Contains(t, chunks[0].Content, "func (c Calculator) Calculate(a, b int) int {")
+}
+
+func TestTreeSitterPreProcessor_MultilineComment(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	content := []byte(`package main
+
+// Divide divides two numbers.
+// It returns an error if the divisor is zero.
+// This is a multi-line comment.
+func Divide(a, b int) (int, error) {
+	if b == 0 {
+		return 0, fmt.Errorf("division by zero")
+	}
+	return a / b, nil
+}
+`)
+
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1)
+
+	// Should include all comment lines
+	assert.Contains(t, chunks[0].Content, "// Divide divides two numbers.")
+	assert.Contains(t, chunks[0].Content, "// It returns an error if the divisor is zero.")
+	assert.Contains(t, chunks[0].Content, "// This is a multi-line comment.")
+	assert.Contains(t, chunks[0].Content, "func Divide(a, b int) (int, error) {")
+}
+
+func TestTreeSitterPreProcessor_BlankLinesBetweenCommentAndFunction(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	// Test with more than one blank line between comment and function
+	// This should NOT include the comment
+	content := []byte(`package main
+
+// This is a detached comment.
+
+
+func Process() {
+	// implementation
+}
+`)
+
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1)
+
+	// Should NOT include the detached comment (too many blank lines)
+	assert.NotContains(t, chunks[0].Content, "This is a detached comment")
+	assert.Contains(t, chunks[0].Content, "func Process() {")
+}
+
+func TestTreeSitterPreProcessor_AdjacentComment(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	// Test with no blank line between comment and function (most common godoc style)
+	content := []byte(`package main
+
+// Handler handles requests.
+func Handler() {
+	// implementation
+}
+`)
+
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1)
+
+	// Should include the adjacent comment
+	assert.Contains(t, chunks[0].Content, "// Handler handles requests.")
+	assert.Contains(t, chunks[0].Content, "func Handler() {")
+}
+
+func TestTreeSitterPreProcessor_MixedCommentsAndFunctions(t *testing.T) {
+	processor := NewDocumentProcessor(80, 0, false)
+
+	content := []byte(`package main
+
+import "fmt"
+
+// First function with comment.
+func First() {
+	fmt.Println("first")
+}
+
+func Second() {
+	fmt.Println("second")
+}
+
+// Third function with a detailed comment.
+// It has multiple lines.
+func Third() {
+	fmt.Println("third")
+}
+`)
+
+	// Use small chunk size to force separate chunks
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 3)
+
+	// First function should have its comment
+	assert.Contains(t, chunks[0].Content, "// First function with comment.")
+	assert.Contains(t, chunks[0].Content, "func First() {")
+
+	// Second function should not have a comment
+	assert.NotContains(t, chunks[1].Content, "//")
+	assert.Contains(t, chunks[1].Content, "func Second() {")
+
+	// Third function should have its multi-line comment
+	assert.Contains(t, chunks[2].Content, "// Third function with a detailed comment.")
+	assert.Contains(t, chunks[2].Content, "// It has multiple lines.")
+	assert.Contains(t, chunks[2].Content, "func Third() {")
+}
+
+func TestTreeSitterPreProcessor_ChunkSizeRespected(t *testing.T) {
+	processor := NewDocumentProcessor(50, 0, false)
+
+	content := []byte(`package main
+
+// Small adds two numbers.
+func Small(a, b int) int {
+	return a + b
+}
+
+// Another small function.
+func Another(x int) int {
+	return x * 2
+}
+`)
+
+	// Set a small chunk size to force separate chunks
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+
+	// With a small chunk size, functions should be in separate chunks
+	assert.Greater(t, len(chunks), 1, "Expected multiple chunks due to size limit")
+}
+
+func TestTreeSitterPreProcessor_LargeFunctionExceedsChunkSize(t *testing.T) {
+	processor := NewDocumentProcessor(50, 0, false)
+
+	content := []byte(`package main
+
+// ProcessData processes a large amount of data.
+// This function is intentionally large.
+func ProcessData() {
+	// Line 1
+	// Line 2
+	// Line 3
+	// Line 4
+	// Line 5
+	// Line 6
+	// Line 7
+	// Line 8
+	// Line 9
+	// Line 10
+}
+`)
+
+	// Set chunk size smaller than the function
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1, "Large function should be in its own chunk despite exceeding limit")
+
+	// Should include the comment even though it's oversized
+	assert.Contains(t, chunks[0].Content, "// ProcessData processes a large amount of data.")
+	assert.Contains(t, chunks[0].Content, "func ProcessData() {")
+}
+
+func TestTreeSitterPreProcessor_UnsupportedExtension(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	content := []byte(`console.log("hello");`)
+
+	// For unsupported extensions, it falls back to text chunking
+	chunks, err := processor.Process("test.js", content)
+	require.NoError(t, err)
+	// Text fallback should produce chunks
+	require.NotNil(t, chunks)
+	require.Len(t, chunks, 1)
+}
+
+func TestTreeSitterPreProcessor_EmptyFile(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	content := []byte(`package main`)
+
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	// Falls back to text chunking for files with no functions
+	require.NotEmpty(t, chunks)
+}
+
+func TestTreeSitterPreProcessor_BlockCommentStyle(t *testing.T) {
+	processor := NewDocumentProcessor(1000, 0, false)
+
+	content := []byte(`package main
+
+/*
+Block comment style.
+Multiple lines.
+*/
+func BlockCommented() {
+	return
+}
+`)
+
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1)
+
+	// Should include block comment
+	assert.Contains(t, chunks[0].Content, "/*")
+	assert.Contains(t, chunks[0].Content, "Block comment style.")
+	assert.Contains(t, chunks[0].Content, "func BlockCommented() {")
+}
+
+func TestTreeSitterPreProcessor_FunctionsGroupedInChunk(t *testing.T) {
+	processor := NewDocumentProcessor(10000, 0, false)
+
+	content := []byte(`package main
+
+// A is small.
+func A() int {
+	return 1
+}
+
+// B is small.
+func B() int {
+	return 2
+}
+
+// C is small.
+func C() int {
+	return 3
+}
+`)
+
+	// Large chunk size should group all functions together
+	chunks, err := processor.Process("test.go", content)
+	require.NoError(t, err)
+	require.Len(t, chunks, 1, "Expected all small functions to be grouped in one chunk")
+
+	// Should contain all functions with their comments
+	assert.Contains(t, chunks[0].Content, "// A is small.")
+	assert.Contains(t, chunks[0].Content, "func A() int {")
+	assert.Contains(t, chunks[0].Content, "// B is small.")
+	assert.Contains(t, chunks[0].Content, "func B() int {")
+	assert.Contains(t, chunks[0].Content, "// C is small.")
+	assert.Contains(t, chunks[0].Content, "func C() int {")
+}

pkg/runtime/runtime_test.go

@@ -407,19 +407,19 @@ func TestContextCancellation(t *testing.T) {
 // stubRAGStrategy is a minimal implementation of strategy.Strategy for testing RAG initialization.
 type stubRAGStrategy struct{}
 
-func (s *stubRAGStrategy) Initialize(context.Context, []string, int, int, bool) error {
+func (s *stubRAGStrategy) Initialize(_ context.Context, _ []string, _ strategy.ChunkingConfig) error {
 	return nil
 }
 
-func (s *stubRAGStrategy) Query(context.Context, string, int, float64) ([]database.SearchResult, error) {
+func (s *stubRAGStrategy) Query(_ context.Context, _ string, _ int, _ float64) ([]database.SearchResult, error) {
 	return nil, nil
 }
 
-func (s *stubRAGStrategy) CheckAndReindexChangedFiles(context.Context, []string, int, int, bool) error {
+func (s *stubRAGStrategy) CheckAndReindexChangedFiles(_ context.Context, _ []string, _ strategy.ChunkingConfig) error {
 	return nil
 }
 
-func (s *stubRAGStrategy) StartFileWatcher(context.Context, []string, int, int, bool) error {
+func (s *stubRAGStrategy) StartFileWatcher(_ context.Context, _ []string, _ strategy.ChunkingConfig) error {
 	return nil
 }