From 7cfd51b6aaec7929f3babf0010f6be8901f3bc4c Mon Sep 17 00:00:00 2001 From: tae2089 Date: Thu, 16 Jul 2026 10:33:09 +0900 Subject: [PATCH 1/7] refactor: stream edge resolution from repeatable sources --- internal/app/ingest/workflow/build.go | 122 +++++++++++-------- internal/app/ingest/workflow/indexer_test.go | 84 +++++++++++++ 2 files changed, 158 insertions(+), 48 deletions(-) diff --git a/internal/app/ingest/workflow/build.go b/internal/app/ingest/workflow/build.go index db2720e..ea680aa 100644 --- a/internal/app/ingest/workflow/build.go +++ b/internal/app/ingest/workflow/build.go @@ -41,6 +41,10 @@ type parsedBuildEdgeBatch struct { edges []graph.Edge } +// buildEdgeBatchSource replays deferred edge batches from the beginning on every invocation. +// @intent let edge resolution make ordered passes over either in-memory batches or disk-backed build records. +type buildEdgeBatchSource func(yield func(parsedBuildEdgeBatch) error) error + // buildParseInput is one validated source file assigned to a parse worker. // @intent keep deterministic input sequencing separate from concurrent filesystem and parser work. type buildParseInput struct { @@ -762,21 +766,38 @@ func (s *Service) flushBuildEdges(ctx context.Context, txStore ingestapp.GraphSt // flushBuildEdgesWithTiming resolves and persists deferred edges while recording each resolver operation when timing is supplied. // @intent measure edge-resolution database reads and writes without altering the existing resolution order or transaction. func (s *Service) flushBuildEdgesWithTiming(ctx context.Context, txStore ingestapp.GraphStore, edgeBatches []parsedBuildEdgeBatch, stats *BuildStats, resolveOptions resolve.ResolveOptions, timing *BuildResolveTiming) error { - implementsEdges, otherBatches := partitionBuildEdges(edgeBatches) - importsByPath := importEdgesByFile(otherBatches) + source := buildEdgeBatchSource(func(yield func(parsedBuildEdgeBatch) error) error { + for _, batch := range edgeBatches { + if err := yield(batch); err != nil { + return err + } + } + return nil + }) + return s.flushBuildEdgeSourceWithTiming(ctx, txStore, source, stats, resolveOptions, timing) +} + +// flushBuildEdgeSourceWithTiming resolves a re-iterable edge source in implements-first order using bounded buffers. +// @intent avoid retaining or repartitioning every build edge while preserving resolver ordering and timing contracts. +// @sideEffect upserts graph edges through the transaction-scoped store. +// @mutates graph edges +func (s *Service) flushBuildEdgeSourceWithTiming(ctx context.Context, txStore ingestapp.GraphStore, source buildEdgeBatchSource, stats *BuildStats, resolveOptions resolve.ResolveOptions, timing *BuildResolveTiming) error { lookup := newBuildResolveLookupWithTiming(txStore, timing) - for start := 0; start < len(implementsEdges); start += buildEdgeResolveChunkSize { + var pendingImplements []graph.Edge + flushImplements := func() error { + if len(pendingImplements) == 0 { + return nil + } if err := ctx.Err(); err != nil { return err } - end := min(start+buildEdgeResolveChunkSize, len(implementsEdges)) resolveStarted := time.Now() - resolved, err := s.edgeResolver()(ctx, lookup, implementsEdges[start:end], resolveOptions) + resolved, err := s.edgeResolver()(ctx, lookup, pendingImplements, resolveOptions) if timing != nil { timing.Resolver.add(time.Since(resolveStarted)) } if err != nil { - s.logger().ErrorContext(ctx, "resolve deferred implements edges failed", append(obs.TraceLogArgs(ctx), "start", start, "end", end, "error", err)...) + s.logger().ErrorContext(ctx, "resolve deferred implements edges failed", append(obs.TraceLogArgs(ctx), "edges", len(pendingImplements), "error", err)...) return trace.Wrap(err, "resolve deferred implements edges") } resolved, diagnostics := resolve.FilterResolvedWithDiagnosticsFiltered(resolved, shouldSuppressExternalImportUnresolved) @@ -790,9 +811,33 @@ func (s *Service) flushBuildEdgesWithTiming(ctx context.Context, txStore ingesta timing.UpsertEdges.add(time.Since(upsertStarted)) } if err != nil { - s.logger().ErrorContext(ctx, "upsert deferred implements edges failed", append(obs.TraceLogArgs(ctx), "start", start, "end", end, "error", err)...) + s.logger().ErrorContext(ctx, "upsert deferred implements edges failed", append(obs.TraceLogArgs(ctx), "edges", len(resolved), "error", err)...) return trace.Wrap(err, "upsert deferred implements edges") } + pendingImplements = nil + return nil + } + if err := source(func(parsed parsedBuildEdgeBatch) error { + if err := ctx.Err(); err != nil { + return err + } + for _, edge := range parsed.edges { + if edge.Kind != graph.EdgeKindImplements { + continue + } + if len(pendingImplements) == buildEdgeResolveChunkSize { + if err := flushImplements(); err != nil { + return err + } + } + pendingImplements = append(pendingImplements, edge) + } + return nil + }); err != nil { + return trace.Wrap(err, "replay deferred implements edges") + } + if err := flushImplements(); err != nil { + return err } // actualEdgeRange identifies parsed edges within a resolver input that also contains import warmup edges. @@ -840,14 +885,16 @@ func (s *Service) flushBuildEdgesWithTiming(ctx context.Context, txStore ingesta actualRanges = nil return nil } - for _, parsed := range otherBatches { + if err := source(func(parsed parsedBuildEdgeBatch) error { if err := ctx.Err(); err != nil { return err } - for start := 0; start < len(parsed.edges); start += buildEdgeResolveChunkSize { - end := min(start+buildEdgeResolveChunkSize, len(parsed.edges)) - chunk := parsed.edges[start:end] - resolveInput := chunkWithImportWarmup(chunk, importsByPath[parsed.relPath]) + _, otherEdges := splitImplementsEdges(parsed.edges) + imports := importEdges(otherEdges) + for start := 0; start < len(otherEdges); start += buildEdgeResolveChunkSize { + end := min(start+buildEdgeResolveChunkSize, len(otherEdges)) + chunk := otherEdges[start:end] + resolveInput := chunkWithImportWarmup(chunk, imports) if len(pending) > 0 && len(pending)+len(resolveInput) > buildEdgeResolveChunkSize { if err := flushPending(); err != nil { return err @@ -857,6 +904,9 @@ func (s *Service) flushBuildEdgesWithTiming(ctx context.Context, txStore ingesta pending = append(pending, resolveInput...) actualRanges = append(actualRanges, actualEdgeRange{start: actualStart, end: actualStart + len(chunk)}) } + return nil + }); err != nil { + return trace.Wrap(err, "replay deferred edges") } if err := flushPending(); err != nil { return err @@ -864,6 +914,18 @@ func (s *Service) flushBuildEdgesWithTiming(ctx context.Context, txStore ingesta return nil } +// importEdges selects file-local import warmup edges from one streamed edge batch. +// @intent preserve import-aware call resolution without retaining a build-wide import map. +func importEdges(edges []graph.Edge) []graph.Edge { + var imports []graph.Edge + for _, edge := range edges { + if edge.Kind == graph.EdgeKindImportsFrom { + imports = append(imports, edge) + } + } + return imports +} + // rewriteImplementsFingerprintScope rewrites an implements fingerprint to use a package anchor file. // @intent keep synthesized package semantic edges idempotent even when they are rebuilt from different files. func rewriteImplementsFingerprintScope(fingerprint, scope string) string { @@ -949,27 +1011,6 @@ func shouldSuppressExternalImportUnresolved(edge graph.Edge, _ string) bool { return edge.Kind == graph.EdgeKindImportsFrom && resolve.IsLikelyExternalImportEdge(edge) } -// partitionBuildEdges keeps implements edges available before resolving call edges in later bounded chunks. -// @intent preserve Go interface dispatch resolution after build edge resolution starts streaming by file. -func partitionBuildEdges(edgeBatches []parsedBuildEdgeBatch) ([]graph.Edge, []parsedBuildEdgeBatch) { - var implementsEdges []graph.Edge - otherBatches := make([]parsedBuildEdgeBatch, 0, len(edgeBatches)) - otherByPath := make(map[string][]graph.Edge, len(edgeBatches)) - var paths []string - for _, parsed := range edgeBatches { - parsedImplements, otherEdges := splitImplementsEdges(parsed.edges) - implementsEdges = append(implementsEdges, parsedImplements...) - if len(otherEdges) > 0 { - otherByPath[parsed.relPath] = append(otherByPath[parsed.relPath], otherEdges...) - paths = append(paths, parsed.relPath) - } - } - for _, path := range paths { - otherBatches = append(otherBatches, parsedBuildEdgeBatch{relPath: path, edges: otherByPath[path]}) - } - return implementsEdges, otherBatches -} - // splitImplementsEdges separates implements edges from other relationship types. // @intent ensure interface fulfillment edges are handled before call dispatch resolution. func splitImplementsEdges(edges []graph.Edge) ([]graph.Edge, []graph.Edge) { @@ -985,21 +1026,6 @@ func splitImplementsEdges(edges []graph.Edge) ([]graph.Edge, []graph.Edge) { return implementsEdges, otherEdges } -// importEdgesByFile groups import-from edges by their source file path. -// @intent optimize edge resolution by pre-loading import context for each file. -func importEdgesByFile(edgeBatches []parsedBuildEdgeBatch) map[string][]graph.Edge { - byFile := make(map[string][]graph.Edge, len(edgeBatches)) - for _, batch := range edgeBatches { - for _, edge := range batch.edges { - if edge.Kind != graph.EdgeKindImportsFrom { - continue - } - byFile[batch.relPath] = append(byFile[batch.relPath], edge) - } - } - return byFile -} - // chunkWithImportWarmup combines a chunk of call edges with their file's import edges. // @intent ensure the edge resolver has enough context to resolve call targets through imports. func chunkWithImportWarmup(chunk []graph.Edge, imports []graph.Edge) []graph.Edge { diff --git a/internal/app/ingest/workflow/indexer_test.go b/internal/app/ingest/workflow/indexer_test.go index 5871a60..a25e254 100644 --- a/internal/app/ingest/workflow/indexer_test.go +++ b/internal/app/ingest/workflow/indexer_test.go @@ -643,6 +643,90 @@ func TestFlushBuildEdges_ResolvesAndUpsertsBoundedBatches(t *testing.T) { } } +func TestFlushBuildEdgeSource_ReplaysTwoPassesInImplementsFirstOrder(t *testing.T) { + ctx := context.Background() + st := newRecordingGraphStore(t) + batches := []parsedBuildEdgeBatch{ + { + relPath: "first.go", + edges: []graph.Edge{ + {Kind: graph.EdgeKindImplements, FilePath: "first.go", FromNodeID: 1, ToNodeID: 2}, + {Kind: graph.EdgeKindContains, FilePath: "first.go", FromNodeID: 1, ToNodeID: 3}, + }, + }, + { + relPath: "second.go", + edges: []graph.Edge{ + {Kind: graph.EdgeKindImplements, FilePath: "second.go", FromNodeID: 4, ToNodeID: 5}, + {Kind: graph.EdgeKindContains, FilePath: "second.go", FromNodeID: 4, ToNodeID: 6}, + }, + }, + } + + passes := 0 + source := buildEdgeBatchSource(func(yield func(parsedBuildEdgeBatch) error) error { + passes++ + for _, batch := range batches { + if err := yield(batch); err != nil { + return err + } + } + return nil + }) + var resolvedKinds [][]graph.EdgeKind + resolver := func(_ context.Context, _ resolve.NodeLookup, edges []graph.Edge, _ resolve.ResolveOptions) ([]graph.Edge, error) { + if len(edges) > buildEdgeResolveChunkSize { + t.Fatalf("resolve batch exceeded limit: got %d want <= %d", len(edges), buildEdgeResolveChunkSize) + } + kinds := make([]graph.EdgeKind, len(edges)) + for i := range edges { + kinds[i] = edges[i].Kind + } + resolvedKinds = append(resolvedKinds, kinds) + return edges, nil + } + + svc := &Service{resolveEdges: resolver} + if err := svc.flushBuildEdgeSourceWithTiming(ctx, st, source, nil, resolve.ResolveOptions{}, nil); err != nil { + t.Fatalf("flushBuildEdgeSourceWithTiming: %v", err) + } + if passes != 2 { + t.Fatalf("source passes = %d, want 2", passes) + } + seenNonImplements := false + implementsCount := 0 + otherCount := 0 + for _, kinds := range resolvedKinds { + for _, kind := range kinds { + if kind == graph.EdgeKindImplements { + if seenNonImplements { + t.Fatal("implements edge resolved after non-implements edge") + } + implementsCount++ + continue + } + seenNonImplements = true + otherCount++ + } + } + if implementsCount != 2 || otherCount != 2 { + t.Fatalf("resolved counts = implements %d, other %d; want 2 and 2", implementsCount, otherCount) + } +} + +func TestFlushBuildEdgeSource_PropagatesSourceError(t *testing.T) { + wantErr := errors.New("read edge spool") + source := buildEdgeBatchSource(func(func(parsedBuildEdgeBatch) error) error { + return wantErr + }) + + svc := &Service{} + err := svc.flushBuildEdgeSourceWithTiming(context.Background(), newRecordingGraphStore(t), source, nil, resolve.ResolveOptions{}, nil) + if !errors.Is(err, wantErr) { + t.Fatalf("flush error = %v, want %v", err, wantErr) + } +} + func TestFlushBuildEdges_RecordsResolverAndUpsertTiming(t *testing.T) { ctx := context.Background() st := newRecordingGraphStore(t) From e87466f54127d13ff48659f23d4934cedd4f22b3 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Thu, 16 Jul 2026 10:34:13 +0900 Subject: [PATCH 2/7] perf: replay deferred build edges from spool --- internal/app/ingest/workflow/build.go | 10 ++---- internal/app/ingest/workflow/indexer_test.go | 34 ++++++++++++++++++++ internal/app/ingest/workflow/spool.go | 29 +++++++++++++++++ 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/internal/app/ingest/workflow/build.go b/internal/app/ingest/workflow/build.go index ea680aa..28f851e 100644 --- a/internal/app/ingest/workflow/build.go +++ b/internal/app/ingest/workflow/build.go @@ -483,7 +483,6 @@ func (s *Service) applyBuildSpoolInTx(ctx context.Context, tx ingestapp.Transact batch := buildPersistBatch{} nodePersistStart := time.Now() - var edgeBatches []parsedBuildEdgeBatch var packageNodeBatches []parsedBuildNodeBatch for _, path := range spool.records { record, err := spool.readRecord(path) @@ -507,10 +506,6 @@ func (s *Service) applyBuildSpoolInTx(ctx context.Context, tx ingestapp.Transact interfaces: record.Interfaces, language: record.Language, }) - edgeBatches = append(edgeBatches, parsedBuildEdgeBatch{ - relPath: record.RelPath, - edges: record.Edges, - }) if batch.shouldFlush() { if err := s.flushBuildBatch(ctx, txStore, &batch); err != nil { return err @@ -523,7 +518,8 @@ func (s *Service) applyBuildSpoolInTx(ctx context.Context, tx ingestapp.Transact if timing != nil { timing.PersistNodesMS = time.Since(nodePersistStart).Milliseconds() } - edgeBatches = append(edgeBatches, s.packageSemanticEdgeBatches(packageNodeBatches)...) + packageEdgeBatches := s.packageSemanticEdgeBatches(packageNodeBatches) + packageNodeBatches = nil if err := upsertPackageContainsEdges(ctx, txStore, spool.packages); err != nil { return trace.Wrap(err, "upsert package file edges") } @@ -533,7 +529,7 @@ func (s *Service) applyBuildSpoolInTx(ctx context.Context, tx ingestapp.Transact if timing != nil { resolveTiming = &timing.Resolve } - if err := s.flushBuildEdgesWithTiming(ctx, txStore, edgeBatches, &spool.stats, resolveOptions, resolveTiming); err != nil { + if err := s.flushBuildEdgeSourceWithTiming(ctx, txStore, spool.edgeBatchSource(ctx, packageEdgeBatches), &spool.stats, resolveOptions, resolveTiming); err != nil { return err } if timing != nil { diff --git a/internal/app/ingest/workflow/indexer_test.go b/internal/app/ingest/workflow/indexer_test.go index a25e254..905e738 100644 --- a/internal/app/ingest/workflow/indexer_test.go +++ b/internal/app/ingest/workflow/indexer_test.go @@ -727,6 +727,40 @@ func TestFlushBuildEdgeSource_PropagatesSourceError(t *testing.T) { } } +func TestBuildSpoolEdgeBatchSource_ReplaysRecordsAndTrailingBatches(t *testing.T) { + spool := &buildSpool{dir: t.TempDir()} + for seq, record := range []spooledBuildRecord{ + {RelPath: "first.go", Edges: []graph.Edge{{Kind: graph.EdgeKindContains, Fingerprint: "first"}}}, + {RelPath: "second.go", Edges: []graph.Edge{{Kind: graph.EdgeKindCalls, Fingerprint: "second"}}}, + } { + if err := spool.writeRecord(seq, record); err != nil { + t.Fatalf("write spool record %d: %v", seq, err) + } + } + trailing := []parsedBuildEdgeBatch{{ + relPath: "package.go", + edges: []graph.Edge{{Kind: graph.EdgeKindImplements, Fingerprint: "package"}}, + }} + + source := spool.edgeBatchSource(context.Background(), trailing) + for pass := 1; pass <= 2; pass++ { + var got []string + err := source(func(batch parsedBuildEdgeBatch) error { + for _, edge := range batch.edges { + got = append(got, batch.relPath+":"+edge.Fingerprint) + } + return nil + }) + if err != nil { + t.Fatalf("source pass %d: %v", pass, err) + } + want := []string{"first.go:first", "second.go:second", "package.go:package"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("source pass %d = %v, want %v", pass, got, want) + } + } +} + func TestFlushBuildEdges_RecordsResolverAndUpsertTiming(t *testing.T) { ctx := context.Background() st := newRecordingGraphStore(t) diff --git a/internal/app/ingest/workflow/spool.go b/internal/app/ingest/workflow/spool.go index 1d5894f..bdfbee3 100644 --- a/internal/app/ingest/workflow/spool.go +++ b/internal/app/ingest/workflow/spool.go @@ -2,6 +2,7 @@ package workflow import ( + "context" "encoding/gob" "fmt" "log/slog" @@ -95,6 +96,34 @@ func (b *buildSpool) readRecord(path string) (spooledBuildRecord, error) { return record, nil } +// edgeBatchSource replays file edge batches from disk before yielding trailing synthesized batches. +// @intent let full builds resolve deferred edges in ordered passes without retaining every parsed edge in memory. +func (b *buildSpool) edgeBatchSource(ctx context.Context, trailing []parsedBuildEdgeBatch) buildEdgeBatchSource { + return func(yield func(parsedBuildEdgeBatch) error) error { + for _, path := range b.records { + if err := ctx.Err(); err != nil { + return err + } + record, err := b.readRecord(path) + if err != nil { + return err + } + if err := yield(parsedBuildEdgeBatch{relPath: record.RelPath, edges: record.Edges}); err != nil { + return err + } + } + for _, batch := range trailing { + if err := ctx.Err(); err != nil { + return err + } + if err := yield(batch); err != nil { + return err + } + } + return nil + } +} + // cleanup removes the spool directory and logs a warning on failure. // @intent reclaim spool disk space whether the build succeeded or failed. // @sideEffect deletes the temporary spool directory. From ef587494976a49ed19c539248e34ae618a226e00 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Thu, 16 Jul 2026 11:44:38 +0900 Subject: [PATCH 3/7] fix: resolve implements before staged calls --- .../app/ingest/incremental/incremental.go | 46 ++++++++++++++++- .../ingest/incremental/incremental_test.go | 50 +++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/internal/app/ingest/incremental/incremental.go b/internal/app/ingest/incremental/incremental.go index c1c8cd5..96ce41a 100644 --- a/internal/app/ingest/incremental/incremental.go +++ b/internal/app/ingest/incremental/incremental.go @@ -306,6 +306,8 @@ func (s *Syncer) syncBatchesWithExisting(ctx context.Context, syncStore Store, s } lookup := newImportIndexedLookup(syncStore) + // Resolve every implementation relationship before any call edge. Interface + // dispatch may depend on an implements edge stored in a later spool record. for _, path := range edgeSpool.records { record, err := edgeSpool.readRecord(path) if err != nil { @@ -315,7 +317,20 @@ func (s *Syncer) syncBatchesWithExisting(ctx context.Context, syncStore Store, s for _, file := range record.Files { parsedFiles = append(parsedFiles, parsedSyncFile{filePath: file.FilePath, edges: file.Edges}) } - if err := s.resolveAndUpsertEdges(ctx, syncStore, lookup, parsedFiles, stats); err != nil { + if err := s.resolveAndUpsertImplements(ctx, syncStore, lookup, parsedFiles, stats); err != nil { + return nil, err + } + } + for _, path := range edgeSpool.records { + record, err := edgeSpool.readRecord(path) + if err != nil { + return nil, err + } + parsedFiles := make([]parsedSyncFile, 0, len(record.Files)) + for _, file := range record.Files { + parsedFiles = append(parsedFiles, parsedSyncFile{filePath: file.FilePath, edges: file.Edges}) + } + if err := s.resolveAndUpsertOtherEdges(ctx, syncStore, lookup, parsedFiles, stats); err != nil { return nil, err } } @@ -419,7 +434,25 @@ func (s *Syncer) stageBatch(ctx context.Context, syncStore Store, files map[stri // @sideEffect upserts resolved graph edges through the sync store. // @mutates graph edges, stats.Unresolved func (s *Syncer) resolveAndUpsertEdges(ctx context.Context, syncStore Store, lookup resolve.NodeLookup, parsedFiles []parsedSyncFile, stats *SyncStats) error { - implementsEdges, otherByFile := partitionParsedSyncEdges(parsedFiles) + if err := s.resolveAndUpsertImplements(ctx, syncStore, lookup, parsedFiles, stats); err != nil { + return err + } + return s.resolveAndUpsertOtherEdges(ctx, syncStore, lookup, parsedFiles, stats) +} + +// resolveAndUpsertImplements resolves only implementation relationships from one parsed file set. +// @intent let staged reconciliation finish a global implements pass before resolving interface-dispatch calls. +// @sideEffect upserts resolved implements edges through the sync store. +// @mutates graph edges, stats.Unresolved +func (s *Syncer) resolveAndUpsertImplements(ctx context.Context, syncStore Store, lookup resolve.NodeLookup, parsedFiles []parsedSyncFile, stats *SyncStats) error { + var implementsEdges []graph.Edge + for _, parsed := range parsedFiles { + for _, edge := range parsed.edges { + if edge.Kind == graph.EdgeKindImplements { + implementsEdges = append(implementsEdges, edge) + } + } + } for _, edgeChunk := range splitEdgeChunks(implementsEdges) { resolved, err := resolve.ResolveWithOptions(ctx, lookup, edgeChunk, s.opts) if err != nil { @@ -434,6 +467,15 @@ func (s *Syncer) resolveAndUpsertEdges(ctx context.Context, syncStore Store, loo return err } } + return nil +} + +// resolveAndUpsertOtherEdges resolves non-implements relationships after implementation state is complete. +// @intent preserve file-local import warmup while making interface call resolution independent of spool record order. +// @sideEffect upserts resolved call, import, contains, and related edges through the sync store. +// @mutates graph edges, stats.Unresolved +func (s *Syncer) resolveAndUpsertOtherEdges(ctx context.Context, syncStore Store, lookup resolve.NodeLookup, parsedFiles []parsedSyncFile, stats *SyncStats) error { + _, otherByFile := partitionParsedSyncEdges(parsedFiles) importsByFile := importEdgesByFile(otherByFile) for _, parsed := range parsedFiles { edges := otherByFile[parsed.filePath] diff --git a/internal/app/ingest/incremental/incremental_test.go b/internal/app/ingest/incremental/incremental_test.go index b7928fa..2b04832 100644 --- a/internal/app/ingest/incremental/incremental_test.go +++ b/internal/app/ingest/incremental/incremental_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/tae2089/code-context-graph/internal/adapters/outbound/treesitter" + "github.com/tae2089/code-context-graph/internal/app/ingest" "github.com/tae2089/code-context-graph/internal/domain/graph" "github.com/tae2089/code-context-graph/internal/domain/reference" ) @@ -701,6 +702,55 @@ func TestIncremental_ResolvesCallEdgesAcrossMultipleFilesInBatch(t *testing.T) { } } +func TestSyncBatchesWithExisting_ResolvesImplementsBeforeEarlierCallBatch(t *testing.T) { + st := newStore() + st.nodes["mcp.FlowTracer"] = &graph.Node{ID: 10, QualifiedName: "mcp.FlowTracer", Kind: graph.NodeKindType, Name: "FlowTracer", FilePath: "mcp/deps.go", StartLine: 1, EndLine: 3, Hash: "iface", Language: "go"} + st.nodes["mcp/deps.go"] = &graph.Node{ID: 11, QualifiedName: "mcp/deps.go", Kind: graph.NodeKindFile, Name: "mcp/deps.go", FilePath: "mcp/deps.go", StartLine: 1, EndLine: 3, Hash: "iface", Language: "go"} + parser := &staticParser{result: map[string]parseResult{ + "cmd/main.go": { + nodes: []graph.Node{ + {QualifiedName: "cmd/main.go", Kind: graph.NodeKindFile, Name: "cmd/main.go", FilePath: "cmd/main.go", StartLine: 1, EndLine: 20, Hash: "main", Language: "go"}, + {QualifiedName: "main.Run", Kind: graph.NodeKindFunction, Name: "Run", FilePath: "cmd/main.go", StartLine: 3, EndLine: 8, Hash: "main", Language: "go"}, + }, + edges: []graph.Edge{ + {Kind: graph.EdgeKindImportsFrom, FilePath: "cmd/main.go", Line: 1, Fingerprint: "imports_from:cmd/main.go:github.com/example/project/mcp:1"}, + {Kind: graph.EdgeKindCalls, FilePath: "cmd/main.go", Line: 4, Fingerprint: "calls:cmd/main.go:h.deps.FlowTracer.TraceFlow:4"}, + }, + }, + "flows/tracer.go": { + nodes: []graph.Node{ + {QualifiedName: "flows/tracer.go", Kind: graph.NodeKindFile, Name: "flows/tracer.go", FilePath: "flows/tracer.go", StartLine: 1, EndLine: 20, Hash: "impl", Language: "go"}, + {QualifiedName: "flows.Tracer", Kind: graph.NodeKindClass, Name: "Tracer", FilePath: "flows/tracer.go", StartLine: 3, EndLine: 7, Hash: "impl", Language: "go"}, + {QualifiedName: "flows.Tracer.TraceFlow", Kind: graph.NodeKindFunction, Name: "TraceFlow", FilePath: "flows/tracer.go", StartLine: 5, EndLine: 6, Hash: "impl", Language: "go"}, + }, + edges: []graph.Edge{{Kind: graph.EdgeKindImplements, FilePath: "flows/tracer.go", Line: 3, Fingerprint: "implements:flows/tracer.go:flows.Tracer:mcp.FlowTracer"}}, + }, + }} + + source := ingest.FileBatchSource(func(visitor ingest.FileBatchVisitor) error { + for _, files := range []map[string]ingest.FileInfo{ + {"cmd/main.go": {Hash: "main", Content: []byte("package main")}}, + {"flows/tracer.go": {Hash: "impl", Content: []byte("package flows")}}, + } { + if err := visitor(files); err != nil { + return err + } + } + return nil + }) + + syncer := New(st, parser) + if _, err := syncer.SyncBatchesWithExisting(context.Background(), source, nil); err != nil { + t.Fatal(err) + } + for _, edge := range st.upsertedEdges { + if edge.Kind == graph.EdgeKindCalls && edge.FromNodeID != 0 && edge.ToNodeID != 0 { + return + } + } + t.Fatalf("expected earlier call batch to resolve after later implements batch, got %+v", st.upsertedEdges) +} + func TestIncremental_BatchesNodesBeforeResolvingEdges(t *testing.T) { st := newStore() st.nodes["mcp.FlowTracer"] = &graph.Node{ID: 10, QualifiedName: "mcp.FlowTracer", Kind: graph.NodeKindType, Name: "FlowTracer", FilePath: "mcp/deps.go", StartLine: 1, EndLine: 3, Hash: "iface", Language: "go"} From 13b42d795abb75d5470a8e729c1a5a5b46607108 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Thu, 16 Jul 2026 11:45:07 +0900 Subject: [PATCH 4/7] fix: rebuild updates for new packages --- internal/app/ingest/workflow/indexer_test.go | 107 +++++++++++++++++- internal/app/ingest/workflow/packages.go | 46 ++++++-- internal/app/ingest/workflow/update.go | 109 +++++++++++++++---- 3 files changed, 230 insertions(+), 32 deletions(-) diff --git a/internal/app/ingest/workflow/indexer_test.go b/internal/app/ingest/workflow/indexer_test.go index 905e738..b71d792 100644 --- a/internal/app/ingest/workflow/indexer_test.go +++ b/internal/app/ingest/workflow/indexer_test.go @@ -422,6 +422,26 @@ type recordingSyncCall struct { existingFiles []string } +type countingIncrementalSyncer struct { + delegate *incremental.Syncer + calls int +} + +func (s *countingIncrementalSyncer) SyncWithExisting(ctx context.Context, files map[string]ingest.FileInfo, existingFiles []string) (*ingest.SyncStats, error) { + s.calls++ + return s.delegate.SyncWithExisting(ctx, files, existingFiles) +} + +func (s *countingIncrementalSyncer) SyncWithExistingStore(ctx context.Context, graphStore ingest.GraphStore, files map[string]ingest.FileInfo, existingFiles []string) (*ingest.SyncStats, error) { + s.calls++ + return s.delegate.SyncWithExistingStore(ctx, graphStore, files, existingFiles) +} + +func (s *countingIncrementalSyncer) SyncBatchesWithExistingStore(ctx context.Context, graphStore ingest.GraphStore, source ingest.FileBatchSource, deletedFiles []string) (*ingest.SyncStats, error) { + s.calls++ + return s.delegate.SyncBatchesWithExistingStore(ctx, graphStore, source, deletedFiles) +} + func (r *recordingIncrementalSyncer) Sync(ctx context.Context, files map[string]incremental.FileInfo) (*incremental.SyncStats, error) { panic("unexpected Sync call") } @@ -1121,7 +1141,8 @@ func TestUpdate_ReconcilesExistingCallerAfterTargetFileAdded(t *testing.T) { } writeFile("target.go", "package sample\n\nfunc Target() {}\n") - syncer := incremental.NewWithRegistry(st, map[string]incremental.Parser{".go": walker}, incremental.WithLogger(slog.Default())) + delegate := incremental.NewWithRegistry(st, map[string]incremental.Parser{".go": walker}, incremental.WithLogger(slog.Default())) + syncer := &countingIncrementalSyncer{delegate: delegate} stats, err := svc.Update(ctx, UpdateOptions{BuildOptions: BuildOptions{Dir: tmpDir}, Syncer: syncer}) if err != nil { t.Fatalf("Update after target add: %v", err) @@ -1129,6 +1150,9 @@ func TestUpdate_ReconcilesExistingCallerAfterTargetFileAdded(t *testing.T) { if stats.Added != 1 { t.Fatalf("Added = %d, want one newly added target", stats.Added) } + if syncer.calls != 1 { + t.Fatalf("incremental sync calls = %d, want existing-package update path", syncer.calls) + } source, err := st.GetNode(ctx, "sample.Source") if err != nil || source == nil { @@ -1150,6 +1174,78 @@ func TestUpdate_ReconcilesExistingCallerAfterTargetFileAdded(t *testing.T) { t.Fatalf("expected existing caller edge from %d to newly added target %d, got %+v", source.ID, target.ID, edges) } +func TestUpdate_ReconcilesExistingCrossPackageCallerAfterTargetPackageAdded(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + if err != nil { + t.Fatalf("open db: %v", err) + } + st := graphgorm.New(db) + if err := st.AutoMigrate(); err != nil { + t.Fatalf("migrate: %v", err) + } + walker := treesitter.NewWalker(treesitter.GoSpec) + svc := &Service{Store: st, UnitOfWork: newTestUnitOfWork(db, nil), Walkers: map[string]Parser{".go": walker}, Logger: slog.Default()} + + tmpDir := t.TempDir() + writeFile := func(rel, content string) { + full := filepath.Join(tmpDir, rel) + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", rel, err) + } + if err := os.WriteFile(full, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", rel, err) + } + } + writeFile("go.mod", "module github.com/example/project\n\ngo 1.25.0\n") + writeFile("app/source.go", `package app + +import "github.com/example/project/api" + +func Source() { api.Target() } +`) + + ctx := context.Background() + if _, err := svc.Build(ctx, BuildOptions{Dir: tmpDir}); err != nil { + t.Fatalf("Build before target package exists: %v", err) + } + writeFile("api/target.go", "package api\n\nfunc Target() {}\n") + + delegate := incremental.NewWithRegistry(st, map[string]incremental.Parser{".go": walker}, incremental.WithLogger(slog.Default())) + syncer := &countingIncrementalSyncer{delegate: delegate} + stats, err := svc.Update(ctx, UpdateOptions{BuildOptions: BuildOptions{Dir: tmpDir}, Syncer: syncer}) + if err != nil { + t.Fatalf("Update after target package add: %v", err) + } + if stats.Added != 1 { + t.Fatalf("Added = %d, want one newly added target file", stats.Added) + } + if stats.Modified != 0 || stats.Skipped != 1 || stats.Deleted != 0 { + t.Fatalf("fallback change counts = %+v, want added=1 modified=0 skipped=1 deleted=0", stats) + } + if syncer.calls != 0 { + t.Fatalf("incremental sync calls = %d, want full-build fallback", syncer.calls) + } + + source, err := st.GetNode(ctx, "app.Source") + if err != nil || source == nil { + t.Fatalf("GetNode source: node=%v err=%v", source, err) + } + target, err := st.GetNode(ctx, "api.Target") + if err != nil || target == nil { + t.Fatalf("GetNode target: node=%v err=%v", target, err) + } + edges, err := st.GetEdgesFrom(ctx, source.ID) + if err != nil { + t.Fatalf("GetEdgesFrom: %v", err) + } + for _, edge := range edges { + if edge.Kind == graph.EdgeKindCalls && edge.ToNodeID == target.ID { + return + } + } + t.Fatalf("expected cross-package caller edge from %d to newly added target %d, got %+v", source.ID, target.ID, edges) +} + func TestUpdate_RemovesStaleCrossFileGoStructuralImplements(t *testing.T) { db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { @@ -3393,6 +3489,15 @@ func TestUpdate_IncludePaths_FiltersExistingFilesWhenReplaceFalse(t *testing.T) if got, want := deleteCall.existingFiles, []string{filepath.Join("src", "api", "handler.go")}; !reflect.DeepEqual(got, want) { t.Fatalf("delete pass existingFiles mismatch: got=%v want=%v (helper.go outside include path must be scoped out)", got, want) } + var outsideCount int64 + if err := db.Model(&graph.Node{}). + Where("namespace = ? AND file_path = ?", requestctx.DefaultNamespace, filepath.Join("src", "other", "helper.go")). + Count(&outsideCount).Error; err != nil { + t.Fatalf("count out-of-scope node: %v", err) + } + if outsideCount != 1 { + t.Fatalf("out-of-scope node count = %d, want preserved graph state", outsideCount) + } } func TestUpdate_ExcludePatterns_LeavesMatchingFilesOutOfSync(t *testing.T) { diff --git a/internal/app/ingest/workflow/packages.go b/internal/app/ingest/workflow/packages.go index 985f9d3..f1916dc 100644 --- a/internal/app/ingest/workflow/packages.go +++ b/internal/app/ingest/workflow/packages.go @@ -504,13 +504,13 @@ func affectedPackageImportPaths(packages map[string]languagePackageInfo, affecte return affected } -// addUnchangedPeersForAddedFiles marks unchanged persisted peers for reparse when a new file joins their package or directory. -// @intent let local symbol lookup observe declarations introduced by newly added source files during incremental update. -// @domainRule discovered packages cover multi-directory language packages; a same-directory fallback preserves local resolution when discovery is unavailable. +// addUnchangedPeersForAddedFiles marks local unchanged peers and reports whether added files require graph-wide reconciliation. +// @intent let existing-package additions stay incremental while signaling new package/directory additions that can affect unresolved external callers. +// @domainRule additions to existing packages reparse package/directory peers; a wholly new package or source directory requires full-graph reconciliation because unresolved external callers are not persisted. // @mutates forceFiles -func addUnchangedPeersForAddedFiles(forceFiles map[string]struct{}, packages map[string]languagePackageInfo, existingNodesByFile map[string][]graph.Node, currentHashes map[string]string) { - if forceFiles == nil || len(existingNodesByFile) == 0 || len(currentHashes) == 0 { - return +func addUnchangedPeersForAddedFiles(forceFiles map[string]struct{}, packages map[string]languagePackageInfo, existingNodesByFile map[string][]graph.Node, currentHashes map[string]string) bool { + if forceFiles == nil || len(currentHashes) == 0 { + return false } addedFiles := make([]string, 0) addedDirs := make(map[string]struct{}) @@ -521,20 +521,36 @@ func addUnchangedPeersForAddedFiles(forceFiles map[string]struct{}, packages map } } if len(addedFiles) == 0 { - return + return false } peerFiles := make(map[string]struct{}) + wideReparse := false for _, importPath := range affectedPackageImportPaths(packages, addedFiles) { - for _, filePath := range packages[importPath].Files { + pkg := packages[importPath] + hasPersistedMember := false + for _, filePath := range pkg.Files { + if len(existingNodesByFile[filepath.ToSlash(filePath)]) > 0 { + hasPersistedMember = true + } peerFiles[filepath.ToSlash(filePath)] = struct{}{} } + if !hasPersistedMember { + wideReparse = true + } } + existingDirs := make(map[string]struct{}) for filePath := range existingNodesByFile { filePath = filepath.ToSlash(filePath) + existingDirs[path.Dir(filePath)] = struct{}{} if _, sameDir := addedDirs[path.Dir(filePath)]; sameDir { peerFiles[filePath] = struct{}{} } } + for dir := range addedDirs { + if _, existed := existingDirs[dir]; !existed { + wideReparse = true + } + } for filePath := range peerFiles { nodes := existingNodesByFile[filePath] currentHash, present := currentHashes[filePath] @@ -543,6 +559,20 @@ func addUnchangedPeersForAddedFiles(forceFiles map[string]struct{}, packages map } forceFiles[filePath] = struct{}{} } + return wideReparse +} + +// addAllUnchangedFiles expands a conservative incremental fallback to every unchanged file in the current scope. +// @intent preserve graph correctness when full-build fallback is unsafe or unavailable. +// @mutates forceFiles +func addAllUnchangedFiles(forceFiles map[string]struct{}, existingNodesByFile map[string][]graph.Node, currentHashes map[string]string) { + for filePath, nodes := range existingNodesByFile { + currentHash, present := currentHashes[filePath] + if len(nodes) == 0 || !present || nodes[0].Hash != currentHash { + continue + } + forceFiles[filepath.ToSlash(filePath)] = struct{}{} + } } // appendUniqueString appends a string to a slice only if it is not already present. diff --git a/internal/app/ingest/workflow/update.go b/internal/app/ingest/workflow/update.go index 1b97e14..48082ac 100644 --- a/internal/app/ingest/workflow/update.go +++ b/internal/app/ingest/workflow/update.go @@ -61,18 +61,69 @@ func (s *Service) Update(ctx context.Context, opts UpdateOptions) (*ingest.SyncS defer spool.cleanup(s.logger()) spool.packages = packages - stats := &ingest.SyncStats{} + outcome := updateOutcome{stats: &ingest.SyncStats{}} err = s.withUpdateTx(ctx, func(tx ingest.Transaction) error { - var err error - stats, err = s.applyUpdateSpoolInTx(ctx, tx, opts, spool) + outcome, err = s.applyUpdateSpoolInTx(ctx, tx, opts, spool) return err }) if err != nil { return nil, err } + if outcome.fullBuild { + spool.cleanup(s.logger()) + return s.buildForUpdate(ctx, opts.BuildOptions, outcome.stats) + } + return outcome.stats, nil +} + +// @intent carry the transaction-scoped update decision out to the orchestration layer without exposing a public result type. +type updateOutcome struct { + stats *ingest.SyncStats + fullBuild bool +} + +// buildForUpdate executes a safe full-build fallback while preserving incremental change-count semantics. +// @intent use the faster full-build write path for new packages without changing the Update result contract. +// @sideEffect rebuilds the graph and search index through Service.Build. +func (s *Service) buildForUpdate(ctx context.Context, opts BuildOptions, stats *ingest.SyncStats) (*ingest.SyncStats, error) { + s.logger().Info("new package or source directory detected; switching update to full build", "dir", opts.Dir) + buildStats, err := s.Build(ctx, opts) + if err != nil { + return nil, trace.Wrap(err, "full build fallback") + } + stats.Unresolved = buildStats.Unresolved return stats, nil } +// canBuildForUpdate reports whether a full rebuild preserves the caller's requested graph scope. +// @intent prevent partial non-replacing updates from deleting graph data outside their include paths. +func (s *Service) canBuildForUpdate(opts UpdateOptions) bool { + return s.Store != nil && s.UnitOfWork != nil && (opts.Replace || len(opts.IncludePaths) == 0) +} + +// classifyUpdateSnapshot preserves incremental file outcome counts when execution switches to a full build. +// @intent keep Added/Modified/Skipped/Deleted based on source changes rather than the chosen persistence path. +func classifyUpdateSnapshot(existingFiles []string, existingNodesByFile map[string][]graph.Node, currentHashes map[string]string) *ingest.SyncStats { + stats := &ingest.SyncStats{} + for filePath, currentHash := range currentHashes { + existing := existingNodesByFile[filePath] + switch { + case len(existing) == 0: + stats.Added++ + case existing[0].Hash == currentHash: + stats.Skipped++ + default: + stats.Modified++ + } + } + for _, filePath := range existingFiles { + if _, present := currentHashes[filePath]; !present { + stats.Deleted++ + } + } + return stats +} + // withUpdateTx selects the right transaction scope for incremental update based on syncer and store capability. // @intent prefer a single coupled tx for graph and search rebuild while gracefully degrading when the syncer or store cannot participate. func (s *Service) withUpdateTx(ctx context.Context, fn func(ingest.Transaction) error) error { @@ -177,29 +228,38 @@ func (s *Service) prepareUpdateSpool(ctx context.Context, absDir string, opts Up // @intent prefer staged reconciliation so every changed or forced file node is current before any cross-file edge is resolved. // @sideEffect adds, modifies, and deletes graph nodes/edges/annotations for changed files and refreshes affected search documents. // @mutates graph nodes, edges, annotations, and search_documents -func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transaction, opts UpdateOptions, spool *updateSpool) (*ingest.SyncStats, error) { +func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transaction, opts UpdateOptions, spool *updateSpool) (updateOutcome, error) { txStore := tx.Graph() if txStore == nil { - return nil, trace.New("incremental update requires transaction-scoped store") + return updateOutcome{}, trace.New("incremental update requires transaction-scoped store") } syncer := opts.Syncer stats := &ingest.SyncStats{} existingFiles, existingNodesByFile, err := existingGraphFileState(ctx, txStore) if err != nil { - return nil, trace.Wrap(err, "load existing graph files") + return updateOutcome{}, trace.Wrap(err, "load existing graph files") } if !opts.Replace && len(opts.IncludePaths) > 0 { existingFiles, existingNodesByFile = filterExistingStateByInclude(existingFiles, existingNodesByFile, opts.IncludePaths) } - if err := upsertPackageNodes(ctx, txStore, spool.packages); err != nil { - return nil, trace.Wrap(err, "upsert package nodes") - } forceFiles, err := forceReparseFiles(ctx, txStore, existingNodesByFile, spool.currentHashes) if err != nil { - return nil, trace.Wrap(err, "load edge source files for changed graph") + return updateOutcome{}, trace.Wrap(err, "load edge source files for changed graph") + } + requiresFullBuild := addUnchangedPeersForAddedFiles(forceFiles, spool.packages, existingNodesByFile, spool.currentHashes) + if requiresFullBuild && s.canBuildForUpdate(opts) { + return updateOutcome{ + stats: classifyUpdateSnapshot(existingFiles, existingNodesByFile, spool.currentHashes), + fullBuild: true, + }, nil + } + if requiresFullBuild { + addAllUnchangedFiles(forceFiles, existingNodesByFile, spool.currentHashes) + } + if err := upsertPackageNodes(ctx, txStore, spool.packages); err != nil { + return updateOutcome{}, trace.Wrap(err, "upsert package nodes") } - addUnchangedPeersForAddedFiles(forceFiles, spool.packages, existingNodesByFile, spool.currentHashes) spool.forceFiles = forceFiles deletedFiles := make([]string, 0, len(existingFiles)) for _, fp := range existingFiles { @@ -210,13 +270,13 @@ func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transactio if stagedSyncer, ok := syncer.(transactionalBatchIncrementalSyncer); ok { stats, err = stagedSyncer.SyncBatchesWithExistingStore(ctx, txStore, newUpdateSpoolBatchSource(ctx, spool), deletedFiles) if err != nil { - return nil, trace.Wrap(err, "staged incremental sync") + return updateOutcome{}, trace.Wrap(err, "staged incremental sync") } } else { for _, path := range spool.records { record, err := spool.readRecord(path) if err != nil { - return nil, err + return updateOutcome{}, err } normalFiles, _ := splitForcedFiles(record.Files, spool.forceFiles) if len(normalFiles) == 0 { @@ -224,14 +284,14 @@ func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transactio } batchStats, err := syncIncrementalBatch(ctx, syncer, txStore, normalFiles, nil) if err != nil { - return nil, trace.Wrap(err, "incremental sync") + return updateOutcome{}, trace.Wrap(err, "incremental sync") } addSyncStats(stats, batchStats) } if len(deletedFiles) > 0 { batchStats, err := syncIncrementalBatch(ctx, syncer, txStore, nil, deletedFiles) if err != nil { - return nil, trace.Wrap(err, "incremental sync") + return updateOutcome{}, trace.Wrap(err, "incremental sync") } addSyncStats(stats, batchStats) } @@ -239,7 +299,7 @@ func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transactio for _, path := range spool.records { record, err := spool.readRecord(path) if err != nil { - return nil, err + return updateOutcome{}, err } _, forcedFiles := splitForcedFiles(record.Files, spool.forceFiles) if len(forcedFiles) == 0 { @@ -247,29 +307,29 @@ func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transactio } batchStats, err := syncIncrementalBatch(ctx, syncer, txStore, forcedFiles, nil) if err != nil { - return nil, trace.Wrap(err, "incremental force sync") + return updateOutcome{}, trace.Wrap(err, "incremental force sync") } addSyncStats(stats, batchStats) } } changedFiles := affectedUpdateFiles(spool.currentHashes, existingNodesByFile, spool.forceFiles) if err := s.refreshPackageSemanticEdges(ctx, txStore, opts.Dir, spool.packages, changedFiles, deletedFiles, resolve.ResolveOptions{FallbackCalls: opts.FallbackCalls}); err != nil { - return nil, trace.Wrap(err, "refresh package semantic edges") + return updateOutcome{}, trace.Wrap(err, "refresh package semantic edges") } if err := upsertPackageContainsEdges(ctx, txStore, spool.packages); err != nil { - return nil, trace.Wrap(err, "upsert package file edges") + return updateOutcome{}, trace.Wrap(err, "upsert package file edges") } if !opts.SkipSearchRebuild { nodeIDs, err := affectedNodeIDsForUpdate(ctx, txStore, existingNodesByFile, affectedUpdateFiles(spool.currentHashes, existingNodesByFile, spool.forceFiles), deletedFiles) if err != nil { - return nil, trace.Wrap(err, "load affected search nodes") + return updateOutcome{}, trace.Wrap(err, "load affected search nodes") } if err := tx.Search().RebuildNodes(ctx, nodeIDs); err != nil { - return nil, err + return updateOutcome{}, err } } - return stats, nil + return updateOutcome{stats: stats}, nil } // @intent run incremental sync without a shared DB transaction while replaying spooled file batches to bound memory. @@ -285,7 +345,10 @@ func (s *Service) updateGraphWithoutTx(ctx context.Context, absDir string, opts if err != nil { return nil, trace.Wrap(err, "load edge source files for changed graph") } - addUnchangedPeersForAddedFiles(forceFiles, packages, existingNodesByFile, spool.currentHashes) + requiresFullBuild := addUnchangedPeersForAddedFiles(forceFiles, packages, existingNodesByFile, spool.currentHashes) + if requiresFullBuild { + addAllUnchangedFiles(forceFiles, existingNodesByFile, spool.currentHashes) + } spool.forceFiles = forceFiles if s.Store != nil { if err := upsertPackageNodes(ctx, s.Store, packages); err != nil { From a636fec925e0dc4f9817ab25ba8e032132b5399a Mon Sep 17 00:00:00 2001 From: tae2089 Date: Thu, 16 Jul 2026 13:51:28 +0900 Subject: [PATCH 5/7] perf: cache parsing and batch graph updates Reuse versioned parse results and unresolved candidates during builds and updates. Batch flow traversal and reduce resolver allocation churn for large Kotlin corpora. --- internal/adapters/inbound/cli/build.go | 3 + internal/adapters/inbound/cli/update.go | 3 + .../adapters/inbound/mcp/handler_parse.go | 2 + internal/adapters/outbound/graphgorm/store.go | 49 ++++ .../adapters/outbound/graphgorm/store_test.go | 82 ++++++ .../adapters/outbound/graphgorm/unresolved.go | 148 +++++++++++ .../outbound/treesitter/semantics_jvm.go | 11 +- .../semantics_jvm_allocation_test.go | 24 ++ .../adapters/outbound/treesitter/walker.go | 28 ++- .../outbound/treesitter/walker_test.go | 19 ++ internal/app/analyze/flow/flow.go | 63 ++--- internal/app/analyze/flow/flow_test.go | 38 ++- internal/app/analyze/ports.go | 2 +- .../app/ingest/incremental/incremental.go | 27 +- internal/app/ingest/ports.go | 35 +++ internal/app/ingest/resolve/resolve.go | 111 ++++++++- internal/app/ingest/resolve/resolve_test.go | 17 ++ internal/app/ingest/workflow/build.go | 108 ++++++-- internal/app/ingest/workflow/indexer.go | 15 +- internal/app/ingest/workflow/indexer_test.go | 233 +++++++++++++++++- internal/app/ingest/workflow/parsecache.go | 107 ++++++++ .../app/ingest/workflow/unresolved_version.go | 48 ++++ internal/app/ingest/workflow/update.go | 97 +++++++- internal/db/migration/migration.go | 18 +- internal/db/migration/migration_test.go | 34 +++ .../postgres/000008_parse_cache.down.sql | 1 + .../postgres/000008_parse_cache.up.sql | 17 ++ .../postgres/000009_unresolved_edges.down.sql | 2 + .../postgres/000009_unresolved_edges.up.sql | 22 ++ .../000010_unresolved_index_version.down.sql | 2 + .../000010_unresolved_index_version.up.sql | 2 + .../sqlite/000008_parse_cache.down.sql | 1 + .../sqlite/000008_parse_cache.up.sql | 17 ++ .../sqlite/000009_unresolved_edges.down.sql | 2 + .../sqlite/000009_unresolved_edges.up.sql | 22 ++ .../000010_unresolved_index_version.down.sql | 2 + .../000010_unresolved_index_version.up.sql | 2 + internal/domain/graph/parse_cache.go | 18 ++ internal/domain/graph/unresolved.go | 31 +++ internal/domain/reference/similarity.go | 16 +- internal/domain/reference/similarity_test.go | 14 ++ internal/runtime/remote/http.go | 2 +- 42 files changed, 1411 insertions(+), 84 deletions(-) create mode 100644 internal/adapters/outbound/graphgorm/unresolved.go create mode 100644 internal/adapters/outbound/treesitter/semantics_jvm_allocation_test.go create mode 100644 internal/app/ingest/workflow/parsecache.go create mode 100644 internal/app/ingest/workflow/unresolved_version.go create mode 100644 internal/db/migration/migration_test.go create mode 100644 internal/db/migration/postgres/000008_parse_cache.down.sql create mode 100644 internal/db/migration/postgres/000008_parse_cache.up.sql create mode 100644 internal/db/migration/postgres/000009_unresolved_edges.down.sql create mode 100644 internal/db/migration/postgres/000009_unresolved_edges.up.sql create mode 100644 internal/db/migration/postgres/000010_unresolved_index_version.down.sql create mode 100644 internal/db/migration/postgres/000010_unresolved_index_version.up.sql create mode 100644 internal/db/migration/sqlite/000008_parse_cache.down.sql create mode 100644 internal/db/migration/sqlite/000008_parse_cache.up.sql create mode 100644 internal/db/migration/sqlite/000009_unresolved_edges.down.sql create mode 100644 internal/db/migration/sqlite/000009_unresolved_edges.up.sql create mode 100644 internal/db/migration/sqlite/000010_unresolved_index_version.down.sql create mode 100644 internal/db/migration/sqlite/000010_unresolved_index_version.up.sql create mode 100644 internal/domain/graph/parse_cache.go create mode 100644 internal/domain/graph/unresolved.go diff --git a/internal/adapters/inbound/cli/build.go b/internal/adapters/inbound/cli/build.go index 8c5b73d..58caeb8 100644 --- a/internal/adapters/inbound/cli/build.go +++ b/internal/adapters/inbound/cli/build.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/cobra" "github.com/tae2089/trace" + "github.com/tae2089/code-context-graph/internal/app/ingest" "github.com/tae2089/code-context-graph/internal/app/ingest/workflow" requestctx "github.com/tae2089/code-context-graph/internal/ctx" ) @@ -37,10 +38,12 @@ func newBuildCmd(deps *Deps) *cobra.Command { paths := resolveIncludePaths(includePaths) fileLimit := resolveMaxFileBytes(maxFileBytes) totalLimit := resolveMaxTotalParsedBytes(maxTotalParsedBytes) + parseCache, _ := deps.Store.(ingest.ParseCache) svc := &workflow.Service{ Store: deps.Store, UnitOfWork: deps.UnitOfWork, Search: deps.Search, + ParseCache: parseCache, Walkers: deps.Walkers, Logger: deps.Logger, } diff --git a/internal/adapters/inbound/cli/update.go b/internal/adapters/inbound/cli/update.go index 590b13b..2f5d1fc 100644 --- a/internal/adapters/inbound/cli/update.go +++ b/internal/adapters/inbound/cli/update.go @@ -6,6 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/tae2089/trace" + "github.com/tae2089/code-context-graph/internal/app/ingest" "github.com/tae2089/code-context-graph/internal/app/ingest/workflow" requestctx "github.com/tae2089/code-context-graph/internal/ctx" ) @@ -40,10 +41,12 @@ func newUpdateCmd(deps *Deps) *cobra.Command { ns := resolveNamespace(cmd) ctx = requestctx.WithNamespace(ctx, ns) + parseCache, _ := deps.Store.(ingest.ParseCache) svc := &workflow.Service{ Store: deps.Store, UnitOfWork: deps.UnitOfWork, Search: deps.Search, + ParseCache: parseCache, Walkers: deps.Walkers, Logger: deps.Logger, } diff --git a/internal/adapters/inbound/mcp/handler_parse.go b/internal/adapters/inbound/mcp/handler_parse.go index ed3b0ff..33a7207 100644 --- a/internal/adapters/inbound/mcp/handler_parse.go +++ b/internal/adapters/inbound/mcp/handler_parse.go @@ -71,10 +71,12 @@ func (h *handlers) graphService() *workflow.Service { if candidate, ok := h.deps.Build.Store.(ingest.GraphStore); ok { graphStore = candidate } + parseCache, _ := h.deps.Build.Store.(ingest.ParseCache) return &workflow.Service{ Store: graphStore, UnitOfWork: h.deps.Build.UnitOfWork, Search: h.deps.Build.Search, + ParseCache: parseCache, Parsers: walkers, Logger: h.logger(), } diff --git a/internal/adapters/outbound/graphgorm/store.go b/internal/adapters/outbound/graphgorm/store.go index da430b4..ad68163 100644 --- a/internal/adapters/outbound/graphgorm/store.go +++ b/internal/adapters/outbound/graphgorm/store.go @@ -46,6 +46,9 @@ func (s *Store) AutoMigrate() error { &graph.CommunityMembership{}, &graph.Flow{}, &graph.FlowMembership{}, + &graph.ParseCacheEntry{}, + &graph.UnresolvedEdgeCandidate{}, + &graph.UnresolvedIndexState{}, ); err != nil { return err } @@ -57,6 +60,43 @@ func (s *Store) AutoMigrate() error { return nil } +// LoadParseResult returns a cached parser payload only when every identity field matches. +// @intent prevent stale parser output from being reused across content, path, parser, context, or namespace changes. +func (s *Store) LoadParseResult(ctx context.Context, key ingest.ParseCacheKey) ([]byte, bool, error) { + var entry graph.ParseCacheEntry + ns := requestctx.FromContext(ctx) + result := s.db.WithContext(ctx).Where( + "namespace = ? AND file_path = ? AND source_hash = ? AND parser_version = ? AND context_hash = ?", + ns, key.FilePath, key.SourceHash, key.ParserVersion, key.ContextHash, + ).First(&entry) + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return nil, false, nil + } + if result.Error != nil { + return nil, false, trace.Wrap(result.Error, "load parse cache result") + } + return entry.Payload, true, nil +} + +// StoreParseResult replaces the latest cached parser payload for one namespace/file path. +// @intent retain one bounded current cache entry per source path instead of accumulating every historical hash. +// @sideEffect inserts or updates a parse_cache_entries row. +func (s *Store) StoreParseResult(ctx context.Context, key ingest.ParseCacheKey, payload []byte) error { + entry := graph.ParseCacheEntry{ + Namespace: requestctx.FromContext(ctx), FilePath: key.FilePath, SourceHash: key.SourceHash, + ParserVersion: key.ParserVersion, ContextHash: key.ContextHash, Payload: payload, + } + if err := s.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "namespace"}, {Name: "file_path"}}, + DoUpdates: clause.AssignmentColumns([]string{ + "source_hash", "parser_version", "context_hash", "payload", "updated_at", + }), + }).Create(&entry).Error; err != nil { + return trace.Wrap(err, "store parse cache result") + } + return nil +} + // UpsertNodes stores a batch of nodes keyed by qualified_name. // @intent apply parsed result nodes in bulk without creating duplicates. // @sideEffect performs batch inserts and updates on the nodes table. @@ -257,6 +297,9 @@ func (s *Store) GetFileNodesByPathSuffix(ctx context.Context, suffix string) ([] // @domainRule connected edges and annotations must also be removed when deleting a file. func (s *Store) DeleteNodesByFile(ctx context.Context, filePath string) error { ns := requestctx.FromContext(ctx) + if err := s.db.WithContext(ctx).Where("namespace = ? AND file_path = ?", ns, filePath).Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil { + return trace.Wrap(err, "delete file unresolved edges") + } var nodeIDs []uint if err := s.db.WithContext(ctx). Model(&graph.Node{}). @@ -342,6 +385,12 @@ func (s *Store) DeleteGraph(ctx context.Context) error { } return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if err := tx.Where("namespace = ?", ns).Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil { + return trace.Wrap(err, "delete namespace unresolved edges") + } + if err := tx.Where("namespace = ?", ns).Delete(&graph.UnresolvedIndexState{}).Error; err != nil { + return trace.Wrap(err, "delete namespace unresolved index state") + } if len(filePaths) > 0 { if err := tx. Where("namespace = ? AND file_path IN ?", ns, filePaths). diff --git a/internal/adapters/outbound/graphgorm/store_test.go b/internal/adapters/outbound/graphgorm/store_test.go index ff53f33..802d230 100644 --- a/internal/adapters/outbound/graphgorm/store_test.go +++ b/internal/adapters/outbound/graphgorm/store_test.go @@ -69,6 +69,88 @@ func TestAutoMigrate_SQLite(t *testing.T) { } } +func TestParseCache_RequiresCompleteIdentityAndReplacesLatestFileEntry(t *testing.T) { + store := setupTestDB(t) + ctx := requestctx.WithNamespace(context.Background(), "repo-a") + v1 := ingest.ParseCacheKey{FilePath: "src/a.go", SourceHash: "hash-1", ParserVersion: "go-v1", ContextHash: "ctx-1"} + if err := store.StoreParseResult(ctx, v1, []byte("first")); err != nil { + t.Fatalf("store v1: %v", err) + } + if got, ok, err := store.LoadParseResult(ctx, v1); err != nil || !ok || string(got) != "first" { + t.Fatalf("load v1 = %q, %v, %v", got, ok, err) + } + if _, ok, err := store.LoadParseResult(ctx, ingest.ParseCacheKey{FilePath: "src/a.go", SourceHash: "hash-2", ParserVersion: "go-v1", ContextHash: "ctx-1"}); err != nil || ok { + t.Fatalf("changed source identity should miss, ok=%v err=%v", ok, err) + } + + v2 := ingest.ParseCacheKey{FilePath: "src/a.go", SourceHash: "hash-2", ParserVersion: "go-v2", ContextHash: "ctx-2"} + if err := store.StoreParseResult(ctx, v2, []byte("second")); err != nil { + t.Fatalf("store v2: %v", err) + } + if _, ok, err := store.LoadParseResult(ctx, v1); err != nil || ok { + t.Fatalf("replaced identity should miss, ok=%v err=%v", ok, err) + } + if got, ok, err := store.LoadParseResult(ctx, v2); err != nil || !ok || string(got) != "second" { + t.Fatalf("load v2 = %q, %v, %v", got, ok, err) + } + otherCtx := requestctx.WithNamespace(context.Background(), "repo-b") + if _, ok, err := store.LoadParseResult(otherCtx, v2); err != nil || ok { + t.Fatalf("other namespace should miss, ok=%v err=%v", ok, err) + } +} + +func TestDeleteNodesByFile_RemovesOnlyMatchingNamespaceUnresolvedCandidates(t *testing.T) { + store := setupTestDB(t) + ctxA := requestctx.WithNamespace(context.Background(), "ns-a") + ctxB := requestctx.WithNamespace(context.Background(), "ns-b") + + for _, ctx := range []context.Context{ctxA, ctxB} { + if err := store.UpsertNodes(ctx, []graph.Node{{ + QualifiedName: "pkg.Source", Kind: graph.NodeKindFunction, Name: "Source", FilePath: "source.go", StartLine: 1, + }}); err != nil { + t.Fatalf("UpsertNodes: %v", err) + } + if err := store.UpsertUnresolvedEdges(ctx, []graph.UnresolvedEdgeCandidate{{ + LookupKey: "Target", Fingerprint: "calls:source.go:Target:1", FilePath: "source.go", Kind: graph.EdgeKindCalls, Line: 1, + }}); err != nil { + t.Fatalf("UpsertUnresolvedEdges: %v", err) + } + } + + if err := store.DeleteNodesByFile(ctxA, "source.go"); err != nil { + t.Fatalf("DeleteNodesByFile: %v", err) + } + gotA, err := store.FindUnresolvedEdgesByLookupKeys(ctxA, []string{"Target"}) + if err != nil || len(gotA) != 0 { + t.Fatalf("ns-a candidates = %+v, err=%v; want none", gotA, err) + } + gotB, err := store.FindUnresolvedEdgesByLookupKeys(ctxB, []string{"Target"}) + if err != nil || len(gotB) != 1 { + t.Fatalf("ns-b candidates = %+v, err=%v; want one preserved", gotB, err) + } +} + +func TestUnresolvedIndexReady_RequiresMatchingVersion(t *testing.T) { + store := setupTestDB(t) + ctx := requestctx.WithNamespace(context.Background(), "repo-a") + + if err := store.MarkUnresolvedIndexReady(ctx, "index-v1"); err != nil { + t.Fatalf("MarkUnresolvedIndexReady: %v", err) + } + ready, err := store.UnresolvedIndexReady(ctx, "index-v1") + if err != nil || !ready { + t.Fatalf("matching version ready = %v, err=%v; want true", ready, err) + } + ready, err = store.UnresolvedIndexReady(ctx, "index-v2") + if err != nil || ready { + t.Fatalf("mismatched version ready = %v, err=%v; want false", ready, err) + } + ready, err = store.UnresolvedIndexReady(requestctx.WithNamespace(context.Background(), "repo-b"), "index-v1") + if err != nil || ready { + t.Fatalf("other namespace ready = %v, err=%v; want false", ready, err) + } +} + func TestUpsertNodes_Insert(t *testing.T) { s := setupTestDB(t) ctx := context.Background() diff --git a/internal/adapters/outbound/graphgorm/unresolved.go b/internal/adapters/outbound/graphgorm/unresolved.go new file mode 100644 index 0000000..d38da2f --- /dev/null +++ b/internal/adapters/outbound/graphgorm/unresolved.go @@ -0,0 +1,148 @@ +// @index GORM persistence for unresolved-edge reverse lookup and readiness state. +package graphgorm + +import ( + "context" + "errors" + + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/tae2089/trace" + + requestctx "github.com/tae2089/code-context-graph/internal/ctx" + "github.com/tae2089/code-context-graph/internal/domain/graph" +) + +const unresolvedQueryChunkSize = 400 + +// UpsertUnresolvedEdges stores reverse-index rows without exposing them as traversable graph edges. +// @intent retain unresolved syntax candidates until a future symbol addition can resolve them. +// @sideEffect inserts unresolved_edge_candidates rows. +func (s *Store) UpsertUnresolvedEdges(ctx context.Context, candidates []graph.UnresolvedEdgeCandidate) error { + if len(candidates) == 0 { + return nil + } + ns := requestctx.FromContext(ctx) + for i := range candidates { + candidates[i].Namespace = ns + } + if err := s.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "namespace"}, {Name: "fingerprint"}}, + DoNothing: true, + }).CreateInBatches(candidates, 500).Error; err != nil { + return trace.Wrap(err, "upsert unresolved edges") + } + return nil +} + +// FindUnresolvedEdgesByLookupKeys finds deduplicated syntax edges matching newly added symbol keys. +// @intent use the reverse index to identify affected unchanged source files. +func (s *Store) FindUnresolvedEdgesByLookupKeys(ctx context.Context, keys []string) ([]graph.Edge, error) { + if len(keys) == 0 { + return nil, nil + } + ns := requestctx.FromContext(ctx) + seen := make(map[string]struct{}) + var out []graph.Edge + for start := 0; start < len(keys); start += unresolvedQueryChunkSize { + end := min(start+unresolvedQueryChunkSize, len(keys)) + var rows []graph.UnresolvedEdgeCandidate + if err := s.db.WithContext(ctx). + Where("namespace = ? AND lookup_key IN ?", ns, keys[start:end]). + Order("file_path ASC").Order("line ASC").Order("fingerprint ASC"). + Find(&rows).Error; err != nil { + return nil, trace.Wrap(err, "find unresolved edges by lookup keys") + } + for _, row := range rows { + if _, ok := seen[row.Fingerprint]; ok { + continue + } + seen[row.Fingerprint] = struct{}{} + out = append(out, row.Edge()) + } + } + return out, nil +} + +// FindUnresolvedEdgesByFiles loads every candidate edge owned by the selected affected files. +// @intent replay import warmup and related edges together after reverse-index selection narrows source files. +func (s *Store) FindUnresolvedEdgesByFiles(ctx context.Context, filePaths []string) ([]graph.Edge, error) { + if len(filePaths) == 0 { + return nil, nil + } + ns := requestctx.FromContext(ctx) + seen := make(map[string]struct{}) + var out []graph.Edge + for start := 0; start < len(filePaths); start += unresolvedQueryChunkSize { + end := min(start+unresolvedQueryChunkSize, len(filePaths)) + var rows []graph.UnresolvedEdgeCandidate + if err := s.db.WithContext(ctx). + Where("namespace = ? AND file_path IN ?", ns, filePaths[start:end]). + Order("file_path ASC").Order("line ASC").Order("fingerprint ASC"). + Find(&rows).Error; err != nil { + return nil, trace.Wrap(err, "find unresolved edges by files") + } + for _, row := range rows { + if _, ok := seen[row.Fingerprint]; ok { + continue + } + seen[row.Fingerprint] = struct{}{} + out = append(out, row.Edge()) + } + } + return out, nil +} + +// DeleteUnresolvedEdgesByFingerprints removes every lookup-key row for resolved syntax edges. +// @intent keep the reverse index limited to relationships that still lack endpoints. +func (s *Store) DeleteUnresolvedEdgesByFingerprints(ctx context.Context, fingerprints []string) error { + if len(fingerprints) == 0 { + return nil + } + ns := requestctx.FromContext(ctx) + for start := 0; start < len(fingerprints); start += unresolvedQueryChunkSize { + end := min(start+unresolvedQueryChunkSize, len(fingerprints)) + if err := s.db.WithContext(ctx). + Where("namespace = ? AND fingerprint IN ?", ns, fingerprints[start:end]). + Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil { + return trace.Wrap(err, "delete resolved unresolved edges") + } + } + return nil +} + +// UnresolvedIndexReady reports whether a compatible full build populated the namespace's reverse index. +// @intent gate semi-naive update on complete historical unresolved-edge coverage produced by the expected algorithm and parsers. +func (s *Store) UnresolvedIndexReady(ctx context.Context, version string) (bool, error) { + if version == "" { + return false, nil + } + var state graph.UnresolvedIndexState + ns := requestctx.FromContext(ctx) + result := s.db.WithContext(ctx).Where("namespace = ? AND version = ?", ns, version).First(&state) + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return false, nil + } + if result.Error != nil { + return false, trace.Wrap(result.Error, "load unresolved index state") + } + return true, nil +} + +// MarkUnresolvedIndexReady marks the current namespace and producer version after a successful full candidate pass. +// @intent distinguish a compatible legitimately empty reverse index from stale or uninitialized state. +// @sideEffect inserts or updates unresolved_index_states. +func (s *Store) MarkUnresolvedIndexReady(ctx context.Context, version string) error { + if version == "" { + return trace.New("unresolved index version is empty") + } + state := graph.UnresolvedIndexState{Namespace: requestctx.FromContext(ctx), Version: version} + if err := s.db.WithContext(ctx).Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "namespace"}}, + DoUpdates: clause.AssignmentColumns([]string{"version", "updated_at"}), + }).Create(&state).Error; err != nil { + return trace.Wrap(err, "mark unresolved index ready") + } + return nil +} diff --git a/internal/adapters/outbound/treesitter/semantics_jvm.go b/internal/adapters/outbound/treesitter/semantics_jvm.go index ca9a741..c1598d0 100644 --- a/internal/adapters/outbound/treesitter/semantics_jvm.go +++ b/internal/adapters/outbound/treesitter/semantics_jvm.go @@ -347,6 +347,11 @@ func collectKotlinMemberTypes(root *sitter.Node, content []byte, pkgName string, return members } +var ( + jvmMemberOwnerPattern = regexp.MustCompile(`^(?:public\s+|private\s+|protected\s+|abstract\s+|final\s+|open\s+)?(?:class|interface)\s+([A-Za-z_][A-Za-z0-9_]*)\b`) + jvmMemberFieldPattern = regexp.MustCompile(`^(?:public\s+|private\s+|protected\s+|final\s+|open\s+|override\s+|static\s+)?(?:val\s+|var\s+)?([A-Za-z_][A-Za-z0-9_\.]*)\s+([A-Za-z_][A-Za-z0-9_]*)\s*(?:=|;|$)|^(?:public\s+|private\s+|protected\s+|final\s+|open\s+|override\s+)?(?:val|var)\s+([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z_][A-Za-z0-9_\.]*)`) +) + // collectJVMMembersFromText scans JVM source text for explicitly typed fields and properties. // @intent share one conservative member-type extractor across Java and Kotlin receiver rewriting. func collectJVMMembersFromText(src string, pkgName string, imports map[string]string, blocked map[string]struct{}, hasWildcard bool) map[string]map[string]string { @@ -354,19 +359,17 @@ func collectJVMMembersFromText(src string, pkgName string, imports map[string]st members := make(map[string]map[string]string) owner := "" depth := 0 - ownerPattern := regexp.MustCompile(`^(?:public\s+|private\s+|protected\s+|abstract\s+|final\s+|open\s+)?(?:class|interface)\s+([A-Za-z_][A-Za-z0-9_]*)\b`) - fieldPattern := regexp.MustCompile(`^(?:public\s+|private\s+|protected\s+|final\s+|open\s+|override\s+|static\s+)?(?:val\s+|var\s+)?([A-Za-z_][A-Za-z0-9_\.]*)\s+([A-Za-z_][A-Za-z0-9_]*)\s*(?:=|;|$)|^(?:public\s+|private\s+|protected\s+|final\s+|open\s+|override\s+)?(?:val|var)\s+([A-Za-z_][A-Za-z0-9_]*)\s*:\s*([A-Za-z_][A-Za-z0-9_\.]*)`) for _, line := range lines { trimmed := strings.TrimSpace(line) if owner == "" { - if match := ownerPattern.FindStringSubmatch(trimmed); len(match) == 2 { + if match := jvmMemberOwnerPattern.FindStringSubmatch(trimmed); len(match) == 2 { owner = qualifyTypeName(pkgName, match[1]) depth = strings.Count(line, "{") - strings.Count(line, "}") continue } } if owner != "" { - if match := fieldPattern.FindStringSubmatch(trimmed); len(match) > 0 { + if match := jvmMemberFieldPattern.FindStringSubmatch(trimmed); len(match) > 0 { var name, typeName string if len(match) >= 3 && match[1] != "" && match[2] != "" { typeName = qualifyJVMReceiverTypeName(match[1], pkgName, imports, blocked, hasWildcard) diff --git a/internal/adapters/outbound/treesitter/semantics_jvm_allocation_test.go b/internal/adapters/outbound/treesitter/semantics_jvm_allocation_test.go new file mode 100644 index 0000000..9fb2443 --- /dev/null +++ b/internal/adapters/outbound/treesitter/semantics_jvm_allocation_test.go @@ -0,0 +1,24 @@ +package treesitter + +import "testing" + +func TestCollectJVMMembersFromTextReusesCompiledPatterns(t *testing.T) { + const source = `class User { + Address address; + val profile: Profile +}` + + var members map[string]map[string]string + allocs := testing.AllocsPerRun(100, func() { + members = collectJVMMembersFromText(source, "example", nil, nil, false) + }) + if got := members["example.User"]["address"]; got != "example.Address" { + t.Fatalf("address type = %q, want %q", got, "example.Address") + } + if got := members["example.User"]["profile"]; got != "example.Profile" { + t.Fatalf("profile type = %q, want %q", got, "example.Profile") + } + if allocs > 100 { + t.Fatalf("collectJVMMembersFromText() allocations = %.0f, want <= 100", allocs) + } +} diff --git a/internal/adapters/outbound/treesitter/walker.go b/internal/adapters/outbound/treesitter/walker.go index dc67bfd..fe00568 100644 --- a/internal/adapters/outbound/treesitter/walker.go +++ b/internal/adapters/outbound/treesitter/walker.go @@ -3,6 +3,7 @@ package treesitter import ( "context" + "crypto/sha256" "embed" "fmt" "log/slog" @@ -38,12 +39,13 @@ var errUnsupportedLanguage = trace.New("unsupported language") // @intent turn language-specific ASTs into the project's normalized code graph representation // @mutates parser and query during one-time initialization in NewWalker type Walker struct { - spec *LangSpec - logger *slog.Logger - parser *sitter.Parser // tests/debug helpers inspect this prototype parser - query *sitter.Query // tags.scm 쿼리 1회 컴파일 후 재사용 (nil이면 쿼리 없음) - lang *sitter.Language - pool sync.Pool + spec *LangSpec + logger *slog.Logger + parser *sitter.Parser // tests/debug helpers inspect this prototype parser + query *sitter.Query // tags.scm 쿼리 1회 컴파일 후 재사용 (nil이면 쿼리 없음) + lang *sitter.Language + pool sync.Pool + parseCacheVersion string } // interfaceInfo captures interface method names for later implementation inference. @@ -101,7 +103,9 @@ func NewWalker(spec *LangSpec, opts ...WalkerOption) *Walker { } qPath := fmt.Sprintf("queries/%s/tags.scm", spec.Name) + var queryContent []byte if qContent, err := queriesFS.ReadFile(qPath); err == nil { + queryContent = qContent if q, err := sitter.NewQuery(qContent, lang); err == nil { w.query = q } else { @@ -110,11 +114,23 @@ func NewWalker(spec *LangSpec, opts ...WalkerOption) *Walker { } else { w.logger.Debug("no tags.scm found for language", "language", spec.Name) } + queryHash := sha256.Sum256(queryContent) + w.parseCacheVersion = fmt.Sprintf("walker-v1:%s:%x", spec.Name, queryHash) } return w } +// ParseCacheVersion returns the parser/query generation used to validate serialized parse results. +// @intent invalidate full-build parse cache entries when language queries or parser semantics change. +// @domainRule bump walker-v1 whenever non-query Walker semantics change parsed nodes, edges, comments, or metadata. +func (w *Walker) ParseCacheVersion() string { + if w == nil { + return "" + } + return w.parseCacheVersion +} + // Spec returns the language specification backing this walker. // @intent expose the configured language rules and query paths for this walker instance func (w *Walker) Spec() *LangSpec { diff --git a/internal/adapters/outbound/treesitter/walker_test.go b/internal/adapters/outbound/treesitter/walker_test.go index 1a808c8..8e15da6 100644 --- a/internal/adapters/outbound/treesitter/walker_test.go +++ b/internal/adapters/outbound/treesitter/walker_test.go @@ -24,6 +24,25 @@ func wantInheritsFingerprint(filePath, child, parent string) string { return graph.BuildInheritsFingerprintV2(filePath, child, parent) } +func TestWalkerParseCacheVersion_IsStableAndLanguageScoped(t *testing.T) { + goA := NewWalker(GoSpec) + defer goA.Close() + goB := NewWalker(GoSpec) + defer goB.Close() + kotlin := NewWalker(KotlinSpec) + defer kotlin.Close() + + if goA.ParseCacheVersion() == "" { + t.Fatal("expected non-empty Go parse cache version") + } + if goA.ParseCacheVersion() != goB.ParseCacheVersion() { + t.Fatalf("same language versions differ: %q vs %q", goA.ParseCacheVersion(), goB.ParseCacheVersion()) + } + if goA.ParseCacheVersion() == kotlin.ParseCacheVersion() { + t.Fatalf("different languages share parse cache version %q", goA.ParseCacheVersion()) + } +} + func TestParseWithContext_RespectsContextCancellation(t *testing.T) { w := NewWalker(GoSpec) ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/app/analyze/flow/flow.go b/internal/app/analyze/flow/flow.go index c7ddcad..187e9dc 100644 --- a/internal/app/analyze/flow/flow.go +++ b/internal/app/analyze/flow/flow.go @@ -12,7 +12,7 @@ import ( // EdgeReader exposes graph traversal primitives needed for flow tracing. // @intent abstract graph reads so flow tracing can follow call edges from any store type EdgeReader interface { - GetEdgesFrom(ctx context.Context, nodeID uint) ([]graph.Edge, error) + GetEdgesFromNodes(ctx context.Context, nodeIDs []uint) ([]graph.Edge, error) GetNodeByID(ctx context.Context, id uint) (*graph.Node, error) } @@ -77,7 +77,7 @@ func (t *Tracer) TraceFlow(ctx context.Context, startNodeID uint) (*graph.Flow, // @param startNodeID graph node where traversal begins // @param opts traversal limits applied during BFS // @return TraceResult with the built flow and truncation metadata -// @domainRule only calls edges enqueue new BFS nodes +// @domainRule only calls edges enqueue new BFS nodes; outgoing edges are fetched once per BFS depth // @ensures Truncated is true only when MaxNodes stopped traversal func (t *Tracer) TraceFlowBounded(ctx context.Context, startNodeID uint, opts TraceOptions) (*TraceResult, error) { opts = defaultTraceOptions(opts) @@ -89,42 +89,49 @@ func (t *Tracer) TraceFlowBounded(ctx context.Context, startNodeID uint, opts Tr fallbackEdges := 0 containsFallbackCalls := false - queue := []uint{startNodeID} + frontier := []uint{startNodeID} visited[startNodeID] = true - for len(queue) > 0 { - current := queue[0] - queue = queue[1:] - - members = append(members, graph.FlowMembership{ - Namespace: ns, - NodeID: current, - Ordinal: ordinal, - }) - ordinal++ + for len(frontier) > 0 { + for _, current := range frontier { + members = append(members, graph.FlowMembership{ + Namespace: ns, + NodeID: current, + Ordinal: ordinal, + }) + ordinal++ + } - edges, err := t.store.GetEdgesFrom(ctx, current) + edges, err := t.store.GetEdgesFromNodes(ctx, frontier) if err != nil { return nil, err } - for _, e := range edges { - if !graph.IsCallKind(e.Kind) || visited[e.ToNodeID] { - continue - } - if e.Kind == graph.EdgeKindFallbackCalls { - if !*opts.IncludeFallbackCalls { + edgesBySource := make(map[uint][]graph.Edge, len(frontier)) + for _, edge := range edges { + edgesBySource[edge.FromNodeID] = append(edgesBySource[edge.FromNodeID], edge) + } + nextFrontier := make([]uint, 0) + for _, current := range frontier { + for _, e := range edgesBySource[current] { + if !graph.IsCallKind(e.Kind) || visited[e.ToNodeID] { continue } - fallbackEdges++ - containsFallbackCalls = true - } - if opts.MaxNodes > 0 && len(visited) >= opts.MaxNodes { - truncated = true - break + if e.Kind == graph.EdgeKindFallbackCalls { + if !*opts.IncludeFallbackCalls { + continue + } + fallbackEdges++ + containsFallbackCalls = true + } + if opts.MaxNodes > 0 && len(visited) >= opts.MaxNodes { + truncated = true + break + } + visited[e.ToNodeID] = true + nextFrontier = append(nextFrontier, e.ToNodeID) } - visited[e.ToNodeID] = true - queue = append(queue, e.ToNodeID) } + frontier = nextFrontier } node, _ := t.store.GetNodeByID(ctx, startNodeID) diff --git a/internal/app/analyze/flow/flow_test.go b/internal/app/analyze/flow/flow_test.go index 8bea1d3..90caaa7 100644 --- a/internal/app/analyze/flow/flow_test.go +++ b/internal/app/analyze/flow/flow_test.go @@ -3,6 +3,7 @@ package flow import ( "context" "fmt" + "reflect" "strings" "sync/atomic" "testing" @@ -25,6 +26,7 @@ type mockStore struct { nodes map[uint]*graph.Node edges map[uint][]graph.Edge fetchedNodeIDs []uint + edgeBatchCalls [][]uint } func (m *mockStore) GetEdgesFrom(_ context.Context, nodeID uint) ([]graph.Edge, error) { @@ -32,6 +34,15 @@ func (m *mockStore) GetEdgesFrom(_ context.Context, nodeID uint) ([]graph.Edge, return m.edges[nodeID], nil } +func (m *mockStore) GetEdgesFromNodes(_ context.Context, nodeIDs []uint) ([]graph.Edge, error) { + m.edgeBatchCalls = append(m.edgeBatchCalls, append([]uint(nil), nodeIDs...)) + var edges []graph.Edge + for _, nodeID := range nodeIDs { + edges = append(edges, m.edges[nodeID]...) + } + return edges, nil +} + func (m *mockStore) GetNodeByID(_ context.Context, id uint) (*graph.Node, error) { n, ok := m.nodes[id] if !ok { @@ -213,6 +224,29 @@ func TestTraceFlow_Merge(t *testing.T) { } } +func TestTraceFlowBounded_ReadsOutgoingEdgesOncePerBFSDepth(t *testing.T) { + ms := &mockStore{ + nodes: map[uint]*graph.Node{ + 1: newNode(1, "A"), 2: newNode(2, "B"), 3: newNode(3, "C"), + 4: newNode(4, "D"), 5: newNode(5, "E"), + }, + edges: map[uint][]graph.Edge{ + 1: {callEdge(1, 2, 1), callEdge(1, 3, 2)}, + 2: {callEdge(2, 4, 3)}, + 3: {callEdge(3, 5, 4)}, + }, + } + result, err := New(ms).TraceFlowBounded(context.Background(), 1, TraceOptions{}) + if err != nil { + t.Fatal(err) + } + assertUintSliceEqual(t, flowMemberIDs(result.Flow), []uint{1, 2, 3, 4, 5}) + want := [][]uint{{1}, {2, 3}, {4, 5}} + if !reflect.DeepEqual(ms.edgeBatchCalls, want) { + t.Fatalf("edge batch calls = %v, want %v", ms.edgeBatchCalls, want) + } +} + func TestTraceFlow_NoEdges(t *testing.T) { ms := &mockStore{ nodes: map[uint]*graph.Node{1: newNode(1, "A")}, @@ -294,7 +328,9 @@ func TestTraceFlowBounded_DoesNotEnqueueBeyondMaxNodesInHighFanout(t *testing.T) } assertUintSliceEqual(t, flowMemberIDs(result.Flow), []uint{1, 2, 3}) - assertUintSliceEqual(t, ms.fetchedNodeIDs, []uint{1, 2, 3}) + if want := [][]uint{{1}, {2, 3}}; !reflect.DeepEqual(ms.edgeBatchCalls, want) { + t.Fatalf("edge batch calls = %v, want %v", ms.edgeBatchCalls, want) + } if !result.Truncated { t.Fatalf("expected high fan-out traversal to report truncation") } diff --git a/internal/app/analyze/ports.go b/internal/app/analyze/ports.go index 9165bf6..bc3fa21 100644 --- a/internal/app/analyze/ports.go +++ b/internal/app/analyze/ports.go @@ -10,7 +10,7 @@ import ( // FlowRebuildStore is the transaction-scoped graph and flow persistence surface used by a rebuild. // @intent let flow application policy trace and replace flows without importing a database adapter. type FlowRebuildStore interface { - GetEdgesFrom(ctx context.Context, nodeID uint) ([]graph.Edge, error) + GetEdgesFromNodes(ctx context.Context, nodeIDs []uint) ([]graph.Edge, error) GetNodeByID(ctx context.Context, id uint) (*graph.Node, error) DeleteFlows(ctx context.Context) error FindFlowEntrypoints(ctx context.Context) ([]graph.Node, error) diff --git a/internal/app/ingest/incremental/incremental.go b/internal/app/ingest/incremental/incremental.go index 96ce41a..4864abe 100644 --- a/internal/app/ingest/incremental/incremental.go +++ b/internal/app/ingest/incremental/incremental.go @@ -458,8 +458,11 @@ func (s *Syncer) resolveAndUpsertImplements(ctx context.Context, syncStore Store if err != nil { return err } - resolved, diagnostics := resolve.FilterResolvedWithDiagnosticsFiltered(resolved, shouldSuppressExternalImportUnresolved) + resolved, unresolved, diagnostics := resolve.PartitionResolvedWithDiagnosticsFiltered(resolved, shouldSuppressExternalImportUnresolved) mergeSyncUnresolvedDiagnostics(stats, diagnostics) + if err := persistUnresolvedEdges(ctx, syncStore, unresolved); err != nil { + return err + } if len(resolved) == 0 { continue } @@ -485,8 +488,11 @@ func (s *Syncer) resolveAndUpsertOtherEdges(ctx context.Context, syncStore Store if err != nil { return err } - resolved, diagnostics := resolve.FilterResolvedWithDiagnosticsFiltered(resolved[len(resolveInput)-len(edgeChunk):], shouldSuppressExternalImportUnresolved) + resolved, unresolved, diagnostics := resolve.PartitionResolvedWithDiagnosticsFiltered(resolved[len(resolveInput)-len(edgeChunk):], shouldSuppressExternalImportUnresolved) mergeSyncUnresolvedDiagnostics(stats, diagnostics) + if err := persistUnresolvedEdges(ctx, syncStore, unresolved); err != nil { + return err + } if len(resolved) == 0 { continue } @@ -498,6 +504,23 @@ func (s *Syncer) resolveAndUpsertOtherEdges(ctx context.Context, syncStore Store return nil } +// persistUnresolvedEdges records dropped syntax candidates when the active store supports semi-naive replay. +// @intent keep incremental candidate maintenance optional for legacy/custom store implementations. +// @sideEffect writes unresolved candidate rows through ingest.UnresolvedEdgeStore. +func persistUnresolvedEdges(ctx context.Context, store Store, edges []graph.Edge) error { + if len(edges) == 0 { + return nil + } + unresolvedStore, ok := store.(ingest.UnresolvedEdgeStore) + if !ok { + return nil + } + if err := unresolvedStore.UpsertUnresolvedEdges(ctx, resolve.BuildUnresolvedCandidates(edges)); err != nil { + return trace.Wrap(err, "persist unresolved incremental edges") + } + return nil +} + // resolveParser picks an extension-specific parser when configured, otherwise the legacy single parser. // @intent let multi-language projects sync without losing the single-parser fallback for callers using New. func (s *Syncer) resolveParser(filePath string) Parser { diff --git a/internal/app/ingest/ports.go b/internal/app/ingest/ports.go index 8f4dfc5..d942ebd 100644 --- a/internal/app/ingest/ports.go +++ b/internal/app/ingest/ports.go @@ -69,6 +69,41 @@ type Parser interface { ParseWithContext(ctx context.Context, filePath string, content []byte) ([]graph.Node, []graph.Edge, error) } +// VersionedParser exposes the semantic parser/query identity used for parse-result cache invalidation. +// @intent let ingest reuse parsed output only while parser behavior and embedded queries remain compatible. +type VersionedParser interface { + Parser + ParseCacheVersion() string +} + +// ParseCacheKey identifies one correctness-equivalent parsed source result. +// @intent include every input known to affect parser output instead of trusting source content alone. +type ParseCacheKey struct { + FilePath string + SourceHash string + ParserVersion string + ContextHash string +} + +// ParseCache stores opaque serialized parse results outside the graph transaction. +// @intent make repeated full builds skip Tree-sitter work without coupling ingest to one cache backend. +// @domainRule cache failures are optimization misses and must not fail graph builds. +type ParseCache interface { + LoadParseResult(ctx context.Context, key ParseCacheKey) ([]byte, bool, error) + StoreParseResult(ctx context.Context, key ParseCacheKey, payload []byte) error +} + +// UnresolvedEdgeStore persists and replays unresolved syntax relationships for semi-naive updates. +// @intent select unchanged source edges affected by newly added symbols without exposing persistence details. +type UnresolvedEdgeStore interface { + UpsertUnresolvedEdges(ctx context.Context, candidates []graph.UnresolvedEdgeCandidate) error + FindUnresolvedEdgesByLookupKeys(ctx context.Context, keys []string) ([]graph.Edge, error) + FindUnresolvedEdgesByFiles(ctx context.Context, filePaths []string) ([]graph.Edge, error) + DeleteUnresolvedEdgesByFingerprints(ctx context.Context, fingerprints []string) error + UnresolvedIndexReady(ctx context.Context, version string) (bool, error) + MarkUnresolvedIndexReady(ctx context.Context, version string) error +} + // AnnotatingParser adds comments and language identity for annotation restoration. // @intent make comment-aware parsing an optional ingest capability. type AnnotatingParser interface { diff --git a/internal/app/ingest/resolve/resolve.go b/internal/app/ingest/resolve/resolve.go index 3e432c6..e0b5165 100644 --- a/internal/app/ingest/resolve/resolve.go +++ b/internal/app/ingest/resolve/resolve.go @@ -343,10 +343,18 @@ func FilterResolvedWithDiagnostics(edges []graph.Edge) ([]graph.Edge, FilterReso // callers suppress selected unresolved edges from the reported summary. // @intent keep edge filtering behavior stable while controlling noise from known-unresolvable patterns. func FilterResolvedWithDiagnosticsFiltered(edges []graph.Edge, filterUnresolved UnresolvedEdgeFilter) ([]graph.Edge, FilterResolvedDiagnostics) { + resolved, _, diagnostics := PartitionResolvedWithDiagnosticsFiltered(edges, filterUnresolved) + return resolved, diagnostics +} + +// PartitionResolvedWithDiagnosticsFiltered separates traversable edges from durable unresolved candidates. +// @intent let build and update persist unresolved syntax edges without changing query-visible graph semantics. +func PartitionResolvedWithDiagnosticsFiltered(edges []graph.Edge, filterUnresolved UnresolvedEdgeFilter) ([]graph.Edge, []graph.Edge, FilterResolvedDiagnostics) { if len(edges) == 0 { - return nil, FilterResolvedDiagnostics{} + return nil, nil, FilterResolvedDiagnostics{} } out := make([]graph.Edge, 0, len(edges)) + unresolved := make([]graph.Edge, 0) diagnostics := FilterResolvedDiagnostics{} for _, edge := range edges { reason, dropped := unresolvedReason(edge) @@ -355,11 +363,110 @@ func FilterResolvedWithDiagnosticsFiltered(edges []graph.Edge, filterUnresolved continue } diagnostics.add(edge, reason) + unresolved = append(unresolved, edge) continue } out = append(out, edge) } - return out, diagnostics + return out, unresolved, diagnostics +} + +// BuildUnresolvedCandidates expands unresolved fingerprints into conservative reverse lookup keys. +// @intent prefer extra candidate replay over missing a caller that a newly added symbol can resolve. +func BuildUnresolvedCandidates(edges []graph.Edge) []graph.UnresolvedEdgeCandidate { + var candidates []graph.UnresolvedEdgeCandidate + for _, edge := range edges { + if edge.FromNodeID == 0 || edge.ToNodeID != 0 || edge.Kind == graph.EdgeKindContains { + continue + } + key := primaryUnresolvedLookupKey(edge) + if key == "" { + continue + } + candidates = append(candidates, graph.UnresolvedEdgeCandidate{ + LookupKey: key, Fingerprint: edge.Fingerprint, FilePath: edge.FilePath, Kind: edge.Kind, Line: edge.Line, + }) + } + return candidates +} + +// primaryUnresolvedLookupKey chooses one target-oriented key for an edge whose source already resolved. +// @intent bound reverse-index rows to one per edge while matching newly added node simple names. +func primaryUnresolvedLookupKey(edge graph.Edge) string { + var target string + switch edge.Kind { + case graph.EdgeKindCalls: + target, _ = callCallee(edge) + case graph.EdgeKindImportsFrom: + target, _ = importsFromTarget(edge) + case graph.EdgeKindInherits: + _, target, _ = graph.ParseInheritsFingerprint(edge.FilePath, edge.Fingerprint) + default: + parts := strings.Split(edge.Fingerprint, ":") + if len(parts) > 0 { + target = parts[len(parts)-1] + } + } + normalized := strings.ReplaceAll(strings.TrimSpace(target), "::", ".") + normalized = strings.Trim(normalized, "<>[]{}()\"'") + if idx := strings.LastIndex(normalized, "."); idx >= 0 { + normalized = normalized[idx+1:] + } + trimmedPath := strings.Trim(normalized, "/") + if idx := strings.LastIndex(trimmedPath, "/"); idx >= 0 { + trimmedPath = trimmedPath[idx+1:] + } + return trimmedPath +} + +// LookupKeysForNodes creates the key set used to probe unresolved candidates after node additions. +// @intent match qualified names, simple names, and package/file path suffixes conservatively. +func LookupKeysForNodes(nodes []graph.Node) []string { + var values []string + for _, node := range nodes { + values = append(values, node.QualifiedName, node.Name) + if node.FilePath != "" { + values = append(values, path.Dir(node.FilePath)) + } + } + return expandLookupValues(values) +} + +// expandLookupValues derives stable qualified-name, simple-name, and path-suffix variants. +// @intent make added-node lookup keys match the bounded simple target keys stored for unresolved edges. +func expandLookupValues(values []string) []string { + seen := make(map[string]struct{}) + var out []string + add := func(value string) { + value = strings.TrimSpace(strings.Trim(value, "<>[]{}()\"'")) + if value == "" { + return + } + if _, ok := seen[value]; ok { + return + } + seen[value] = struct{}{} + out = append(out, value) + } + for _, value := range values { + add(value) + normalized := strings.ReplaceAll(value, "::", ".") + add(normalized) + if idx := strings.LastIndex(normalized, "."); idx >= 0 { + add(normalized[idx+1:]) + } + trimmedPath := strings.Trim(normalized, "/") + for trimmedPath != "" && trimmedPath != "." { + add(trimmedPath) + idx := strings.Index(trimmedPath, "/") + if idx < 0 { + break + } + trimmedPath = trimmedPath[idx+1:] + } + } + sort.Strings(out) + return out } // unresolvedReason classifies why an edge will be dropped from the resolved output. diff --git a/internal/app/ingest/resolve/resolve_test.go b/internal/app/ingest/resolve/resolve_test.go index 5920dae..e91507d 100644 --- a/internal/app/ingest/resolve/resolve_test.go +++ b/internal/app/ingest/resolve/resolve_test.go @@ -855,6 +855,23 @@ func TestFilterResolvedWithDiagnosticsFilteredSkipsExternalImportUnresolvedEdges } } +func TestBuildUnresolvedCandidates_IndexesOnlyResolvedSourcesWithMissingTargets(t *testing.T) { + edges := []graph.Edge{ + {FromNodeID: 10, Kind: graph.EdgeKindCalls, FilePath: "app/source.go", Line: 3, Fingerprint: "calls:app/source.go:api.Target:3"}, + {Kind: graph.EdgeKindCalls, FilePath: "app/missing.go", Line: 4, Fingerprint: "calls:app/missing.go:api.Other:4"}, + {FromNodeID: 10, ToNodeID: 20, Kind: graph.EdgeKindCalls, FilePath: "app/source.go", Line: 5, Fingerprint: "calls:app/source.go:api.Ready:5"}, + {FromNodeID: 10, Kind: graph.EdgeKindContains, FilePath: "app/source.go", Line: 1, Fingerprint: "contains:app/source.go:app.Source"}, + } + + candidates := BuildUnresolvedCandidates(edges) + if len(candidates) != 1 { + t.Fatalf("candidate count = %d, want 1: %+v", len(candidates), candidates) + } + if got := candidates[0]; got.LookupKey != "Target" || got.Fingerprint != edges[0].Fingerprint || got.FilePath != "app/source.go" { + t.Fatalf("candidate = %+v, want one target-oriented row for api.Target", got) + } +} + func TestIsLikelyExternalImportPath(t *testing.T) { cases := map[string]bool{ "fmt": true, diff --git a/internal/app/ingest/workflow/build.go b/internal/app/ingest/workflow/build.go index 28f851e..1642e26 100644 --- a/internal/app/ingest/workflow/build.go +++ b/internal/app/ingest/workflow/build.go @@ -48,18 +48,21 @@ type buildEdgeBatchSource func(yield func(parsedBuildEdgeBatch) error) error // buildParseInput is one validated source file assigned to a parse worker. // @intent keep deterministic input sequencing separate from concurrent filesystem and parser work. type buildParseInput struct { - seq int - path string - relPath string - parser Parser + seq int + path string + relPath string + parser Parser + contextHash string } // buildParseResult is one parsed source file returned to the spool coordinator. // @intent let workers finish out of order while the coordinator preserves record order. type buildParseResult struct { - seq int - record spooledBuildRecord - err error + seq int + record spooledBuildRecord + err error + cacheHit bool + cacheMiss bool } // newParsedBuildNodeBatch packages parsed nodes plus comment metadata for later persistence. @@ -209,6 +212,8 @@ func (s *Service) Build(ctx context.Context, opts BuildOptions) (BuildStats, err ) s.logger().Debug("build timing", "parse_ms", stats.Timing.ParseMS, + "parse_cache_hits", stats.Timing.ParseCacheHits, + "parse_cache_misses", stats.Timing.ParseCacheMisses, "persist_nodes_ms", stats.Timing.PersistNodesMS, "resolve_edges_ms", stats.Timing.ResolveEdgesMS, "resolver_calls", stats.Timing.Resolve.Resolver.Calls, @@ -248,7 +253,7 @@ func (s *Service) prepareBuildSpool(ctx context.Context, absDir string, opts Bui return nil, trace.Wrap(err, "create build spool") } spool := &buildSpool{dir: dir} - inputs, err := s.collectBuildParseInputs(ctx, absDir, opts) + inputs, err := s.collectBuildParseInputs(ctx, absDir, opts, parseSemanticContextHash(ctx)) if err != nil { spool.cleanup(s.logger()) return nil, err @@ -266,6 +271,12 @@ func (s *Service) prepareBuildSpool(ctx context.Context, absDir string, opts Bui spool.stats.TotalFiles++ spool.stats.TotalNodes += len(result.record.Nodes) spool.stats.TotalEdges += len(result.record.Edges) + if result.cacheHit { + spool.stats.Timing.ParseCacheHits++ + } + if result.cacheMiss { + spool.stats.Timing.ParseCacheMisses++ + } return nil }) if err != nil { @@ -278,7 +289,7 @@ func (s *Service) prepareBuildSpool(ctx context.Context, absDir string, opts Bui // collectBuildParseInputs walks eligible files once and validates their scheduled parse budget. // @intent preserve build traversal policy and deterministic file order before concurrent parsing starts. -func (s *Service) collectBuildParseInputs(ctx context.Context, absDir string, opts BuildOptions) ([]buildParseInput, error) { +func (s *Service) collectBuildParseInputs(ctx context.Context, absDir string, opts BuildOptions, contextHash string) ([]buildParseInput, error) { var inputs []buildParseInput var scheduledParsedBytes int64 if err := walkMatchingFiles(ctx, absDir, opts, func(path, relPath string) error { @@ -299,10 +310,11 @@ func (s *Service) collectBuildParseInputs(ctx context.Context, absDir string, op } scheduledParsedBytes += info.Size() inputs = append(inputs, buildParseInput{ - seq: len(inputs), - path: path, - relPath: relPath, - parser: parser, + seq: len(inputs), + path: path, + relPath: relPath, + parser: parser, + contextHash: contextHash, }) return nil }); err != nil { @@ -440,16 +452,32 @@ func (s *Service) parseBuildInput(ctx context.Context, input buildParseInput) bu result.err = err return result } + hash := sha256.Sum256(content) + hashString := hex.EncodeToString(hash[:]) + cacheKey, cacheable := parseResultCacheKey(input, hashString) + if cacheable && s.ParseCache != nil { + payload, found, loadErr := s.ParseCache.LoadParseResult(ctx, cacheKey) + if loadErr != nil { + s.logger().DebugContext(ctx, "parse cache load failed; parsing source", "file", input.relPath, "error", loadErr) + } + if found && loadErr == nil { + cached, decodeErr := decodeCachedParseRecord(payload) + if decodeErr == nil { + setBuildNodeHashes(cached.Nodes, hashString) + result.record = cached.toSpooledRecord(input.relPath, content) + result.cacheHit = true + return result + } + s.logger().DebugContext(ctx, "parse cache decode failed; parsing source", "file", input.relPath, "error", decodeErr) + } + result.cacheMiss = true + } nodes, edges, tsComments, meta, language, err := parseForBuild(ctx, input.parser, input.relPath, content) if err != nil { result.err = trace.Wrap(err, "parse build file "+input.relPath) return result } - hash := sha256.Sum256(content) - hashString := hex.EncodeToString(hash[:]) - for i := range nodes { - nodes[i].Hash = hashString - } + setBuildNodeHashes(nodes, hashString) nodeBatch := newParsedBuildNodeBatch(input.relPath, content, nodes, meta.Package, meta.Interfaces, tsComments, language) result.record = spooledBuildRecord{ RelPath: input.relPath, @@ -462,6 +490,14 @@ func (s *Service) parseBuildInput(ctx context.Context, input buildParseInput) bu Edges: edges, Bytes: int64(len(content)), } + if cacheable && s.ParseCache != nil { + payload, encodeErr := encodeCachedParseRecord(cachedParseRecordFrom(result.record)) + if encodeErr != nil { + s.logger().DebugContext(ctx, "parse cache encode failed", "file", input.relPath, "error", encodeErr) + } else if storeErr := s.ParseCache.StoreParseResult(ctx, cacheKey, payload); storeErr != nil { + s.logger().DebugContext(ctx, "parse cache store failed", "file", input.relPath, "error", storeErr) + } + } return result } @@ -532,6 +568,13 @@ func (s *Service) applyBuildSpoolInTx(ctx context.Context, tx ingestapp.Transact if err := s.flushBuildEdgeSourceWithTiming(ctx, txStore, spool.edgeBatchSource(ctx, packageEdgeBatches), &spool.stats, resolveOptions, resolveTiming); err != nil { return err } + if unresolvedStore, ok := txStore.(ingestapp.UnresolvedEdgeStore); ok { + if version, versioned := s.unresolvedIndexVersion(); versioned { + if err := unresolvedStore.MarkUnresolvedIndexReady(ctx, version); err != nil { + return trace.Wrap(err, "mark unresolved edge index ready") + } + } + } if timing != nil { timing.ResolveEdgesMS = time.Since(edgeResolveStart).Milliseconds() } @@ -796,8 +839,11 @@ func (s *Service) flushBuildEdgeSourceWithTiming(ctx context.Context, txStore in s.logger().ErrorContext(ctx, "resolve deferred implements edges failed", append(obs.TraceLogArgs(ctx), "edges", len(pendingImplements), "error", err)...) return trace.Wrap(err, "resolve deferred implements edges") } - resolved, diagnostics := resolve.FilterResolvedWithDiagnosticsFiltered(resolved, shouldSuppressExternalImportUnresolved) + resolved, unresolved, diagnostics := resolve.PartitionResolvedWithDiagnosticsFiltered(resolved, shouldSuppressExternalImportUnresolved) mergeBuildUnresolvedDiagnostics(stats, diagnostics) + if err := persistBuildUnresolvedEdges(ctx, txStore, unresolved); err != nil { + return err + } if diagnostics.DroppedCount > 0 { s.logger().DebugContext(ctx, "dropped unresolved implements edges", append(obs.TraceLogArgs(ctx), "count", diagnostics.DroppedCount, "by_kind", formatEdgeKindCounts(diagnostics.ByKind), "by_reason", diagnostics.ByReason)...) } @@ -855,6 +901,7 @@ func (s *Service) flushBuildEdgeSourceWithTiming(ctx context.Context, txStore in return trace.Wrap(err, "resolve deferred edge batch") } resolvedEdges := make([]graph.Edge, 0, len(resolved)) + var unresolvedEdges []graph.Edge for _, actual := range actualRanges { if actual.end > len(resolved) { actual.end = len(resolved) @@ -862,9 +909,13 @@ func (s *Service) flushBuildEdgeSourceWithTiming(ctx context.Context, txStore in if actual.start >= actual.end { continue } - resolvedChunk, diagnostics := resolve.FilterResolvedWithDiagnosticsFiltered(resolved[actual.start:actual.end], shouldSuppressExternalImportUnresolved) + resolvedChunk, unresolvedChunk, diagnostics := resolve.PartitionResolvedWithDiagnosticsFiltered(resolved[actual.start:actual.end], shouldSuppressExternalImportUnresolved) mergeBuildUnresolvedDiagnostics(stats, diagnostics) resolvedEdges = append(resolvedEdges, resolvedChunk...) + unresolvedEdges = append(unresolvedEdges, unresolvedChunk...) + } + if err := persistBuildUnresolvedEdges(ctx, txStore, unresolvedEdges); err != nil { + return err } if len(resolvedEdges) > 0 { upsertStarted := time.Now() @@ -910,6 +961,23 @@ func (s *Service) flushBuildEdgeSourceWithTiming(ctx context.Context, txStore in return nil } +// persistBuildUnresolvedEdges records syntax candidates separately from traversable graph edges. +// @intent populate the reverse index during full builds while keeping stores without the optional capability compatible. +// @sideEffect writes unresolved candidate rows when supported by the graph store. +func persistBuildUnresolvedEdges(ctx context.Context, store ingestapp.GraphStore, edges []graph.Edge) error { + if len(edges) == 0 { + return nil + } + unresolvedStore, ok := store.(ingestapp.UnresolvedEdgeStore) + if !ok { + return nil + } + if err := unresolvedStore.UpsertUnresolvedEdges(ctx, resolve.BuildUnresolvedCandidates(edges)); err != nil { + return trace.Wrap(err, "persist unresolved build edges") + } + return nil +} + // importEdges selects file-local import warmup edges from one streamed edge batch. // @intent preserve import-aware call resolution without retaining a build-wide import map. func importEdges(edges []graph.Edge) []graph.Edge { diff --git a/internal/app/ingest/workflow/indexer.go b/internal/app/ingest/workflow/indexer.go index 1334809..2e541b0 100644 --- a/internal/app/ingest/workflow/indexer.go +++ b/internal/app/ingest/workflow/indexer.go @@ -51,6 +51,7 @@ type Service struct { Store ingestapp.GraphStore UnitOfWork ingestapp.UnitOfWork Search ingestapp.SearchWriter + ParseCache ingestapp.ParseCache Walkers map[string]Parser Parsers map[string]Parser Logger *slog.Logger @@ -116,12 +117,14 @@ type BuildStats struct { // BuildTiming reports elapsed milliseconds for the major full-build stages. // @intent expose actionable stage-level evidence so large-build regressions can be diagnosed without guessing. type BuildTiming struct { - ParseMS int64 - PersistNodesMS int64 - ResolveEdgesMS int64 - SearchRebuildMS int64 - TotalMS int64 - Resolve BuildResolveTiming + ParseMS int64 + ParseCacheHits int + ParseCacheMisses int + PersistNodesMS int64 + ResolveEdgesMS int64 + SearchRebuildMS int64 + TotalMS int64 + Resolve BuildResolveTiming } // BuildResolveOperationTiming reports one resolver operation's call count and elapsed time. diff --git a/internal/app/ingest/workflow/indexer_test.go b/internal/app/ingest/workflow/indexer_test.go index b71d792..00aea6f 100644 --- a/internal/app/ingest/workflow/indexer_test.go +++ b/internal/app/ingest/workflow/indexer_test.go @@ -517,6 +517,56 @@ type blockingBuildParser struct { release <-chan struct{} } +type countingVersionedBuildParser struct { + calls int + version string +} + +func (p *countingVersionedBuildParser) Parse(filePath string, content []byte) ([]graph.Node, []graph.Edge, error) { + return p.ParseWithContext(context.Background(), filePath, content) +} + +func (p *countingVersionedBuildParser) ParseWithContext(ctx context.Context, filePath string, content []byte) ([]graph.Node, []graph.Edge, error) { + p.calls++ + return failingBuildParser{}.ParseWithContext(ctx, filePath, content) +} + +func (p *countingVersionedBuildParser) ParseCacheVersion() string { + return p.version +} + +type memoryParseCache struct { + entries map[ingest.ParseCacheKey][]byte +} + +type failingParseCache struct { + payload []byte + found bool + loadErr error + storeErr error +} + +func (c failingParseCache) LoadParseResult(_ context.Context, _ ingest.ParseCacheKey) ([]byte, bool, error) { + return c.payload, c.found, c.loadErr +} + +func (c failingParseCache) StoreParseResult(_ context.Context, _ ingest.ParseCacheKey, _ []byte) error { + return c.storeErr +} + +func (c *memoryParseCache) LoadParseResult(_ context.Context, key ingest.ParseCacheKey) ([]byte, bool, error) { + payload, ok := c.entries[key] + return append([]byte(nil), payload...), ok, nil +} + +func (c *memoryParseCache) StoreParseResult(_ context.Context, key ingest.ParseCacheKey, payload []byte) error { + if c.entries == nil { + c.entries = make(map[ingest.ParseCacheKey][]byte) + } + c.entries[key] = append([]byte(nil), payload...) + return nil +} + func (p blockingBuildParser) Parse(filePath string, content []byte) ([]graph.Node, []graph.Edge, error) { return p.ParseWithContext(context.Background(), filePath, content) } @@ -1199,15 +1249,33 @@ func TestUpdate_ReconcilesExistingCrossPackageCallerAfterTargetPackageAdded(t *t writeFile("go.mod", "module github.com/example/project\n\ngo 1.25.0\n") writeFile("app/source.go", `package app -import "github.com/example/project/api" +import ( + "github.com/example/project/api" + "github.com/example/project/other" +) -func Source() { api.Target() } +func Source() { + api.Target() + other.Second() +} `) ctx := context.Background() if _, err := svc.Build(ctx, BuildOptions{Dir: tmpDir}); err != nil { t.Fatalf("Build before target package exists: %v", err) } + indexVersion, versioned := svc.unresolvedIndexVersion() + if !versioned { + t.Fatal("expected Go walker to provide unresolved index version") + } + ready, err := st.UnresolvedIndexReady(ctx, indexVersion) + if err != nil || !ready { + t.Fatalf("unresolved index ready = %v, err=%v", ready, err) + } + indexed, err := st.FindUnresolvedEdgesByLookupKeys(ctx, []string{"api.Target", "Target"}) + if err != nil || len(indexed) == 0 { + t.Fatalf("expected unresolved caller candidate before target add, candidates=%+v err=%v", indexed, err) + } writeFile("api/target.go", "package api\n\nfunc Target() {}\n") delegate := incremental.NewWithRegistry(st, map[string]incremental.Parser{".go": walker}, incremental.WithLogger(slog.Default())) @@ -1222,8 +1290,8 @@ func Source() { api.Target() } if stats.Modified != 0 || stats.Skipped != 1 || stats.Deleted != 0 { t.Fatalf("fallback change counts = %+v, want added=1 modified=0 skipped=1 deleted=0", stats) } - if syncer.calls != 0 { - t.Fatalf("incremental sync calls = %d, want full-build fallback", syncer.calls) + if syncer.calls != 1 { + t.Fatalf("incremental sync calls = %d, want semi-naive update path", syncer.calls) } source, err := st.GetNode(ctx, "app.Source") @@ -1238,12 +1306,57 @@ func Source() { api.Target() } if err != nil { t.Fatalf("GetEdgesFrom: %v", err) } + foundTarget := false for _, edge := range edges { if edge.Kind == graph.EdgeKindCalls && edge.ToNodeID == target.ID { + remaining, lookupErr := st.FindUnresolvedEdgesByLookupKeys(ctx, []string{"api.Target", "Target"}) + if lookupErr != nil { + t.Fatalf("load remaining candidates: %v", lookupErr) + } + if len(remaining) != 0 { + t.Fatalf("resolved caller candidate remained indexed: %+v", remaining) + } + foundTarget = true + break + } + } + if !foundTarget { + t.Fatalf("expected cross-package caller edge from %d to newly added target %d, got %+v", source.ID, target.ID, edges) + } + + if err := st.MarkUnresolvedIndexReady(ctx, "stale-index-version"); err != nil { + t.Fatalf("mark stale unresolved index: %v", err) + } + writeFile("other/second.go", "package other\n\nfunc Second() {}\n") + staleSyncer := &countingIncrementalSyncer{delegate: delegate} + staleStats, err := svc.Update(ctx, UpdateOptions{BuildOptions: BuildOptions{Dir: tmpDir}, Syncer: staleSyncer}) + if err != nil { + t.Fatalf("Update after stale-version target add: %v", err) + } + if staleSyncer.calls != 0 { + t.Fatalf("incremental sync calls = %d, want full-build fallback for stale index", staleSyncer.calls) + } + if staleStats.Added != 1 || staleStats.Modified != 0 || staleStats.Skipped != 2 || staleStats.Deleted != 0 { + t.Fatalf("stale fallback change counts = %+v, want added=1 modified=0 skipped=2 deleted=0", staleStats) + } + second, err := st.GetNode(ctx, "other.Second") + if err != nil || second == nil { + t.Fatalf("GetNode second: node=%v err=%v", second, err) + } + source, err = st.GetNode(ctx, "app.Source") + if err != nil || source == nil { + t.Fatalf("GetNode source after fallback: node=%v err=%v", source, err) + } + edges, err = st.GetEdgesFrom(ctx, source.ID) + if err != nil { + t.Fatalf("GetEdgesFrom after fallback: %v", err) + } + for _, edge := range edges { + if edge.Kind == graph.EdgeKindCalls && edge.ToNodeID == second.ID { return } } - t.Fatalf("expected cross-package caller edge from %d to newly added target %d, got %+v", source.ID, target.ID, edges) + t.Fatalf("expected full-build fallback edge from %d to newly added second target %d, got %+v", source.ID, second.ID, edges) } func TestUpdate_RemovesStaleCrossFileGoStructuralImplements(t *testing.T) { @@ -2175,6 +2288,116 @@ func TestPrepareBuildSpool_ParsesFilesConcurrentlyAndKeepsOrder(t *testing.T) { } } +func TestBuildParseCache_ReusesParsedRecordOnWarmBuild(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + if err != nil { + t.Fatalf("open db: %v", err) + } + st := graphgorm.New(db) + if err := st.AutoMigrate(); err != nil { + t.Fatalf("migrate: %v", err) + } + parser := &countingVersionedBuildParser{version: "stub-v1"} + cache := &memoryParseCache{} + svc := &Service{ + Store: st, + UnitOfWork: newTestUnitOfWork(db, nil), + Parsers: map[string]Parser{".stub": parser}, + ParseCache: cache, + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "sample.stub"), []byte("same content"), 0o644); err != nil { + t.Fatal(err) + } + + cold, err := svc.Build(context.Background(), BuildOptions{Dir: dir, SkipSearchRebuild: true}) + if err != nil { + t.Fatalf("cold build: %v", err) + } + warm, err := svc.Build(context.Background(), BuildOptions{Dir: dir, SkipSearchRebuild: true}) + if err != nil { + t.Fatalf("warm build: %v", err) + } + + if parser.calls != 1 { + t.Fatalf("parser calls = %d, want 1 across cold and warm builds", parser.calls) + } + if cold.Timing.ParseCacheMisses != 1 || cold.Timing.ParseCacheHits != 0 { + t.Fatalf("cold cache timing = hits:%d misses:%d, want hits:0 misses:1", cold.Timing.ParseCacheHits, cold.Timing.ParseCacheMisses) + } + if warm.Timing.ParseCacheHits != 1 || warm.Timing.ParseCacheMisses != 0 { + t.Fatalf("warm cache timing = hits:%d misses:%d, want hits:1 misses:0", warm.Timing.ParseCacheHits, warm.Timing.ParseCacheMisses) + } +} + +func TestBuildParseCache_FailsOpenOnReadDecodeAndWriteErrors(t *testing.T) { + tests := []struct { + name string + cache failingParseCache + }{ + {name: "read error", cache: failingParseCache{loadErr: errors.New("cache unavailable"), storeErr: errors.New("cache unavailable")}}, + {name: "corrupt payload", cache: failingParseCache{payload: []byte("not-gob"), found: true, storeErr: errors.New("cache unavailable")}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + if err != nil { + t.Fatalf("open db: %v", err) + } + st := graphgorm.New(db) + if err := st.AutoMigrate(); err != nil { + t.Fatalf("migrate: %v", err) + } + parser := &countingVersionedBuildParser{version: "stub-v1"} + svc := &Service{ + Store: st, UnitOfWork: newTestUnitOfWork(db, nil), + Parsers: map[string]Parser{".stub": parser}, ParseCache: tt.cache, + } + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "sample.stub"), []byte("content"), 0o644); err != nil { + t.Fatal(err) + } + + stats, err := svc.Build(context.Background(), BuildOptions{Dir: dir, SkipSearchRebuild: true}) + if err != nil { + t.Fatalf("build with failed cache: %v", err) + } + if parser.calls != 1 || stats.Timing.ParseCacheHits != 0 || stats.Timing.ParseCacheMisses != 1 { + t.Fatalf("parser calls=%d cache hits=%d misses=%d, want 1/0/1", parser.calls, stats.Timing.ParseCacheHits, stats.Timing.ParseCacheMisses) + } + }) + } +} + +func TestUnresolvedIndexVersion_IsDeterministicAndParserVersionScoped(t *testing.T) { + goParser := &countingVersionedBuildParser{version: "go-v1"} + kotlinParser := &countingVersionedBuildParser{version: "kotlin-v1"} + first := &Service{Parsers: map[string]Parser{".go": goParser, ".kt": kotlinParser}} + second := &Service{Parsers: map[string]Parser{".kt": kotlinParser, ".go": goParser}} + + firstVersion, ok := first.unresolvedIndexVersion() + if !ok || firstVersion == "" { + t.Fatal("expected version for versioned parsers") + } + secondVersion, ok := second.unresolvedIndexVersion() + if !ok || secondVersion != firstVersion { + t.Fatalf("registration order changed version: first=%q second=%q", firstVersion, secondVersion) + } + + kotlinParser.version = "kotlin-v2" + changedVersion, ok := first.unresolvedIndexVersion() + if !ok || changedVersion == firstVersion { + t.Fatalf("parser version change did not invalidate unresolved index: before=%q after=%q", firstVersion, changedVersion) + } +} + +func TestUnresolvedIndexVersion_IsUnavailableForUnversionedParser(t *testing.T) { + svc := &Service{Parsers: map[string]Parser{".stub": failingBuildParser{}}} + if version, ok := svc.unresolvedIndexVersion(); ok || version != "" { + t.Fatalf("unversioned parser produced unresolved index version %q, ok=%v", version, ok) + } +} + func TestImportPackageContext_TypeScriptAliasUsesCanonicalImportPath(t *testing.T) { packages := map[string]languagePackageInfo{ "@acme/app/src/base": { diff --git a/internal/app/ingest/workflow/parsecache.go b/internal/app/ingest/workflow/parsecache.go new file mode 100644 index 0000000..1566cdb --- /dev/null +++ b/internal/app/ingest/workflow/parsecache.go @@ -0,0 +1,107 @@ +// @index Parse-result cache identity and serialization for repeatable full builds. +package workflow + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "sort" + + ingestapp "github.com/tae2089/code-context-graph/internal/app/ingest" + "github.com/tae2089/code-context-graph/internal/domain/graph" +) + +// cachedParseRecord is the parser-owned portion of a build spool record. +// @intent cache reusable syntax results without duplicating source text or invocation-local byte accounting. +type cachedParseRecord struct { + Nodes []graph.Node + PackageName string + Interfaces []ingestapp.PackageInterfaceInfo + Comments []ingestapp.CommentBlock + Language string + Edges []graph.Edge +} + +// cachedParseRecordFrom removes invocation-local source and byte data before serialization. +// @intent keep durable cache payloads limited to parser output reused by later builds. +func cachedParseRecordFrom(record spooledBuildRecord) cachedParseRecord { + return cachedParseRecord{ + Nodes: record.Nodes, PackageName: record.PackageName, Interfaces: record.Interfaces, + Comments: record.Comments, Language: record.Language, Edges: record.Edges, + } +} + +// toSpooledRecord restores source-derived fields that intentionally are not cached. +// @intent reconstruct the same build spool contract on a cache hit as on a fresh parse. +func (r cachedParseRecord) toSpooledRecord(relPath string, content []byte) spooledBuildRecord { + nodeBatch := newParsedBuildNodeBatch(relPath, content, r.Nodes, r.PackageName, r.Interfaces, r.Comments, r.Language) + return spooledBuildRecord{ + RelPath: relPath, Nodes: r.Nodes, PackageName: r.PackageName, Interfaces: r.Interfaces, + Comments: r.Comments, Language: r.Language, SourceLines: nodeBatch.sourceLines, + Edges: r.Edges, Bytes: int64(len(content)), + } +} + +// encodeCachedParseRecord serializes parser output into an opaque adapter payload. +// @intent keep cache persistence independent of workflow-internal record types. +func encodeCachedParseRecord(record cachedParseRecord) ([]byte, error) { + var out bytes.Buffer + err := gob.NewEncoder(&out).Encode(record) + return out.Bytes(), err +} + +// decodeCachedParseRecord validates and restores an opaque cached parser payload. +// @intent treat malformed or incompatible cache bytes as a recoverable cache miss. +func decodeCachedParseRecord(payload []byte) (cachedParseRecord, error) { + var record cachedParseRecord + err := gob.NewDecoder(bytes.NewReader(payload)).Decode(&record) + return record, err +} + +// parseResultCacheKey builds a complete semantic cache identity for versioned parsers. +// @intent bypass caching when a parser cannot prove how its output version is invalidated. +func parseResultCacheKey(input buildParseInput, sourceHash string) (ingestapp.ParseCacheKey, bool) { + versioned, ok := input.parser.(ingestapp.VersionedParser) + if !ok || versioned.ParseCacheVersion() == "" { + return ingestapp.ParseCacheKey{}, false + } + return ingestapp.ParseCacheKey{ + FilePath: input.relPath, SourceHash: sourceHash, + ParserVersion: versioned.ParseCacheVersion(), ContextHash: input.contextHash, + }, true +} + +// parseSemanticContextHash fingerprints repository maps that can change semantic parser output. +// @intent invalidate cached syntax results when import or file-package normalization changes. +func parseSemanticContextHash(ctx context.Context) string { + h := sha256.New() + writeMap := func(prefix string, values map[string]string) { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + h.Write([]byte(prefix)) + h.Write([]byte{0}) + h.Write([]byte(key)) + h.Write([]byte{0}) + h.Write([]byte(values[key])) + h.Write([]byte{0}) + } + } + writeMap("import", ingestapp.ImportPackagesFromContext(ctx)) + writeMap("file", ingestapp.FilePackagesFromContext(ctx)) + return hex.EncodeToString(h.Sum(nil)) +} + +// setBuildNodeHashes applies the current source hash after a fresh parse or cache hit. +// @intent keep change detection based on current content rather than serialized node state. +// @mutates nodes +func setBuildNodeHashes(nodes []graph.Node, hash string) { + for i := range nodes { + nodes[i].Hash = hash + } +} diff --git a/internal/app/ingest/workflow/unresolved_version.go b/internal/app/ingest/workflow/unresolved_version.go new file mode 100644 index 0000000..bc09664 --- /dev/null +++ b/internal/app/ingest/workflow/unresolved_version.go @@ -0,0 +1,48 @@ +// @index Unresolved-edge index compatibility identity for safe semi-naive updates. +package workflow + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + + ingestapp "github.com/tae2089/code-context-graph/internal/app/ingest" +) + +const unresolvedIndexAlgorithmVersion = "missing-target-v1" + +// unresolvedIndexVersion fingerprints the candidate algorithm and every parser that can influence the built graph. +// @intent prevent semi-naive replay from consuming unresolved candidates produced by incompatible parser/query or resolution behavior. +// @domainRule bump unresolvedIndexAlgorithmVersion whenever unresolved candidate selection or endpoint resolution semantics change. +func (s *Service) unresolvedIndexVersion() (string, bool) { + parsers := make(map[string]Parser, len(s.Parsers)+len(s.Walkers)) + for ext, parser := range s.Parsers { + parsers["parser:"+ext] = parser + } + for ext, parser := range s.Walkers { + parsers["walker:"+ext] = parser + } + if len(parsers) == 0 { + return "", false + } + keys := make([]string, 0, len(parsers)) + for key := range parsers { + keys = append(keys, key) + } + sort.Strings(keys) + + h := sha256.New() + h.Write([]byte(unresolvedIndexAlgorithmVersion)) + h.Write([]byte{0}) + for _, key := range keys { + versioned, ok := parsers[key].(ingestapp.VersionedParser) + if !ok || versioned.ParseCacheVersion() == "" { + return "", false + } + h.Write([]byte(key)) + h.Write([]byte{0}) + h.Write([]byte(versioned.ParseCacheVersion())) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)), true +} diff --git a/internal/app/ingest/workflow/update.go b/internal/app/ingest/workflow/update.go index 48082ac..8b50651 100644 --- a/internal/app/ingest/workflow/update.go +++ b/internal/app/ingest/workflow/update.go @@ -8,6 +8,7 @@ import ( "errors" "os" "path/filepath" + "slices" "strings" "github.com/tae2089/trace" @@ -248,13 +249,24 @@ func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transactio return updateOutcome{}, trace.Wrap(err, "load edge source files for changed graph") } requiresFullBuild := addUnchangedPeersForAddedFiles(forceFiles, spool.packages, existingNodesByFile, spool.currentHashes) + useSemiNaiveReplay := false if requiresFullBuild && s.canBuildForUpdate(opts) { + version, versioned := s.unresolvedIndexVersion() + if unresolvedStore, ok := txStore.(ingest.UnresolvedEdgeStore); ok && versioned { + ready, readyErr := unresolvedStore.UnresolvedIndexReady(ctx, version) + if readyErr != nil { + return updateOutcome{}, trace.Wrap(readyErr, "check unresolved edge index readiness") + } + useSemiNaiveReplay = ready + } + } + if requiresFullBuild && s.canBuildForUpdate(opts) && !useSemiNaiveReplay { return updateOutcome{ stats: classifyUpdateSnapshot(existingFiles, existingNodesByFile, spool.currentHashes), fullBuild: true, }, nil } - if requiresFullBuild { + if requiresFullBuild && !useSemiNaiveReplay { addAllUnchangedFiles(forceFiles, existingNodesByFile, spool.currentHashes) } if err := upsertPackageNodes(ctx, txStore, spool.packages); err != nil { @@ -319,6 +331,12 @@ func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transactio if err := upsertPackageContainsEdges(ctx, txStore, spool.packages); err != nil { return updateOutcome{}, trace.Wrap(err, "upsert package file edges") } + if useSemiNaiveReplay { + addedFiles := addedUpdateFiles(existingNodesByFile, spool.currentHashes) + if err := replayUnresolvedEdgesForAddedFiles(ctx, txStore, addedFiles, resolve.ResolveOptions{FallbackCalls: opts.FallbackCalls}, stats); err != nil { + return updateOutcome{}, err + } + } if !opts.SkipSearchRebuild { nodeIDs, err := affectedNodeIDsForUpdate(ctx, txStore, existingNodesByFile, affectedUpdateFiles(spool.currentHashes, existingNodesByFile, spool.forceFiles), deletedFiles) @@ -332,6 +350,83 @@ func (s *Service) applyUpdateSpoolInTx(ctx context.Context, tx ingest.Transactio return updateOutcome{stats: stats}, nil } +// addedUpdateFiles returns current paths that had no persisted nodes before the update. +// @intent seed semi-naive unresolved lookup from newly introduced source files only. +func addedUpdateFiles(existingNodesByFile map[string][]graph.Node, currentHashes map[string]string) []string { + files := make([]string, 0) + for filePath := range currentHashes { + if len(existingNodesByFile[filePath]) == 0 { + files = append(files, filePath) + } + } + slices.Sort(files) + return files +} + +// replayUnresolvedEdgesForAddedFiles resolves only unchanged-source candidates selected by newly added symbols. +// @intent replace graph-wide reparsing with reverse-index-driven edge reconciliation for new packages. +// @sideEffect reads unresolved candidates, upserts newly resolved edges, and removes resolved candidate rows. +func replayUnresolvedEdgesForAddedFiles(ctx context.Context, graphStore ingest.GraphStore, addedFiles []string, options resolve.ResolveOptions, stats *ingest.SyncStats) error { + unresolvedStore, ok := graphStore.(ingest.UnresolvedEdgeStore) + if !ok || len(addedFiles) == 0 { + return nil + } + nodesByFile, err := graphStore.GetNodesByFiles(ctx, addedFiles) + if err != nil { + return trace.Wrap(err, "load added nodes for unresolved replay") + } + var addedNodes []graph.Node + for _, filePath := range addedFiles { + addedNodes = append(addedNodes, nodesByFile[filePath]...) + } + matched, err := unresolvedStore.FindUnresolvedEdgesByLookupKeys(ctx, resolve.LookupKeysForNodes(addedNodes)) + if err != nil { + return trace.Wrap(err, "select unresolved edges by added symbols") + } + if len(matched) == 0 { + return nil + } + fileSet := make(map[string]struct{}) + for _, edge := range matched { + fileSet[edge.FilePath] = struct{}{} + } + files := make([]string, 0, len(fileSet)) + for filePath := range fileSet { + files = append(files, filePath) + } + slices.Sort(files) + candidates, err := unresolvedStore.FindUnresolvedEdgesByFiles(ctx, files) + if err != nil { + return trace.Wrap(err, "load affected unresolved source edges") + } + implementsEdges, otherEdges := splitImplementsEdges(candidates) + for _, phase := range [][]graph.Edge{implementsEdges, otherEdges} { + if len(phase) == 0 { + continue + } + resolvedEdges, err := resolve.ResolveWithOptions(ctx, graphStore, phase, options) + if err != nil { + return trace.Wrap(err, "resolve semi-naive edge candidates") + } + resolvedEdges, _, diagnostics := resolve.PartitionResolvedWithDiagnosticsFiltered(resolvedEdges, shouldSuppressExternalImportUnresolved) + mergeFilterResolvedDiagnostics(&stats.Unresolved, diagnostics) + if len(resolvedEdges) == 0 { + continue + } + if err := graphStore.UpsertEdges(ctx, resolvedEdges); err != nil { + return trace.Wrap(err, "upsert semi-naive resolved edges") + } + fingerprints := make([]string, 0, len(resolvedEdges)) + for _, edge := range resolvedEdges { + fingerprints = append(fingerprints, edge.Fingerprint) + } + if err := unresolvedStore.DeleteUnresolvedEdgesByFingerprints(ctx, fingerprints); err != nil { + return trace.Wrap(err, "delete resolved semi-naive candidates") + } + } + return nil +} + // @intent run incremental sync without a shared DB transaction while replaying spooled file batches to bound memory. func (s *Service) updateGraphWithoutTx(ctx context.Context, absDir string, opts UpdateOptions, packages map[string]languagePackageInfo, spool *updateSpool) (*ingest.SyncStats, error) { existingFiles, existingNodesByFile, err := existingGraphFileState(ctx, s.Store) diff --git a/internal/db/migration/migration.go b/internal/db/migration/migration.go index f926235..5b37a5f 100644 --- a/internal/db/migration/migration.go +++ b/internal/db/migration/migration.go @@ -23,7 +23,7 @@ import ( ) const ( - RequiredSchemaVersion = 7 + RequiredSchemaVersion = 10 SchemaVersionKey = "schema" LegacySchemaVersionTable = "ccg_schema_versions" ) @@ -382,6 +382,9 @@ func RequiredSchemaTables() []string { "flows", "flow_memberships", "search_documents", + "parse_cache_entries", + "unresolved_edge_candidates", + "unresolved_index_states", } } @@ -403,6 +406,19 @@ func ModelNullabilityColumns() []SchemaColumn { {Table: "flows", Column: "name"}, {Table: "flow_memberships", Column: "flow_id"}, {Table: "flow_memberships", Column: "node_id"}, + {Table: "parse_cache_entries", Column: "namespace"}, + {Table: "parse_cache_entries", Column: "file_path"}, + {Table: "parse_cache_entries", Column: "source_hash"}, + {Table: "parse_cache_entries", Column: "parser_version"}, + {Table: "parse_cache_entries", Column: "context_hash"}, + {Table: "parse_cache_entries", Column: "payload"}, + {Table: "unresolved_edge_candidates", Column: "namespace"}, + {Table: "unresolved_edge_candidates", Column: "lookup_key"}, + {Table: "unresolved_edge_candidates", Column: "fingerprint"}, + {Table: "unresolved_edge_candidates", Column: "file_path"}, + {Table: "unresolved_edge_candidates", Column: "kind"}, + {Table: "unresolved_index_states", Column: "namespace"}, + {Table: "unresolved_index_states", Column: "version"}, } } diff --git a/internal/db/migration/migration_test.go b/internal/db/migration/migration_test.go new file mode 100644 index 0000000..6ad7db3 --- /dev/null +++ b/internal/db/migration/migration_test.go @@ -0,0 +1,34 @@ +package migration + +import "testing" + +func TestRequiredSchemaTables_IncludesOptimizationState(t *testing.T) { + want := []string{"parse_cache_entries", "unresolved_edge_candidates", "unresolved_index_states"} + got := make(map[string]struct{}) + for _, table := range RequiredSchemaTables() { + got[table] = struct{}{} + } + for _, table := range want { + if _, ok := got[table]; !ok { + t.Errorf("RequiredSchemaTables missing %q", table) + } + } +} + +func TestModelNullabilityColumns_IncludesOptimizationState(t *testing.T) { + want := []SchemaColumn{ + {Table: "parse_cache_entries", Column: "payload"}, + {Table: "unresolved_edge_candidates", Column: "lookup_key"}, + {Table: "unresolved_index_states", Column: "namespace"}, + {Table: "unresolved_index_states", Column: "version"}, + } + got := make(map[SchemaColumn]struct{}) + for _, column := range ModelNullabilityColumns() { + got[column] = struct{}{} + } + for _, column := range want { + if _, ok := got[column]; !ok { + t.Errorf("ModelNullabilityColumns missing %+v", column) + } + } +} diff --git a/internal/db/migration/postgres/000008_parse_cache.down.sql b/internal/db/migration/postgres/000008_parse_cache.down.sql new file mode 100644 index 0000000..200e3f0 --- /dev/null +++ b/internal/db/migration/postgres/000008_parse_cache.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS parse_cache_entries; diff --git a/internal/db/migration/postgres/000008_parse_cache.up.sql b/internal/db/migration/postgres/000008_parse_cache.up.sql new file mode 100644 index 0000000..4ce8a1d --- /dev/null +++ b/internal/db/migration/postgres/000008_parse_cache.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS parse_cache_entries ( + id bigserial PRIMARY KEY, + namespace varchar(256) NOT NULL DEFAULT 'default', + file_path varchar(768) NOT NULL, + source_hash varchar(64) NOT NULL, + parser_version varchar(160) NOT NULL, + context_hash varchar(64) NOT NULL, + payload bytea NOT NULL, + created_at timestamptz, + updated_at timestamptz +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_parse_cache_ns_file +ON parse_cache_entries(namespace, file_path); + +CREATE INDEX IF NOT EXISTS idx_parse_cache_source_hash +ON parse_cache_entries(source_hash); diff --git a/internal/db/migration/postgres/000009_unresolved_edges.down.sql b/internal/db/migration/postgres/000009_unresolved_edges.down.sql new file mode 100644 index 0000000..1d53621 --- /dev/null +++ b/internal/db/migration/postgres/000009_unresolved_edges.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS unresolved_index_states; +DROP TABLE IF EXISTS unresolved_edge_candidates; diff --git a/internal/db/migration/postgres/000009_unresolved_edges.up.sql b/internal/db/migration/postgres/000009_unresolved_edges.up.sql new file mode 100644 index 0000000..e772902 --- /dev/null +++ b/internal/db/migration/postgres/000009_unresolved_edges.up.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS unresolved_edge_candidates ( + id bigserial PRIMARY KEY, + namespace varchar(256) NOT NULL DEFAULT 'default', + lookup_key varchar(512) NOT NULL, + fingerprint text NOT NULL, + file_path varchar(1024) NOT NULL, + kind varchar(32) NOT NULL, + line integer, + created_at timestamptz +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_unresolved_ns_fp +ON unresolved_edge_candidates(namespace, fingerprint); +CREATE INDEX IF NOT EXISTS idx_unresolved_edge_candidates_lookup_key +ON unresolved_edge_candidates(lookup_key); +CREATE INDEX IF NOT EXISTS idx_unresolved_edge_candidates_file_path +ON unresolved_edge_candidates(file_path); + +CREATE TABLE IF NOT EXISTS unresolved_index_states ( + namespace varchar(256) NOT NULL PRIMARY KEY, + updated_at timestamptz +); diff --git a/internal/db/migration/postgres/000010_unresolved_index_version.down.sql b/internal/db/migration/postgres/000010_unresolved_index_version.down.sql new file mode 100644 index 0000000..66c60e9 --- /dev/null +++ b/internal/db/migration/postgres/000010_unresolved_index_version.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE unresolved_index_states +DROP COLUMN version; diff --git a/internal/db/migration/postgres/000010_unresolved_index_version.up.sql b/internal/db/migration/postgres/000010_unresolved_index_version.up.sql new file mode 100644 index 0000000..4af652a --- /dev/null +++ b/internal/db/migration/postgres/000010_unresolved_index_version.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE unresolved_index_states +ADD COLUMN version varchar(64) NOT NULL DEFAULT ''; diff --git a/internal/db/migration/sqlite/000008_parse_cache.down.sql b/internal/db/migration/sqlite/000008_parse_cache.down.sql new file mode 100644 index 0000000..200e3f0 --- /dev/null +++ b/internal/db/migration/sqlite/000008_parse_cache.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS parse_cache_entries; diff --git a/internal/db/migration/sqlite/000008_parse_cache.up.sql b/internal/db/migration/sqlite/000008_parse_cache.up.sql new file mode 100644 index 0000000..b2baa56 --- /dev/null +++ b/internal/db/migration/sqlite/000008_parse_cache.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE IF NOT EXISTS parse_cache_entries ( + id integer PRIMARY KEY AUTOINCREMENT, + namespace text NOT NULL DEFAULT 'default', + file_path text NOT NULL, + source_hash text NOT NULL, + parser_version text NOT NULL, + context_hash text NOT NULL, + payload blob NOT NULL, + created_at datetime, + updated_at datetime +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_parse_cache_ns_file +ON parse_cache_entries(namespace, file_path); + +CREATE INDEX IF NOT EXISTS idx_parse_cache_source_hash +ON parse_cache_entries(source_hash); diff --git a/internal/db/migration/sqlite/000009_unresolved_edges.down.sql b/internal/db/migration/sqlite/000009_unresolved_edges.down.sql new file mode 100644 index 0000000..1d53621 --- /dev/null +++ b/internal/db/migration/sqlite/000009_unresolved_edges.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS unresolved_index_states; +DROP TABLE IF EXISTS unresolved_edge_candidates; diff --git a/internal/db/migration/sqlite/000009_unresolved_edges.up.sql b/internal/db/migration/sqlite/000009_unresolved_edges.up.sql new file mode 100644 index 0000000..5ee5692 --- /dev/null +++ b/internal/db/migration/sqlite/000009_unresolved_edges.up.sql @@ -0,0 +1,22 @@ +CREATE TABLE IF NOT EXISTS unresolved_edge_candidates ( + id integer PRIMARY KEY AUTOINCREMENT, + namespace text NOT NULL DEFAULT 'default', + lookup_key text NOT NULL, + fingerprint text NOT NULL, + file_path text NOT NULL, + kind text NOT NULL, + line integer, + created_at datetime +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_unresolved_ns_fp +ON unresolved_edge_candidates(namespace, fingerprint); +CREATE INDEX IF NOT EXISTS idx_unresolved_edge_candidates_lookup_key +ON unresolved_edge_candidates(lookup_key); +CREATE INDEX IF NOT EXISTS idx_unresolved_edge_candidates_file_path +ON unresolved_edge_candidates(file_path); + +CREATE TABLE IF NOT EXISTS unresolved_index_states ( + namespace text NOT NULL PRIMARY KEY, + updated_at datetime +); diff --git a/internal/db/migration/sqlite/000010_unresolved_index_version.down.sql b/internal/db/migration/sqlite/000010_unresolved_index_version.down.sql new file mode 100644 index 0000000..66c60e9 --- /dev/null +++ b/internal/db/migration/sqlite/000010_unresolved_index_version.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE unresolved_index_states +DROP COLUMN version; diff --git a/internal/db/migration/sqlite/000010_unresolved_index_version.up.sql b/internal/db/migration/sqlite/000010_unresolved_index_version.up.sql new file mode 100644 index 0000000..c090d3e --- /dev/null +++ b/internal/db/migration/sqlite/000010_unresolved_index_version.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE unresolved_index_states +ADD COLUMN version text NOT NULL DEFAULT ''; diff --git a/internal/domain/graph/parse_cache.go b/internal/domain/graph/parse_cache.go new file mode 100644 index 0000000..b3801c9 --- /dev/null +++ b/internal/domain/graph/parse_cache.go @@ -0,0 +1,18 @@ +// @index Persisted parser-result cache entries scoped by namespace and source path. +package graph + +import "time" + +// ParseCacheEntry stores the latest serialized parse result for one namespace/file path. +// @intent bound cache growth per active source path while validating the complete semantic cache identity. +type ParseCacheEntry struct { + ID uint `gorm:"primaryKey"` + Namespace string `gorm:"size:256;not null;default:'default';uniqueIndex:idx_parse_cache_ns_file"` + FilePath string `gorm:"size:768;not null;uniqueIndex:idx_parse_cache_ns_file"` + SourceHash string `gorm:"size:64;not null;index"` + ParserVersion string `gorm:"size:160;not null"` + ContextHash string `gorm:"size:64;not null"` + Payload []byte `gorm:"not null"` + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/internal/domain/graph/unresolved.go b/internal/domain/graph/unresolved.go new file mode 100644 index 0000000..b398826 --- /dev/null +++ b/internal/domain/graph/unresolved.go @@ -0,0 +1,31 @@ +// @index Durable unresolved-edge reverse index for semi-naive incremental resolution. +package graph + +import "time" + +// UnresolvedEdgeCandidate stores one lookup key for a syntax edge that lacks a graph endpoint. +// @intent let newly added symbols select affected unchanged callers without reparsing the whole graph. +type UnresolvedEdgeCandidate struct { + ID uint `gorm:"primaryKey"` + Namespace string `gorm:"size:256;not null;default:'default';uniqueIndex:idx_unresolved_ns_fp"` + LookupKey string `gorm:"size:512;not null;index"` + Fingerprint string `gorm:"type:text;not null;uniqueIndex:idx_unresolved_ns_fp"` + FilePath string `gorm:"size:1024;not null;index"` + Kind EdgeKind `gorm:"size:32;not null"` + Line int + CreatedAt time.Time +} + +// Edge converts the durable candidate back into resolver input. +// @intent keep unresolved storage separate from traversable graph edges while reusing the resolver contract. +func (c UnresolvedEdgeCandidate) Edge() Edge { + return Edge{Kind: c.Kind, FilePath: c.FilePath, Line: c.Line, Fingerprint: c.Fingerprint} +} + +// UnresolvedIndexState marks a namespace whose last full build populated the unresolved reverse index. +// @intent prevent upgraded databases with an empty, uninitialized index from taking an unsafe incremental shortcut. +type UnresolvedIndexState struct { + Namespace string `gorm:"primaryKey;size:256"` + Version string `gorm:"size:64;not null;default:''"` + UpdatedAt time.Time +} diff --git a/internal/domain/reference/similarity.go b/internal/domain/reference/similarity.go index f7e1ca6..7b8570f 100644 --- a/internal/domain/reference/similarity.go +++ b/internal/domain/reference/similarity.go @@ -11,14 +11,18 @@ func CommonSuffixDepth(a, b string) int { if a == "" || b == "" { return 0 } - aParts := strings.Split(a, "/") - bParts := strings.Split(b, "/") depth := 0 - for i, j := len(aParts)-1, len(bParts)-1; i >= 0 && j >= 0; i, j = i-1, j-1 { - if aParts[i] != bParts[j] { - break + for { + aSlash := strings.LastIndexByte(a, '/') + bSlash := strings.LastIndexByte(b, '/') + if a[aSlash+1:] != b[bSlash+1:] { + return depth } depth++ + if aSlash < 0 || bSlash < 0 { + return depth + } + a = a[:aSlash] + b = b[:bSlash] } - return depth } diff --git a/internal/domain/reference/similarity_test.go b/internal/domain/reference/similarity_test.go index 4aee4d3..1723ea7 100644 --- a/internal/domain/reference/similarity_test.go +++ b/internal/domain/reference/similarity_test.go @@ -12,6 +12,7 @@ func TestCommonSuffixDepth(t *testing.T) { {name: "exact path", a: "github.com/acme/api", b: "github.com/acme/api", want: 3}, {name: "shared trailing segments", a: "github.com/acme/api", b: "example.com/team/api", want: 1}, {name: "leading and trailing slashes", a: "/acme/api/", b: "/team/api/", want: 1}, + {name: "repeated separators", a: "acme//api", b: "team//api", want: 2}, {name: "no shared suffix", a: "acme/api", b: "team/web", want: 0}, {name: "empty input", a: "", b: "team/api", want: 0}, } @@ -24,3 +25,16 @@ func TestCommonSuffixDepth(t *testing.T) { }) } } + +func TestCommonSuffixDepthDoesNotAllocate(t *testing.T) { + var got int + allocs := testing.AllocsPerRun(1_000, func() { + got = CommonSuffixDepth("github.com/acme/api/internal/handler", "example.com/team/api/internal/handler") + }) + if got != 3 { + t.Fatalf("CommonSuffixDepth() = %d, want 3", got) + } + if allocs != 0 { + t.Fatalf("CommonSuffixDepth() allocations = %.0f, want 0", allocs) + } +} diff --git a/internal/runtime/remote/http.go b/internal/runtime/remote/http.go index 54452e4..ad9b42d 100644 --- a/internal/runtime/remote/http.go +++ b/internal/runtime/remote/http.go @@ -99,7 +99,7 @@ func buildRepoSyncHTTP(rt *ccgruntime.Runtime, cfg httpin.Config, inst *mcprunti for ext, walker := range rt.Walkers { walkers[ext] = walker } - graphSvc := &workflow.Service{Store: rt.Store, UnitOfWork: rt.UnitOfWork, Search: rt.Search, Walkers: walkers, Logger: rt.Logger} + graphSvc := &workflow.Service{Store: rt.Store, UnitOfWork: rt.UnitOfWork, Search: rt.Search, ParseCache: rt.Store, Walkers: walkers, Logger: rt.Logger} syncService := &reposync.Service{ Checkout: gitrepo.NewCheckout(cfg.RepoRoot, repoLocker, nil), IncludePaths: configfiles.IncludePaths{}, Graph: reposyncgraph.Updater{Service: graphSvc, Syncer: rt.Syncer}, From 6c4c3cd9fae08430f8288fafca74ebd2bdc668c0 Mon Sep 17 00:00:00 2001 From: tae2089 Date: Thu, 16 Jul 2026 16:58:01 +0900 Subject: [PATCH 6/7] perf: batch PostgreSQL graph persistence --- cmd/ccg/main_test.go | 14 +- internal/adapters/inbound/cli/build_test.go | 2 +- internal/adapters/inbound/cli/update_test.go | 4 +- .../graphgorm/annotation_bulk_test.go | 161 +++++++++ .../graphgorm/deletegraph_query_test.go | 105 ++++++ .../outbound/graphgorm/deletion_bulk_test.go | 70 ++++ internal/adapters/outbound/graphgorm/store.go | 315 +++++++++--------- .../outbound/graphgorm/store_postgres_test.go | 31 ++ .../adapters/outbound/graphgorm/store_test.go | 2 +- .../adapters/outbound/graphgorm/unresolved.go | 27 +- .../outbound/graphgorm/unresolved_test.go | 86 +++++ .../app/ingest/incremental/incremental.go | 142 ++++---- .../ingest/incremental/incremental_test.go | 117 ++++++- internal/app/ingest/ports.go | 4 +- internal/app/ingest/ports_test.go | 6 +- internal/app/ingest/workflow/build.go | 24 +- internal/app/ingest/workflow/indexer_test.go | 23 +- internal/db/migration/migration.go | 30 +- internal/db/migration/migration_test.go | 111 +++++- .../000011_unresolved_index_hashes.down.sql | 17 + .../000011_unresolved_index_hashes.up.sql | 17 + .../postgres/000012_annotation_text.down.sql | 5 + .../postgres/000012_annotation_text.up.sql | 3 + ...13_node_namespace_file_path_index.down.sql | 1 + ...0013_node_namespace_file_path_index.up.sql | 2 + .../000011_unresolved_index_hashes.down.sql | 15 + .../000011_unresolved_index_hashes.up.sql | 15 + .../sqlite/000012_annotation_text.down.sql | 2 + .../sqlite/000012_annotation_text.up.sql | 2 + ...13_node_namespace_file_path_index.down.sql | 1 + ...0013_node_namespace_file_path_index.up.sql | 2 + internal/domain/graph/annotation.go | 4 +- internal/domain/graph/node.go | 4 +- internal/domain/graph/unresolved.go | 18 +- 34 files changed, 1095 insertions(+), 287 deletions(-) create mode 100644 internal/adapters/outbound/graphgorm/annotation_bulk_test.go create mode 100644 internal/adapters/outbound/graphgorm/deletegraph_query_test.go create mode 100644 internal/adapters/outbound/graphgorm/deletion_bulk_test.go create mode 100644 internal/adapters/outbound/graphgorm/unresolved_test.go create mode 100644 internal/db/migration/postgres/000011_unresolved_index_hashes.down.sql create mode 100644 internal/db/migration/postgres/000011_unresolved_index_hashes.up.sql create mode 100644 internal/db/migration/postgres/000012_annotation_text.down.sql create mode 100644 internal/db/migration/postgres/000012_annotation_text.up.sql create mode 100644 internal/db/migration/postgres/000013_node_namespace_file_path_index.down.sql create mode 100644 internal/db/migration/postgres/000013_node_namespace_file_path_index.up.sql create mode 100644 internal/db/migration/sqlite/000011_unresolved_index_hashes.down.sql create mode 100644 internal/db/migration/sqlite/000011_unresolved_index_hashes.up.sql create mode 100644 internal/db/migration/sqlite/000012_annotation_text.down.sql create mode 100644 internal/db/migration/sqlite/000012_annotation_text.up.sql create mode 100644 internal/db/migration/sqlite/000013_node_namespace_file_path_index.down.sql create mode 100644 internal/db/migration/sqlite/000013_node_namespace_file_path_index.up.sql diff --git a/cmd/ccg/main_test.go b/cmd/ccg/main_test.go index 180ac9f..387b4d6 100644 --- a/cmd/ccg/main_test.go +++ b/cmd/ccg/main_test.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -250,8 +251,8 @@ func TestRunMigrations_SQLiteDownRestoresNullableColumns(t *testing.T) { if err != nil { t.Fatalf("create migrator: %v", err) } - if err := migrator.Steps(-6); err != nil { - t.Fatalf("run down migration: %v", err) + if err := migrator.Migrate(1); err != nil { + t.Fatalf("migrate down to version 1: %v", err) } var version migration.MigrationSchemaVersion @@ -295,8 +296,8 @@ func TestRunMigrations_SQLiteDownFromVersionThreeDropsPolicyTables(t *testing.T) if err != nil { t.Fatalf("create migrator: %v", err) } - if err := migrator.Steps(-5); err != nil { - t.Fatalf("run down migration: %v", err) + if err := migrator.Migrate(2); err != nil { + t.Fatalf("migrate down to version 2: %v", err) } var version migration.MigrationSchemaVersion @@ -409,7 +410,8 @@ func TestEnsureSchemaVersion_LogsRuntimeSchemaPassAndFail(t *testing.T) { t.Fatalf("ensure schema version: %v", err) } passLog := logs.String() - for _, want := range []string{"database runtime schema check passed", "driver=sqlite", "required_version=7", "auto_migrated=true"} { + requiredVersionLog := "required_version=" + strconv.Itoa(migration.RequiredSchemaVersion) + for _, want := range []string{"database runtime schema check passed", "driver=sqlite", requiredVersionLog, "auto_migrated=true"} { if !strings.Contains(passLog, want) { t.Fatalf("expected runtime schema pass log to contain %q, got %q", want, passLog) } @@ -423,7 +425,7 @@ func TestEnsureSchemaVersion_LogsRuntimeSchemaPassAndFail(t *testing.T) { t.Fatal("expected parity failure") } failLog := logs.String() - for _, want := range []string{"database runtime schema check failed", "driver=sqlite", "required_version=7", "auto_migrated=false", "error.message="} { + for _, want := range []string{"database runtime schema check failed", "driver=sqlite", requiredVersionLog, "auto_migrated=false", "error.message="} { if !strings.Contains(failLog, want) { t.Fatalf("expected runtime schema failure log to contain %q, got %q", want, failLog) } diff --git a/internal/adapters/inbound/cli/build_test.go b/internal/adapters/inbound/cli/build_test.go index c610d4c..aa42d74 100644 --- a/internal/adapters/inbound/cli/build_test.go +++ b/internal/adapters/inbound/cli/build_test.go @@ -22,7 +22,7 @@ func setupBuildTest(t *testing.T) (*Deps, *bytes.Buffer, *bytes.Buffer, *gorm.DB t.Helper() deps, stdout, stderr := newTestDeps() - db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "ccg.db")), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { t.Fatal(err) } diff --git a/internal/adapters/inbound/cli/update_test.go b/internal/adapters/inbound/cli/update_test.go index 829d147..040e8a6 100644 --- a/internal/adapters/inbound/cli/update_test.go +++ b/internal/adapters/inbound/cli/update_test.go @@ -26,7 +26,7 @@ func setupUpdateTest(t *testing.T) (*Deps, *bytes.Buffer, *bytes.Buffer, *gorm.D t.Helper() deps, stdout, stderr := newTestDeps() - db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "ccg.db")), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { t.Fatal(err) } @@ -61,7 +61,7 @@ func setupUpdateGraphOnlyTest(t *testing.T) (*Deps, *bytes.Buffer, *bytes.Buffer t.Helper() deps, stdout, stderr := newTestDeps() - db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: gormlogger.Discard}) + db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "ccg.db")), &gorm.Config{Logger: gormlogger.Discard}) if err != nil { t.Fatal(err) } diff --git a/internal/adapters/outbound/graphgorm/annotation_bulk_test.go b/internal/adapters/outbound/graphgorm/annotation_bulk_test.go new file mode 100644 index 0000000..c2da694 --- /dev/null +++ b/internal/adapters/outbound/graphgorm/annotation_bulk_test.go @@ -0,0 +1,161 @@ +package graphgorm + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + requestctx "github.com/tae2089/code-context-graph/internal/ctx" + "github.com/tae2089/code-context-graph/internal/domain/graph" +) + +type annotationSQLCounter struct { + statements atomic.Int64 +} + +func (c *annotationSQLCounter) LogMode(logger.LogLevel) logger.Interface { return c } +func (c *annotationSQLCounter) Info(context.Context, string, ...any) {} +func (c *annotationSQLCounter) Warn(context.Context, string, ...any) {} +func (c *annotationSQLCounter) Error(context.Context, string, ...any) {} +func (c *annotationSQLCounter) Trace(_ context.Context, _ time.Time, fc func() (string, int64), _ error) { + c.statements.Add(1) + _, _ = fc() +} + +func (c *annotationSQLCounter) reset() { + c.statements.Store(0) +} + +func setupAnnotationSQLMeasurement(t *testing.T, count int) (*Store, context.Context, []*graph.Annotation, *annotationSQLCounter) { + t.Helper() + counter := &annotationSQLCounter{} + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: counter}) + if err != nil { + t.Fatalf("open SQLite: %v", err) + } + store := New(db) + if err := store.AutoMigrate(); err != nil { + t.Fatalf("auto migrate: %v", err) + } + ctx := requestctx.WithNamespace(context.Background(), "annotation-sql-measurement") + nodes := make([]graph.Node, count) + for i := range nodes { + nodes[i] = graph.Node{ + QualifiedName: fmt.Sprintf("sample.F%d", i), + Kind: graph.NodeKindFunction, + Name: fmt.Sprintf("F%d", i), + FilePath: "sample.go", + StartLine: i + 1, + EndLine: i + 1, + Language: "go", + } + } + if err := store.UpsertNodes(ctx, nodes); err != nil { + t.Fatalf("upsert nodes: %v", err) + } + stored, err := store.GetNodesByFile(ctx, "sample.go") + if err != nil { + t.Fatalf("load nodes: %v", err) + } + annotations := make([]*graph.Annotation, len(stored)) + for i := range stored { + annotations[i] = &graph.Annotation{ + NodeID: stored[i].ID, + Summary: fmt.Sprintf("summary %d", i), + Tags: []graph.DocTag{{ + Kind: graph.TagIntent, + Value: fmt.Sprintf("intent %d", i), + Ordinal: 0, + }}, + } + } + counter.reset() + return store, ctx, annotations, counter +} + +func TestUpsertAnnotations_BatchesSQLAndPreservesTags(t *testing.T) { + store, ctx, annotations, counter := setupAnnotationSQLMeasurement(t, 20) + started := time.Now() + if err := store.UpsertAnnotations(ctx, annotations); err != nil { + t.Fatalf("upsert annotations: %v", err) + } + statements := counter.statements.Load() + t.Logf("bulk annotation candidate: annotations=%d statements=%d elapsed=%s", len(annotations), statements, time.Since(started)) + if statements > 5 { + t.Fatalf("bulk annotation statements = %d, want <= 5", statements) + } + + for i, annotation := range annotations { + got, err := store.GetAnnotation(ctx, annotation.NodeID) + if err != nil { + t.Fatalf("get annotation %d: %v", i, err) + } + if got == nil || got.Summary != annotation.Summary { + t.Fatalf("annotation %d = %+v, want summary %q", i, got, annotation.Summary) + } + if len(got.Tags) != 1 || got.Tags[0].Value != annotation.Tags[0].Value { + t.Fatalf("annotation %d tags = %+v, want %+v", i, got.Tags, annotation.Tags) + } + } + + for i, annotation := range annotations { + annotation.Summary = fmt.Sprintf("updated summary %d", i) + annotation.Tags = []graph.DocTag{{Kind: graph.TagDomainRule, Value: fmt.Sprintf("rule %d", i), Ordinal: 0}} + } + counter.reset() + if err := store.UpsertAnnotations(ctx, annotations); err != nil { + t.Fatalf("update annotations: %v", err) + } + if statements := counter.statements.Load(); statements > 5 { + t.Fatalf("bulk annotation update statements = %d, want <= 5", statements) + } + for i, annotation := range annotations { + got, err := store.GetAnnotation(ctx, annotation.NodeID) + if err != nil { + t.Fatalf("get updated annotation %d: %v", i, err) + } + if got == nil || got.Summary != annotation.Summary || len(got.Tags) != 1 || got.Tags[0].Value != annotation.Tags[0].Value { + t.Fatalf("updated annotation %d = %+v, want %+v", i, got, annotation) + } + } +} + +func TestUpsertAnnotations_RejectsForeignNodeBeforeWrites(t *testing.T) { + store, ctx, annotations, _ := setupAnnotationSQLMeasurement(t, 1) + foreignCtx := requestctx.WithNamespace(context.Background(), "foreign") + foreignNodes := []graph.Node{{ + QualifiedName: "foreign.F", + Kind: graph.NodeKindFunction, + Name: "F", + FilePath: "foreign.go", + StartLine: 1, + EndLine: 1, + Language: "go", + }} + if err := store.UpsertNodes(foreignCtx, foreignNodes); err != nil { + t.Fatalf("upsert foreign node: %v", err) + } + storedForeign, err := store.GetNodesByFile(foreignCtx, "foreign.go") + if err != nil || len(storedForeign) != 1 { + t.Fatalf("load foreign node: nodes=%+v err=%v", storedForeign, err) + } + foreignAnnotation := &graph.Annotation{NodeID: storedForeign[0].ID, Summary: "foreign"} + + err = store.UpsertAnnotations(ctx, []*graph.Annotation{annotations[0], foreignAnnotation}) + if err == nil { + t.Fatal("expected foreign-node batch to be rejected") + } + got, getErr := store.GetAnnotation(ctx, annotations[0].NodeID) + if getErr != nil { + t.Fatalf("get owned annotation after rejection: %v", getErr) + } + if got != nil { + t.Fatalf("owned annotation was written before batch rejection: %+v", got) + } +} diff --git a/internal/adapters/outbound/graphgorm/deletegraph_query_test.go b/internal/adapters/outbound/graphgorm/deletegraph_query_test.go new file mode 100644 index 0000000..53145cf --- /dev/null +++ b/internal/adapters/outbound/graphgorm/deletegraph_query_test.go @@ -0,0 +1,105 @@ +package graphgorm + +import ( + "context" + "strings" + "sync" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/logger" + + requestctx "github.com/tae2089/code-context-graph/internal/ctx" + "github.com/tae2089/code-context-graph/internal/domain/graph" +) + +type deleteGraphSQLRecorder struct { + mu sync.Mutex + statements []string +} + +func (r *deleteGraphSQLRecorder) LogMode(logger.LogLevel) logger.Interface { return r } +func (r *deleteGraphSQLRecorder) Info(context.Context, string, ...any) {} +func (r *deleteGraphSQLRecorder) Warn(context.Context, string, ...any) {} +func (r *deleteGraphSQLRecorder) Error(context.Context, string, ...any) {} +func (r *deleteGraphSQLRecorder) Trace(_ context.Context, _ time.Time, fc func() (string, int64), _ error) { + sql, _ := fc() + r.mu.Lock() + r.statements = append(r.statements, sql) + r.mu.Unlock() +} + +func (r *deleteGraphSQLRecorder) reset() { + r.mu.Lock() + r.statements = nil + r.mu.Unlock() +} + +func (r *deleteGraphSQLRecorder) searchDocumentDelete() string { + r.mu.Lock() + defer r.mu.Unlock() + for _, statement := range r.statements { + if strings.Contains(statement, "DELETE FROM `search_documents`") { + return statement + } + } + return "" +} + +func TestDeleteGraph_DeletesSearchDocumentsByNamespace(t *testing.T) { + recorder := &deleteGraphSQLRecorder{} + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: recorder}) + if err != nil { + t.Fatalf("open SQLite: %v", err) + } + store := New(db) + if err := store.AutoMigrate(); err != nil { + t.Fatalf("auto migrate: %v", err) + } + if err := db.AutoMigrate(&graph.SearchDocument{}); err != nil { + t.Fatalf("migrate search documents: %v", err) + } + + ctx := requestctx.WithNamespace(context.Background(), "delete-me") + if err := store.UpsertNodes(ctx, []graph.Node{{ + QualifiedName: "pkg.DeleteMe", + Kind: graph.NodeKindFunction, + Name: "DeleteMe", + FilePath: "delete.go", + StartLine: 1, + EndLine: 1, + Language: "go", + }}); err != nil { + t.Fatalf("upsert node: %v", err) + } + node, err := store.GetNode(ctx, "pkg.DeleteMe") + if err != nil { + t.Fatalf("get node: %v", err) + } + if err := db.Create(&graph.SearchDocument{ + Namespace: "delete-me", + NodeID: node.ID, + Content: "delete me", + Language: "go", + }).Error; err != nil { + t.Fatalf("create search document: %v", err) + } + + recorder.reset() + if err := store.DeleteGraph(ctx); err != nil { + t.Fatalf("DeleteGraph: %v", err) + } + + statement := recorder.searchDocumentDelete() + if statement == "" { + t.Fatal("search_documents DELETE statement not recorded") + } + if !strings.Contains(statement, "WHERE namespace = \"delete-me\"") { + t.Fatalf("search_documents DELETE = %q, want direct namespace predicate", statement) + } + if strings.Contains(statement, "SELECT `id` FROM `nodes`") { + t.Fatalf("search_documents DELETE = %q, must not select node IDs", statement) + } +} diff --git a/internal/adapters/outbound/graphgorm/deletion_bulk_test.go b/internal/adapters/outbound/graphgorm/deletion_bulk_test.go new file mode 100644 index 0000000..37f3c44 --- /dev/null +++ b/internal/adapters/outbound/graphgorm/deletion_bulk_test.go @@ -0,0 +1,70 @@ +package graphgorm + +import ( + "context" + "fmt" + "testing" + "time" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" + + requestctx "github.com/tae2089/code-context-graph/internal/ctx" + "github.com/tae2089/code-context-graph/internal/domain/graph" +) + +func setupDeletionSQLMeasurement(t *testing.T, fileCount int) (*Store, context.Context, []string, *annotationSQLCounter) { + t.Helper() + counter := &annotationSQLCounter{} + db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{Logger: counter}) + if err != nil { + t.Fatalf("open SQLite: %v", err) + } + store := New(db) + if err := store.AutoMigrate(); err != nil { + t.Fatalf("auto migrate graph store: %v", err) + } + if err := db.AutoMigrate(&graph.SearchDocument{}); err != nil { + t.Fatalf("auto migrate search documents: %v", err) + } + ctx := requestctx.WithNamespace(context.Background(), "bulk-delete") + filePaths := make([]string, fileCount) + nodes := make([]graph.Node, fileCount) + for i := range fileCount { + filePaths[i] = fmt.Sprintf("src/file-%03d.go", i) + nodes[i] = graph.Node{ + QualifiedName: fmt.Sprintf("sample.F%d", i), + Kind: graph.NodeKindFunction, + Name: fmt.Sprintf("F%d", i), + FilePath: filePaths[i], + StartLine: 1, + EndLine: 1, + Language: "go", + } + } + if err := store.UpsertNodes(ctx, nodes); err != nil { + t.Fatalf("upsert nodes: %v", err) + } + counter.reset() + return store, ctx, filePaths, counter +} + +func TestDeleteNodesByFiles_UsesBoundedSQL(t *testing.T) { + store, ctx, filePaths, counter := setupDeletionSQLMeasurement(t, 5) + started := time.Now() + if err := store.DeleteNodesByFiles(ctx, filePaths); err != nil { + t.Fatalf("delete nodes by files: %v", err) + } + statements := counter.statements.Load() + t.Logf("bulk file deletion: files=%d statements=%d elapsed=%s", len(filePaths), statements, time.Since(started)) + if statements > 12 { + t.Fatalf("bulk file deletion statements = %d, want <= 12", statements) + } + var remaining int64 + if err := store.db.Model(&graph.Node{}).Where("namespace = ?", requestctx.FromContext(ctx)).Count(&remaining).Error; err != nil { + t.Fatalf("count remaining nodes: %v", err) + } + if remaining != 0 { + t.Fatalf("remaining nodes = %d, want 0", remaining) + } +} diff --git a/internal/adapters/outbound/graphgorm/store.go b/internal/adapters/outbound/graphgorm/store.go index ad68163..abc5ecd 100644 --- a/internal/adapters/outbound/graphgorm/store.go +++ b/internal/adapters/outbound/graphgorm/store.go @@ -292,98 +292,87 @@ func (s *Store) GetFileNodesByPathSuffix(ctx context.Context, suffix string) ([] } // DeleteNodesByFile removes nodes in a file and their related data. -// @intent clean out prior nodes, edges, and annotations before reparsing a file. +// @intent keep the single-file API compatible while delegating cleanup to the bounded batch path. // @sideEffect deletes related records from the nodes, edges, annotations, and doc_tags tables. // @domainRule connected edges and annotations must also be removed when deleting a file. func (s *Store) DeleteNodesByFile(ctx context.Context, filePath string) error { - ns := requestctx.FromContext(ctx) - if err := s.db.WithContext(ctx).Where("namespace = ? AND file_path = ?", ns, filePath).Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil { - return trace.Wrap(err, "delete file unresolved edges") - } - var nodeIDs []uint - if err := s.db.WithContext(ctx). - Model(&graph.Node{}). - Where("namespace = ? AND file_path = ?", ns, filePath). - Pluck("id", &nodeIDs).Error; err != nil { - return trace.Wrap(err, "pluck node ids") - } - if len(nodeIDs) == 0 { + return s.DeleteNodesByFiles(ctx, []string{filePath}) +} + +// DeleteNodesByFiles removes nodes and dependent graph rows for bounded file-path batches. +// @intent replace per-file deletion round trips with one transaction and a fixed deletion sequence per path chunk. +// @sideEffect deletes unresolved candidates, edges, annotations/tags, memberships, search documents, and nodes. +// @domainRule only rows owned by the request namespace and rows connected to its deleted nodes may be removed. +func (s *Store) DeleteNodesByFiles(ctx context.Context, filePaths []string) error { + if len(filePaths) == 0 { return nil } + ns := requestctx.FromContext(ctx) + hasFlowMemberships := s.db.Migrator().HasTable(&graph.FlowMembership{}) + hasSearchDocuments := s.db.Migrator().HasTable(&graph.SearchDocument{}) return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - if err := tx. - Where("namespace = ? AND file_path = ?", ns, filePath). - Delete(&graph.Edge{}).Error; err != nil { - return trace.Wrap(err, "delete file-owned edges") - } - - if err := tx. - Where("from_node_id IN ? OR to_node_id IN ?", nodeIDs, nodeIDs). - Delete(&graph.Edge{}).Error; err != nil { - return trace.Wrap(err, "cascade delete edges") - } - - if err := tx. - Where("annotation_id IN (?)", - tx.Model(&graph.Annotation{}).Select("id").Where("node_id IN ?", nodeIDs), - ).Delete(&graph.DocTag{}).Error; err != nil { - return trace.Wrap(err, "cascade delete doc_tags") - } - - if err := tx. - Where("node_id IN ?", nodeIDs). - Delete(&graph.Annotation{}).Error; err != nil { - return trace.Wrap(err, "cascade delete annotations") - } - - if err := tx. - Where("node_id IN ?", nodeIDs). - Delete(&graph.CommunityMembership{}).Error; err != nil { - return trace.Wrap(err, "cascade delete community memberships") - } - - if tx.Migrator().HasTable(&graph.FlowMembership{}) { - if err := tx. - Where("node_id IN ?", nodeIDs). - Delete(&graph.FlowMembership{}).Error; err != nil { - return trace.Wrap(err, "cascade delete flow memberships") + for start := 0; start < len(filePaths); start += 500 { + end := min(start+500, len(filePaths)) + chunk := filePaths[start:end] + if err := tx.Where("namespace = ? AND file_path IN ?", ns, chunk).Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil { + return trace.Wrap(err, "delete file unresolved edges") } - } - - if tx.Migrator().HasTable(&graph.SearchDocument{}) { - if err := tx. - Where("node_id IN ?", nodeIDs). - Delete(&graph.SearchDocument{}).Error; err != nil { - return trace.Wrap(err, "cascade delete search_documents") + if err := tx.Where("namespace = ? AND file_path IN ?", ns, chunk).Delete(&graph.Edge{}).Error; err != nil { + return trace.Wrap(err, "delete file-owned edges") + } + if err := deleteNodeScope(tx, "namespace = ? AND file_path IN ?", []any{ns, chunk}, hasFlowMemberships, hasSearchDocuments); err != nil { + return err } } - - return tx.Where("id IN ?", nodeIDs).Delete(&graph.Node{}).Error + return nil }) } +// deleteNodeScope removes every dependent row for nodes selected by a reusable WHERE scope. +// @intent centralize node-dependent cleanup while keeping node IDs inside database subqueries. +// @sideEffect deletes connected edges, documentation, memberships, search documents, and nodes. +func deleteNodeScope(tx *gorm.DB, nodeWhere string, nodeArgs []any, hasFlowMemberships, hasSearchDocuments bool) error { + nodeIDs := func() *gorm.DB { + return tx.Model(&graph.Node{}).Select("id").Where(nodeWhere, nodeArgs...) + } + if err := tx.Where("from_node_id IN (?) OR to_node_id IN (?)", nodeIDs(), nodeIDs()).Delete(&graph.Edge{}).Error; err != nil { + return trace.Wrap(err, "cascade delete edges") + } + annotationIDs := tx.Model(&graph.Annotation{}).Select("id").Where("node_id IN (?)", nodeIDs()) + if err := tx.Where("annotation_id IN (?)", annotationIDs).Delete(&graph.DocTag{}).Error; err != nil { + return trace.Wrap(err, "cascade delete doc_tags") + } + if err := tx.Where("node_id IN (?)", nodeIDs()).Delete(&graph.Annotation{}).Error; err != nil { + return trace.Wrap(err, "cascade delete annotations") + } + if err := tx.Where("node_id IN (?)", nodeIDs()).Delete(&graph.CommunityMembership{}).Error; err != nil { + return trace.Wrap(err, "cascade delete community memberships") + } + if hasFlowMemberships { + if err := tx.Where("node_id IN (?)", nodeIDs()).Delete(&graph.FlowMembership{}).Error; err != nil { + return trace.Wrap(err, "cascade delete flow memberships") + } + } + if hasSearchDocuments { + if err := tx.Where("node_id IN (?)", nodeIDs()).Delete(&graph.SearchDocument{}).Error; err != nil { + return trace.Wrap(err, "cascade delete search_documents") + } + } + if err := tx.Where(nodeWhere, nodeArgs...).Delete(&graph.Node{}).Error; err != nil { + return trace.Wrap(err, "cascade delete nodes") + } + return nil +} + // DeleteGraph removes the entire graph state for the current namespace. // @intent replace namespace-scoped state before a full rebuild or include_paths rebuild. // @sideEffect deletes all nodes, edges, annotations, and doc_tags in the namespace. +// @domainRule namespace-owned search documents are deleted directly before node-scoped dependent cleanup. func (s *Store) DeleteGraph(ctx context.Context) error { ns := requestctx.FromContext(ctx) - var nodeIDs []uint - var filePaths []string - if err := s.db.WithContext(ctx). - Model(&graph.Node{}). - Where("namespace = ?", ns). - Pluck("id", &nodeIDs).Error; err != nil { - return trace.Wrap(err, "pluck namespace node ids") - } - if err := s.db.WithContext(ctx). - Model(&graph.Node{}). - Where("namespace = ?", ns). - Distinct(). - Pluck("file_path", &filePaths).Error; err != nil { - return trace.Wrap(err, "pluck namespace file paths") - } - + hasFlowMemberships := s.db.Migrator().HasTable(&graph.FlowMembership{}) + hasSearchDocuments := s.db.Migrator().HasTable(&graph.SearchDocument{}) return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { if err := tx.Where("namespace = ?", ns).Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil { return trace.Wrap(err, "delete namespace unresolved edges") @@ -391,65 +380,21 @@ func (s *Store) DeleteGraph(ctx context.Context) error { if err := tx.Where("namespace = ?", ns).Delete(&graph.UnresolvedIndexState{}).Error; err != nil { return trace.Wrap(err, "delete namespace unresolved index state") } - if len(filePaths) > 0 { - if err := tx. - Where("namespace = ? AND file_path IN ?", ns, filePaths). - Delete(&graph.Edge{}).Error; err != nil { - return trace.Wrap(err, "delete namespace file-owned edges") - } + if err := tx.Where("namespace = ?", ns).Delete(&graph.Edge{}).Error; err != nil { + return trace.Wrap(err, "delete namespace edges") } - - if len(nodeIDs) > 0 { - if err := tx. - Where("annotation_id IN (?)", - tx.Model(&graph.Annotation{}).Select("id").Where("node_id IN ?", nodeIDs), - ).Delete(&graph.DocTag{}).Error; err != nil { - return trace.Wrap(err, "delete namespace doc_tags") - } - - if err := tx. - Where("node_id IN ?", nodeIDs). - Delete(&graph.Annotation{}).Error; err != nil { - return trace.Wrap(err, "delete namespace annotations") - } - - if err := tx. - Where("node_id IN ?", nodeIDs). - Delete(&graph.CommunityMembership{}).Error; err != nil { - return trace.Wrap(err, "delete namespace community memberships") - } - - if tx.Migrator().HasTable(&graph.FlowMembership{}) { - flowIDs := tx.Model(&graph.Flow{}).Select("id").Where("namespace = ?", ns) - if err := tx. - Where("node_id IN ? OR flow_id IN (?)", nodeIDs, flowIDs). - Delete(&graph.FlowMembership{}).Error; err != nil { - return trace.Wrap(err, "delete namespace flow memberships") - } - } - - if tx.Migrator().HasTable(&graph.SearchDocument{}) { - if err := tx. - Where("node_id IN ?", nodeIDs). - Delete(&graph.SearchDocument{}).Error; err != nil { - return trace.Wrap(err, "delete namespace search_documents") - } - } - - if err := tx. - Where("from_node_id IN ? OR to_node_id IN ?", nodeIDs, nodeIDs). - Delete(&graph.Edge{}).Error; err != nil { - return trace.Wrap(err, "delete namespace connected edges") + if hasFlowMemberships { + flowIDs := tx.Model(&graph.Flow{}).Select("id").Where("namespace = ?", ns) + if err := tx.Where("flow_id IN (?)", flowIDs).Delete(&graph.FlowMembership{}).Error; err != nil { + return trace.Wrap(err, "delete namespace flow memberships") } - - if err := tx. - Where("id IN ?", nodeIDs). - Delete(&graph.Node{}).Error; err != nil { - return trace.Wrap(err, "delete namespace nodes") + } + if hasSearchDocuments { + if err := tx.Where("namespace = ?", ns).Delete(&graph.SearchDocument{}).Error; err != nil { + return trace.Wrap(err, "delete namespace search documents") } } - - return nil + return deleteNodeScope(tx, "namespace = ?", []any{ns}, hasFlowMemberships, false) }) } @@ -578,44 +523,100 @@ func (s *Store) DeletePackageSemanticEdges(ctx context.Context, anchors []string } // UpsertAnnotation stores a node's annotation and tags. -// @intent replace structured comments for each node with the latest state. +// @intent keep the single-annotation API compatible while delegating persistence to the batch path. // @sideEffect performs inserts, updates, and deletes on the annotations and doc_tags tables. // @mutates may overwrite ann.ID with the existing record ID. // @domainRule only one annotation must be kept per node_id. func (s *Store) UpsertAnnotation(ctx context.Context, ann *graph.Annotation) error { - var existing graph.Annotation - ns := requestctx.FromContext(ctx) - result := s.db.WithContext(ctx). - Joins("JOIN nodes ON nodes.id = annotations.node_id"). - Where("annotations.node_id = ? AND nodes.namespace = ?", ann.NodeID, ns). - First(&existing) + return s.UpsertAnnotations(ctx, []*graph.Annotation{ann}) +} - if errors.Is(result.Error, gorm.ErrRecordNotFound) { - return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - // The read path is namespace-scoped; the create path must be too, or a caller - // could attach an annotation to another namespace's node (or a missing node). - var owned int64 - if err := tx.Model(&graph.Node{}). - Where("id = ? AND namespace = ?", ann.NodeID, ns). - Count(&owned).Error; err != nil { - return trace.Wrap(err, "verify annotation node namespace") - } - if owned == 0 { - return fmt.Errorf("annotation node %d not found in namespace %q", ann.NodeID, ns) - } - return tx.Create(ann).Error - }) +// UpsertAnnotations replaces annotations and tags for a batch of namespace-owned nodes. +// @intent collapse per-annotation lookup and write round trips into bounded batch operations. +// @sideEffect inserts or updates annotations, deletes prior tags, and inserts replacement tags in one transaction. +// @mutates assigns persisted annotation and annotation-tag foreign IDs to input values. +// @domainRule every node must belong to the request namespace before any annotation in the batch is mutated. +func (s *Store) UpsertAnnotations(ctx context.Context, annotations []*graph.Annotation) error { + if len(annotations) == 0 { + return nil } - if result.Error != nil { - return result.Error + + nodeIDs := make([]uint, len(annotations)) + seen := make(map[uint]struct{}, len(annotations)) + rows := make([]graph.Annotation, len(annotations)) + for i, annotation := range annotations { + if annotation == nil { + return fmt.Errorf("annotation at index %d is nil", i) + } + if _, duplicate := seen[annotation.NodeID]; duplicate { + return fmt.Errorf("duplicate annotation node id %d", annotation.NodeID) + } + seen[annotation.NodeID] = struct{}{} + nodeIDs[i] = annotation.NodeID + rows[i] = *annotation + rows[i].ID = 0 + rows[i].Tags = nil } + ns := requestctx.FromContext(ctx) return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - if err := tx.Where("annotation_id = ?", existing.ID).Delete(&graph.DocTag{}).Error; err != nil { - return trace.Wrap(err, "delete doc tags") + var ownedNodeIDs []uint + if err := tx.Model(&graph.Node{}). + Where("id IN ? AND namespace = ?", nodeIDs, ns). + Pluck("id", &ownedNodeIDs).Error; err != nil { + return trace.Wrap(err, "verify annotation node namespaces") + } + if len(ownedNodeIDs) != len(nodeIDs) { + return fmt.Errorf("annotation nodes are missing from namespace %q: owned %d of %d", ns, len(ownedNodeIDs), len(nodeIDs)) + } + + if err := tx.Omit("Tags"). + Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "node_id"}}, + DoUpdates: clause.AssignmentColumns([]string{"summary", "context", "raw_text", "updated_at"}), + }). + CreateInBatches(rows, 100).Error; err != nil { + return trace.Wrap(err, "batch upsert annotations") + } + + var stored []graph.Annotation + if err := tx.Select("id", "node_id").Where("node_id IN ?", nodeIDs).Find(&stored).Error; err != nil { + return trace.Wrap(err, "reload batch annotation ids") } - ann.ID = existing.ID - return tx.Save(ann).Error + if len(stored) != len(annotations) { + return fmt.Errorf("reloaded %d annotations, want %d", len(stored), len(annotations)) + } + + annotationIDByNode := make(map[uint]uint, len(stored)) + annotationIDs := make([]uint, len(stored)) + for i := range stored { + annotationIDByNode[stored[i].NodeID] = stored[i].ID + annotationIDs[i] = stored[i].ID + } + if err := tx.Where("annotation_id IN ?", annotationIDs).Delete(&graph.DocTag{}).Error; err != nil { + return trace.Wrap(err, "delete batch doc tags") + } + + tagCount := 0 + for _, annotation := range annotations { + tagCount += len(annotation.Tags) + } + tags := make([]graph.DocTag, 0, tagCount) + for _, annotation := range annotations { + annotationID := annotationIDByNode[annotation.NodeID] + annotation.ID = annotationID + for i := range annotation.Tags { + annotation.Tags[i].ID = 0 + annotation.Tags[i].AnnotationID = annotationID + tags = append(tags, annotation.Tags[i]) + } + } + if len(tags) > 0 { + if err := tx.CreateInBatches(tags, 500).Error; err != nil { + return trace.Wrap(err, "batch insert doc tags") + } + } + return nil }) } diff --git a/internal/adapters/outbound/graphgorm/store_postgres_test.go b/internal/adapters/outbound/graphgorm/store_postgres_test.go index 9c672bd..bc8101c 100644 --- a/internal/adapters/outbound/graphgorm/store_postgres_test.go +++ b/internal/adapters/outbound/graphgorm/store_postgres_test.go @@ -51,6 +51,37 @@ func TestWithTxDB_PostgresRollsBackGraphAndSearchDocumentTogether(t *testing.T) assertGraphAndSearchDocumentCounts(t, db, "txdb.PostgresRollback", "postgres rolled back search document", 0) } +func TestDeleteGraph_PostgresHandlesMoreThanBindParameterLimit(t *testing.T) { + s, db := setupIsolatedPostgresStore(t) + ctx := context.Background() + const nodeCount = 66_000 + nodes := make([]graph.Node, nodeCount) + for i := range nodes { + nodes[i] = graph.Node{ + QualifiedName: fmt.Sprintf("bindlimit.Node%d", i), + Kind: graph.NodeKindFunction, + Name: fmt.Sprintf("Node%d", i), + FilePath: "bind-limit.go", + StartLine: i + 1, + EndLine: i + 1, + Language: "go", + } + } + if err := s.UpsertNodes(ctx, nodes); err != nil { + t.Fatalf("upsert %d nodes: %v", nodeCount, err) + } + if err := s.DeleteGraph(ctx); err != nil { + t.Fatalf("delete graph with %d nodes: %v", nodeCount, err) + } + var remaining int64 + if err := db.Model(&graph.Node{}).Count(&remaining).Error; err != nil { + t.Fatalf("count remaining nodes: %v", err) + } + if remaining != 0 { + t.Fatalf("remaining nodes = %d, want 0", remaining) + } +} + func setupIsolatedPostgresStore(t *testing.T) (*Store, *gorm.DB) { t.Helper() diff --git a/internal/adapters/outbound/graphgorm/store_test.go b/internal/adapters/outbound/graphgorm/store_test.go index 802d230..cab8954 100644 --- a/internal/adapters/outbound/graphgorm/store_test.go +++ b/internal/adapters/outbound/graphgorm/store_test.go @@ -1689,7 +1689,7 @@ func TestDeleteGraph_CleansSearchDocuments(t *testing.T) { }) node, _ := s.GetNode(ctx, "pkg.B") - if err := s.db.Create(&graph.SearchDocument{NodeID: node.ID, Content: "pkg.B content", Language: "go"}).Error; err != nil { + if err := s.db.Create(&graph.SearchDocument{Namespace: "ns-x", NodeID: node.ID, Content: "pkg.B content", Language: "go"}).Error; err != nil { t.Fatalf("insert search_document: %v", err) } diff --git a/internal/adapters/outbound/graphgorm/unresolved.go b/internal/adapters/outbound/graphgorm/unresolved.go index d38da2f..97bb74c 100644 --- a/internal/adapters/outbound/graphgorm/unresolved.go +++ b/internal/adapters/outbound/graphgorm/unresolved.go @@ -3,6 +3,8 @@ package graphgorm import ( "context" + "crypto/sha256" + "encoding/hex" "errors" "gorm.io/gorm" @@ -26,9 +28,11 @@ func (s *Store) UpsertUnresolvedEdges(ctx context.Context, candidates []graph.Un ns := requestctx.FromContext(ctx) for i := range candidates { candidates[i].Namespace = ns + candidates[i].LookupKeyHash = unresolvedIndexHash(candidates[i].LookupKey) + candidates[i].FingerprintHash = unresolvedIndexHash(candidates[i].Fingerprint) } if err := s.db.WithContext(ctx).Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "namespace"}, {Name: "fingerprint"}}, + Columns: []clause.Column{{Name: "namespace"}, {Name: "fingerprint_hash"}}, DoNothing: true, }).CreateInBatches(candidates, 500).Error; err != nil { return trace.Wrap(err, "upsert unresolved edges") @@ -43,13 +47,14 @@ func (s *Store) FindUnresolvedEdgesByLookupKeys(ctx context.Context, keys []stri return nil, nil } ns := requestctx.FromContext(ctx) + keyHashes := unresolvedIndexHashes(keys) seen := make(map[string]struct{}) var out []graph.Edge for start := 0; start < len(keys); start += unresolvedQueryChunkSize { end := min(start+unresolvedQueryChunkSize, len(keys)) var rows []graph.UnresolvedEdgeCandidate if err := s.db.WithContext(ctx). - Where("namespace = ? AND lookup_key IN ?", ns, keys[start:end]). + Where("namespace = ? AND lookup_key_hash IN ? AND lookup_key IN ?", ns, keyHashes[start:end], keys[start:end]). Order("file_path ASC").Order("line ASC").Order("fingerprint ASC"). Find(&rows).Error; err != nil { return nil, trace.Wrap(err, "find unresolved edges by lookup keys") @@ -101,10 +106,11 @@ func (s *Store) DeleteUnresolvedEdgesByFingerprints(ctx context.Context, fingerp return nil } ns := requestctx.FromContext(ctx) + fingerprintHashes := unresolvedIndexHashes(fingerprints) for start := 0; start < len(fingerprints); start += unresolvedQueryChunkSize { end := min(start+unresolvedQueryChunkSize, len(fingerprints)) if err := s.db.WithContext(ctx). - Where("namespace = ? AND fingerprint IN ?", ns, fingerprints[start:end]). + Where("namespace = ? AND fingerprint_hash IN ? AND fingerprint IN ?", ns, fingerprintHashes[start:end], fingerprints[start:end]). Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil { return trace.Wrap(err, "delete resolved unresolved edges") } @@ -112,6 +118,21 @@ func (s *Store) DeleteUnresolvedEdgesByFingerprints(ctx context.Context, fingerp return nil } +// unresolvedIndexHashes derives fixed-width database lookup values while preserving caller order. +// @intent keep arbitrary source-derived text out of PostgreSQL B-tree indexes without weakening exact-match queries. +func unresolvedIndexHashes(values []string) []string { + hashes := make([]string, len(values)) + for i, value := range values { + hashes[i] = unresolvedIndexHash(value) + } + return hashes +} + +func unresolvedIndexHash(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + // UnresolvedIndexReady reports whether a compatible full build populated the namespace's reverse index. // @intent gate semi-naive update on complete historical unresolved-edge coverage produced by the expected algorithm and parsers. func (s *Store) UnresolvedIndexReady(ctx context.Context, version string) (bool, error) { diff --git a/internal/adapters/outbound/graphgorm/unresolved_test.go b/internal/adapters/outbound/graphgorm/unresolved_test.go new file mode 100644 index 0000000..f01e48f --- /dev/null +++ b/internal/adapters/outbound/graphgorm/unresolved_test.go @@ -0,0 +1,86 @@ +package graphgorm + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + + requestctx "github.com/tae2089/code-context-graph/internal/ctx" + "github.com/tae2089/code-context-graph/internal/domain/graph" +) + +func TestUnresolvedEdges_PreservesAndIndexesLongValues(t *testing.T) { + store := setupTestDB(t) + ctx := requestctx.WithNamespace(context.Background(), "long-values") + lookupKey := strings.Repeat("lookup.", 600) + fingerprint := "calls:source.go:" + strings.Repeat("receiver.", 34_000) + ":1" + candidate := graph.UnresolvedEdgeCandidate{ + LookupKey: lookupKey, Fingerprint: fingerprint, FilePath: "source.go", Kind: graph.EdgeKindCalls, Line: 1, + } + + if err := store.UpsertUnresolvedEdges(ctx, []graph.UnresolvedEdgeCandidate{candidate}); err != nil { + t.Fatalf("UpsertUnresolvedEdges: %v", err) + } + var hashes struct { + LookupKeyHash string + FingerprintHash string + } + if err := store.db.Model(&graph.UnresolvedEdgeCandidate{}). + Select("lookup_key_hash", "fingerprint_hash"). + Where("namespace = ?", "long-values"). + First(&hashes).Error; err != nil { + t.Fatalf("load unresolved hashes: %v", err) + } + if hashes.LookupKeyHash != sha256HexForTest(lookupKey) { + t.Fatalf("lookup key hash = %q, want SHA-256", hashes.LookupKeyHash) + } + if hashes.FingerprintHash != sha256HexForTest(fingerprint) { + t.Fatalf("fingerprint hash = %q, want SHA-256", hashes.FingerprintHash) + } + + got, err := store.FindUnresolvedEdgesByLookupKeys(ctx, []string{lookupKey}) + if err != nil || len(got) != 1 { + t.Fatalf("FindUnresolvedEdgesByLookupKeys() = %d edges, err=%v; want one", len(got), err) + } + if got[0].Fingerprint != fingerprint { + t.Fatalf("fingerprint length = %d, want exact length %d", len(got[0].Fingerprint), len(fingerprint)) + } + if err := store.DeleteUnresolvedEdgesByFingerprints(ctx, []string{fingerprint}); err != nil { + t.Fatalf("DeleteUnresolvedEdgesByFingerprints: %v", err) + } + if got, err := store.FindUnresolvedEdgesByLookupKeys(ctx, []string{lookupKey}); err != nil || len(got) != 0 { + t.Fatalf("remaining unresolved edges = %d, err=%v; want none", len(got), err) + } +} + +func TestFindUnresolvedEdgesByLookupKeys_FiltersHashCollisionByOriginalText(t *testing.T) { + store := setupTestDB(t) + ctx := requestctx.WithNamespace(context.Background(), "hash-collision") + candidates := []graph.UnresolvedEdgeCandidate{ + {LookupKey: "Wanted", Fingerprint: "calls:a.go:Wanted:1", FilePath: "a.go", Kind: graph.EdgeKindCalls, Line: 1}, + {LookupKey: "Other", Fingerprint: "calls:b.go:Other:1", FilePath: "b.go", Kind: graph.EdgeKindCalls, Line: 1}, + } + if err := store.UpsertUnresolvedEdges(ctx, candidates); err != nil { + t.Fatalf("UpsertUnresolvedEdges: %v", err) + } + if err := store.db.Model(&graph.UnresolvedEdgeCandidate{}). + Where("namespace = ? AND lookup_key = ?", "hash-collision", "Other"). + Update("lookup_key_hash", sha256HexForTest("Wanted")).Error; err != nil { + t.Fatalf("force lookup hash collision: %v", err) + } + + got, err := store.FindUnresolvedEdgesByLookupKeys(ctx, []string{"Wanted"}) + if err != nil { + t.Fatalf("FindUnresolvedEdgesByLookupKeys: %v", err) + } + if len(got) != 1 || got[0].Fingerprint != "calls:a.go:Wanted:1" { + t.Fatalf("collision lookup returned %+v, want only Wanted", got) + } +} + +func sha256HexForTest(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/app/ingest/incremental/incremental.go b/internal/app/ingest/incremental/incremental.go index 4864abe..56d9ea7 100644 --- a/internal/app/ingest/incremental/incremental.go +++ b/internal/app/ingest/incremental/incremental.go @@ -20,12 +20,12 @@ import ( // @intent abstract graph storage so changed files can be reparsed and upserted type Store interface { GetNodesByIDs(ctx context.Context, ids []uint) ([]graph.Node, error) - GetNodesByFile(ctx context.Context, filePath string) ([]graph.Node, error) GetNodesByFiles(ctx context.Context, filePaths []string) (map[string][]graph.Node, error) GetNodesByQualifiedNames(ctx context.Context, names []string) (map[string][]graph.Node, error) UpsertNodes(ctx context.Context, nodes []graph.Node) error UpsertEdges(ctx context.Context, edges []graph.Edge) error - DeleteNodesByFile(ctx context.Context, filePath string) error + UpsertAnnotations(ctx context.Context, annotations []*graph.Annotation) error + DeleteNodesByFiles(ctx context.Context, filePaths []string) error } // Parser parses one file into graph nodes and edges. @@ -42,12 +42,6 @@ type AnnotatingParser interface { Language() string } -// annotationWriter is the optional store capability needed to persist comment-derived annotations. -// @intent allow incremental sync to skip annotation writes when the underlying store does not support them. -type annotationWriter interface { - UpsertAnnotation(ctx context.Context, ann *graph.Annotation) error -} - // FileInfo holds change-tracking data for one file. // @intent carry file content and hash so sync can detect modifications cheaply type FileInfo = ingest.FileInfo @@ -217,31 +211,25 @@ func (s *Syncer) syncWithExisting(ctx context.Context, syncStore Store, files ma parsedFiles = append(parsedFiles, parsed) } + modifiedPaths := make([]string, 0, len(parsedFiles)) for _, parsed := range parsedFiles { if !parsed.hadExisting { s.logger.Debug("file added", "file", parsed.filePath) stats.Added++ continue } - if err := syncStore.DeleteNodesByFile(ctx, parsed.filePath); err != nil { - return nil, err - } + modifiedPaths = append(modifiedPaths, parsed.filePath) s.logger.Debug("file modified", "file", parsed.filePath) stats.Modified++ } - - for _, parsed := range parsedFiles { - if len(parsed.nodes) > 0 { - if err := syncStore.UpsertNodes(ctx, parsed.nodes); err != nil { - return nil, err - } - if len(parsed.comments) > 0 { - if err := s.restoreAnnotations(ctx, syncStore, parsed.filePath, parsed.info.Content, parsed.nodes, parsed.comments, parsed.language); err != nil { - return nil, err - } - } + if len(modifiedPaths) > 0 { + if err := syncStore.DeleteNodesByFiles(ctx, modifiedPaths); err != nil { + return nil, err } } + if err := s.persistParsedNodesAndAnnotations(ctx, syncStore, parsedFiles); err != nil { + return nil, err + } if err := s.resolveAndUpsertEdges(ctx, syncStore, syncStore, parsedFiles, stats); err != nil { return nil, err } @@ -249,15 +237,19 @@ func (s *Syncer) syncWithExisting(ctx context.Context, syncStore Store, files ma releaseContent(files, parsed.filePath) } + deletedPaths := make([]string, 0, len(existingFiles)) for _, ep := range existingFiles { if _, stillPresent := files[ep]; !stillPresent { - if err := syncStore.DeleteNodesByFile(ctx, ep); err != nil { - return nil, err - } + deletedPaths = append(deletedPaths, ep) s.logger.Debug("file deleted", "file", ep) stats.Deleted++ } } + if len(deletedPaths) > 0 { + if err := syncStore.DeleteNodesByFiles(ctx, deletedPaths); err != nil { + return nil, err + } + } s.logger.Info("sync completed", "added", stats.Added, @@ -297,10 +289,12 @@ func (s *Syncer) syncBatchesWithExisting(ctx context.Context, syncStore Store, s return nil, err } - for _, filePath := range deletedFiles { - if err := syncStore.DeleteNodesByFile(ctx, filePath); err != nil { + if len(deletedFiles) > 0 { + if err := syncStore.DeleteNodesByFiles(ctx, deletedFiles); err != nil { return nil, err } + } + for _, filePath := range deletedFiles { s.logger.Debug("file deleted", "file", filePath) stats.Deleted++ } @@ -395,32 +389,29 @@ func (s *Syncer) stageBatch(ctx context.Context, syncStore Store, files map[stri parsedFiles = append(parsedFiles, parsed) } + modifiedPaths := make([]string, 0, len(parsedFiles)) for _, parsed := range parsedFiles { if !parsed.hadExisting { s.logger.Debug("file added", "file", parsed.filePath) stats.Added++ continue } - if err := syncStore.DeleteNodesByFile(ctx, parsed.filePath); err != nil { - return err - } + modifiedPaths = append(modifiedPaths, parsed.filePath) s.logger.Debug("file modified", "file", parsed.filePath) stats.Modified++ } + if len(modifiedPaths) > 0 { + if err := syncStore.DeleteNodesByFiles(ctx, modifiedPaths); err != nil { + return err + } + } + if err := s.persistParsedNodesAndAnnotations(ctx, syncStore, parsedFiles); err != nil { + return err + } edgeRecord := deferredEdgeRecord{Files: make([]deferredEdgeFile, 0, len(parsedFiles))} for i := range parsedFiles { parsed := &parsedFiles[i] - if len(parsed.nodes) > 0 { - if err := syncStore.UpsertNodes(ctx, parsed.nodes); err != nil { - return err - } - if len(parsed.comments) > 0 { - if err := s.restoreAnnotations(ctx, syncStore, parsed.filePath, parsed.info.Content, parsed.nodes, parsed.comments, parsed.language); err != nil { - return err - } - } - } if len(parsed.edges) > 0 { edgeRecord.Files = append(edgeRecord.Files, deferredEdgeFile{FilePath: parsed.filePath, Edges: parsed.edges}) } @@ -533,19 +524,55 @@ func (s *Syncer) resolveParser(filePath string) Parser { return s.parser } -// restoreAnnotations re-binds parsed comment blocks to the freshly persisted nodes for one file. -// @intent keep doc comments associated with their owning declarations after incremental reparses. -// @sideEffect upserts annotation rows through the store's annotation writer. -// @mutates graph annotations -func (s *Syncer) restoreAnnotations(ctx context.Context, syncStore Store, filePath string, content []byte, nodes []graph.Node, comments []ingest.CommentBlock, language string) error { - writer, ok := syncStore.(annotationWriter) - if !ok || language == "" { +// persistParsedNodesAndAnnotations writes one parsed file set with one node call and one annotation call. +// @intent keep incremental persistence proportional to bounded batches instead of individual files or comments. +// @sideEffect upserts graph nodes and annotations through the supplied store. +// @mutates persisted node IDs and annotation ownership. +func (s *Syncer) persistParsedNodesAndAnnotations(ctx context.Context, syncStore Store, parsedFiles []parsedSyncFile) error { + nodeCount := 0 + annotationFilePaths := make([]string, 0, len(parsedFiles)) + for i := range parsedFiles { + nodeCount += len(parsedFiles[i].nodes) + if len(parsedFiles[i].comments) > 0 && parsedFiles[i].language != "" { + annotationFilePaths = append(annotationFilePaths, parsedFiles[i].filePath) + } + } + nodes := make([]graph.Node, 0, nodeCount) + for i := range parsedFiles { + nodes = append(nodes, parsedFiles[i].nodes...) + } + if len(nodes) > 0 { + if err := syncStore.UpsertNodes(ctx, nodes); err != nil { + return trace.Wrap(err, "upsert incremental batch nodes") + } + } + if len(annotationFilePaths) == 0 { return nil } + storedNodesByFile, err := syncStore.GetNodesByFiles(ctx, annotationFilePaths) + if err != nil { + return trace.Wrap(err, "get incremental annotation nodes") + } + annotations := make([]*graph.Annotation, 0) + for i := range parsedFiles { + annotations = append(annotations, collectAnnotations(parsedFiles[i], storedNodesByFile[parsedFiles[i].filePath])...) + } + if err := syncStore.UpsertAnnotations(ctx, annotations); err != nil { + return trace.Wrap(err, "upsert incremental batch annotations") + } + return nil +} +// collectAnnotations re-binds one parsed file's comments to its freshly persisted nodes. +// @intent prepare annotation rows for the flush-scoped bulk write without issuing per-file SQL. +// @mutates returned annotations receive persisted node IDs. +func collectAnnotations(parsed parsedSyncFile, storedNodes []graph.Node) []*graph.Annotation { + if len(parsed.comments) == 0 || parsed.language == "" { + return nil + } binder := binding.NewBinder() - bindingComments := make([]binding.CommentBlock, len(comments)) - for i, c := range comments { + bindingComments := make([]binding.CommentBlock, len(parsed.comments)) + for i, c := range parsed.comments { bindingComments[i] = binding.CommentBlock{ StartLine: c.StartLine, EndLine: c.EndLine, @@ -554,33 +581,26 @@ func (s *Syncer) restoreAnnotations(ctx context.Context, syncStore Store, filePa OwnerStartLine: c.OwnerStartLine, } } - sourceLines := strings.Split(string(content), "\n") - bindings := binder.Bind(bindingComments, nodes, language, sourceLines) + sourceLines := strings.Split(string(parsed.info.Content), "\n") + bindings := binder.Bind(bindingComments, parsed.nodes, parsed.language, sourceLines) if len(bindings) == 0 { return nil } - - storedNodes, err := syncStore.GetNodesByFile(ctx, filePath) - if err != nil { - return err - } storedByKey := make(map[string]*graph.Node, len(storedNodes)) for i := range storedNodes { storedByKey[annotationBindingKey(storedNodes[i].QualifiedName, storedNodes[i].StartLine)] = &storedNodes[i] } + annotations := make([]*graph.Annotation, 0, len(bindings)) for _, binding := range bindings { stored := storedByKey[annotationBindingKey(binding.Node.QualifiedName, binding.Node.StartLine)] if stored == nil { continue } binding.Annotation.NodeID = stored.ID - if err := writer.UpsertAnnotation(ctx, binding.Annotation); err != nil { - return err - } + annotations = append(annotations, binding.Annotation) } - - return nil + return annotations } // parsedSyncFile holds parsed output and file metadata for one incremental sync input. diff --git a/internal/app/ingest/incremental/incremental_test.go b/internal/app/ingest/incremental/incremental_test.go index 2b04832..a5088d4 100644 --- a/internal/app/ingest/incremental/incremental_test.go +++ b/internal/app/ingest/incremental/incremental_test.go @@ -14,13 +14,15 @@ import ( ) type recordingStore struct { - nodes map[string]*graph.Node - upserted []string - deleted []string - upsertedEdges []graph.Edge - annotations []*graph.Annotation - nextID uint - operations []string + nodes map[string]*graph.Node + upserted []string + deleted []string + upsertedEdges []graph.Edge + annotations []*graph.Annotation + nextID uint + operations []string + deleteCalls int + annotationCalls int } func (r *recordingStore) GetNodesByFile(_ context.Context, filePath string) ([]graph.Node, error) { @@ -129,12 +131,17 @@ func (r *recordingStore) UpsertEdges(_ context.Context, edges []graph.Edge) erro return nil } -func (r *recordingStore) DeleteNodesByFile(_ context.Context, filePath string) error { +func (r *recordingStore) DeleteNodesByFiles(_ context.Context, filePaths []string) error { + r.deleteCalls++ r.operations = append(r.operations, "delete_nodes") - r.deleted = append(r.deleted, filePath) + r.deleted = append(r.deleted, filePaths...) + fileSet := make(map[string]struct{}, len(filePaths)) + for _, filePath := range filePaths { + fileSet[filePath] = struct{}{} + } toDelete := []string{} for qn, n := range r.nodes { - if n.FilePath == filePath { + if _, ok := fileSet[n.FilePath]; ok { toDelete = append(toDelete, qn) } } @@ -144,8 +151,9 @@ func (r *recordingStore) DeleteNodesByFile(_ context.Context, filePath string) e return nil } -func (r *recordingStore) UpsertAnnotation(_ context.Context, ann *graph.Annotation) error { - r.annotations = append(r.annotations, ann) +func (r *recordingStore) UpsertAnnotations(_ context.Context, annotations []*graph.Annotation) error { + r.annotationCalls++ + r.annotations = append(r.annotations, annotations...) return nil } @@ -370,6 +378,31 @@ func TestIncremental_ModifiedFile(t *testing.T) { } } +func TestIncremental_BatchesModifiedFileDeletion(t *testing.T) { + st := newStore() + st.nodes["pkg.OldA"] = &graph.Node{QualifiedName: "pkg.OldA", Kind: graph.NodeKindFunction, Name: "OldA", FilePath: "a.go", Hash: "old_a", Language: "go"} + st.nodes["pkg.OldB"] = &graph.Node{QualifiedName: "pkg.OldB", Kind: graph.NodeKindFunction, Name: "OldB", FilePath: "b.go", Hash: "old_b", Language: "go"} + parser := &staticParser{result: map[string]parseResult{ + "a.go": {nodes: []graph.Node{{QualifiedName: "pkg.NewA", Kind: graph.NodeKindFunction, Name: "NewA", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}}}, + "b.go": {nodes: []graph.Node{{QualifiedName: "pkg.NewB", Kind: graph.NodeKindFunction, Name: "NewB", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"}}}, + }} + + syncer := New(st, parser) + stats, err := syncer.Sync(context.Background(), map[string]FileInfo{ + "a.go": {Hash: "new_a", Content: []byte("package pkg")}, + "b.go": {Hash: "new_b", Content: []byte("package pkg")}, + }) + if err != nil { + t.Fatal(err) + } + if stats.Modified != 2 { + t.Fatalf("expected 2 modified files, got %d", stats.Modified) + } + if st.deleteCalls != 1 { + t.Fatalf("expected one batched deletion call, got %d", st.deleteCalls) + } +} + func TestIncremental_ModifiedFileParseFailurePreservesExistingNodes(t *testing.T) { st := newStore() st.nodes["pkg.Old"] = &graph.Node{QualifiedName: "pkg.Old", Kind: graph.NodeKindFunction, Name: "Old", FilePath: "mod.go", Hash: "old_hash", Language: "go"} @@ -484,6 +517,43 @@ func TestSyncWithExisting_RestoresAnnotationsForModifiedFile(t *testing.T) { } } +func TestSyncWithExisting_BatchesAnnotationsAcrossFiles(t *testing.T) { + st := newStore() + parser := &commentAwareParser{result: map[string]commentParseResult{ + "a.go": { + parseResult: parseResult{nodes: []graph.Node{ + {QualifiedName: "a.go", Kind: graph.NodeKindFile, Name: "a.go", FilePath: "a.go", StartLine: 1, EndLine: 2, Language: "go"}, + {QualifiedName: "pkg.A", Kind: graph.NodeKindFunction, Name: "A", FilePath: "a.go", StartLine: 2, EndLine: 2, Language: "go"}, + }}, + comments: []treesitter.CommentBlock{{StartLine: 1, EndLine: 1, Text: "// @intent A"}}, + language: "go", + }, + "b.go": { + parseResult: parseResult{nodes: []graph.Node{ + {QualifiedName: "b.go", Kind: graph.NodeKindFile, Name: "b.go", FilePath: "b.go", StartLine: 1, EndLine: 2, Language: "go"}, + {QualifiedName: "pkg.B", Kind: graph.NodeKindFunction, Name: "B", FilePath: "b.go", StartLine: 2, EndLine: 2, Language: "go"}, + }}, + comments: []treesitter.CommentBlock{{StartLine: 1, EndLine: 1, Text: "// @intent B"}}, + language: "go", + }, + }} + + syncer := New(st, parser) + _, err := syncer.Sync(context.Background(), map[string]FileInfo{ + "a.go": {Hash: "new_a", Content: []byte("// @intent A\nfunc A() {}")}, + "b.go": {Hash: "new_b", Content: []byte("// @intent B\nfunc B() {}")}, + }) + if err != nil { + t.Fatal(err) + } + if len(st.annotations) < 2 { + t.Fatalf("expected annotations from both files, got %d", len(st.annotations)) + } + if st.annotationCalls != 1 { + t.Fatalf("expected one batched annotation call, got %d", st.annotationCalls) + } +} + func TestIncremental_DispatchesParserByExtension(t *testing.T) { st := newStore() goParser := &staticParser{result: map[string]parseResult{ @@ -751,6 +821,25 @@ func TestSyncBatchesWithExisting_ResolvesImplementsBeforeEarlierCallBatch(t *tes t.Fatalf("expected earlier call batch to resolve after later implements batch, got %+v", st.upsertedEdges) } +func TestSyncBatchesWithExisting_BatchesDeletedFiles(t *testing.T) { + st := newStore() + st.nodes["pkg.A"] = &graph.Node{QualifiedName: "pkg.A", Kind: graph.NodeKindFunction, FilePath: "a.go"} + st.nodes["pkg.B"] = &graph.Node{QualifiedName: "pkg.B", Kind: graph.NodeKindFunction, FilePath: "b.go"} + source := ingest.FileBatchSource(func(ingest.FileBatchVisitor) error { return nil }) + + syncer := New(st, &staticParser{}) + stats, err := syncer.SyncBatchesWithExisting(context.Background(), source, []string{"a.go", "b.go"}) + if err != nil { + t.Fatal(err) + } + if stats.Deleted != 2 { + t.Fatalf("expected 2 deleted files, got %d", stats.Deleted) + } + if st.deleteCalls != 1 { + t.Fatalf("expected one batched deletion call, got %d", st.deleteCalls) + } +} + func TestIncremental_BatchesNodesBeforeResolvingEdges(t *testing.T) { st := newStore() st.nodes["mcp.FlowTracer"] = &graph.Node{ID: 10, QualifiedName: "mcp.FlowTracer", Kind: graph.NodeKindType, Name: "FlowTracer", FilePath: "mcp/deps.go", StartLine: 1, EndLine: 3, Hash: "iface", Language: "go"} @@ -787,8 +876,8 @@ func TestIncremental_BatchesNodesBeforeResolvingEdges(t *testing.T) { if firstEdge == -1 { t.Fatalf("expected edge upsert operation, got %v", st.operations) } - if nodeOps < 2 { - t.Fatalf("expected both files to upsert nodes before edges, got ops %v", st.operations) + if nodeOps != 1 { + t.Fatalf("expected one batched node upsert before edges, got ops %v", st.operations) } for i := 0; i < firstEdge; i++ { if st.operations[i] == "upsert_edges" { diff --git a/internal/app/ingest/ports.go b/internal/app/ingest/ports.go index d942ebd..1bb504d 100644 --- a/internal/app/ingest/ports.go +++ b/internal/app/ingest/ports.go @@ -145,8 +145,8 @@ type GraphStore interface { GetEdgesToNodes(ctx context.Context, nodeIDs []uint) ([]graph.Edge, error) UpsertNodes(ctx context.Context, nodes []graph.Node) error UpsertEdges(ctx context.Context, edges []graph.Edge) error - UpsertAnnotation(ctx context.Context, annotation *graph.Annotation) error - DeleteNodesByFile(ctx context.Context, filePath string) error + UpsertAnnotations(ctx context.Context, annotations []*graph.Annotation) error + DeleteNodesByFiles(ctx context.Context, filePaths []string) error DeleteEdgesByFile(ctx context.Context, filePath string) error DeletePackageSemanticEdges(ctx context.Context, anchorFiles []string) error DeleteGraph(ctx context.Context) error diff --git a/internal/app/ingest/ports_test.go b/internal/app/ingest/ports_test.go index 58ed6bb..7149f89 100644 --- a/internal/app/ingest/ports_test.go +++ b/internal/app/ingest/ports_test.go @@ -35,11 +35,11 @@ func (graphStoreContractStub) GetEdgesToNodes(context.Context, []uint) ([]graph. } func (graphStoreContractStub) UpsertNodes(context.Context, []graph.Node) error { return nil } func (graphStoreContractStub) UpsertEdges(context.Context, []graph.Edge) error { return nil } -func (graphStoreContractStub) UpsertAnnotation(context.Context, *graph.Annotation) error { +func (graphStoreContractStub) UpsertAnnotations(context.Context, []*graph.Annotation) error { return nil } -func (graphStoreContractStub) DeleteNodesByFile(context.Context, string) error { return nil } -func (graphStoreContractStub) DeleteEdgesByFile(context.Context, string) error { return nil } +func (graphStoreContractStub) DeleteNodesByFiles(context.Context, []string) error { return nil } +func (graphStoreContractStub) DeleteEdgesByFile(context.Context, string) error { return nil } func (graphStoreContractStub) DeletePackageSemanticEdges(context.Context, []string) error { return nil } diff --git a/internal/app/ingest/workflow/build.go b/internal/app/ingest/workflow/build.go index 1642e26..a9add6c 100644 --- a/internal/app/ingest/workflow/build.go +++ b/internal/app/ingest/workflow/build.go @@ -114,12 +114,12 @@ func (b *buildPersistBatch) reset() { b.bytes = 0 } -// bindAndReleaseNodeBatch binds a parsed file's comments after its nodes have persisted, then releases comment buffers. -// @intent preserve per-file annotation binding and release behavior after the enclosing flush persists all nodes together. -// @sideEffect writes annotation rows via the transaction-scoped store. -// @mutates graph annotations -func (s *Service) bindAndReleaseNodeBatch(ctx context.Context, txStore ingestapp.GraphStore, storedNodesByFile map[string][]graph.Node, batches []parsedBuildNodeBatch, idx int) error { +// collectAndReleaseNodeBatch binds a parsed file's comments after its nodes have persisted, then releases comment buffers. +// @intent collect per-file annotations for one flush-scoped bulk write while preserving eager buffer release. +// @mutates releases parsed comment and source-line buffers and returns annotations with stored node IDs. +func (s *Service) collectAndReleaseNodeBatch(storedNodesByFile map[string][]graph.Node, batches []parsedBuildNodeBatch, idx int) []*graph.Annotation { parsed := &batches[idx] + var annotations []*graph.Annotation if len(parsed.tsComments) > 0 { binderComments := toBinderComments(parsed.tsComments) @@ -140,9 +140,7 @@ func (s *Service) bindAndReleaseNodeBatch(ctx context.Context, txStore ingestapp continue } b.Annotation.NodeID = stored.ID - if err := txStore.UpsertAnnotation(ctx, b.Annotation); err != nil { - return trace.Wrap(err, "upsert annotation for "+stored.QualifiedName) - } + annotations = append(annotations, b.Annotation) } } @@ -151,7 +149,7 @@ func (s *Service) bindAndReleaseNodeBatch(ctx context.Context, txStore ingestapp if s.onBatchRelease != nil { s.onBatchRelease(batches, idx) } - return nil + return annotations } // Build walks source files, stores parsed graph data, and rebuilds search docs. @@ -677,13 +675,15 @@ func (s *Service) flushBuildBatch(ctx context.Context, txStore ingestapp.GraphSt storedNodesByFile = stored } + var annotations []*graph.Annotation for i := range batch.nodeBatches { if err := ctx.Err(); err != nil { return err } - if err := s.bindAndReleaseNodeBatch(ctx, txStore, storedNodesByFile, batch.nodeBatches, i); err != nil { - return err - } + annotations = append(annotations, s.collectAndReleaseNodeBatch(storedNodesByFile, batch.nodeBatches, i)...) + } + if err := txStore.UpsertAnnotations(ctx, annotations); err != nil { + return trace.Wrap(err, "upsert batch annotations") } batch.reset() diff --git a/internal/app/ingest/workflow/indexer_test.go b/internal/app/ingest/workflow/indexer_test.go index 00aea6f..31e1f83 100644 --- a/internal/app/ingest/workflow/indexer_test.go +++ b/internal/app/ingest/workflow/indexer_test.go @@ -117,6 +117,7 @@ type recordingGraphStore struct { nodesByFP map[string][]graph.Node edges []graph.Edge upsertedNodeBatches [][]graph.Node + upsertedAnnotations [][]*graph.Annotation upsertedEdges [][]graph.Edge fileSuffixLookupCalls int importFileNodeCalls int @@ -155,8 +156,9 @@ func (r *recordingGraphStore) GetNodesByFile(ctx context.Context, filePath strin return out, nil } -func (r *recordingGraphStore) UpsertAnnotation(ctx context.Context, ann *graph.Annotation) error { - r.record("UpsertAnnotation") +func (r *recordingGraphStore) UpsertAnnotations(ctx context.Context, annotations []*graph.Annotation) error { + r.record("UpsertAnnotations") + r.upsertedAnnotations = append(r.upsertedAnnotations, append([]*graph.Annotation(nil), annotations...)) return nil } @@ -393,7 +395,7 @@ func (r *recordingGraphStore) GetEdgesToNodes(ctx context.Context, nodeIDs []uin return result, nil } -func (r *recordingGraphStore) DeleteNodesByFile(ctx context.Context, filePath string) error { +func (r *recordingGraphStore) DeleteNodesByFiles(ctx context.Context, filePaths []string) error { return nil } @@ -2774,7 +2776,7 @@ func Keep() {} t.Fatalf("Build: %v", err) } - want := []string{"DeleteGraph", "UpsertNodes", "GetNodesByFiles", "UpsertAnnotation", "UpsertEdges"} + want := []string{"DeleteGraph", "UpsertNodes", "GetNodesByFiles", "UpsertAnnotations", "UpsertEdges"} for i, op := range want[:4] { if len(fakeStore.ops) <= i { t.Fatalf("ops too short: got %v", fakeStore.ops) @@ -2786,7 +2788,7 @@ func Keep() {} firstEdge := slices.Index(fakeStore.ops, "UpsertEdges") lastAnn := -1 for i := len(fakeStore.ops) - 1; i >= 0; i-- { - if fakeStore.ops[i] == "UpsertAnnotation" { + if fakeStore.ops[i] == "UpsertAnnotations" { lastAnn = i break } @@ -2795,7 +2797,7 @@ func Keep() {} t.Fatalf("expected annotations and edges in ops: %v", fakeStore.ops) } if firstEdge <= lastAnn { - t.Fatalf("expected UpsertEdges after all UpsertAnnotation calls, got %v", fakeStore.ops) + t.Fatalf("expected UpsertEdges after all UpsertAnnotations calls, got %v", fakeStore.ops) } } @@ -2863,8 +2865,11 @@ func TestFlushBuildBatch_BatchesNodesAndAnnotationLookups(t *testing.T) { if got := countOperation("GetNodesByFile"); got != 0 { t.Fatalf("per-file annotation node lookups = %d, want 0 (ops=%v)", got, fakeStore.ops) } - if got := countOperation("UpsertAnnotation"); got != 2 { - t.Fatalf("annotation upserts = %d, want 2 (ops=%v)", got, fakeStore.ops) + if got := countOperation("UpsertAnnotations"); got != 1 { + t.Fatalf("annotation batch upserts = %d, want 1 (ops=%v)", got, fakeStore.ops) + } + if got := len(fakeStore.upsertedAnnotations[0]); got != 2 { + t.Fatalf("annotations in batch = %d, want 2", got) } if released != 2 { t.Fatalf("released batches = %d, want 2", released) @@ -2930,7 +2935,7 @@ func Keep` + strconv.Itoa(i) + `() {} firstEdge := slices.Index(fakeStore.ops, "UpsertEdges") lastAnn := -1 for i := len(fakeStore.ops) - 1; i >= 0; i-- { - if fakeStore.ops[i] == "UpsertAnnotation" { + if fakeStore.ops[i] == "UpsertAnnotations" { lastAnn = i break } diff --git a/internal/db/migration/migration.go b/internal/db/migration/migration.go index 5b37a5f..2e243cd 100644 --- a/internal/db/migration/migration.go +++ b/internal/db/migration/migration.go @@ -23,7 +23,7 @@ import ( ) const ( - RequiredSchemaVersion = 10 + RequiredSchemaVersion = 13 SchemaVersionKey = "schema" LegacySchemaVersionTable = "ccg_schema_versions" ) @@ -414,7 +414,9 @@ func ModelNullabilityColumns() []SchemaColumn { {Table: "parse_cache_entries", Column: "payload"}, {Table: "unresolved_edge_candidates", Column: "namespace"}, {Table: "unresolved_edge_candidates", Column: "lookup_key"}, + {Table: "unresolved_edge_candidates", Column: "lookup_key_hash"}, {Table: "unresolved_edge_candidates", Column: "fingerprint"}, + {Table: "unresolved_edge_candidates", Column: "fingerprint_hash"}, {Table: "unresolved_edge_candidates", Column: "file_path"}, {Table: "unresolved_edge_candidates", Column: "kind"}, {Table: "unresolved_index_states", Column: "namespace"}, @@ -566,7 +568,12 @@ func validateSQLiteSchemaParity(db *gorm.DB) error { return fmt.Errorf("required column %q.%q is nullable", column.Table, column.Column) } } - for _, indexName := range []string{"idx_edges_ns_from_kind_to", "idx_edges_ns_to_kind_from"} { + for _, indexName := range []string{ + "idx_edges_ns_from_kind_to", + "idx_edges_ns_to_kind_from", + "idx_unresolved_ns_fp_hash", + "idx_unresolved_lookup_hash", + } { exists, err := sqliteIndexExists(db, indexName) if err != nil { return trace.Wrap(err, "inspect sqlite edge index") @@ -600,7 +607,12 @@ func validatePostgresSchemaParity(db *gorm.DB) error { } else if !ok { return fmt.Errorf("required trigger %q is missing", "trg_search_documents_tsv") } - for _, indexName := range []string{"idx_edges_ns_from_kind_to", "idx_edges_ns_to_kind_from"} { + for _, indexName := range []string{ + "idx_edges_ns_from_kind_to", + "idx_edges_ns_to_kind_from", + "idx_unresolved_ns_fp_hash", + "idx_unresolved_lookup_hash", + } { exists, err := postgresIndexExists(db, indexName) if err != nil { return trace.Wrap(err, "inspect postgres edge index") @@ -609,6 +621,18 @@ func validatePostgresSchemaParity(db *gorm.DB) error { return fmt.Errorf("required index %q is missing", indexName) } } + for _, column := range []SchemaColumn{ + {Table: "annotations", Column: "summary"}, + {Table: "annotations", Column: "context"}, + } { + dataType, err := postgresColumnDataType(db, column.Table, column.Column) + if err != nil { + return trace.Wrap(err, "inspect postgres annotation column type") + } + if dataType != "text" { + return fmt.Errorf("required column %q.%q has type %q, want text", column.Table, column.Column, dataType) + } + } return nil } diff --git a/internal/db/migration/migration_test.go b/internal/db/migration/migration_test.go index 6ad7db3..179c165 100644 --- a/internal/db/migration/migration_test.go +++ b/internal/db/migration/migration_test.go @@ -1,6 +1,18 @@ package migration -import "testing" +import ( + "path/filepath" + "testing" + + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestRequiredSchemaVersion_IncludesUnboundedAnnotationsAndResolverFileLookupIndex(t *testing.T) { + if RequiredSchemaVersion != 13 { + t.Fatalf("RequiredSchemaVersion = %d, want 13", RequiredSchemaVersion) + } +} func TestRequiredSchemaTables_IncludesOptimizationState(t *testing.T) { want := []string{"parse_cache_entries", "unresolved_edge_candidates", "unresolved_index_states"} @@ -19,6 +31,8 @@ func TestModelNullabilityColumns_IncludesOptimizationState(t *testing.T) { want := []SchemaColumn{ {Table: "parse_cache_entries", Column: "payload"}, {Table: "unresolved_edge_candidates", Column: "lookup_key"}, + {Table: "unresolved_edge_candidates", Column: "lookup_key_hash"}, + {Table: "unresolved_edge_candidates", Column: "fingerprint_hash"}, {Table: "unresolved_index_states", Column: "namespace"}, {Table: "unresolved_index_states", Column: "version"}, } @@ -32,3 +46,98 @@ func TestModelNullabilityColumns_IncludesOptimizationState(t *testing.T) { } } } + +func TestSQLiteMigrationEleven_InvalidatesUnresolvedIndexAndCanMigrateDown(t *testing.T) { + dsn := filepath.Join(t.TempDir(), "migration.db") + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatalf("open SQLite: %v", err) + } + migrator, _, err := NewMigrator(db, "sqlite", "") + if err != nil { + t.Fatalf("NewMigrator: %v", err) + } + if err := migrator.Steps(10); err != nil { + t.Fatalf("migrate to version 10: %v", err) + } + if err := db.Table("unresolved_edge_candidates").Create(map[string]any{ + "namespace": "repo", "lookup_key": "Target", "fingerprint": "calls:a.go:Target:1", + "file_path": "a.go", "kind": "calls", "line": 1, + }).Error; err != nil { + t.Fatalf("insert version-10 candidate: %v", err) + } + if err := db.Table("unresolved_index_states").Create(map[string]any{ + "namespace": "repo", "version": "old", + }).Error; err != nil { + t.Fatalf("insert version-10 state: %v", err) + } + + if err := migrator.Steps(1); err != nil { + t.Fatalf("migrate to version 11: %v", err) + } + for _, table := range []string{"unresolved_edge_candidates", "unresolved_index_states"} { + var count int64 + if err := db.Table(table).Count(&count).Error; err != nil { + t.Fatalf("count %s: %v", table, err) + } + if count != 0 { + t.Fatalf("%s rows after migration = %d, want 0", table, count) + } + } + for _, column := range []string{"lookup_key_hash", "fingerprint_hash"} { + if !db.Migrator().HasColumn("unresolved_edge_candidates", column) { + t.Fatalf("version 11 missing column %q", column) + } + } + for _, indexName := range []string{"idx_unresolved_ns_fp_hash", "idx_unresolved_lookup_hash"} { + exists, err := sqliteIndexExists(db, indexName) + if err != nil { + t.Fatalf("inspect index %q: %v", indexName, err) + } + if !exists { + t.Fatalf("version 11 missing index %q", indexName) + } + } + if err := migrator.Steps(-1); err != nil { + t.Fatalf("migrate down to version 10: %v", err) + } + for _, column := range []string{"lookup_key_hash", "fingerprint_hash"} { + if db.Migrator().HasColumn("unresolved_edge_candidates", column) { + t.Fatalf("version 10 retained column %q", column) + } + } +} + +func TestSQLiteMigrationThirteen_AddsResolverFileLookupIndexAndCanMigrateDown(t *testing.T) { + dsn := filepath.Join(t.TempDir(), "migration.db") + db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{}) + if err != nil { + t.Fatalf("open SQLite: %v", err) + } + migrator, _, err := NewMigrator(db, "sqlite", "") + if err != nil { + t.Fatalf("NewMigrator: %v", err) + } + if err := migrator.Steps(13); err != nil { + t.Fatalf("migrate to version 13: %v", err) + } + + const indexName = "idx_nodes_ns_file_path" + exists, err := sqliteIndexExists(db, indexName) + if err != nil { + t.Fatalf("inspect index %q: %v", indexName, err) + } + if !exists { + t.Fatalf("version 13 missing index %q", indexName) + } + if err := migrator.Steps(-1); err != nil { + t.Fatalf("migrate down to version 12: %v", err) + } + exists, err = sqliteIndexExists(db, indexName) + if err != nil { + t.Fatalf("inspect index %q after down: %v", indexName, err) + } + if exists { + t.Fatalf("version 12 retained index %q", indexName) + } +} diff --git a/internal/db/migration/postgres/000011_unresolved_index_hashes.down.sql b/internal/db/migration/postgres/000011_unresolved_index_hashes.down.sql new file mode 100644 index 0000000..84b3505 --- /dev/null +++ b/internal/db/migration/postgres/000011_unresolved_index_hashes.down.sql @@ -0,0 +1,17 @@ +DELETE FROM unresolved_edge_candidates; +DELETE FROM unresolved_index_states; + +DROP INDEX IF EXISTS idx_unresolved_ns_fp_hash; +DROP INDEX IF EXISTS idx_unresolved_lookup_hash; + +ALTER TABLE unresolved_edge_candidates +DROP COLUMN fingerprint_hash; +ALTER TABLE unresolved_edge_candidates +DROP COLUMN lookup_key_hash; +ALTER TABLE unresolved_edge_candidates +ALTER COLUMN lookup_key TYPE varchar(512); + +CREATE UNIQUE INDEX idx_unresolved_ns_fp +ON unresolved_edge_candidates(namespace, fingerprint); +CREATE INDEX idx_unresolved_edge_candidates_lookup_key +ON unresolved_edge_candidates(lookup_key); diff --git a/internal/db/migration/postgres/000011_unresolved_index_hashes.up.sql b/internal/db/migration/postgres/000011_unresolved_index_hashes.up.sql new file mode 100644 index 0000000..fec3277 --- /dev/null +++ b/internal/db/migration/postgres/000011_unresolved_index_hashes.up.sql @@ -0,0 +1,17 @@ +DELETE FROM unresolved_edge_candidates; +DELETE FROM unresolved_index_states; + +DROP INDEX IF EXISTS idx_unresolved_ns_fp; +DROP INDEX IF EXISTS idx_unresolved_edge_candidates_lookup_key; + +ALTER TABLE unresolved_edge_candidates +ALTER COLUMN lookup_key TYPE text; +ALTER TABLE unresolved_edge_candidates +ADD COLUMN lookup_key_hash varchar(64) NOT NULL DEFAULT ''; +ALTER TABLE unresolved_edge_candidates +ADD COLUMN fingerprint_hash varchar(64) NOT NULL DEFAULT ''; + +CREATE UNIQUE INDEX idx_unresolved_ns_fp_hash +ON unresolved_edge_candidates(namespace, fingerprint_hash); +CREATE INDEX idx_unresolved_lookup_hash +ON unresolved_edge_candidates(lookup_key_hash); diff --git a/internal/db/migration/postgres/000012_annotation_text.down.sql b/internal/db/migration/postgres/000012_annotation_text.down.sql new file mode 100644 index 0000000..c6c353c --- /dev/null +++ b/internal/db/migration/postgres/000012_annotation_text.down.sql @@ -0,0 +1,5 @@ +-- PostgreSQL rejects this downgrade instead of truncating annotation content +-- when existing values exceed the historical limits. +ALTER TABLE annotations + ALTER COLUMN summary TYPE varchar(1024), + ALTER COLUMN context TYPE varchar(2048); diff --git a/internal/db/migration/postgres/000012_annotation_text.up.sql b/internal/db/migration/postgres/000012_annotation_text.up.sql new file mode 100644 index 0000000..c0c4214 --- /dev/null +++ b/internal/db/migration/postgres/000012_annotation_text.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE annotations + ALTER COLUMN summary TYPE text, + ALTER COLUMN context TYPE text; diff --git a/internal/db/migration/postgres/000013_node_namespace_file_path_index.down.sql b/internal/db/migration/postgres/000013_node_namespace_file_path_index.down.sql new file mode 100644 index 0000000..e11a1b1 --- /dev/null +++ b/internal/db/migration/postgres/000013_node_namespace_file_path_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_nodes_ns_file_path; diff --git a/internal/db/migration/postgres/000013_node_namespace_file_path_index.up.sql b/internal/db/migration/postgres/000013_node_namespace_file_path_index.up.sql new file mode 100644 index 0000000..eb3fd14 --- /dev/null +++ b/internal/db/migration/postgres/000013_node_namespace_file_path_index.up.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_nodes_ns_file_path + ON nodes(namespace, file_path); diff --git a/internal/db/migration/sqlite/000011_unresolved_index_hashes.down.sql b/internal/db/migration/sqlite/000011_unresolved_index_hashes.down.sql new file mode 100644 index 0000000..7de1cd9 --- /dev/null +++ b/internal/db/migration/sqlite/000011_unresolved_index_hashes.down.sql @@ -0,0 +1,15 @@ +DELETE FROM unresolved_edge_candidates; +DELETE FROM unresolved_index_states; + +DROP INDEX IF EXISTS idx_unresolved_ns_fp_hash; +DROP INDEX IF EXISTS idx_unresolved_lookup_hash; + +ALTER TABLE unresolved_edge_candidates +DROP COLUMN fingerprint_hash; +ALTER TABLE unresolved_edge_candidates +DROP COLUMN lookup_key_hash; + +CREATE UNIQUE INDEX idx_unresolved_ns_fp +ON unresolved_edge_candidates(namespace, fingerprint); +CREATE INDEX idx_unresolved_edge_candidates_lookup_key +ON unresolved_edge_candidates(lookup_key); diff --git a/internal/db/migration/sqlite/000011_unresolved_index_hashes.up.sql b/internal/db/migration/sqlite/000011_unresolved_index_hashes.up.sql new file mode 100644 index 0000000..d123dfb --- /dev/null +++ b/internal/db/migration/sqlite/000011_unresolved_index_hashes.up.sql @@ -0,0 +1,15 @@ +DELETE FROM unresolved_edge_candidates; +DELETE FROM unresolved_index_states; + +DROP INDEX IF EXISTS idx_unresolved_ns_fp; +DROP INDEX IF EXISTS idx_unresolved_edge_candidates_lookup_key; + +ALTER TABLE unresolved_edge_candidates +ADD COLUMN lookup_key_hash text NOT NULL DEFAULT ''; +ALTER TABLE unresolved_edge_candidates +ADD COLUMN fingerprint_hash text NOT NULL DEFAULT ''; + +CREATE UNIQUE INDEX idx_unresolved_ns_fp_hash +ON unresolved_edge_candidates(namespace, fingerprint_hash); +CREATE INDEX idx_unresolved_lookup_hash +ON unresolved_edge_candidates(lookup_key_hash); diff --git a/internal/db/migration/sqlite/000012_annotation_text.down.sql b/internal/db/migration/sqlite/000012_annotation_text.down.sql new file mode 100644 index 0000000..7f811e6 --- /dev/null +++ b/internal/db/migration/sqlite/000012_annotation_text.down.sql @@ -0,0 +1,2 @@ +-- Version alignment only; SQLite requires no schema reversal. +SELECT 1; diff --git a/internal/db/migration/sqlite/000012_annotation_text.up.sql b/internal/db/migration/sqlite/000012_annotation_text.up.sql new file mode 100644 index 0000000..6931f69 --- /dev/null +++ b/internal/db/migration/sqlite/000012_annotation_text.up.sql @@ -0,0 +1,2 @@ +-- SQLite annotation summary/context columns have always used unbounded text. +SELECT 1; diff --git a/internal/db/migration/sqlite/000013_node_namespace_file_path_index.down.sql b/internal/db/migration/sqlite/000013_node_namespace_file_path_index.down.sql new file mode 100644 index 0000000..e11a1b1 --- /dev/null +++ b/internal/db/migration/sqlite/000013_node_namespace_file_path_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_nodes_ns_file_path; diff --git a/internal/db/migration/sqlite/000013_node_namespace_file_path_index.up.sql b/internal/db/migration/sqlite/000013_node_namespace_file_path_index.up.sql new file mode 100644 index 0000000..eb3fd14 --- /dev/null +++ b/internal/db/migration/sqlite/000013_node_namespace_file_path_index.up.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_nodes_ns_file_path + ON nodes(namespace, file_path); diff --git a/internal/domain/graph/annotation.go b/internal/domain/graph/annotation.go index f38836c..297dc82 100644 --- a/internal/domain/graph/annotation.go +++ b/internal/domain/graph/annotation.go @@ -26,8 +26,8 @@ const ( type Annotation struct { ID uint `gorm:"primaryKey"` NodeID uint `gorm:"uniqueIndex;not null"` - Summary string `gorm:"size:1024"` - Context string `gorm:"size:2048"` + Summary string `gorm:"type:text"` + Context string `gorm:"type:text"` RawText string `gorm:"type:text"` CreatedAt time.Time UpdatedAt time.Time diff --git a/internal/domain/graph/node.go b/internal/domain/graph/node.go index dea6d1f..a13057d 100644 --- a/internal/domain/graph/node.go +++ b/internal/domain/graph/node.go @@ -19,11 +19,11 @@ const ( // @intent 파일 내 선언의 정체성과 위치 정보를 영속화한다. type Node struct { ID uint `gorm:"primaryKey"` - Namespace string `gorm:"size:256;not null;default:'default';uniqueIndex:idx_ns_qn_fp_sl"` + Namespace string `gorm:"size:256;not null;default:'default';uniqueIndex:idx_ns_qn_fp_sl;index:idx_nodes_ns_file_path,priority:1"` QualifiedName string `gorm:"size:512;not null;uniqueIndex:idx_ns_qn_fp_sl"` Kind NodeKind `gorm:"size:32;not null;index"` Name string `gorm:"size:256;not null"` - FilePath string `gorm:"size:768;not null;index;uniqueIndex:idx_ns_qn_fp_sl"` + FilePath string `gorm:"size:768;not null;index;uniqueIndex:idx_ns_qn_fp_sl;index:idx_nodes_ns_file_path,priority:2"` StartLine int `gorm:"not null;uniqueIndex:idx_ns_qn_fp_sl"` EndLine int `gorm:"not null"` Hash string `gorm:"size:64"` diff --git a/internal/domain/graph/unresolved.go b/internal/domain/graph/unresolved.go index b398826..5f468f0 100644 --- a/internal/domain/graph/unresolved.go +++ b/internal/domain/graph/unresolved.go @@ -6,14 +6,16 @@ import "time" // UnresolvedEdgeCandidate stores one lookup key for a syntax edge that lacks a graph endpoint. // @intent let newly added symbols select affected unchanged callers without reparsing the whole graph. type UnresolvedEdgeCandidate struct { - ID uint `gorm:"primaryKey"` - Namespace string `gorm:"size:256;not null;default:'default';uniqueIndex:idx_unresolved_ns_fp"` - LookupKey string `gorm:"size:512;not null;index"` - Fingerprint string `gorm:"type:text;not null;uniqueIndex:idx_unresolved_ns_fp"` - FilePath string `gorm:"size:1024;not null;index"` - Kind EdgeKind `gorm:"size:32;not null"` - Line int - CreatedAt time.Time + ID uint `gorm:"primaryKey"` + Namespace string `gorm:"size:256;not null;default:'default';uniqueIndex:idx_unresolved_ns_fp_hash"` + LookupKey string `gorm:"type:text;not null"` + LookupKeyHash string `gorm:"size:64;not null;index:idx_unresolved_lookup_hash"` + Fingerprint string `gorm:"type:text;not null"` + FingerprintHash string `gorm:"size:64;not null;uniqueIndex:idx_unresolved_ns_fp_hash"` + FilePath string `gorm:"size:1024;not null;index"` + Kind EdgeKind `gorm:"size:32;not null"` + Line int + CreatedAt time.Time } // Edge converts the durable candidate back into resolver input. From 45ca5a4cb5b106709e995536f27bdd150b18b48f Mon Sep 17 00:00:00 2001 From: tae2089 Date: Thu, 16 Jul 2026 17:13:46 +0900 Subject: [PATCH 7/7] test: retain reposync queue start signals --- internal/app/reposync/queue_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/reposync/queue_test.go b/internal/app/reposync/queue_test.go index 6b8bf62..bfa3d0b 100644 --- a/internal/app/reposync/queue_test.go +++ b/internal/app/reposync/queue_test.go @@ -491,7 +491,7 @@ func TestNewSyncQueueWithOptions_NilContextDefaultsToBackground(t *testing.T) { } func TestSyncQueue_AddRejectsWhenMaxTrackedReposExceeded(t *testing.T) { - started := make(chan struct{}) + started := make(chan struct{}, 1) release := make(chan struct{}) handler := func(_ context.Context, repoFullName, cloneURL, branch string) error { select { @@ -526,7 +526,7 @@ func TestSyncQueue_AddRejectsWhenMaxTrackedReposExceeded(t *testing.T) { } func TestSyncQueue_AddAllowsExistingRepoUpdateWhenAtCapacity(t *testing.T) { - started := make(chan struct{}) + started := make(chan struct{}, 1) release := make(chan struct{}) urls := make(chan string, 2) handler := func(_ context.Context, repoFullName, cloneURL, branch string) error {