From 94fb8c4b627288aae8e0cd436672893431262a35 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 21:42:02 +0900 Subject: [PATCH 01/12] Bound initial-index SQLite write batches --- DEVELOPER_GUIDE.md | 2 + TESTING_GUIDE.md | 4 +- .../+initial-index-write-batches.changed.md | 16 ++ src/CodeIndex/Database/DbWriter.BatchSql.cs | 14 ++ .../Database/DbWriter.ChunkSymbolBatches.cs | 10 +- src/CodeIndex/Database/DbWriter.Issues.cs | 5 +- src/CodeIndex/Database/DbWriter.References.cs | 39 ++++- tests/CodeIndex.Tests/DatabaseTests.cs | 146 ++++++++++++++++-- .../ReferencePersistenceBindingTests.cs | 21 ++- 9 files changed, 227 insertions(+), 30 deletions(-) create mode 100644 changelog.d/unreleased/+initial-index-write-batches.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 035292d21..329f5039d 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1274,6 +1274,7 @@ Current stable codes and triggers: | Maintenance error contract | `vacuum`, `backfill-fold`, `optimize` / `index --optimize`, and `db integrity` route failures through `MaintenanceDatabaseErrorClassifier` version `1` and one JSON/human writer. SQLite primary codes `5`/`6`, `8`, `11`, and `26` classify locked/busy, not-writable, corrupt, and not-a-database failures without inspecting exception wording. The shared response carries a stable error code/category, conditional recovery hint, redacted path metadata, and optional primary/extended SQLite codes. Absolute paths are redacted by default; `--show-paths` is the explicit diagnostic opt-in. | | Durable WAL file set | When WAL is active, the durable SQLite index is the `.db` file plus sibling `.db-wal` and `.db-shm` files. Backups, diagnostics bundles, and manual copies must include all three files when the siblings exist, or use SQLite's `.backup` command/API from a live connection. Copying only `codeindex.db` can produce a stale snapshot because committed pages may still live in `codeindex.db-wal`. | | `synchronous=NORMAL` | Under WAL, `NORMAL` avoids per-commit fsync pressure during 500-row indexing batches while preserving database consistency after crashes. | +| Caller-owned write batching | Full-scan and other atomic file writes already run inside one caller-owned transaction, so their language-neutral chunk, symbol, issue, reference-line, and reference inserts cap each named-parameter statement at 32 parameters. `Microsoft.Data.Sqlite` resolves every parameter name again on execution; this smaller shape avoids dense binding lookup without adding transaction scopes. Public writer APIs retain the SQLite-variable-limit batch shape and their existing per-batch transaction/SAVEPOINT contract. | | Checkpointing | `DbWriter` runs `PRAGMA wal_checkpoint(PASSIVE)` after each outer transaction commit, and SQLite may also checkpoint automatically after the configured 1000-page threshold. Both checkpoint paths are opportunistic: active readers are not blocked, and an uncheckpointed WAL is expected state rather than corruption. | | Checkpoint result contract | Explicit `PRAGMA wal_checkpoint(TRUNCATE)` paths execute a reader and return a structured result containing SQLite's `(busy, log, checkpointed)` values. Non-zero `busy` or positive remaining pages is unsuccessful with a bounded machine reason. `(0, -1, -1)` is SQLite's successful non-WAL no-op. Instance checkpointing, the static read-only-fallback preflight, query diagnostics, top-level status, and nested connection-policy status preserve the same result and counts. Raw exception text and paths must not enter diagnostics. | | Crash recovery | If the process is killed after SQLite has committed a transaction but before checkpointing, the next normal opener rolls the WAL forward; no manual recovery step is required. If the process dies before a transaction commits, SQLite rolls that transaction back. | @@ -5039,6 +5040,7 @@ apply 時は `PRAGMA optimize` を実行します。 | maintenance error contract | `vacuum`、`backfill-fold`、`optimize` / `index --optimize`、`db integrity` の失敗は `MaintenanceDatabaseErrorClassifier` version `1` と単一の JSON / human writer を通ります。SQLite primary code `5` / `6`、`8`、`11`、`26` から locked / busy、not-writable、corrupt、not-a-database を分類し、例外 message は判定に使いません。共有 response は stable error code / category、条件別 recovery hint、redaction 済み path metadata、任意の primary / extended SQLite code を返します。absolute path は既定で redaction し、`--show-paths` を明示的な diagnostic opt-in とします。 | | durable WAL file set | WAL が有効な場合、永続化された SQLite index は `.db` file と sibling の `.db-wal` / `.db-shm` file の組です。backup、diagnostics bundle、手動 copy では sibling が存在する場合に 3 file すべてを含めるか、live connection から SQLite の `.backup` command/API を使う必要があります。`codeindex.db` だけを copy すると、committed page がまだ `codeindex.db-wal` に残っているため stale snapshot になる可能性があります。 | | `synchronous=NORMAL` | WAL では `NORMAL` により 500 row 単位の indexing batch ごとの fsync 負荷を避けつつ、crash 後の database consistency を保ちます。 | +| caller-owned write batch | full-scan などの atomic file write は既に1つの caller-owned transaction 内で実行されるため、言語共通の chunk、symbol、issue、reference-line、reference insert は named parameter statement を32 parameter以下に制限します。`Microsoft.Data.Sqlite` は実行ごとに全 parameter name を再解決するため、この小さい形状で追加 transaction scope を増やさず dense binding lookup を避けます。public writer API は SQLite variable limit までの batch 形状と既存の batch ごとの transaction / SAVEPOINT 契約を維持します。 | | checkpoint | `DbWriter` は outer transaction commit 後に `PRAGMA wal_checkpoint(PASSIVE)` を実行し、SQLite も設定済みの 1000 page threshold を超えると自動 checkpoint する場合があります。どちらの checkpoint path も opportunistic で、active reader は block されず、未 checkpoint の WAL は corruption ではなく期待される状態です。 | | checkpoint result contract | 明示的な `PRAGMA wal_checkpoint(TRUNCATE)` path は reader を実行し、SQLite の `(busy, log, checkpointed)` を含む構造化結果を返します。`busy` が 0 以外、または remaining page が正の場合は、上限付き machine reason を伴う unsuccessful result です。`(0, -1, -1)` は SQLite の非 WAL database に対する成功 no-op です。instance checkpoint、read-only fallback 前の static preflight、query diagnostics、top-level status、nested connection-policy status は同じ結果と count を保持します。raw exception text や path を diagnostics に含めてはいけません。 | | crash recovery | SQLite が transaction を commit した後、checkpoint 前に process が kill された場合、次の通常 open が WAL を roll forward するため手動 recovery は不要です。commit 前に process が終了した transaction は SQLite により rollback されます。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index bae41c94c..28e6c2c44 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -745,7 +745,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` keeps 8,000 members under three nested C# containers on one reusable assignment path buffer; its `net8.0` allocation and practical-time budgets are blocking. `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` keeps 12,000 no-alias calls from paying for alias dedupe, `CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` guards pre-sized stable compaction after alias rewrites, and `MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` covers repeated qualified C#- and Python-style cycle names without per-edge normalization strings. `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` warms a 1,024-row typed snapshot with its exact capacity, then prevents a second candidate collection or redundant path values from returning to the default `net8.0` path. - `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. + `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. `CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` additionally fixes the atomic full-index parameter budget at 32 across language-neutral chunks (6 rows), symbols (1), issues (5), reference lines (10), and references (2), while the public reference API retains 71-row batches. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. - `HotspotReferenceAggregateTests.cs` @@ -1813,7 +1813,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` は3階層の C# container 内の8,000 member を1つの再利用 assignment path buffer で処理します。`net8.0` の allocation と実用時間 budget は blocking です。 `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` は alias のない 12,000 call が alias dedupe のコストを負わないこと、`CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` は alias rewrite 後の事前 capacity 付き stable compaction、`MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` は C# / Python 形式の qualified cycle name が edge ごとの正規化文字列を作らないことを保証します。 `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` は exact capacity を渡した1,024行の typed snapshot を warm-up し、2つ目の候補 collection や重複 path value が通常の `net8.0` 経路へ戻らないようにします。 - `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 + `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。`CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` はさらに、言語共通のchunk（6行）、symbol（1行）、issue（5行）、reference line（10行）、reference（2行）に対するatomic full-index parameter budgetを32に固定し、public reference APIの71行batchは維持します。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 - `HotspotReferenceAggregateTests.cs` diff --git a/changelog.d/unreleased/+initial-index-write-batches.changed.md b/changelog.d/unreleased/+initial-index-write-batches.changed.md new file mode 100644 index 000000000..9bf47b033 --- /dev/null +++ b/changelog.d/unreleased/+initial-index-write-batches.changed.md @@ -0,0 +1,16 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.BatchSql.cs + - src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs + - src/CodeIndex/Database/DbWriter.Issues.cs + - src/CodeIndex/Database/DbWriter.References.cs +--- + +## English + +- **Initial full indexing now uses binding-efficient SQLite write batches** — caller-owned file transactions cap named parameters per chunk, symbol, issue, reference-line, and reference statement, avoiding repeated dense parameter-name lookup across every supported language while preserving public writer transaction and SAVEPOINT contracts. + +## 日本語 + +- **初回フル索引がSQLite binding効率のよいwrite batchを使うようになりました** — caller-owned file transaction内のchunk、symbol、issue、reference-line、reference statementでnamed parameter数を制限し、全対応言語に共通する密なparameter name再探索を避けつつ、public writerのtransaction / SAVEPOINT契約を維持します。 diff --git a/src/CodeIndex/Database/DbWriter.BatchSql.cs b/src/CodeIndex/Database/DbWriter.BatchSql.cs index 900709c93..e7a6d5b5e 100644 --- a/src/CodeIndex/Database/DbWriter.BatchSql.cs +++ b/src/CodeIndex/Database/DbWriter.BatchSql.cs @@ -6,6 +6,10 @@ namespace CodeIndex.Database; public partial class DbWriter { private const int BatchSize = 500; + // Microsoft.Data.Sqlite resolves every named parameter through SQLite again on + // each execution. Caller-owned transactions let us split dense writes without + // adding transaction scopes, so keep those statements below this binding budget. + private const int MaxCallerTransactionBatchParameters = 32; private const int MaxFoldedNameCacheEntries = 4096; private static object FoldedNameDbValue(string? name, Dictionary cache) @@ -72,4 +76,14 @@ private static int GetRowsPerInsertStatement(int columnCount) return Math.Max(1, Math.Min(BatchSize, SqliteDynamicSql.MaxSqlVariables / columnCount)); } + + private static int GetRowsPerCallerTransactionInsertStatement(int columnCount) + { + if (columnCount <= 0) + throw new ArgumentOutOfRangeException(nameof(columnCount)); + + return Math.Max(1, Math.Min( + GetRowsPerInsertStatement(columnCount), + MaxCallerTransactionBatchParameters / columnCount)); + } } diff --git a/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs b/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs index f2c453334..1f476803e 100644 --- a/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs +++ b/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs @@ -20,7 +20,9 @@ public void InsertChunks(IReadOnlyList chunks, CancellationToken ca { if (chunks.Count == 0) return; - int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 5); + int rowsPerStatement = IsInTransaction() + ? GetRowsPerCallerTransactionInsertStatement(columnCount: 5) + : GetRowsPerInsertStatement(columnCount: 5); for (int i = 0; i < chunks.Count; i += rowsPerStatement) { CheckBatchCancellationAndReportProgress("insert_chunks", i, chunks.Count, cancellationToken); @@ -59,7 +61,9 @@ public void InsertSymbols(IReadOnlyList symbols, CancellationToken TrackReferenceGraphInsertedSymbols(symbols); InvalidateReferenceIdentityContractForMutation(); - int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 25); + int rowsPerStatement = IsInTransaction() + ? GetRowsPerCallerTransactionInsertStatement(columnCount: 25) + : GetRowsPerInsertStatement(columnCount: 25); var foldedNameCache = CreateFoldedNameCache( Math.Min(symbols.Count, rowsPerStatement), namesPerRow: 1); @@ -221,6 +225,7 @@ private void InsertChunkBatch(IReadOnlyList chunks, int start, int cmd.Parameters[parameterIndex++].Value = chunk.Content; } + ReportBatchStatementForTesting("insert_chunks", batchCount, batchCount); cmd.ExecuteNonQuery(); } finally @@ -280,6 +285,7 @@ private void InsertSymbolBatch(IReadOnlyList symbols, int start, i symbol.DisplayNameFolded); } + ReportBatchStatementForTesting("insert_symbols", batchCount, batchCount); cmd.ExecuteNonQuery(); } finally diff --git a/src/CodeIndex/Database/DbWriter.Issues.cs b/src/CodeIndex/Database/DbWriter.Issues.cs index 977c812f9..50c9efa67 100644 --- a/src/CodeIndex/Database/DbWriter.Issues.cs +++ b/src/CodeIndex/Database/DbWriter.Issues.cs @@ -45,7 +45,9 @@ private void InsertIssues(long fileId, IReadOnlyList return; } - int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 6); + int rowsPerStatement = IsInTransaction() + ? GetRowsPerCallerTransactionInsertStatement(columnCount: 6) + : GetRowsPerInsertStatement(columnCount: 6); for (int i = 0; i < issues.Count; i += rowsPerStatement) { int end = Math.Min(i + rowsPerStatement, issues.Count); @@ -111,6 +113,7 @@ private void InsertIssueBatch(long fileId, IReadOnlyList references, bool referenceLinesAreNew, Dictionary<(long FileId, int Line, string Context), long>? newReferenceLineIds, + bool useCallerTransactionParameterBudget, Dictionary foldedNameCache, int rowsPerStatement, int referenceBatchCount, @@ -1833,6 +1839,7 @@ private void InsertAtomicReferenceBatches( windowEnd, referenceLinesAreNew, newReferenceLineIds, + useCallerTransactionParameterBudget, cancellationToken); for (int batchIndex = windowStartBatch; batchIndex < windowEndBatch; batchIndex++) @@ -1890,10 +1897,22 @@ private ReferenceLineBatchMap MaterializeReferenceLines( int end, bool referenceLinesAreNew, Dictionary<(long FileId, int Line, string Context), long>? newReferenceLineIds, + bool useCallerTransactionParameterBudget, CancellationToken cancellationToken) => referenceLinesAreNew - ? InsertNewReferenceLines(references, start, end, newReferenceLineIds!, cancellationToken) - : UpsertReferenceLines(references, start, end, cancellationToken); + ? InsertNewReferenceLines( + references, + start, + end, + newReferenceLineIds!, + useCallerTransactionParameterBudget, + cancellationToken) + : UpsertReferenceLines( + references, + start, + end, + useCallerTransactionParameterBudget, + cancellationToken); private void InsertReferenceBatch( IReadOnlyList references, @@ -2078,6 +2097,7 @@ private ReferenceLineBatchMap UpsertReferenceLines( IReadOnlyList references, int start, int end, + bool useCallerTransactionParameterBudget, CancellationToken cancellationToken) { var lineIds = ReferenceLineBatchMap.Create( @@ -2086,7 +2106,9 @@ private ReferenceLineBatchMap UpsertReferenceLines( end, cancellationToken); var rows = lineIds.Keys; - int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 3); + int rowsPerStatement = useCallerTransactionParameterBudget + ? GetRowsPerCallerTransactionInsertStatement(columnCount: 3) + : GetRowsPerInsertStatement(columnCount: 3); for (int i = 0; i < rows.Length; i += rowsPerStatement) { CheckBatchCancellationAndReportProgress("upsert_reference_lines", i, rows.Length, cancellationToken); @@ -2106,7 +2128,7 @@ private ReferenceLineBatchMap UpsertReferenceLines( } } - int keysPerStatement = GetRowsPerInsertStatement(columnCount: 3); + int keysPerStatement = rowsPerStatement; for (int i = 0; i < rows.Length; i += keysPerStatement) { CheckBatchCancellationAndReportProgress("lookup_reference_lines", i, rows.Length, cancellationToken); @@ -2146,6 +2168,7 @@ private ReferenceLineBatchMap InsertNewReferenceLines( int start, int end, Dictionary<(long FileId, int Line, string Context), long> knownLineIds, + bool useCallerTransactionParameterBudget, CancellationToken cancellationToken) { var lineIds = ReferenceLineBatchMap.Create( @@ -2163,7 +2186,9 @@ private ReferenceLineBatchMap InsertNewReferenceLines( rows.Add(key); } - int rowsPerStatement = GetRowsPerInsertStatement(columnCount: 3); + int rowsPerStatement = useCallerTransactionParameterBudget + ? GetRowsPerCallerTransactionInsertStatement(columnCount: 3) + : GetRowsPerInsertStatement(columnCount: 3); for (int i = 0; i < rows.Count; i += rowsPerStatement) { CheckBatchCancellationAndReportProgress("insert_reference_lines", i, rows.Count, cancellationToken); diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 17ee56a46..3ed8f1b4c 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -8250,6 +8250,118 @@ public void ReferenceBatchStatements_EightFiveOneInputsUseExactRowCounts() } } + [Fact] + public void CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables() + { + const int ParameterBudget = 32; + var fileId = UpsertTestFile( + "src/caller-transaction-batches.cs", + checksum: "caller-transaction-batches"); + var chunks = Enumerable.Range(0, 7) + .Select(index => new ChunkRecord + { + FileId = fileId, + ChunkIndex = index, + StartLine = index + 1, + EndLine = index + 1, + Content = $"chunk_{index}", + }) + .ToArray(); + var symbols = Enumerable.Range(0, 3) + .Select(index => new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = $"target_{index}", + Line = index + 1, + StartLine = index + 1, + EndLine = index + 1, + }) + .ToArray(); + var issues = Enumerable.Range(0, 6) + .Select(index => new FileIssue + { + Path = "src/caller-transaction-batches.cs", + Kind = $"test_issue_{index}", + Line = index + 1, + Message = $"test issue {index}", + }) + .ToArray(); + var references = Enumerable.Range(0, 5) + .Select(index => new ReferenceRecord + { + FileId = fileId, + SymbolName = $"target_{index % symbols.Length}", + ReferenceKind = "call", + Line = index + 1, + Column = 1, + Context = $"target_{index % symbols.Length}();", + ContainerKind = "function", + ContainerName = "caller", + }) + .ToArray(); + var statements = new List(); + var previousStatementHook = DbWriter.BatchStatementExecutingForTesting; + try + { + DbWriter.BatchStatementExecutingForTesting = statement => + { + statements.Add(statement); + previousStatementHook?.Invoke(statement); + }; + + using var transaction = _writer.BeginTransaction(); + _writer.InsertChunks(chunks); + _writer.InsertSymbols(symbols); + _writer.InsertIssuesForNewFile(fileId, issues); + _writer.InsertReferencesForNewFilesInAtomicFileScope( + references, + refreshMutualRecursionFlags: false, + CancellationToken.None); + transaction.Commit(); + } + finally + { + DbWriter.BatchStatementExecutingForTesting = previousStatementHook; + } + + Assert.Equal( + [(6, 6), (1, 1)], + BatchRows("insert_chunks")); + Assert.Equal( + [(1, 1), (1, 1), (1, 1)], + BatchRows("insert_symbols")); + Assert.Equal( + [(5, 5), (1, 1)], + BatchRows("insert_issues")); + Assert.Equal( + [(5, 5)], + BatchRows("insert_reference_lines")); + Assert.Equal( + [(2, 2), (2, 2), (1, 1)], + BatchRows("insert_references")); + + var columnsByOperation = new Dictionary(StringComparer.Ordinal) + { + ["insert_chunks"] = 5, + ["insert_symbols"] = 25, + ["insert_issues"] = 6, + ["insert_reference_lines"] = 3, + ["insert_references"] = 14, + }; + Assert.All( + statements, + statement => Assert.True( + statement.StatementRows * columnsByOperation[statement.Operation] <= ParameterBudget, + $"{statement.Operation} used {statement.StatementRows * columnsByOperation[statement.Operation]} parameters.")); + + (int ActiveRows, int StatementRows)[] BatchRows(string operation) + => statements + .Where(statement => statement.Operation == operation) + .Select(statement => (statement.ActiveRows, statement.StatementRows)) + .ToArray(); + } + [Fact] public void ReferenceLineLookup_BatchedInputUsesUniqueAutoIndexPlan() { @@ -10107,19 +10219,27 @@ public void InsertReferences_AtomicFileScopeGroupsWholeBatchesWithoutMovingRefer } Assert.Equal(0, transactionCount); - Assert.Equal([0, 71, 142, 213, 284, 355, 356], progressRows); - Assert.Equal( - [(71, 71), (71, 71), (71, 71), (71, 71), (71, 71), (1, 1)], - statements.Where(statement => statement.Operation == "insert_references") - .Select(statement => (statement.ActiveRows, statement.StatementRows)) - .ToArray()); Assert.Equal( - [(284, 284), (72, 72)], - statements.Where(statement => statement.Operation == lineWriteOperation) - .Select(statement => (statement.ActiveRows, statement.StatementRows)) - .ToArray()); + Enumerable.Range(0, (ReferenceCount / 2) + 1).Select(index => index * 2), + progressRows); + var atomicReferenceStatements = statements + .Where(statement => statement.Operation == "insert_references") + .ToArray(); + Assert.Equal(ReferenceCount / 2, atomicReferenceStatements.Length); + Assert.All(atomicReferenceStatements, statement => + { + Assert.Equal((2, 2), (statement.ActiveRows, statement.StatementRows)); + Assert.True(statement.StatementRows * 14 <= 32); + }); + var atomicLineWriteStatements = statements + .Where(statement => statement.Operation == lineWriteOperation) + .ToArray(); + Assert.Equal(ReferenceCount, atomicLineWriteStatements.Sum(statement => statement.ActiveRows)); + Assert.All( + atomicLineWriteStatements, + statement => Assert.True(statement.StatementRows * 3 <= 32)); Assert.Equal( - referenceLinesAreNew ? 0 : 2, + referenceLinesAreNew ? 0 : atomicLineWriteStatements.Length, statements.Count(statement => statement.Operation == "lookup_reference_lines")); } finally @@ -10133,7 +10253,7 @@ public void InsertReferences_AtomicFileScopeGroupsWholeBatchesWithoutMovingRefer [Fact] public void InsertReferences_AtomicFileScopeCapsReferenceLineWindowAtThirtyTwoBatches() { - const int ReferenceCount = 71 * 33; + const int ReferenceCount = 2 * 33; var fileId = UpsertTestFile( "src/atomic-reference-window-cap.cs", checksum: "atomic-reference-window-cap"); @@ -10398,7 +10518,7 @@ public void InsertReferences_AtomicFileScopeCancellationRollsBackReferenceAndCon { DbWriter.BatchProgressCheckpointForTesting = progress => { - if (progress.Operation == "insert_references" && progress.RowsProcessed == 71) + if (progress.Operation == "insert_references" && progress.RowsProcessed == 2) cancellation.Cancel(); previousProgressHook?.Invoke(progress); }; diff --git a/tests/CodeIndex.Tests/ReferencePersistenceBindingTests.cs b/tests/CodeIndex.Tests/ReferencePersistenceBindingTests.cs index 0bc30438e..526ce3b2b 100644 --- a/tests/CodeIndex.Tests/ReferencePersistenceBindingTests.cs +++ b/tests/CodeIndex.Tests/ReferencePersistenceBindingTests.cs @@ -92,11 +92,22 @@ public void InsertReferences_AllPersistenceModesBindNormalizedContextByOrdinal( DbWriter.ReferenceInsertBindingWorkForTesting = previousWorkHook; } - var work = Assert.Single(observedWork); - Assert.Equal(3, work.StatementRows); - Assert.Equal(3 * 14, work.BoundParameterCount); - Assert.Equal(3, work.MaterializedReferenceCount); - Assert.Equal(2, work.MaterializedReferenceLineCount); + if (atomicFileScope) + { + Assert.Equal([2, 1], observedWork.Select(work => work.StatementRows)); + Assert.Equal([2 * 14, 14], observedWork.Select(work => work.BoundParameterCount)); + } + else + { + var work = Assert.Single(observedWork); + Assert.Equal(3, work.StatementRows); + Assert.Equal(3 * 14, work.BoundParameterCount); + } + Assert.All(observedWork, work => + { + Assert.Equal(3, work.MaterializedReferenceCount); + Assert.Equal(2, work.MaterializedReferenceLineCount); + }); using var command = _db.Connection.CreateCommand(); command.Parameters.AddWithValue("@fileId", fileId); From c4fc1972c23d29d693421e909cef0edb3ef587ea Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 22:24:21 +0900 Subject: [PATCH 02/12] Make manual performance tests explicitly runnable --- TESTING_GUIDE.md | 4 +-- .../+manual-performance-opt-in.changed.md | 14 ++++++++++ docs/test-doc-maintenance-plan.md | 4 +-- .../ManualPerformanceFactAttribute.cs | 21 ++++++++++++++ .../ManualPerformanceFactAttributeTests.cs | 28 +++++++++++++++++++ tests/CodeIndex.Tests/PerformanceTests.cs | 8 +++--- 6 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 changelog.d/unreleased/+manual-performance-opt-in.changed.md create mode 100644 tests/CodeIndex.Tests/ManualPerformanceFactAttribute.cs create mode 100644 tests/CodeIndex.Tests/ManualPerformanceFactAttributeTests.cs diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 28e6c2c44..c2fae8f89 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -713,7 +713,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result - `ConcurrencyTests.cs` Concurrent read and read-during-write scenarios (WAL mode validation), including the issue #180 bug-catching snapshot-isolation regressions for all three multi-statement reader entry points: (1) `GetStatus` seeds `refs == files * refsPerFile` and asserts every concurrent observation preserves that invariant; (2) `AnalyzeSymbol` seeds one symbol `S` plus matching reference/caller pairs, toggles a second file symmetrically, and asserts `references.Count == callers.Count` across every `inspect`/`analyze_symbol` bundle; (3) `GetRepoMap` seeds a baseline modified timestamp and toggles a newer file, asserting `latest_modified == workspace_latest_modified` across every map call. Each test fails without the DEFERRED-transaction wrap on the matching reader and passes with it. - `PerformanceTests.cs` - Bounded CI smoke coverage plus large-scale data benchmarks. `CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` and the allocation budget guards run in the default `net8.0` suite, so they are blocking PR/CI checks on the production target, but their broad budgets are intended to catch only severe indexing/search or allocation regressions rather than act as benchmarks. `ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` uses dense C# private-property receivers and Python imported-type calls to prevent per-candidate full-symbol rescans from returning. `ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` covers dense C# declaration containers and GitHub Actions jobs so name/range ownership resolution stays indexed. `Extraction_DenseDelimitedLists_StayWithinAllocationBudget` covers Python imports, YAML needs, JSON paths, and Fortran procedure lists without temporary split-array growth. `ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` keeps all-language dedupe identities value-based when qualified names are long. The 10K+ large-scale tests remain skip-by-default; run them manually with `--filter`. + Bounded CI smoke coverage plus large-scale data benchmarks. `CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` and the allocation budget guards run in the default `net8.0` suite, so they are blocking PR/CI checks on the production target, but their broad budgets are intended to catch only severe indexing/search or allocation regressions rather than act as benchmarks. `ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` uses dense C# private-property receivers and Python imported-type calls to prevent per-candidate full-symbol rescans from returning. `ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` covers dense C# declaration containers and GitHub Actions jobs so name/range ownership resolution stays indexed. `Extraction_DenseDelimitedLists_StayWithinAllocationBudget` covers Python imports, YAML needs, JSON paths, and Fortran procedure lists without temporary split-array growth. `ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` keeps all-language dedupe identities value-based when qualified names are long. Large-scale manual tests remain skip-by-default; run a selected test on the production target with `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter `. Focused authoritative-fresh fold-readiness coverage spans `DatabaseTests`, `IndexCommandRunnerTests`, and `McpServerToolsCallTests`: the built-in empty-database CLI/MCP path must consume its claim once, retain the NULL-column verification, skip the stored-value re-fold scan, and produce the same readiness/version/fingerprint/language stamps and Unicode, Markdown, C#, Nim, and TypeScript query results as full validation. Pair it with fail-closed cases for each initially nonempty ownership table (`files`, `symbols`, or `symbol_references`), a wrong or reused claim, an intervening external commit observed through `PRAGMA data_version`, rebuild/update/legacy/public-writer paths, and custom plugins, patterns, or post-extraction hooks; full validation must still reject NULL and stale non-NULL folds. A run-barrier regression must also activate a custom producer and then reload back to built-in-only before readiness: the current producer count returns to zero, but the monotonic mutation generation changes and forces full validation. Unchanged missing-directory and diagnostic-only publications must not change that generation. A deterministic cancel-after-`BEGIN IMMEDIATE` test must prove that the raw transaction is rolled back and the same writer can immediately start and commit another transaction. For performance audits, alternate identical repository-scale fresh fixtures, isolate the readiness-finalization interval, and report elapsed time plus `GC.GetAllocatedBytesForCurrentThread`; adoption requires removing row-count-proportional managed allocation without changing rows, stamps, or query results. Keep wall-clock measurements out of blocking CI assertions and remove temporary instrumentation after recording the result. @@ -1781,7 +1781,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `ConcurrencyTests.cs` 並行読み取りと書き込み中読み取りシナリオ（WALモード検証）。issue #180 の bug-catching な snapshot 隔離回帰テストを 3 つの multi-statement reader 経路について含む。(1) `GetStatus` は `refs == files * refsPerFile` の seed 不変条件を立て、並行観測が常にこの条件を維持することを要求する。(2) `AnalyzeSymbol` はシンボル `S` に対して reference/caller を対称に 1 対 1 で seed し、もう 1 ファイルを対称に toggle することで `inspect` / `analyze_symbol` bundle の `references.Count == callers.Count` を常に保証する。(3) `GetRepoMap` はベースラインの modified と新しい toggle 対象ファイルを用意し、`latest_modified == workspace_latest_modified` が常に一致することを要求する。各テストは対応する reader の DEFERRED transaction を外すと落ち、戻すと通ることを確認済み。 - `PerformanceTests.cs` - bounded な CI smoke と大規模データベンチマークを扱います。`CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` と allocation budget guard は通常の `net8.0` suite で実行されるため production target 上の PR / CI blocking check ですが、benchmark ではなく重大な indexing/search または allocation 退行だけを拾う広めの budget を使います。`ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` は密な C# private-property receiver と Python imported-type call を使い、candidate ごとの full-symbol 再走査が戻るのを防ぎます。`ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` は密な C# declaration container と GitHub Actions job を扱い、name / range ownership 解決の索引化を維持します。`Extraction_DenseDelimitedLists_StayWithinAllocationBudget` は Python import、YAML needs、JSON path、Fortran procedure list を使い、一時 split-array の増加を防ぎます。`ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` は長い qualified name でも全言語共通 dedupe identity を value-based に維持します。10K+ の大規模テストは引き続きデフォルト Skip で、`--filter` で手動実行します。 + bounded な CI smoke と大規模データベンチマークを扱います。`CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` と allocation budget guard は通常の `net8.0` suite で実行されるため production target 上の PR / CI blocking check ですが、benchmark ではなく重大な indexing/search または allocation 退行だけを拾う広めの budget を使います。`ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` は密な C# private-property receiver と Python imported-type call を使い、candidate ごとの full-symbol 再走査が戻るのを防ぎます。`ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` は密な C# declaration container と GitHub Actions job を扱い、name / range ownership 解決の索引化を維持します。`Extraction_DenseDelimitedLists_StayWithinAllocationBudget` は Python import、YAML needs、JSON path、Fortran procedure list を使い、一時 split-array の増加を防ぎます。`ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` は長い qualified name でも全言語共通 dedupe identity を value-based に維持します。大規模な手動 test は引き続きデフォルト Skip とし、production target で `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 dotnet test tests/CodeIndex.Tests/CodeIndex.Tests.csproj -f net8.0 --filter ` を実行します。 authoritative-fresh fold readiness の focused coverage は `DatabaseTests`、`IndexCommandRunnerTests`、`McpServerToolsCallTests` で分担します。built-in の empty-database CLI / MCP 経路が claim を一度だけ consume し、NULL column の検証を維持しつつ、保存 value の再 fold scan を省き、full validation と同じ readiness / version / fingerprint / language stamp、および Unicode、Markdown、C#、Nim、TypeScript の query result を生成することを固定します。初期状態で ownership table（`files`、`symbols`、`symbol_references`）のいずれかが非空の場合、owner が異なるか再利用された claim、`PRAGMA data_version` で観測される外部 commit、rebuild / update / legacy / public-writer 経路、custom plugin / pattern / post-extraction hook は fail closed であることも対にし、full validation が NULL と stale な非 NULL fold を引き続き拒否することを確認します。run barrier では custom producer を一度 active にしてから readiness 前に built-in-only へ reload し、最終 producer count が zero に戻っていても monotonic mutation generation の変化で full validation へ戻ることを固定します。状態不変の missing-directory と diagnostic-only publication では generation が変わらないことも確認します。 `BEGIN IMMEDIATE`成功直後のdeterministicなcancel testでは、raw transactionがrollbackされ、同じwriterが直後に別transactionを開始・commitできることを必須とします。 性能監査では、同一の repository-scale fresh fixture を交互に実行し、readiness finalization 区間を分離して、経過時間と `GC.GetAllocatedBytesForCurrentThread` を報告します。row 数に比例する managed allocation を取り除きつつ、row、stamp、query result が変わらないことを採用条件にします。wall-clock 計測は blocking CI assertion にせず、結果を記録したら一時 instrumentation を削除してください。 diff --git a/changelog.d/unreleased/+manual-performance-opt-in.changed.md b/changelog.d/unreleased/+manual-performance-opt-in.changed.md new file mode 100644 index 000000000..37da386e9 --- /dev/null +++ b/changelog.d/unreleased/+manual-performance-opt-in.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - tests/CodeIndex.Tests/ManualPerformanceFactAttribute.cs + - tests/CodeIndex.Tests/PerformanceTests.cs +--- + +## English + +- **Manual performance tests now have an executable opt-in contract** — setting `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1` together with a focused test filter runs the selected production-runtime benchmark, while ordinary CI still skips it. The 1,000-file search fixture is also named for its actual scale. + +## 日本語 + +- **手動performance testに実行可能なopt-in契約を追加しました** — `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1` とfocused test filterを併用すると選択したproduction-runtime benchmarkが実行され、通常CIでは引き続きskipされます。1,000-file search fixtureの名前も実際の規模に合わせました。 diff --git a/docs/test-doc-maintenance-plan.md b/docs/test-doc-maintenance-plan.md index 4ab96f796..4ae697bb6 100644 --- a/docs/test-doc-maintenance-plan.md +++ b/docs/test-doc-maintenance-plan.md @@ -70,7 +70,7 @@ candidate before changing a skip. |---|---|---| | Target-framework or platform-specific | `ProductionCliFactAttribute`, `ProductionCliTheoryAttribute`, `ExternalProcessFactAttribute`, `ExternalProcessTheoryAttribute`, and practical-budget guards that run only on the production `net8.0` target. | Keep the shared skip-reason constants and their contract tests beside the attributes. Do not duplicate literal reasons at call sites. | | External process or toolchain limitation | Published/trimmed CLI and installer paths that can be reported as skipped when SDK/ILLink/runtime availability prevents the test from reaching `cdidx` (#2586, #3571). | Keep the tracking issue in the reason or surrounding guide text, and prefer narrowing the environment guard over disabling broader coverage. | -| Performance-only or manual benchmark | `PerformanceTests` large-scale checks such as `Insert10KFiles` and extractor stress tests with manual `dotnet test --filter ...` instructions. | Keep them skipped by default, keep the command in the reason, and do not treat them as required PR gates. | +| Performance-only or manual benchmark | `PerformanceTests` large-scale checks such as `Insert10KFiles` and extractor stress tests using `ManualPerformanceFactAttribute`. | Keep them skipped by default, require `CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1` plus a focused `dotnet test --filter ...` invocation, and do not treat them as required PR gates. | | Temporary investigation skip | Skips with owner, expiration, and `blocked by #NNNN` metadata. | A temporary skip must cite a tracking issue, owner, and expiry date. If that metadata is missing, remove the skip or open the tracking issue before adding it. | | Intentionally disabled coverage | No standing class of untracked intentional disables should exist. | If coverage must remain disabled, create an issue first and use the temporary-skip format until the replacement coverage lands. | @@ -204,7 +204,7 @@ exact-substring query を使い、変更前に候補を 1 件ずつ確認して |---|---|---| | 対象フレームワークまたはプラットフォーム固有 | `ProductionCliFactAttribute`、`ProductionCliTheoryAttribute`、`ExternalProcessFactAttribute`、`ExternalProcessTheoryAttribute`、production `net8.0` target だけで走る practical-budget guard。 | 共有 skip-reason constant とその contract test を attribute の近くに置く。call site に literal reason を重複させない。 | | 外部プロセスまたはツールチェーン制約 | SDK/ILLink/runtime の可用性により `cdidx` に到達する前に skipped として報告されうる published/trimmed CLI と installer 経路（#2586、#3571）。 | reason または周辺 guide text に tracking issue を残し、広い coverage を止めるより environment guard を狭める。 | -| 性能専用または手動ベンチマーク | `PerformanceTests` の `Insert10KFiles` などの大規模チェックと、手動 `dotnet test --filter ...` 指示付き extractor stress test。 | 既定では skipped のままにし、reason に実行コマンドを残し、PR 必須 gate として扱わない。 | +| 性能専用または手動ベンチマーク | `ManualPerformanceFactAttribute` を使う `PerformanceTests` の `Insert10KFiles` などの大規模チェックと extractor stress test。 | 既定では skipped のままにし、`CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1` と focused `dotnet test --filter ...` の両方を必須とし、PR 必須 gate として扱わない。 | | 一時調査用 skip | owner、expiration、`blocked by #NNNN` metadata を持つ skip。 | temporary skip は tracking issue、owner、expiry date を必ず持つ。metadata がなければ skip を削除するか、追加前に tracking issue を起票する。 | | 意図的な無効化 | 未追跡の intentional disable を常設カテゴリとして持たない。 | coverage を無効化したままにする必要がある場合は先に issue を作り、replacement coverage が入るまで temporary-skip format を使う。 | diff --git a/tests/CodeIndex.Tests/ManualPerformanceFactAttribute.cs b/tests/CodeIndex.Tests/ManualPerformanceFactAttribute.cs new file mode 100644 index 000000000..f7c70ec18 --- /dev/null +++ b/tests/CodeIndex.Tests/ManualPerformanceFactAttribute.cs @@ -0,0 +1,21 @@ +namespace CodeIndex.Tests; + +public sealed class ManualPerformanceFactAttribute : FactAttribute +{ + internal const string EnvironmentVariable = "CDIDX_RUN_MANUAL_PERFORMANCE_TESTS"; + internal const string OptInSkipReason = + "Manual performance test. Set CDIDX_RUN_MANUAL_PERFORMANCE_TESTS=1 and select this test with --filter to run it."; + + public ManualPerformanceFactAttribute() + { +#if NET8_0 + if (!IsEnabled(Environment.GetEnvironmentVariable(EnvironmentVariable))) + Skip = OptInSkipReason; +#else + Skip = ProductionRuntimeTestTarget.SecondaryTargetSkipReason; +#endif + } + + internal static bool IsEnabled(string? value) + => string.Equals(value, "1", StringComparison.Ordinal); +} diff --git a/tests/CodeIndex.Tests/ManualPerformanceFactAttributeTests.cs b/tests/CodeIndex.Tests/ManualPerformanceFactAttributeTests.cs new file mode 100644 index 000000000..8e20855b1 --- /dev/null +++ b/tests/CodeIndex.Tests/ManualPerformanceFactAttributeTests.cs @@ -0,0 +1,28 @@ +namespace CodeIndex.Tests; + +public class ManualPerformanceFactAttributeTests +{ + [Theory] + [InlineData(null, false)] + [InlineData("", false)] + [InlineData("0", false)] + [InlineData("true", false)] + [InlineData("01", false)] + [InlineData("1", true)] + public void IsEnabled_RequiresExactExplicitOptIn(string? value, bool expected) + => Assert.Equal(expected, ManualPerformanceFactAttribute.IsEnabled(value)); + + [Fact] + public void Constructor_AppliesRuntimeAndExplicitOptInPolicy() + { + var attribute = new ManualPerformanceFactAttribute(); + +#if NET8_0 + var enabled = ManualPerformanceFactAttribute.IsEnabled( + Environment.GetEnvironmentVariable(ManualPerformanceFactAttribute.EnvironmentVariable)); + Assert.Equal(enabled ? null : ManualPerformanceFactAttribute.OptInSkipReason, attribute.Skip); +#else + Assert.Equal(ProductionRuntimeTestTarget.SecondaryTargetSkipReason, attribute.Skip); +#endif + } +} diff --git a/tests/CodeIndex.Tests/PerformanceTests.cs b/tests/CodeIndex.Tests/PerformanceTests.cs index f596cdb20..9cc0fc0ef 100644 --- a/tests/CodeIndex.Tests/PerformanceTests.cs +++ b/tests/CodeIndex.Tests/PerformanceTests.cs @@ -55,7 +55,7 @@ public void ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesC atomicFileScope: true)); } - [Fact(Skip = "Performance test — run manually with: dotnet test --filter Insert10KFiles")] + [ManualPerformanceFact] public void Insert10KFiles_CompletesInReasonableTime() { var writer = new DbWriter(_db.Connection); @@ -86,8 +86,8 @@ public void Insert10KFiles_CompletesInReasonableTime() Assert.Equal(10_000, files); } - [Fact(Skip = "Performance test — run manually with: dotnet test --filter Search10KFileIndex")] - public void Search10KFileIndex_ReturnsInReasonableTime() + [ManualPerformanceFact] + public void Search1KFileIndex_ReturnsInReasonableTime() { var writer = new DbWriter(_db.Connection); @@ -125,7 +125,7 @@ public void Search10KFileIndex_ReturnsInReasonableTime() Assert.True(results.Count > 0); } - [Fact(Skip = "Performance test — run manually with: dotnet test --filter ExtractLargeSameLineSymbolFixture_CompletesInReasonableTime")] + [ManualPerformanceFact] public void ExtractLargeSameLineSymbolFixture_CompletesInReasonableTime() { var content = string.Join( From a3c831014e4c6422259b9a0ab1a48234ef48ff18 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 22:42:17 +0900 Subject: [PATCH 03/12] Use compact lower-rank candidate guards --- DEVELOPER_GUIDE.md | 15 ++++ TESTING_GUIDE.md | 2 + .../+compact-lower-rank-candidates.changed.md | 14 ++++ .../DbWriter.ReferenceGraphRefreshScope.cs | 9 ++- src/CodeIndex/Database/DbWriter.References.cs | 47 ++++++++---- tests/CodeIndex.Tests/DatabaseTests.cs | 71 ++++++++++++++++++- 6 files changed, 143 insertions(+), 15 deletions(-) create mode 100644 changelog.d/unreleased/+compact-lower-rank-candidates.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 329f5039d..b874ae869 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -415,6 +415,14 @@ for every join candidate. Scoped refreshes must limit symbol facts to their lookup-name set and derive identity facts from that bounded population; full and retained rebuilds use the complete C# symbol-fact population. +After rank 0–4 candidate construction, graph finalization materializes the +distinct matching reference IDs into a compact `WITHOUT ROWID` TEMP table. All +language-independent and C# rank-5 fallbacks consult that set instead of probing +the physical candidate table, while the persisted row-per-symbol candidate and +ambiguity contracts remain unchanged. Scoped refreshes must build the set by +driving from dirty reference IDs into the candidate primary key, and every graph +pass must clear it before materialization so retries cannot observe stale rows. + Repository-wide incremental scans load stat-reuse candidates with one SQLite statement before the C# contract prepass and parallel extraction. Each candidate is still compared with a fresh filesystem size and UTC modification time, and @@ -4247,6 +4255,13 @@ scalar function へ再入したりせず、primary-key の fact lookup を使い lookup-name 集合だけに限定し、identity fact もその限定済み集合から作ります。full / retained rebuild は C# symbol fact の全対象を使います。 +rank 0〜4 の candidate 構築後は、一致した reference ID の distinct 集合を compact な +`WITHOUT ROWID` TEMP table に materialize します。言語共通および C# の rank 5 fallback は +巨大な物理 candidate table ではなくこの集合を参照し、永続化される symbol ごとの candidate 行と +ambiguity 契約は変更しません。scoped refresh は dirty reference ID から candidate primary key を +seek して集合を作り、retry が古い行を参照しないよう graph pass ごとに materialize 前の clear を +維持してください。 + リポジトリ全体の incremental scan は、C# contract prepass と parallel extraction の前に stat-reuse 候補を 1 回の SQLite statement で読みます。各候補は引き続き最新の filesystem size と UTC 更新時刻と照合し、language extractor version、extraction cap、古い issue metadata、 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index c2fae8f89..b56f546a9 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -748,6 +748,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. `CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` additionally fixes the atomic full-index parameter budget at 32 across language-neutral chunks (6 rows), symbols (1), issues (5), reference lines (10), and references (2), while the public reference API retains 71-row batches. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. + `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. Keep the full/scoped/retained resolution oracles beside this structural guard so physical candidate rows and multi-language ambiguity remain unchanged. - `HotspotReferenceAggregateTests.cs` Regression coverage for the maintained per-file hotspot aggregate, including legacy transactional backfill, high-cardinality limited file/name queries, aggregate/raw logical-site parity, cross-file context and identity invalidation, and cancellation after aggregate SQL begins. Query-plan assertions require the rank index on `hotspot_reference_counts`, reject raw `symbol_references` scans, and use the shared broad deterministic timeout instead of a benchmark-grade threshold. - `.github/scripts/run-dotnet-tests.ps1` @@ -1816,6 +1817,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。`CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` はさらに、言語共通のchunk（6行）、symbol（1行）、issue（5行）、reference line（10行）、reference（2行）に対するatomic full-index parameter budgetを32に固定し、public reference APIの71行batchは維持します。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 + `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。物理candidate行と多言語ambiguityが変わらないよう、この構造guardとfull / scoped / retained resolution oracleを対で維持してください。 - `HotspotReferenceAggregateTests.cs` file 単位 maintained hotspot aggregate の回帰 coverage です。legacy database の transactional backfill、高カーディナリティな limit 付き file/name query、aggregate/raw の logical-site parity、cross-file context / identity の無効化、aggregate SQL 開始後の cancellation を含みます。query-plan assertion は `hotspot_reference_counts` の rank index 利用を必須とし、raw `symbol_references` scan を拒否します。benchmark 用の厳しい閾値ではなく、共有の広い deterministic timeout を使います。 - `.github/scripts/run-dotnet-tests.ps1` diff --git a/changelog.d/unreleased/+compact-lower-rank-candidates.changed.md b/changelog.d/unreleased/+compact-lower-rank-candidates.changed.md new file mode 100644 index 000000000..ff35c13dd --- /dev/null +++ b/changelog.d/unreleased/+compact-lower-rank-candidates.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +--- + +## English + +- **Initial reference-graph finalization now uses a compact lower-rank match set** — all rank-5 fallbacks probe one TEMP row per reference that matched ranks 0–4 instead of repeatedly searching the much larger physical candidate table, while full, scoped, and retained refreshes preserve candidate and ambiguity results across languages. + +## 日本語 + +- **初回reference graph確定がcompactな下位rank一致集合を使うようになりました** — 全rank 5 fallbackは巨大な物理candidate tableを反復検索せず、rank 0〜4で一致したreferenceごとに1行のTEMP集合を参照します。full / scoped / retained refreshのcandidate・ambiguity結果は全言語で維持されます。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index d0a9059ab..8806eb775 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -227,6 +227,8 @@ private static string BuildScopedReferenceCandidatesSql() const string fullInstantiateNamePredicateSql = "AND s.name_folded IS NOT NULL"; const string fullCSharpTypeSymbolSourceSql = "FROM symbols AS type_symbol"; const string fullCSharpTypeNamePredicateSql = "AND type_symbol.name_folded IS NOT NULL"; + const string fullLowerRankCandidateSourceSql = + "FROM symbol_reference_candidates AS lower_rank_candidate"; const int expectedReferenceSourceCount = 14; if (CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullDeleteSql) != 1 @@ -235,7 +237,8 @@ private static string BuildScopedReferenceCandidatesSql() || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullInstantiateSymbolSourceSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullInstantiateNamePredicateSql) != 1 || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullCSharpTypeSymbolSourceSql) != 1 - || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullCSharpTypeNamePredicateSql) != 1) + || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullCSharpTypeNamePredicateSql) != 1 + || CountOrdinalOccurrences(RefreshReferenceCandidatesSql, fullLowerRankCandidateSourceSql) != 1) { throw new InvalidOperationException( "The reference-candidate SQL shape changed without updating the dirty-scope projection."); @@ -265,6 +268,10 @@ private static string BuildScopedReferenceCandidatesSql() .Replace( fullCSharpTypeNamePredicateSql, "AND type_lookup_name.lang = 'csharp'\n AND type_symbol.name_folded = type_lookup_name.name_folded", + StringComparison.Ordinal) + .Replace( + fullLowerRankCandidateSourceSql, + $"FROM temp.{ReferenceGraphDirtyReferencesTable} AS dirty_lower_rank\n CROSS JOIN symbol_reference_candidates AS lower_rank_candidate\n ON lower_rank_candidate.reference_id = dirty_lower_rank.reference_id", StringComparison.Ordinal); } diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index dce21eeea..e8b4627ce 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -10,6 +10,8 @@ public partial class DbWriter private const string NonIdentifierReceiverQualifier = NonTypeReceiverQualifierPrefix + "\u001fqualified"; private const int MaxReferenceLineWindowBatchCount = 32; + private const string ReferenceLowerRankCandidateMatchesTable = + "reference_lower_rank_candidate_matches"; private const string MutualRecursionValueSql = """ CASE @@ -110,7 +112,7 @@ UPDATE symbol_references AS r WHERE r.source_symbol_id IS NOT {ReferenceSourceSymbolValueSql} """; - private const string CreateReferenceUniqueFamiliesSql = """ + private static readonly string CreateReferenceUniqueFamiliesSql = $""" CREATE TEMP TABLE IF NOT EXISTS reference_unique_symbol_families ( lang TEXT NOT NULL, name_folded TEXT NOT NULL, @@ -155,6 +157,10 @@ CREATE TEMP TABLE IF NOT EXISTS csharp_constructor_identity_facts ( symbol_id INTEGER NOT NULL PRIMARY KEY, type_identity TEXT COLLATE BINARY, type_arity INTEGER + ) WITHOUT ROWID; + + CREATE TEMP TABLE IF NOT EXISTS {ReferenceLowerRankCandidateMatchesTable} ( + reference_id INTEGER NOT NULL PRIMARY KEY ) WITHOUT ROWID """; @@ -1156,6 +1162,19 @@ SELECT 1 FROM symbol_reference_candidates AS existing WHERE existing.reference_id = r.id ); + -- Rank-5 fallbacks only need to know whether a lower rank matched. Keep that + -- one-row-per-reference fact compact instead of probing the much larger + -- physical-candidate table once for every fallback candidate. + -- rank 5 fallbackが必要とするのは下位rankの一致有無だけであるため、各fallback + -- candidateから巨大な物理candidate表を参照せず、referenceごと1行の集合に縮約する。 + DELETE FROM temp.{ReferenceLowerRankCandidateMatchesTable}; + + INSERT INTO temp.{ReferenceLowerRankCandidateMatchesTable}(reference_id) + SELECT lower_rank_candidate.reference_id + FROM symbol_reference_candidates AS lower_rank_candidate + WHERE lower_rank_candidate.scope_rank < 5 + GROUP BY lower_rank_candidate.reference_id; + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) SELECT r.id, unique_target.symbol_id, 5 FROM symbol_references AS r @@ -1198,8 +1217,9 @@ OR unique_target.type_arity = {CSharpReferenceTypeAritySql} ) AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id + SELECT 1 + FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match + WHERE lower_rank_match.reference_id = r.id ); INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) @@ -1240,8 +1260,9 @@ OR source_file.lang IN ( ) ) AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id + SELECT 1 + FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match + WHERE lower_rank_match.reference_id = r.id ) AND (source_file.lang <> 'dependency_lock' OR target.file_id = r.file_id); @@ -1263,8 +1284,9 @@ JOIN files AS target_file AND r.target_qualifier IS NULL AND r.reference_kind NOT IN ('instantiate', 'type_reference') AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id + SELECT 1 + FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match + WHERE lower_rank_match.reference_id = r.id ); INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) @@ -1285,9 +1307,9 @@ JOIN files AS target_file AND r.target_qualifier IS NULL AND r.reference_kind = 'attribute' AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id - AND existing.scope_rank < 5 + SELECT 1 + FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match + WHERE lower_rank_match.reference_id = r.id ); INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) @@ -1443,8 +1465,9 @@ explicit_zero_constructor.name COLLATE BINARY ) ) AND NOT EXISTS ( - SELECT 1 FROM symbol_reference_candidates AS existing - WHERE existing.reference_id = r.id + SELECT 1 + FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match + WHERE lower_rank_match.reference_id = r.id ); """; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 3ed8f1b4c..7b9fa6478 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1046,6 +1046,39 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() using var scope = _writer.BeginReferenceGraphRefreshScope(); _writer.RefreshMutualRecursionFlags(); + var fullCandidateSql = Assert.Single( + DbWriter.CSharpGraphCandidateSqlForTesting, + static entry => entry.Scope == "full").Sql; + Assert.Equal( + 1, + fullCandidateSql.Split( + "FROM symbol_reference_candidates AS lower_rank_candidate", + StringSplitOptions.None).Length - 1); + Assert.Equal( + 5, + fullCandidateSql.Split( + "FROM temp.reference_lower_rank_candidate_matches AS lower_rank_match", + StringSplitOptions.None).Length - 1); + var fullCandidateStatements = fullCandidateSql + .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var fullMaterializationIndex = Array.FindIndex(fullCandidateStatements, static statement => + statement.StartsWith( + "INSERT INTO temp.reference_lower_rank_candidate_matches", + StringComparison.Ordinal)); + Assert.True(fullMaterializationIndex >= 0); + Assert.Equal( + 9, + fullCandidateStatements[..fullMaterializationIndex].Count(static statement => + statement.StartsWith( + "INSERT INTO symbol_reference_candidates", + StringComparison.Ordinal))); + Assert.Equal( + 5, + fullCandidateStatements[(fullMaterializationIndex + 1)..].Count(static statement => + statement.StartsWith( + "INSERT INTO symbol_reference_candidates", + StringComparison.Ordinal))); + var candidateSql = DbWriter.RefreshScopedReferenceCandidatesSqlForTesting; Assert.DoesNotContain("AND s.name_folded IS NOT NULL", candidateSql, StringComparison.Ordinal); Assert.Contains( @@ -1065,9 +1098,35 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() candidateSql.Split( "FROM temp.reference_graph_dirty_references AS dirty_reference", StringSplitOptions.None).Length - 1); + Assert.Equal( + 5, + candidateSql.Split( + "FROM temp.reference_lower_rank_candidate_matches AS lower_rank_match", + StringSplitOptions.None).Length - 1); - var candidateInserts = candidateSql - .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + var candidateStatements = candidateSql + .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + var lowerRankMaterialization = Assert.Single(candidateStatements.Where(static statement => + statement.StartsWith( + "INSERT INTO temp.reference_lower_rank_candidate_matches", + StringComparison.Ordinal))); + Assert.Contains( + "FROM temp.reference_graph_dirty_references AS dirty_lower_rank", + lowerRankMaterialization, + StringComparison.Ordinal); + Assert.Contains( + "CROSS JOIN symbol_reference_candidates AS lower_rank_candidate", + lowerRankMaterialization, + StringComparison.Ordinal); + var lowerRankPlan = ReadQueryPlanDetails(_db.Connection, lowerRankMaterialization); + Assert.Contains(lowerRankPlan, static detail => detail.Contains( + "SEARCH lower_rank_candidate USING INDEX", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(lowerRankPlan, static detail => + detail.Equals("SCAN lower_rank_candidate", StringComparison.OrdinalIgnoreCase) + || detail.StartsWith("SCAN lower_rank_candidate ", StringComparison.OrdinalIgnoreCase)); + + var candidateInserts = candidateStatements .Where(static statement => statement.StartsWith( "INSERT INTO symbol_reference_candidates", StringComparison.Ordinal)) @@ -1084,6 +1143,14 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() Assert.DoesNotContain(plan, static detail => detail.Equals("SCAN r", StringComparison.OrdinalIgnoreCase) || detail.StartsWith("SCAN r ", StringComparison.OrdinalIgnoreCase)); + if (statement.Contains( + "FROM temp.reference_lower_rank_candidate_matches AS lower_rank_match", + StringComparison.Ordinal)) + { + Assert.Contains(plan, static detail => detail.Contains( + "SEARCH lower_rank_match USING PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + } } Assert.Contains(candidatePlans, static detail => detail.Contains( "SEARCH type_identity_fact USING PRIMARY KEY", From bdee187a737e11b742d6893aa722b173b7f6dc3f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 22:57:00 +0900 Subject: [PATCH 04/12] Materialize reference resolution target keys --- DEVELOPER_GUIDE.md | 15 +++ TESTING_GUIDE.md | 2 + ...ference-resolution-symbol-facts.changed.md | 14 +++ .../DbWriter.ReferenceGraphRefreshScope.cs | 31 +++++ src/CodeIndex/Database/DbWriter.References.cs | 62 +++++++--- tests/CodeIndex.Tests/DatabaseTests.cs | 33 ++++++ .../FreshReferenceResolutionTests.cs | 106 ++++++++++++++++++ 7 files changed, 245 insertions(+), 18 deletions(-) create mode 100644 changelog.d/unreleased/+reference-resolution-symbol-facts.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b874ae869..f07c4d42f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -423,6 +423,14 @@ ambiguity contracts remain unchanged. Scoped refreshes must build the set by driving from dirty reference IDs into the candidate primary key, and every graph pass must clear it before materialization so retries cannot observe stale rows. +Resolution also materializes the nullable target-family key once per target symbol +into a primary-keyed TEMP fact table. Full, fresh, differential, and retained +refreshes populate all symbols; scoped refreshes first deduplicate target symbol +IDs reachable from dirty-reference candidates. Resolution must join candidates to +that fact by symbol ID instead of rebuilding the language/path/container/name key +for every physical candidate. Preserve a `NULL` key when legacy target language is +missing, while still resolving a single valid candidate by ID. + Repository-wide incremental scans load stat-reuse candidates with one SQLite statement before the C# contract prepass and parallel extraction. Each candidate is still compared with a fresh filesystem size and UTC modification time, and @@ -4262,6 +4270,13 @@ ambiguity 契約は変更しません。scoped refresh は dirty reference ID seek して集合を作り、retry が古い行を参照しないよう graph pass ごとに materialize 前の clear を 維持してください。 +resolution は nullable な target-family key も target symbol ごとに1回だけ primary-keyed TEMP +fact table へ materialize します。full / fresh / differential / retained refresh は全 symbol を投入し、 +scoped refresh は dirty-reference candidate から到達する target symbol ID を先に重複排除します。 +resolution は物理 candidate ごとに language / path / container / name key を再構築せず、symbol IDで +このfactへjoinしてください。legacy targetのlanguageが欠ける場合はkeyを`NULL`のまま保ちつつ、 +有効candidateが1件ならIDによるresolved状態を維持します。 + リポジトリ全体の incremental scan は、C# contract prepass と parallel extraction の前に stat-reuse 候補を 1 回の SQLite statement で読みます。各候補は引き続き最新の filesystem size と UTC 更新時刻と照合し、language extractor version、extraction cap、古い issue metadata、 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index b56f546a9..82896cd06 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -749,6 +749,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. Keep the full/scoped/retained resolution oracles beside this structural guard so physical candidate rows and multi-language ambiguity remain unchanged. + `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` fixes target-family key construction to one per target symbol and candidate resolution to TEMP primary-key facts across fresh, full, differential, scoped, and retained paths. Its legacy-null-key and C#/Python oracle coverage preserves resolved IDs, exact keys, grouped families, ambiguity, and self-reference semantics. - `HotspotReferenceAggregateTests.cs` Regression coverage for the maintained per-file hotspot aggregate, including legacy transactional backfill, high-cardinality limited file/name queries, aggregate/raw logical-site parity, cross-file context and identity invalidation, and cancellation after aggregate SQL begins. Query-plan assertions require the rank index on `hotspot_reference_counts`, reject raw `symbol_references` scans, and use the shared broad deterministic timeout instead of a benchmark-grade threshold. - `.github/scripts/run-dotnet-tests.ps1` @@ -1818,6 +1819,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。物理candidate行と多言語ambiguityが変わらないよう、この構造guardとfull / scoped / retained resolution oracleを対で維持してください。 + `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` は、target-family key構築をtarget symbolごと1回に限定し、fresh / full / differential / scoped / retainedのcandidate resolutionがTEMP primary-key factを使う契約を固定します。legacy null-keyとC# / Python oracle coverageにより、resolved ID、exact key、group family、ambiguity、self-reference semanticsを維持します。 - `HotspotReferenceAggregateTests.cs` file 単位 maintained hotspot aggregate の回帰 coverage です。legacy database の transactional backfill、高カーディナリティな limit 付き file/name query、aggregate/raw の logical-site parity、cross-file context / identity の無効化、aggregate SQL 開始後の cancellation を含みます。query-plan assertion は `hotspot_reference_counts` の rank index 利用を必須とし、raw `symbol_references` scan を拒否します。benchmark 用の厳しい閾値ではなく、共有の広い deterministic timeout を使います。 - `.github/scripts/run-dotnet-tests.ps1` diff --git a/changelog.d/unreleased/+reference-resolution-symbol-facts.changed.md b/changelog.d/unreleased/+reference-resolution-symbol-facts.changed.md new file mode 100644 index 000000000..70bcda78d --- /dev/null +++ b/changelog.d/unreleased/+reference-resolution-symbol-facts.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +--- + +## English + +- **Initial reference resolution now constructs target-family keys once per symbol** — full, fresh, differential, scoped, and retained graph refreshes resolve candidate rows through primary-keyed TEMP facts instead of rebuilding long language/path/container/name keys for every physical candidate, with unchanged cross-language resolution states. + +## 日本語 + +- **初回reference resolutionがtarget-family keyをsymbolごとに1回だけ構築するようになりました** — full / fresh / differential / scoped / retained graph refreshは、物理candidateごとに長いlanguage / path / container / name keyを再構築せずprimary-keyed TEMP factを通して解決し、言語横断のresolution stateを維持します。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 8806eb775..dadea7902 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -107,6 +107,24 @@ HAVING COUNT(DISTINCT target_file.path || char(31) || private static readonly string RefreshScopedReferenceCandidatesSql = BuildScopedReferenceCandidatesSql(); + private static readonly string RefreshScopedReferenceResolutionSymbolFactsSql = $""" + DELETE FROM temp.{ReferenceResolutionSymbolFactsTable}; + + WITH dirty_target_symbols(symbol_id) AS MATERIALIZED ( + SELECT candidate.symbol_id + FROM temp.{ReferenceGraphDirtyReferencesTable} AS dirty_target_reference + CROSS JOIN symbol_reference_candidates AS candidate + ON candidate.reference_id = dirty_target_reference.reference_id + GROUP BY candidate.symbol_id + ) + INSERT INTO temp.{ReferenceResolutionSymbolFactsTable}(symbol_id, target_key) + SELECT target.id, + {ReferenceResolutionTargetKeySql} + FROM dirty_target_symbols AS dirty_target + JOIN symbols AS target ON target.id = dirty_target.symbol_id + JOIN files AS target_file ON target_file.id = target.file_id; + """; + private static readonly string RefreshScopedReferenceResolutionValuesSql = $""" UPDATE symbol_references AS r SET (target_symbol_id, target_symbol_key, resolution_candidate_count, resolution_state) = {ReferenceResolutionValueSql} @@ -144,6 +162,19 @@ FROM temp.{ReferenceGraphDirtyReferencesTable} internal static string RefreshScopedReferenceCandidatesSqlForTesting => RefreshScopedReferenceCandidatesSql; + internal static IReadOnlyList<( + string Scope, + string MaterializationSql, + string ResolutionSql)> ReferenceResolutionFactSqlForTesting + => + [ + ("fresh", RefreshReferenceResolutionSymbolFactsFullSql, RefreshReferenceResolutionFreshSparseSql), + ("full", RefreshReferenceResolutionSymbolFactsFullSql, RefreshReferenceResolutionFullSql), + ("differential", RefreshReferenceResolutionSymbolFactsFullSql, RefreshReferenceResolutionDifferentialSql), + ("scoped", RefreshScopedReferenceResolutionSymbolFactsSql, RefreshScopedReferenceResolutionSql), + ("retained", RefreshReferenceResolutionSymbolFactsFullSql, RefreshReferenceResolutionFullSql), + ]; + internal static IReadOnlyList<(string Scope, string Sql)> CSharpGraphFactEvaluationSqlForTesting => diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index e8b4627ce..3ecba765d 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -12,6 +12,21 @@ public partial class DbWriter private const int MaxReferenceLineWindowBatchCount = 32; private const string ReferenceLowerRankCandidateMatchesTable = "reference_lower_rank_candidate_matches"; + private const string ReferenceResolutionSymbolFactsTable = + "reference_resolution_symbol_facts"; + + private const string ReferenceResolutionTargetKeySql = """ + target_file.lang || char(31) || target_file.path || char(31) || + COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || + COALESCE(target.name, '') + """; + + private static readonly string CreateReferenceResolutionSymbolFactsTableSql = $""" + CREATE TEMP TABLE IF NOT EXISTS {ReferenceResolutionSymbolFactsTable} ( + symbol_id INTEGER NOT NULL PRIMARY KEY, + target_key TEXT COLLATE BINARY + ) WITHOUT ROWID + """; private const string MutualRecursionValueSql = """ CASE @@ -159,11 +174,23 @@ CREATE TEMP TABLE IF NOT EXISTS csharp_constructor_identity_facts ( type_arity INTEGER ) WITHOUT ROWID; + {CreateReferenceResolutionSymbolFactsTableSql}; + CREATE TEMP TABLE IF NOT EXISTS {ReferenceLowerRankCandidateMatchesTable} ( reference_id INTEGER NOT NULL PRIMARY KEY ) WITHOUT ROWID """; + private static readonly string RefreshReferenceResolutionSymbolFactsFullSql = $""" + DELETE FROM temp.{ReferenceResolutionSymbolFactsTable}; + + INSERT INTO temp.{ReferenceResolutionSymbolFactsTable}(symbol_id, target_key) + SELECT target.id, + {ReferenceResolutionTargetKeySql} + FROM symbols AS target + JOIN files AS target_file ON target_file.id = target.file_id; + """; + private static string BuildRefreshCSharpReferenceFactsSql(string scopePredicate) => $""" DELETE FROM temp.csharp_reference_facts; @@ -1486,15 +1513,11 @@ FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match FROM ( SELECT COUNT(*) AS candidate_count, MIN(c.symbol_id) AS minimum_symbol_id, - COUNT(DISTINCT target_file.lang || char(31) || target_file.path || char(31) || - COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || - COALESCE(target.name, '')) AS target_family_count, - MIN(target_file.lang || char(31) || target_file.path || char(31) || - COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || - COALESCE(target.name, '')) AS minimum_target_key + COUNT(DISTINCT target_fact.target_key) AS target_family_count, + MIN(target_fact.target_key) AS minimum_target_key FROM symbol_reference_candidates AS c - JOIN symbols AS target ON target.id = c.symbol_id - JOIN files AS target_file ON target_file.id = target.file_id + JOIN temp.reference_resolution_symbol_facts AS target_fact + ON target_fact.symbol_id = c.symbol_id WHERE c.reference_id = r.id ) AS resolution ) @@ -1518,7 +1541,9 @@ UPDATE symbol_references """; internal static string RefreshReferenceResolutionFullSqlForTesting - => RefreshReferenceResolutionFullSql; + => CreateReferenceResolutionSymbolFactsTableSql + ";\n" + + RefreshReferenceResolutionSymbolFactsFullSql + "\n" + + RefreshReferenceResolutionFullSql; private static readonly string RefreshReferenceResolutionDifferentialSql = $""" UPDATE symbol_references AS r @@ -1539,15 +1564,11 @@ WITH resolution_facts AS MATERIALIZED ( SELECT candidate.reference_id, COUNT(*) AS candidate_count, MIN(candidate.symbol_id) AS minimum_symbol_id, - COUNT(DISTINCT target_file.lang || char(31) || target_file.path || char(31) || - COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || - COALESCE(target.name, '')) AS target_family_count, - MIN(target_file.lang || char(31) || target_file.path || char(31) || - COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || - COALESCE(target.name, '')) AS minimum_target_key + COUNT(DISTINCT target_fact.target_key) AS target_family_count, + MIN(target_fact.target_key) AS minimum_target_key FROM symbol_reference_candidates AS candidate - JOIN symbols AS target ON target.id = candidate.symbol_id - JOIN files AS target_file ON target_file.id = target.file_id + JOIN temp.{ReferenceResolutionSymbolFactsTable} AS target_fact + ON target_fact.symbol_id = candidate.symbol_id GROUP BY candidate.reference_id ) UPDATE symbol_references AS r @@ -1578,7 +1599,9 @@ FROM resolution_facts AS resolution """; internal static string RefreshReferenceResolutionFreshSparseSqlForTesting - => RefreshReferenceResolutionFreshSparseSql; + => CreateReferenceResolutionSymbolFactsTableSql + ";\n" + + RefreshReferenceResolutionSymbolFactsFullSql + "\n" + + RefreshReferenceResolutionFreshSparseSql; private static readonly string RefreshMutualRecursionFlagsSql = $""" WITH desired_mutual_recursion(id, desired_value) AS MATERIALIZED ( @@ -1631,6 +1654,7 @@ internal static void RebuildRetainedReferenceGraph( NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + RefreshReferenceCandidatesSql + "\n" + + RefreshReferenceResolutionSymbolFactsFullSql + "\n" + RefreshReferenceResolutionFullSql + "\n" + RefreshMutualRecursionFlagsSql; using var cancellationRegistration = cancellationToken.Register(command.Cancel); @@ -2454,6 +2478,7 @@ internal void RefreshMutualRecursionFlags( NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + RefreshReferenceCandidatesSql + "\n" + + RefreshReferenceResolutionSymbolFactsFullSql + "\n" + refreshReferenceResolutionSql + "\n"; } else @@ -2467,6 +2492,7 @@ internal void RefreshMutualRecursionFlags( NormalizeCSharpPropertyReceiverReferencesScopedSql + "\n" + RefreshScopedReferenceUniqueFamiliesSql + "\n" + RefreshScopedReferenceCandidatesSql + "\n" + + RefreshScopedReferenceResolutionSymbolFactsSql + "\n" + RefreshScopedReferenceResolutionSql + "\n" + ExpandReferenceGraphNewMutualScopeSql + "\n"; } diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 7b9fa6478..ecc832aa6 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1191,6 +1191,31 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() detail.Equals("SCAN type_symbol", StringComparison.OrdinalIgnoreCase) || detail.StartsWith("SCAN type_symbol ", StringComparison.OrdinalIgnoreCase)); + var scopedResolutionFacts = Assert.Single( + DbWriter.ReferenceResolutionFactSqlForTesting, + static entry => entry.Scope == "scoped"); + var scopedResolutionFactInsert = Assert.Single( + scopedResolutionFacts.MaterializationSql + .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(static statement => statement.StartsWith( + "WITH dirty_target_symbols", + StringComparison.Ordinal))); + var scopedResolutionFactPlan = ReadQueryPlanDetails( + _db.Connection, + scopedResolutionFactInsert); + Assert.Contains(scopedResolutionFactPlan, static detail => detail.Contains( + "SEARCH candidate USING COVERING INDEX", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains(scopedResolutionFactPlan, static detail => detail.Contains( + "SEARCH target USING INTEGER PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains(scopedResolutionFactPlan, static detail => detail.Contains( + "SEARCH target_file USING INTEGER PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + Assert.DoesNotContain(scopedResolutionFactPlan, static detail => + detail.Equals("SCAN candidate", StringComparison.OrdinalIgnoreCase) + || detail.StartsWith("SCAN candidate ", StringComparison.OrdinalIgnoreCase)); + foreach (var statement in DbWriter.ScopedReferenceGraphUpdateStatementsForTesting) { var plan = ReadQueryPlanDetails(_db.Connection, statement); @@ -1201,6 +1226,14 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() detail.Equals("SCAN r", StringComparison.OrdinalIgnoreCase) || detail.StartsWith("SCAN r ", StringComparison.OrdinalIgnoreCase)); } + var scopedResolutionPlan = ReadQueryPlanDetails( + _db.Connection, + scopedResolutionFacts.ResolutionSql.Split( + ';', + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)[0]); + Assert.Contains(scopedResolutionPlan, static detail => detail.Contains( + "SEARCH target_fact USING PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); } [Fact] diff --git a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs index f118239a5..6f2c3a6ac 100644 --- a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs +++ b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs @@ -144,6 +144,93 @@ public void FreshResolutionSql_MaterializesCandidateFactsWithoutOuterReferenceSc Assert.Equal(1, CountOccurrences(sql, "UPDATE symbol_references AS r")); } + [Fact] + public void ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope() + { + foreach (var (scope, materializationSql, resolutionSql) in + DbWriter.ReferenceResolutionFactSqlForTesting) + { + Assert.Equal( + 1, + CountOccurrences( + materializationSql, + "target_file.lang || char(31) || target_file.path")); + Assert.Contains( + "INSERT INTO temp.reference_resolution_symbol_facts", + materializationSql, + StringComparison.Ordinal); + Assert.DoesNotContain( + "target_file.lang || char(31) || target_file.path", + resolutionSql, + StringComparison.Ordinal); + Assert.Contains( + "JOIN temp.reference_resolution_symbol_facts AS target_fact", + resolutionSql, + StringComparison.Ordinal); + Assert.Equal( + scope is "differential" or "scoped" ? 2 : 1, + CountOccurrences(resolutionSql, "JOIN temp.reference_resolution_symbol_facts AS target_fact")); + } + + var scoped = Assert.Single( + DbWriter.ReferenceResolutionFactSqlForTesting, + static entry => entry.Scope == "scoped"); + Assert.Contains( + "WITH dirty_target_symbols(symbol_id) AS MATERIALIZED", + scoped.MaterializationSql, + StringComparison.Ordinal); + Assert.Contains( + "FROM temp.reference_graph_dirty_references AS dirty_target_reference", + scoped.MaterializationSql, + StringComparison.Ordinal); + Assert.Contains("GROUP BY candidate.symbol_id", scoped.MaterializationSql, StringComparison.Ordinal); + + foreach (var allSymbolsScope in new[] { "fresh", "full", "differential", "retained" }) + { + var materialization = Assert.Single( + DbWriter.ReferenceResolutionFactSqlForTesting, + entry => entry.Scope == allSymbolsScope).MaterializationSql; + Assert.Contains("FROM symbols AS target", materialization, StringComparison.Ordinal); + Assert.DoesNotContain("dirty_target_symbols", materialization, StringComparison.Ordinal); + } + } + + [Fact] + public void ReferenceResolutionFacts_PreserveResolvedLegacyCandidateWithNullTargetKey() + { + var callerFileId = InsertFile("src/legacy-caller.py", "python"); + var targetFileId = InsertFile("src/legacy-target.py", "python"); + _writer.InsertSymbols([CreateSymbol(targetFileId, "LegacyTarget", line: 1)]); + _writer.InsertReferences( + [CreateReference(callerFileId, "LegacyTarget", line: 10)], + refreshMutualRecursionFlags: false); + + Execute($""" + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) + SELECT reference.id, target.id, 0 + FROM symbol_references AS reference + CROSS JOIN symbols AS target + WHERE reference.symbol_name = 'LegacyTarget' + AND target.name = 'LegacyTarget'; + + UPDATE files SET lang = NULL WHERE id = {targetFileId}; + """); + + Execute(DbWriter.RefreshReferenceResolutionFullSqlForTesting); + + Assert.Equal( + 1, + ScalarLong(""" + SELECT COUNT(*) + FROM symbol_references + WHERE symbol_name = 'LegacyTarget' + AND resolution_state = 'resolved' + AND resolution_candidate_count = 1 + AND target_symbol_id IS NOT NULL + AND target_symbol_key IS NULL + """)); + } + [Fact] public void FreshResolutionSql_MatchesFullOracleForAllStatesAcrossCSharpAndPython() { @@ -238,6 +325,15 @@ INSERT INTO observed_fresh_resolution_updates(reference_id) Assert.Equal( new ResolutionRow("resolved", 1, HasTargetId: true, HasTargetKey: true, IsSelf: false), ReadResolutionRow("src/caller.cs", 10)); + Assert.Equal( + "csharp\u001fsrc/target.cs\u001f\u001fCsTarget", + ScalarString(""" + SELECT reference.target_symbol_key + FROM symbol_references AS reference + JOIN files AS file ON file.id = reference.file_id + WHERE file.path = 'src/caller.cs' + AND reference.line = 10 + """)); Assert.Equal( new ResolutionRow("unresolved", 0, HasTargetId: false, HasTargetKey: false, IsSelf: false), ReadResolutionRow("src/caller.cs", 11)); @@ -444,6 +540,16 @@ private long ScalarLong(string sql) return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture); } + private string? ScalarString(string sql) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = sql; + var value = command.ExecuteScalar(); + return value == null || value == DBNull.Value + ? null + : Convert.ToString(value, CultureInfo.InvariantCulture); + } + private static int CountOccurrences(string text, string value) { var count = 0; From e2e6bc64727b6b078f174b424ad25fb635101ff9 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 23:06:48 +0900 Subject: [PATCH 05/12] Avoid flattened C# prepass workspaces --- DEVELOPER_GUIDE.md | 8 +- TESTING_GUIDE.md | 2 + ...arp-prepass-segmented-workspace.changed.md | 13 ++ .../Indexer/CSharpStaticInterfacePrepass.cs | 161 +++++++++++++++--- .../CSharpPrepassSymbolArtifactCacheTests.cs | 74 ++++++++ tests/CodeIndex.Tests/PerformanceTests.cs | 45 +++++ 6 files changed, 282 insertions(+), 21 deletions(-) create mode 100644 changelog.d/unreleased/+csharp-prepass-segmented-workspace.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index f07c4d42f..e97c6bd61 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -456,7 +456,9 @@ artifact once only after that checksum matches. Generic cache admission keeps deep-clone isolation for direct callers. The fresh-index production path instead materializes both workspace lookup snapshots first, then transfers ownership of each admitted per-file symbol list and releases the redundant workspace-symbol -fallback list. Main-pass mutation therefore remains isolated from the lookup +fallback list. It enumerates those per-file lists through a non-owning segmented +view while building the snapshots, so neither a transient flattened list nor a +second workspace-sized pointer buffer is required. Main-pass mutation therefore remains isolated from the lookup snapshots without retaining duplicate `SymbolRecord` objects. Artifact-producing extraction receives the main pass's absolute file path and project root so file-local family identities stay identical. File ID assignment, family scope, @@ -4296,7 +4298,9 @@ pass は引き続き authoritative な content read と hook、stat snapshot / T 実行します。正規化 path の artifact は checksum 一致後に1回だけ取り出します。汎用 cache admission は direct caller 向けの deep-clone isolation を維持します。fresh-index の production 経路では、先に2種類の workspace lookup snapshot を materialize し、その後で admit した file ごとの symbol list の所有権を -cache へ移し、重複する workspace-symbol fallback list を解放します。これにより main-pass mutation と +cache へ移し、重複する workspace-symbol fallback list を解放します。snapshot 構築中は file ごとの +list を non-owning な segmented view で列挙し、一時的な flattened list と workspace 規模の2つ目の +pointer bufferを作りません。これにより main-pass mutation と lookup snapshot の分離を保ったまま、重複する `SymbolRecord` object を保持しません。artifact を生成する extraction には main pass と同じ absolute file path / project root を渡し、file-local family identity を 一致させてください。FileId、family scope、source observation、post-extraction hook、kind filter、cap、line 検証、 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 82896cd06..2eed6602b 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -462,6 +462,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result assertions, or repository-wide extraction to this test; measure cold-index performance externally and pair it with natural-key database parity. - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` builds 20,000 unrelated generic interfaces around one contract and keeps lookup allocation below 64 KiB. Discover contract containers before parsing generic declarations, while the paired `ReferenceExtractorTests` case preserves declaration/member ordering, partial-interface last-write behavior, and contract member order. +- `CSharpPrepassSymbolArtifactCacheTests.CSharpWorkspaceAssembly_PreservesOrderIdentityAndEvidence` and `PerformanceTests.CSharpPrepassWorkspaceSegments_AvoidFlattenedReferenceBuffers` keep prepass workspace assembly in existing-row/candidate/file-symbol order without transient flattened buffers. The allocation guard enumerates the full 131,072-symbol cache ceiling below 16 KiB; preserve the non-owning view until both immutable lookups exist and transfer owned lists only afterward. - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` keeps a 1,200-statement, regex-and-brace-heavy source at one emitted symbol and pins current-thread allocations below 6.0 MB for JavaScript, 8.2 MB for TypeScript, and 14.1 MB combined. This is a physical-pass contract: do not restore a second full-file sanitizer for private-scope analysis. Pair allocation changes with the JavaScript/TypeScript literal-and-comment scope theory so templates, comments, and regex braces cannot create or leak private classes. - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` keeps persisted C# contract-member candidates and the pending-path contract flag on one `files(lang)` → `symbols(file_id, kind)` row pass. Derive both results from that reader; do not restore a second all-contract query for excluded paths. Interface declarations may be loaded only for exact retained contract container names through bounded `symbols(name)` batches, and those dynamic tail shapes must stay out of the prepared cache. `DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` pins both index plans, while `LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` requires negative/LIKE-decoy-only reads to execute no interface phase and keeps unrelated plain interfaces out of managed materialization. `PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` also splits a 503-path language lookup at 500 parameters, returns only persisted C# paths plus sorted purge-plan IDs/bytes, keeps both SQL tail shapes out of the prepared cache, and stops before the second batch when cancellation is requested. - `IndexCommandRunnerTests.Run_UpdateFiles_CsharpContractPreflightAvoidsRedundantWorkspacePasses` keeps a false source-evidence marker authoritative, leaves the repository-wide persisted-member preflight count at zero, and defers candidate reads plus workspace materialization to the single expanded C# pass. Its plain-interface, newly-added-contract, and persisted-contract phases pin the respective one scoped pass, discovery-plus-expanded passes, and one expanded pass. The paired full-scan/MCP known-evidence no-op fixtures require zero source loads, raw prepasses, persisted C# symbol loads, and lookup builds after the initial true or false snapshot. The shortcut must be rejected when explicit `index_completeness=complete` or GraphReady is absent, when symbols-only/filter/version/root/hotspot contracts drift, or when a persisted C# path changes language. Their final-stat mutation fixtures require one complete raw prepass plus every C# content load, and the explicit/shared-DB root-switch fixtures reject cross-root stat reuse. @@ -1518,6 +1519,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `Run_FullScan_PostPrepassCsharpContractLeavesReadinessPartialUntilCleanRetry` は fresh、rebuild、incremental-existing の各 route で full-scan extraction state が単調に維持されることを固定します。Python を C# より先に処理する fixture で、後段の C# workspace snapshot drift 後も先行 raw chunk が保存され、standard / trigram FTS の両方から検索でき、bulk-load optimization が厳密に1回であることを証明してください。 - `Run_FullScan_FatalParallelResultKeepsWorkerResourcesAliveUntilPeersStop` は一方の C# symbols worker を block し、peer に fatal extraction stall を返させます。command が速やかに戻ること、block 中の peer を release する前に worker completion と artifact-cache clear のどちらも起きないことを assertion し、process-wide hook の復元や fixture 削除の前に両 cleanup signal を待ってください。 - `CSharpPrepassSymbolArtifactCacheTests`、`FileIndexerTests`、CLI/MCP の fresh-index fixture は bounded prepass artifact reuse を固定します。汎用 admission の deep-clone 独立性を維持し、production の owned-list admission は原子的な publish 成功後だけ list / symbol identity を保持し、reject または cancel された input は caller-owned のままにしてください。2種類の lookup snapshot を materialize した後だけ workspace fallback symbol を解放し、lookup parity と mutation isolation を保ちます。checksum 一致時の take-once、不一致時の消費、file / symbol / estimated-byte cap の原子性、partial admission を残さない cancellation、bounded-regex timeout 後の partial symbol をadmitしない契約も維持してください。encoding theory は UTF-8、UTF-16 LE/BE、不正 UTF-8 の prepass checksum を authoritative loader と比較します。integration coverage では空 database の非 rebuild full index だけが再利用し、rebuild / symbols-only / existing / incomplete-or-stall 経路は通常 extraction、main read 中の mutation は checksum fallback、post hook と family/kind 処理は従来どおり、graph 開始前に cache が clear されることを証明してください。 +- `CSharpPrepassSymbolArtifactCacheTests.CSharpWorkspaceAssembly_PreservesOrderIdentityAndEvidence` と `PerformanceTests.CSharpPrepassWorkspaceSegments_AvoidFlattenedReferenceBuffers` は、existing row / candidate / file内symbol順を保ったまま一時flatten bufferなしでprepass workspaceを組み立てる契約を固定します。allocation guardはcache上限131,072 symbolを16 KiB未満で全列挙します。2種類のimmutable lookupが完成するまでnon-owning viewを維持し、その後だけowned listを移譲してください。 - `SymbolExtractorRequiredLiteralGateTests` は built-in required-literal gate の決定性と output 不変性を固定します。51 の case-sensitive 言語にまたがる監査済み Tier A pattern 400件、出力順を 含む `SymbolRecord` の readable field 29個すべて、Python、JavaScript、TypeScript、Go、Rust、 diff --git a/changelog.d/unreleased/+csharp-prepass-segmented-workspace.changed.md b/changelog.d/unreleased/+csharp-prepass-segmented-workspace.changed.md new file mode 100644 index 000000000..a033c781f --- /dev/null +++ b/changelog.d/unreleased/+csharp-prepass-segmented-workspace.changed.md @@ -0,0 +1,13 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs +--- + +## English + +- **Fresh C# prepass lookup assembly no longer flattens symbol lists** — per-file artifacts are enumerated through a bounded non-owning segmented view until immutable workspace lookups are complete, avoiding duplicate workspace-sized pointer buffers while preserving extraction order, evidence, checksum, and cache ownership contracts. + +## 日本語 + +- **初回C# prepassのlookup構築がsymbol listをflattenしなくなりました** — immutableなworkspace lookupが完成するまでfile単位artifactをboundedなnon-owning segmented viewで列挙し、extraction順、evidence、checksum、cache所有権契約を維持しながらworkspace規模の重複pointer bufferを避けます。 diff --git a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs index 845d22ef9..0975d134f 100644 --- a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs +++ b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs @@ -200,19 +200,6 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( } }); - var pendingSymbolCount = 0; - foreach (var extracted in extractedByCandidate) - pendingSymbolCount += extracted?.Count ?? 0; - - var pendingSymbols = new List(pendingSymbolCount); - for (var candidateIndex = 0; candidateIndex < extractedByCandidate.Length; candidateIndex++) - { - var extracted = extractedByCandidate[candidateIndex]; - if (extracted != null) - pendingSymbols.AddRange(extracted); - } - - var hasSourceStaticInterfaceContracts = HasCSharpStaticInterfaceContractSymbol(pendingSymbols); var hadPendingContracts = false; var hadPendingMemberReadTargets = false; var shouldLoadExistingSymbols = includeExistingSymbols @@ -227,20 +214,39 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( out hadPendingMemberReadTargets, cancellationToken) : []; - symbols.AddRange(pendingSymbols); - var hasStaticInterfaceContracts = HasCSharpStaticInterfaceContractSymbol(symbols) || hadPendingContracts; + IReadOnlyList workspaceSymbols; + CSharpWorkspaceSymbolEvidence pendingEvidence; + if (symbolArtifactCache == null) + { + pendingEvidence = AppendExtractedWorkspaceSymbols( + extractedByCandidate, + symbols); + workspaceSymbols = symbols; + } + else + { + pendingEvidence = InspectExtractedWorkspaceSymbols(extractedByCandidate); + workspaceSymbols = new CSharpWorkspaceSymbolSegments( + symbols, + extractedByCandidate, + pendingEvidence.SymbolCount); + } + var hasSourceStaticInterfaceContracts = pendingEvidence.HasStaticInterfaceContracts; + var hasStaticInterfaceContracts = + HasCSharpStaticInterfaceContractSymbol(symbols) + || hadPendingContracts + || hasSourceStaticInterfaceContracts; var requiresMemberReadReferenceRefresh = hadPendingMemberReadTargets - || pendingSymbols.Any( - ReferenceExtractor.IsCSharpQualifiedMemberReadTargetSymbol); + || pendingEvidence.HasMemberReadTargets; IReadOnlyList incompletePaths = firstIncompleteSourcePath == null ? [] : [firstIncompleteSourcePath]; var isSourceEvidenceComplete = sourceEvidenceComplete != 0; var staticInterfaceMemberLookups = - ReferenceExtractor.BuildCSharpStaticInterfaceMemberLookups(symbols); + ReferenceExtractor.BuildCSharpStaticInterfaceMemberLookups(workspaceSymbols); var qualifiedPatternLookups = - ReferenceExtractor.BuildCSharpQualifiedPatternLookups(symbols); + ReferenceExtractor.BuildCSharpQualifiedPatternLookups(workspaceSymbols); // Materialize every immutable workspace lookup before transferring the raw // per-file lists. The main pass may mutate owned symbols after take, while // reference extraction must continue to observe this prepass snapshot. @@ -278,6 +284,61 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( requiresMemberReadReferenceRefresh); } + internal static CSharpWorkspaceSymbolEvidence AppendExtractedWorkspaceSymbols( + IReadOnlyList?[] extractedByCandidate, + List workspaceSymbols) + { + var evidence = InspectExtractedWorkspaceSymbols(extractedByCandidate); + workspaceSymbols.EnsureCapacity(checked(workspaceSymbols.Count + evidence.SymbolCount)); + foreach (var extracted in extractedByCandidate) + { + if (extracted != null) + workspaceSymbols.AddRange(extracted); + } + + return evidence; + } + + private static CSharpWorkspaceSymbolEvidence InspectExtractedWorkspaceSymbols( + IReadOnlyList?[] extractedByCandidate) + { + var symbolCount = 0; + var hasStaticInterfaceContracts = false; + var hasMemberReadTargets = false; + foreach (var extracted in extractedByCandidate) + { + if (extracted == null) + continue; + + symbolCount = checked(symbolCount + extracted.Count); + if (hasStaticInterfaceContracts && hasMemberReadTargets) + continue; + + foreach (var symbol in extracted) + { + if (!hasStaticInterfaceContracts + && IsCSharpStaticInterfaceContractSymbol(symbol)) + { + hasStaticInterfaceContracts = true; + } + + if (!hasMemberReadTargets + && ReferenceExtractor.IsCSharpQualifiedMemberReadTargetSymbol(symbol)) + { + hasMemberReadTargets = true; + } + + if (hasStaticInterfaceContracts && hasMemberReadTargets) + break; + } + } + + return new CSharpWorkspaceSymbolEvidence( + symbolCount, + hasStaticInterfaceContracts, + hasMemberReadTargets); + } + internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( DbWriter writer, FileIndexer indexer, @@ -315,6 +376,68 @@ internal static bool HasCSharpStaticInterfaceContractSymbol(IEnumerable + { + private readonly IReadOnlyList _prefix; + private readonly IReadOnlyList?[] _candidateSegments; + + internal CSharpWorkspaceSymbolSegments( + IReadOnlyList prefix, + IReadOnlyList?[] candidateSegments, + int candidateSymbolCount) + { + _prefix = prefix; + _candidateSegments = candidateSegments; + Count = checked(prefix.Count + candidateSymbolCount); + } + + public int Count { get; } + + public SymbolRecord this[int index] + { + get + { + if ((uint)index >= (uint)Count) + throw new ArgumentOutOfRangeException(nameof(index)); + if (index < _prefix.Count) + return _prefix[index]; + + index -= _prefix.Count; + foreach (var segment in _candidateSegments) + { + if (segment == null) + continue; + if (index < segment.Count) + return segment[index]; + index -= segment.Count; + } + + throw new InvalidOperationException("The workspace symbol segments changed after construction."); + } + } + + public IEnumerator GetEnumerator() + { + foreach (var symbol in _prefix) + yield return symbol; + foreach (var segment in _candidateSegments) + { + if (segment == null) + continue; + foreach (var symbol in segment) + yield return symbol; + } + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + => GetEnumerator(); + } + private static IEnumerable EnumerateFileTargets(string projectRoot, IEnumerable filePaths) { foreach (var path in filePaths) diff --git a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs index 8bafe7620..812d45815 100644 --- a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs +++ b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs @@ -7,6 +7,80 @@ namespace CodeIndex.Tests; public class CSharpPrepassSymbolArtifactCacheTests { + [Fact] + public void CSharpWorkspaceAssembly_PreservesOrderIdentityAndEvidence() + { + var existing = CreateMinimalSymbols(1); + existing[0].Name = "Existing"; + var contract = new SymbolRecord + { + Kind = "function", + Name = "Create", + Signature = "static abstract T Create();", + ContainerKind = "interface", + ContainerName = "IContract", + }; + var enumMember = new SymbolRecord + { + Kind = "enum", + Name = "Red", + ContainerKind = "enum", + ContainerName = "Shade", + }; + var ordinary = CreateMinimalSymbols(1)[0]; + ordinary.Name = "Ordinary"; + IReadOnlyList?[] segments = + [ + null, + new List { contract }, + [], + new List { enumMember, ordinary }, + ]; + + var appended = new List(existing); + var evidence = CSharpStaticInterfacePrepass.AppendExtractedWorkspaceSymbols( + segments, + appended); + Assert.Equal(3, evidence.SymbolCount); + Assert.True(evidence.HasStaticInterfaceContracts); + Assert.True(evidence.HasMemberReadTargets); + Assert.Equal( + new[] { existing[0], contract, enumMember, ordinary }, + appended, + ReferenceEqualityComparer.Instance); + + var ordinaryOnly = new List(); + var ordinaryEvidence = CSharpStaticInterfacePrepass.AppendExtractedWorkspaceSymbols( + [new List { ordinary }], + ordinaryOnly); + Assert.False(ordinaryEvidence.HasStaticInterfaceContracts); + Assert.False(ordinaryEvidence.HasMemberReadTargets); + Assert.Same(ordinary, Assert.Single(ordinaryOnly)); + } + + [Fact] + public void CSharpWorkspaceSymbolSegments_PreservesPrefixCandidateOrderCountAndIdentity() + { + var prefix = CreateMinimalSymbols(2); + var first = CreateMinimalSymbols(2); + var last = CreateMinimalSymbols(1); + IReadOnlyList?[] segments = [null, first, [], last]; + var view = new CSharpStaticInterfacePrepass.CSharpWorkspaceSymbolSegments( + prefix, + segments, + candidateSymbolCount: 3); + + Assert.Equal(5, view.Count); + Assert.Equal( + prefix.Concat(first).Concat(last), + view, + ReferenceEqualityComparer.Instance); + Assert.Same(prefix[0], view[0]); + Assert.Same(last[0], view[^1]); + Assert.Throws(() => view[-1]); + Assert.Throws(() => view[view.Count]); + } + [Fact] public void TryTake_MatchingChecksumOwnsDeepCloneAndIsTakeOnce() { diff --git a/tests/CodeIndex.Tests/PerformanceTests.cs b/tests/CodeIndex.Tests/PerformanceTests.cs index 9cc0fc0ef..0782f662d 100644 --- a/tests/CodeIndex.Tests/PerformanceTests.cs +++ b/tests/CodeIndex.Tests/PerformanceTests.cs @@ -340,6 +340,51 @@ public void CSharpStaticInterfacePrepass_LargeSemanticProbeStaysWithinAllocation Assert.True(allocatedBytes < 4_096, $"C# static-interface semantic probes allocated {allocatedBytes:N0} bytes"); } +#if NET8_0 + [Fact] +#else + [Fact(Skip = PracticalBudgetTestTarget.SecondaryTargetSkipReason)] +#endif + public void CSharpPrepassWorkspaceSegments_AvoidFlattenedReferenceBuffers() + { + const int segmentCount = 32; + const int symbolsPerSegment = 4_096; + IReadOnlyList?[] segments = Enumerable.Range(0, segmentCount) + .Select(_ => (IReadOnlyList)Enumerable.Repeat( + new SymbolRecord { Kind = "function", Name = "Ordinary" }, + symbolsPerSegment).ToArray()) + .ToArray(); + var prefix = Array.Empty(); + + var warmup = new CSharpStaticInterfacePrepass.CSharpWorkspaceSymbolSegments( + prefix, + segments, + segmentCount * symbolsPerSegment); + Assert.Equal(segmentCount * symbolsPerSegment, warmup.Count); + Assert.Equal(warmup.Count, warmup.Count(static _ => true)); + + CSharpStaticInterfacePrepass.CSharpWorkspaceSymbolSegments? view = null; + var observed = 0; + var allocatedBytes = MeasureAllocatedBytes(() => + { + view = new CSharpStaticInterfacePrepass.CSharpWorkspaceSymbolSegments( + prefix, + segments, + segmentCount * symbolsPerSegment); + foreach (var symbol in view) + { + if (symbol.Kind == "function") + observed++; + } + }); + + Assert.NotNull(view); + Assert.Equal(segmentCount * symbolsPerSegment, observed); + Assert.True( + allocatedBytes < 16_384, + $"Segmented C# prepass workspace allocated {allocatedBytes:N0} bytes"); + } + #if NET8_0 [Fact] #else From 0ca3e55955c5a7a79ae2f2084d2a301dd8dbce3e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 23:12:15 +0900 Subject: [PATCH 06/12] Avoid discarded C# workspace type lookups --- DEVELOPER_GUIDE.md | 8 +++++ TESTING_GUIDE.md | 4 +-- .../+csharp-workspace-type-lookups.changed.md | 13 +++++++++ .../CSharpReferenceExtractor.Support.cs | 25 +++++++++++++--- .../CSharpPrepassSymbolArtifactCacheTests.cs | 29 +++++++++++++++++++ tests/CodeIndex.Tests/PerformanceTests.cs | 14 +++++++++ 6 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 changelog.d/unreleased/+csharp-workspace-type-lookups.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e97c6bd61..259c12038 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -471,6 +471,10 @@ partial and must not make that transient result authoritative. Keep admission bounded to 4,096 files, 131,072 symbols, and an estimated 32 MiB, and clear all unconsumed artifacts before reference-graph work begins. +The workspace qualified-pattern lookup needs only raw non-enum type names for +enum-shadowing decisions. Build that conflict set directly; do not call the +per-file type-name builder and discard its normalized and qualified known-type set. + Authoritative full scans collect the C#, VB, F#, and MSBuild project-marker fingerprints during the shared source-directory enumeration. The same pass also builds a budget-independent directory marker-count snapshot used by @@ -4310,6 +4314,10 @@ extraction へ fallback します。timeout した prepass 結果は partial で authoritative にしてはいけません。admission は 4,096 file、131,072 symbol、推定 32 MiB に制限し、未消費 artifact は reference graph 開始前にすべて clear してください。 +workspace qualified-pattern lookup が enum shadowing 判定に必要とするのは raw な non-enum type +nameだけです。このconflict setは直接構築し、per-file type-name builderを呼んでnormalized / qualified +known-type setを直後に捨てないでください。 + authoritative な full scan は、共有 source-directory enumeration 中に C#、VB、F#、 MSBuild の project-marker fingerprint を収集します。同じ pass で budget 非依存の directory marker-count snapshot も構築し、`GetFamilyScopeKey` が file ごとに各 ancestor の diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 2eed6602b..32157ee31 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -461,7 +461,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result current-worktree paths, `.cdidx` databases, allocation/wall-clock assertions, or repository-wide extraction to this test; measure cold-index performance externally and pair it with natural-key database parity. -- `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` builds 20,000 unrelated generic interfaces around one contract and keeps lookup allocation below 64 KiB. Discover contract containers before parsing generic declarations, while the paired `ReferenceExtractorTests` case preserves declaration/member ordering, partial-interface last-write behavior, and contract member order. +- `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` builds 20,000 unrelated namespaced generic interfaces around one contract, keeps static-interface lookup allocation below 64 KiB, and keeps qualified-pattern lookup allocation below 7 MB by constructing only its required raw non-enum conflict set. Discover contract containers before parsing generic declarations, while the paired `ReferenceExtractorTests` case preserves declaration/member ordering, partial-interface last-write behavior, contract member order, and enum-shadowing semantics. - `CSharpPrepassSymbolArtifactCacheTests.CSharpWorkspaceAssembly_PreservesOrderIdentityAndEvidence` and `PerformanceTests.CSharpPrepassWorkspaceSegments_AvoidFlattenedReferenceBuffers` keep prepass workspace assembly in existing-row/candidate/file-symbol order without transient flattened buffers. The allocation guard enumerates the full 131,072-symbol cache ceiling below 16 KiB; preserve the non-owning view until both immutable lookups exist and transfer owned lists only afterward. - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` keeps a 1,200-statement, regex-and-brace-heavy source at one emitted symbol and pins current-thread allocations below 6.0 MB for JavaScript, 8.2 MB for TypeScript, and 14.1 MB combined. This is a physical-pass contract: do not restore a second full-file sanitizer for private-scope analysis. Pair allocation changes with the JavaScript/TypeScript literal-and-comment scope theory so templates, comments, and regex braces cannot create or leak private classes. - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` keeps persisted C# contract-member candidates and the pending-path contract flag on one `files(lang)` → `symbols(file_id, kind)` row pass. Derive both results from that reader; do not restore a second all-contract query for excluded paths. Interface declarations may be loaded only for exact retained contract container names through bounded `symbols(name)` batches, and those dynamic tail shapes must stay out of the prepared cache. `DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` pins both index plans, while `LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` requires negative/LIKE-decoy-only reads to execute no interface phase and keeps unrelated plain interfaces out of managed materialization. `PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` also splits a 503-path language lookup at 500 parameters, returns only persisted C# paths plus sorted purge-plan IDs/bytes, keeps both SQL tail shapes out of the prepared cache, and stops before the second batch when cancellation is requested. @@ -1535,7 +1535,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" current worktree path、`.cdidx` database、allocation / wall-clock assertion、repository 全体の extraction をこの test に追加してはいけません。cold-index performance は外部で測定し、database の natural-key parity と対にしてください。 -- `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` は、1件のcontractの周囲に20,000件の無関係なgeneric interfaceを構築し、lookup allocationを64 KiB未満に固定します。generic宣言を解析する前にcontract containerを検出し、対になる`ReferenceExtractorTests`で宣言/member順、partial interfaceの後勝ち、contract member順を維持してください。 +- `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` は、1件のcontractの周囲に20,000件の無関係なnamespaced generic interfaceを構築し、static-interface lookup allocationを64 KiB未満、qualified-pattern lookup allocationを7 MB未満に固定します。後者は必要なraw non-enum conflict setだけを構築してください。generic宣言を解析する前にcontract containerを検出し、対になる`ReferenceExtractorTests`で宣言/member順、partial interfaceの後勝ち、contract member順、enum-shadowing semanticsを維持してください。 - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` は、regex と brace が密な1,200 statement の source を出力 symbol 1件に保ち、current-thread allocation を JavaScript 6.0 MB未満、TypeScript 8.2 MB未満、合計14.1 MB未満に固定します。これは physical-pass 契約であり、private-scope 解析専用の2回目の全file sanitizerを戻さないでください。allocation変更時はJavaScript/TypeScriptのliteral/comment scope theoryも対にし、template、comment、regexのbraceがprivate classを生成またはscope外へ漏らさないことを維持してください。 - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` は、永続化済み C# contract member 候補と pending-path contract flag を1回の `files(lang)` → `symbols(file_id, kind)` row passで得ることを固定します。両方を同じ reader から導出し、除外 path 判定のための2回目の全 contract query を戻さないでください。interface 宣言は厳密検証後に保持した contract container 名だけを bounded な `symbols(name)` batch で取得でき、その dynamic tail shape は prepared cache に残してはいけません。`DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` は両方の index plan を固定し、`LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` は negative / LIKE decoy だけの読込で interface phase が0回となり、無関係な通常 interface を managed materialization へ入れないことを要求します。`PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` は503件のpath言語lookupを500 parameterで分割し、永続C# pathと昇順のpurge-plan ID/byteだけを返し、両方のSQL tail shapeをprepared cache外に保ち、1つ目のbatch後のcancellationで2つ目を実行しないことも固定します。 - `IndexCommandRunnerTests.Run_UpdateFiles_CsharpContractPreflightAvoidsRedundantWorkspacePasses` は false の source-evidence marker を authoritative に保ち、repository 全体の persisted-member preflight 回数を0に固定して、candidate read と workspace materialization を1回の expanded C# passへ委譲します。plain-interface、新規contract追加、永続化済みcontractの各phaseで、それぞれscoped 1回、discovery＋expanded、expanded 1回を固定します。対になるfull-scan/MCPのknown-evidence no-op fixtureでは初回のtrueまたはfalse snapshot以降、source load、raw prepass、永続C# symbol load、lookup buildをすべて0に固定します。明示的な `index_completeness=complete` または GraphReady がない場合、symbols-only/filter/version/root/hotspot contract が変わった場合、永続 C# path が別言語へ変わった場合はshortcutを拒否しなければなりません。final-stat mutation fixtureではcomplete raw prepass 1回と全C# content loadを要求し、explicit/shared DBのroot-switch fixtureではcross-root stat reuseを拒否します。 diff --git a/changelog.d/unreleased/+csharp-workspace-type-lookups.changed.md b/changelog.d/unreleased/+csharp-workspace-type-lookups.changed.md new file mode 100644 index 000000000..0219695a2 --- /dev/null +++ b/changelog.d/unreleased/+csharp-workspace-type-lookups.changed.md @@ -0,0 +1,13 @@ +--- +category: changed +affected: + - src/CodeIndex/Indexer/References/Languages/CSharpReferenceExtractor.Support.cs +--- + +## English + +- **C# workspace pattern lookups no longer construct a discarded known-type set** — enum-shadowing analysis now builds only the raw non-enum conflict names it consumes, reducing fresh-prepass allocation while preserving qualified type and member lookup behavior. + +## 日本語 + +- **C# workspace pattern lookupが未使用のknown-type setを構築しなくなりました** — enum shadowing解析は実際に使うraw non-enum conflict nameだけを構築し、qualified type / member lookupの挙動を維持しながら初回prepassのallocationを削減します。 diff --git a/src/CodeIndex/Indexer/References/Languages/CSharpReferenceExtractor.Support.cs b/src/CodeIndex/Indexer/References/Languages/CSharpReferenceExtractor.Support.cs index b3b8e5312..5577a3cfc 100644 --- a/src/CodeIndex/Indexer/References/Languages/CSharpReferenceExtractor.Support.cs +++ b/src/CodeIndex/Indexer/References/Languages/CSharpReferenceExtractor.Support.cs @@ -263,6 +263,24 @@ private static (IReadOnlySet KnownTypeNames, IReadOnlySet NonEnu return (knownTypeNames ?? EmptyCSharpStringSet, nonEnumTypeNames ?? EmptyCSharpStringSet); } + private static IReadOnlySet BuildCSharpNonEnumTypeNames( + IReadOnlyList symbols) + { + HashSet? names = null; + foreach (var symbol in symbols) + { + if (symbol.Kind is not ("class" or "struct" or "interface" or "delegate") + || string.IsNullOrWhiteSpace(symbol.Name)) + { + continue; + } + + (names ??= new HashSet(StringComparer.Ordinal)).Add(symbol.Name); + } + + return names ?? EmptyCSharpStringSet; + } + private static HashSet? BuildCallableDefinitionNames(string language, IReadOnlyList symbols) { if (language != "csharp") @@ -525,10 +543,9 @@ private static void AddCSharpContainingTypeValueReceiverName( internal static CSharpQualifiedPatternLookups BuildCSharpQualifiedPatternLookups( IReadOnlyList symbols) - { - var typeNameSets = BuildCSharpTypeNameSets("csharp", symbols); - return BuildCSharpQualifiedPatternLookups(symbols, typeNameSets.NonEnumTypeNames); - } + => BuildCSharpQualifiedPatternLookups( + symbols, + BuildCSharpNonEnumTypeNames(symbols)); private static CSharpQualifiedPatternLookups BuildCSharpQualifiedPatternLookups( IReadOnlyList symbols, diff --git a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs index 812d45815..0e4d37efd 100644 --- a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs +++ b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs @@ -81,6 +81,35 @@ public void CSharpWorkspaceSymbolSegments_PreservesPrefixCandidateOrderCountAndI Assert.Throws(() => view[view.Count]); } + [Fact] + public void QualifiedPatternLookups_PreserveRawNonEnumTypeShadowing() + { + var lookups = ReferenceExtractor.BuildCSharpQualifiedPatternLookups( + [ + new SymbolRecord { Kind = "enum", Name = "Shade" }, + new SymbolRecord + { + Kind = "enum", + Name = "Red", + ContainerKind = "enum", + ContainerName = "Shade", + }, + new SymbolRecord { Kind = "class", Name = "Shade" }, + new SymbolRecord { Kind = "enum", Name = "Tone" }, + new SymbolRecord + { + Kind = "enum", + Name = "Warm", + ContainerKind = "enum", + ContainerName = "Tone", + }, + new SymbolRecord { Kind = "field", Name = "Tone" }, + ]); + + Assert.False(Assert.Single(lookups.EnumMemberLookup["Red"]).AllowShortNameFallback); + Assert.True(Assert.Single(lookups.EnumMemberLookup["Warm"]).AllowShortNameFallback); + } + [Fact] public void TryTake_MatchingChecksumOwnsDeepCloneAndIsTakeOnce() { diff --git a/tests/CodeIndex.Tests/PerformanceTests.cs b/tests/CodeIndex.Tests/PerformanceTests.cs index 0782f662d..cf25721c9 100644 --- a/tests/CodeIndex.Tests/PerformanceTests.cs +++ b/tests/CodeIndex.Tests/PerformanceTests.cs @@ -400,6 +400,9 @@ public void CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationB { Kind = "interface", Name = $"IUnrelated{index}", + ContainerKind = "namespace", + ContainerName = "Demo", + ContainerQualifiedName = "Demo", Signature = $"public interface IUnrelated{index}", }); } @@ -408,6 +411,9 @@ public void CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationB { Kind = "interface", Name = "IContract", + ContainerKind = "namespace", + ContainerName = "Demo", + ContainerQualifiedName = "Demo", Signature = "public interface IContract", }); workspaceSymbols.Add(new SymbolRecord @@ -421,15 +427,23 @@ public void CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationB }); _ = ReferenceExtractor.BuildCSharpStaticInterfaceMemberLookups(workspaceSymbols); + _ = ReferenceExtractor.BuildCSharpQualifiedPatternLookups(workspaceSymbols); ReferenceExtractor.CSharpStaticInterfaceMemberLookups? lookups = null; + ReferenceExtractor.CSharpQualifiedPatternLookups? qualifiedLookups = null; var allocatedBytes = MeasureAllocatedBytes( () => lookups = ReferenceExtractor.BuildCSharpStaticInterfaceMemberLookups(workspaceSymbols)); + var qualifiedAllocatedBytes = MeasureAllocatedBytes( + () => qualifiedLookups = ReferenceExtractor.BuildCSharpQualifiedPatternLookups(workspaceSymbols)); Assert.True( allocatedBytes < 64_000, $"C# static-interface lookup allocated {allocatedBytes:N0} bytes for unrelated interfaces"); Assert.Single(lookups!.ContractsByType); Assert.Single(lookups.InterfaceGenericParameters); + Assert.Equal(unrelatedInterfaceCount + 1, qualifiedLookups!.TypePatternLookup.Count); + Assert.True( + qualifiedAllocatedBytes < 7_000_000, + $"C# qualified workspace lookup allocated {qualifiedAllocatedBytes:N0} bytes"); } #if NET8_0 From 42115a0024a6b0ddcb53989f12fdc3315a5f8074 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 23:19:51 +0900 Subject: [PATCH 07/12] Remove duplicate reference-line window hashing --- DEVELOPER_GUIDE.md | 6 ++- TESTING_GUIDE.md | 2 + .../+reference-line-window-sizing.changed.md | 13 +++++++ src/CodeIndex/Database/DbWriter.References.cs | 39 ++++++++----------- tests/CodeIndex.Tests/DatabaseTests.cs | 19 +++++++++ 5 files changed, 55 insertions(+), 24 deletions(-) create mode 100644 changelog.d/unreleased/+reference-line-window-sizing.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 259c12038..ebc5fb21b 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -383,7 +383,9 @@ batch-local key-to-ordinal map, resolves each unique line ID, then releases the tuple lookup before binding `symbol_references` from ordinal arrays. Preserve this path for both new-file inserts and replacement upserts, including atomic file windows, so large multi-language reference sets do not rehash file/line/ -context tuples for every persisted edge. +context tuples for every persisted edge. Atomic window sizing uses the worst-case +rows-per-statement bound and leaves the materializer as the only tuple-hash pass; +do not restore a duplicate key-sizing set before it. Use the same secondary-index deferral for an existing-database full scan when the established FTS dirty-byte policy selects bulk loading. Scoped updates have @@ -4240,6 +4242,8 @@ map を構築し、unique な line ID を解決した後、`symbol_references` する前に tuple lookup を解放します。巨大な multi-language reference 集合で file / line / context tuple を edge ごとに再 hash しないよう、新規 file insert と replacement upsert の両方、 atomic file window を含む全経路でこの契約を維持してください。 +atomic window の size は rows-per-statement の最悪ケース境界から算出し、materializer だけを +tuple-hash pass として保ちます。その前段に重複した key-sizing set を戻さないでください。 既存DBの full scan でも、既定の FTS dirty-byte policy が bulk load を選ぶ場合は同じ secondary index 退避を使います。scoped update には workspace 全体の authoritative な byte estimate が diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 32157ee31..0071ced56 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -747,6 +747,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` keeps 12,000 no-alias calls from paying for alias dedupe, `CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` guards pre-sized stable compaction after alias rewrites, and `MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` covers repeated qualified C#- and Python-style cycle names without per-edge normalization strings. `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` warms a 1,024-row typed snapshot with its exact capacity, then prevents a second candidate collection or redundant path values from returning to the default `net8.0` path. `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. `CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` additionally fixes the atomic full-index parameter budget at 32 across language-neutral chunks (6 rows), symbols (1), issues (5), reference lines (10), and references (2), while the public reference API retains 71-row batches. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. + `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` fixes atomic reference-line windows to a worst-case arithmetic bound. Keep it paired with whole-batch grouping, 32-batch caps, cross-boundary context reuse, rollback, and cancellation tests so the materializer remains the only tuple-hash pass without changing persistence boundaries. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. Keep the full/scoped/retained resolution oracles beside this structural guard so physical candidate rows and multi-language ambiguity remain unchanged. @@ -1818,6 +1819,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` は alias のない 12,000 call が alias dedupe のコストを負わないこと、`CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` は alias rewrite 後の事前 capacity 付き stable compaction、`MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` は C# / Python 形式の qualified cycle name が edge ごとの正規化文字列を作らないことを保証します。 `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` は exact capacity を渡した1,024行の typed snapshot を warm-up し、2つ目の候補 collection や重複 path value が通常の `net8.0` 経路へ戻らないようにします。 `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。`CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` はさらに、言語共通のchunk（6行）、symbol（1行）、issue（5行）、reference line（10行）、reference（2行）に対するatomic full-index parameter budgetを32に固定し、public reference APIの71行batchは維持します。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 + `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` はatomic reference-line windowを最悪ケースの算術境界へ固定します。materializerだけをtuple-hash passとして保ちつつ永続化境界を変えないよう、whole-batch grouping、32-batch cap、batch境界をまたぐcontext再利用、rollback、cancellation testと対で維持してください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。物理candidate行と多言語ambiguityが変わらないよう、この構造guardとfull / scoped / retained resolution oracleを対で維持してください。 diff --git a/changelog.d/unreleased/+reference-line-window-sizing.changed.md b/changelog.d/unreleased/+reference-line-window-sizing.changed.md new file mode 100644 index 000000000..359d56bfc --- /dev/null +++ b/changelog.d/unreleased/+reference-line-window-sizing.changed.md @@ -0,0 +1,13 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.References.cs +--- + +## English + +- **Initial reference-line persistence no longer hashes every context twice** — language-neutral atomic windows are bounded from worst-case batch rows before the single deduplicating materialization pass, removing a redundant tuple set while preserving window, rollback, and context-identity contracts. + +## 日本語 + +- **初回reference-line永続化で全contextを二重hashしなくなりました** — 言語共通のatomic windowは単一の重複排除materializationより前にbatch rowの最悪ケースからboundされ、window、rollback、context identity契約を維持しながら冗長なtuple setを除去します。 diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 3ecba765d..5bf157452 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -1875,7 +1875,6 @@ private void InsertAtomicReferenceBatches( references.Count, cancellationToken); int windowEndBatch = GetAtomicReferenceLineWindowEndBatch( - references, windowStartBatch, referenceBatchCount, rowsPerStatement); @@ -1909,35 +1908,29 @@ private void InsertAtomicReferenceBatches( } private static int GetAtomicReferenceLineWindowEndBatch( - IReadOnlyList references, int windowStartBatch, int referenceBatchCount, int rowsPerStatement) { int maxReferenceLines = GetRowsPerInsertStatement(columnCount: 3); - var windowKeys = new HashSet<(long FileId, int Line, string Context)>(maxReferenceLines); - int windowEndBatch = windowStartBatch; - while (windowEndBatch < referenceBatchCount - && windowEndBatch - windowStartBatch < MaxReferenceLineWindowBatchCount) - { - int batchStart = windowEndBatch * rowsPerStatement; - int batchEnd = Math.Min(batchStart + rowsPerStatement, references.Count); - for (int index = batchStart; index < batchEnd; index++) - { - var reference = references[index]; - var key = (reference.FileId, reference.Line, reference.Context); - windowKeys.Add(key); - } - - if (windowKeys.Count > maxReferenceLines && windowEndBatch > windowStartBatch) - break; - - windowEndBatch++; - } - - return windowEndBatch; + int worstCaseBatches = Math.Max(1, maxReferenceLines / rowsPerStatement); + int windowBatchCount = Math.Min( + MaxReferenceLineWindowBatchCount, + worstCaseBatches); + return Math.Min( + referenceBatchCount, + windowStartBatch + windowBatchCount); } + internal static int GetAtomicReferenceLineWindowEndBatchForTesting( + int windowStartBatch, + int referenceBatchCount, + int rowsPerStatement) + => GetAtomicReferenceLineWindowEndBatch( + windowStartBatch, + referenceBatchCount, + rowsPerStatement); + private ReferenceLineBatchMap MaterializeReferenceLines( IReadOnlyList references, int start, diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index ecc832aa6..03118b470 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -10401,6 +10401,25 @@ public void InsertReferences_AtomicFileScopeCapsReferenceLineWindowAtThirtyTwoBa Assert.Equal(2, statements.Count(statement => statement.Operation == "lookup_reference_lines")); } + [Theory] + [InlineData(0, 33, 2, 32)] + [InlineData(32, 33, 2, 33)] + [InlineData(0, 10, 71, 4)] + [InlineData(0, 10, 334, 1)] + public void AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing( + int windowStartBatch, + int referenceBatchCount, + int rowsPerStatement, + int expectedEndBatch) + { + Assert.Equal( + expectedEndBatch, + DbWriter.GetAtomicReferenceLineWindowEndBatchForTesting( + windowStartBatch, + referenceBatchCount, + rowsPerStatement)); + } + [Theory] [InlineData(false, 72)] [InlineData(true, 142)] From d5f918c1e27dc8e3d95ba6875aa25e8fbc09b7f6 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Thu, 13 Aug 2026 23:36:56 +0900 Subject: [PATCH 08/12] Drive C# property normalization from compact facts --- DEVELOPER_GUIDE.md | 14 ++- TESTING_GUIDE.md | 2 + ...+csharp-property-receiver-facts.changed.md | 14 +++ .../DbWriter.ReferenceGraphRefreshScope.cs | 32 ++++++ src/CodeIndex/Database/DbWriter.References.cs | 107 ++++++++++++++---- tests/CodeIndex.Tests/DatabaseTests.cs | 72 +++++++++++- 6 files changed, 213 insertions(+), 28 deletions(-) create mode 100644 changelog.d/unreleased/+csharp-property-receiver-facts.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ebc5fb21b..887490d15 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -409,13 +409,17 @@ C# reference-graph finalization materializes reference arity, invocation arity, member-receiver, definition arity, constructor arity, and value-type facts once per applicable row in TEMP tables. Full, scoped, and retained-graph rebuilds must then materialize project/file-local type identities and constructor-owner identity -and arity facts from those symbol facts. Populate all four fact tables before +and arity facts from those symbol facts. Before property-receiver normalization, +also materialize C# field/property target identities into a primary-keyed TEMP +fact set. Populate all fact sets before property-receiver normalization, candidate construction, and resolution. Keep candidate SQL on primary-key fact lookups instead of rebuilding identity strings, rescanning constructor-owner ranges, or re-entering managed SQLite scalar functions for every join candidate. Scoped refreshes must limit symbol facts to their lookup-name set and derive identity facts from that bounded population; full and -retained rebuilds use the complete C# symbol-fact population. +retained rebuilds use the complete C# symbol-fact population. Property-receiver +normalization must likewise drive from flagged reference facts and the target fact +primary key; scoped target materialization is restricted to its lookup-name set. After rank 0–4 candidate construction, graph finalization materializes the distinct matching reference IDs into a compact `WITHOUT ROWID` TEMP table. All @@ -4266,12 +4270,14 @@ single-evaluation の契約を維持してください。 C# の reference-graph finalization は、reference arity、invocation arity、member receiver、 definition arity、constructor arity、value-type の fact を、対象 row ごとに TEMP table へ1回だけ materialize し、その symbol fact から project / file-local type identity と constructor-owner の identity / arity -も materialize します。full / scoped / retained graph rebuild の全経路で4つの fact tableを +も materialize します。property-receiver normalization の前に C# field / property の target identity も +primary-keyed TEMP fact 集合へ materialize します。full / scoped / retained graph rebuild の全経路で fact 集合を property-receiver normalization、candidate 構築、resolution より前に投入してください。candidate SQL は join candidate ごとに identity 文字列を再構築したり constructor-owner range を再走査したり managed SQLite scalar function へ再入したりせず、primary-key の fact lookup を使います。scoped refresh の symbol fact は lookup-name 集合だけに限定し、identity fact もその限定済み集合から作ります。full / retained rebuild は -C# symbol fact の全対象を使います。 +C# symbol fact の全対象を使います。property-receiver normalization も flag 済み reference fact と target fact の +primary key から駆動し、scoped target materialization は lookup-name 集合だけに限定してください。 rank 0〜4 の candidate 構築後は、一致した reference ID の distinct 集合を compact な `WITHOUT ROWID` TEMP table に materialize します。言語共通および C# の rank 5 fallback は diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 0071ced56..30b5d3099 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -751,6 +751,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. Keep the full/scoped/retained resolution oracles beside this structural guard so physical candidate rows and multi-language ambiguity remain unchanged. + `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` requires both normalization updates to seek flagged reference IDs and the primary-keyed field/property target facts rather than scan all references or persistent target symbols. Keep its full/scoped/retained stage-order assertions and property-resolution fixtures paired so lookup-name scoping cannot change inherited-member semantics. `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` fixes target-family key construction to one per target symbol and candidate resolution to TEMP primary-key facts across fresh, full, differential, scoped, and retained paths. Its legacy-null-key and C#/Python oracle coverage preserves resolved IDs, exact keys, grouped families, ambiguity, and self-reference semantics. - `HotspotReferenceAggregateTests.cs` Regression coverage for the maintained per-file hotspot aggregate, including legacy transactional backfill, high-cardinality limited file/name queries, aggregate/raw logical-site parity, cross-file context and identity invalidation, and cancellation after aggregate SQL begins. Query-plan assertions require the rank index on `hotspot_reference_counts`, reject raw `symbol_references` scans, and use the shared broad deterministic timeout instead of a benchmark-grade threshold. @@ -1823,6 +1824,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。物理candidate行と多言語ambiguityが変わらないよう、この構造guardとfull / scoped / retained resolution oracleを対で維持してください。 + `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` は、2つのnormalization updateが全referenceや永続target symbolをscanせず、flag済みreference IDとprimary-keyed field / property target factをseekすることを要求します。lookup-name scopeが継承member semanticsを変えないよう、full / scoped / retainedのstage-order assertionとproperty-resolution fixtureを対で維持してください。 `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` は、target-family key構築をtarget symbolごと1回に限定し、fresh / full / differential / scoped / retainedのcandidate resolutionがTEMP primary-key factを使う契約を固定します。legacy null-keyとC# / Python oracle coverageにより、resolved ID、exact key、group family、ambiguity、self-reference semanticsを維持します。 - `HotspotReferenceAggregateTests.cs` file 単位 maintained hotspot aggregate の回帰 coverage です。legacy database の transactional backfill、高カーディナリティな limit 付き file/name query、aggregate/raw の logical-site parity、cross-file context / identity の無効化、aggregate SQL 開始後の cancellation を含みます。query-plan assertion は `hotspot_reference_counts` の rank index 利用を必須とし、raw `symbol_references` scan を拒否します。benchmark 用の厳しい閾値ではなく、共有の広い deterministic timeout を使います。 diff --git a/changelog.d/unreleased/+csharp-property-receiver-facts.changed.md b/changelog.d/unreleased/+csharp-property-receiver-facts.changed.md new file mode 100644 index 000000000..2f691f3ea --- /dev/null +++ b/changelog.d/unreleased/+csharp-property-receiver-facts.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +--- + +## English + +- **Initial C# property-receiver normalization now seeks compact facts** — full, scoped, and retained graph refreshes drive the two normalization updates from flagged reference IDs and primary-keyed field/property target identities instead of scanning all references and repeatedly probing persistent target symbols. + +## 日本語 + +- **初回C# property-receiver normalizationがcompact factをseekするようになりました** — full / scoped / retained graph refreshは、全referenceをscanして永続target symbolを繰り返しprobeせず、flag済みreference IDとprimary-keyed field / property target identityから2つのnormalization updateを駆動します。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index dadea7902..208e1385a 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -59,6 +59,15 @@ JOIN temp.{ReferenceGraphLookupNamesTable} AS symbol_lookup AND symbol_lookup.name_folded = symbol.name_folded """); + private static string RefreshCSharpPropertyTargetFactsScopedSql => + BuildRefreshCSharpPropertyTargetFactsSql( + $""" + FROM temp.{ReferenceGraphLookupNamesTable} AS property_lookup + CROSS JOIN symbols AS target INDEXED BY idx_symbols_name_folded + """, + "property_lookup.lang = 'csharp' " + + "AND target.name_folded = property_lookup.name_folded"); + private static string NormalizeCSharpPropertyReceiverReferencesScopedSql => BuildCSharpPropertyReceiverNormalizationSql( $"r.id IN (SELECT reference_id FROM temp.{ReferenceGraphDirtyReferencesTable})"); @@ -185,6 +194,7 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting + RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceCandidatesSql), ( @@ -193,6 +203,7 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting + RefreshCSharpSymbolFactsScopedSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpPropertyTargetFactsScopedSql + "\n" + NormalizeCSharpPropertyReceiverReferencesScopedSql + "\n" + RefreshScopedReferenceCandidatesSql), ( @@ -201,6 +212,7 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting + RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceCandidatesSql), ]; @@ -214,6 +226,26 @@ internal static string RefreshScopedReferenceCandidatesSqlForTesting ("retained", RefreshReferenceCandidatesSql), ]; + internal static IReadOnlyList<( + string Scope, + string MaterializationSql, + string NormalizationSql)> CSharpPropertyReceiverFactSqlForTesting + => + [ + ( + "full", + RefreshCSharpPropertyTargetFactsFullSql, + NormalizeCSharpPropertyReceiverReferencesFullSql), + ( + "scoped", + RefreshCSharpPropertyTargetFactsScopedSql, + NormalizeCSharpPropertyReceiverReferencesScopedSql), + ( + "retained", + RefreshCSharpPropertyTargetFactsFullSql, + NormalizeCSharpPropertyReceiverReferencesFullSql), + ]; + internal static IReadOnlyList ScopedReferenceGraphUpdateStatementsForTesting => [ diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 5bf157452..1b2f966b1 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -149,10 +149,19 @@ PRIMARY KEY(derived_qualified_name, base_qualified_name) ) WITHOUT ROWID; CREATE TEMP TABLE IF NOT EXISTS csharp_reference_facts ( - reference_id INTEGER NOT NULL PRIMARY KEY, - type_arity INTEGER, - argument_count INTEGER, - is_member_receiver INTEGER NOT NULL + reference_id INTEGER NOT NULL PRIMARY KEY, + type_arity INTEGER, + argument_count INTEGER, + is_member_receiver INTEGER NOT NULL, + is_property_receiver_reference INTEGER NOT NULL + ) WITHOUT ROWID; + + CREATE TEMP TABLE IF NOT EXISTS csharp_property_target_facts ( + name_folded TEXT NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + container_qualified_name TEXT NOT NULL COLLATE BINARY, + symbol_id INTEGER NOT NULL, + PRIMARY KEY(name_folded, name, container_qualified_name, symbol_id) ) WITHOUT ROWID; CREATE TEMP TABLE IF NOT EXISTS csharp_symbol_facts ( @@ -181,6 +190,12 @@ reference_id INTEGER NOT NULL PRIMARY KEY ) WITHOUT ROWID """; + private const string CreateCSharpReferenceFactIndexesSql = """ + CREATE INDEX IF NOT EXISTS temp.idx_csharp_reference_facts_property_receiver + ON csharp_reference_facts(reference_id) + WHERE is_property_receiver_reference = 1 + """; + private static readonly string RefreshReferenceResolutionSymbolFactsFullSql = $""" DELETE FROM temp.{ReferenceResolutionSymbolFactsTable}; @@ -199,7 +214,8 @@ INSERT INTO temp.csharp_reference_facts( reference_id, type_arity, argument_count, - is_member_receiver) + is_member_receiver, + is_property_receiver_reference) SELECT r.id, CASE WHEN r.reference_kind IN ('instantiate', 'type_reference') @@ -234,6 +250,13 @@ THEN csharp_reference_is_member_receiver( r.symbol_name, r.column_number) ELSE 0 + END, + CASE + WHEN r.reference_kind = 'reference' + AND r.target_qualifier LIKE + char(31) || 'property_receiver:%' + THEN 1 + ELSE 0 END FROM symbol_references AS r JOIN files AS source_file @@ -255,6 +278,37 @@ AND r.target_qualifier LIKE private static string RefreshCSharpReferenceFactsFullSql => BuildRefreshCSharpReferenceFactsSql("1 = 1"); + private static string BuildRefreshCSharpPropertyTargetFactsSql( + string symbolSource, + string scopePredicate) + => $""" + DELETE FROM temp.csharp_property_target_facts; + + INSERT INTO temp.csharp_property_target_facts( + name_folded, + name, + container_qualified_name, + symbol_id) + SELECT target.name_folded, + target.name, + target.container_qualified_name, + target.id + {symbolSource} + JOIN files AS target_file + ON target_file.id = target.file_id + AND target_file.lang = 'csharp' + WHERE {scopePredicate} + AND target.kind IN ('field', 'property') + AND target.name_folded IS NOT NULL + AND target.name IS NOT NULL + AND target.container_qualified_name IS NOT NULL; + """; + + private static string RefreshCSharpPropertyTargetFactsFullSql => + BuildRefreshCSharpPropertyTargetFactsSql( + "FROM symbols AS target", + "1 = 1"); + private static string BuildRefreshCSharpSymbolFactsSql(string scopeJoin) => $""" DELETE FROM temp.csharp_symbol_facts; @@ -787,20 +841,22 @@ UPDATE symbol_references AS r SET reference_kind = 'type_reference', target_qualifier = NULL WHERE {scopePredicate} + AND r.id IN ( + SELECT reference_fact.reference_id + FROM temp.csharp_reference_facts AS reference_fact + WHERE reference_fact.is_property_receiver_reference = 1 + ) AND r.reference_kind = 'reference' AND r.target_qualifier LIKE char(31) || 'property_receiver:%' AND NOT EXISTS ( SELECT 1 FROM symbols AS source JOIN files AS source_file ON source_file.id = source.file_id - JOIN symbols AS target + JOIN temp.csharp_property_target_facts AS target ON target.name_folded = r.symbol_name_folded AND target.name = r.symbol_name COLLATE BINARY - JOIN files AS target_file ON target_file.id = target.file_id WHERE source.id = r.source_symbol_id AND source_file.lang = 'csharp' - AND target_file.lang = 'csharp' - AND target.kind IN ('field', 'property') AND target.container_qualified_name IN ( SELECT source.container_qualified_name UNION @@ -820,14 +876,11 @@ UPDATE symbol_references AS r SELECT target.container_qualified_name FROM symbols AS source JOIN files AS source_file ON source_file.id = source.file_id - JOIN symbols AS target + JOIN temp.csharp_property_target_facts AS target ON target.name_folded = r.symbol_name_folded AND target.name = r.symbol_name COLLATE BINARY - JOIN files AS target_file ON target_file.id = target.file_id WHERE source.id = r.source_symbol_id AND source_file.lang = 'csharp' - AND target_file.lang = 'csharp' - AND target.kind IN ('field', 'property') AND target.container_qualified_name IN ( SELECT source.container_qualified_name UNION @@ -849,29 +902,26 @@ source.container_qualified_name COLLATE BINARY target.container_qualified_name COLLATE BINARY ), 33) END, - target.id + target.symbol_id LIMIT 1 ) WHERE {scopePredicate} + AND r.id IN ( + SELECT reference_fact.reference_id + FROM temp.csharp_reference_facts AS reference_fact + WHERE reference_fact.is_member_receiver = 1 + ) AND r.reference_kind = 'type_reference' AND r.target_qualifier IS NULL - AND COALESCE(( - SELECT reference_fact.is_member_receiver - FROM temp.csharp_reference_facts AS reference_fact - WHERE reference_fact.reference_id = r.id - ), 0) = 1 AND EXISTS ( SELECT 1 FROM symbols AS source JOIN files AS source_file ON source_file.id = source.file_id - JOIN symbols AS target + JOIN temp.csharp_property_target_facts AS target ON target.name_folded = r.symbol_name_folded AND target.name = r.symbol_name COLLATE BINARY - JOIN files AS target_file ON target_file.id = target.file_id WHERE source.id = r.source_symbol_id AND source_file.lang = 'csharp' - AND target_file.lang = 'csharp' - AND target.kind IN ('field', 'property') AND target.container_qualified_name IN ( SELECT source.container_qualified_name UNION @@ -1646,11 +1696,13 @@ internal static void RebuildRetainedReferenceGraph( command.Transaction = transaction; command.CommandText = CreateReferenceUniqueFamiliesSql + ";\n" + + CreateCSharpReferenceFactIndexesSql + ";\n" + RefreshReferenceSourceSymbolsFullSql + ";\n" + RefreshCSharpReferenceFactsFullSql + "\n" + RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + RefreshReferenceCandidatesSql + "\n" + @@ -2415,6 +2467,7 @@ internal void RefreshMutualRecursionFlags( if (graphScope != null) graphScope.IsCompleting = true; SqliteCommand? createUniqueFamiliesCommand = null; + SqliteCommand? createCSharpReferenceFactIndexesCommand = null; SqliteCommand? refreshIdentityCommand = null; SqliteCommand? refreshMutualCommand = null; try @@ -2425,6 +2478,10 @@ internal void RefreshMutualRecursionFlags( cancellationToken.ThrowIfCancellationRequested(); createUniqueFamiliesCommand = RentCommand(CreateReferenceUniqueFamiliesSql, static _ => { }); createUniqueFamiliesCommand.ExecuteNonQuery(); + createCSharpReferenceFactIndexesCommand = RentCommand( + CreateCSharpReferenceFactIndexesSql, + static _ => { }); + createCSharpReferenceFactIndexesCommand.ExecuteNonQuery(); cancellationToken.ThrowIfCancellationRequested(); var refreshPlan = graphScope == null ? new ReferenceGraphRefreshPlan(true, 0, 0, 0, 0) @@ -2468,6 +2525,7 @@ internal void RefreshMutualRecursionFlags( RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpPropertyTargetFactsFullSql + "\n" + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + RefreshReferenceUniqueFamiliesSql + "\n" + RefreshReferenceCandidatesSql + "\n" + @@ -2482,6 +2540,7 @@ internal void RefreshMutualRecursionFlags( RefreshCSharpSymbolFactsScopedSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + RefreshCSharpConstructorIdentityFactsSql + "\n" + + RefreshCSharpPropertyTargetFactsScopedSql + "\n" + NormalizeCSharpPropertyReceiverReferencesScopedSql + "\n" + RefreshScopedReferenceUniqueFamiliesSql + "\n" + RefreshScopedReferenceCandidatesSql + "\n" + @@ -2539,6 +2598,8 @@ internal void RefreshMutualRecursionFlags( ReleaseCommand(refreshMutualCommand); if (refreshIdentityCommand != null) ReleaseCommand(refreshIdentityCommand); + if (createCSharpReferenceFactIndexesCommand != null) + ReleaseCommand(createCSharpReferenceFactIndexesCommand); if (createUniqueFamiliesCommand != null) ReleaseCommand(createUniqueFamiliesCommand); if (graphScope != null) diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 03118b470..914b6dc82 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -292,6 +292,12 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() var constructorIdentityInsert = sql.IndexOf( "INSERT INTO temp.csharp_constructor_identity_facts", StringComparison.Ordinal); + var propertyTargetDelete = sql.IndexOf( + "DELETE FROM temp.csharp_property_target_facts", + StringComparison.Ordinal); + var propertyTargetInsert = sql.IndexOf( + "INSERT INTO temp.csharp_property_target_facts", + StringComparison.Ordinal); var normalization = sql.IndexOf( "DELETE FROM temp.csharp_type_inheritance", StringComparison.Ordinal); @@ -308,7 +314,9 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() && typeIdentityDelete < typeIdentityInsert && typeIdentityInsert < constructorIdentityDelete && constructorIdentityDelete < constructorIdentityInsert - && constructorIdentityInsert < normalization + && constructorIdentityInsert < propertyTargetDelete + && propertyTargetDelete < propertyTargetInsert + && propertyTargetInsert < normalization && normalization < candidates, $"Unexpected {scope} C# graph fact stage order."); Assert.Equal(1, CountOccurrences(sql, "WITH type_identity_parts(")); @@ -320,6 +328,33 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() Assert.DoesNotContain("reference_graph_lookup_names AS symbol_lookup", fullSql, StringComparison.Ordinal); Assert.Contains("reference_graph_lookup_names AS symbol_lookup", scopedSql, StringComparison.Ordinal); + var propertyStages = DbWriter.CSharpPropertyReceiverFactSqlForTesting; + Assert.Equal(["full", "scoped", "retained"], propertyStages.Select(static stage => stage.Scope)); + foreach (var (scope, materializationSql, normalizationSql) in propertyStages) + { + Assert.Contains("DELETE FROM temp.csharp_property_target_facts", materializationSql, StringComparison.Ordinal); + Assert.Contains("INSERT INTO temp.csharp_property_target_facts", materializationSql, StringComparison.Ordinal); + Assert.Contains("target.kind IN ('field', 'property')", materializationSql, StringComparison.Ordinal); + Assert.DoesNotContain("JOIN symbols AS target", normalizationSql, StringComparison.Ordinal); + Assert.Equal(3, CountOccurrences(normalizationSql, "temp.csharp_property_target_facts AS target")); + Assert.Equal(1, CountOccurrences( + normalizationSql, + "reference_fact.is_property_receiver_reference = 1")); + Assert.Equal(1, CountOccurrences( + normalizationSql, + "reference_fact.is_member_receiver = 1")); + + if (scope == "scoped") + { + Assert.Contains("reference_graph_lookup_names AS property_lookup", materializationSql, StringComparison.Ordinal); + Assert.Contains("symbols AS target INDEXED BY idx_symbols_name_folded", materializationSql, StringComparison.Ordinal); + } + else + { + Assert.DoesNotContain("reference_graph_lookup_names AS property_lookup", materializationSql, StringComparison.Ordinal); + } + } + var candidateStages = DbWriter.CSharpGraphCandidateSqlForTesting; Assert.Equal(["full", "scoped", "retained"], candidateStages.Select(static stage => stage.Scope)); foreach (var (scope, sql) in candidateStages) @@ -352,6 +387,41 @@ static int CountOccurrences(string text, string value) } } + [Fact] + public void CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets() + { + _writer.RefreshMutualRecursionFlags(); + var fullStage = Assert.Single( + DbWriter.CSharpPropertyReceiverFactSqlForTesting, + static stage => stage.Scope == "full"); + var updates = fullStage.NormalizationSql.Split( + ';', + StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(static statement => statement.StartsWith( + "UPDATE symbol_references", + StringComparison.Ordinal)) + .ToArray(); + Assert.Equal(2, updates.Length); + + foreach (var update in updates) + { + var plan = ReadQueryPlanDetails(_db.Connection, update); + Assert.Contains( + plan, + static detail => detail.Contains( + "SEARCH r USING INTEGER PRIMARY KEY", + StringComparison.Ordinal)); + Assert.DoesNotContain( + plan, + static detail => detail.StartsWith("SCAN r ", StringComparison.Ordinal)); + Assert.True( + plan.Any(static detail => detail.Contains( + "SEARCH target USING PRIMARY KEY", + StringComparison.Ordinal)), + string.Join(Environment.NewLine, plan)); + } + } + [Fact] public void CSharpGraphIdentityFacts_PreserveFullScopedAndRetainedConstructorResolution() { From ebf3ecbd5cf5cf4cb724a331317aa0e6b49afef5 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 14 Aug 2026 00:03:56 +0900 Subject: [PATCH 09/12] Assign fresh reference sources during insertion --- DEVELOPER_GUIDE.md | 9 ++ TESTING_GUIDE.md | 4 +- ...sh-reference-source-attribution.changed.md | 14 ++ .../DbWriter.ReferenceGraphRefreshScope.cs | 4 +- .../Database/DbWriter.ReferenceSql.cs | 67 +++++++++- src/CodeIndex/Database/DbWriter.References.cs | 52 +++++--- .../FreshReferenceResolutionTests.cs | 124 ++++++++++++++++++ .../IndexCommandRunnerFullScanTests.cs | 81 ++++++++++++ 8 files changed, 327 insertions(+), 28 deletions(-) create mode 100644 changelog.d/unreleased/+fresh-reference-source-attribution.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 887490d15..e21253826 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1604,6 +1604,11 @@ SQLite resolves referenced tables while preparing every statement in a command b A true empty-database ordinary CLI full scan (not `--rebuild` or `--symbols-only`) opts into a separate fresh-resolution contract. Reference inserts persist canonical provisional values (`unresolved`, candidate count zero, and zero self/mutual flags) without adding bind parameters. +The fresh CTE also assigns `source_symbol_id` from the same-file symbols already persisted before +each file's references, using the ordinary narrowest-containing-range tie-break and a literal input +ordinal to preserve batch order. Finalization therefore omits the all-reference source-identity +UPDATE on this authoritative path. Ordinary full, differential, scoped, rebuild, retained, and MCP +paths keep their established source refreshes. The early empty observation is advisory: immediately after the authoritative outer write transaction begins, the CLI rechecks `files`, `symbols`, and `symbol_references` in that transaction. If another connection committed any row during the pre-write gap, the graph scope @@ -5409,6 +5414,10 @@ prepared command で作成してください。SQLite は command batch の全st 真に空のdatabaseから始める通常のCLI full scan（`--rebuild` と `--symbols-only` を除く）だけは、 fresh resolution専用の契約をopt-inします。reference insertはbind parameterを増やさず、 `unresolved`、candidate count 0、self/mutual flag 0というcanonicalな暫定値を永続化します。 +fresh CTEは各fileのreferenceより先に永続化済みの同一file symbolから`source_symbol_id`も設定し、 +通常経路と同じ最小包含rangeのtie-breakを使い、literalのinput ordinalでbatch順序を維持します。 +このauthoritative経路のfinalizationはreference全件のsource-identity UPDATEを省略します。通常full、 +differential、scoped、rebuild、retained、MCP経路は従来のsource refreshを維持します。 早期のempty確認はadvisoryです。authoritativeなouter write transaction開始直後に、CLIは同じ transaction内で`files`、`symbols`、`symbol_references`を再確認します。write前のgapで別connectionが 1行でもcommitしていた場合は、最初のrowを永続化する前にgraph scopeのfresh insert defaultを無効化し、 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 30b5d3099..4b0353a8d 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -39,7 +39,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. - Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Lowering or raising the file-size policy must reprocess unchanged files in CLI and MCP indexing so a prior `file_too_large` issue cannot be reused. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps and must not label an incomplete index as fold-only. Human output must identify incomplete generations instead of printing a complete summary. - Reference-identity refresh coverage treats a stable graph rebuild as a physical-write performance contract. Keep NULL-safe changed-row predicates for source identity, the four-column target-resolution tuple, self-reference, and mutual-recursion updates; trigger audits must remain at zero on a stable rerun, repair each corrupted phase once, and prove a later-phase failure rolls back earlier identity writes. SQLite `changes()` must continue to report the final mutual-recursion phase. -- Fresh-reference resolution coverage belongs in `FreshReferenceResolutionTests`. Keep the empty/rebuild/symbols-only policy truth table, the unchanged 14-parameter insert shape with distinct fresh/ordinary SQL cache entries, canonical provisional values, the materialized candidate-side aggregation shape, and exact fresh-versus-full semantic parity for `unresolved`, `resolved`, `resolved_group`, `ambiguous`, and self-reference rows across C# and a non-C# language. Failure tests must prove that fresh defaults remain pending after graph rollback and clear only after a successful commit. Pair these database tests with the full-scan bulk-load theory that observes `unresolved` provisional rows on a fresh CLI scan and NULL resolution state on rebuild. MCP remains outside this opt-in because its per-file durable transactions require the existing graph-failure retry contract. +- Fresh-reference resolution coverage belongs in `FreshReferenceResolutionTests`. Keep the empty/rebuild/symbols-only policy truth table, the unchanged 14-parameter insert shape with distinct fresh/ordinary SQL cache entries, canonical provisional values, the materialized candidate-side aggregation shape, and exact fresh-versus-full semantic parity for `unresolved`, `resolved`, `resolved_group`, `ambiguous`, and self-reference rows across C# and a non-C# language. The fresh insert must preserve literal input order, choose same-file nested source symbols with the ordinary containment tie-break across C# and Python, leave out-of-range sources NULL, and omit the final source UPDATE; ordinary full and differential repair paths remain covered separately. Failure tests must prove that fresh defaults remain pending after graph rollback and clear only after a successful commit. Pair these database tests with the full-scan bulk-load theory that observes `unresolved` provisional rows on a fresh CLI scan and NULL resolution state on rebuild. MCP remains outside this opt-in because its per-file durable transactions require the existing graph-failure retry contract. A full-scan barrier regression must commit a candidate-free row from a second connection after the early empty observation but before the outer write transaction, then prove transaction-local revalidation switches to ordinary full resolution and normalizes that row. - C# reference-graph fact coverage is an SQL-shape performance contract. Each of the six managed arity/receiver/value-type functions must occur once only in its reference or symbol materialization statement, and full, scoped, and retained refresh chains must populate reference, symbol, type-identity, and constructor-identity facts in that order before normalization and candidate consumers. Candidate SQL must use the WITHOUT ROWID fact primary keys without retaining project/file-local identity construction or constructor-owner range scans. Keep full/scoped/retained semantic snapshots across partial generic and file-local constructors, primary constructors with same-leaf generic arities, scoped target-definition mutations, fallback owners, legacy NULL identities, stale-row cleanup, and refresh rollback/retry alongside the focused property-receiver regressions. - C# metadata-target resolver coverage treats propagation work and stable reruns as performance contracts. Keep the reverse-ordered 8,000-class chain at exactly `n - 1` dependency edges and `n` queue visits instead of using a wall-clock threshold; retain cross-file partial fan-in, an unseeded cycle, pre-cancellation, rollback of an earlier row after an injected later update failure, zero trigger-audited writes on a stable rerun, and exactly one write when repairing a corrupted derived row. @@ -1110,7 +1110,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 - index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。file-size policy を下げた場合も上げた場合も、CLI / MCP indexing は unchanged file を再処理し、以前の `file_too_large` issue を再利用してはいけません。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別し、incomplete index を fold-only と表示しない必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 - reference identity refresh coverage は、安定graphの再構築を物理writeのperformance contractとします。source identity、target resolutionの4列tuple、self reference、mutual recursionの更新にはNULL-safeなchanged-row predicateを維持し、安定rerunのtrigger auditは0、各corrupt phaseのrepairは1回、後段phaseの失敗で先行identity writeもrollbackされることを検証してください。SQLite `changes()` は引き続き最後のmutual-recursion phaseを表します。 -- fresh reference resolutionのcoverageは`FreshReferenceResolutionTests`が担当します。empty/rebuild/symbols-onlyのpolicy truth table、fresh/ordinaryでSQL cache entryを分けても14 parameterのinsert shapeが不変であること、canonicalな暫定値、candidate側materialized aggregationのSQL shape、およびC#と非C#言語を横断する`unresolved`、`resolved`、`resolved_group`、`ambiguous`、self-referenceのfresh/full完全同値を維持してください。failure testはgraph rollback後もfresh defaultsがpendingのままで、成功commit後にだけ解除されることを証明します。database testは、fresh CLI scanで暫定`unresolved`、rebuildでNULL resolution stateを観測するfull-scan bulk-load theoryと対にしてください。MCPはfile単位のdurable transactionが既存graph失敗再試行契約を必要とするため、このopt-inの対象外です。 +- fresh reference resolutionのcoverageは`FreshReferenceResolutionTests`が担当します。empty/rebuild/symbols-onlyのpolicy truth table、fresh/ordinaryでSQL cache entryを分けても14 parameterのinsert shapeが不変であること、canonicalな暫定値、candidate側materialized aggregationのSQL shape、およびC#と非C#言語を横断する`unresolved`、`resolved`、`resolved_group`、`ambiguous`、self-referenceのfresh/full完全同値を維持してください。fresh insertはliteral input順序を保ち、C# / Pythonの同一file nested source symbolを通常の包含tie-breakで選び、range外sourceをNULLのままにして、最終source UPDATEを省略する必要があります。通常fullとdifferentialのrepair経路は別途coverageを維持します。failure testはgraph rollback後もfresh defaultsがpendingのままで、成功commit後にだけ解除されることを証明します。database testは、fresh CLI scanで暫定`unresolved`、rebuildでNULL resolution stateを観測するfull-scan bulk-load theoryと対にしてください。MCPはfile単位のdurable transactionが既存graph失敗再試行契約を必要とするため、このopt-inの対象外です。 full-scan barrier回帰では、早期empty確認後からouter write transaction開始前の間に別connectionからcandidate-free rowをcommitし、transaction内の再検証が通常のfull resolutionへ切り替わってそのrowを正規化することを必須とします。 - C# reference-graph fact coverage は SQL shape の performance contract です。arity / receiver / value-type を求める6つの managed function は reference または symbol の materialization statement 内にそれぞれ1回だけ置き、full / scoped / retained refresh chain は reference、symbol、type identity、constructor identity の fact をこの順で normalization と candidate consumer より前に投入してください。candidate SQL は WITHOUT ROWID fact の主キーを使い、project / file-local identity の再構築や constructor-owner range の再走査を残してはいけません。partial generic / file-local constructor、同名別 generic arity を持つ primary constructor、scoped target 定義変更、fallback owner、legacy NULL identity、stale row cleanup、refresh rollback / retry の full / scoped / retained semantic snapshot と、focused property-receiver regression を維持します。 - C# metadata-target resolver coverage は propagation work と安定 rerun を performance contract とします。逆順に保存した 8,000 class の chain では wall-clock threshold を使わず、dependency edge が厳密に `n - 1`、queue visit が `n` であることを維持してください。cross-file partial fan-in、seed を持たない cycle、事前 cancel、後段 update の注入失敗時に先行 row も rollback されること、安定 rerun の trigger audit が write 0 回、破損した derived row の修復が厳密に 1 write であることも残します。 diff --git a/changelog.d/unreleased/+fresh-reference-source-attribution.changed.md b/changelog.d/unreleased/+fresh-reference-source-attribution.changed.md new file mode 100644 index 000000000..cb9fc8c89 --- /dev/null +++ b/changelog.d/unreleased/+fresh-reference-source-attribution.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.ReferenceSql.cs + - src/CodeIndex/Database/DbWriter.References.cs +--- + +## English + +- **Fresh full indexes now assign reference sources during insertion** — the authoritative empty-database path resolves each language-neutral reference to its narrowest containing same-file symbol while preserving the 14-parameter batch shape, so graph finalization no longer scans and rewrites every reference solely to add source identity. + +## 日本語 + +- **fresh full indexがreference挿入時にsourceを設定するようになりました** — authoritativeなempty-database経路は14 parameterのbatch shapeを維持しつつ、言語共通の各referenceを同一file内で最も狭く包含するsymbolへ解決するため、graph finalizationはsource identity追加だけのために全referenceをscan・rewriteしません。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 208e1385a..7fff5842f 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -39,12 +39,12 @@ internal static Action? ReferenceGraphRowCountForTesting private static readonly string RefreshScopedReferenceSourceSymbolsSql = $""" UPDATE symbol_references AS r - SET source_symbol_id = {ReferenceSourceSymbolValueSql} + SET source_symbol_id = {BuildReferenceSourceSymbolValueSql("r")} WHERE r.id IN ( SELECT reference_id FROM temp.{ReferenceGraphDirtyReferencesTable} ) - AND r.source_symbol_id IS NOT {ReferenceSourceSymbolValueSql}; + AND r.source_symbol_id IS NOT {BuildReferenceSourceSymbolValueSql("r")}; """; private static string RefreshCSharpReferenceFactsScopedSql => diff --git a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs index 7a3a9b33d..5aee9ffe8 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs @@ -27,14 +27,63 @@ private static string BuildReferenceInsertSql( bool useFreshReferenceResolutionDefaults) { var sql = CreateBatchSqlBuilder(rowCount, estimatedCharsPerRow: 256); + if (useFreshReferenceResolutionDefaults) + { + sql.Append(@" + WITH fresh_reference( + input_ordinal, + file_id, symbol_name, reference_kind, line, column_number, span_length, + context, reference_line_id, container_kind, container_name, + symbol_name_folded, container_name_folded, is_self_reference, + is_mutual_recursion, target_qualifier) AS ( + VALUES "); + var freshParameterIndex = 0; + for (var row = 0; row < rowCount; row++) + { + if (row > 0) + sql.Append(", "); + AppendReferenceInsertParameterTuple( + sql, + ref freshParameterIndex, + row); + } + sql.Append($@" + ) + INSERT INTO symbol_references ( + file_id, symbol_name, reference_kind, line, column_number, span_length, + context, reference_line_id, container_kind, container_name, + symbol_name_folded, container_name_folded, is_self_reference, + is_mutual_recursion, target_qualifier, source_symbol_id, + resolution_state, resolution_candidate_count) + SELECT r.file_id, + r.symbol_name, + r.reference_kind, + r.line, + r.column_number, + r.span_length, + r.context, + r.reference_line_id, + r.container_kind, + r.container_name, + r.symbol_name_folded, + r.container_name_folded, + r.is_self_reference, + r.is_mutual_recursion, + r.target_qualifier, + {BuildReferenceSourceSymbolValueSql("r")}, + 'unresolved', + 0 + FROM fresh_reference AS r + ORDER BY r.input_ordinal"); + return sql.ToString(); + } + sql.Append(@" INSERT INTO symbol_references ( file_id, symbol_name, reference_kind, line, column_number, span_length, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, is_mutual_recursion, target_qualifier"); - if (useFreshReferenceResolutionDefaults) - sql.Append(", resolution_state, resolution_candidate_count"); sql.Append(@" ) VALUES "); @@ -45,18 +94,24 @@ INSERT INTO symbol_references ( sql.Append(", "); AppendReferenceInsertParameterTuple( sql, - ref parameterIndex, - useFreshReferenceResolutionDefaults); + ref parameterIndex); } return sql.ToString(); } + internal static string BuildReferenceInsertSqlForTesting( + int rowCount, + bool useFreshReferenceResolutionDefaults) + => BuildReferenceInsertSql(rowCount, useFreshReferenceResolutionDefaults); + private static void AppendReferenceInsertParameterTuple( StringBuilder sql, ref int parameterIndex, - bool useFreshReferenceResolutionDefaults) + int? inputOrdinal = null) { sql.Append('('); + if (inputOrdinal is { } ordinal) + sql.Append(ordinal).Append(", "); for (var column = 0; column < 15; column++) { if (column > 0) @@ -66,8 +121,6 @@ private static void AppendReferenceInsertParameterTuple( else sql.Append("@p").Append(parameterIndex++); } - if (useFreshReferenceResolutionDefaults) - sql.Append(", 'unresolved', 0"); sql.Append(')'); } diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 1b2f966b1..7193c2303 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -96,17 +96,18 @@ ELSE 0 END """; - private const string ReferenceSourceSymbolValueSql = """ + private static string BuildReferenceSourceSymbolValueSql(string referenceAlias) + => $""" ( SELECT s.id FROM symbols AS s - WHERE s.file_id = r.file_id - AND r.container_name IS NOT NULL - AND r.container_name <> '' - AND (s.name_folded = r.container_name_folded - OR s.display_name_folded = r.container_name_folded - OR (s.name_folded IS NULL AND s.name = r.container_name COLLATE NOCASE)) - AND r.line BETWEEN COALESCE(s.start_line, s.line) AND COALESCE(s.end_line, s.line) + WHERE s.file_id = {referenceAlias}.file_id + AND {referenceAlias}.container_name IS NOT NULL + AND {referenceAlias}.container_name <> '' + AND (s.name_folded = {referenceAlias}.container_name_folded + OR s.display_name_folded = {referenceAlias}.container_name_folded + OR (s.name_folded IS NULL AND s.name = {referenceAlias}.container_name COLLATE NOCASE)) + AND {referenceAlias}.line BETWEEN COALESCE(s.start_line, s.line) AND COALESCE(s.end_line, s.line) ORDER BY (COALESCE(s.end_line, s.line) - COALESCE(s.start_line, s.line)), COALESCE(s.start_line, s.line) DESC, s.id @@ -116,17 +117,33 @@ LIMIT 1 private static readonly string RefreshReferenceSourceSymbolsFullSql = $""" UPDATE symbol_references AS r - SET source_symbol_id = {ReferenceSourceSymbolValueSql} + SET source_symbol_id = {BuildReferenceSourceSymbolValueSql("r")} """; private static readonly string RefreshReferenceSourceSymbolsDifferentialSql = $""" UPDATE symbol_references AS r - SET source_symbol_id = {ReferenceSourceSymbolValueSql} + SET source_symbol_id = {BuildReferenceSourceSymbolValueSql("r")} -- IS NOT is null-safe: stable NULL identities must not be rewritten either. -- IS NOTはNULL-safeであり、安定したNULL identityも再書込みしない。 - WHERE r.source_symbol_id IS NOT {ReferenceSourceSymbolValueSql} + WHERE r.source_symbol_id IS NOT {BuildReferenceSourceSymbolValueSql("r")} """; + private static string? SelectReferenceSourceRefreshSql( + bool useFreshReferenceResolutionDefaults, + bool hasPersistedReferenceResolutionState) + => useFreshReferenceResolutionDefaults + ? null + : hasPersistedReferenceResolutionState + ? RefreshReferenceSourceSymbolsDifferentialSql + : RefreshReferenceSourceSymbolsFullSql; + + internal static string? SelectReferenceSourceRefreshSqlForTesting( + bool useFreshReferenceResolutionDefaults, + bool hasPersistedReferenceResolutionState) + => SelectReferenceSourceRefreshSql( + useFreshReferenceResolutionDefaults, + hasPersistedReferenceResolutionState); + private static readonly string CreateReferenceUniqueFamiliesSql = $""" CREATE TEMP TABLE IF NOT EXISTS reference_unique_symbol_families ( lang TEXT NOT NULL, @@ -2510,17 +2527,18 @@ internal void RefreshMutualRecursionFlags( { var hasPersistedReferenceResolutionState = !useFreshReferenceResolutionDefaults && HasPersistedReferenceResolutionState(cancellationToken); - var refreshReferenceSourcesSql = useFreshReferenceResolutionDefaults - ? RefreshReferenceSourceSymbolsFullSql - : hasPersistedReferenceResolutionState - ? RefreshReferenceSourceSymbolsDifferentialSql - : RefreshReferenceSourceSymbolsFullSql; + var refreshReferenceSourcesSql = SelectReferenceSourceRefreshSql( + useFreshReferenceResolutionDefaults, + hasPersistedReferenceResolutionState); var refreshReferenceResolutionSql = useFreshReferenceResolutionDefaults ? RefreshReferenceResolutionFreshSparseSql : hasPersistedReferenceResolutionState ? RefreshReferenceResolutionDifferentialSql : RefreshReferenceResolutionFullSql; - refreshIdentitySql = refreshReferenceSourcesSql + ";\n" + + refreshIdentitySql = + (refreshReferenceSourcesSql == null + ? string.Empty + : refreshReferenceSourcesSql + ";\n") + RefreshCSharpReferenceFactsFullSql + "\n" + RefreshCSharpSymbolFactsFullSql + "\n" + RefreshCSharpTypeIdentityFactsSql + "\n" + diff --git a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs index 6f2c3a6ac..5bf015ed8 100644 --- a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs +++ b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs @@ -75,6 +75,7 @@ public void FreshDefaultsRevalidation_RequiresTransactionAndRejectsPersistedRows public void InsertReferences_FreshDefaultsKeepParameterShapeAndUseSeparateCachedSql() { var fileId = InsertFile("src/provisional.py", "python"); + _writer.InsertSymbols([CreateSymbol(fileId, "Caller", line: 1)]); var observedWork = new List(); var previousHook = DbWriter.ReferenceInsertBindingWorkForTesting; try @@ -128,6 +129,97 @@ public void InsertReferences_FreshDefaultsKeepParameterShapeAndUseSeparateCached Assert.Equal( new ProvisionalRow(null, 0, 1, 1), ReadProvisionalRow("Standard")); + Assert.Equal( + 1, + ScalarLong(""" + SELECT COUNT(*) + FROM symbol_references + WHERE symbol_name = 'Fresh' + AND source_symbol_id IS NOT NULL + """)); + Assert.Equal( + 1, + ScalarLong(""" + SELECT COUNT(*) + FROM symbol_references + WHERE symbol_name = 'Standard' + AND source_symbol_id IS NULL + """)); + + var freshSql = DbWriter.BuildReferenceInsertSqlForTesting( + rowCount: 2, + useFreshReferenceResolutionDefaults: true); + var standardSql = DbWriter.BuildReferenceInsertSqlForTesting( + rowCount: 2, + useFreshReferenceResolutionDefaults: false); + Assert.Contains("WITH fresh_reference(", freshSql, StringComparison.Ordinal); + Assert.Contains("input_ordinal", freshSql, StringComparison.Ordinal); + Assert.Contains("source_symbol_id", freshSql, StringComparison.Ordinal); + Assert.Contains("FROM fresh_reference AS r", freshSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY r.input_ordinal", freshSql, StringComparison.Ordinal); + Assert.Contains("ORDER BY (COALESCE(s.end_line", freshSql, StringComparison.Ordinal); + Assert.Equal(28, CountOccurrences(freshSql, "@p")); + Assert.DoesNotContain("WITH fresh_reference(", standardSql, StringComparison.Ordinal); + Assert.DoesNotContain("source_symbol_id", standardSql, StringComparison.Ordinal); + Assert.Equal(28, CountOccurrences(standardSql, "@p")); + + var freshRefresh = DbWriter.SelectReferenceSourceRefreshSqlForTesting( + useFreshReferenceResolutionDefaults: true, + hasPersistedReferenceResolutionState: false); + var ordinaryFull = DbWriter.SelectReferenceSourceRefreshSqlForTesting( + useFreshReferenceResolutionDefaults: false, + hasPersistedReferenceResolutionState: false); + var differential = DbWriter.SelectReferenceSourceRefreshSqlForTesting( + useFreshReferenceResolutionDefaults: false, + hasPersistedReferenceResolutionState: true); + Assert.Null(freshRefresh); + Assert.DoesNotContain("r.source_symbol_id IS NOT", ordinaryFull, StringComparison.Ordinal); + Assert.Contains("r.source_symbol_id IS NOT", differential, StringComparison.Ordinal); + } + + [Fact] + public void FreshReferenceInsert_AssignsCrossLanguageNestedSourcesWithoutFinalUpdate() + { + var csharpFileId = InsertFile("src/nested-source.cs", "csharp"); + var pythonFileId = InsertFile("src/nested_source.py", "python"); + _writer.InsertSymbols([ + CreateRangedSymbol(csharpFileId, "Caller", startLine: 1, endLine: 30), + CreateRangedSymbol(csharpFileId, "Caller", startLine: 10, endLine: 20), + CreateRangedSymbol(pythonFileId, "Caller", startLine: 1, endLine: 30), + CreateRangedSymbol(pythonFileId, "Caller", startLine: 10, endLine: 20), + ]); + + using var freshScope = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + _writer.InsertReferences([ + CreateReference(csharpFileId, "CsOuter", line: 5), + CreateReference(csharpFileId, "CsNested", line: 15), + CreateReference(csharpFileId, "CsOutside", line: 31), + CreateReference(pythonFileId, "PyOuter", line: 5), + CreateReference(pythonFileId, "PyNested", line: 15), + CreateReference(pythonFileId, "PyOutside", line: 31), + ], refreshMutualRecursionFlags: false); + + Assert.Equal(1, ReadSourceLine("CsOuter")); + Assert.Equal(10, ReadSourceLine("CsNested")); + Assert.Null(ReadSourceLine("CsOutside")); + Assert.Equal(1, ReadSourceLine("PyOuter")); + Assert.Equal(10, ReadSourceLine("PyNested")); + Assert.Null(ReadSourceLine("PyOutside")); + Execute(""" + CREATE TEMP TRIGGER reject_fresh_source_rewrite + BEFORE UPDATE OF source_symbol_id ON symbol_references + BEGIN + SELECT RAISE(ABORT, 'fresh source identity must be insert-complete'); + END; + """); + + _writer.RefreshMutualRecursionFlags(stampReferenceIdentityContractReady: false); + + Execute("DROP TRIGGER reject_fresh_source_rewrite;"); + Assert.Equal(10, ReadSourceLine("CsNested")); + Assert.Equal(10, ReadSourceLine("PyNested")); } [Fact] @@ -440,6 +532,22 @@ private static SymbolRecord CreateSymbol( ContainerQualifiedName = container, }; + private static SymbolRecord CreateRangedSymbol( + long fileId, + string name, + int startLine, + int endLine) + => new() + { + FileId = fileId, + Kind = "function", + Name = name, + Line = startLine, + StartLine = startLine, + EndLine = endLine, + Signature = $"function {name}()", + }; + private static ReferenceRecord CreateReference( long fileId, string symbolName, @@ -481,6 +589,22 @@ FROM symbol_references reader.GetInt32(3)); } + private int? ReadSourceLine(string symbolName) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = """ + SELECT source.line + FROM symbol_references AS reference + LEFT JOIN symbols AS source ON source.id = reference.source_symbol_id + WHERE reference.symbol_name = @symbol_name + """; + command.Parameters.AddWithValue("@symbol_name", symbolName); + var value = command.ExecuteScalar(); + return value == null || value == DBNull.Value + ? null + : Convert.ToInt32(value, CultureInfo.InvariantCulture); + } + private ResolutionRow ReadResolutionRow(string path, int line) { using var command = _db.Connection.CreateCommand(); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 40b7991cf..1c7d1851f 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -3493,6 +3493,87 @@ FROM symbol_references AS r } } + [Fact] + public void Run_FreshFullScan_PersistsNestedReferenceSourcesAcrossLanguages() + { + var projectRoot = CreateTempProject(); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "Caller.cs"), + """ + public sealed class Caller + { + public void Outer() + { + Target(); + void Inner() + { + Target(); + } + } + private static void Target() { } + } + """); + File.WriteAllText( + Path.Combine(projectRoot, "caller.py"), + """ + def outer(): + python_target() + def inner(): + python_target() + """); + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var command = db.Connection.CreateCommand(); + command.CommandText = + """ + SELECT source_file.path, + reference.line, + source_symbol.name + FROM symbol_references AS reference + JOIN files AS source_file ON source_file.id = reference.file_id + LEFT JOIN symbols AS source_symbol ON source_symbol.id = reference.source_symbol_id + WHERE (source_file.path = 'Caller.cs' + AND reference.symbol_name = 'Target' + AND reference.reference_kind = 'call') + OR (source_file.path = 'caller.py' + AND reference.symbol_name = 'python_target' + AND reference.reference_kind = 'call') + ORDER BY source_file.path COLLATE BINARY, + reference.line + """; + using var reader = command.ExecuteReader(); + var sources = new List<(string Path, long Line, string? Source)>(); + while (reader.Read()) + { + sources.Add(( + reader.GetString(0), + reader.GetInt64(1), + reader.IsDBNull(2) ? null : reader.GetString(2))); + } + + Assert.Equal( + [ + ("Caller.cs", 5L, "Outer"), + ("Caller.cs", 8L, "Inner"), + ("caller.py", 2L, "outer"), + ("caller.py", 4L, "inner"), + ], + sources); + } + finally + { + DeleteDirectory(projectRoot); + SqliteConnection.ClearAllPools(); + } + } + [Fact] public void Run_FullScan_FreshSnapshotAbortRetainsDiscoveredLanguageFailuresWithoutRows() { From 79f1db629aec52b905a748485804492be15ce1a0 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 14 Aug 2026 00:25:57 +0900 Subject: [PATCH 10/12] Match C# type-reference families once --- DEVELOPER_GUIDE.md | 13 + TESTING_GUIDE.md | 4 +- .../+csharp-type-family-candidates.changed.md | 14 + src/CodeIndex/Database/DbWriter.References.cs | 101 +++++--- tests/CodeIndex.Tests/DatabaseTests.cs | 242 ++++++++++++++++++ 5 files changed, 338 insertions(+), 36 deletions(-) create mode 100644 changelog.d/unreleased/+csharp-type-family-candidates.changed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e21253826..e5729719a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -429,6 +429,13 @@ ambiguity contracts remain unchanged. Scoped refreshes must build the set by driving from dirty reference IDs into the candidate primary key, and every graph pass must clear it before materialization so retries cannot observe stale rows. +The unqualified C# rank-5 type fallback materializes physical type members from +the shared symbol and type-identity facts, groups them into unique logical +families by exact name, arity, and identity, and matches each reference to that +family once. Only the final projection expands a matched family back to every +physical member. This preserves row-per-symbol candidates for partial types while +avoiding repeated compatibility and ambiguity work for every partial declaration. + Resolution also materializes the nullable target-family key once per target symbol into a primary-keyed TEMP fact table. Full, fresh, differential, and retained refreshes populate all symbols; scoped refreshes first deduplicate target symbol @@ -4291,6 +4298,12 @@ ambiguity 契約は変更しません。scoped refresh は dirty reference ID seek して集合を作り、retry が古い行を参照しないよう graph pass ごとに materialize 前の clear を 維持してください。 +qualifier のない C# rank 5 type fallback は、共有 symbol / type-identity fact から物理 type member を +materializeし、exact name・arity・identityごとの一意な論理familyへgroup化して、referenceごとの照合を +family単位で1回だけ行います。一致したfamilyを全物理memberへ展開するのは最終projectionだけです。 +これによりpartial typeのsymbolごとのcandidate行を維持しつつ、各partial宣言でcompatibilityとambiguity +判定を繰り返しません。 + resolution は nullable な target-family key も target symbol ごとに1回だけ primary-keyed TEMP fact table へ materialize します。full / fresh / differential / retained refresh は全 symbol を投入し、 scoped refresh は dirty-reference candidate から到達する target symbol ID を先に重複排除します。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 4b0353a8d..704b356b0 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -750,7 +750,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` fixes atomic reference-line windows to a worst-case arithmetic bound. Keep it paired with whole-batch grouping, 32-batch caps, cross-boundary context reuse, rollback, and cancellation tests so the materializer remains the only tuple-hash pass without changing persistence boundaries. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. - `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. Keep the full/scoped/retained resolution oracles beside this structural guard so physical candidate rows and multi-language ambiguity remain unchanged. + `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. The C# type fallback must materialize physical members, unique logical families, and matched families in that order before its final physical expansion; retain primary-key seeks for scoped symbols, references, and facts. `CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember` proves that same-identity partial declarations still produce every physical candidate while a conflicting identity suppresses the whole rank-5 family across scoped, full, and retained refreshes. Keep the remaining full/scoped/retained resolution oracles beside these structural guards so physical candidate rows and multi-language ambiguity remain unchanged. `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` requires both normalization updates to seek flagged reference IDs and the primary-keyed field/property target facts rather than scan all references or persistent target symbols. Keep its full/scoped/retained stage-order assertions and property-resolution fixtures paired so lookup-name scoping cannot change inherited-member semantics. `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` fixes target-family key construction to one per target symbol and candidate resolution to TEMP primary-key facts across fresh, full, differential, scoped, and retained paths. Its legacy-null-key and C#/Python oracle coverage preserves resolved IDs, exact keys, grouped families, ambiguity, and self-reference semantics. - `HotspotReferenceAggregateTests.cs` @@ -1823,7 +1823,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` はatomic reference-line windowを最悪ケースの算術境界へ固定します。materializerだけをtuple-hash passとして保ちつつ永続化境界を変えないよう、whole-batch grouping、32-batch cap、batch境界をまたぐcontext再利用、rollback、cancellation testと対で維持してください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 - `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。物理candidate行と多言語ambiguityが変わらないよう、この構造guardとfull / scoped / retained resolution oracleを対で維持してください。 + `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。C# type fallbackは最終の物理展開より前に、物理member、一意な論理family、一致familyの順でmaterializeし、scoped symbol / reference / factのprimary-key seekを維持してください。`CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember`は、同一identityのpartial宣言が引き続き全物理candidateを生成し、競合identityがscoped / full / retained refreshを横断してrank 5 family全体を抑止することを証明します。物理candidate行と多言語ambiguityが変わらないよう、残りのfull / scoped / retained resolution oracleもこれらの構造guardと対で維持してください。 `DatabaseTests.CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets` は、2つのnormalization updateが全referenceや永続target symbolをscanせず、flag済みreference IDとprimary-keyed field / property target factをseekすることを要求します。lookup-name scopeが継承member semanticsを変えないよう、full / scoped / retainedのstage-order assertionとproperty-resolution fixtureを対で維持してください。 `FreshReferenceResolutionTests.ReferenceResolutionFacts_ConstructTargetKeysOnceAcrossEveryRefreshScope` は、target-family key構築をtarget symbolごと1回に限定し、fresh / full / differential / scoped / retainedのcandidate resolutionがTEMP primary-key factを使う契約を固定します。legacy null-keyとC# / Python oracle coverageにより、resolved ID、exact key、group family、ambiguity、self-reference semanticsを維持します。 - `HotspotReferenceAggregateTests.cs` diff --git a/changelog.d/unreleased/+csharp-type-family-candidates.changed.md b/changelog.d/unreleased/+csharp-type-family-candidates.changed.md new file mode 100644 index 000000000..65a7d9181 --- /dev/null +++ b/changelog.d/unreleased/+csharp-type-family-candidates.changed.md @@ -0,0 +1,14 @@ +--- +category: changed +affected: + - src/CodeIndex/Database/DbWriter.References.cs + - tests/CodeIndex.Tests/DatabaseTests.cs +--- + +## English + +- **Initial C# type-reference matching now evaluates partial families once** — rank-5 candidate construction groups exact name, arity, and type identity before matching references, then expands successful families back to every physical symbol, preserving candidate and ambiguity behavior without repeating the same compatibility work for each partial declaration. + +## 日本語 + +- **初回C# type-reference照合がpartial familyごとに1回だけ評価されるようになりました** — rank 5 candidate構築はexact name・arity・type identityでgroup化してからreferenceを照合し、一致familyを全物理symbolへ展開するため、candidateとambiguityの動作を維持しながら各partial宣言で同じcompatibility処理を繰り返しません。 diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 7193c2303..1eecef672 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -1270,51 +1270,84 @@ WHERE lower_rank_candidate.scope_rank < 5 GROUP BY lower_rank_candidate.reference_id; INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) - SELECT r.id, unique_target.symbol_id, 5 - FROM symbol_references AS r - JOIN files AS source_file ON source_file.id = r.file_id - JOIN ( - SELECT type_symbol.id AS symbol_id, + WITH csharp_type_reference_members( + symbol_id, + name_folded, + name, + type_arity, + type_identity) AS MATERIALIZED ( + SELECT type_symbol.id, type_symbol.name_folded, type_symbol.name, - {BuildCSharpDefinitionTypeAritySql("type_symbol")} AS type_arity + type_symbol_fact.definition_type_arity, + type_identity_fact.type_identity FROM symbols AS type_symbol JOIN files AS target_file ON target_file.id = type_symbol.file_id + CROSS JOIN temp.csharp_symbol_facts AS type_symbol_fact + ON type_symbol_fact.symbol_id = type_symbol.id + CROSS JOIN temp.csharp_type_identity_facts AS type_identity_fact + ON type_identity_fact.symbol_id = type_symbol.id WHERE target_file.lang = 'csharp' AND type_symbol.name_folded IS NOT NULL AND type_symbol.kind IN ('class', 'struct', 'record', 'interface', 'enum', 'delegate') - AND {BuildCSharpDefinitionTypeAritySql("type_symbol")} IS NOT NULL + AND type_symbol_fact.definition_type_arity IS NOT NULL + ), + csharp_unique_type_reference_families( + name_folded, + name, + type_arity, + type_identity) AS MATERIALIZED ( + SELECT type_member.name_folded, + type_member.name, + type_member.type_arity, + MIN(type_member.type_identity COLLATE BINARY) + FROM csharp_type_reference_members AS type_member + GROUP BY type_member.name_folded, + type_member.name, + type_member.type_arity + HAVING COUNT(DISTINCT type_member.type_identity COLLATE BINARY) = 1 + ), + matched_csharp_type_reference_families( + reference_id, + name_folded, + name, + type_arity, + type_identity) AS MATERIALIZED ( + SELECT r.id, + unique_family.name_folded, + unique_family.name, + unique_family.type_arity, + unique_family.type_identity + FROM symbol_references AS r + JOIN files AS source_file ON source_file.id = r.file_id + JOIN csharp_unique_type_reference_families AS unique_family + ON unique_family.name_folded = r.symbol_name_folded + AND unique_family.name = r.symbol_name COLLATE BINARY + LEFT JOIN temp.csharp_reference_facts AS reference_fact + ON reference_fact.reference_id = r.id + WHERE source_file.lang = 'csharp' + AND r.target_qualifier IS NULL + AND r.reference_kind = 'type_reference' + AND ( + reference_fact.type_arity IS NULL + OR unique_family.type_arity = reference_fact.type_arity + ) AND NOT EXISTS ( SELECT 1 - FROM symbols AS other_type - JOIN files AS other_type_file - ON other_type_file.id = other_type.file_id - AND other_type_file.lang = 'csharp' - WHERE other_type.name_folded = type_symbol.name_folded - AND other_type.name = type_symbol.name COLLATE BINARY - AND other_type.kind IN ('class', 'struct', 'record', 'interface', 'enum', 'delegate') - AND {BuildCSharpDefinitionTypeAritySql("other_type")} - = {BuildCSharpDefinitionTypeAritySql("type_symbol")} - AND {BuildCSharpTypeIdentitySql("other_type")} - <> {BuildCSharpTypeIdentitySql("type_symbol")} COLLATE BINARY + FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match + WHERE lower_rank_match.reference_id = r.id ) - ) AS unique_target - ON unique_target.name_folded = r.symbol_name_folded - AND unique_target.name = r.symbol_name COLLATE BINARY - WHERE source_file.lang = 'csharp' - AND r.target_qualifier IS NULL - AND r.reference_kind = 'type_reference' - AND ( - {CSharpReferenceTypeAritySql} IS NULL - OR unique_target.type_arity - = {CSharpReferenceTypeAritySql} - ) - AND NOT EXISTS ( - SELECT 1 - FROM temp.{ReferenceLowerRankCandidateMatchesTable} AS lower_rank_match - WHERE lower_rank_match.reference_id = r.id - ); + ) + SELECT matched_family.reference_id, + type_member.symbol_id, + 5 + FROM matched_csharp_type_reference_families AS matched_family + JOIN csharp_type_reference_members AS type_member + ON type_member.name_folded = matched_family.name_folded + AND type_member.name = matched_family.name COLLATE BINARY + AND type_member.type_arity = matched_family.type_arity + AND type_member.type_identity = matched_family.type_identity COLLATE BINARY; INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) SELECT r.id, target.id, 5 diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 914b6dc82..d1d11c751 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -361,6 +361,23 @@ public void CSharpGraphFacts_EvaluateManagedScalarsOnceBeforeGraphConsumers() { Assert.Contains("temp.csharp_type_identity_facts", sql, StringComparison.Ordinal); Assert.Contains("temp.csharp_constructor_identity_facts", sql, StringComparison.Ordinal); + Assert.Contains( + "csharp_type_reference_members(", + sql, + StringComparison.Ordinal); + Assert.Contains( + "csharp_unique_type_reference_families(", + sql, + StringComparison.Ordinal); + Assert.Contains( + "matched_csharp_type_reference_families(", + sql, + StringComparison.Ordinal); + Assert.Contains( + "HAVING COUNT(DISTINCT type_member.type_identity COLLATE BINARY) = 1", + sql, + StringComparison.Ordinal); + Assert.DoesNotContain("symbols AS other_type", sql, StringComparison.Ordinal); Assert.DoesNotContain("file-local:", sql, StringComparison.Ordinal); Assert.DoesNotContain("ranked_constructor_owners", sql, StringComparison.Ordinal); Assert.DoesNotContain( @@ -387,6 +404,213 @@ static int CountOccurrences(string text, string value) } } + [Fact] + public void CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember() + { + var partialAFileId = UpsertTestFile("proj/Widget.A.cs", "widget-a"); + var partialBFileId = UpsertTestFile("proj/Widget.B.cs", "widget-b"); + var lowerCaseFileId = UpsertTestFile("proj/lower-widget.cs", "lower-widget"); + var arityTwoFileId = UpsertTestFile("other/Widget2.cs", "widget-arity-two"); + var callerFileId = UpsertTestFile("proj/WidgetCaller.cs", "widget-caller"); + _writer.InsertSymbols([ + CreateType( + partialAFileId, + "Widget", + "Demo", + "proj|Demo+Widget`1", + "public partial class Widget"), + CreateType( + partialBFileId, + "Widget", + "Demo", + "proj|Demo+Widget`1", + "public partial class Widget"), + CreateType( + lowerCaseFileId, + "widget", + "Demo", + "proj|Demo+widget`1", + "public class widget"), + CreateType( + arityTwoFileId, + "Widget", + "Other", + "other|Other+Widget`2", + "public class Widget"), + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = callerFileId, + SymbolName = "Widget", + ReferenceKind = "type_reference", + Line = 1, + Column = 1, + Context = "Widget value;", + }, + new ReferenceRecord + { + FileId = callerFileId, + SymbolName = "Widget", + ReferenceKind = "type_reference", + Line = 2, + Column = 1, + Context = "unparseable", + }, + new ReferenceRecord + { + FileId = callerFileId, + SymbolName = "widget", + ReferenceKind = "type_reference", + Line = 3, + Column = 1, + Context = "widget lower;", + }, + ], refreshMutualRecursionFlags: false); + + _writer.RefreshMutualRecursionFlags(); + + AssertUnconflictedCandidates(); + + using (var scope = _writer.BeginReferenceGraphRefreshScope()) + { + using var transaction = _writer.BeginTransaction(); + var conflictFileId = _writer.InsertNewFile(new FileRecord + { + Path = "other/Widget.cs", + Lang = "csharp", + Size = 100, + Lines = 5, + Modified = new DateTime(2026, 8, 14, 0, 0, 0, DateTimeKind.Utc), + Checksum = "widget-conflict", + }); + _writer.InsertSymbols([ + CreateType( + conflictFileId, + "Widget", + "Other", + "other|Other+Widget`1", + "public class Widget"), + ]); + transaction.Commit(); + _writer.RefreshMutualRecursionFlags(); + } + + AssertConflictedCandidates(); + var scopedConflictSnapshot = ReadReferenceGraphSemanticSnapshot(); + + _writer.RefreshMutualRecursionFlags(); + AssertConflictedCandidates(); + var fullConflictSnapshot = ReadReferenceGraphSemanticSnapshot(); + Assert.Equal(scopedConflictSnapshot, fullConflictSnapshot); + + using (var transaction = _db.Connection.BeginTransaction()) + { + DbWriter.RebuildRetainedReferenceGraph( + _db.Connection, + transaction, + CancellationToken.None); + transaction.Commit(); + } + + AssertConflictedCandidates(); + Assert.Equal(fullConflictSnapshot, ReadReferenceGraphSemanticSnapshot()); + + using (var scope = _writer.BeginReferenceGraphRefreshScope()) + { + using var transaction = _writer.BeginTransaction(); + Assert.True(_writer.DeleteFileByPath("other/Widget.cs")); + transaction.Commit(); + _writer.RefreshMutualRecursionFlags(); + } + + AssertUnconflictedCandidates(); + var scopedSnapshot = ReadReferenceGraphSemanticSnapshot(); + + _writer.RefreshMutualRecursionFlags(); + AssertUnconflictedCandidates(); + var fullSnapshot = ReadReferenceGraphSemanticSnapshot(); + Assert.Equal(scopedSnapshot, fullSnapshot); + + using (var transaction = _db.Connection.BeginTransaction()) + { + DbWriter.RebuildRetainedReferenceGraph( + _db.Connection, + transaction, + CancellationToken.None); + transaction.Commit(); + } + + AssertUnconflictedCandidates(); + Assert.Equal(fullSnapshot, ReadReferenceGraphSemanticSnapshot()); + + static SymbolRecord CreateType( + long fileId, + string name, + string container, + string familyKey, + string signature) + => new() + { + FileId = fileId, + Kind = "class", + Name = name, + Line = 1, + StartLine = 1, + EndLine = 5, + Signature = signature, + ContainerQualifiedName = container, + FamilyKey = familyKey, + }; + + void AssertConflictedCandidates() + { + Assert.Equal( + 0, + ExecuteScalarLong(""" + SELECT COUNT(*) + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference + ON reference.id = candidate.reference_id + WHERE reference.symbol_name = 'Widget' + AND reference.line = 1 + """)); + AssertCandidatePaths(line: 2, "other/Widget2.cs"); + AssertCandidatePaths(line: 3, "proj/lower-widget.cs"); + } + + void AssertUnconflictedCandidates() + { + AssertCandidatePaths( + line: 1, + "proj/Widget.A.cs|proj/Widget.B.cs"); + AssertCandidatePaths( + line: 2, + "other/Widget2.cs|proj/Widget.A.cs|proj/Widget.B.cs"); + AssertCandidatePaths(line: 3, "proj/lower-widget.cs"); + } + + void AssertCandidatePaths(int line, string expected) + { + Assert.Equal( + expected, + ExecuteScalarString($""" + SELECT group_concat(candidate_path.path, '|') + FROM ( + SELECT target_file.path + FROM symbol_reference_candidates AS candidate + JOIN symbol_references AS reference + ON reference.id = candidate.reference_id + JOIN symbols AS target ON target.id = candidate.symbol_id + JOIN files AS target_file ON target_file.id = target.file_id + WHERE reference.line = {line} + AND candidate.scope_rank = 5 + ORDER BY target_file.path COLLATE BINARY + ) AS candidate_path + """)); + } + } + [Fact] public void CSharpPropertyReceiverNormalization_SeeksFactBackedReferencesAndTargets() { @@ -1257,6 +1481,24 @@ public void ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks() Assert.Contains(csharpTypePlan, static detail => detail.Contains( "SEARCH type_lookup_name USING PRIMARY KEY", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(csharpTypePlan, static detail => detail.Contains( + "MATERIALIZE csharp_type_reference_members", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains(csharpTypePlan, static detail => detail.Contains( + "MATERIALIZE csharp_unique_type_reference_families", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains(csharpTypePlan, static detail => detail.Contains( + "MATERIALIZE matched_csharp_type_reference_families", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains(csharpTypePlan, static detail => detail.Contains( + "SEARCH type_symbol_fact USING PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains(csharpTypePlan, static detail => detail.Contains( + "SEARCH type_identity_fact USING PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); + Assert.Contains(csharpTypePlan, static detail => detail.Contains( + "SEARCH reference_fact USING PRIMARY KEY", + StringComparison.OrdinalIgnoreCase)); Assert.DoesNotContain(csharpTypePlan, static detail => detail.Equals("SCAN type_symbol", StringComparison.OrdinalIgnoreCase) || detail.StartsWith("SCAN type_symbol ", StringComparison.OrdinalIgnoreCase)); From 678dbd5410cbedd1433badeb50378e2ed742372d Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 14 Aug 2026 01:08:36 +0900 Subject: [PATCH 11/12] Throttle persistence batch progress logging --- DEVELOPER_GUIDE.md | 4 +- TESTING_GUIDE.md | 12 +- .../+initial-index-write-batches.changed.md | 4 +- .../Database/DbWriter.ChunkSymbolBatches.cs | 71 ++++++++-- src/CodeIndex/Database/DbWriter.References.cs | 32 ++++- tests/CodeIndex.Tests/DatabaseTests.cs | 128 +++++++++++++++++- 6 files changed, 222 insertions(+), 29 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index e5729719a..796d3c769 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1309,7 +1309,7 @@ Current stable codes and triggers: | Maintenance error contract | `vacuum`, `backfill-fold`, `optimize` / `index --optimize`, and `db integrity` route failures through `MaintenanceDatabaseErrorClassifier` version `1` and one JSON/human writer. SQLite primary codes `5`/`6`, `8`, `11`, and `26` classify locked/busy, not-writable, corrupt, and not-a-database failures without inspecting exception wording. The shared response carries a stable error code/category, conditional recovery hint, redacted path metadata, and optional primary/extended SQLite codes. Absolute paths are redacted by default; `--show-paths` is the explicit diagnostic opt-in. | | Durable WAL file set | When WAL is active, the durable SQLite index is the `.db` file plus sibling `.db-wal` and `.db-shm` files. Backups, diagnostics bundles, and manual copies must include all three files when the siblings exist, or use SQLite's `.backup` command/API from a live connection. Copying only `codeindex.db` can produce a stale snapshot because committed pages may still live in `codeindex.db-wal`. | | `synchronous=NORMAL` | Under WAL, `NORMAL` avoids per-commit fsync pressure during 500-row indexing batches while preserving database consistency after crashes. | -| Caller-owned write batching | Full-scan and other atomic file writes already run inside one caller-owned transaction, so their language-neutral chunk, symbol, issue, reference-line, and reference inserts cap each named-parameter statement at 32 parameters. `Microsoft.Data.Sqlite` resolves every parameter name again on execution; this smaller shape avoids dense binding lookup without adding transaction scopes. Public writer APIs retain the SQLite-variable-limit batch shape and their existing per-batch transaction/SAVEPOINT contract. | +| Caller-owned write batching | Full-scan and other atomic file writes already run inside one caller-owned transaction, so their language-neutral chunk, symbol, issue, reference-line, and reference inserts cap each named-parameter statement at 32 parameters. `Microsoft.Data.Sqlite` resolves every parameter name again on execution; this smaller shape avoids dense binding lookup without adding transaction scopes. Cancellation and test checkpoints remain at every statement. For operations above 500 rows, persistent `db_writer_batch_checkpoint` records are emitted only when progress crosses a 500-row boundary and at completion, avoiding a synchronous log flush for every tiny statement. Public writer APIs retain the SQLite-variable-limit batch shape and their existing per-batch transaction/SAVEPOINT contract. | | Checkpointing | `DbWriter` runs `PRAGMA wal_checkpoint(PASSIVE)` after each outer transaction commit, and SQLite may also checkpoint automatically after the configured 1000-page threshold. Both checkpoint paths are opportunistic: active readers are not blocked, and an uncheckpointed WAL is expected state rather than corruption. | | Checkpoint result contract | Explicit `PRAGMA wal_checkpoint(TRUNCATE)` paths execute a reader and return a structured result containing SQLite's `(busy, log, checkpointed)` values. Non-zero `busy` or positive remaining pages is unsuccessful with a bounded machine reason. `(0, -1, -1)` is SQLite's successful non-WAL no-op. Instance checkpointing, the static read-only-fallback preflight, query diagnostics, top-level status, and nested connection-policy status preserve the same result and counts. Raw exception text and paths must not enter diagnostics. | | Crash recovery | If the process is killed after SQLite has committed a transaction but before checkpointing, the next normal opener rolls the WAL forward; no manual recovery step is required. If the process dies before a transaction commits, SQLite rolls that transaction back. | @@ -5110,7 +5110,7 @@ apply 時は `PRAGMA optimize` を実行します。 | maintenance error contract | `vacuum`、`backfill-fold`、`optimize` / `index --optimize`、`db integrity` の失敗は `MaintenanceDatabaseErrorClassifier` version `1` と単一の JSON / human writer を通ります。SQLite primary code `5` / `6`、`8`、`11`、`26` から locked / busy、not-writable、corrupt、not-a-database を分類し、例外 message は判定に使いません。共有 response は stable error code / category、条件別 recovery hint、redaction 済み path metadata、任意の primary / extended SQLite code を返します。absolute path は既定で redaction し、`--show-paths` を明示的な diagnostic opt-in とします。 | | durable WAL file set | WAL が有効な場合、永続化された SQLite index は `.db` file と sibling の `.db-wal` / `.db-shm` file の組です。backup、diagnostics bundle、手動 copy では sibling が存在する場合に 3 file すべてを含めるか、live connection から SQLite の `.backup` command/API を使う必要があります。`codeindex.db` だけを copy すると、committed page がまだ `codeindex.db-wal` に残っているため stale snapshot になる可能性があります。 | | `synchronous=NORMAL` | WAL では `NORMAL` により 500 row 単位の indexing batch ごとの fsync 負荷を避けつつ、crash 後の database consistency を保ちます。 | -| caller-owned write batch | full-scan などの atomic file write は既に1つの caller-owned transaction 内で実行されるため、言語共通の chunk、symbol、issue、reference-line、reference insert は named parameter statement を32 parameter以下に制限します。`Microsoft.Data.Sqlite` は実行ごとに全 parameter name を再解決するため、この小さい形状で追加 transaction scope を増やさず dense binding lookup を避けます。public writer API は SQLite variable limit までの batch 形状と既存の batch ごとの transaction / SAVEPOINT 契約を維持します。 | +| caller-owned write batch | full-scan などの atomic file write は既に1つの caller-owned transaction 内で実行されるため、言語共通の chunk、symbol、issue、reference-line、reference insert は named parameter statement を32 parameter以下に制限します。`Microsoft.Data.Sqlite` は実行ごとに全 parameter name を再解決するため、この小さい形状で追加 transaction scope を増やさず dense binding lookup を避けます。cancellation / test checkpoint はstatementごとに維持します。500 rowを超えるoperationでは、永続 `db_writer_batch_checkpoint` を500 row境界をまたいだ時点と完了時だけ出力することで、小さなstatementごとの同期log flushを避けます。public writer API は SQLite variable limit までの batch 形状と既存の batch ごとの transaction / SAVEPOINT 契約を維持します。 | | checkpoint | `DbWriter` は outer transaction commit 後に `PRAGMA wal_checkpoint(PASSIVE)` を実行し、SQLite も設定済みの 1000 page threshold を超えると自動 checkpoint する場合があります。どちらの checkpoint path も opportunistic で、active reader は block されず、未 checkpoint の WAL は corruption ではなく期待される状態です。 | | checkpoint result contract | 明示的な `PRAGMA wal_checkpoint(TRUNCATE)` path は reader を実行し、SQLite の `(busy, log, checkpointed)` を含む構造化結果を返します。`busy` が 0 以外、または remaining page が正の場合は、上限付き machine reason を伴う unsuccessful result です。`(0, -1, -1)` は SQLite の非 WAL database に対する成功 no-op です。instance checkpoint、read-only fallback 前の static preflight、query diagnostics、top-level status、nested connection-policy status は同じ結果と count を保持します。raw exception text や path を diagnostics に含めてはいけません。 | | crash recovery | SQLite が transaction を commit した後、checkpoint 前に process が kill された場合、次の通常 open が WAL を roll forward するため手動 recovery は不要です。commit 前に process が終了した transaction は SQLite により rollback されます。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 704b356b0..0e889a222 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -43,7 +43,7 @@ Use the full suite by default. Use targeted filters only while iterating locally A full-scan barrier regression must commit a candidate-free row from a second connection after the early empty observation but before the outer write transaction, then prove transaction-local revalidation switches to ordinary full resolution and normalizes that row. - C# reference-graph fact coverage is an SQL-shape performance contract. Each of the six managed arity/receiver/value-type functions must occur once only in its reference or symbol materialization statement, and full, scoped, and retained refresh chains must populate reference, symbol, type-identity, and constructor-identity facts in that order before normalization and candidate consumers. Candidate SQL must use the WITHOUT ROWID fact primary keys without retaining project/file-local identity construction or constructor-owner range scans. Keep full/scoped/retained semantic snapshots across partial generic and file-local constructors, primary constructors with same-leaf generic arities, scoped target-definition mutations, fallback owners, legacy NULL identities, stale-row cleanup, and refresh rollback/retry alongside the focused property-receiver regressions. - C# metadata-target resolver coverage treats propagation work and stable reruns as performance contracts. Keep the reverse-ordered 8,000-class chain at exactly `n - 1` dependency edges and `n` queue visits instead of using a wall-clock threshold; retain cross-file partial fan-in, an unseeded cycle, pre-cancellation, rollback of an earlier row after an injected later update failure, zero trigger-audited writes on a stable rerun, and exactly one write when repairing a corrupted derived row. -- Reference-insert transaction coverage keeps the public APIs' #1518 transaction/SAVEPOINT per 71-row batch, while the explicit atomic-file APIs must reject calls without a live caller-owned transaction and open zero reference-batch scopes. Atomic-file reference-line materialization may group only complete 71-row reference batches, stopping before the union of `(file_id, line, context)` keys would exceed 333 rows or after 32 batches; reference INSERT executions and their progress/cancellation checkpoints remain on the original 71-row boundaries. Preserve exact public/atomic statement counts, the 333-row and 32-batch stops, the unique `reference_lines` autoindex lookup plan, batch-two/three failure rollback for both normal and new-file reference-line paths, same/different contexts across a batch boundary, cancellation and empty-input ordering, and guarded multi-language integration coverage for full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. The controlled 321,352-reference/856-file performance contract retains the repository snapshot's five/six-batch large-file distribution and compares 5,009 public batch scopes with zero atomic-file batch scopes without using a wall-clock threshold. +- Reference-insert transaction coverage keeps the public APIs' #1518 transaction/SAVEPOINT per 71-row batch, while the explicit atomic-file APIs must reject calls without a live caller-owned transaction and open zero reference-batch scopes. Atomic-file inserts cap every named-parameter statement at 32 parameters: reference INSERTs use two-row statements, reference-line writes and lookups use at most ten rows, and reference-line materialization groups complete reference statements into worst-case arithmetic windows capped at 32 statements. Progress/cancellation test checkpoints remain on every reference-statement boundary, while production progress logs are row-cadenced separately. Preserve exact public/atomic statement counts, worst-case window sizing and the 32-statement cap, the unique `reference_lines` autoindex lookup plan, second-/third-window failure rollback for both normal and new-file reference-line paths, same/different contexts across a materialization-window boundary, cancellation and empty-input ordering, and guarded multi-language integration coverage for full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. The controlled 321,352-reference/856-file performance contract retains the repository snapshot's public five/six-batch large-file distribution and compares 5,009 public batch scopes with zero atomic-file batch scopes without using a wall-clock threshold. - Deferred hotspot-aggregate coverage treats one physical set-based refresh per indexing batch as a performance contract across full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. Keep the statement hook at exactly one through the controlled 1,001-dirty-file case, prove all four query indexes are absent only in bulk mode and restored after success or cancellation rollback, and keep them present for the default small-update path. Deferral requires at least 64 dirty file IDs and existing dirty aggregate rows covering three-fifths of the table; an empty pre-refresh aggregate qualifies for fresh/rebuild work. Exercise the overflow-safe boundary, the bounded total-row probe, and a skewed case whose many zero-reference dirty files leave high-cardinality stable rows indexed. Reuse a persistence command prepared before the schema cycle. Recheck/clear readiness in every mutation transaction, retain only successful transaction/savepoint dirty IDs, and preserve prior-false readiness plus cancellation demotion. Unit coverage must include rolled-back files, zero-reference cleanup, stale deletion, cross-file `reference_lines`, and C#/Python/Markdown/SQL aggregate-to-raw parity; standalone public writer APIs remain immediate. - Adaptive FTS maintenance coverage shares the 25-run threshold across full scan, scoped update, and MCP indexing, but threshold maintenance is an incremental merge with a 1,000-page minimum work target rather than a full optimize. SQLite processes complete segments, so coverage must not treat that target as a maximum: the actual page count can exceed it. Assert that the first mutating incremental run records one write without invoking the merge hook, the threshold run invokes it once and resets only the dedicated merge counter, and the since-optimize counter keeps accumulating so `optimize --dry-run` recommendations retain their meaning. Scoped-update JSON reports `fts_merge_ran` while preserving the legacy `fts_optimize_ran` field as false. Full-scan and MCP coverage must also hold the exact three-fifths dirty-byte boundary: below it keeps triggers and incremental maintenance, while the boundary and above use trigger-free bulk writes followed by FTS rebuild and optimize. For each current rewrite, the dirty numerator must use the larger of its persisted and current sizes, then add persisted bytes for planned stale deletions (including rename-old paths). The comparison total must add those deleted bytes and the positive persisted-minus-current excess for shrunk rewrites to known readable current-workspace bytes. Keep deletion-only, rename, shrink, and combined delete-plus-modify boundary cases, assert that the purge runs inside the selected guard, and retain the conservative non-bulk fallback for an invalid persisted size, scan error, or overflow. Batched purge cancellation before commit must roll back every file/chunk/FTS delete; cancellation after a committed bulk purge must exercise guard abandonment and verify rebuilt searchable rows plus restored triggers. A post-commit WAL-checkpoint fault during an MCP bulk purge must retain the durable deletion, make the guard rebuild even before the caller can set its mutation flag, restore all three triggers, and clear the recovery marker only after FTS parity is restored. Stale-purge planning stays before the C# prepass, while the `memory_timeline` `purge` sample represents physical deletion inside the selected guard and therefore follows `csharp_prepass`. The planned IDs must be excluded from persisted C# static-interface workspace symbols without copying the deletion set; CLI and MCP deletion cases must prove that a removed contract cannot regenerate an `implicit_implementation` reference in the same run and that no-stale reruns skip the contract preflight. A three-run MCP recovery case must purge a contract while a scan error omits its implementer, keep the C# contract marker invalid, and prove that the next clean retry rewrites the implementer, removes the stale implicit reference, and then restamps readiness. Reusable-stat snapshots must exclude the same sorted IDs, and an MCP plan-to-scan reappearance race must prove that a live file, its chunks, and its FTS row are reindexed rather than skipped after the old row is purged. For reusable-row snapshots, assert that ordinary runs do not allocate a current-target path filter, while an indexed workspace whose non-purged rows unused by the current target set outnumber the current targets does enable filtering. Keep fresh/rebuild runs on immediate bulk rebuild/optimize without allocating stat-preflight buffers, preserve request cancellation and the existing per-file error contract for recoverable stat failures, and propagate an MCP authorization denial immediately without retrying it in the real file loop. - FTS bulk-guard failure coverage injects one-shot trigger-drop, trigger-restore, and FTS-rebuild failures separately. The start-failure case must remove exactly one trigger before throwing, return no guard, and leave two triggers; every case must rethrow the exact injected exception and downgrade the process-owned marker to owner-independent `true`. After disabling the fault, prove same-process recovery restores all three triggers, rebuilds searchable chunk state, clears the marker, and becomes a no-op on the next recovery attempt. New owner state keeps the primary marker in legacy-readable `pid:` form and atomically writes a PID-bound process-start generation or per-process random-token fallback afterward. Preserve legacy integer parsing; prove insert/update/delete cleanup triggers invalidate the generation when an older writer mutates only the primary key; ignore the generation when the trigger set is incomplete or its PID association is mismatched; reject a reused current PID for either valid generation form; and keep PID-only markers conservatively active. @@ -746,8 +746,8 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` keeps 8,000 members under three nested C# containers on one reusable assignment path buffer; its `net8.0` allocation and practical-time budgets are blocking. `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` keeps 12,000 no-alias calls from paying for alias dedupe, `CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` guards pre-sized stable compaction after alias rewrites, and `MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` covers repeated qualified C#- and Python-style cycle names without per-edge normalization strings. `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` warms a 1,024-row typed snapshot with its exact capacity, then prevents a second candidate collection or redundant path values from returning to the default `net8.0` path. - `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. `CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` additionally fixes the atomic full-index parameter budget at 32 across language-neutral chunks (6 rows), symbols (1), issues (5), reference lines (10), and references (2), while the public reference API retains 71-row batches. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. - `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` fixes atomic reference-line windows to a worst-case arithmetic bound. Keep it paired with whole-batch grouping, 32-batch caps, cross-boundary context reuse, rollback, and cancellation tests so the materializer remains the only tuple-hash pass without changing persistence boundaries. + `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. `CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` additionally fixes the atomic full-index parameter budget at 32 across language-neutral chunks (6 rows), symbols (1), issues (5), reference lines (10), and references (2), while the public reference API retains 71-row batches. `CallerOwnedTransactionBatchProgress_LogsByRowsWhileCheckpointsStayPerStatement` keeps cancellation/test checkpoints at every persistence-statement boundary, but for operations above 500 rows emits production batch-progress logs only when processed rows cross each 500-row boundary and at completion. For reference-line window performance audits, measure identical prebuilt reference rows against one-statement and 32-statement limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. + `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` fixes atomic reference-line windows to a worst-case arithmetic bound. Keep it paired with whole-statement grouping, 32-statement caps, materialization-window-boundary context reuse, rollback, and cancellation tests so the materializer remains the only tuple-hash pass without changing persistence boundaries. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` also fixes the rank-5 lower-rank guard at one compact TEMP row per matched reference: scoped materialization must drive from dirty IDs into the candidate primary key, and each of the five rank-5 fallbacks must use a TEMP primary-key seek. The C# type fallback must materialize physical members, unique logical families, and matched families in that order before its final physical expansion; retain primary-key seeks for scoped symbols, references, and facts. `CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember` proves that same-identity partial declarations still produce every physical candidate while a conflicting identity suppresses the whole rank-5 family across scoped, full, and retained refreshes. Keep the remaining full/scoped/retained resolution oracles beside these structural guards so physical candidate rows and multi-language ambiguity remain unchanged. @@ -1114,7 +1114,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" full-scan barrier回帰では、早期empty確認後からouter write transaction開始前の間に別connectionからcandidate-free rowをcommitし、transaction内の再検証が通常のfull resolutionへ切り替わってそのrowを正規化することを必須とします。 - C# reference-graph fact coverage は SQL shape の performance contract です。arity / receiver / value-type を求める6つの managed function は reference または symbol の materialization statement 内にそれぞれ1回だけ置き、full / scoped / retained refresh chain は reference、symbol、type identity、constructor identity の fact をこの順で normalization と candidate consumer より前に投入してください。candidate SQL は WITHOUT ROWID fact の主キーを使い、project / file-local identity の再構築や constructor-owner range の再走査を残してはいけません。partial generic / file-local constructor、同名別 generic arity を持つ primary constructor、scoped target 定義変更、fallback owner、legacy NULL identity、stale row cleanup、refresh rollback / retry の full / scoped / retained semantic snapshot と、focused property-receiver regression を維持します。 - C# metadata-target resolver coverage は propagation work と安定 rerun を performance contract とします。逆順に保存した 8,000 class の chain では wall-clock threshold を使わず、dependency edge が厳密に `n - 1`、queue visit が `n` であることを維持してください。cross-file partial fan-in、seed を持たない cycle、事前 cancel、後段 update の注入失敗時に先行 row も rollback されること、安定 rerun の trigger audit が write 0 回、破損した derived row の修復が厳密に 1 write であることも残します。 -- reference insert の transaction coverage は、public API の #1518 契約として71 row batchごとの transaction/SAVEPOINTを維持し、明示atomic-file APIは呼出元所有のlive transactionなしでは拒否され、reference batch scopeを0回に保つことを検証します。atomic-fileのreference-line materializationは完全な71 row reference batchだけをまとめ、`(file_id, line, context)` keyの和集合が333行を超える直前、または32 batchで停止します。reference INSERTの実行回数とprogress/cancellation checkpointは元の71 row境界に保ってください。public/atomicの正確なstatement数、333行/32 batch停止、`reference_lines` unique autoindexのlookup plan、通常/new-file両方のreference-line pathでbatch 2/3失敗時の全rollback、batch境界をまたぐ同一/異なるcontext、cancelとempty入力の順序、full scan・scoped update・MCP indexing・TypeScript augmentation rebuildのmulti-language guard付きintegrationを維持してください。321,352 refs / 856 filesの制御performance契約は自己snapshotの5/6 batch巨大file分布を保ち、wall-clock閾値を使わずpublic 5,009 scopeとatomic-file 0 scopeを比較します。 +- reference insert の transaction coverage は、public API の #1518 契約として71 row batchごとの transaction/SAVEPOINTを維持し、明示atomic-file APIは呼出元所有のlive transactionなしでは拒否され、reference batch scopeを0回に保つことを検証します。atomic-file insertは各named-parameter statementを32 parameter以下に制限し、reference INSERTは2 row、reference-line write / lookupは最大10 rowのstatementを使い、reference-line materializationは完全なreference statementを最悪ケースの算術windowへまとめて最大32 statementで停止します。progress/cancellationのtest checkpointは各reference statement境界を維持し、本番progress logは別のrow cadenceで間引きます。public/atomicの正確なstatement数、最悪ケースのwindow sizingと32 statement cap、`reference_lines` unique autoindexのlookup plan、通常/new-file両方のreference-line pathでwindow 2/3失敗時の全rollback、materialization window境界をまたぐ同一/異なるcontext、cancelとempty入力の順序、full scan・scoped update・MCP indexing・TypeScript augmentation rebuildのmulti-language guard付きintegrationを維持してください。321,352 refs / 856 filesの制御performance契約は自己snapshotのpublic 5/6 batch巨大file分布を保ち、wall-clock閾値を使わずpublic 5,009 scopeとatomic-file 0 scopeを比較します。 - deferred hotspot aggregate coverage は、full scan、scoped update、MCP indexing、TypeScript augmentation rebuild を横断して、変更を含む indexing batch ごとの物理 set-based refresh が1回であることを performance contract とします。1,001 dirty files の制御caseまでstatement hookの厳密な1回を保ち、bulk mode 中だけ query index 4本が不在で、成功後または cancellation rollback 後に復元され、default の小規模 update 経路では常に存在することを証明してください。index 遅延は dirty file ID 64件以上かつ既存dirty aggregate rowがtableの5分の3以上を占める場合だけ有効にし、fresh / rebuild のrefresh前aggregateが空の場合も条件を満たすものとします。overflow-safeな境界、bounded total-row probe、多数のzero-reference dirty fileと高cardinalityのstable rowが共存する偏りfixtureを検証し、schema cycle 前にprepareされた persistence command も再利用してください。各 mutation transaction での readiness 再確認/clear、成功した transaction/savepoint の dirty ID だけの採用、prior-false readiness と cancel 時の demotion を維持してください。unit coverage には rollback file、zero-reference cleanup、stale delete、cross-file `reference_lines`、C#/Python/Markdown/SQL の aggregate/raw parity を含め、standalone public writer API は即時更新のままにします。 - adaptive FTS maintenance coverage は、full scan、scoped update、MCP indexing で25 run の threshold を共有しますが、threshold maintenance は full optimize ではなく1,000 page の最小 work target を持つ incremental merge とします。SQLite は完全な segment 単位で処理するため、この target を最大値と扱わず、実際の page 数が target を超えることを契約に含めます。最初の変更付き incremental run は write を1回記録して merge hook を呼ばず、threshold 到達 run は hook を1回呼んで専用 merge counter だけをresetし、since-optimize counter は蓄積を続けて `optimize --dry-run` recommendation の意味を維持することを検証してください。scoped-update JSON は従来の `fts_optimize_ran` を false のまま残しつつ `fts_merge_ran` を報告します。full-scan と MCP では dirty byte が厳密に5分の3となる境界も固定し、境界未満は trigger と incremental maintenance を維持し、境界以上は trigger-free bulk write の後に FTS rebuild / optimize を行います。dirty numerator は current rewrite ごとに永続化済み size と current size の大きい方を使い、rename の旧 path を含む stale 削除予定 row の永続化済み byte を加算します。比較対象の total には読み取り可能と判明した current workspace byte、削除予定 byte、および縮小した rewrite の永続化済み size が current size を上回る正の差分を含めてください。delete-only、rename、shrink、delete と modify の合算境界を維持し、purge が選択された guard 内で実行されること、永続化 size 不正・scan error・overflow では保守的な non-bulk fallback になることを検証します。batch purge の commit 前 cancellation は file / chunk / FTS の全削除を rollback し、bulk purge commit 後の cancellation は guard abandon を通して検索可能 row の rebuild と trigger 復元を確認してください。MCP bulk purge の commit 後 WAL checkpoint fault では durable な削除を保持し、caller が mutation flag を立てる前でも guard が rebuild を行い、trigger 3本を復元して FTS parity 回復後にだけ recovery marker を clear することを確認します。stale purge の plan は C# prepass より前に維持しますが、`memory_timeline` の `purge` sample は選択した guard 内の物理削除を表すため `csharp_prepass` の後になります。削除 set を複製せず、plan 済み ID を永続 C# static-interface workspace symbol から除外すること、CLI / MCP の削除 case で削除済み contract が同一 run 内に `implicit_implementation` reference を再生成しないこと、および stale のない rerun では contract preflight を省くことも検証します。MCP の3 run recovery case では contract purge と同時に scan error で implementer を対象外にし、C# contract marker が invalid のまま残ること、次の clean retry が implementer を書き換えて stale implicit reference を除去した後に readiness を再 stamp することを確認します。reusable-stat snapshot からも同じ昇順 ID を除外し、MCP の plan 後から scan までに同じ path が再出現する race では現存 file、chunk、FTS row が旧 row の purge 後に skip されず再indexされることを確認します。reusable-row snapshot は通常 run で current-target path filter を確保せず、purge 後も残る indexed row のうち current target に使われない row 数が current target 数を上回る workspace だけで filter を有効にすることを検証します。fresh/rebuild では stat-preflight buffer を確保せず、引き続き即時 bulk rebuild / optimize を行います。recoverable な stat 失敗に対する request cancellation と既存 per-file error 契約を維持し、MCP authorization denial は実ファイル loop で再試行せず即時に再送出してください。 - FTS bulk guard の failure coverage は trigger drop、trigger restore、FTS rebuild の一回だけの失敗を個別に注入します。start failure は1本だけ trigger を削除してから例外を投げ、guard を返さず trigger 2本を残すことを確認します。全 case で注入した同一例外を再送出し、process owner 付き marker を owner 非依存の `true` へ降格した後、fault を無効化して同一 process recovery が trigger 3本、検索可能な chunk state、marker clear を復元し、次の recovery が no-op になることを確認してください。新しい owner state は primary marker を旧 reader が読める `pid:` に保ち、その後で PID を含む process-start generation または process ごとの random-token fallback を atomic に書きます。旧形式の整数 parse、旧 writer が primary key だけを insert/update/delete したとき cleanup trigger が generation を無効化すること、trigger 3本が不完全または PID 関連付けが不一致なら generation を無視すること、有効な両 generation 形式で再利用された current PID を拒否すること、および PID-only marker を保守的に active と扱うことを維持してください。 @@ -1819,8 +1819,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` は3階層の C# container 内の8,000 member を1つの再利用 assignment path buffer で処理します。`net8.0` の allocation と実用時間 budget は blocking です。 `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` は alias のない 12,000 call が alias dedupe のコストを負わないこと、`CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` は alias rewrite 後の事前 capacity 付き stable compaction、`MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` は C# / Python 形式の qualified cycle name が edge ごとの正規化文字列を作らないことを保証します。 `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` は exact capacity を渡した1,024行の typed snapshot を warm-up し、2つ目の候補 collection や重複 path value が通常の `net8.0` 経路へ戻らないようにします。 - `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。`CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` はさらに、言語共通のchunk（6行）、symbol（1行）、issue（5行）、reference line（10行）、reference（2行）に対するatomic full-index parameter budgetを32に固定し、public reference APIの71行batchは維持します。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 - `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` はatomic reference-line windowを最悪ケースの算術境界へ固定します。materializerだけをtuple-hash passとして保ちつつ永続化境界を変えないよう、whole-batch grouping、32-batch cap、batch境界をまたぐcontext再利用、rollback、cancellation testと対で維持してください。 + `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file batch scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。`CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` はさらに、言語共通のchunk（6行）、symbol（1行）、issue（5行）、reference line（10行）、reference（2行）に対するatomic full-index parameter budgetを32に固定し、public reference APIの71行batchは維持します。`CallerOwnedTransactionBatchProgress_LogsByRowsWhileCheckpointsStayPerStatement` はcancellation/test checkpointを各persistence statement境界に維持しつつ、500 rowを超えるoperationの本番batch-progress logを処理済みrowが500 row境界を越えた時点と完了時だけに固定します。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-statement上限と32-statement上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 + `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` はatomic reference-line windowを最悪ケースの算術境界へ固定します。materializerだけをtuple-hash passとして保ちつつ永続化境界を変えないよう、whole-statement grouping、32-statement cap、materialization window境界をまたぐcontext再利用、rollback、cancellation testと対で維持してください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 `DatabaseTests.ReferenceGraphDirtyScope_GeneratedSqlUsesDirtyPrimaryKeySeeks` は、rank 5 の下位rank guardも、一致したreferenceごとに1行のcompact TEMP集合へ固定します。scoped materializationはdirty IDからcandidate primary keyをseekし、5つのrank 5 fallbackはそれぞれTEMP primary-key seekを使う必要があります。C# type fallbackは最終の物理展開より前に、物理member、一意な論理family、一致familyの順でmaterializeし、scoped symbol / reference / factのprimary-key seekを維持してください。`CSharpTypeReferenceFamilies_MatchOnceAndExpandEveryPartialMember`は、同一identityのpartial宣言が引き続き全物理candidateを生成し、競合identityがscoped / full / retained refreshを横断してrank 5 family全体を抑止することを証明します。物理candidate行と多言語ambiguityが変わらないよう、残りのfull / scoped / retained resolution oracleもこれらの構造guardと対で維持してください。 diff --git a/changelog.d/unreleased/+initial-index-write-batches.changed.md b/changelog.d/unreleased/+initial-index-write-batches.changed.md index 9bf47b033..afdaede8b 100644 --- a/changelog.d/unreleased/+initial-index-write-batches.changed.md +++ b/changelog.d/unreleased/+initial-index-write-batches.changed.md @@ -9,8 +9,8 @@ affected: ## English -- **Initial full indexing now uses binding-efficient SQLite write batches** — caller-owned file transactions cap named parameters per chunk, symbol, issue, reference-line, and reference statement, avoiding repeated dense parameter-name lookup across every supported language while preserving public writer transaction and SAVEPOINT contracts. +- **Initial full indexing now uses binding-efficient SQLite write batches** — caller-owned file transactions cap named parameters per chunk, symbol, issue, reference-line, and reference statement, avoiding repeated dense parameter-name lookup across every supported language while preserving public writer transaction and SAVEPOINT contracts. Per-statement cancellation checkpoints remain intact. For operations above 500 rows, persistent progress logs are emitted at 500-row boundaries and completion so the smaller statements do not introduce a synchronous log flush per statement. ## 日本語 -- **初回フル索引がSQLite binding効率のよいwrite batchを使うようになりました** — caller-owned file transaction内のchunk、symbol、issue、reference-line、reference statementでnamed parameter数を制限し、全対応言語に共通する密なparameter name再探索を避けつつ、public writerのtransaction / SAVEPOINT契約を維持します。 +- **初回フル索引がSQLite binding効率のよいwrite batchを使うようになりました** — caller-owned file transaction内のchunk、symbol、issue、reference-line、reference statementでnamed parameter数を制限し、全対応言語に共通する密なparameter name再探索を避けつつ、public writerのtransaction / SAVEPOINT契約を維持します。statementごとのcancellation checkpointは保ちます。500 rowを超えるoperationでは、永続progress logを500 row境界と完了時だけに出力し、小さなstatementごとの同期log flushを防ぎます。 diff --git a/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs b/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs index 1f476803e..b942c340e 100644 --- a/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs +++ b/src/CodeIndex/Database/DbWriter.ChunkSymbolBatches.cs @@ -25,7 +25,12 @@ public void InsertChunks(IReadOnlyList chunks, CancellationToken ca : GetRowsPerInsertStatement(columnCount: 5); for (int i = 0; i < chunks.Count; i += rowsPerStatement) { - CheckBatchCancellationAndReportProgress("insert_chunks", i, chunks.Count, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_chunks", + i, + chunks.Count, + rowsPerStatement, + cancellationToken); int end = Math.Min(i + rowsPerStatement, chunks.Count); try { @@ -40,7 +45,12 @@ public void InsertChunks(IReadOnlyList chunks, CancellationToken ca InsertChunksWithRowSkip(chunks, i, end, batchException, cancellationToken); } } - CheckBatchCancellationAndReportProgress("insert_chunks", chunks.Count, chunks.Count, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_chunks", + chunks.Count, + chunks.Count, + rowsPerStatement, + cancellationToken); } /// @@ -69,7 +79,12 @@ public void InsertSymbols(IReadOnlyList symbols, CancellationToken namesPerRow: 1); for (int i = 0; i < symbols.Count; i += rowsPerStatement) { - CheckBatchCancellationAndReportProgress("insert_symbols", i, symbols.Count, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_symbols", + i, + symbols.Count, + rowsPerStatement, + cancellationToken); int end = Math.Min(i + rowsPerStatement, symbols.Count); try { @@ -84,7 +99,12 @@ public void InsertSymbols(IReadOnlyList symbols, CancellationToken InsertSymbolsWithRowSkip(symbols, i, end, batchException, cancellationToken); } } - CheckBatchCancellationAndReportProgress("insert_symbols", symbols.Count, symbols.Count, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_symbols", + symbols.Count, + symbols.Count, + rowsPerStatement, + cancellationToken); } private void TrackCurrentWriterCSharpFamilyRows(IReadOnlyList symbols) @@ -111,7 +131,12 @@ private void InsertChunksWithRowSkip(IReadOnlyList chunks, int star using var transaction = !IsInTransaction() ? BeginTransaction(cancellationToken, "insert chunks row skip") : null; for (int i = start; i < end; i++) { - CheckBatchCancellationAndReportProgress("insert_chunks_row_skip", i, end, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_chunks_row_skip", + i, + end, + rowsAdvancedSincePreviousCheckpoint: 1, + cancellationToken); var chunk = chunks[i]; ExecuteWithRowSavepoint( () => InsertChunkBatch(chunks, i, i + 1), @@ -127,7 +152,12 @@ private void InsertSymbolsWithRowSkip(IReadOnlyList symbols, int s var foldedNameCache = CreateFoldedNameCache(end - start, namesPerRow: 1); for (int i = start; i < end; i++) { - CheckBatchCancellationAndReportProgress("insert_symbols_row_skip", i, end, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_symbols_row_skip", + i, + end, + rowsAdvancedSincePreviousCheckpoint: 1, + cancellationToken); var symbol = symbols[i]; ExecuteWithRowSavepoint( () => InsertSymbolBatch(symbols, i, i + 1, foldedNameCache), @@ -141,14 +171,23 @@ private void CheckBatchCancellationAndReportProgress( string operation, int rowsProcessed, int rowsTotal, + int rowsAdvancedSincePreviousCheckpoint, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (rowsTotal <= BatchSize && BatchProgressCheckpointForTesting == null) + var checkpoint = BatchProgressCheckpointForTesting; + var shouldWriteLog = ShouldWriteBatchProgressLog( + rowsProcessed, + rowsTotal, + rowsAdvancedSincePreviousCheckpoint); + if (checkpoint == null && !shouldWriteLog) return; var progress = new DbWriterBatchProgress(operation, rowsProcessed, rowsTotal); - BatchProgressCheckpointForTesting?.Invoke(progress); + checkpoint?.Invoke(progress); + if (!shouldWriteLog) + return; + GlobalToolLog.Info( "db_writer_batch_checkpoint" + $" operation={operation}" @@ -156,6 +195,22 @@ private void CheckBatchCancellationAndReportProgress( + $" rows_total={rowsTotal.ToString(System.Globalization.CultureInfo.InvariantCulture)}"); } + private static bool ShouldWriteBatchProgressLog( + int rowsProcessed, + int rowsTotal, + int rowsAdvancedSincePreviousCheckpoint) + { + if (rowsTotal <= BatchSize) + return false; + if (rowsProcessed >= rowsTotal) + return true; + + var previousRowsProcessed = Math.Max( + 0, + rowsProcessed - Math.Max(1, rowsAdvancedSincePreviousCheckpoint)); + return rowsProcessed / BatchSize > previousRowsProcessed / BatchSize; + } + private void ExecuteWithRowSavepoint(Action insertRow, Action onSkip) { var savepointName = $"row_skip_{Interlocked.Increment(ref _rowSkipSavepointCounter)}"; diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 1eecef672..8ed5d9e5a 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -1931,6 +1931,7 @@ private void InsertReferencesCore( "insert_references", start, references.Count, + rowsPerStatement, cancellationToken); using var transaction = BeginReferenceBatchTransaction(cancellationToken); var referenceLineIds = MaterializeReferenceLines( @@ -1946,7 +1947,12 @@ private void InsertReferencesCore( } } - CheckBatchCancellationAndReportProgress("insert_references", references.Count, references.Count, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_references", + references.Count, + references.Count, + rowsPerStatement, + cancellationToken); RefreshHotspotReferenceCounts(references, cancellationToken); RestoreHotspotReferenceAggregateReady(aggregateWasReady); if (refreshMutualRecursionFlags) @@ -1975,6 +1981,7 @@ private void InsertAtomicReferenceBatches( "insert_references", windowStart, references.Count, + rowsPerStatement, cancellationToken); int windowEndBatch = GetAtomicReferenceLineWindowEndBatch( windowStartBatch, @@ -1999,6 +2006,7 @@ private void InsertAtomicReferenceBatches( "insert_references", start, references.Count, + rowsPerStatement, cancellationToken); } int end = Math.Min(start + rowsPerStatement, references.Count); @@ -2209,6 +2217,7 @@ private void RefreshHotspotReferenceCounts( "refresh_hotspot_reference_counts", completed, fileIds.Count, + rowsAdvancedSincePreviousCheckpoint: 1, cancellationToken); cmd.Parameters["@file_id"].Value = fileId; try @@ -2253,7 +2262,12 @@ private ReferenceLineBatchMap UpsertReferenceLines( : GetRowsPerInsertStatement(columnCount: 3); for (int i = 0; i < rows.Length; i += rowsPerStatement) { - CheckBatchCancellationAndReportProgress("upsert_reference_lines", i, rows.Length, cancellationToken); + CheckBatchCancellationAndReportProgress( + "upsert_reference_lines", + i, + rows.Length, + rowsPerStatement, + cancellationToken); int batchEnd = Math.Min(i + rowsPerStatement, rows.Length); var statementRowCount = batchEnd - i; var sql = ReferenceLineUpsertSqlCache.GetOrAdd(statementRowCount, static count => BuildReferenceLineUpsertSql(count)); @@ -2273,7 +2287,12 @@ private ReferenceLineBatchMap UpsertReferenceLines( int keysPerStatement = rowsPerStatement; for (int i = 0; i < rows.Length; i += keysPerStatement) { - CheckBatchCancellationAndReportProgress("lookup_reference_lines", i, rows.Length, cancellationToken); + CheckBatchCancellationAndReportProgress( + "lookup_reference_lines", + i, + rows.Length, + keysPerStatement, + cancellationToken); int keyEnd = Math.Min(i + keysPerStatement, rows.Length); var statementRowCount = keyEnd - i; var sql = ReferenceLineLookupSqlCache.GetOrAdd(statementRowCount, static count => BuildReferenceLineLookupSql(count)); @@ -2333,7 +2352,12 @@ private ReferenceLineBatchMap InsertNewReferenceLines( : GetRowsPerInsertStatement(columnCount: 3); for (int i = 0; i < rows.Count; i += rowsPerStatement) { - CheckBatchCancellationAndReportProgress("insert_reference_lines", i, rows.Count, cancellationToken); + CheckBatchCancellationAndReportProgress( + "insert_reference_lines", + i, + rows.Count, + rowsPerStatement, + cancellationToken); int batchEnd = Math.Min(i + rowsPerStatement, rows.Count); var statementRowCount = batchEnd - i; var sql = ReferenceLineInsertSqlCache.GetOrAdd(statementRowCount, static count => BuildReferenceLineInsertSql(count)); diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index d1d11c751..d2d621aac 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -8774,6 +8774,120 @@ public void CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPers .ToArray(); } + [Fact] + public void CallerOwnedTransactionBatchProgress_LogsByRowsWhileCheckpointsStayPerStatement() + { + const int RowCount = 1001; + var fileId = UpsertTestFile( + "src/caller-transaction-progress.cs", + checksum: "caller-transaction-progress"); + var symbols = Enumerable.Range(0, RowCount) + .Select(index => new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = $"target_{index}", + Line = index + 1, + StartLine = index + 1, + EndLine = index + 1, + }) + .ToArray(); + var references = Enumerable.Range(0, RowCount) + .Select(index => new ReferenceRecord + { + FileId = fileId, + SymbolName = symbols[index].Name, + ReferenceKind = "call", + Line = index + 1, + Column = 1, + Context = $"target_{index}();", + ContainerKind = "function", + ContainerName = "caller", + }) + .ToArray(); + var checkpointRows = new Dictionary>(StringComparer.Ordinal) + { + ["insert_symbols"] = [], + ["insert_references"] = [], + }; + using var env = EnvironmentVariableScope.Capture( + "CDIDX_FORCE_GLOBAL_TOOL_LOG", + "CDIDX_DISABLE_PERSISTENT_LOG", + "CDIDX_GLOBAL_TOOL_LOG_DIR"); + env.Set("CDIDX_FORCE_GLOBAL_TOOL_LOG", "1"); + env.Set("CDIDX_DISABLE_PERSISTENT_LOG", null); + env.Set("CDIDX_GLOBAL_TOOL_LOG_DIR", _dbDir); + using var logStream = new MemoryStream(); + using var logWriter = new StreamWriter( + logStream, + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + bufferSize: 1024, + leaveOpen: true); + var previousCheckpointHook = DbWriter.BatchProgressCheckpointForTesting; + try + { + DbWriter.BatchProgressCheckpointForTesting = progress => + { + if (checkpointRows.TryGetValue(progress.Operation, out var rows)) + rows.Add(progress.RowsProcessed); + }; + + using (var logSession = GlobalToolLog.TryStartForTesting( + ["index", "."], + "test", + createWriter: _ => logWriter)) + { + Assert.NotNull(logSession); + using var transaction = _writer.BeginTransaction(); + _writer.InsertSymbols(symbols); + _writer.InsertReferencesForNewFilesInAtomicFileScope( + references, + refreshMutualRecursionFlags: false, + CancellationToken.None); + transaction.Commit(); + } + } + finally + { + DbWriter.BatchProgressCheckpointForTesting = previousCheckpointHook; + } + + Assert.Equal( + Enumerable.Range(0, RowCount + 1).ToArray(), + checkpointRows["insert_symbols"]); + Assert.Equal( + Enumerable.Range(0, (RowCount + 1) / 2) + .Select(index => index * 2) + .Append(RowCount) + .ToArray(), + checkpointRows["insert_references"]); + + logStream.Position = 0; + using var logReader = new StreamReader(logStream, Encoding.UTF8, leaveOpen: true); + var progressLogRows = logReader.ReadToEnd() + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.Contains("db_writer_batch_checkpoint", StringComparison.Ordinal)) + .ToArray(); + Assert.Equal(6, progressLogRows.Length); + AssertProgressLogRows("insert_symbols", [500, 1000, RowCount]); + AssertProgressLogRows("insert_references", [500, 1000, RowCount]); + + void AssertProgressLogRows(string operation, int[] expectedRows) + { + var operationRows = progressLogRows + .Where(line => line.Contains($" operation={operation} ", StringComparison.Ordinal)) + .Select(line => + { + var marker = "rows_processed="; + var start = line.IndexOf(marker, StringComparison.Ordinal) + marker.Length; + var end = line.IndexOf(' ', start); + return int.Parse(line[start..end], CultureInfo.InvariantCulture); + }) + .ToArray(); + Assert.Equal(expectedRows, operationRows); + } + } + [Fact] public void ReferenceLineLookup_BatchedInputUsesUniqueAutoIndexPlan() { @@ -10802,7 +10916,7 @@ BEFORE INSERT ON symbol_references [Theory] [InlineData(false)] [InlineData(true)] - public void InsertReferences_AtomicFileScopeReusesSameContextAcrossBatchBoundary(bool referenceLinesAreNew) + public void InsertReferences_AtomicFileScopeReusesSameContextAcrossWindowBoundary(bool referenceLinesAreNew) { var fileId = UpsertTestFile( $"src/atomic-reference-boundary-{referenceLinesAreNew}.cs", @@ -10813,18 +10927,18 @@ public void InsertReferences_AtomicFileScopeReusesSameContextAcrossBatchBoundary FileId = fileId, SymbolName = index switch { - 70 => "boundary_same_first", - 71 => "boundary_same_second", - 72 => "boundary_different_context", + 63 => "boundary_same_first", + 64 => "boundary_same_second", + 66 => "boundary_different_context", _ => $"callee_{index}", }, ReferenceKind = "call", - Line = index is 70 or 71 or 72 ? 500 : index + 1, + Line = index is 63 or 64 or 66 ? 500 : index + 1, Column = index + 1, Context = index switch { - 70 or 71 => "shared boundary context", - 72 => "different boundary context", + 63 or 64 => "shared boundary context", + 66 => "different boundary context", _ => $"line {index}", }, ContainerKind = "function", From d4bd43090c091971f147cac467976da1203d84d9 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 14 Aug 2026 01:43:53 +0900 Subject: [PATCH 12/12] Complete hotspot refresh progress reporting --- TESTING_GUIDE.md | 4 +- src/CodeIndex/Database/DbWriter.References.cs | 11 +++ tests/CodeIndex.Tests/DatabaseTests.cs | 68 +++++++++++++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 0e889a222..7ad74d9a9 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -746,7 +746,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` keeps 8,000 members under three nested C# containers on one reusable assignment path buffer; its `net8.0` allocation and practical-time budgets are blocking. `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` keeps 12,000 no-alias calls from paying for alias dedupe, `CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` guards pre-sized stable compaction after alias rewrites, and `MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` covers repeated qualified C#- and Python-style cycle names without per-edge normalization strings. `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` warms a 1,024-row typed snapshot with its exact capacity, then prevents a second candidate collection or redundant path values from returning to the default `net8.0` path. - `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. `CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` additionally fixes the atomic full-index parameter budget at 32 across language-neutral chunks (6 rows), symbols (1), issues (5), reference lines (10), and references (2), while the public reference API retains 71-row batches. `CallerOwnedTransactionBatchProgress_LogsByRowsWhileCheckpointsStayPerStatement` keeps cancellation/test checkpoints at every persistence-statement boundary, but for operations above 500 rows emits production batch-progress logs only when processed rows cross each 500-row boundary and at completion. For reference-line window performance audits, measure identical prebuilt reference rows against one-statement and 32-statement limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. + `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. `CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` additionally fixes the atomic full-index parameter budget at 32 across language-neutral chunks (6 rows), symbols (1), issues (5), reference lines (10), and references (2), while the public reference API retains 71-row batches. `CallerOwnedTransactionBatchProgress_LogsByRowsWhileCheckpointsStayPerStatement` keeps cancellation/test checkpoints at every persistence-statement boundary, but for operations above 500 rows emits production batch-progress logs only when processed rows cross each 500-row boundary and at completion. `HotspotAggregateRefreshProgress_LogsExactBoundaryAndCompletion` applies the same production-log cadence to the immediate per-file hotspot refresh, pinning completion at an exact 1,000-row boundary and one row beyond it. For reference-line window performance audits, measure identical prebuilt reference rows against one-statement and 32-statement limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` fixes atomic reference-line windows to a worst-case arithmetic bound. Keep it paired with whole-statement grouping, 32-statement caps, materialization-window-boundary context reuse, rollback, and cancellation tests so the materializer remains the only tuple-hash pass without changing persistence boundaries. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. @@ -1819,7 +1819,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `SymbolExtractorTests.Extract_CSharp_ManyNestedMembers_ReusesContainerPathBuffer` は3階層の C# container 内の8,000 member を1つの再利用 assignment path buffer で処理します。`net8.0` の allocation と実用時間 budget は blocking です。 `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` は alias のない 12,000 call が alias dedupe のコストを負わないこと、`CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` は alias rewrite 後の事前 capacity 付き stable compaction、`MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` は C# / Python 形式の qualified cycle name が edge ごとの正規化文字列を作らないことを保証します。 `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` は exact capacity を渡した1,024行の typed snapshot を warm-up し、2つ目の候補 collection や重複 path value が通常の `net8.0` 経路へ戻らないようにします。 - `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file batch scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。`CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` はさらに、言語共通のchunk（6行）、symbol（1行）、issue（5行）、reference line（10行）、reference（2行）に対するatomic full-index parameter budgetを32に固定し、public reference APIの71行batchは維持します。`CallerOwnedTransactionBatchProgress_LogsByRowsWhileCheckpointsStayPerStatement` はcancellation/test checkpointを各persistence statement境界に維持しつつ、500 rowを超えるoperationの本番batch-progress logを処理済みrowが500 row境界を越えた時点と完了時だけに固定します。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-statement上限と32-statement上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 + `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file batch scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。`CallerOwnedTransactionBatchStatements_BoundNamedParametersAcrossPersistenceTables` はさらに、言語共通のchunk（6行）、symbol（1行）、issue（5行）、reference line（10行）、reference（2行）に対するatomic full-index parameter budgetを32に固定し、public reference APIの71行batchは維持します。`CallerOwnedTransactionBatchProgress_LogsByRowsWhileCheckpointsStayPerStatement` はcancellation/test checkpointを各persistence statement境界に維持しつつ、500 rowを超えるoperationの本番batch-progress logを処理済みrowが500 row境界を越えた時点と完了時だけに固定します。`HotspotAggregateRefreshProgress_LogsExactBoundaryAndCompletion` は同じ本番log cadenceを即時のfile別hotspot refreshへ適用し、1,000 rowの厳密な境界とその1 row後の完了を固定します。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-statement上限と32-statement上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 `DatabaseTests.AtomicReferenceLineWindowSizing_UsesWorstCaseRowsWithoutTupleHashing` はatomic reference-line windowを最悪ケースの算術境界へ固定します。materializerだけをtuple-hash passとして保ちつつ永続化境界を変えないよう、whole-statement grouping、32-statement cap、materialization window境界をまたぐcontext再利用、rollback、cancellation testと対で維持してください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 8ed5d9e5a..78005c67f 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -2235,6 +2235,12 @@ private void RefreshHotspotReferenceCounts( cancellationToken.ThrowIfCancellationRequested(); completed++; } + CheckBatchCancellationAndReportProgress( + "refresh_hotspot_reference_counts", + completed, + fileIds.Count, + rowsAdvancedSincePreviousCheckpoint: 1, + cancellationToken); } finally { @@ -2244,6 +2250,11 @@ private void RefreshHotspotReferenceCounts( transaction.Commit(); } + internal void RefreshHotspotReferenceCountsForTesting( + IReadOnlyCollection fileIds, + CancellationToken cancellationToken) + => RefreshHotspotReferenceCounts(fileIds, cancellationToken); + private ReferenceLineBatchMap UpsertReferenceLines( IReadOnlyList references, int start, diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index d2d621aac..67e8bb47e 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -8888,6 +8888,74 @@ void AssertProgressLogRows(string operation, int[] expectedRows) } } + [Theory] + [InlineData(1_000)] + [InlineData(1_001)] + public void HotspotAggregateRefreshProgress_LogsExactBoundaryAndCompletion(int fileCount) + { + using var env = EnvironmentVariableScope.Capture( + "CDIDX_FORCE_GLOBAL_TOOL_LOG", + "CDIDX_DISABLE_PERSISTENT_LOG", + "CDIDX_GLOBAL_TOOL_LOG_DIR"); + env.Set("CDIDX_FORCE_GLOBAL_TOOL_LOG", "1"); + env.Set("CDIDX_DISABLE_PERSISTENT_LOG", null); + env.Set("CDIDX_GLOBAL_TOOL_LOG_DIR", _dbDir); + using var logStream = new MemoryStream(); + using var logWriter = new StreamWriter( + logStream, + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + bufferSize: 1024, + leaveOpen: true); + var checkpointRows = new List(fileCount + 1); + var previousCheckpointHook = DbWriter.BatchProgressCheckpointForTesting; + try + { + DbWriter.BatchProgressCheckpointForTesting = progress => + { + if (progress.Operation == "refresh_hotspot_reference_counts") + checkpointRows.Add(progress.RowsProcessed); + previousCheckpointHook?.Invoke(progress); + }; + using var logSession = GlobalToolLog.TryStartForTesting( + ["index", "."], + "test", + createWriter: _ => logWriter); + Assert.NotNull(logSession); + _writer.RefreshHotspotReferenceCountsForTesting( + Enumerable.Range(1, fileCount) + .Select(static index => (long)index) + .ToArray(), + CancellationToken.None); + } + finally + { + DbWriter.BatchProgressCheckpointForTesting = previousCheckpointHook; + } + + Assert.Equal(Enumerable.Range(0, fileCount + 1), checkpointRows); + + logStream.Position = 0; + using var logReader = new StreamReader(logStream, Encoding.UTF8, leaveOpen: true); + var progressRows = logReader.ReadToEnd() + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Where(line => line.Contains( + "db_writer_batch_checkpoint operation=refresh_hotspot_reference_counts ", + StringComparison.Ordinal)) + .Select(line => + { + var marker = "rows_processed="; + var start = line.IndexOf(marker, StringComparison.Ordinal) + marker.Length; + var end = line.IndexOf(' ', start); + return int.Parse(line[start..end], CultureInfo.InvariantCulture); + }) + .ToArray(); + + int[] expectedProgressRows = fileCount == 1_000 + ? [500, 1_000] + : [500, 1_000, 1_001]; + Assert.Equal(expectedProgressRows, progressRows); + } + [Fact] public void ReferenceLineLookup_BatchedInputUsesUniqueAutoIndexPlan() {