From c3e51350fab1baa43accce097b2b6312ad4c24c1 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 04:27:17 +0900 Subject: [PATCH 01/16] Optimize fresh graph recursion finalization --- DEVELOPER_GUIDE.md | 8 +++++ TESTING_GUIDE.md | 2 ++ .../+large-codebase-initial-indexing.fixed.md | 15 ++++++++++ src/CodeIndex/Database/DbWriter.References.cs | 30 ++++++++++++++++--- tests/CodeIndex.Tests/DatabaseTests.cs | 26 +++++++++++++++- 5 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index f65f6e330..dfd26c042 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -340,6 +340,10 @@ guard forces any active dirty graph scope onto its full-refresh plan before the indexes disappear. Immediately before mutual-recursion evaluation, its graph transaction restores only the unresolved-folded, legacy NOCASE, and resolved reverse-edge indexes; the remaining query indexes return after the mutual update. +The full mutual-recursion update materializes one desired flag per call-like or +non-canonical row before applying changes. Keep the correlated reverse-edge +expression single-evaluation: repeating it in both `SET` and `WHERE` causes +fresh large graphs to perform the same random B-tree probes twice. When a TypeScript augmentation rebuild owns the sole graph pass, restore every ordinary graph/query index before readiness, then drop the reverse candidate-symbol lookup immediately before augmentation candidate population and keep it deferred @@ -4089,6 +4093,10 @@ unchanged、または sparse mutation の target 集合では全 index を維持 してください。identity / resolution 中は query-only 集合を遅延したままにし、mutual recursion の 直前に reverse-edge 用3本を復元して、その update 後に残りを戻してください。小規模 scoped update は固定的な再構築 cost が更新時間を支配しないよう、全 index を維持します。 +full mutual-recursion update は、call-like または非canonicalな row ごとに望ましい flag を +1回 materialize してから変更を適用します。相関 reverse-edge 式を `SET` と `WHERE` の +両方で評価すると、巨大な fresh graph で同じランダム B-tree probe が二重になるため、 +single-evaluation の契約を維持してください。 C# の reference-graph finalization は、reference arity、invocation arity、member receiver、 definition arity、constructor arity、value-type の fact を、対象 row ごとに TEMP table へ1回だけ diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 860c895e5..073a6acd9 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -706,6 +706,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` keeps 12,000 no-alias calls from paying for alias dedupe, `CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` guards pre-sized stable compaction after alias rewrites, and `MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` covers repeated qualified C#- and Python-style cycle names without per-edge normalization strings. `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` warms a 1,024-row typed snapshot with its exact capacity, then prevents a second candidate collection or redundant path values from returning to the default `net8.0` path. `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. + `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. - `HotspotReferenceAggregateTests.cs` Regression coverage for the maintained per-file hotspot aggregate, including legacy transactional backfill, high-cardinality limited file/name queries, aggregate/raw logical-site parity, cross-file context and identity invalidation, and cancellation after aggregate SQL begins. Query-plan assertions require the rank index on `hotspot_reference_counts`, reject raw `symbol_references` scans, and use the shared broad deterministic timeout instead of a benchmark-grade threshold. - `.github/scripts/run-dotnet-tests.ps1` @@ -1710,6 +1711,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `ReferenceExtraction_CSharpNoAliasDenseReferences_StaysWithinAllocationBudget` は alias のない 12,000 call が alias dedupe のコストを負わないこと、`CSharpAliasCompaction_DenseDuplicates_StaysWithinAllocationBudget` は alias rewrite 後の事前 capacity 付き stable compaction、`MutualRecursion_DenseRepeatedQualifiedNames_StaysWithinAllocationBudget` は C# / Python 形式の qualified cycle name が edge ごとの正規化文字列を作らないことを保証します。 `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` は exact capacity を渡した1,024行の typed snapshot を warm-up し、2つ目の候補 collection や重複 path value が通常の `net8.0` 経路へ戻らないようにします。 `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 + `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 - `HotspotReferenceAggregateTests.cs` file 単位 maintained hotspot aggregate の回帰 coverage です。legacy database の transactional backfill、高カーディナリティな limit 付き file/name query、aggregate/raw の logical-site parity、cross-file context / identity の無効化、aggregate SQL 開始後の cancellation を含みます。query-plan assertion は `hotspot_reference_counts` の rank index 利用を必須とし、raw `symbol_references` scan を拒否します。benchmark 用の厳しい閾値ではなく、共有の広い deterministic timeout を使います。 - `.github/scripts/run-dotnet-tests.ps1` diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md new file mode 100644 index 000000000..92a604335 --- /dev/null +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -0,0 +1,15 @@ +--- +category: fixed +affected: + - src/CodeIndex/Database/DbWriter.References.cs + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md +--- + +## English + +- **Fresh large-codebase indexes finalize mutual-recursion edges without duplicate reverse lookups** — reference graph finalization now materializes each candidate edge's desired recursion flag once, avoiding a costly second set of random B-tree probes when only a small number of flags change. + +## 日本語 + +- **巨大コードベースの新規インデックスで相互再帰 edge の reverse lookup を重複実行しないようにしました** — reference graph の確定時に各候補 edge の望ましい recursion flag を一度だけ materialize し、変更対象の flag が少数の場合に発生していた高コストな2回目のランダム B-tree probe を避けます。 diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index f6a9f13e4..4413a048d 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -1509,11 +1509,33 @@ UPDATE symbol_references """; private static readonly string RefreshMutualRecursionFlagsSql = $""" + WITH desired_mutual_recursion(id, desired_value) AS MATERIALIZED ( + SELECT r.id, + {MutualRecursionValueSql} + FROM symbol_references AS r + -- Ordinary non-call rows are already persisted with the canonical zero value. + -- Keep them out of the materialized working set while still repairing legacy or + -- externally modified non-boolean values. + -- 通常の非call rowはcanonicalな0で永続化済みなのでwork setから除外しつつ、 + -- legacyまたは外部変更による非boolean値は引き続き修復する。 + WHERE r.reference_kind IN ( + 'call', + 'instantiate', + 'subscribe', + 'unsubscribe', + 'razor_event_binding') + OR r.is_mutual_recursion IS NOT 0 + ) UPDATE symbol_references AS r - SET is_mutual_recursion = {MutualRecursionValueSql} - -- IS NOT is null-safe and also normalizes legacy non-boolean values. - -- IS NOT により NULL と legacy の非boolean値も安全に正規化する。 - WHERE r.is_mutual_recursion IS NOT ({MutualRecursionValueSql}) + SET is_mutual_recursion = desired.desired_value + FROM desired_mutual_recursion AS desired + WHERE r.id = desired.id + -- IS NOT is null-safe and also normalizes legacy non-boolean values. Materializing + -- the desired value keeps each correlated reverse-edge lookup to one evaluation + -- per candidate instead of repeating it in both SET and WHERE. + -- IS NOTによりNULLとlegacyの非boolean値も安全に正規化する。desired valueを + -- materializeし、相関reverse-edge lookupをSETとWHEREで二重評価しない。 + AND r.is_mutual_recursion IS NOT desired.desired_value """; internal static string RefreshMutualRecursionFlagsSqlForTesting diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index d58a37fae..6b06ad953 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -198,11 +198,22 @@ public void InitializeSchema_CreatesOnlyCanonicalReferenceSecondaryIndexes() public void MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans() { const string indexName = "idx_symbol_refs_unresolved_mutual_folded"; + var fullSql = DbWriter.RefreshMutualRecursionFlagsSqlForTesting; + Assert.Contains("AS MATERIALIZED", fullSql, StringComparison.Ordinal); + Assert.Equal( + 1, + CountOccurrences(fullSql, "INDEXED BY idx_symbol_refs_resolved_source_target_kind")); + Assert.Equal( + 1, + CountOccurrences(fullSql, "INDEXED BY idx_symbol_refs_unresolved_mutual_folded")); + Assert.Equal( + 1, + CountOccurrences(fullSql, "INDEXED BY idx_symbol_refs_container_nocase_kind")); AssertUsesPartialIndex( "full", ReadQueryPlanDetails( _db.Connection, - DbWriter.RefreshMutualRecursionFlagsSqlForTesting)); + fullSql)); using var scope = _writer.BeginReferenceGraphRefreshScope(); var scopedLookups = DbWriter.ScopedUnresolvedMutualLookupStatementsForTesting; @@ -221,6 +232,19 @@ static void AssertUsesPartialIndex(string scopeName, IReadOnlyList plan) || detail.StartsWith("SCAN reverse ", StringComparison.OrdinalIgnoreCase)), $"Unexpected reverse scan in {scopeName} plan:{Environment.NewLine}{string.Join(Environment.NewLine, plan)}"); } + + static int CountOccurrences(string value, string search) + { + var count = 0; + for (var start = 0; ;) + { + var found = value.IndexOf(search, start, StringComparison.Ordinal); + if (found < 0) + return count; + count++; + start = found + search.Length; + } + } } [Fact] From 85be9d02d276a3f0a9eb65f85bd16ebd05b1b90a Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 04:41:54 +0900 Subject: [PATCH 02/16] Skip dirty graph tracking for full refresh --- DEVELOPER_GUIDE.md | 9 ++- TESTING_GUIDE.md | 2 + .../+large-codebase-initial-indexing.fixed.md | 3 + .../DbWriter.ReferenceGraphRefreshScope.cs | 12 +++- tests/CodeIndex.Tests/DatabaseTests.cs | 69 +++++++++++++++++++ 5 files changed, 92 insertions(+), 3 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index dfd26c042..3efb15baf 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -337,7 +337,10 @@ materialization and resolution, and the file and reference-line maintenance inde remain available during the load, while identity and resolution finalization continues without query indexes. The guard forces any active dirty graph scope onto its full-refresh plan before the -indexes disappear. Immediately before mutual-recursion evaluation, its graph +indexes disappear. While that force-full plan is known, do not populate the +dirty-file/name/reference TEMP scope: fresh indexes and rebuilds never consume +it, and per-batch tracking otherwise adds avoidable set materialization across +every language. Immediately before mutual-recursion evaluation, its graph transaction restores only the unresolved-folded, legacy NOCASE, and resolved reverse-edge indexes; the remaining query indexes return after the mutual update. The full mutual-recursion update materializes one desired flag per call-like or @@ -4058,7 +4061,9 @@ reverse lookup は raw persistence 中は維持し、実際の graph refresh が 再構築しません。reference scope の materialization / resolution に使う candidate primary key は 維持します。load 中も file と reference-line の保守用 index は残し、identity / resolution finalization 中は query index を遅延したままにします。guard は index を外す前に active な dirty graph scope を full refresh へ -昇格します。mutual-recursion 評価の直前に、その graph transaction 内で unresolved-folded、 +昇格します。この force-full plan が確定している間は dirty file / name / reference の TEMP scope を +投入しないでください。fresh index と rebuild はその scope を参照せず、追跡すると全言語の batch ごとに +不要な set materialization が発生します。mutual-recursion 評価の直前に、その graph transaction 内で unresolved-folded、 legacy NOCASE、resolved reverse-edge の3本だけを復元し、残りの query index は mutual update 後に戻します。TypeScript augmentation rebuild が唯一の graph pass を担当する場合は、readiness 前に通常の graph / query index を復元し、augmentation の candidate 構築直前にだけ diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 073a6acd9..aa802f392 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -707,6 +707,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` warms a 1,024-row typed snapshot with its exact capacity, then prevents a second candidate collection or redundant path values from returning to the default `net8.0` path. `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` models 321,352 references distributed over 856 files in the repository snapshot's five/six-batch shape and fixes the control-SQL contract at 5,009 public batch transaction scopes versus zero explicit atomic-file batch scopes; keep it deterministic and allocation-light instead of inserting all modeled rows or asserting elapsed time. For reference-line window performance audits, measure identical prebuilt reference rows against one-batch and 32-batch limits in alternating order, report both elapsed time and `GC.GetAllocatedBytesForCurrentThread`, and remove the timing harness after recording the result; end-to-end `--memory-trace` rebuilds remain corroborating evidence because extraction, graph finalization, and OS page-cache variance can dominate the persistence delta. `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` also pins the full-refresh desired-value CTE as materialized and requires each reverse-edge index lookup to occur only once in the SQL text. Keep this structural assertion together with the query-plan checks: duplicating the correlated expression between `SET` and `WHERE` turns fresh large-graph finalization into repeated random B-tree probes even when only a handful of recursion flags change. + `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` verifies that a forced full refresh still finalizes the language-independent graph while leaving the unused dirty file/name TEMP scope empty. Keep this paired with the ordinary scoped-refresh tests so fresh/rebuild batching cannot silently regain per-file tracking work. - `HotspotReferenceAggregateTests.cs` Regression coverage for the maintained per-file hotspot aggregate, including legacy transactional backfill, high-cardinality limited file/name queries, aggregate/raw logical-site parity, cross-file context and identity invalidation, and cancellation after aggregate SQL begins. Query-plan assertions require the rank index on `hotspot_reference_counts`, reject raw `symbol_references` scans, and use the shared broad deterministic timeout instead of a benchmark-grade threshold. - `.github/scripts/run-dotnet-tests.ps1` @@ -1712,6 +1713,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" `ReusableStatSnapshot_OnePassMaterialization_StaysWithinAllocationBudget` は exact capacity を渡した1,024行の typed snapshot を warm-up し、2つ目の候補 collection や重複 path value が通常の `net8.0` 経路へ戻らないようにします。 `ReferenceBatchTransactions_RepositoryScaleAtomicFileScopeEliminatesControlledSqlScopes` は321,352 refsを自己snapshotの5/6 batch形状で856 filesへ分配し、制御SQL契約をpublic batch transaction scope 5,009回対explicit atomic-file scope 0回に固定します。全model rowを挿入したり経過時間をassertしたりせず、deterministicでallocation-lightなまま維持してください。reference-line windowの性能監査では、同一の事前構築済みreference rowを1-batch上限と32-batch上限で交互に測り、経過時間と`GC.GetAllocatedBytesForCurrentThread`の両方を報告して、結果記録後にtiming harnessを削除します。end-to-endの`--memory-trace` rebuildは、extraction・graph finalize・OS page cacheの変動が永続化差を支配し得るため、補助証拠として扱ってください。 `DatabaseTests.MutualRecursionLookups_UsePartialUnresolvedIndexInFullAndScopedPlans` は、full refresh の desired-value CTE が materialize され、各 reverse-edge index lookup が SQL text 内で1回だけ現れることも固定します。この構造 assertion は query-plan check と一緒に維持してください。相関式を `SET` と `WHERE` で重複させると、変更される recursion flag が少数でも、巨大な fresh graph でランダム B-tree probe が反復されます。 + `DatabaseTests.ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking` は、forced full refresh が複数言語に共通する graph 確定を完了しつつ、未使用の dirty file / name TEMP scope を空のまま保つことを検証します。fresh / rebuild の batch に file ごとの追跡処理が戻らないよう、通常の scoped-refresh test と対で維持してください。 - `HotspotReferenceAggregateTests.cs` file 単位 maintained hotspot aggregate の回帰 coverage です。legacy database の transactional backfill、高カーディナリティな limit 付き file/name query、aggregate/raw の logical-site parity、cross-file context / identity の無効化、aggregate SQL 開始後の cancellation を含みます。query-plan assertion は `hotspot_reference_counts` の rank index 利用を必須とし、raw `symbol_references` scan を拒否します。benchmark 用の厳しい閾値ではなく、共有の広い deterministic timeout を使います。 - `.github/scripts/run-dotnet-tests.ps1` diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 92a604335..10aee2926 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -2,6 +2,7 @@ category: fixed affected: - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md --- @@ -9,7 +10,9 @@ affected: ## English - **Fresh large-codebase indexes finalize mutual-recursion edges without duplicate reverse lookups** — reference graph finalization now materializes each candidate edge's desired recursion flag once, avoiding a costly second set of random B-tree probes when only a small number of flags change. +- **Fresh and rebuilt indexes skip unused incremental graph bookkeeping** — once a full reference-graph refresh is known, symbol and reference batches no longer populate dirty-scope tables that the full plan never reads, removing repeated set construction across all indexed languages. ## 日本語 - **巨大コードベースの新規インデックスで相互再帰 edge の reverse lookup を重複実行しないようにしました** — reference graph の確定時に各候補 edge の望ましい recursion flag を一度だけ materialize し、変更対象の flag が少数の場合に発生していた高コストな2回目のランダム B-tree probe を避けます。 +- **新規作成および rebuild 時に未使用の差分 graph bookkeeping を省くようにしました** — reference graph の full refresh が確定した後は、その plan が参照しない dirty scope table を symbol / reference batch ごとに投入せず、全インデックス対象言語にまたがる反復的な set 構築を取り除きます。 diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 075333124..771ccc8b5 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -541,7 +541,17 @@ reference_id INTEGER PRIMARY KEY } private bool IsTrackingReferenceGraphRefresh - => _referenceGraphRefreshScope is { IsCompleting: false, IsDisposed: false }; + // A forced full refresh never consumes the dirty TEMP tables. Fresh indexes and + // rebuilds can therefore skip the per-batch Distinct/materialization work across + // every language while preserving the scoped path for existing databases. + // forced full refreshはdirty TEMP tableを参照しないため、全言語のbatchごとの + // Distinct/materializeを省き、既存DBのscoped pathだけ追跡する。 + => _referenceGraphRefreshScope is + { + IsCompleting: false, + IsDisposed: false, + ForceFullRefresh: false, + }; private void RequireFullReferenceGraphRefresh() { diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 6b06ad953..355d2c114 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -1182,6 +1182,75 @@ public void ReferenceGraphDirtyScope_TinySetSkipsGlobalReferenceCountWithoutDiag } } + [Fact] + public void ReferenceGraphDirtyScope_ForcedFullRefreshSkipsUnusedDirtyTracking() + { + DbWriter.ReferenceGraphRefreshScopeStats? observed = null; + var previousHook = DbWriter.ReferenceGraphRefreshScopeForTesting; + try + { + DbWriter.ReferenceGraphRefreshScopeForTesting = stats => observed = stats; + using var scope = _writer.BeginReferenceGraphRefreshScope(forceFullRefresh: true); + long fileId; + using (var transaction = _writer.BeginTransaction()) + { + fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/forced-full.py", + Lang = "python", + Size = 100, + Lines = 1, + Modified = new DateTime(2025, 1, 2, 0, 0, 0, DateTimeKind.Utc), + Checksum = "forced-full-replacement", + }); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "After", + Line = 1, + }, + ]); + _writer.InsertReferences([ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "After", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "After()", + }, + ], refreshMutualRecursionFlags: false); + transaction.Commit(); + } + + Assert.Equal(0, ExecuteScalarLong( + "SELECT COUNT(*) FROM temp.reference_graph_dirty_files")); + Assert.Equal(0, ExecuteScalarLong( + "SELECT COUNT(*) FROM temp.reference_graph_dirty_names")); + Assert.Equal(0, ExecuteScalarLong( + "SELECT COUNT(*) FROM temp.reference_graph_removed_references")); + Assert.Equal(0, ExecuteScalarLong( + "SELECT COUNT(*) FROM temp.reference_graph_dirty_references")); + + _writer.RefreshMutualRecursionFlags(); + + Assert.NotNull(observed); + Assert.True(observed!.UsedFullRefresh); + Assert.Equal(0, observed.DirtyFileCount); + Assert.Equal(0, observed.DirtyNameCount); + Assert.Equal(1, observed.DirtyReferenceCount); + Assert.Equal(1, observed.TotalReferenceCount); + Assert.Equal("resolved", ReadReferenceResolutionState(fileId)); + } + finally + { + DbWriter.ReferenceGraphRefreshScopeForTesting = previousHook; + } + } + [Fact] public void ReferenceGraphDirtyScope_TracksLanguageTransitionsAndMatchesFullRefresh() { From 91a1c7272008d0a926b4f8a5c35fe53bf47a52f7 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 04:55:51 +0900 Subject: [PATCH 03/16] Reuse loaded configs in C# workspace prepass --- DEVELOPER_GUIDE.md | 9 +++ TESTING_GUIDE.md | 2 + .../+large-codebase-initial-indexing.fixed.md | 3 + ...Runner.FullScan.CSharpFinalRevalidation.cs | 1 + ...xCommandRunner.FullScan.CSharpPreflight.cs | 1 + ...xCommandRunner.FullScan.TargetSelection.cs | 1 + ...dexCommandRunner.Update.CSharpPreflight.cs | 2 + .../Indexer/CSharpStaticInterfacePrepass.cs | 21 ++++-- .../Mcp/McpToolHandlers.Indexing.Execution.cs | 2 + .../ExtractorPluginRegistryTests.cs | 71 +++++++++++++++++++ 10 files changed, 107 insertions(+), 6 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 3efb15baf..1e08ab85b 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -406,6 +406,11 @@ is still compared with a fresh filesystem size and UTC modification time, and language extractor versions, extraction caps, stale issue metadata, and generated code suppression must all remain part of the snapshot eligibility contract. Do not replace this snapshot with per-file database probes in either CLI or MCP. +CLI full scans, scoped updates, and MCP indexing also load the workspace pattern +configuration once before this prepass. Their C# candidate extraction must reuse +that loaded snapshot instead of refreshing default plugins for every candidate; +direct prepass callers retain the discovery-enabled default unless they explicitly +prove the snapshot is already loaded. Rows with missing or invalid legacy stat values are excluded so normal checksum reuse or reindexing can repair them, and CLI/MCP cancellation must interrupt the snapshot query as well as the later extraction pipeline. @@ -4120,6 +4125,10 @@ generated-code suppression も snapshot eligibility contract に含めます。C この snapshot を file ごとの database probe に戻さないでください。旧 DB の欠損または不正な stat 値を持つ row は除外して通常の checksum reuse / 再 index で修復し、CLI/MCP の cancellation は 後続の extraction pipeline だけでなく snapshot query も中断できる状態を保ってください。 +CLI full scan、scoped update、MCP indexing は、この prepass より前に workspace pattern config も +1回だけ読み込みます。C# candidate extraction は candidate ごとに default plugin を refresh せず、 +その読込済み snapshot を再利用してください。直接 prepass を呼ぶ側は、snapshot 読込済みを明示的に +保証しない限り、従来どおり discovery 有効の既定経路を維持します。 authoritative な full scan は、共有 source-directory enumeration 中に C#、VB、F#、 MSBuild の project-marker fingerprint を収集します。同じ pass で budget 非依存の diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index aa802f392..e8885d6cf 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -785,6 +785,7 @@ Use the inventory below before adding or moving a test class: - SQLite pool resets, direct `SqliteConnection.ClearAllPools()` calls, process current-directory changes, or process-global environment variable mutation: put the class in the `SQLite pool sensitive` non-parallel collection. - Extractor plugin registry tests belong in the dedicated non-parallel `Plugin registry sensitive` collection: they mutate process-global registry reload state, worker inventories, staging hooks, and user plugin directory overrides. Keep `TrustedPluginAssemblyFixture` on that collection only, so ordinary console-sensitive tests do not build an unused plugin fixture. +- `ExtractorPluginRegistryTests.CSharpWorkspacePrepass_ReusesLoadedPatternConfigSnapshot` preloads one invalid-plugin discovery snapshot, runs several C# candidates in parallel without restaging that plugin, then proves the direct-caller default still performs discovery. Keep the staging hook and user-directory override inside the same plugin-sensitive lock and `finally` cleanup. - Environment variables: use `EnvironmentVariableScope.Capture(...)` so setup failures and assertion failures restore the original values through one cleanup path. - JSON API-version utility coverage belongs in the non-parallel collection because its combined command scenario temporarily disables update checks through a process-global environment variable; console locking alone does not isolate that lifetime. - `Console.Out` or `Console.Error` replacement: prefer `ConsoleCapture`, which owns the shared gate and checks the restored writer identity. If a specialized fixture must swap a writer directly, lock `TestConsoleLock.Gate` around the whole capture/swap window, restore in `finally`, and restore before disposing the captured writer. @@ -1791,6 +1792,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - SQLite pool reset、`SqliteConnection.ClearAllPools()` の直接呼び出し、プロセスの current directory 変更、process-global な環境変数変更: クラスを non-parallel な `SQLite pool sensitive` collection に入れる。 - extractor plugin registry testは専用のnon-parallelな`Plugin registry sensitive` collectionに置いてください。process-globalなregistry reload state、worker inventory、staging hook、user plugin directory overrideを変更します。通常のconsole-sensitive testが未使用plugin fixtureを構築しないよう、`TrustedPluginAssemblyFixture`はこのcollectionだけで所有します。 +- `ExtractorPluginRegistryTests.CSharpWorkspacePrepass_ReusesLoadedPatternConfigSnapshot` は invalid plugin の discovery snapshot を1回preloadし、複数の C# candidate を並列処理してもpluginを再stageしないことと、直接callerの既定経路ではdiscoveryが引き続き行われることを検証します。staging hookとuser-directory overrideは同じplugin-sensitive lockおよび`finally` cleanup内に維持してください。 - 環境変数: `EnvironmentVariableScope.Capture(...)` を使い、setup failure や assertion failure でも単一の cleanup 経路で元の値に戻す。 - JSON API-versionのutility coverageは、combined command scenarioがprocess-globalな環境変数でupdate checkを一時無効化するためnon-parallel collectionに置きます。console lockだけではそのlifetimeを分離できません。 - `Console.Out` / `Console.Error` の差し替え: shared gate の所有と復元writerの同一性確認を行う `ConsoleCapture` を優先する。特殊なfixtureで直接差し替える必要がある場合は、capture / swap 期間全体を `TestConsoleLock.Gate` で lockし、`finally`で復元し、capture writerをdisposeする前に元writerへ戻す。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 10aee2926..076e9edf7 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -3,6 +3,7 @@ category: fixed affected: - src/CodeIndex/Database/DbWriter.References.cs - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md --- @@ -11,8 +12,10 @@ affected: - **Fresh large-codebase indexes finalize mutual-recursion edges without duplicate reverse lookups** — reference graph finalization now materializes each candidate edge's desired recursion flag once, avoiding a costly second set of random B-tree probes when only a small number of flags change. - **Fresh and rebuilt indexes skip unused incremental graph bookkeeping** — once a full reference-graph refresh is known, symbol and reference batches no longer populate dirty-scope tables that the full plan never reads, removing repeated set construction across all indexed languages. +- **Initial C# workspace prepasses reuse the loaded extractor configuration** — CLI and MCP indexing no longer rediscover default plugins under a shared lock for every static/enum/const candidate after the workspace pattern snapshot has already been loaded. ## 日本語 - **巨大コードベースの新規インデックスで相互再帰 edge の reverse lookup を重複実行しないようにしました** — reference graph の確定時に各候補 edge の望ましい recursion flag を一度だけ materialize し、変更対象の flag が少数の場合に発生していた高コストな2回目のランダム B-tree probe を避けます。 - **新規作成および rebuild 時に未使用の差分 graph bookkeeping を省くようにしました** — reference graph の full refresh が確定した後は、その plan が参照しない dirty scope table を symbol / reference batch ごとに投入せず、全インデックス対象言語にまたがる反復的な set 構築を取り除きます。 +- **初回 C# workspace prepass で読込済み extractor config を再利用するようにしました** — workspace pattern snapshot の読込後に、static / enum / const の候補ごとに共有lock下でdefault pluginを再探索しないよう、CLIとMCP indexingを既読込経路へ接続します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpFinalRevalidation.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpFinalRevalidation.cs index 6739dddac..a637c8e14 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpFinalRevalidation.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpFinalRevalidation.cs @@ -158,6 +158,7 @@ private static FullScanCSharpFinalRevalidationResult context.StaleFilePurgePlan.FileIds, isExistingSymbolPathExcluded: context.IsExistingCSharpSymbolPathNowNonCSharp, + patternConfigsAlreadyLoaded: true, cancellationToken: context.CancellationToken), context.CancellationToken); if (!workspace.SourceContractEvidenceComplete) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs index ef9935fbe..35f5893f6 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs @@ -312,6 +312,7 @@ private static CSharpStaticInterfaceWorkspaceSymbols isExistingSymbolPathExcluded: context .IsExistingCSharpSymbolPathNowNonCSharp, + patternConfigsAlreadyLoaded: true, cancellationToken: context.CancellationToken), context.CancellationToken); } diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.TargetSelection.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.TargetSelection.cs index dd9345451..8d8f8f105 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.TargetSelection.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.TargetSelection.cs @@ -305,6 +305,7 @@ private static void RebuildInvalidatedFullScanCSharpNoOp( context.StaleFilePurgePlan.FileIds, isExistingSymbolPathExcluded: context.IsExistingCSharpSymbolPathNowNonCSharp, + patternConfigsAlreadyLoaded: true, cancellationToken: context.CancellationToken), context.CancellationToken); state.CSharpWorkspaceFileSnapshots = workspaceFileSnapshots; diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Update.CSharpPreflight.cs b/src/CodeIndex/Cli/IndexCommandRunner.Update.CSharpPreflight.cs index ded37587f..360d90d47 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Update.CSharpPreflight.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Update.CSharpPreflight.cs @@ -282,6 +282,7 @@ private static void BuildInitialUpdateCSharpWorkspaceSnapshot( loadExistingSymbolsOnlyForPendingQualifiedMemberAccess: true, parallelism: context.Options.Parallelism, + patternConfigsAlreadyLoaded: true, cancellationToken: context.CancellationToken); if (!CSharpStaticInterfacePrepass.TryValidateFileStatSnapshots( state.CSharpPrepassTargets, @@ -424,6 +425,7 @@ private static void BuildExpandedUpdateCSharpWorkspace( parallelism: context.Options.Parallelism, excludedExistingFileIds: context.ScopedCleanupPlan.FileIds, + patternConfigsAlreadyLoaded: true, cancellationToken: cancellationToken); } diff --git a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs index 0951a1575..8fa90ebcf 100644 --- a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs +++ b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs @@ -25,6 +25,7 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( IReadOnlyList? excludedExistingFileIds = null, Func? isExistingSymbolPathExcluded = null, bool loadExistingSymbolsOnlyForPendingQualifiedMemberAccess = false, + bool patternConfigsAlreadyLoaded = false, CancellationToken cancellationToken = default) { var targetCount = fileTargets.TryGetNonEnumeratedCount(out var count) ? count : 0; @@ -114,12 +115,19 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( if (MayContainCSharpWorkspaceReferenceTargets(content)) { extractedByCandidate[candidateIndex] = - SymbolExtractor.Extract( - 0, - "csharp", - content, - target.IndexPath, - cancellationToken: cancellationToken); + patternConfigsAlreadyLoaded + ? SymbolExtractor.ExtractWithPatternConfigsLoaded( + 0, + "csharp", + content, + target.IndexPath, + cancellationToken: cancellationToken) + : SymbolExtractor.Extract( + 0, + "csharp", + content, + target.IndexPath, + cancellationToken: cancellationToken); } } } @@ -216,6 +224,7 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( parallelism: 1, excludedExistingFileIds: null, loadExistingSymbolsOnlyForPendingQualifiedMemberAccess: false, + patternConfigsAlreadyLoaded: false, cancellationToken: cancellationToken); } diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index d19328090..66b864093 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -715,6 +715,7 @@ CSharpStaticInterfaceWorkspaceSymbols BuildStableCSharpWorkspace( parallelism: 1, excludedExistingFileIds: staleFilePurgePlan.FileIds, isExistingSymbolPathExcluded: IsExistingCSharpSymbolPathNowNonCSharp, + patternConfigsAlreadyLoaded: true, cancellationToken: requestToken)); forceFullCSharpRefreshFromInvalidatedNoOp = indexSnapshot.CSharpStaticInterfaceSourceEvidence == true @@ -1036,6 +1037,7 @@ void CountFreshInsertedRows( parallelism: 1, excludedExistingFileIds: staleFilePurgePlan.FileIds, isExistingSymbolPathExcluded: IsExistingCSharpSymbolPathNowNonCSharp, + patternConfigsAlreadyLoaded: true, cancellationToken: requestToken)); preservePriorPositiveCSharpSourceNoOp = false; if (!csharpWorkspace.SourceContractEvidenceComplete) diff --git a/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs index cd2b28880..16c5d0da4 100644 --- a/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs +++ b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs @@ -555,6 +555,77 @@ public void DefaultPluginDiscovery_RepairsAndReplacesFingerprintWithoutLeakingWo } } + [Fact] + public void CSharpWorkspacePrepass_ReusesLoadedPatternConfigSnapshot() + { + var projectRoot = TestProjectHelper.CreateExecutableExtensionTestProject( + "extractor_registry_csharp_prepass_snapshot"); + lock (TestConsoleLock.Gate) + { + var pluginDirectory = Path.Combine(projectRoot, "plugins"); + var pluginPath = Path.Combine(pluginDirectory, "invalid.dll"); + var pluginStageCount = 0; + try + { + Directory.CreateDirectory(pluginDirectory); + File.WriteAllText(pluginPath, "invalid plugin assembly"); + var sourceDirectory = Path.Combine(projectRoot, "src"); + Directory.CreateDirectory(sourceDirectory); + var targets = new List(); + for (var index = 0; index < 8; index++) + { + var sourcePath = Path.Combine(sourceDirectory, $"Static{index}.cs"); + File.WriteAllText( + sourcePath, + $"public static class Static{index} {{ public const int Value = {index}; }}"); + targets.Add(CodeIndex.Indexer.CSharpStaticInterfacePrepass.FileTarget.Create( + projectRoot, + sourcePath, + "csharp")); + } + + ExtractorPluginRegistry.ReloadForTests(); + ExtractorPluginRegistry.UserPluginDirectoryForTesting = pluginDirectory; + ExecutableExtensionBoundary.StagedForTesting = (source, _) => + { + if (string.Equals(source, pluginPath, StringComparison.Ordinal)) + pluginStageCount++; + }; + ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(projectRoot); + Assert.Equal(1, pluginStageCount); + + var dbPath = TestProjectHelper.CreateProjectDb(projectRoot); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + var indexer = new CodeIndex.Indexer.FileIndexer(projectRoot, ignoreCase: false); + + var workspace = CodeIndex.Indexer.CSharpStaticInterfacePrepass.BuildWorkspaceSymbols( + writer, + indexer, + targets, + includeExistingSymbols: false, + parallelism: 4, + patternConfigsAlreadyLoaded: true); + + Assert.True(workspace.SourceContractEvidenceComplete); + Assert.Equal(1, pluginStageCount); + + CodeIndex.Indexer.CSharpStaticInterfacePrepass.BuildWorkspaceSymbols( + writer, + indexer, + [targets[0]], + includeExistingSymbols: false); + Assert.Equal(2, pluginStageCount); + } + finally + { + ExecutableExtensionBoundary.StagedForTesting = null; + ExtractorPluginRegistry.ResetForTests(); + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + } + [Fact] public void LoadPlugin_KillsTimedOutAndCrashedConstructorWorkers_Issue4598() { From 386f22e22d25d94e92883cdd7958f025c4f39ce7 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 05:30:15 +0900 Subject: [PATCH 04/16] Optimize fresh reference resolution --- DEVELOPER_GUIDE.md | 40 +- TESTING_GUIDE.md | 2 + .../+large-codebase-initial-indexing.fixed.md | 7 + .../Cli/IndexCommandRunner.FullScan.cs | 13 +- .../DbWriter.ReferenceGraphRefreshScope.cs | 28 +- .../Database/DbWriter.ReferenceSql.cs | 22 +- src/CodeIndex/Database/DbWriter.References.cs | 127 +++-- src/CodeIndex/Database/DbWriter.cs | 3 +- .../FreshReferenceResolutionTests.cs | 458 ++++++++++++++++++ ...ommandRunnerReferenceIndexBulkLoadTests.cs | 58 +++ 10 files changed, 711 insertions(+), 47 deletions(-) create mode 100644 tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 1e08ab85b..31d7743dc 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1513,15 +1513,27 @@ The persisted reference-identity contract is versioned for this rule, so indexes the compatibility filter are treated as non-authoritative until a normal index refresh rebuilds their candidates. -Reference finalization computes candidate count, minimum symbol ID, distinct target-family -count, and stable target key in one correlated aggregate per reference. Keep these four -resolution fields on the row-value assignment path; separate scalar subqueries multiply the -candidate-index and symbol/file lookup work on large graphs. The language/name families that +Existing-index, rebuild, and retained-graph finalization paths compute candidate count, minimum +symbol ID, distinct target-family count, and stable target key in one correlated aggregate per +reference. Keep these four resolution fields on the row-value assignment path; separate scalar +subqueries multiply the candidate-index and symbol/file lookup work on large graphs. The +language/name families that are globally unique are aggregated once into the connection-local `temp.reference_unique_symbol_families` table and reused by non-C#, C#, and C# attribute fallbacks. Create that temp table in a separate prepared command before preparing the refresh; SQLite resolves referenced tables while preparing every statement in a command batch. +A true empty-database ordinary CLI full scan (not `--rebuild` or `--symbols-only`) opts into a +separate fresh-resolution contract. Reference inserts persist canonical provisional values +(`unresolved`, candidate count zero, and zero self/mutual flags) without adding bind parameters. +Finalization scans `symbol_reference_candidates` once into materialized per-reference facts and +updates only candidate-bearing references by primary key; candidate-free references retain their +provisional values, and the self flag is derived in that same sparse update. The opt-in remains +pending after a failed graph transaction and clears only after the graph commit. Existing-index +updates, rebuilds, retained-graph rebuilds, and MCP indexing keep the established path. In +particular, MCP can durably commit per-file batches before graph finalization, so it must retain +its existing retry semantics until a separately designed recovery contract can cover that state. + For C# explicit-interface members, `symbols.name` remains the short display/discovery alias, while `symbols.name_folded` stores the normalized interface qualifier plus terminal method generic arity. When that identity differs, `symbols.display_name_folded` stores the short @@ -5188,15 +5200,27 @@ Java の reference resolution は変更しません。 この規則は persisted reference-identity contract の version 対象であり、compatibility filter 導入前に 作成された index は、通常の index 更新で candidate を再構築するまで非 authoritative として扱います。 -reference finalization は、candidate count、最小 symbol ID、distinct target-family count、安定 target -key を reference ごとに1回の correlated aggregate で計算します。この4つの resolution field は -row-value assignment のまま維持してください。scalar subquery を分けると、大規模 graph で -candidate index と symbol/file lookup が重複します。global に一意な language/name family は +既存index、rebuild、retained graph の reference finalization は、candidate count、最小 symbol ID、 +distinct target-family count、安定 target key を reference ごとに1回の correlated aggregate で +計算します。この4つの resolution field は row-value assignment のまま維持してください。 +scalar subquery を分けると、大規模 graph で candidate index と symbol/file lookup が重複します。 +global に一意な language/name family は connection-local な `temp.reference_unique_symbol_families` table へ1回だけ集約し、non-C#、C#、 C# attribute fallback で共有します。この temp table は refresh command を prepare する前に別の prepared command で作成してください。SQLite は command batch の全statementをprepareする時点で 参照tableを解決します。 +真に空のdatabaseから始める通常のCLI full scan（`--rebuild` と `--symbols-only` を除く）だけは、 +fresh resolution専用の契約をopt-inします。reference insertはbind parameterを増やさず、 +`unresolved`、candidate count 0、self/mutual flag 0というcanonicalな暫定値を永続化します。 +finalizationは`symbol_reference_candidates`をreferenceごとのmaterialized factsへ1回走査し、 +candidateを持つreferenceだけをprimary keyで更新します。candidateを持たないreferenceは暫定値を +維持し、self flagも同じsparse update内で導出します。このopt-inはgraph transaction失敗後も +pendingのまま残り、graph commit後にだけ解除します。既存indexのupdate、rebuild、retained graph +rebuild、MCP indexingは従来経路を維持します。特にMCPはgraph finalization前にfile batchをdurable +commitできるため、その状態を扱う独立したrecovery契約が設計されるまでは既存の再試行semanticsを +変更してはいけません。 + C# の明示的 interface member では、`symbols.name` は短い表示用 / discovery alias のままにし、 `symbols.name_folded` に正規化した interface qualifier と末尾 method の generic arity を 保存します。identity が異なる場合は `symbols.display_name_folded` に短い Unicode-folded diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index e8885d6cf..6a66b3766 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -38,6 +38,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. - Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Lowering or raising the file-size policy must reprocess unchanged files in CLI and MCP indexing so a prior `file_too_large` issue cannot be reused. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps and must not label an incomplete index as fold-only. Human output must identify incomplete generations instead of printing a complete summary. - Reference-identity refresh coverage treats a stable graph rebuild as a physical-write performance contract. Keep NULL-safe changed-row predicates for source identity, the four-column target-resolution tuple, self-reference, and mutual-recursion updates; trigger audits must remain at zero on a stable rerun, repair each corrupted phase once, and prove a later-phase failure rolls back earlier identity writes. SQLite `changes()` must continue to report the final mutual-recursion phase. +- Fresh-reference resolution coverage belongs in `FreshReferenceResolutionTests`. Keep the empty/rebuild/symbols-only policy truth table, the unchanged 14-parameter insert shape with distinct fresh/ordinary SQL cache entries, canonical provisional values, the materialized candidate-side aggregation shape, and exact fresh-versus-full semantic parity for `unresolved`, `resolved`, `resolved_group`, `ambiguous`, and self-reference rows across C# and a non-C# language. Failure tests must prove that fresh defaults remain pending after graph rollback and clear only after a successful commit. Pair these database tests with the full-scan bulk-load theory that observes `unresolved` provisional rows on a fresh CLI scan and NULL resolution state on rebuild. MCP remains outside this opt-in because its per-file durable transactions require the existing graph-failure retry contract. - C# reference-graph fact coverage is an SQL-shape performance contract. Each of the six managed arity/receiver/value-type functions must occur once only in its reference or symbol materialization statement, and full, scoped, and retained refresh chains must populate reference, symbol, type-identity, and constructor-identity facts in that order before normalization and candidate consumers. Candidate SQL must use the WITHOUT ROWID fact primary keys without retaining project/file-local identity construction or constructor-owner range scans. Keep full/scoped/retained semantic snapshots across partial generic and file-local constructors, primary constructors with same-leaf generic arities, scoped target-definition mutations, fallback owners, legacy NULL identities, stale-row cleanup, and refresh rollback/retry alongside the focused property-receiver regressions. - C# metadata-target resolver coverage treats propagation work and stable reruns as performance contracts. Keep the reverse-ordered 8,000-class chain at exactly `n - 1` dependency edges and `n` queue visits instead of using a wall-clock threshold; retain cross-file partial fan-in, an unseeded cycle, pre-cancellation, rollback of an earlier row after an injected later update failure, zero trigger-audited writes on a stable rerun, and exactly one write when repairing a corrupted derived row. - Reference-insert transaction coverage keeps the public APIs' #1518 transaction/SAVEPOINT per 71-row batch, while the explicit atomic-file APIs must reject calls without a live caller-owned transaction and open zero reference-batch scopes. Atomic-file reference-line materialization may group only complete 71-row reference batches, stopping before the union of `(file_id, line, context)` keys would exceed 333 rows or after 32 batches; reference INSERT executions and their progress/cancellation checkpoints remain on the original 71-row boundaries. Preserve exact public/atomic statement counts, the 333-row and 32-batch stops, the unique `reference_lines` autoindex lookup plan, batch-two/three failure rollback for both normal and new-file reference-line paths, same/different contexts across a batch boundary, cancellation and empty-input ordering, and guarded multi-language integration coverage for full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. The controlled 321,352-reference/856-file performance contract retains the repository snapshot's five/six-batch large-file distribution and compares 5,009 public batch scopes with zero atomic-file batch scopes without using a wall-clock threshold. @@ -1044,6 +1045,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 - index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。file-size policy を下げた場合も上げた場合も、CLI / MCP indexing は unchanged file を再処理し、以前の `file_too_large` issue を再利用してはいけません。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別し、incomplete index を fold-only と表示しない必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 - reference identity refresh coverage は、安定graphの再構築を物理writeのperformance contractとします。source identity、target resolutionの4列tuple、self reference、mutual recursionの更新にはNULL-safeなchanged-row predicateを維持し、安定rerunのtrigger auditは0、各corrupt phaseのrepairは1回、後段phaseの失敗で先行identity writeもrollbackされることを検証してください。SQLite `changes()` は引き続き最後のmutual-recursion phaseを表します。 +- fresh reference resolutionのcoverageは`FreshReferenceResolutionTests`が担当します。empty/rebuild/symbols-onlyのpolicy truth table、fresh/ordinaryでSQL cache entryを分けても14 parameterのinsert shapeが不変であること、canonicalな暫定値、candidate側materialized aggregationのSQL shape、およびC#と非C#言語を横断する`unresolved`、`resolved`、`resolved_group`、`ambiguous`、self-referenceのfresh/full完全同値を維持してください。failure testはgraph rollback後もfresh defaultsがpendingのままで、成功commit後にだけ解除されることを証明します。database testは、fresh CLI scanで暫定`unresolved`、rebuildでNULL resolution stateを観測するfull-scan bulk-load theoryと対にしてください。MCPはfile単位のdurable transactionが既存graph失敗再試行契約を必要とするため、このopt-inの対象外です。 - C# reference-graph fact coverage は SQL shape の performance contract です。arity / receiver / value-type を求める6つの managed function は reference または symbol の materialization statement 内にそれぞれ1回だけ置き、full / scoped / retained refresh chain は reference、symbol、type identity、constructor identity の fact をこの順で normalization と candidate consumer より前に投入してください。candidate SQL は WITHOUT ROWID fact の主キーを使い、project / file-local identity の再構築や constructor-owner range の再走査を残してはいけません。partial generic / file-local constructor、同名別 generic arity を持つ primary constructor、scoped target 定義変更、fallback owner、legacy NULL identity、stale row cleanup、refresh rollback / retry の full / scoped / retained semantic snapshot と、focused property-receiver regression を維持します。 - C# metadata-target resolver coverage は propagation work と安定 rerun を performance contract とします。逆順に保存した 8,000 class の chain では wall-clock threshold を使わず、dependency edge が厳密に `n - 1`、queue visit が `n` であることを維持してください。cross-file partial fan-in、seed を持たない cycle、事前 cancel、後段 update の注入失敗時に先行 row も rollback されること、安定 rerun の trigger audit が write 0 回、破損した derived row の修復が厳密に 1 write であることも残します。 - reference insert の transaction coverage は、public API の #1518 契約として71 row batchごとの transaction/SAVEPOINTを維持し、明示atomic-file APIは呼出元所有のlive transactionなしでは拒否され、reference batch scopeを0回に保つことを検証します。atomic-fileのreference-line materializationは完全な71 row reference batchだけをまとめ、`(file_id, line, context)` keyの和集合が333行を超える直前、または32 batchで停止します。reference INSERTの実行回数とprogress/cancellation checkpointは元の71 row境界に保ってください。public/atomicの正確なstatement数、333行/32 batch停止、`reference_lines` unique autoindexのlookup plan、通常/new-file両方のreference-line pathでbatch 2/3失敗時の全rollback、batch境界をまたぐ同一/異なるcontext、cancelとempty入力の順序、full scan・scoped update・MCP indexing・TypeScript augmentation rebuildのmulti-language guard付きintegrationを維持してください。321,352 refs / 856 filesの制御performance契約は自己snapshotの5/6 batch巨大file分布を保ち、wall-clock閾値を使わずpublic 5,009 scopeとatomic-file 0 scopeを比較します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 076e9edf7..30094c43b 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -1,9 +1,14 @@ --- category: fixed affected: + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Database/DbWriter.cs - src/CodeIndex/Database/DbWriter.References.cs + - src/CodeIndex/Database/DbWriter.ReferenceSql.cs - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs + - tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md --- @@ -13,9 +18,11 @@ affected: - **Fresh large-codebase indexes finalize mutual-recursion edges without duplicate reverse lookups** — reference graph finalization now materializes each candidate edge's desired recursion flag once, avoiding a costly second set of random B-tree probes when only a small number of flags change. - **Fresh and rebuilt indexes skip unused incremental graph bookkeeping** — once a full reference-graph refresh is known, symbol and reference batches no longer populate dirty-scope tables that the full plan never reads, removing repeated set construction across all indexed languages. - **Initial C# workspace prepasses reuse the loaded extractor configuration** — CLI and MCP indexing no longer rediscover default plugins under a shared lock for every static/enum/const candidate after the workspace pattern snapshot has already been loaded. +- **First-time full indexes resolve only references that have candidates** — empty-database CLI scans persist canonical unresolved defaults during bulk insertion, aggregate the candidate table once, and update candidate-bearing references by primary key instead of probing every reference row. Rebuilds, updates, retained graphs, and MCP indexing keep their existing recovery contracts. ## 日本語 - **巨大コードベースの新規インデックスで相互再帰 edge の reverse lookup を重複実行しないようにしました** — reference graph の確定時に各候補 edge の望ましい recursion flag を一度だけ materialize し、変更対象の flag が少数の場合に発生していた高コストな2回目のランダム B-tree probe を避けます。 - **新規作成および rebuild 時に未使用の差分 graph bookkeeping を省くようにしました** — reference graph の full refresh が確定した後は、その plan が参照しない dirty scope table を symbol / reference batch ごとに投入せず、全インデックス対象言語にまたがる反復的な set 構築を取り除きます。 - **初回 C# workspace prepass で読込済み extractor config を再利用するようにしました** — workspace pattern snapshot の読込後に、static / enum / const の候補ごとに共有lock下でdefault pluginを再探索しないよう、CLIとMCP indexingを既読込経路へ接続します。 +- **初回full indexでcandidateを持つreferenceだけを解決するようにしました** — 空databaseからのCLI scanではbulk insert中にcanonicalな未解決値を保存し、candidate tableを1回集約して、全reference rowをprobeせずcandidateを持つrowだけをprimary keyで更新します。rebuild、update、retained graph、MCP indexingの既存recovery契約は変更しません。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 9804120e8..7bd2b5e75 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -21,6 +21,12 @@ public static partial class IndexCommandRunner private const int PartialIndexFileErrorLimit = 50; + internal static bool ShouldUseFreshReferenceResolutionDefaults( + bool startedWithNoIndexedFiles, + bool rebuild, + bool symbolsOnly) + => startedWithNoIndexedFiles && !rebuild && !symbolsOnly; + @@ -175,6 +181,10 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) if (!options.Json && !options.Quiet) purgeCts = ConsoleUi.StartSpinner("Cleaning up stale entries...", spinnerFrames); var startedWithNoIndexedFiles = !writer.HasAnyIndexedFiles(); + var useFreshReferenceResolutionDefaults = ShouldUseFreshReferenceResolutionDefaults( + startedWithNoIndexedFiles, + options.Rebuild, + options.SymbolsOnly); var priorCSharpStaticInterfaceSourceEvidence = options.Rebuild || startedWithNoIndexedFiles ? null : writer.GetCSharpStaticInterfaceSourceEvidence(); @@ -817,7 +827,8 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis if (options.Rebuild) db.RepairIncompleteBatchReadiness(); using var referenceGraphRefresh = writer.BeginReferenceGraphRefreshScope( - options.Rebuild || !writer.HasAnyIndexedFiles()); + forceFullRefresh: options.Rebuild || startedWithNoIndexedFiles, + useFreshReferenceResolutionDefaults: useFreshReferenceResolutionDefaults); using var hotspotAggregateRefresh = writer.BeginDeferredHotspotReferenceAggregateRefresh( deferSecondaryIndexes: !options.SymbolsOnly && useFtsBulkLoad); using var fullScanTxn = writer.BeginTransaction(cancellationToken, "full scan write phase"); diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 771ccc8b5..9a39014cf 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -493,10 +493,18 @@ AND reverse.symbol_name_folded <> '' /// 参加するため、rollbackされたfile batchのdirty状態は残らない。空・stale契約・広範な /// dirty集合では従来のfull refreshを維持する。 /// - internal ReferenceGraphRefreshScope BeginReferenceGraphRefreshScope(bool forceFullRefresh = false) + internal ReferenceGraphRefreshScope BeginReferenceGraphRefreshScope( + bool forceFullRefresh = false, + bool useFreshReferenceResolutionDefaults = false) { if (_referenceGraphRefreshScope != null) throw new InvalidOperationException("A reference graph refresh scope is already active for this writer."); + if (useFreshReferenceResolutionDefaults && !forceFullRefresh) + { + throw new ArgumentException( + "Fresh reference resolution defaults require a forced full graph refresh.", + nameof(useFreshReferenceResolutionDefaults)); + } using (var command = _conn.CreateCommand()) { @@ -535,7 +543,8 @@ reference_id INTEGER PRIMARY KEY var scope = new ReferenceGraphRefreshScope( this, - forceFullRefresh || !ReferenceIdentityContractMatchesCurrent()); + forceFullRefresh || !ReferenceIdentityContractMatchesCurrent(), + useFreshReferenceResolutionDefaults); _referenceGraphRefreshScope = scope; return scope; } @@ -824,20 +833,31 @@ internal sealed class ReferenceGraphRefreshScope : IDisposable { private readonly DbWriter _writer; private bool _forceFullRefresh; + private bool _freshReferenceResolutionDefaultsPending; - internal ReferenceGraphRefreshScope(DbWriter writer, bool forceFullRefresh) + internal ReferenceGraphRefreshScope( + DbWriter writer, + bool forceFullRefresh, + bool useFreshReferenceResolutionDefaults) { _writer = writer; _forceFullRefresh = forceFullRefresh; + _freshReferenceResolutionDefaultsPending = useFreshReferenceResolutionDefaults; } internal bool IsCompleting { get; set; } internal bool IsDisposed { get; private set; } internal bool ForceFullRefresh => _forceFullRefresh; + internal bool FreshReferenceResolutionDefaultsPending + => _freshReferenceResolutionDefaultsPending; internal void RequireFullRefresh() => _forceFullRefresh = true; - internal void MarkRefreshCompleted() => _forceFullRefresh = false; + internal void MarkRefreshCompleted() + { + _forceFullRefresh = false; + _freshReferenceResolutionDefaultsPending = false; + } public void Dispose() { diff --git a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs index 3c52a5f70..7a3a9b33d 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceSql.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceSql.cs @@ -13,7 +13,8 @@ internal sealed record ReferenceInsertBindingWork( int StatementRows, int BoundParameterCount, int MaterializedReferenceCount, - int MaterializedReferenceLineCount); + int MaterializedReferenceLineCount, + bool UsesFreshResolutionDefaults); internal static Action? ReferenceInsertBindingWorkForTesting { @@ -21,7 +22,9 @@ internal static Action? ReferenceInsertBindingWorkFo set => ScopedReferenceInsertBindingWorkForTesting.Value = value; } - private static string BuildReferenceInsertSql(int rowCount) + private static string BuildReferenceInsertSql( + int rowCount, + bool useFreshReferenceResolutionDefaults) { var sql = CreateBatchSqlBuilder(rowCount, estimatedCharsPerRow: 256); sql.Append(@" @@ -29,7 +32,10 @@ INSERT INTO symbol_references ( file_id, symbol_name, reference_kind, line, column_number, span_length, context, reference_line_id, container_kind, container_name, symbol_name_folded, container_name_folded, is_self_reference, - is_mutual_recursion, target_qualifier + is_mutual_recursion, target_qualifier"); + if (useFreshReferenceResolutionDefaults) + sql.Append(", resolution_state, resolution_candidate_count"); + sql.Append(@" ) VALUES "); var parameterIndex = 0; @@ -37,14 +43,18 @@ INSERT INTO symbol_references ( { if (row > 0) sql.Append(", "); - AppendReferenceInsertParameterTuple(sql, ref parameterIndex); + AppendReferenceInsertParameterTuple( + sql, + ref parameterIndex, + useFreshReferenceResolutionDefaults); } return sql.ToString(); } private static void AppendReferenceInsertParameterTuple( StringBuilder sql, - ref int parameterIndex) + ref int parameterIndex, + bool useFreshReferenceResolutionDefaults) { sql.Append('('); for (var column = 0; column < 15; column++) @@ -56,6 +66,8 @@ private static void AppendReferenceInsertParameterTuple( else sql.Append("@p").Append(parameterIndex++); } + if (useFreshReferenceResolutionDefaults) + sql.Append(", 'unresolved', 0"); sql.Append(')'); } diff --git a/src/CodeIndex/Database/DbWriter.References.cs b/src/CodeIndex/Database/DbWriter.References.cs index 4413a048d..936e340d6 100644 --- a/src/CodeIndex/Database/DbWriter.References.cs +++ b/src/CodeIndex/Database/DbWriter.References.cs @@ -1494,6 +1494,9 @@ UPDATE symbol_references SET is_self_reference = {SelfReferenceValueSql}; """; + internal static string RefreshReferenceResolutionFullSqlForTesting + => RefreshReferenceResolutionFullSql; + private static readonly string RefreshReferenceResolutionDifferentialSql = $""" UPDATE symbol_references AS r SET (target_symbol_id, target_symbol_key, resolution_candidate_count, resolution_state) = {ReferenceResolutionValueSql} @@ -1508,6 +1511,52 @@ UPDATE symbol_references WHERE is_self_reference IS NOT ({SelfReferenceValueSql}); """; + private static readonly string RefreshReferenceResolutionFreshSparseSql = $""" + WITH resolution_facts AS MATERIALIZED ( + SELECT candidate.reference_id, + COUNT(*) AS candidate_count, + MIN(candidate.symbol_id) AS minimum_symbol_id, + COUNT(DISTINCT target_file.lang || char(31) || target_file.path || char(31) || + COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || + COALESCE(target.name, '')) AS target_family_count, + MIN(target_file.lang || char(31) || target_file.path || char(31) || + COALESCE(target.container_qualified_name, target.container_name, '') || char(31) || + COALESCE(target.name, '')) AS minimum_target_key + FROM symbol_reference_candidates AS candidate + JOIN symbols AS target ON target.id = candidate.symbol_id + JOIN files AS target_file ON target_file.id = target.file_id + GROUP BY candidate.reference_id + ) + UPDATE symbol_references AS r + SET target_symbol_id = CASE + WHEN resolution.candidate_count = 1 THEN resolution.minimum_symbol_id + END, + target_symbol_key = CASE + WHEN resolution.target_family_count = 1 THEN resolution.minimum_target_key + END, + resolution_candidate_count = resolution.candidate_count, + resolution_state = CASE + WHEN resolution.candidate_count = 1 THEN 'resolved' + WHEN resolution.target_family_count = 1 THEN 'resolved_group' + ELSE 'ambiguous' + END, + is_self_reference = CASE + WHEN r.source_symbol_id IS NOT NULL + AND resolution.candidate_count = 1 + AND r.source_symbol_id = resolution.minimum_symbol_id THEN 1 + ELSE 0 + END + FROM resolution_facts AS resolution + -- Fresh inserts already carry canonical candidate-free values. Aggregate and write + -- only rows that gained a candidate during this graph build. + -- fresh insertはcandidate-freeのcanonical値を保持するため、このgraph buildで + -- candidateを得たrowだけを集約・更新する。 + WHERE r.id = resolution.reference_id; + """; + + internal static string RefreshReferenceResolutionFreshSparseSqlForTesting + => RefreshReferenceResolutionFreshSparseSql; + private static readonly string RefreshMutualRecursionFlagsSql = $""" WITH desired_mutual_recursion(id, desired_value) AS MATERIALIZED ( SELECT r.id, @@ -1854,7 +1903,17 @@ private void InsertReferenceBatch( Dictionary foldedNameCache) { var rowsInBatch = end - start; - var sql = ReferenceInsertSqlCache.GetOrAdd(rowsInBatch, static count => BuildReferenceInsertSql(count)); + var useFreshReferenceResolutionDefaults = _referenceGraphRefreshScope is + { + IsDisposed: false, + FreshReferenceResolutionDefaultsPending: true, + }; + var cacheKey = ( + Rows: rowsInBatch, + FreshResolutionDefaults: useFreshReferenceResolutionDefaults); + var sql = ReferenceInsertSqlCache.GetOrAdd( + cacheKey, + static key => BuildReferenceInsertSql(key.Rows, key.FreshResolutionDefaults)); var cmd = RentCommand(sql, c => AddReferenceInsertParameters(c, rowsInBatch)); try { @@ -1884,8 +1943,10 @@ private void InsertReferenceBatch( reference.ContainerName, reference.IdentityContainerNameFolded, foldedNameCache); - cmd.Parameters[parameterIndex++].Value = reference.IsSelfReference ? 1 : 0; - cmd.Parameters[parameterIndex++].Value = reference.IsMutualRecursion ? 1 : 0; + cmd.Parameters[parameterIndex++].Value = + !useFreshReferenceResolutionDefaults && reference.IsSelfReference ? 1 : 0; + cmd.Parameters[parameterIndex++].Value = + !useFreshReferenceResolutionDefaults && reference.IsMutualRecursion ? 1 : 0; cmd.Parameters[parameterIndex++].Value = (object?)ExtractTargetQualifier(reference) ?? DBNull.Value; } @@ -1894,7 +1955,8 @@ private void InsertReferenceBatch( rowsInBatch, cmd.Parameters.Count, referenceLineIds.ReferenceCount, - referenceLineIds.ReferenceLineCount)); + referenceLineIds.ReferenceLineCount, + useFreshReferenceResolutionDefaults)); ReportBatchStatementForTesting("insert_references", rowsInBatch, rowsInBatch); cmd.ExecuteNonQuery(); } @@ -2309,33 +2371,42 @@ internal void RefreshMutualRecursionFlags( refreshPlan.DirtyReferenceCount, refreshPlan.TotalReferenceCount)); cancellationToken.ThrowIfCancellationRequested(); - // A fresh graph evaluates each correlated identity expression once. Once any - // persisted resolution exists, differential SQL avoids rewriting the stable - // majority while newly inserted or invalidated rows still repair normally. - // fresh graphでは相関identity式を1回だけ評価し、既存resolutionがあれば - // differential SQLで安定多数の再書込みを避けつつ新規/無効rowを修復する。 + var useFreshReferenceResolutionDefaults = graphScope?.FreshReferenceResolutionDefaultsPending == true; + if (useFreshReferenceResolutionDefaults && !refreshPlan.UseFullRefresh) + { + throw new InvalidOperationException( + "Fresh reference resolution defaults require a full graph refresh."); + } + // A true empty-database graph aggregates candidate-side resolution facts once and + // seeks only candidate-bearing references. Other persisted resolutions retain the + // differential path so stable rows are not rewritten. + // 真に空のdatabaseではcandidate側resolution factsを1回集約し、candidateを持つ + // referenceだけをseekする。その他の既存resolutionはstable rowを書き換えない + // differential pathを維持する。 string refreshIdentitySql; if (refreshPlan.UseFullRefresh) { - refreshIdentitySql = HasPersistedReferenceResolutionState(cancellationToken) - ? RefreshReferenceSourceSymbolsDifferentialSql + ";\n" + - RefreshCSharpReferenceFactsFullSql + "\n" + - RefreshCSharpSymbolFactsFullSql + "\n" + - RefreshCSharpTypeIdentityFactsSql + "\n" + - RefreshCSharpConstructorIdentityFactsSql + "\n" + - NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + - RefreshReferenceUniqueFamiliesSql + "\n" + - RefreshReferenceCandidatesSql + "\n" + - RefreshReferenceResolutionDifferentialSql + "\n" - : RefreshReferenceSourceSymbolsFullSql + ";\n" + - RefreshCSharpReferenceFactsFullSql + "\n" + - RefreshCSharpSymbolFactsFullSql + "\n" + - RefreshCSharpTypeIdentityFactsSql + "\n" + - RefreshCSharpConstructorIdentityFactsSql + "\n" + - NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + - RefreshReferenceUniqueFamiliesSql + "\n" + - RefreshReferenceCandidatesSql + "\n" + - RefreshReferenceResolutionFullSql + "\n"; + var hasPersistedReferenceResolutionState = !useFreshReferenceResolutionDefaults + && HasPersistedReferenceResolutionState(cancellationToken); + var refreshReferenceSourcesSql = useFreshReferenceResolutionDefaults + ? RefreshReferenceSourceSymbolsFullSql + : hasPersistedReferenceResolutionState + ? RefreshReferenceSourceSymbolsDifferentialSql + : RefreshReferenceSourceSymbolsFullSql; + var refreshReferenceResolutionSql = useFreshReferenceResolutionDefaults + ? RefreshReferenceResolutionFreshSparseSql + : hasPersistedReferenceResolutionState + ? RefreshReferenceResolutionDifferentialSql + : RefreshReferenceResolutionFullSql; + refreshIdentitySql = refreshReferenceSourcesSql + ";\n" + + RefreshCSharpReferenceFactsFullSql + "\n" + + RefreshCSharpSymbolFactsFullSql + "\n" + + RefreshCSharpTypeIdentityFactsSql + "\n" + + RefreshCSharpConstructorIdentityFactsSql + "\n" + + NormalizeCSharpPropertyReceiverReferencesFullSql + "\n" + + RefreshReferenceUniqueFamiliesSql + "\n" + + RefreshReferenceCandidatesSql + "\n" + + refreshReferenceResolutionSql + "\n"; } else { diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index b1fd384fd..6868c3e3d 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -155,7 +155,8 @@ internal static Action? ReferenceSecondaryIndexBulkLoa private const int TypeScriptModuleSyntaxFallbackMaxLines = 16384; private static readonly ConcurrentDictionary ChunkInsertSqlCache = new(); private static readonly ConcurrentDictionary SymbolInsertSqlCache = new(); - private static readonly ConcurrentDictionary ReferenceInsertSqlCache = new(); + private static readonly ConcurrentDictionary<(int Rows, bool FreshResolutionDefaults), string> + ReferenceInsertSqlCache = new(); private static readonly ConcurrentDictionary ReferenceLineUpsertSqlCache = new(); private static readonly ConcurrentDictionary ReferenceLineLookupSqlCache = new(); private static readonly ConcurrentDictionary ReferenceLineInsertSqlCache = new(); diff --git a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs new file mode 100644 index 000000000..91535b571 --- /dev/null +++ b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs @@ -0,0 +1,458 @@ +using System.Globalization; +using CodeIndex.Cli; +using CodeIndex.Database; +using CodeIndex.Models; +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Tests; + +[Collection("SQLite pool sensitive")] +public sealed class FreshReferenceResolutionTests : IDisposable +{ + private readonly string _projectRoot; + private readonly DbContext _db; + private readonly DbWriter _writer; + + public FreshReferenceResolutionTests() + { + _projectRoot = TestProjectHelper.CreateTempProject("cdidx_fresh_reference_resolution"); + _db = new DbContext( + DbOpenIntent.WriteIndex, + Path.Combine(_projectRoot, "codeindex.db")); + _db.InitializeSchema(); + _writer = new DbWriter(_db.Connection); + } + + [Theory] + [InlineData(false, false, false, false)] + [InlineData(false, false, true, false)] + [InlineData(false, true, false, false)] + [InlineData(false, true, true, false)] + [InlineData(true, false, false, true)] + [InlineData(true, false, true, false)] + [InlineData(true, true, false, false)] + [InlineData(true, true, true, false)] + public void ShouldUseFreshReferenceResolutionDefaults_RequiresEmptyOrdinaryFullScan( + bool startedWithNoIndexedFiles, + bool rebuild, + bool symbolsOnly, + bool expected) + { + Assert.Equal( + expected, + IndexCommandRunner.ShouldUseFreshReferenceResolutionDefaults( + startedWithNoIndexedFiles, + rebuild, + symbolsOnly)); + } + + [Fact] + public void BeginReferenceGraphRefreshScope_RejectsFreshDefaultsWithoutForcedFullRefresh() + { + var exception = Assert.Throws(() => + _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: false, + useFreshReferenceResolutionDefaults: true)); + + Assert.Equal("useFreshReferenceResolutionDefaults", exception.ParamName); + } + + [Fact] + public void InsertReferences_FreshDefaultsKeepParameterShapeAndUseSeparateCachedSql() + { + var fileId = InsertFile("src/provisional.py", "python"); + var observedWork = new List(); + var previousHook = DbWriter.ReferenceInsertBindingWorkForTesting; + try + { + DbWriter.ReferenceInsertBindingWorkForTesting = work => + { + observedWork.Add(work); + previousHook?.Invoke(work); + }; + + using (var freshScope = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true)) + { + _writer.InsertReferences( + [CreateReference(fileId, "Fresh", line: 1, extractorFlags: true)], + refreshMutualRecursionFlags: false); + } + + using (var ordinaryFullScope = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true)) + { + _writer.InsertReferences( + [CreateReference(fileId, "Standard", line: 2, extractorFlags: true)], + refreshMutualRecursionFlags: false); + } + } + finally + { + DbWriter.ReferenceInsertBindingWorkForTesting = previousHook; + } + + Assert.Collection( + observedWork, + fresh => + { + Assert.True(fresh.UsesFreshResolutionDefaults); + Assert.Equal(1, fresh.StatementRows); + Assert.Equal(14, fresh.BoundParameterCount); + }, + standard => + { + Assert.False(standard.UsesFreshResolutionDefaults); + Assert.Equal(1, standard.StatementRows); + Assert.Equal(14, standard.BoundParameterCount); + }); + + Assert.Equal( + new ProvisionalRow("unresolved", 0, 0, 0), + ReadProvisionalRow("Fresh")); + Assert.Equal( + new ProvisionalRow(null, 0, 1, 1), + ReadProvisionalRow("Standard")); + } + + [Fact] + public void FreshResolutionSql_MaterializesCandidateFactsWithoutOuterReferenceScan() + { + var sql = DbWriter.RefreshReferenceResolutionFreshSparseSqlForTesting; + + Assert.Contains("WITH resolution_facts AS MATERIALIZED", sql, StringComparison.Ordinal); + Assert.Contains("GROUP BY candidate.reference_id", sql, StringComparison.Ordinal); + Assert.Contains("FROM resolution_facts AS resolution", sql, StringComparison.Ordinal); + Assert.Contains("WHERE r.id = resolution.reference_id", sql, StringComparison.Ordinal); + Assert.Contains("is_self_reference = CASE", sql, StringComparison.Ordinal); + Assert.DoesNotContain("WHERE EXISTS", sql, StringComparison.Ordinal); + Assert.Equal(1, CountOccurrences(sql, "UPDATE symbol_references AS r")); + } + + [Fact] + public void FreshResolutionSql_MatchesFullOracleForAllStatesAcrossCSharpAndPython() + { + var csharpCallerFileId = InsertFile("src/caller.cs", "csharp"); + var csharpTargetFileId = InsertFile("src/target.cs", "csharp"); + var pythonCallerFileId = InsertFile("src/caller.py", "python"); + var pythonGroupFileId = InsertFile("src/group.py", "python"); + var pythonAmbiguousAFileId = InsertFile("src/ambiguous_a.py", "python"); + var pythonAmbiguousBFileId = InsertFile("src/ambiguous_b.py", "python"); + var pythonUniqueFileId = InsertFile("src/unique.py", "python"); + _writer.InsertSymbols([ + CreateSymbol(csharpCallerFileId, "CsCaller", line: 1), + CreateSymbol(csharpCallerFileId, "SelfTarget", line: 2), + CreateSymbol(csharpTargetFileId, "CsTarget", line: 1), + CreateSymbol(pythonCallerFileId, "PyCaller", line: 1), + CreateSymbol(pythonGroupFileId, "GroupTarget", line: 1, container: "group_module"), + CreateSymbol(pythonGroupFileId, "GroupTarget", line: 2, container: "group_module"), + CreateSymbol(pythonAmbiguousAFileId, "AmbiguousTarget", line: 1, container: "module_a"), + CreateSymbol(pythonAmbiguousBFileId, "AmbiguousTarget", line: 1, container: "module_b"), + CreateSymbol(pythonUniqueFileId, "PyUnique", line: 1, container: "unique_module"), + ]); + + using var freshScope = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + _writer.InsertReferences([ + CreateReference(csharpCallerFileId, "CsTarget", line: 10), + CreateReference(csharpCallerFileId, "MissingCs", line: 11), + CreateReference(csharpCallerFileId, "SelfTarget", line: 12, container: "SelfTarget"), + CreateReference(pythonCallerFileId, "GroupTarget", line: 20), + CreateReference(pythonCallerFileId, "AmbiguousTarget", line: 21), + CreateReference(pythonCallerFileId, "PyUnique", line: 22), + CreateReference(pythonCallerFileId, "MissingPy", line: 23), + ], refreshMutualRecursionFlags: false); + + Assert.Equal( + 7, + ScalarLong(""" + SELECT COUNT(*) + FROM symbol_references + WHERE resolution_state = 'unresolved' + AND resolution_candidate_count = 0 + AND target_symbol_id IS NULL + AND target_symbol_key IS NULL + AND is_self_reference = 0 + AND is_mutual_recursion = 0 + """)); + + Execute($""" + UPDATE symbol_references AS reference + SET source_symbol_id = ( + SELECT source.id + FROM symbols AS source + WHERE source.file_id = reference.file_id + AND source.name = CASE + WHEN reference.line = 12 THEN 'SelfTarget' + WHEN reference.file_id = {csharpCallerFileId} THEN 'CsCaller' + ELSE 'PyCaller' + END + ORDER BY source.id + LIMIT 1 + ); + + INSERT INTO symbol_reference_candidates(reference_id, symbol_id, scope_rank) + SELECT reference.id, target.id, 0 + FROM symbol_references AS reference + JOIN files AS source_file ON source_file.id = reference.file_id + JOIN symbols AS target ON target.name = reference.symbol_name + JOIN files AS target_file + ON target_file.id = target.file_id + AND target_file.lang = source_file.lang; + + CREATE TEMP TABLE observed_fresh_resolution_updates ( + reference_id INTEGER PRIMARY KEY + ) WITHOUT ROWID; + CREATE TEMP TRIGGER observe_fresh_resolution_updates + AFTER UPDATE OF target_symbol_id, target_symbol_key, + resolution_candidate_count, resolution_state, + is_self_reference + ON symbol_references + BEGIN + INSERT INTO observed_fresh_resolution_updates(reference_id) + VALUES (NEW.id); + END; + """); + + Execute(DbWriter.RefreshReferenceResolutionFreshSparseSqlForTesting); + + Assert.Equal( + ScalarLong("SELECT COUNT(DISTINCT reference_id) FROM symbol_reference_candidates"), + ScalarLong("SELECT COUNT(*) FROM observed_fresh_resolution_updates")); + Assert.Equal( + new ResolutionRow("resolved", 1, HasTargetId: true, HasTargetKey: true, IsSelf: false), + ReadResolutionRow("src/caller.cs", 10)); + Assert.Equal( + new ResolutionRow("unresolved", 0, HasTargetId: false, HasTargetKey: false, IsSelf: false), + ReadResolutionRow("src/caller.cs", 11)); + Assert.Equal( + new ResolutionRow("resolved", 1, HasTargetId: true, HasTargetKey: true, IsSelf: true), + ReadResolutionRow("src/caller.cs", 12)); + Assert.Equal( + new ResolutionRow("resolved_group", 2, HasTargetId: false, HasTargetKey: true, IsSelf: false), + ReadResolutionRow("src/caller.py", 20)); + Assert.Equal( + new ResolutionRow("ambiguous", 2, HasTargetId: false, HasTargetKey: false, IsSelf: false), + ReadResolutionRow("src/caller.py", 21)); + Assert.Equal( + new ResolutionRow("resolved", 1, HasTargetId: true, HasTargetKey: true, IsSelf: false), + ReadResolutionRow("src/caller.py", 22)); + Assert.Equal( + new ResolutionRow("unresolved", 0, HasTargetId: false, HasTargetKey: false, IsSelf: false), + ReadResolutionRow("src/caller.py", 23)); + + var freshSnapshot = ReadReferenceIdentitySnapshot(); + Execute("DROP TRIGGER observe_fresh_resolution_updates;"); + Execute(DbWriter.RefreshReferenceResolutionFullSqlForTesting); + Assert.Equal(freshSnapshot, ReadReferenceIdentitySnapshot()); + } + + [Fact] + public void FreshResolutionScope_RetainsDefaultsAfterFailureAndClearsThemAfterCompletion() + { + var fileId = InsertFile("src/retry.py", "python"); + _writer.InsertSymbols([ + CreateSymbol(fileId, "Caller", line: 1), + CreateSymbol(fileId, "Target", line: 2), + ]); + + using var freshScope = _writer.BeginReferenceGraphRefreshScope( + forceFullRefresh: true, + useFreshReferenceResolutionDefaults: true); + _writer.InsertReferences( + [CreateReference(fileId, "Target", line: 10, extractorFlags: true)], + refreshMutualRecursionFlags: false); + Execute(""" + CREATE TEMP TRIGGER fail_fresh_resolution + BEFORE UPDATE OF resolution_state ON symbol_references + WHEN OLD.symbol_name = 'Target' + BEGIN + SELECT RAISE(ABORT, 'fail fresh resolution'); + END; + """); + + Assert.Throws(() => + _writer.RefreshMutualRecursionFlags(stampReferenceIdentityContractReady: false)); + _writer.InsertReferences( + [CreateReference(fileId, "MissingAfterFailure", line: 11, extractorFlags: true)], + refreshMutualRecursionFlags: false); + Assert.Equal( + new ProvisionalRow("unresolved", 0, 0, 0), + ReadProvisionalRow("MissingAfterFailure")); + + Execute("DROP TRIGGER fail_fresh_resolution;"); + _writer.RefreshMutualRecursionFlags(stampReferenceIdentityContractReady: false); + Assert.Equal("resolved", ReadProvisionalRow("Target").ResolutionState); + + _writer.InsertReferences( + [CreateReference(fileId, "MissingAfterCompletion", line: 12, extractorFlags: true)], + refreshMutualRecursionFlags: false); + Assert.Equal( + new ProvisionalRow(null, 0, 1, 1), + ReadProvisionalRow("MissingAfterCompletion")); + } + + public void Dispose() + { + _db.Dispose(); + TestProjectHelper.DeleteDirectory(_projectRoot); + } + + private long InsertFile(string path, string language) + => _writer.UpsertFile(new FileRecord + { + Path = path, + Lang = language, + Size = 100, + Lines = 30, + Checksum = path, + Modified = new DateTime(2026, 8, 11, 0, 0, 0, DateTimeKind.Utc), + }); + + private static SymbolRecord CreateSymbol( + long fileId, + string name, + int line, + string? container = null) + => new() + { + FileId = fileId, + Kind = "function", + Name = name, + Line = line, + StartLine = line, + EndLine = line, + Signature = $"function {name}()", + ContainerKind = container == null ? null : "module", + ContainerName = container, + ContainerQualifiedName = container, + }; + + private static ReferenceRecord CreateReference( + long fileId, + string symbolName, + int line, + string container = "Caller", + bool extractorFlags = false) + => new() + { + FileId = fileId, + SymbolName = symbolName, + ReferenceKind = "call", + Line = line, + Column = 1, + Context = $"{symbolName}();", + ContainerKind = "function", + ContainerName = container, + IsSelfReference = extractorFlags, + IsMutualRecursion = extractorFlags, + }; + + private ProvisionalRow ReadProvisionalRow(string symbolName) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = """ + SELECT resolution_state, + resolution_candidate_count, + is_self_reference, + is_mutual_recursion + FROM symbol_references + WHERE symbol_name = @symbol_name + """; + command.Parameters.AddWithValue("@symbol_name", symbolName); + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + return new ProvisionalRow( + reader.IsDBNull(0) ? null : reader.GetString(0), + reader.GetInt32(1), + reader.GetInt32(2), + reader.GetInt32(3)); + } + + private ResolutionRow ReadResolutionRow(string path, int line) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = """ + SELECT reference.resolution_state, + reference.resolution_candidate_count, + reference.target_symbol_id IS NOT NULL, + reference.target_symbol_key IS NOT NULL, + reference.is_self_reference + FROM symbol_references AS reference + JOIN files AS file ON file.id = reference.file_id + WHERE file.path = @path + AND reference.line = @line + """; + command.Parameters.AddWithValue("@path", path); + command.Parameters.AddWithValue("@line", line); + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + return new ResolutionRow( + reader.GetString(0), + reader.GetInt32(1), + reader.GetBoolean(2), + reader.GetBoolean(3), + reader.GetBoolean(4)); + } + + private string ReadReferenceIdentitySnapshot() + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = """ + SELECT group_concat( + id || ':' || + COALESCE(source_symbol_id, -1) || ':' || + COALESCE(target_symbol_id, -1) || ':' || + COALESCE(target_symbol_key, '') || ':' || + resolution_candidate_count || ':' || + COALESCE(resolution_state, '') || ':' || + is_self_reference || ':' || + is_mutual_recursion, + '|') + FROM (SELECT * FROM symbol_references ORDER BY id) + """; + return Convert.ToString(command.ExecuteScalar(), CultureInfo.InvariantCulture) ?? string.Empty; + } + + private void Execute(string sql) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + private long ScalarLong(string sql) + { + using var command = _db.Connection.CreateCommand(); + command.CommandText = sql; + return Convert.ToInt64(command.ExecuteScalar(), CultureInfo.InvariantCulture); + } + + private static int CountOccurrences(string text, string value) + { + var count = 0; + var offset = 0; + while ((offset = text.IndexOf(value, offset, StringComparison.Ordinal)) >= 0) + { + count++; + offset += value.Length; + } + + return count; + } + + private sealed record ProvisionalRow( + string? ResolutionState, + int CandidateCount, + int IsSelf, + int IsMutual); + + private sealed record ResolutionRow( + string ResolutionState, + int CandidateCount, + bool HasTargetId, + bool HasTargetKey, + bool IsSelf); +} diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs index ba4b98911..5e1d173c0 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs @@ -66,6 +66,7 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza var scopeSnapshots = new ConcurrentQueue(); SqliteConnection? activeConnection = null; string[]? hotspotIndexNamesDuringRefresh = null; + ProvisionalReferenceRows? provisionalRowsAtIdentityStart = null; var missingConnectionObservations = 0; var refreshCount = 0; try @@ -81,6 +82,8 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza { Volatile.Write(ref activeConnection, connection); snapshots.Enqueue(CaptureReferenceIndexSnapshot(phase, connection)); + if (string.Equals(phase, "identity_started", StringComparison.Ordinal)) + provisionalRowsAtIdentityStart = CaptureProvisionalReferenceRows(connection); previousStateHook?.Invoke(connection, phase); }; DbWriter.BatchStatementExecutingForTesting = statement => @@ -177,6 +180,23 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza Assert.Equal(1, refreshCount); var scope = Assert.Single(scopeSnapshots); Assert.True(scope.UsedFullRefresh); + var provisionalRows = Assert.IsType(provisionalRowsAtIdentityStart); + Assert.True(provisionalRows.Total > 0); + Assert.Equal(provisionalRows.Total, provisionalRows.ZeroCandidateCount); + Assert.Equal(provisionalRows.Total, provisionalRows.NullTargetCount); + Assert.Equal(provisionalRows.Total, provisionalRows.NullTargetKeyCount); + Assert.Equal(provisionalRows.Total, provisionalRows.ZeroSelfCount); + Assert.Equal(provisionalRows.Total, provisionalRows.ZeroMutualCount); + if (rebuild) + { + Assert.Equal(provisionalRows.Total, provisionalRows.NullResolutionStateCount); + Assert.Equal(0, provisionalRows.UnresolvedResolutionStateCount); + } + else + { + Assert.Equal(0, provisionalRows.NullResolutionStateCount); + Assert.Equal(provisionalRows.Total, provisionalRows.UnresolvedResolutionStateCount); + } Assert.Equal(2, CountMutualRecursionReferences(dbPath)); } finally @@ -754,6 +774,34 @@ private static ReferenceIndexStageSnapshot CaptureReferenceIndexSnapshot( return new ReferenceIndexStageSnapshot(stage, names); } + private static ProvisionalReferenceRows CaptureProvisionalReferenceRows( + SqliteConnection connection) + { + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT COUNT(*), + SUM(CASE WHEN resolution_state IS NULL THEN 1 ELSE 0 END), + SUM(CASE WHEN resolution_state = 'unresolved' THEN 1 ELSE 0 END), + SUM(CASE WHEN resolution_candidate_count = 0 THEN 1 ELSE 0 END), + SUM(CASE WHEN target_symbol_id IS NULL THEN 1 ELSE 0 END), + SUM(CASE WHEN target_symbol_key IS NULL THEN 1 ELSE 0 END), + SUM(CASE WHEN is_self_reference = 0 THEN 1 ELSE 0 END), + SUM(CASE WHEN is_mutual_recursion = 0 THEN 1 ELSE 0 END) + FROM symbol_references + """; + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + return new ProvisionalReferenceRows( + reader.GetInt64(0), + reader.GetInt64(1), + reader.GetInt64(2), + reader.GetInt64(3), + reader.GetInt64(4), + reader.GetInt64(5), + reader.GetInt64(6), + reader.GetInt64(7)); + } + private static string? ReadCommittedTypeScriptAugmentationVersion(string dbPath) { using var connection = new SqliteConnection( @@ -877,4 +925,14 @@ private static string[] WriteHighCardinalityTypeScriptReferenceFixture(string pr } private sealed record ReferenceIndexStageSnapshot(string Stage, string[] Names); + + private sealed record ProvisionalReferenceRows( + long Total, + long NullResolutionStateCount, + long UnresolvedResolutionStateCount, + long ZeroCandidateCount, + long NullTargetCount, + long NullTargetKeyCount, + long ZeroSelfCount, + long ZeroMutualCount); } From 8d4f433d4a75d2c695b07e4cc31535ab583110f3 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 06:01:48 +0900 Subject: [PATCH 05/16] Refresh fresh graph planner statistics --- DEVELOPER_GUIDE.md | 4 + TESTING_GUIDE.md | 2 + .../+large-codebase-initial-indexing.fixed.md | 6 + .../Cli/IndexCommandRunner.FullScan.cs | 4 +- src/CodeIndex/Database/DbWriter.cs | 8 + .../ReferenceSecondaryIndexBulkLoadGuard.cs | 73 +++- .../Mcp/McpToolHandlers.Indexing.Execution.cs | 7 +- ...ommandRunnerReferenceIndexBulkLoadTests.cs | 93 ++++- .../McpServerToolsCallTests.cs | 16 + ...ferenceSecondaryIndexBulkLoadGuardTests.cs | 341 ++++++++++++++++++ 10 files changed, 542 insertions(+), 12 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 31d7743dc..98736d53f 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1273,6 +1273,8 @@ Operators can override the defaults with environment variables: After a successful `cdidx index` run, the writer refreshes SQLite planner statistics so large repositories do not rely on default selectivity estimates for `search`, `references`, `callers`, and related joins. A brand-new index database runs full `ANALYZE` once after the initial population; later successful index runs use SQLite's lighter `PRAGMA optimize`. This maintenance is best-effort and never changes the schema contract. +Truly empty-database bulk loads also perform a separate, targeted planner-statistics refresh immediately before reference-candidate population. An enabled reference-secondary-index bulk-load guard runs `ANALYZE main.files`, `ANALYZE main.symbols`, and `ANALYZE main.symbol_references` exactly once after dropping the candidate reverse index and before preparing the identity-resolution SQL; the TypeScript-deferred path has already restored its ordinary graph/query indexes at this point, while the direct graph path proceeds without that extra restoration phase. The CLI enables this only when it started with no indexed files and is neither rebuilding nor symbols-only; MCP captures the same pre-rebuild empty state explicitly. Existing-database runs, updates, rebuilds, symbols-only runs, and disabled guards retain the prior behavior. Cancellation aborts the indexing operation, while a non-cancellation SQLite failure rolls back the nested statistics savepoint and continues graph construction with the previous planner state. This pre-graph phase has a dedicated testing hook and is independent of final planner maintenance. + ### MCP request correlation Each JSON-RPC MCP request gets a server-generated `correlation_id` in addition to the client-controlled JSON-RPC `id`. Successful MCP responses include it under `result._meta.correlation_id`, and error responses include it in `error.data.correlation_id` or tool-error `result.structuredContent.correlation_id`. The serialized JSON-RPC id is echoed as `request_id` in the same metadata when one exists. `batch_query` assigns child correlation IDs to each slot by suffixing the parent value with `.1`, `.2`, and so on. @@ -4945,6 +4947,8 @@ operator は environment variable で既定値を上書きできる。 `cdidx index` が成功すると、writer は SQLite planner statistics を更新し、大規模 repository で `search`、`references`、`callers` などの join が default selectivity estimate に依存しないようにする。新規 index database は初回 population 後に full `ANALYZE` を一度実行し、それ以降の成功した index run では軽量な `PRAGMA optimize` を使う。この maintenance は best-effort であり、schema contract は変更しない。 +真に空の database からの bulk load では、reference candidate の構築直前にも独立した対象限定の planner-statistics refresh を実行する。有効な reference-secondary-index bulk-load guard は candidate reverse index の drop 後、identity-resolution SQL の prepare 前に `ANALYZE main.files`、`ANALYZE main.symbols`、`ANALYZE main.symbol_references` を正確に1回実行する。TypeScript に委譲する経路ではこの時点までに通常の graph / query index が復元済みであり、direct graph 経路では追加の復元 phase を挟まずに進む。CLI は indexed file が0件の状態から開始し、rebuild でも symbols-only でもない場合だけ有効化する。MCP も rebuild 前の空状態を明示的に保持して同じ条件を適用する。既存 database、update、rebuild、symbols-only、guard 無効時は従来どおりである。cancellation は indexing 全体へ伝播し、cancellation 以外の SQLite failure は nested statistics savepoint だけを rollback して、従来の planner state で graph 構築を続ける。この pre-graph phase は専用 testing hook を持ち、最終 planner maintenance とは独立している。 + ### MCP リクエスト相関 各 JSON-RPC MCP request には、client-controlled な JSON-RPC `id` に加えて、server-generated な `correlation_id` を付与する。成功 response は `result._meta.correlation_id`、error response は `error.data.correlation_id` または tool-error の `result.structuredContent.correlation_id` に含める。serialized JSON-RPC id がある場合は同じ metadata に `request_id` として echo する。`batch_query` は parent value に `.1`、`.2` のような suffix を付けて slot ごとの child correlation ID を割り当てる。 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 6a66b3766..7c6da021a 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -837,6 +837,7 @@ Use the inventory below before adding or moving a test class: - SQLite connection policy builder coverage should verify connection strings, command timeout, and status diagnostics in one test; these allocation-only checks do not need three runner cases. - SQLite command parameter builders should verify primitive types, stable dates, and copied parameter shapes on one command fixture rather than allocating three independent test cases. - Reference-secondary-index bulk-load coverage should assert the exact canonical schema at raw persistence, candidate deferral, identity start, graph-required restore, mutual-recursion start, and final query restore. Raw persistence keeps the reverse candidate-symbol lookup while ordinary query/graph indexes are absent; an actual graph refresh drops it immediately before candidate deletion/materialization. The ordinary middle graph set is exactly unresolved-folded, legacy NOCASE, and resolved reverse-edge, and its three plans plus candidate reference-primary-key lookups must remain usable. When TypeScript augmentation owns the graph pass, assert that all ordinary graph/query indexes return before readiness, the reverse candidate-symbol lookup remains available through marker/grouping work, and it is absent only from candidate population through mutual recursion. For transactional full scans, assert that it returns after readiness work but before the outer full-scan transaction commits; for recoverable scoped updates and MCP indexing, assert that guard ownership remains live through the readiness transaction commit and that the index returns immediately afterward. Also prove direct `changes()` preservation, no-graph completion without candidate-index DDL, graph-stage cancellation/disposal recovery, abandoned-stage schema repair, and transactional rollback without relying on initialization repair. A guard still removes legacy single-prefix/all-row-mutual indexes without restoring them recoverably. +- Fresh bulk-load planner-statistics coverage should seed only the minimum file, symbol, and reference rows, clear `sqlite_stat1`, and prove that exactly `files`, `symbols`, and `symbol_references` are analyzed once after candidate-index deferral but before identity SQL starts. At the start hook, neither those tables nor deferred query indexes may have statistics and the candidate reverse index must already be absent; after completion, TypeScript-deferred query indexes must have statistics while the candidate table/index remains unanalyzed. Cover both the TypeScript-deferred CLI lifecycle and a C#-only direct graph lifecycle, plus fresh MCP. Rebuilds, existing/update paths, symbols-only or disabled guards, and a false fresh flag must emit no statistics phase. Outer rollback must remove the new statistics; a non-cancellation SQLite failure must roll back sentinel/statistics writes inside the nested savepoint and continue graph resolution, while cancellation must propagate. Keep this phase on its dedicated hook and assert that the final `DbContext` maintenance hook is not invoked by the pre-graph refresh. - High-churn update reference-index coverage should materialize exactly the production minimum target count, cross the 60% boundary with pure decision cases, and assert candidate-index deferral through the TypeScript-owned graph pass plus full restoration afterwards. Re-run the same high-cardinality scoped target set unchanged and prove that candidate deferral and graph refresh never start, so the existing candidate index is not rebuilt. Change exactly one target before invocation and prove that the raw target-count gate alone cannot stage indexes or force a full graph: the estimated mutating-target count must cross the same production threshold, while the authoritative file loop still persists the sparse change with a scoped graph refresh. A single duplicate-hardlink cleanup must likewise remain scoped instead of treating identity detection as preflight uncertainty. Fresh/rebuild CLI and fresh MCP TypeScript fixtures must also observe all four hotspot aggregate indexes absent during their qualifying bulk refresh and restored at completion. Existing-database MCP FTS bulk load must prove that the guard centrally forces a full graph plan; low-churn scoped updates must remain scoped. Do not lower the production boundary through a global testing override. - Reference persistence binding coverage should exercise atomic and public transaction paths for both new-file inserts and replacement upserts. Assert 14 bound parameters per row, one ordinal slot per reference, one materialized ID per unique reference line, legacy `symbol_references.context` NULL storage, and intact normalized `reference_lines.context` text in one data-driven fixture. - Timeout-origin coverage should exercise timer cancellation and caller cancellation sequentially in one async test so the distinguishing assertion does not duplicate runner setup. @@ -1447,6 +1448,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - case-folded cleanup coverageではSQLite `NOCASE` とmanaged foldingを候補filterだけに使い、leaf case・ancestor case・Unicode foldが一致してもfile identityが異なるpathは両方のexact rowを保持し、真のcase-only aliasだけがchecksum readなしで旧spellingを削除できることを固定します。無関係なhardlinkをaliasとするにはpathとidentityの両方が同じretained case-fold bucketに一致しなければならず、`ScopedFileCleanupReappearance_FoldBucketsDoNotCrossMatchTargetIdentities` は別target由来のpath一致とidentity一致の合成を防ぎます。Git name-status coverageはold/new pathにtabと改行を含む実NUL区切りrenameを使い、commit/range helperの両方がunquotedな正確な文字列を返すことを検証します。 - project-marker budget の integration coverage は directory budget を最小境界に override し、child を 1 件だけ列挙します。warning 伝播の検証だけのために本番の 8,192-directory cap を実体化しないでください。shared discovery の coverage は1回の directory enumeration から C#、VB、F#、MSBuild の全 fingerprint が得られることを検証し、family-scope coverage は scan 後の ancestor filesystem probe が0回、fingerprint budget 枯渇後も complete な scope snapshot を利用可能、不完全 discovery では live fallback を維持することを固定してください。 - reference secondary-index の bulk-load coverage は raw persistence、candidate 遅延、identity 開始、graph-required 復元、mutual-recursion 開始、最終 query 復元の各境界で正確な canonical schema を検証してください。raw persistence 中は通常の query / graph index を外しても reverse candidate-symbol lookup を維持し、実際の graph refresh が candidate row を削除・構築する直前だけ外します。通常の中間 graph 集合は unresolved-folded、legacy NOCASE、resolved reverse-edge の正確に3本であり、candidate の reference-primary-key lookup を保ったまま3経路の plan が利用可能でなければなりません。TypeScript augmentation が graph pass を担当する場合は、readiness 前に通常の graph / query index を復元し、marker / grouping 中は reverse candidate-symbol lookup を維持し、candidate 構築から mutual recursion の完了までだけ不在にします。transactional full scan では readiness work 後かつ outer full-scan transaction の commit 前の復元を、recoverable scoped update / MCP indexing では readiness transaction の commit までの guard ownership 保持とその直後の復元を固定します。直接の `changes()` 保持、candidate index DDL を伴わない graph 不要時の完了、graph 段階での cancellation / dispose recovery、途中終了後の schema repair、initialization repair に依存しない transactional rollback も証明してください。guard は legacy single-prefix / 全row mutual index も削除し、recoverable mode では復元しません。 +- fresh bulk-load の planner-statistics coverage は必要最小限の file / symbol / reference row だけを seed し、`sqlite_stat1` を消去して、candidate index の遅延後かつ identity SQL の開始前に `files`、`symbols`、`symbol_references` の正確に3 tableだけを1回 ANALYZE することを証明してください。開始 hook では対象 table と deferred query index の統計がまだ存在せず、candidate reverse index は既に不在でなければなりません。完了後は TypeScript-deferred query index に統計があり、candidate table / index は未解析のままであることを固定します。TypeScript-deferred CLI lifecycle、C#-only direct graph lifecycle、fresh MCP のすべてを対象にしてください。rebuild、既存 / update 経路、symbols-only、guard 無効、fresh flag false では statistics phase を発生させません。outer rollback は新しい統計を消去し、cancellation 以外の SQLite failure は nested savepoint 内の sentinel / statistics write を rollback して graph resolution を継続し、cancellation は伝播しなければなりません。この phase は専用 hook に限定し、pre-graph refresh が最終 `DbContext` maintenance hook を呼ばないことも検証してください。 - high-churn update の reference-index coverage は production の最小 target 数だけを実体化し、pure decision case で60%境界を越え、TypeScript-owned graph pass の完了まで candidate index が遅延し、最後に全 index が復元されることを検証してください。同じ高 cardinality scoped target 集合を unchanged のまま再実行し、candidate 遅延と graph refresh が始まらず既存 candidate index を再構築しないことも証明します。実行前に target を正確に1件だけ変更し、raw target-count gate だけでは index staging や full graph 強制に入らず、変更を起こし得る target の見積もり件数が同じ production threshold を超える必要があること、authoritative file loop が疎な変更を scoped graph refresh で永続化することも固定します。また、duplicate-hardlink cleanup が1件だけなら identity detection を preflight の不確実性として扱わず、scoped のままであることも固定します。fresh / rebuild CLI と fresh MCP の TypeScript fixture では、条件を満たす bulk refresh 中に hotspot aggregate の4 indexも不在で、完了後に復元されることを観測します。既存DBに対する MCP FTS bulk load では guard が full graph plan を中央強制することを証明し、low-churn scoped update は scoped のまま維持してください。global testing override で production 境界を下げないでください。 - reference persistence binding の coverage は、新規 file insert と replacement upsert の両方について atomic 経路と public transaction 経路を検証してください。1 row あたり14 bound parameter、reference ごとに1つの ordinal slot、unique reference line ごとに1つの materialized ID、legacy `symbol_references.context` の NULL、正規化済み `reference_lines.context` text の保持を1つの data-driven fixture で固定してください。 - search guard と LSP の candidate/materialization cap coverage は、各 active limit を sentinel 1 件だけで超えます。利用可能な production constant は再利用し、境界を観測できた後に数百件の余分な row や symbol を残さないでください。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 30094c43b..62fd1261a 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -6,9 +6,13 @@ affected: - src/CodeIndex/Database/DbWriter.References.cs - src/CodeIndex/Database/DbWriter.ReferenceSql.cs - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs + - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs - tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs + - tests/CodeIndex.Tests/McpServerToolsCallTests.cs + - tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md --- @@ -19,6 +23,7 @@ affected: - **Fresh and rebuilt indexes skip unused incremental graph bookkeeping** — once a full reference-graph refresh is known, symbol and reference batches no longer populate dirty-scope tables that the full plan never reads, removing repeated set construction across all indexed languages. - **Initial C# workspace prepasses reuse the loaded extractor configuration** — CLI and MCP indexing no longer rediscover default plugins under a shared lock for every static/enum/const candidate after the workspace pattern snapshot has already been loaded. - **First-time full indexes resolve only references that have candidates** — empty-database CLI scans persist canonical unresolved defaults during bulk insertion, aggregate the candidate table once, and update candidate-bearing references by primary key instead of probing every reference row. Rebuilds, updates, retained graphs, and MCP indexing keep their existing recovery contracts. +- **Fresh graph planning uses post-load cardinalities** — truly empty CLI and MCP bulk loads now analyze the populated file, symbol, and reference tables immediately before candidate resolution, allowing SQLite to plan the expensive first graph build from current statistics. Rebuilds, updates, existing databases, and symbols-only runs keep their prior lifecycle; cancellation still aborts, while a statistics-only SQLite failure rolls back its savepoint and continues best-effort. ## 日本語 @@ -26,3 +31,4 @@ affected: - **新規作成および rebuild 時に未使用の差分 graph bookkeeping を省くようにしました** — reference graph の full refresh が確定した後は、その plan が参照しない dirty scope table を symbol / reference batch ごとに投入せず、全インデックス対象言語にまたがる反復的な set 構築を取り除きます。 - **初回 C# workspace prepass で読込済み extractor config を再利用するようにしました** — workspace pattern snapshot の読込後に、static / enum / const の候補ごとに共有lock下でdefault pluginを再探索しないよう、CLIとMCP indexingを既読込経路へ接続します。 - **初回full indexでcandidateを持つreferenceだけを解決するようにしました** — 空databaseからのCLI scanではbulk insert中にcanonicalな未解決値を保存し、candidate tableを1回集約して、全reference rowをprobeせずcandidateを持つrowだけをprimary keyで更新します。rebuild、update、retained graph、MCP indexingの既存recovery契約は変更しません。 +- **新規graph planningでbulk load後のcardinalityを使うようにしました** — 真に空のdatabaseから始まるCLI / MCP bulk loadでは、candidate解決の直前に投入済みのfile、symbol、reference tableを解析し、SQLiteが初回の高コストなgraph構築を最新統計から計画できるようにします。rebuild、update、既存database、symbols-onlyのlifecycleは従来どおりです。cancellationは引き続き中断し、統計更新だけのSQLite failureはsavepointを戻してbest-effortで継続します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 7bd2b5e75..e5e78f969 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -868,7 +868,9 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis ReferenceSecondaryIndexBulkLoadGuard.StartTransactional( writer, enabled: !options.SymbolsOnly && useFtsBulkLoad, - cancellationToken); + cancellationToken, + refreshPlannerStatisticsBeforeCandidatePopulation: + useFreshReferenceResolutionDefaults); using var ftsBulkLoad = FtsBulkLoadTriggerGuard.Start(writer, useFtsBulkLoad); if (staleFilePurgePlan.Count > 0) diff --git a/src/CodeIndex/Database/DbWriter.cs b/src/CodeIndex/Database/DbWriter.cs index 6868c3e3d..0500cf46c 100644 --- a/src/CodeIndex/Database/DbWriter.cs +++ b/src/CodeIndex/Database/DbWriter.cs @@ -49,6 +49,8 @@ private static readonly AsyncLocal?> private static readonly AsyncLocal?> ScopedBatchStatementExecutingForTesting = new(); private static readonly AsyncLocal?> ScopedReferenceSecondaryIndexBulkLoadStateForTesting = new(); + private static readonly AsyncLocal?> + ScopedFreshBulkLoadPlannerStatisticsStateForTesting = new(); internal static Action? LanguagePresenceCheckForTesting { get => ScopedLanguagePresenceCheckForTesting.Value; @@ -139,6 +141,12 @@ internal static Action? ReferenceSecondaryIndexBulkLoa set => ScopedReferenceSecondaryIndexBulkLoadStateForTesting.Value = value; } + internal static Action? FreshBulkLoadPlannerStatisticsStateForTesting + { + get => ScopedFreshBulkLoadPlannerStatisticsStateForTesting.Value; + set => ScopedFreshBulkLoadPlannerStatisticsStateForTesting.Value = value; + } + // Transaction ownership (#4154): the semaphore is held for the outermost writer // transaction lifetime. Same-stack nested calls from the owning thread and // AsyncLocal token skip the semaphore and become SAVEPOINTs; other flows wait even diff --git a/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs b/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs index f05ec2061..71c68e547 100644 --- a/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs +++ b/src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs @@ -1,3 +1,5 @@ +using Microsoft.Data.Sqlite; + namespace CodeIndex.Database; /// @@ -8,14 +10,19 @@ namespace CodeIndex.Database; internal sealed class ReferenceSecondaryIndexBulkLoadGuard : IDisposable { private readonly bool _restoreOnDispose; + private readonly bool _refreshPlannerStatisticsBeforeCandidatePopulation; private DbWriter? _writer; + private bool _plannerStatisticsRefreshAttempted; private ReferenceSecondaryIndexBulkLoadGuard( DbWriter writer, bool restoreOnDispose, + bool refreshPlannerStatisticsBeforeCandidatePopulation, CancellationToken cancellationToken) { _restoreOnDispose = restoreOnDispose; + _refreshPlannerStatisticsBeforeCandidatePopulation = + refreshPlannerStatisticsBeforeCandidatePopulation; try { // Scoped graph planning names deferred indexes explicitly. Force the active @@ -75,7 +82,8 @@ private ReferenceSecondaryIndexBulkLoadGuard( internal static ReferenceSecondaryIndexBulkLoadGuard? StartTransactional( DbWriter writer, bool enabled, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + bool refreshPlannerStatisticsBeforeCandidatePopulation = false) { if (!enabled) return null; @@ -84,17 +92,20 @@ private ReferenceSecondaryIndexBulkLoadGuard( return new ReferenceSecondaryIndexBulkLoadGuard( writer, restoreOnDispose: false, + refreshPlannerStatisticsBeforeCandidatePopulation, cancellationToken); } internal static ReferenceSecondaryIndexBulkLoadGuard? StartRecoverable( DbWriter writer, bool enabled, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + bool refreshPlannerStatisticsBeforeCandidatePopulation = false) => enabled ? new ReferenceSecondaryIndexBulkLoadGuard( writer, restoreOnDispose: true, + refreshPlannerStatisticsBeforeCandidatePopulation, cancellationToken) : null; @@ -118,7 +129,19 @@ internal void ReportIdentityRefreshStarted() /// graph更新がcandidate rowを実際に再構築する直前だけ逆引きindexをdropする。 /// internal void PrepareForCandidatePopulation(CancellationToken cancellationToken = default) - => _writer?.DropCandidatePopulationReferenceSecondaryIndexes(cancellationToken); + { + var writer = _writer; + if (writer == null) + return; + + writer.DropCandidatePopulationReferenceSecondaryIndexes(cancellationToken); + if (!_refreshPlannerStatisticsBeforeCandidatePopulation + || _plannerStatisticsRefreshAttempted) + return; + + _plannerStatisticsRefreshAttempted = true; + writer.RefreshFreshBulkLoadPlannerStatistics(cancellationToken); + } internal void PrepareForMutualRecursion(CancellationToken cancellationToken = default) => _writer?.RestoreGraphFinalizationRequiredReferenceSecondaryIndexes(cancellationToken); @@ -160,6 +183,12 @@ public void Dispose() public partial class DbWriter { + private const string RefreshFreshBulkLoadPlannerStatisticsSql = """ + ANALYZE main.files; + ANALYZE main.symbols; + ANALYZE main.symbol_references; + """; + internal void DropDeferredReferenceSecondaryIndexes(CancellationToken cancellationToken) { // Old binaries may have recreated retired indexes after a database was pruned by a @@ -241,6 +270,44 @@ private void RestoreReferenceSecondaryIndexes( internal void ReportReferenceSecondaryIndexBulkLoadState(string phase) => ReferenceSecondaryIndexBulkLoadStateForTesting?.Invoke(_conn, phase); + internal void RefreshFreshBulkLoadPlannerStatistics( + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + using var transaction = BeginTransaction( + cancellationToken, + "refresh fresh bulk-load planner statistics"); + try + { + FreshBulkLoadPlannerStatisticsStateForTesting?.Invoke( + _conn, + "post_load_statistics_started"); + Execute(RefreshFreshBulkLoadPlannerStatisticsSql, cancellationToken); + FreshBulkLoadPlannerStatisticsStateForTesting?.Invoke( + _conn, + "post_load_statistics_completed"); + transaction.Commit(); + } + catch + { + transaction.Rollback(); + throw; + } + } + catch (SqliteException) + { + // Fresh statistics improve only this graph plan. If SQLite rejects ANALYZE, + // keep the enclosing bulk load alive with the prior planner state. + // fresh statisticsは今回のgraph planだけを改善するため、SQLiteがANALYZEを + // 拒否した場合はsavepointを戻し、従来のplanner stateでbulk loadを続行する。 + FreshBulkLoadPlannerStatisticsStateForTesting?.Invoke( + _conn, + "post_load_statistics_failed"); + } + } + internal void RequireCallerOwnedTransactionForReferenceSecondaryIndexBulkLoad() => RequireCallerOwnedTransaction(nameof(ReferenceSecondaryIndexBulkLoadGuard)); } diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index 66b864093..de6f9cd70 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -163,7 +163,8 @@ bool IsPathAuthorized(string path) || !typeScriptAugmentationVersionMatchesCurrent; var typeScriptAugmentationReadyCleared = !typeScriptAugmentationVersionMatchesCurrent; var ftsMutated = false; - var startedWithNoIndexedFiles = rebuild || !writer.HasAnyIndexedFiles(); + var startedWithNoIndexedFilesBeforeRebuild = !writer.HasAnyIndexedFiles(); + var startedWithNoIndexedFiles = rebuild || startedWithNoIndexedFilesBeforeRebuild; if (rebuild || startedWithNoIndexedFiles) indexSnapshot.CSharpStaticInterfaceSourceEvidence = null; var requiresConservativeCSharpSourceRefresh = !rebuild @@ -1288,7 +1289,9 @@ await EmitProgressNotificationAsync( ReferenceSecondaryIndexBulkLoadGuard.StartRecoverable( writer, enabled: useFtsBulkLoad, - requestToken); + requestToken, + refreshPlannerStatisticsBeforeCandidatePopulation: + startedWithNoIndexedFilesBeforeRebuild && !rebuild); if (staleFilePurgePlan.Count > 0) { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs index 5e1d173c0..6965e80ac 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs @@ -58,12 +58,15 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza var projectRoot = TestProjectHelper.CreateTempProject("cdidx_reference_index_bulk_load"); var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); var previousStateHook = DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting; + var previousStatisticsHook = DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting; var previousStatementHook = DbWriter.BatchStatementExecutingForTesting; var previousGraphHook = DbWriter.MutualRecursionRefreshForTesting; var previousScopeHook = DbWriter.ReferenceGraphRefreshScopeForTesting; var previousHotspotHook = DbWriter.HotspotAggregateRefreshStatementExecutingForTesting; var snapshots = new ConcurrentQueue(); var scopeSnapshots = new ConcurrentQueue(); + var lifecycle = new ConcurrentQueue(); + var statisticsPhases = new ConcurrentQueue(); SqliteConnection? activeConnection = null; string[]? hotspotIndexNamesDuringRefresh = null; ProvisionalReferenceRows? provisionalRowsAtIdentityStart = null; @@ -82,10 +85,17 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza { Volatile.Write(ref activeConnection, connection); snapshots.Enqueue(CaptureReferenceIndexSnapshot(phase, connection)); + lifecycle.Enqueue(phase); if (string.Equals(phase, "identity_started", StringComparison.Ordinal)) provisionalRowsAtIdentityStart = CaptureProvisionalReferenceRows(connection); previousStateHook?.Invoke(connection, phase); }; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = (connection, phase) => + { + statisticsPhases.Enqueue(phase); + lifecycle.Enqueue(phase); + previousStatisticsHook?.Invoke(connection, phase); + }; DbWriter.BatchStatementExecutingForTesting = statement => { previousStatementHook?.Invoke(statement); @@ -146,6 +156,16 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza captured .Where(snapshot => snapshot.Stage != "insert_references") .Select(snapshot => snapshot.Stage)); + Assert.Equal( + rebuild + ? [] + : ["post_load_statistics_started", "post_load_statistics_completed"], + statisticsPhases); + Assert.Equal( + rebuild + ? ["dropped", "deferred_graph_prepared", "candidate_deferred", "identity_started", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"] + : ["dropped", "deferred_graph_prepared", "candidate_deferred", "post_load_statistics_started", "post_load_statistics_completed", "identity_started", "graph_required_restored", "mutual_started", "readiness_completed", "restored", "full_scan_committed"], + lifecycle); var requiredNames = GetRequiredReferenceIndexNames(); var initialBulkNames = GetInitialBulkPersistenceReferenceIndexNames(); @@ -202,6 +222,7 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza finally { DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting = previousStateHook; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = previousStatisticsHook; DbWriter.BatchStatementExecutingForTesting = previousStatementHook; DbWriter.MutualRecursionRefreshForTesting = previousGraphHook; DbWriter.ReferenceGraphRefreshScopeForTesting = previousScopeHook; @@ -210,6 +231,61 @@ public void Run_FreshAndRebuildFullScan_DefersReferenceIndexesUntilGraphFinaliza } } + [Fact] + public void Run_FreshFullScan_DirectGraphRefreshesPostLoadPlannerStatistics() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_reference_index_direct_graph_statistics"); + var previousStateHook = DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting; + var previousStatisticsHook = DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting; + var previousGraphHook = DbWriter.MutualRecursionRefreshForTesting; + var previousAugmentationGroupingHook = DbWriter.TypeScriptAugmentationGroupingForTesting; + var lifecycle = new ConcurrentQueue(); + var graphRefreshCount = 0; + var augmentationGroupingCount = 0; + try + { + WriteDirectReferenceIndexCycleFixture(projectRoot); + DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting = (connection, phase) => + { + lifecycle.Enqueue(phase); + previousStateHook?.Invoke(connection, phase); + }; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = (connection, phase) => + { + lifecycle.Enqueue(phase); + previousStatisticsHook?.Invoke(connection, phase); + }; + DbWriter.MutualRecursionRefreshForTesting = () => + { + Interlocked.Increment(ref graphRefreshCount); + previousGraphHook?.Invoke(); + }; + DbWriter.TypeScriptAugmentationGroupingForTesting = stats => + { + Interlocked.Increment(ref augmentationGroupingCount); + previousAugmentationGroupingHook?.Invoke(stats); + }; + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(1, graphRefreshCount); + Assert.Equal(0, augmentationGroupingCount); + Assert.Equal( + ["dropped", "candidate_deferred", "post_load_statistics_started", "post_load_statistics_completed", "identity_started", "graph_required_restored", "mutual_started", "restored"], + lifecycle); + } + finally + { + DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting = previousStateHook; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = previousStatisticsHook; + DbWriter.MutualRecursionRefreshForTesting = previousGraphHook; + DbWriter.TypeScriptAugmentationGroupingForTesting = previousAugmentationGroupingHook; + DeleteDirectory(projectRoot); + } + } + [Theory] [InlineData("dropped")] [InlineData("candidate_deferred")] @@ -889,12 +965,7 @@ ORDER BY name private static void WriteReferenceIndexCycleFixture(string projectRoot) { - File.WriteAllText( - Path.Combine(projectRoot, "cycle_a.cs"), - "public static class BulkCycleA { public static void CallA() { CallB(); } }\n"); - File.WriteAllText( - Path.Combine(projectRoot, "cycle_b.cs"), - "public static class BulkCycleB { public static void CallB() { CallA(); } }\n"); + WriteDirectReferenceIndexCycleFixture(projectRoot); for (var index = 0; index < IndexCommandRunner.UpdateReferenceSecondaryIndexBulkLoadMinimumTargetCount; index++) @@ -906,6 +977,16 @@ private static void WriteReferenceIndexCycleFixture(string projectRoot) } } + private static void WriteDirectReferenceIndexCycleFixture(string projectRoot) + { + File.WriteAllText( + Path.Combine(projectRoot, "cycle_a.cs"), + "public static class BulkCycleA { public static void CallA() { CallB(); } }\n"); + File.WriteAllText( + Path.Combine(projectRoot, "cycle_b.cs"), + "public static class BulkCycleB { public static void CallB() { CallA(); } }\n"); + } + private static string[] WriteHighCardinalityTypeScriptReferenceFixture(string projectRoot) { var relativePaths = Enumerable diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 0a563a65e..76fa39197 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -7743,9 +7743,12 @@ public void ToolsCall_Index_FreshTypeScriptBulkDefersReferenceAndHotspotSecondar var previousRefreshHook = DbWriter.MutualRecursionRefreshForTesting; var previousReferenceIndexHook = DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting; + var previousStatisticsHook = + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting; var previousHotspotRefreshHook = DbWriter.HotspotAggregateRefreshStatementExecutingForTesting; var referenceIndexStates = new List<(string Phase, string[] PresentIndexNames)>(); + var statisticsPhases = new List(); var hotspotIndexStates = new List(); var lifecycle = new List(); SqliteConnection? writerConnection = null; @@ -7818,6 +7821,12 @@ static string[] ReadPresentIndexes( ReadPresentIndexes(connection, canonicalReferenceIndexNames))); previousReferenceIndexHook?.Invoke(connection, phase); }; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = (connection, phase) => + { + lifecycle.Add(phase); + statisticsPhases.Add(phase); + previousStatisticsHook?.Invoke(connection, phase); + }; McpServer.McpIndexTypeScriptAugmentationRebuildForTesting = () => { augmentationRebuildCount++; @@ -7869,6 +7878,8 @@ static string[] ReadPresentIndexes( "typescript_augmentation", "graph_refresh", "candidate_deferred", + "post_load_statistics_started", + "post_load_statistics_completed", "identity_started", "graph_required_restored", "mutual_started", @@ -7877,6 +7888,9 @@ static string[] ReadPresentIndexes( "hotspot_refresh", }, lifecycle); + Assert.Equal( + ["post_load_statistics_started", "post_load_statistics_completed"], + statisticsPhases); Assert.Equal( new[] @@ -7933,6 +7947,8 @@ static string[] ReadPresentIndexes( DbWriter.MutualRecursionRefreshForTesting = previousRefreshHook; DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting = previousReferenceIndexHook; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = + previousStatisticsHook; DbWriter.HotspotAggregateRefreshStatementExecutingForTesting = previousHotspotRefreshHook; TestProjectHelper.DeleteDirectory(fixtureDir); diff --git a/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs b/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs index 6810d2fc5..5f5db2c2b 100644 --- a/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs +++ b/tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs @@ -295,6 +295,262 @@ public void RecoverableComplete_WithoutGraphRefreshRestoresAllIndexes() } } + [Fact] + public void FreshPlannerStatistics_RunOnceAfterCandidateDropAndAnalyzeOnlyGraphTables() + { + SeedPlannerStatisticsFixture(); + ResetPlannerStatistics(); + var previousBulkStateHook = DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting; + var previousStatisticsHook = DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting; + var previousFinalMaintenanceHook = DbContext.PlannerStatisticsCommandCreatedForTesting; + var lifecycle = new List(); + string[]? tablesBeforeAnalyze = null; + string[]? tablesAfterAnalyze = null; + string[]? indexesBeforeAnalyze = null; + string[]? indexesAfterAnalyze = null; + var finalMaintenanceHookCalls = 0; + try + { + DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting = (connection, phase) => + { + lifecycle.Add(phase); + previousBulkStateHook?.Invoke(connection, phase); + }; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = (connection, phase) => + { + lifecycle.Add(phase); + if (phase == "post_load_statistics_started") + { + tablesBeforeAnalyze = ReadPlannerStatisticTables(connection); + indexesBeforeAnalyze = ReadPlannerStatisticIndexes(connection); + Assert.DoesNotContain( + "idx_symbol_ref_candidates_symbol", + ReadReferenceIndexNames(connection)); + } + else if (phase == "post_load_statistics_completed") + { + tablesAfterAnalyze = ReadPlannerStatisticTables(connection); + indexesAfterAnalyze = ReadPlannerStatisticIndexes(connection); + } + previousStatisticsHook?.Invoke(connection, phase); + }; + DbContext.PlannerStatisticsCommandCreatedForTesting = command => + { + finalMaintenanceHookCalls++; + previousFinalMaintenanceHook?.Invoke(command); + }; + + using var transaction = _writer.BeginTransaction(); + using var guard = ReferenceSecondaryIndexBulkLoadGuard.StartTransactional( + _writer, + enabled: true, + refreshPlannerStatisticsBeforeCandidatePopulation: true); + Assert.NotNull(guard); + + guard.PrepareForDeferredGraphRefresh(); + guard.PrepareForCandidatePopulation(); + guard.PrepareForCandidatePopulation(); + guard.ReportIdentityRefreshStarted(); + guard.Complete(); + transaction.Commit(); + + Assert.Equal( + [ + "dropped", + "deferred_graph_prepared", + "candidate_deferred", + "post_load_statistics_started", + "post_load_statistics_completed", + "candidate_deferred", + "identity_started", + "restored", + ], + lifecycle); + Assert.Empty(Assert.IsType(tablesBeforeAnalyze)); + Assert.Empty(Assert.IsType(indexesBeforeAnalyze)); + Assert.Equal( + ["files", "symbol_references", "symbols"], + Assert.IsType(tablesAfterAnalyze)); + var analyzedIndexes = Assert.IsType(indexesAfterAnalyze); + Assert.All( + ReferenceSecondaryIndexSql.DeferredGraphPreparation, + definition => Assert.Contains(definition.Name, analyzedIndexes)); + Assert.DoesNotContain("idx_symbol_ref_candidates_symbol", analyzedIndexes); + Assert.DoesNotContain("symbol_reference_candidates", tablesAfterAnalyze); + Assert.Equal(0, finalMaintenanceHookCalls); + } + finally + { + DbWriter.ReferenceSecondaryIndexBulkLoadStateForTesting = previousBulkStateHook; + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = previousStatisticsHook; + DbContext.PlannerStatisticsCommandCreatedForTesting = previousFinalMaintenanceHook; + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + public void FreshPlannerStatistics_DisabledGuardOrFlagLeavesStatisticsUntouched( + bool guardEnabled, + bool statisticsEnabled) + { + SeedPlannerStatisticsFixture(); + ResetPlannerStatistics(); + var previousStatisticsHook = DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting; + var phases = new List(); + try + { + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = (connection, phase) => + { + phases.Add(phase); + previousStatisticsHook?.Invoke(connection, phase); + }; + + using var transaction = _writer.BeginTransaction(); + using var guard = ReferenceSecondaryIndexBulkLoadGuard.StartTransactional( + _writer, + guardEnabled, + refreshPlannerStatisticsBeforeCandidatePopulation: statisticsEnabled); + if (guard != null) + { + guard.PrepareForDeferredGraphRefresh(); + guard.PrepareForCandidatePopulation(); + guard.Complete(); + } + transaction.Commit(); + + Assert.Empty(phases); + Assert.Empty(ReadPlannerStatisticTables(_db.Connection)); + } + finally + { + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = previousStatisticsHook; + } + } + + [Fact] + public void FreshPlannerStatistics_OuterRollbackRemovesAnalyzeResults() + { + SeedPlannerStatisticsFixture(); + ResetPlannerStatistics(); + + using (var transaction = _writer.BeginTransaction()) + { + using var guard = ReferenceSecondaryIndexBulkLoadGuard.StartTransactional( + _writer, + enabled: true, + refreshPlannerStatisticsBeforeCandidatePopulation: true); + Assert.NotNull(guard); + + guard.PrepareForDeferredGraphRefresh(); + guard.PrepareForCandidatePopulation(); + + Assert.Equal( + ["files", "symbol_references", "symbols"], + ReadPlannerStatisticTables(_db.Connection)); + } + + Assert.Empty(ReadPlannerStatisticTables(_db.Connection)); + AssertDeferredIndexesPresent(_db.Connection); + } + + [Fact] + public void FreshPlannerStatistics_NonCancellationSqliteFailureRollsBackAndGraphContinues() + { + SeedPlannerStatisticsFixture(); + ResetPlannerStatistics(); + var previousStatisticsHook = DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting; + var phases = new List(); + try + { + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = (connection, phase) => + { + phases.Add(phase); + if (phase == "post_load_statistics_started") + { + ExecuteNonQuery(connection, """ + INSERT INTO codeindex_meta(key, value) + VALUES ('fresh_statistics_savepoint_probe', 'pending'); + ANALYZE main.files; + """); + throw new SqliteException("forced fresh statistics failure", 1); + } + previousStatisticsHook?.Invoke(connection, phase); + }; + + using var guard = ReferenceSecondaryIndexBulkLoadGuard.StartRecoverable( + _writer, + enabled: true, + refreshPlannerStatisticsBeforeCandidatePopulation: true); + Assert.NotNull(guard); + guard.PrepareForDeferredGraphRefresh(); + + _writer.RefreshMutualRecursionFlags( + stampReferenceIdentityContractReady: false, + referenceSecondaryIndexBulkLoad: guard); + + Assert.Equal( + ["post_load_statistics_started", "post_load_statistics_failed"], + phases); + Assert.Equal( + 0, + ReadScalarLong( + _db.Connection, + "SELECT COUNT(*) FROM codeindex_meta WHERE key = 'fresh_statistics_savepoint_probe'")); + Assert.Empty(ReadPlannerStatisticTables(_db.Connection)); + Assert.Equal( + 1, + ReadScalarLong( + _db.Connection, + "SELECT COUNT(*) FROM symbol_references WHERE resolution_state IS NOT NULL")); + guard.Complete(); + } + finally + { + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = previousStatisticsHook; + } + } + + [Fact] + public void FreshPlannerStatistics_CancellationPropagatesAndRollsBackSavepoint() + { + SeedPlannerStatisticsFixture(); + ResetPlannerStatistics(); + var previousStatisticsHook = DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting; + using var cancellation = new CancellationTokenSource(); + var phases = new List(); + try + { + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = (connection, phase) => + { + phases.Add(phase); + if (phase == "post_load_statistics_started") + cancellation.Cancel(); + previousStatisticsHook?.Invoke(connection, phase); + }; + + using var transaction = _writer.BeginTransaction(); + using var guard = ReferenceSecondaryIndexBulkLoadGuard.StartTransactional( + _writer, + enabled: true, + refreshPlannerStatisticsBeforeCandidatePopulation: true); + Assert.NotNull(guard); + guard.PrepareForDeferredGraphRefresh(); + + var exception = Assert.Throws( + () => guard.PrepareForCandidatePopulation(cancellation.Token)); + + Assert.Equal(cancellation.Token, exception.CancellationToken); + Assert.Equal(["post_load_statistics_started"], phases); + Assert.Empty(ReadPlannerStatisticTables(_db.Connection)); + } + finally + { + DbWriter.FreshBulkLoadPlannerStatisticsStateForTesting = previousStatisticsHook; + } + } + [Fact] public void StagedRefresh_PreservesMutualChangesCountAcrossRemainingIndexRestore() { @@ -439,6 +695,91 @@ private static long ReadChanges(SqliteConnection connection) return (long)command.ExecuteScalar()!; } + private void SeedPlannerStatisticsFixture() + { + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/fresh-statistics.cs", + Lang = "csharp", + Size = 100, + Lines = 3, + Modified = new DateTime(2026, 8, 11, 0, 0, 0, DateTimeKind.Utc), + Checksum = "fresh-statistics", + }); + _writer.InsertSymbols( + [ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "Run", + Line = 1, + StartLine = 1, + EndLine = 3, + }, + ]); + _writer.InsertReferences( + [ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "Run", + ReferenceKind = "call", + Line = 2, + Column = 1, + Context = "Run();", + ContainerName = "Run", + }, + ], + refreshMutualRecursionFlags: false); + } + + private void ResetPlannerStatistics() + { + ExecuteNonQuery(_db.Connection, """ + ANALYZE main.files; + ANALYZE main.symbols; + ANALYZE main.symbol_references; + DELETE FROM sqlite_stat1; + """); + } + + private static string[] ReadPlannerStatisticTables(SqliteConnection connection) + { + using var command = connection.CreateCommand(); + command.CommandText = "SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY tbl"; + using var reader = command.ExecuteReader(); + var tables = new List(); + while (reader.Read()) + tables.Add(reader.GetString(0)); + return tables.ToArray(); + } + + private static string[] ReadPlannerStatisticIndexes(SqliteConnection connection) + { + using var command = connection.CreateCommand(); + command.CommandText = "SELECT idx FROM sqlite_stat1 WHERE idx IS NOT NULL ORDER BY idx"; + using var reader = command.ExecuteReader(); + var indexes = new List(); + while (reader.Read()) + indexes.Add(reader.GetString(0)); + return indexes.ToArray(); + } + + private static long ReadScalarLong(SqliteConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + return Convert.ToInt64(command.ExecuteScalar()); + } + + private static void ExecuteNonQuery(SqliteConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + private static void AssertRetiredIndexesAbsent(SqliteConnection connection) { var names = ReadReferenceIndexNames(connection); From 918b9e97b8fcc6f5c636b14272fef1d05b923fc8 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 06:29:56 +0900 Subject: [PATCH 06/16] Cache symbol worker pattern discovery --- DEVELOPER_GUIDE.md | 29 +- TESTING_GUIDE.md | 2 + .../+large-codebase-initial-indexing.fixed.md | 6 + .../Extensibility/ExtractorPluginRegistry.cs | 43 +- .../Indexer/Symbols/SymbolExtractionWorker.cs | 177 +++++- .../ExtractorPluginRegistryTests.cs | 34 ++ .../IndexCommandRunnerTests.cs | 560 ++++++++++++++++++ 7 files changed, 812 insertions(+), 39 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 98736d53f..5a798b2b6 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -3636,18 +3636,28 @@ Downstream users can add lightweight language support without rebuilding | Sidecar size | 64 KiB per file | | Rules per sidecar | 128 | | Configured rules | 128 per immutable workspace snapshot | +| Worker project-root snapshots | 32 per persistent symbol worker | +| Worker pattern-directory snapshots | 4,096 per root and 8,192 per worker; overflow uses live discovery | | Regex match timeout | 100 ms | | Timed-out rule cooldown | At most one minute in the owning workspace snapshot | - Sidecars must be regular files inside non-symlink pattern directories. - Each sidecar is parsed, compiled, and checked against `SymbolKindCatalog` before its path, rules, or budget are committed. -- Rejected content is fingerprinted to suppress duplicate diagnostics. Content - or metadata changes and recovery from a transient read failure trigger a retry - without restarting the process. +- Rejected content is fingerprinted to suppress duplicate diagnostics. On paths + that perform another discovery or explicit refresh, content or metadata changes + and recovery from a transient read failure trigger a retry without restarting + the process. - Workspace discovery requires an explicit trust root and never probes above it. Nested sidecars inside that boundary are loaded for the current file by the bounded extraction worker. +- A persistent symbol-worker command treats pattern discovery as a project-root + snapshot. Root reload reads user and workspace-root configs once; each nested + pattern directory's first result, including missing, unsafe, and known discovery + failures, remains fixed for the run. Sidecars added or repaired afterward become + visible on the next worker command, while unexpected exceptions remain retryable. + A saturated directory cache falls back to uncached discovery and never skips a + config merely to preserve the memory bound. - Path identity follows the active filesystem's case-sensitivity, so case-distinct sidecars remain distinct on case-sensitive volumes. - `status --json` reports accepted files in `extractors.pattern_configs[]`, @@ -6754,16 +6764,25 @@ cleared range を証明するテストが必要です。Bounded accumulation pat | sidecar size | 1 file あたり 64 KiB | | sidecar 内の rule | 128 件 | | configured rule | immutable workspace snapshot ごとに 128 件 | +| worker の project-root snapshot | persistent symbol worker ごとに 32 件 | +| worker の pattern-directory snapshot | root ごとに 4,096 件、worker ごとに 8,192 件。超過分は live discovery | | regex match timeout | 100 ms | | timeout rule の cooldown | 所有する workspace snapshot 内で最大 1 分 | - sidecar は symlink ではない pattern directory 配下の通常 file に限定します。 - 各 sidecar は path・rule・budget を commit する前に parse / compile し、 `SymbolKindCatalog` に対して kind を検証します。 -- 拒否された内容は fingerprint で重複診断を抑制します。内容や metadata の変更、 - 一時的な read failure からの回復後は、process を再起動せず再試行します。 +- 拒否された内容は fingerprint で重複診断を抑制します。再度 discovery または明示的 + refresh を行う経路では、内容や metadata の変更、一時的な read failure からの回復後に + process を再起動せず再試行します。 - workspace 探索には明示的な trust root が必要で、それより上は探索しません。 境界内の nested sidecar は対象 file の上限付き extraction worker で読み込みます。 +- persistent symbol-worker command は pattern discovery を project-root snapshot として扱います。 + root reload は user / workspace-root config を1回だけ読み込み、missing、unsafe、既知の + discovery failure を含む各 nested pattern directory の初回結果を run 中は固定します。 + その後に追加または修復された sidecar は次の worker command で可視になり、想定外例外は + 引き続き再試行します。directory cache が飽和した場合は uncached discovery に fallback し、 + memory 上限を守るために config を skip することはありません。 - path identity は実際の filesystem の case-sensitivity に従うため、case-sensitive volume では大小文字だけが異なる sidecar も別々に扱います。 - `status --json` の `extractors.pattern_configs[]` は、受理済み file の diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 7c6da021a..db83400ca 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -812,6 +812,7 @@ Use the inventory below before adding or moving a test class: - JSON API-version fixtures use locked console capture and scoped project cleanup; deleting the project before an unconditional pool reset makes that reset both redundant and too late, so keep this contract suite parallelizable. - Golden JSON snapshot fixtures use the non-parallel console-sensitive collection because empty stderr is part of their contract, while retaining per-instance database cleanup; they should not pay a process-wide pool reset after each of the status, search, references, impact, and excerpt snapshots. - Extractor plugin-registry fixtures share that console-sensitive collection because rejected pattern configs emit diagnostics through the process-wide stderr writer. +- Persistent symbol-worker pattern-discovery fixtures should feed multiple JSON frames through one in-process `TryRunCommand` when they need registry hooks, and restore both those hooks and `PatternConfigDiscoveryCacheFactoryForTesting` in `finally`. Pin that project-root reload is the only user/root discovery for a run; the first nested-directory observation, including missing, rejected, or known-failure results, remains frozen until the next run, while unexpected exceptions retry. Capacity tests must prove saturation continues with uncached discovery instead of skipping configs. - Executable extension boundary fixtures should cover unsafe Unix directory modes, symlink ancestors, and a source rename-swap after staging; keep permission assertions Unix-only and restore staging test hooks in `finally`. - Plugin-worker fixtures should prove metadata rejection before process start, bounded timeout/crash/memory/output behavior, proxy extraction, failed-fingerprint retry after an explicit refresh, and no restaging during repeated hot-path lookups; use `CDIDX_TEST_` variables so the isolated-worker environment policy carries only explicit fixtures. - Hook-discovery worker fixtures should force a module initializer to record its worker PID and spawn a persistent descendant, prove both PIDs terminate after a manifest, cover discovery timeout/memory/output caps, and copy one hook assembly under two names to prove stable IDs and per-ID disablement without `Type.FullName` collisions. @@ -1822,6 +1823,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - JSON API-version fixture は locked console capture と scoped project cleanup を使う。project 削除後の無条件 pool reset は冗長なうえ遅すぎるため、この contract suite は parallel 実行可能な状態を保つ。 - golden JSON snapshot fixture は空の stderr 自体が契約なので non-parallel な console-sensitive collection を使い、instance ごとの database cleanup を維持する。status、search、references、impact、excerpt の各 snapshot 後に process-wide pool reset を支払わないこと。 - extractor plugin-registry fixture も、reject された pattern config が process-wide stderr writer 経由で診断を出すため、同じ console-sensitive collection を共有する。 +- persistent symbol worker の pattern discovery fixture で registry hook を使う場合は、1回の process 内 `TryRunCommand` に複数 JSON frame を渡し、hook と `PatternConfigDiscoveryCacheFactoryForTesting` の両方を `finally` で復元する。run 内の user/root discovery は project-root reload の1回だけであること、missing・reject・既知 failure を含む nested directory の初回観測は次の run まで固定される一方、想定外例外は再試行されることを固定する。capacity test では飽和時も config を skip せず uncached discovery を続けることを検証する。 - executable extension boundary fixture は unsafe な Unix directory mode、symlink ancestor、staging 後の source rename-swap を検証する。permission assertion は Unix のみにし、staging の test hook は `finally` で必ず復元する。 - plugin-worker fixture はprocess開始前のmetadata拒否、timeout / crash / memory / output上限、proxy extraction、明示的refreshによるfile修復後のfailed-fingerprint再試行、hot-path lookup反復時に再stagingしないことを検証する。isolated-worker environment policyが明示的なfixtureだけを渡すよう、`CDIDX_TEST_` variableを使う。 - hook-discovery worker fixture は module initializer に worker PID を記録させて persistent な descendant を起動し、manifest 後に両方の PID が終了することと discovery の timeout / memory / output cap を検証する。同じ hook assembly を 2 つの名前で copy し、stable ID と `Type.FullName` が衝突しない ID 単位 disablement も固定する。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 62fd1261a..e2c94824b 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -8,8 +8,12 @@ affected: - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs - src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs + - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs + - tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs - tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs - tests/CodeIndex.Tests/McpServerToolsCallTests.cs - tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs @@ -24,6 +28,7 @@ affected: - **Initial C# workspace prepasses reuse the loaded extractor configuration** — CLI and MCP indexing no longer rediscover default plugins under a shared lock for every static/enum/const candidate after the workspace pattern snapshot has already been loaded. - **First-time full indexes resolve only references that have candidates** — empty-database CLI scans persist canonical unresolved defaults during bulk insertion, aggregate the candidate table once, and update candidate-bearing references by primary key instead of probing every reference row. Rebuilds, updates, retained graphs, and MCP indexing keep their existing recovery contracts. - **Fresh graph planning uses post-load cardinalities** — truly empty CLI and MCP bulk loads now analyze the populated file, symbol, and reference tables immediately before candidate resolution, allowing SQLite to plan the expensive first graph build from current statistics. Rebuilds, updates, existing databases, and symbols-only runs keep their prior lifecycle; cancellation still aborts, while a statistics-only SQLite failure rolls back its savepoint and continues best-effort. +- **Persistent symbol workers reuse bounded pattern-directory snapshots across languages** — each project-root reload discovers user and root configs once, while nested ancestor directories, including missing and rejected results, are observed once per worker command. The cache follows live filesystem casing and falls back to uncached discovery when full so configs are never skipped; direct registry callers keep dynamic discovery. ## 日本語 @@ -32,3 +37,4 @@ affected: - **初回 C# workspace prepass で読込済み extractor config を再利用するようにしました** — workspace pattern snapshot の読込後に、static / enum / const の候補ごとに共有lock下でdefault pluginを再探索しないよう、CLIとMCP indexingを既読込経路へ接続します。 - **初回full indexでcandidateを持つreferenceだけを解決するようにしました** — 空databaseからのCLI scanではbulk insert中にcanonicalな未解決値を保存し、candidate tableを1回集約して、全reference rowをprobeせずcandidateを持つrowだけをprimary keyで更新します。rebuild、update、retained graph、MCP indexingの既存recovery契約は変更しません。 - **新規graph planningでbulk load後のcardinalityを使うようにしました** — 真に空のdatabaseから始まるCLI / MCP bulk loadでは、candidate解決の直前に投入済みのfile、symbol、reference tableを解析し、SQLiteが初回の高コストなgraph構築を最新統計から計画できるようにします。rebuild、update、既存database、symbols-onlyのlifecycleは従来どおりです。cancellationは引き続き中断し、統計更新だけのSQLite failureはsavepointを戻してbest-effortで継続します。 +- **persistent symbol worker が全言語で上限付き pattern-directory snapshot を再利用するようにしました** — project-root reload ごとに user / root config を1回だけ探索し、missing や reject を含む nested ancestor directory は worker command ごとに初回結果を再利用します。cache は実 filesystem の case policy に従い、飽和時は uncached discovery に fallback して config を skip しません。registry の direct caller は従来どおり動的に探索します。 diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index 71ff55c29..831258af7 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -585,7 +585,10 @@ internal static void LoadPatternConfigsForPath( bool includeWorkspaceRoot = true, Func? openFile = null, Func? directoryExists = null, - Action?, long?>? observeInput = null) + Action?, long?>? observeInput = null, + bool includeUserDirectory = true, + Func? shouldInspectWorkspacePatternDirectory = null, + Action? workspacePatternDirectoryInspected = null) { EnsurePluginsLoaded(); if (string.IsNullOrWhiteSpace(path) || string.IsNullOrWhiteSpace(workspaceRoot)) @@ -600,25 +603,39 @@ internal static void LoadPatternConfigsForPath( return; var state = GetOrCreatePatternWorkspace(fullRoot); - foreach (var patternPath in EnumerateUserPatternConfigPaths( - state, - directoryExists, - observeInput)) - TryLoadPatternConfig(state, patternPath, "user", openFile, observeInput); + if (includeUserDirectory) + { + foreach (var patternPath in EnumerateUserPatternConfigPaths( + state, + directoryExists, + observeInput)) + { + TryLoadPatternConfig(state, patternPath, "user", openFile, observeInput); + } + } while (PathCasing.IsFullPathEqualOrParent(fullRoot, directory)) { if (!includeWorkspaceRoot && PathCasing.PathsEqual(directory, fullRoot)) break; - foreach (var patternPath in EnumeratePatternConfigPaths( - state, - directory, - includeUserDirectory: false, - directoryExists, - observeInput)) + var patternDirectory = Path.Combine(directory, ".cdidx", "patterns"); + if (shouldInspectWorkspacePatternDirectory?.Invoke(patternDirectory) != false) { - TryLoadPatternConfig(state, patternPath, "workspace", openFile, observeInput); + foreach (var patternPath in EnumeratePatternConfigPaths( + state, + directory, + includeUserDirectory: false, + directoryExists, + observeInput)) + { + TryLoadPatternConfig(state, patternPath, "workspace", openFile, observeInput); + } + + // Complete only after the existing discovery/diagnostic path returns. + // Expected missing, unsafe, and enumeration-failure outcomes therefore + // become snapshot entries, while unexpected exceptions remain retryable. + workspacePatternDirectoryInspected?.Invoke(patternDirectory); } if (PathCasing.PathsEqual(directory, fullRoot)) diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs index 63371e585..2610eccb8 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs @@ -387,6 +387,138 @@ private static void ForwardCapturedStderr(string? capturedStderr) } } +internal sealed class WorkerPatternConfigRootSnapshot(string projectRoot) +{ + private readonly HashSet inspectedOrdinal = new(StringComparer.Ordinal); + private readonly HashSet inspectedIgnoreCase = new(StringComparer.OrdinalIgnoreCase); + + internal string ProjectRoot { get; } = projectRoot; + internal long LastAccessSequence { get; set; } + internal int RetainedDirectoryCount => inspectedOrdinal.Count + inspectedIgnoreCase.Count; + + internal bool Contains(string normalizedPatternDirectory, bool ignoreCase) + => (ignoreCase ? inspectedIgnoreCase : inspectedOrdinal).Contains(normalizedPatternDirectory); + + internal bool Add(string normalizedPatternDirectory, bool ignoreCase) + => (ignoreCase ? inspectedIgnoreCase : inspectedOrdinal).Add(normalizedPatternDirectory); +} + +internal sealed class WorkerPatternConfigDiscoveryCache +{ + internal const int DefaultMaxRootSnapshots = ExtractorPluginRegistry.MaxRetainedWorkspaceSnapshots; + internal const int DefaultMaxDirectoriesPerRoot = 4096; + internal const int DefaultMaxDirectoriesPerWorker = 8192; + + private readonly int maxRootSnapshots; + private readonly int maxDirectoriesPerRoot; + private readonly int maxDirectoriesPerWorker; + private readonly List roots = []; + private long accessSequence; + + internal WorkerPatternConfigDiscoveryCache( + int maxRootSnapshots = DefaultMaxRootSnapshots, + int maxDirectoriesPerRoot = DefaultMaxDirectoriesPerRoot, + int maxDirectoriesPerWorker = DefaultMaxDirectoriesPerWorker) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxRootSnapshots, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(maxDirectoriesPerRoot, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(maxDirectoriesPerWorker, 1); + this.maxRootSnapshots = maxRootSnapshots; + this.maxDirectoriesPerRoot = maxDirectoriesPerRoot; + this.maxDirectoriesPerWorker = maxDirectoriesPerWorker; + } + + internal int RootCount => roots.Count; + internal int RetainedDirectoryCount { get; private set; } + + internal bool TryGetRoot(string projectRoot, out WorkerPatternConfigRootSnapshot snapshot) + { + var normalizedRoot = PathCasing.NormalizeBoundaryPath(projectRoot); + foreach (var candidate in roots) + { + if (!PathCasing.PathsEqual(candidate.ProjectRoot, normalizedRoot)) + continue; + + Touch(candidate); + snapshot = candidate; + return true; + } + + snapshot = null!; + return false; + } + + internal WorkerPatternConfigRootSnapshot AddReloadedRoot(string projectRoot) + { + var normalizedRoot = PathCasing.NormalizeBoundaryPath(projectRoot); + if (TryGetRoot(normalizedRoot, out var existing)) + return existing; + + if (roots.Count >= maxRootSnapshots) + { + var evicted = roots.MinBy(candidate => candidate.LastAccessSequence)!; + roots.Remove(evicted); + RetainedDirectoryCount -= evicted.RetainedDirectoryCount; + } + + var snapshot = new WorkerPatternConfigRootSnapshot(normalizedRoot); + Touch(snapshot); + roots.Add(snapshot); + return snapshot; + } + + internal bool ShouldInspectPatternDirectory( + WorkerPatternConfigRootSnapshot root, + string patternDirectory) + { + var normalizedDirectory = PathCasing.NormalizeBoundaryPath(patternDirectory); + var ignoreCase = PathCasing.IsIgnoreCase(GetPatternDirectoryCaseReference(normalizedDirectory)); + return !root.Contains(normalizedDirectory, ignoreCase); + } + + internal void RecordInspectedPatternDirectory( + WorkerPatternConfigRootSnapshot root, + string patternDirectory) + { + var normalizedDirectory = PathCasing.NormalizeBoundaryPath(patternDirectory); + var ignoreCase = PathCasing.IsIgnoreCase(GetPatternDirectoryCaseReference(normalizedDirectory)); + if (root.Contains(normalizedDirectory, ignoreCase)) + return; + + // Retain existing entries when saturated. New directories continue through + // uncached discovery so a bounded cache never causes a config to be skipped. + if (root.RetainedDirectoryCount >= maxDirectoriesPerRoot + || RetainedDirectoryCount >= maxDirectoriesPerWorker) + { + return; + } + + if (root.Add(normalizedDirectory, ignoreCase)) + RetainedDirectoryCount++; + } + + internal void Reset() + { + roots.Clear(); + RetainedDirectoryCount = 0; + accessSequence = 0; + } + + private void Touch(WorkerPatternConfigRootSnapshot root) + => root.LastAccessSequence = ++accessSequence; + + private static string GetPatternDirectoryCaseReference(string normalizedPatternDirectory) + { + // The registry has not performed its reparse-point checks when the cache + // predicate runs. Probe the owning source directory, not an untrusted + // .cdidx/patterns link target, while retaining the full lexical cache key. + var cdidxDirectory = Path.GetDirectoryName(normalizedPatternDirectory); + return string.IsNullOrEmpty(cdidxDirectory) + ? normalizedPatternDirectory + : Path.GetDirectoryName(cdidxDirectory) ?? normalizedPatternDirectory; + } +} + internal static class SymbolExtractionWorker { internal const string CommandName = "__cdidx-symbol-extraction"; @@ -395,8 +527,8 @@ internal static class SymbolExtractionWorker private const string TestDelayMillisecondsOption = "--test-delay-ms"; private const string TestConsoleStdoutOption = "--test-console-stdout"; private const int CapturedConsoleMaxChars = 32 * 1024; - private static readonly object PatternConfigProjectRootsGate = new(); - private static readonly List LoadedPatternConfigProjectRoots = []; + private static readonly object PatternConfigDiscoveryGate = new(); + private static WorkerPatternConfigDiscoveryCache patternConfigDiscoveryCache = new(); internal static string FormatExecutionFailure(Exception ex) => SafeDiagnosticFormatter.FormatExceptionCategoryWithOrigin("worker_execution_failed", ex); @@ -404,6 +536,7 @@ internal static string FormatExecutionFailure(Exception ex) WorkerProtocolJsonValidator.CreateSerializerOptions(SymbolExtractionWorkerJsonContext.Default.Options); internal static int? DelayMillisecondsForTesting { get; set; } internal static string? ConsoleStdoutForTesting { get; set; } + internal static Func? PatternConfigDiscoveryCacheFactoryForTesting { get; set; } internal static bool TryRunCommand( string[] args, @@ -554,7 +687,7 @@ private static int RunCommand( return 2; } - ResetPatternConfigProjectRootsForWorker(); + ResetPatternConfigDiscoveryForWorker(); maxProtocolLineCharacters = workerOptions.MaxProtocolLineCharacters; maxProtocolLineUtf8Bytes = workerOptions.MaxProtocolLineUtf8Bytes; @@ -724,23 +857,13 @@ private static bool EnsurePatternConfigsLoadedForWorker(string? projectRoot, str if (string.IsNullOrWhiteSpace(projectRoot)) return false; - var fullRoot = Path.GetFullPath(projectRoot); - lock (PatternConfigProjectRootsGate) + var fullRoot = PathCasing.NormalizeBoundaryPath(projectRoot); + lock (PatternConfigDiscoveryGate) { - var rootAlreadyLoaded = false; - foreach (var loadedRoot in LoadedPatternConfigProjectRoots) - { - if (PathCasing.PathsEqual(loadedRoot, fullRoot)) - { - rootAlreadyLoaded = true; - break; - } - } - - if (!rootAlreadyLoaded) + if (!patternConfigDiscoveryCache.TryGetRoot(fullRoot, out var rootSnapshot)) { ExtractorPluginRegistry.ReloadPatternConfigsForProjectRoot(fullRoot); - LoadedPatternConfigProjectRoots.Add(fullRoot); + rootSnapshot = patternConfigDiscoveryCache.AddReloadedRoot(fullRoot); } if (!string.IsNullOrWhiteSpace(filePath)) @@ -748,17 +871,29 @@ private static bool EnsurePatternConfigsLoadedForWorker(string? projectRoot, str var fullPath = Path.IsPathRooted(filePath) ? Path.GetFullPath(filePath) : Path.GetFullPath(Path.Combine(fullRoot, filePath)); - ExtractorPluginRegistry.LoadPatternConfigsForPath(fullPath, fullRoot, includeWorkspaceRoot: false); + ExtractorPluginRegistry.LoadPatternConfigsForPath( + fullPath, + fullRoot, + includeWorkspaceRoot: false, + includeUserDirectory: false, + shouldInspectWorkspacePatternDirectory: patternDirectory => + patternConfigDiscoveryCache.ShouldInspectPatternDirectory(rootSnapshot, patternDirectory), + workspacePatternDirectoryInspected: patternDirectory => + patternConfigDiscoveryCache.RecordInspectedPatternDirectory(rootSnapshot, patternDirectory)); } return true; } } - private static void ResetPatternConfigProjectRootsForWorker() + private static void ResetPatternConfigDiscoveryForWorker() { - lock (PatternConfigProjectRootsGate) - LoadedPatternConfigProjectRoots.Clear(); + lock (PatternConfigDiscoveryGate) + { + patternConfigDiscoveryCache = PatternConfigDiscoveryCacheFactoryForTesting?.Invoke() + ?? new WorkerPatternConfigDiscoveryCache(); + patternConfigDiscoveryCache.Reset(); + } } private static void AddTestingArguments(ProcessStartInfo startInfo) diff --git a/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs index 16c5d0da4..e3e15b2f0 100644 --- a/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs +++ b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs @@ -1142,6 +1142,40 @@ public void LoadPatternConfigsForPath_StopsAtWorkspaceRootAndReportsProvenance_I } } + [Fact] + public void LoadPatternConfigsForPath_DefaultCallerDiscoversSidecarAddedAfterEarlierLookup() + { + var projectRoot = TestProjectHelper.CreateTempProject("extractor_registry_dynamic_pattern_sidecar"); + lock (TestConsoleLock.Gate) + { + try + { + var sourceDirectory = Path.Combine(projectRoot, "src", "nested"); + var filePath = Path.Combine(sourceDirectory, "sample.dynamicpattern"); + Directory.CreateDirectory(sourceDirectory); + ExtractorPluginRegistry.ResetForTests(); + ExtractorPluginRegistry.UserPatternDirectoryOverrideForTests = + Path.Combine(projectRoot, "missing-user-patterns"); + + ExtractorPluginRegistry.LoadPatternConfigsForPath(filePath, projectRoot); + Assert.False(ExtractorPluginRegistry.TryGetSymbolExtractor("dynamicpatterndsl", projectRoot, out _)); + + WritePatternConfig( + sourceDirectory, + "dynamic.yaml", + "language: \"dynamicpatterndsl\"\nextensions:\n - extension: \".dynamicpattern\"\npatterns:\n - kind: \"class\"\n regex: \"^dynamic (?\\\\w+)\"\n"); + ExtractorPluginRegistry.LoadPatternConfigsForPath(filePath, projectRoot); + + Assert.True(ExtractorPluginRegistry.TryGetSymbolExtractor("dynamicpatterndsl", projectRoot, out _)); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + } + [Fact] public void LoadPatternConfigsForProjectRoot_LoadsCaseDistinctFilesWhenFilesystemIsCaseSensitive_Issue4597() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index a6735c1e1..d5d9db587 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -707,6 +707,523 @@ public void SymbolExtractionWorker_ReusesPatternConfigDiscoveryPerProjectRoot() } } + [Fact] + public void SymbolExtractionWorker_CachesUserRootAndAncestorPatternDiscoveryForRun() + { + var projectRoot = CreateTempProject(); + var userRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + WorkerPatternConfigDiscoveryCache? cache = null; + var inspected = new Dictionary(StringComparer.Ordinal); + try + { + var userPatternDirectory = Path.Combine(userRoot, ".cdidx", "patterns"); + var firstDirectory = Path.Combine(projectRoot, "src", "shared", "first"); + var secondDirectory = Path.Combine(projectRoot, "src", "shared", "second"); + Directory.CreateDirectory(firstDirectory); + Directory.CreateDirectory(secondDirectory); + WriteSymbolWorkerPatternConfig( + userRoot, + "user.yaml", + "language: \"workeruserdsl\"\nextensions:\n - extension: \".workeruser\"\npatterns:\n - kind: \"class\"\n regex: \"^user (?\\\\w+)\"\n"); + WriteSymbolWorkerPatternConfig( + projectRoot, + "root.yaml", + "language: \"workerrootdsl\"\nextensions:\n - extension: \".workerroot\"\npatterns:\n - kind: \"class\"\n regex: \"^root (?\\\\w+)\"\n"); + ExtractorPluginRegistry.ResetForTests(); + ExtractorPluginRegistry.UserPatternDirectoryOverrideForTests = userPatternDirectory; + ExtractorPluginRegistry.InspectPatternDirectoryForTesting = path => + { + var normalized = PathCasing.NormalizeBoundaryPath(path); + inspected[normalized] = inspected.GetValueOrDefault(normalized) + 1; + }; + SymbolExtractionWorker.PatternConfigDiscoveryCacheFactoryForTesting = () => + cache = new WorkerPatternConfigDiscoveryCache(); + + var responses = RunSymbolWorkerRequestsInProcess( + CreateSymbolWorkerRequest(projectRoot, Path.Combine(firstDirectory, "one.cs")), + CreateSymbolWorkerRequest(projectRoot, Path.Combine(secondDirectory, "two.cs")), + CreateSymbolWorkerRequest(projectRoot, Path.Combine(firstDirectory, "three.cs"))); + + Assert.All(responses, response => Assert.Null(response.WorkerError)); + Assert.NotNull(cache); + Assert.Equal(1, cache.RootCount); + Assert.Equal(4, cache.RetainedDirectoryCount); + Assert.Equal(1, inspected[PathCasing.NormalizeBoundaryPath(userPatternDirectory)]); + Assert.Equal( + 1, + inspected[PathCasing.NormalizeBoundaryPath(Path.Combine(projectRoot, ".cdidx"))]); + Assert.Equal( + 1, + inspected[PathCasing.NormalizeBoundaryPath(Path.Combine(projectRoot, ".cdidx", "patterns"))]); + } + finally + { + SymbolExtractionWorker.PatternConfigDiscoveryCacheFactoryForTesting = null; + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + DeleteDirectory(userRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_MissingNestedSidecarRemainsAbsentUntilNewWorkerSnapshot() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + try + { + var sourceDirectory = Path.Combine(projectRoot, "src", "nested"); + var filePath = Path.Combine(sourceDirectory, "sample.laternested"); + Directory.CreateDirectory(sourceDirectory); + ExtractorPluginRegistry.ResetForTests(); + + using (var worker = new SymbolExtractionWorkerClient()) + { + var beforeConfig = worker.Invoke( + 0, + "laternesteddsl", + "entity Before", + filePath, + projectRoot, + contentIsNormalized: true, + hasOversizeLine: false, + conflictMarkerLine: null, + TimeSpan.FromSeconds(5)); + Assert.True(beforeConfig.Success, beforeConfig.WorkerError); + Assert.Empty(beforeConfig.Symbols!); + + WriteSymbolWorkerPatternConfig( + sourceDirectory, + "later.yaml", + "language: \"laternesteddsl\"\nextensions:\n - extension: \".laternested\"\npatterns:\n - kind: \"class\"\n regex: \"^entity (?\\\\w+)\"\n"); + + var sameSnapshot = worker.Invoke( + 0, + "laternesteddsl", + "entity SameSnapshot", + filePath, + projectRoot, + contentIsNormalized: true, + hasOversizeLine: false, + conflictMarkerLine: null, + TimeSpan.FromSeconds(5)); + Assert.True(sameSnapshot.Success, sameSnapshot.WorkerError); + Assert.Empty(sameSnapshot.Symbols!); + } + + using var nextWorker = new SymbolExtractionWorkerClient(); + var nextSnapshot = nextWorker.Invoke( + 0, + "laternesteddsl", + "entity NextSnapshot", + filePath, + projectRoot, + contentIsNormalized: true, + hasOversizeLine: false, + conflictMarkerLine: null, + TimeSpan.FromSeconds(5)); + Assert.True(nextSnapshot.Success, nextSnapshot.WorkerError); + Assert.Equal("NextSnapshot", Assert.Single(nextSnapshot.Symbols!).Name); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_NewRunCommandResetsPatternDiscoverySnapshot() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + try + { + var sourceDirectory = Path.Combine(projectRoot, "src", "run-reset"); + var filePath = Path.Combine(sourceDirectory, "sample.runreset"); + Directory.CreateDirectory(sourceDirectory); + ExtractorPluginRegistry.ResetForTests(); + ExtractorPluginRegistry.UserPatternDirectoryOverrideForTests = + Path.Combine(projectRoot, "missing-user-patterns"); + + var beforeConfig = Assert.Single(RunSymbolWorkerRequestsInProcess( + CreateSymbolWorkerRequest( + projectRoot, + filePath, + "runresetdsl", + "reset Before"))); + Assert.Null(beforeConfig.WorkerError); + Assert.Empty(beforeConfig.Symbols!); + + WriteSymbolWorkerPatternConfig( + sourceDirectory, + "run-reset.yaml", + "language: \"runresetdsl\"\nextensions:\n - extension: \".runreset\"\npatterns:\n - kind: \"class\"\n regex: \"^reset (?\\\\w+)\"\n"); + + var nextRun = Assert.Single(RunSymbolWorkerRequestsInProcess( + CreateSymbolWorkerRequest( + projectRoot, + filePath, + "runresetdsl", + "reset NextRun"))); + Assert.Null(nextRun.WorkerError); + Assert.Equal("NextRun", Assert.Single(nextRun.Symbols!).Name); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_CachesKnownPatternDiscoveryFailureForRun() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + try + { + var sourceDirectory = Path.Combine(projectRoot, "src", "known-failure"); + WriteSymbolWorkerPatternConfig( + sourceDirectory, + "known.yaml", + "language: \"knownfailuredsl\"\nextensions:\n - extension: \".knownfailure\"\npatterns:\n - kind: \"class\"\n regex: \"^known (?\\\\w+)\"\n"); + var patternDirectory = PathCasing.NormalizeBoundaryPath( + Path.Combine(sourceDirectory, ".cdidx", "patterns")); + var yamlAttempts = 0; + ExtractorPluginRegistry.ResetForTests(); + ExtractorPluginRegistry.UserPatternDirectoryOverrideForTests = + Path.Combine(projectRoot, "missing-user-patterns"); + ExtractorPluginRegistry.EnumeratePatternFilesForTesting = (directory, searchPattern) => + { + if (PathCasing.PathsEqual(patternDirectory, PathCasing.NormalizeBoundaryPath(directory)) + && string.Equals(searchPattern, "*.yaml", StringComparison.Ordinal)) + { + yamlAttempts++; + throw new IOException("simulated known pattern discovery failure"); + } + + return Directory.EnumerateFiles(directory, searchPattern, SearchOption.TopDirectoryOnly); + }; + + var responses = RunSymbolWorkerRequestsInProcess( + CreateSymbolWorkerRequest( + projectRoot, + Path.Combine(sourceDirectory, "first.knownfailure"), + "knownfailuredsl", + "known First"), + CreateSymbolWorkerRequest( + projectRoot, + Path.Combine(sourceDirectory, "second.knownfailure"), + "knownfailuredsl", + "known Second")); + + Assert.Equal(1, yamlAttempts); + Assert.Contains("pattern directory", responses[0].CapturedStderr, StringComparison.OrdinalIgnoreCase); + Assert.Equal(string.Empty, responses[1].CapturedStderr); + Assert.Empty(responses[0].Symbols!); + Assert.Empty(responses[1].Symbols!); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_CachesRejectedSymlinkPatternDirectoryForRun() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + try + { + var sourceDirectory = Path.Combine(projectRoot, "src", "unsafe"); + var externalPatternDirectory = Path.Combine(projectRoot, "external-patterns"); + Directory.CreateDirectory(sourceDirectory); + Directory.CreateDirectory(externalPatternDirectory); + var cdidxDirectory = Path.Combine(sourceDirectory, ".cdidx"); + Directory.CreateDirectory(cdidxDirectory); + var patternDirectory = Path.Combine(cdidxDirectory, "patterns"); + try + { + Directory.CreateSymbolicLink(patternDirectory, externalPatternDirectory); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException) + { + return; + } + + var normalizedPatternDirectory = PathCasing.NormalizeBoundaryPath(patternDirectory); + var patternInspections = 0; + ExtractorPluginRegistry.ResetForTests(); + ExtractorPluginRegistry.UserPatternDirectoryOverrideForTests = + Path.Combine(projectRoot, "missing-user-patterns"); + ExtractorPluginRegistry.InspectPatternDirectoryForTesting = path => + { + if (PathCasing.PathsEqual( + normalizedPatternDirectory, + PathCasing.NormalizeBoundaryPath(path))) + { + patternInspections++; + } + }; + + var responses = RunSymbolWorkerRequestsInProcess( + CreateSymbolWorkerRequest(projectRoot, Path.Combine(sourceDirectory, "first.cs")), + CreateSymbolWorkerRequest(projectRoot, Path.Combine(sourceDirectory, "second.cs"))); + + Assert.Equal(1, patternInspections); + Assert.Contains("symbolic links", responses[0].CapturedStderr, StringComparison.Ordinal); + Assert.Equal(string.Empty, responses[1].CapturedStderr); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_RetriesUnexpectedPatternDiscoveryFailure() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + try + { + var sourceDirectory = Path.Combine(projectRoot, "src", "unexpected-failure"); + WriteSymbolWorkerPatternConfig( + sourceDirectory, + "retry.yaml", + "language: \"retryworkerdsl\"\nextensions:\n - extension: \".retryworker\"\npatterns:\n - kind: \"class\"\n regex: \"^retry (?\\\\w+)\"\n"); + var patternDirectory = PathCasing.NormalizeBoundaryPath( + Path.Combine(sourceDirectory, ".cdidx", "patterns")); + var yamlAttempts = 0; + ExtractorPluginRegistry.ResetForTests(); + ExtractorPluginRegistry.UserPatternDirectoryOverrideForTests = + Path.Combine(projectRoot, "missing-user-patterns"); + ExtractorPluginRegistry.EnumeratePatternFilesForTesting = (directory, searchPattern) => + { + if (PathCasing.PathsEqual(patternDirectory, PathCasing.NormalizeBoundaryPath(directory)) + && string.Equals(searchPattern, "*.yaml", StringComparison.Ordinal) + && ++yamlAttempts == 1) + { + throw new InvalidOperationException("simulated unexpected pattern discovery failure"); + } + + return Directory.EnumerateFiles(directory, searchPattern, SearchOption.TopDirectoryOnly); + }; + + var responses = RunSymbolWorkerRequestsInProcess( + CreateSymbolWorkerRequest( + projectRoot, + Path.Combine(sourceDirectory, "first.retryworker"), + "retryworkerdsl", + "retry First"), + CreateSymbolWorkerRequest( + projectRoot, + Path.Combine(sourceDirectory, "second.retryworker"), + "retryworkerdsl", + "retry Second")); + + Assert.NotNull(responses[0].WorkerError); + Assert.Null(responses[1].WorkerError); + Assert.Equal(2, yamlAttempts); + Assert.Equal("Second", Assert.Single(responses[1].Symbols!).Name); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void SymbolExtractionWorker_SaturatedPatternCacheFallsBackToUncachedDiscovery() + { + var projectRoot = CreateTempProject(); + lock (TestConsoleLock.Gate) + { + WorkerPatternConfigDiscoveryCache? cache = null; + try + { + var firstDirectory = Path.Combine(projectRoot, "src", "first"); + var overflowDirectory = Path.Combine(projectRoot, "src", "overflow"); + Directory.CreateDirectory(firstDirectory); + WriteSymbolWorkerPatternConfig( + overflowDirectory, + "overflow.yaml", + "language: \"overflowworkerdsl\"\nextensions:\n - extension: \".overflowworker\"\npatterns:\n - kind: \"class\"\n regex: \"^overflow (?\\\\w+)\"\n"); + var overflowPatternDirectory = PathCasing.NormalizeBoundaryPath( + Path.Combine(overflowDirectory, ".cdidx", "patterns")); + var overflowYamlEnumerations = 0; + ExtractorPluginRegistry.ResetForTests(); + ExtractorPluginRegistry.UserPatternDirectoryOverrideForTests = + Path.Combine(projectRoot, "missing-user-patterns"); + ExtractorPluginRegistry.EnumeratePatternFilesForTesting = (directory, searchPattern) => + { + if (PathCasing.PathsEqual( + overflowPatternDirectory, + PathCasing.NormalizeBoundaryPath(directory)) + && string.Equals(searchPattern, "*.yaml", StringComparison.Ordinal)) + { + overflowYamlEnumerations++; + } + + return Directory.EnumerateFiles(directory, searchPattern, SearchOption.TopDirectoryOnly); + }; + SymbolExtractionWorker.PatternConfigDiscoveryCacheFactoryForTesting = () => + cache = new WorkerPatternConfigDiscoveryCache( + maxRootSnapshots: 1, + maxDirectoriesPerRoot: 1, + maxDirectoriesPerWorker: 1); + + var responses = RunSymbolWorkerRequestsInProcess( + CreateSymbolWorkerRequest(projectRoot, Path.Combine(firstDirectory, "first.cs")), + CreateSymbolWorkerRequest( + projectRoot, + Path.Combine(overflowDirectory, "first.overflowworker"), + "overflowworkerdsl", + "overflow First"), + CreateSymbolWorkerRequest( + projectRoot, + Path.Combine(overflowDirectory, "second.overflowworker"), + "overflowworkerdsl", + "overflow Second")); + + Assert.All(responses, response => Assert.Null(response.WorkerError)); + Assert.Equal("First", Assert.Single(responses[1].Symbols!).Name); + Assert.Equal("Second", Assert.Single(responses[2].Symbols!).Name); + Assert.Equal(2, overflowYamlEnumerations); + Assert.NotNull(cache); + Assert.Equal(1, cache.RetainedDirectoryCount); + } + finally + { + SymbolExtractionWorker.PatternConfigDiscoveryCacheFactoryForTesting = null; + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void WorkerPatternConfigDiscoveryCache_UsesFilesystemCasingAndBoundedRootLru() + { + var projectRoot = CreateTempProject(); + lock (PathCasingTestLock.Gate) + { + var previousProbe = PathCasing.IgnoreCaseProbeForTesting; + try + { + PathCasing.ResetCacheForTests(); + PathCasing.IgnoreCaseProbeForTesting = _ => true; + var cache = new WorkerPatternConfigDiscoveryCache( + maxRootSnapshots: 2, + maxDirectoriesPerRoot: 1, + maxDirectoriesPerWorker: 2); + var rootAPath = Path.Combine(projectRoot, "root-a"); + var rootBPath = Path.Combine(projectRoot, "root-b"); + var rootCPath = Path.Combine(projectRoot, "root-c"); + var rootA = cache.AddReloadedRoot(rootAPath); + var upperDirectory = Path.Combine(rootAPath, "Src", ".cdidx", "patterns"); + var lowerDirectory = Path.Combine(rootAPath, "src", ".cdidx", "patterns"); + Assert.True(cache.ShouldInspectPatternDirectory(rootA, upperDirectory)); + cache.RecordInspectedPatternDirectory(rootA, upperDirectory); + Assert.False(cache.ShouldInspectPatternDirectory(rootA, lowerDirectory)); + + var overflowDirectory = Path.Combine(rootAPath, "other", ".cdidx", "patterns"); + Assert.True(cache.ShouldInspectPatternDirectory(rootA, overflowDirectory)); + cache.RecordInspectedPatternDirectory(rootA, overflowDirectory); + Assert.True(cache.ShouldInspectPatternDirectory(rootA, overflowDirectory)); + + _ = cache.AddReloadedRoot(rootBPath); + Assert.True(cache.TryGetRoot(rootAPath, out _)); + _ = cache.AddReloadedRoot(rootCPath); + Assert.True(cache.TryGetRoot(rootAPath, out _)); + Assert.False(cache.TryGetRoot(rootBPath, out _)); + Assert.True(cache.TryGetRoot(rootCPath, out _)); + Assert.Equal(2, cache.RootCount); + + PathCasing.ResetCacheForTests(); + PathCasing.IgnoreCaseProbeForTesting = _ => false; + var sensitiveCache = new WorkerPatternConfigDiscoveryCache(); + var sensitiveRoot = sensitiveCache.AddReloadedRoot(rootAPath); + sensitiveCache.RecordInspectedPatternDirectory(sensitiveRoot, upperDirectory); + Assert.True(sensitiveCache.ShouldInspectPatternDirectory(sensitiveRoot, lowerDirectory)); + } + finally + { + PathCasing.IgnoreCaseProbeForTesting = previousProbe; + PathCasing.ResetCacheForTests(); + DeleteDirectory(projectRoot); + } + } + } + + [Fact] + public void WorkerPatternConfigDiscoveryCache_DoesNotProbeSymlinkedPatternTargetForCasing() + { + var projectRoot = CreateTempProject(); + lock (PathCasingTestLock.Gate) + { + var previousProbe = PathCasing.IgnoreCaseProbeForTesting; + try + { + var sourceDirectory = Path.Combine(projectRoot, "src", "unsafe-case-probe"); + var cdidxDirectory = Path.Combine(sourceDirectory, ".cdidx"); + var externalPatternDirectory = Path.Combine(projectRoot, "external-patterns"); + Directory.CreateDirectory(cdidxDirectory); + Directory.CreateDirectory(externalPatternDirectory); + var patternDirectory = Path.Combine(cdidxDirectory, "patterns"); + try + { + Directory.CreateSymbolicLink(patternDirectory, externalPatternDirectory); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or PlatformNotSupportedException) + { + return; + } + + var probedAnchors = new List(); + PathCasing.ResetCacheForTests(); + PathCasing.IgnoreCaseProbeForTesting = anchor => + { + probedAnchors.Add(PathCasing.NormalizeBoundaryPath(anchor)); + return false; + }; + var cache = new WorkerPatternConfigDiscoveryCache(); + var root = cache.AddReloadedRoot(projectRoot); + + Assert.True(cache.ShouldInspectPatternDirectory(root, patternDirectory)); + cache.RecordInspectedPatternDirectory(root, patternDirectory); + + Assert.Contains(PathCasing.NormalizeBoundaryPath(sourceDirectory), probedAnchors); + Assert.DoesNotContain(PathCasing.NormalizeBoundaryPath(patternDirectory), probedAnchors); + Assert.DoesNotContain(PathCasing.NormalizeBoundaryPath(externalPatternDirectory), probedAnchors); + } + finally + { + PathCasing.IgnoreCaseProbeForTesting = previousProbe; + PathCasing.ResetCacheForTests(); + DeleteDirectory(projectRoot); + } + } + } + [Fact] public void Run_NestedPatternSidecarReachesBoundedSymbolWorker_Issue4597() { @@ -9882,6 +10399,49 @@ private static string GetBuiltCliDllPath() private static void WriteSymbolWorkerPatternConfig(string projectRoot, string content) => WriteSymbolWorkerPatternConfig(projectRoot, "toydsl.yaml", content); + private static SymbolExtractionWorker.WorkerRequest CreateSymbolWorkerRequest( + string projectRoot, + string filePath, + string lang = "csharp", + string content = "class WorkerCacheSample { }") + => new( + 0, + lang, + content, + filePath, + projectRoot, + ContentIsNormalized: true, + HasOversizeLine: false, + ConflictMarkerLine: null); + + private static List RunSymbolWorkerRequestsInProcess( + params SymbolExtractionWorker.WorkerRequest[] requests) + { + var frames = string.Join( + '\n', + requests.Select(request => JsonSerializer.Serialize(request, SymbolExtractionWorker.JsonOptions))); + using var input = new StringReader(frames + "\n"); + using var output = new StringWriter(CultureInfo.InvariantCulture); + using var error = new StringWriter(CultureInfo.InvariantCulture); + + var handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out var exitCode); + + Assert.True(handled); + Assert.Equal(0, exitCode); + Assert.Equal(string.Empty, error.ToString()); + return output.ToString() + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(line => JsonSerializer.Deserialize( + line, + SymbolExtractionWorker.JsonOptions)!) + .ToList(); + } + private static void WriteSymbolWorkerPatternConfig(string projectRoot, string fileName, string content) { var path = Path.Combine(projectRoot, ".cdidx", "patterns", fileName); From 9b89682cced2b6f54166978ba018897be063ed17 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 08:10:24 +0900 Subject: [PATCH 07/16] Reuse C# prepass symbol artifacts --- DEVELOPER_GUIDE.md | 32 +++ TESTING_GUIDE.md | 2 + .../+large-codebase-initial-indexing.fixed.md | 7 + ...xCommandRunner.FullScan.CSharpPreflight.cs | 16 +- ...mmandRunner.FullScan.ExtractionPipeline.cs | 9 + ...ommandRunner.FullScan.ExtractionWorkers.cs | 47 ++-- ...xCommandRunner.FullScan.FilePersistence.cs | 25 +- ...exCommandRunner.FullScan.ResultConsumer.cs | 3 + .../Cli/IndexCommandRunner.FullScan.cs | 8 + .../CSharpPrepassSymbolArtifactCache.cs | 170 +++++++++++++ .../Indexer/CSharpStaticInterfacePrepass.cs | 79 +++++- .../PostExtractionHookMutationMaterializer.cs | 2 +- .../Indexer/Scanning/FileContentLoader.cs | 25 +- .../Scanning/FileIndexer.RecordLoading.cs | 30 +++ .../Mcp/McpToolHandlers.Indexing.Execution.cs | 37 ++- .../CSharpPrepassSymbolArtifactCacheTests.cs | 182 ++++++++++++++ tests/CodeIndex.Tests/FileIndexerTests.cs | 58 +++++ .../IndexCommandRunnerFullScanTests.cs | 228 +++++++++++++++++- .../McpServerToolsCallTests.cs | 31 +++ 19 files changed, 951 insertions(+), 40 deletions(-) create mode 100644 src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs create mode 100644 tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 5a798b2b6..781692420 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -415,6 +415,24 @@ Rows with missing or invalid legacy stat values are excluded so normal checksum reuse or reindexing can repair them, and CLI/MCP cancellation must interrupt the snapshot query as well as the later extraction pipeline. +Only a first-time full index that started with no indexed files may reuse raw +built-in C# symbol artifacts from the static-interface prepass. The CLI excludes +rebuilds and symbols-only runs; MCP evaluates the same empty-database condition +before any rebuild mutation. The cache never owns source text or bytes: the main +pass still performs its authoritative content read and hook, stat-snapshot and +TOCTOU validation, and checksum calculation. It consumes a normalized-path +artifact once only after that checksum matches. Cached symbols are deep clones. +Artifact-producing extraction receives the main pass's absolute file path and +project root so file-local family identities stay identical. File ID assignment, +family scope, source observation, post-extraction hooks, +kind filtering, caps, line validation, persistence, reference extraction, and +bounded-regex issue reporting remain on the normal main-pass path. Incomplete +prepasses, extraction-stall test seams, checksum drift, regex timeouts, and cache +admission limits fall back to ordinary extraction. A timed-out prepass result is +partial and must not make that transient result authoritative. Keep admission +bounded to 4,096 files, 131,072 symbols, and an estimated 32 MiB, and clear all +unconsumed artifacts before reference-graph work begins. + Authoritative full scans collect the C#, VB, F#, and MSBuild project-marker fingerprints during the shared source-directory enumeration. The same pass also builds a budget-independent directory marker-count snapshot used by @@ -4154,6 +4172,20 @@ CLI full scan、scoped update、MCP indexing は、この prepass より前に w その読込済み snapshot を再利用してください。直接 prepass を呼ぶ側は、snapshot 読込済みを明示的に 保証しない限り、従来どおり discovery 有効の既定経路を維持します。 +static-interface prepass の raw built-in C# symbol artifact を再利用できるのは、indexed file が +0件の状態から開始した初回 full index だけです。CLI は rebuild と symbols-only を除外し、MCP は +rebuild mutation より前の空 database 条件を使います。cache は source text / byte を保持せず、main +pass は引き続き authoritative な content read と hook、stat snapshot / TOCTOU 検証、checksum 計算を +実行します。正規化 path の artifact は checksum 一致後に1回だけ取り出します。cached symbol は deep +clone とします。artifact を生成する extraction には main pass と同じ absolute file path / project root を +渡し、file-local family identity を一致させてください。FileId、family scope、source observation、 +post-extraction hook、kind filter、cap、line 検証、 +persistence、reference extraction、bounded-regex issue は通常の main-pass 経路で処理してください。 +不完全な prepass、extraction-stall test seam、checksum drift、regex timeout、cache 上限では通常 +extraction へ fallback します。timeout した prepass 結果は partial であり、一過性の結果を +authoritative にしてはいけません。admission は 4,096 file、131,072 symbol、推定 32 MiB に制限し、未消費 +artifact は reference graph 開始前にすべて clear してください。 + authoritative な full scan は、共有 source-directory enumeration 中に C#、VB、F#、 MSBuild の project-marker fingerprint を収集します。同じ pass で budget 非依存の directory marker-count snapshot も構築し、`GetFamilyScopeKey` が file ごとに各 ancestor の diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index db83400ca..fe79ff5fd 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -427,6 +427,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result cover multi-frame remainder reuse, CRLF split across a 4 KiB read boundary, Unicode bytes, an unterminated final frame, stable EOF, and rejection at one byte over the negotiated cap. They protect direct worker-response deserialization without constructing decoded JSON strings. - `IndexCommandRunnerTests`, `FileIndexerTests`, and `PerformanceTests` also cover `CSharpStaticInterfacePrepass` text, raw-byte, chunked raw-token, and streaming file contract probes. Stable candidate reads must authorize and open each file once, keep raw-negative reads to one bounded pass, and rewind that same handle only for a raw-positive full decode. A detected in-place or atomic-replacement mutation must discard that snapshot and reauthorize/reopen once so the prepass cannot diverge from the main indexing pass. Preserve UTF-8 / UTF-16, NUL rejection, growth, cancellation, and lexical-boundary behavior. The 576 KiB semantic-negative/positive allocation guard runs each probe 12 times and stays below 4 KiB of current-thread allocation so a whole-content mask cannot return. - The parallel C# static-interface full-scan fixture uses 64 implementation files and treats one workspace lookup build as a performance contract. Keep the contract lookup attached to the immutable prepass snapshot across CLI full scan, scoped update, and MCP indexing; do not rebuild it once per C# file. +- `CSharpPrepassSymbolArtifactCacheTests`, `FileIndexerTests`, and the CLI/MCP fresh-index fixtures protect bounded prepass artifact reuse. Keep deep-clone independence, take-once checksum matching, mismatch consumption, atomic file/symbol/estimated-byte caps, cancellation without partial admission, and non-admission of partial symbols after any bounded-regex timeout. Encoding theories must compare UTF-8, UTF-16 LE/BE, and invalid-UTF-8 prepass checksums with the authoritative loader. Integration coverage must prove reuse only for an empty non-rebuild full index, ordinary extraction for rebuild/symbols-only/existing/incomplete-or-stall paths, authoritative main-read mutation fallback, unchanged post hooks and family/kind processing, and cache clearing before graph work. - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` builds 20,000 unrelated generic interfaces around one contract and keeps lookup allocation below 64 KiB. Discover contract containers before parsing generic declarations, while the paired `ReferenceExtractorTests` case preserves declaration/member ordering, partial-interface last-write behavior, and contract member order. - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` keeps a 1,200-statement, regex-and-brace-heavy source at one emitted symbol and pins current-thread allocations below 6.0 MB for JavaScript, 8.2 MB for TypeScript, and 14.1 MB combined. This is a physical-pass contract: do not restore a second full-file sanitizer for private-scope analysis. Pair allocation changes with the JavaScript/TypeScript literal-and-comment scope theory so templates, comments, and regex braces cannot create or leak private classes. - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` keeps persisted C# contract-member candidates and the pending-path contract flag on one `files(lang)` → `symbols(file_id, kind)` row pass. Derive both results from that reader; do not restore a second all-contract query for excluded paths. Interface declarations may be loaded only for exact retained contract container names through bounded `symbols(name)` batches, and those dynamic tail shapes must stay out of the prepared cache. `DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` pins both index plans, while `LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` requires negative/LIKE-decoy-only reads to execute no interface phase and keeps unrelated plain interfaces out of managed materialization. `PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` also splits a 503-path language lookup at 500 parameters, returns only persisted C# paths plus sorted purge-plan IDs/bytes, keeps both SQL tail shapes out of the prepared cache, and stops before the second batch when cancellation is requested. @@ -1440,6 +1441,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" multi-frame remainder の再利用、4 KiB read 境界をまたぐ CRLF、Unicode byte、改行なし最終 frame、安定した EOF、合意済み上限を1 byte 超えた時点での拒否を検証します。decode 済み JSON string を作らず worker response を直接 deserialize する経路を固定します。 - `IndexCommandRunnerTests`、`FileIndexerTests`、`PerformanceTests` は `CSharpStaticInterfacePrepass` のテキスト判定、raw-byte、chunked raw-token、streaming file 契約 probe も扱います。安定した候補読み取りは各 file を1回だけ認可・openし、raw-negative は bounded pass 1回に留め、raw-positive の full decode だけ同じ handle を rewind してください。in-place mutation または atomic replacement を検知した場合は snapshot を破棄し、prepass と main indexing pass が乖離しないよう1回だけ再認可・再openします。UTF-8 / UTF-16、NUL 拒否、growth、cancellation、lexical boundary を維持してください。576 KiB の semantic-negative/positive allocation guard は各 probe を12回実行して current-thread allocation を4 KiB未満に保ち、content 全体 mask の再導入を防ぎます。 - parallel C# static-interface full-scan fixture は64個のimplementation fileを使い、workspace lookup buildが1回であることをperformance contractとします。CLI full scan、scoped update、MCP indexingを横断してcontract lookupをimmutable prepass snapshotに保持し、C# fileごとの再構築を戻さないでください。 +- `CSharpPrepassSymbolArtifactCacheTests`、`FileIndexerTests`、CLI/MCP の fresh-index fixture は bounded prepass artifact reuse を固定します。deep-clone の独立性、checksum 一致時の take-once、不一致時の消費、file / symbol / estimated-byte cap の原子性、partial admission を残さない cancellation、bounded-regex timeout 後の partial symbol をadmitしない契約を維持してください。encoding theory は UTF-8、UTF-16 LE/BE、不正 UTF-8 の prepass checksum を authoritative loader と比較します。integration coverage では空 database の非 rebuild full index だけが再利用し、rebuild / symbols-only / existing / incomplete-or-stall 経路は通常 extraction、main read 中の mutation は checksum fallback、post hook と family/kind 処理は従来どおり、graph 開始前に cache が clear されることを証明してください。 - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` は、1件のcontractの周囲に20,000件の無関係なgeneric interfaceを構築し、lookup allocationを64 KiB未満に固定します。generic宣言を解析する前にcontract containerを検出し、対になる`ReferenceExtractorTests`で宣言/member順、partial interfaceの後勝ち、contract member順を維持してください。 - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` は、regex と brace が密な1,200 statement の source を出力 symbol 1件に保ち、current-thread allocation を JavaScript 6.0 MB未満、TypeScript 8.2 MB未満、合計14.1 MB未満に固定します。これは physical-pass 契約であり、private-scope 解析専用の2回目の全file sanitizerを戻さないでください。allocation変更時はJavaScript/TypeScriptのliteral/comment scope theoryも対にし、template、comment、regexのbraceがprivate classを生成またはscope外へ漏らさないことを維持してください。 - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` は、永続化済み C# contract member 候補と pending-path contract flag を1回の `files(lang)` → `symbols(file_id, kind)` row passで得ることを固定します。両方を同じ reader から導出し、除外 path 判定のための2回目の全 contract query を戻さないでください。interface 宣言は厳密検証後に保持した contract container 名だけを bounded な `symbols(name)` batch で取得でき、その dynamic tail shape は prepared cache に残してはいけません。`DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` は両方の index plan を固定し、`LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` は negative / LIKE decoy だけの読込で interface phase が0回となり、無関係な通常 interface を managed materialization へ入れないことを要求します。`PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` は503件のpath言語lookupを500 parameterで分割し、永続C# pathと昇順のpurge-plan ID/byteだけを返し、両方のSQL tail shapeをprepared cache外に保ち、1つ目のbatch後のcancellationで2つ目を実行しないことも固定します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index e2c94824b..f8ec6332a 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -8,11 +8,16 @@ affected: - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs - src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs + - src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs + - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs - tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs + - tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs + - tests/CodeIndex.Tests/FileIndexerTests.cs - tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs - tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -29,6 +34,7 @@ affected: - **First-time full indexes resolve only references that have candidates** — empty-database CLI scans persist canonical unresolved defaults during bulk insertion, aggregate the candidate table once, and update candidate-bearing references by primary key instead of probing every reference row. Rebuilds, updates, retained graphs, and MCP indexing keep their existing recovery contracts. - **Fresh graph planning uses post-load cardinalities** — truly empty CLI and MCP bulk loads now analyze the populated file, symbol, and reference tables immediately before candidate resolution, allowing SQLite to plan the expensive first graph build from current statistics. Rebuilds, updates, existing databases, and symbols-only runs keep their prior lifecycle; cancellation still aborts, while a statistics-only SQLite failure rolls back its savepoint and continues best-effort. - **Persistent symbol workers reuse bounded pattern-directory snapshots across languages** — each project-root reload discovers user and root configs once, while nested ancestor directories, including missing and rejected results, are observed once per worker command. The cache follows live filesystem casing and falls back to uncached discovery when full so configs are never skipped; direct registry callers keep dynamic discovery. +- **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once deep clones of built-in symbols already extracted for the static-interface workspace. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. ## 日本語 @@ -38,3 +44,4 @@ affected: - **初回full indexでcandidateを持つreferenceだけを解決するようにしました** — 空databaseからのCLI scanではbulk insert中にcanonicalな未解決値を保存し、candidate tableを1回集約して、全reference rowをprobeせずcandidateを持つrowだけをprimary keyで更新します。rebuild、update、retained graph、MCP indexingの既存recovery契約は変更しません。 - **新規graph planningでbulk load後のcardinalityを使うようにしました** — 真に空のdatabaseから始まるCLI / MCP bulk loadでは、candidate解決の直前に投入済みのfile、symbol、reference tableを解析し、SQLiteが初回の高コストなgraph構築を最新統計から計画できるようにします。rebuild、update、既存database、symbols-onlyのlifecycleは従来どおりです。cancellationは引き続き中断し、統計更新だけのSQLite failureはsavepointを戻してbest-effortで継続します。 - **persistent symbol worker が全言語で上限付き pattern-directory snapshot を再利用するようにしました** — project-root reload ごとに user / root config を1回だけ探索し、missing や reject を含む nested ancestor directory は worker command ごとに初回結果を再利用します。cache は実 filesystem の case policy に従い、飽和時は uncached discovery に fallback して config を skip しません。registry の direct caller は従来どおり動的に探索します。 +- **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once の deep clone として利用できます。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs index 35f5893f6..a02960a21 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.CSharpPreflight.cs @@ -44,6 +44,7 @@ private sealed record FullScanCSharpPreflightResult( Dictionary? CSharpPrepassStatReuse, Dictionary? CSharpWorkspaceFileSnapshots, CSharpStaticInterfaceWorkspaceSymbols CSharpWorkspace, + CSharpPrepassSymbolArtifactCache? CSharpPrepassSymbolArtifacts, bool ForceFullCSharpRefreshFromInvalidatedNoOp, bool PreservePriorPositiveCSharpSourceNoOp, bool CSharpSourceEvidenceForStamp, @@ -177,6 +178,13 @@ bool CanReuseCSharpPrepassTargetWithoutRead( && context.CSharpPrepassTargets.Count > 0 && !(priorPositiveCSharpSourceNoOpCandidate && allCSharpPrepassTargetsReusable); + var csharpPrepassSymbolArtifacts = CSharpPrepassSymbolArtifactCache + .CreateForFreshBuiltInExtraction( + csharpWorkspaceMaterialized + && context.StartedWithNoIndexedFiles + && !options.Rebuild + && !options.SymbolsOnly + && IndexExtractionStallTimeoutForTesting == null); if (options.SymbolsOnly || context.GetDeferCSharpMutationsForIncompleteScan()) { @@ -190,6 +198,7 @@ bool CanReuseCSharpPrepassTargetWithoutRead( priorPositiveCSharpSourceNoOpCandidate, allCSharpPrepassTargetsReusable, CanReuseCSharpPrepassTargetWithoutRead, + csharpPrepassSymbolArtifacts, out csharpWorkspaceFileSnapshots); forceFullCSharpRefreshFromInvalidatedNoOp = csharpWorkspaceMaterialized @@ -202,6 +211,8 @@ bool CanReuseCSharpPrepassTargetWithoutRead( if (!options.SymbolsOnly && !csharpWorkspace.SourceContractEvidenceComplete) { + csharpPrepassSymbolArtifacts?.Clear(); + csharpPrepassSymbolArtifacts = null; var incompleteSourcePaths = csharpWorkspace.IncompleteSourcePaths; context.DeferCSharpMutationsForIncompleteWorkspace( @@ -248,6 +259,7 @@ bool CanReuseCSharpPrepassTargetWithoutRead( csharpPrepassStatReuse, csharpWorkspaceFileSnapshots, csharpWorkspace, + csharpPrepassSymbolArtifacts, forceFullCSharpRefreshFromInvalidatedNoOp, preservePriorPositiveCSharpSourceNoOp, csharpSourceEvidenceForStamp, @@ -261,6 +273,7 @@ private static CSharpStaticInterfaceWorkspaceSymbols bool allCSharpPrepassTargetsReusable, Func canReuseCSharpPrepassTargetWithoutRead, + CSharpPrepassSymbolArtifactCache? symbolArtifactCache, out Dictionary? csharpWorkspaceFileSnapshots) @@ -313,7 +326,8 @@ private static CSharpStaticInterfaceWorkspaceSymbols context .IsExistingCSharpSymbolPathNowNonCSharp, patternConfigsAlreadyLoaded: true, - cancellationToken: context.CancellationToken), + cancellationToken: context.CancellationToken, + symbolArtifactCache: symbolArtifactCache), context.CancellationToken); } catch (OperationCanceledException) when ( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs index c168f5f92..7caab71f4 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs @@ -69,6 +69,9 @@ internal required Func internal required Func GetCSharpWorkspace { get; init; } + internal required Func + GetCSharpPrepassSymbolArtifacts + { get; init; } internal required Func?> GetCSharpWorkspaceFileSnapshots @@ -126,6 +129,7 @@ private static FullScanExtractionPipelineResult { if (context.ExtractionWorkItemCount == 0) { + context.GetCSharpPrepassSymbolArtifacts()?.Clear(); FullScanExtractionSchedulingForTesting?.Invoke(false, null); return new FullScanExtractionPipelineResult(null, null); } @@ -154,6 +158,7 @@ private static FullScanExtractionPipelineResult } finally { + context.GetCSharpPrepassSymbolArtifacts()?.Clear(); context.SetCurrentJsonIndexFile(null); context.FullScanProgress.StopJsonHeartbeat(); postExtractionHooks.Dispose(); @@ -326,6 +331,8 @@ private static FullScanExtractionConsumerState ParallelizeExtraction = parallelizeExtraction, ExtractionTailSchedule = extractionTailSchedule, CSharpWorkspace = context.GetCSharpWorkspace(), + CSharpPrepassSymbolArtifacts = + context.GetCSharpPrepassSymbolArtifacts(), CSharpWorkspaceFileSnapshots = context.GetCSharpWorkspaceFileSnapshots(), PostExtractionHooks = postExtractionHooks, @@ -437,6 +444,8 @@ private static FullScanExtractionConsumerContext context.GetDeferCSharpMutationsForIncompleteScan, GetFtsMutated = context.GetFtsMutated, GetCSharpWorkspace = context.GetCSharpWorkspace, + GetCSharpPrepassSymbolArtifacts = + context.GetCSharpPrepassSymbolArtifacts, GetCSharpWorkspaceFileSnapshots = context.GetCSharpWorkspaceFileSnapshots, DeferCSharpMutationsForLoadedSnapshotDrift = diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs index 7120d254f..dcb4b521d 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionWorkers.cs @@ -20,6 +20,7 @@ private sealed class FullScanExtractionWorkerContext internal required bool ParallelizeExtraction { get; init; } internal required int[] ExtractionTailSchedule { get; init; } internal required CSharpStaticInterfaceWorkspaceSymbols CSharpWorkspace { get; init; } + internal CSharpPrepassSymbolArtifactCache? CSharpPrepassSymbolArtifacts { get; init; } internal Dictionary? CSharpWorkspaceFileSnapshots { get; init; } internal required PostExtractionHookRunner PostExtractionHooks { get; init; } internal required ActiveExtractionPhase?[] ActiveExtractionPhases { get; init; } @@ -41,6 +42,7 @@ private static Task[] StartFullScanExtractionWorkers( var parallelizeExtraction = context.ParallelizeExtraction; var extractionTailSchedule = context.ExtractionTailSchedule; var csharpWorkspace = context.CSharpWorkspace; + var csharpPrepassSymbolArtifacts = context.CSharpPrepassSymbolArtifacts; var csharpWorkspaceFileSnapshots = context.CSharpWorkspaceFileSnapshots; var postExtractionHooks = context.PostExtractionHooks; var activeExtractionPhases = context.ActiveExtractionPhases; @@ -146,21 +148,36 @@ private static Task[] StartFullScanExtractionWorkers( } Volatile.Write(ref activeExtractionPhases[workerIndex], new(record.Path, "symbols")); FullScanFilePhaseForTesting?.Invoke(record.Path, "symbols"); - var symbolExtraction = ExtractSymbolsWithStallTimeout( - 0, - record.Lang, - content, - filePath, - projectRoot, - record.Path, - Volatile.Read(ref activeExtractionPhases[workerIndex])!.Format(), - true, - hasOversizeLine, - loaded.ConflictMarkerLine, - workerSymbolExtractionWorker.Value, - extractionCancellationToken); - symbols = symbolExtraction.Symbols; - var symbolRegexTimeoutIssue = symbolExtraction.RegexTimeoutIssue; + FileIssue? symbolRegexTimeoutIssue; + if (string.Equals(record.Lang, "csharp", StringComparison.Ordinal) + && record.Checksum is { } checksum + && csharpPrepassSymbolArtifacts?.TryTake( + record.Path, + checksum, + out var symbolArtifact) == true) + { + symbols = symbolArtifact.Symbols; + symbolRegexTimeoutIssue = null; + } + else + { + var symbolExtraction = ExtractSymbolsWithStallTimeout( + 0, + record.Lang, + content, + filePath, + projectRoot, + record.Path, + Volatile.Read(ref activeExtractionPhases[workerIndex])!.Format(), + true, + hasOversizeLine, + loaded.ConflictMarkerLine, + workerSymbolExtractionWorker.Value, + extractionCancellationToken); + symbols = symbolExtraction.Symbols; + symbolRegexTimeoutIssue = + symbolExtraction.RegexTimeoutIssue; + } if (string.Equals(record.Lang, "csharp", StringComparison.Ordinal)) { var sourceFileContext = new FileContext( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs index 2bfda829c..6279bfc38 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.FilePersistence.cs @@ -20,6 +20,7 @@ private sealed class FullScanFilePersistenceContext internal required bool StartedWithNoIndexedFiles { get; init; } internal required bool DeferCSharpMutationsForIncompleteScan { get; init; } internal required CSharpStaticInterfaceWorkspaceSymbols CSharpWorkspace { get; init; } + internal CSharpPrepassSymbolArtifactCache? CSharpPrepassSymbolArtifacts { get; init; } internal required PostExtractionHookRunner PostExtractionHooks { get; init; } internal required SymbolExtractionWorkerClient SymbolExtractionWorker { get; init; } internal required CancellationToken CancellationToken { get; init; } @@ -117,8 +118,23 @@ private static FullScanFilePersistenceResult PersistFullScanFile( context.SetPhase(FormatIndexPhasePath(record.Path, "symbols"), "symbols"); FullScanFilePhaseForTesting?.Invoke(record.Path, "symbols"); SymbolExtractionResult? symbolExtraction = null; - var symbols = item.Symbols == null - ? (symbolExtraction = ExtractSymbolsWithStallTimeout( + IReadOnlyList symbols; + if (item.Symbols != null) + { + symbols = ReassignSymbolFileIds(item.Symbols, fileId); + } + else if (string.Equals(record.Lang, "csharp", StringComparison.Ordinal) + && record.Checksum is { } checksum + && context.CSharpPrepassSymbolArtifacts?.TryTake( + record.Path, + checksum, + out var symbolArtifact) == true) + { + symbols = ReassignSymbolFileIds(symbolArtifact.Symbols, fileId); + } + else + { + symbolExtraction = ExtractSymbolsWithStallTimeout( fileId, record.Lang, item.Content!, @@ -130,8 +146,9 @@ private static FullScanFilePersistenceResult PersistFullScanFile( item.HasOversizeLine, item.ConflictMarkerLine, context.SymbolExtractionWorker, - cancellationToken)).Symbols - : ReassignSymbolFileIds(item.Symbols, fileId); + cancellationToken); + symbols = symbolExtraction.Symbols; + } var extractedSymbolCount = symbols.Count; var symbolRegexTimeoutIssue = symbolExtraction?.RegexTimeoutIssue; var fileContext = new FileContext( diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ResultConsumer.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ResultConsumer.cs index bb3ed8dca..373bbc458 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ResultConsumer.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ResultConsumer.cs @@ -46,6 +46,7 @@ private sealed class FullScanExtractionConsumerContext internal required Func GetDeferCSharpMutationsForIncompleteScan { get; init; } internal required Func GetFtsMutated { get; init; } internal required Func GetCSharpWorkspace { get; init; } + internal required Func GetCSharpPrepassSymbolArtifacts { get; init; } internal required Func?> GetCSharpWorkspaceFileSnapshots { get; init; } internal required Action DeferCSharpMutationsForLoadedSnapshotDrift { get; init; } internal required Func TargetRequiresJavaScriptTypeScriptRefresh { get; init; } @@ -180,6 +181,8 @@ private static void ProcessFullScanExtractionItem( DeferCSharpMutationsForIncompleteScan = context.GetDeferCSharpMutationsForIncompleteScan(), CSharpWorkspace = context.GetCSharpWorkspace(), + CSharpPrepassSymbolArtifacts = + context.GetCSharpPrepassSymbolArtifacts(), PostExtractionHooks = context.PostExtractionHooks, SymbolExtractionWorker = context.SymbolExtractionWorker, CancellationToken = context.CancellationToken, diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index e5e78f969..10303b646 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -445,6 +445,8 @@ bool IsExistingCSharpSymbolPathNowNonCSharp(string indexPath) var csharpWorkspaceFileSnapshots = csharpPreflight.CSharpWorkspaceFileSnapshots; var csharpWorkspace = csharpPreflight.CSharpWorkspace; + var csharpPrepassSymbolArtifacts = + csharpPreflight.CSharpPrepassSymbolArtifacts; var forceFullCSharpRefreshFromInvalidatedNoOp = csharpPreflight.ForceFullCSharpRefreshFromInvalidatedNoOp; var preservePriorPositiveCSharpSourceNoOp = @@ -456,6 +458,8 @@ bool IsExistingCSharpSymbolPathNowNonCSharp(string indexPath) void DeferCSharpMutationsForLoadedSnapshotDrift(string path) { + csharpPrepassSymbolArtifacts?.Clear(); + csharpPrepassSymbolArtifacts = null; path = FormatCSharpWorkspaceSnapshotPath(projectRoot, path); deferCSharpMutationsForIncompleteScan = true; preservePriorPositiveCSharpSourceNoOp = false; @@ -759,6 +763,8 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis cancellationToken); if (!stableFiles) { + csharpPrepassSymbolArtifacts?.Clear(); + csharpPrepassSymbolArtifacts = null; var driftPath = FormatCSharpWorkspaceSnapshotPath(projectRoot, changedFilePath); var incompleteWorkspace = new CSharpStaticInterfaceWorkspaceSymbols( [], @@ -968,6 +974,8 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis () => deferCSharpMutationsForIncompleteScan, GetFtsMutated = () => ftsMutated, GetCSharpWorkspace = () => csharpWorkspace, + GetCSharpPrepassSymbolArtifacts = + () => csharpPrepassSymbolArtifacts, GetCSharpWorkspaceFileSnapshots = () => csharpWorkspaceFileSnapshots, DeferCSharpMutationsForLoadedSnapshotDrift = diff --git a/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs b/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs new file mode 100644 index 000000000..3d346517c --- /dev/null +++ b/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs @@ -0,0 +1,170 @@ +using System.Collections.Concurrent; +using CodeIndex.Indexer.Hooks; +using CodeIndex.Models; + +namespace CodeIndex.Indexer; + +internal sealed class CSharpPrepassSymbolArtifactCache +{ + internal const int DefaultMaxFiles = 4_096; + internal const int DefaultMaxSymbols = 131_072; + internal const long DefaultMaxEstimatedBytes = 32L * 1024 * 1024; + + private const long EstimatedArtifactBytes = 128; + private const long EstimatedSymbolBytes = 256; + private static readonly AsyncLocal?> + ScopedEventForTesting = new(); + + private readonly ConcurrentDictionary _artifacts = + new(StringComparer.Ordinal); + private readonly object _admissionLock = new(); + private readonly int _maxFiles; + private readonly int _maxSymbols; + private readonly long _maxEstimatedBytes; + private int _admittedFiles; + private int _admittedSymbols; + private long _admittedEstimatedBytes; + + internal CSharpPrepassSymbolArtifactCache( + int maxFiles = DefaultMaxFiles, + int maxSymbols = DefaultMaxSymbols, + long maxEstimatedBytes = DefaultMaxEstimatedBytes) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxFiles, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(maxSymbols, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(maxEstimatedBytes, 1); + _maxFiles = maxFiles; + _maxSymbols = maxSymbols; + _maxEstimatedBytes = maxEstimatedBytes; + } + + internal static Action? EventForTesting + { + get => ScopedEventForTesting.Value; + set => ScopedEventForTesting.Value = value; + } + + internal int Count => _artifacts.Count; + internal int AdmittedFileCount => Volatile.Read(ref _admittedFiles); + internal int AdmittedSymbolCount => Volatile.Read(ref _admittedSymbols); + internal long AdmittedEstimatedBytes => Interlocked.Read(ref _admittedEstimatedBytes); + + internal static CSharpPrepassSymbolArtifactCache? CreateForFreshBuiltInExtraction( + bool enabled) + => enabled + ? new CSharpPrepassSymbolArtifactCache() + : null; + + internal bool TryAdmit( + string path, + string checksum, + IReadOnlyList symbols, + bool hadRegexTimeout, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(path); + ArgumentNullException.ThrowIfNull(checksum); + ArgumentNullException.ThrowIfNull(symbols); + cancellationToken.ThrowIfCancellationRequested(); + var normalizedPath = FileIndexer.NormalizeIndexPath(path); + if (hadRegexTimeout) + { + Report("regex_timeout_skipped", normalizedPath); + return false; + } + var estimatedBytes = EstimateBytes(normalizedPath, checksum, symbols.Count); + + lock (_admissionLock) + { + cancellationToken.ThrowIfCancellationRequested(); + if (_artifacts.ContainsKey(normalizedPath)) + return false; + + if (_admittedFiles >= _maxFiles + || symbols.Count > _maxSymbols - _admittedSymbols + || estimatedBytes > _maxEstimatedBytes - _admittedEstimatedBytes) + { + Report("capacity_skipped", normalizedPath); + return false; + } + + var clonedSymbols = new List(symbols.Count); + foreach (var symbol in symbols) + { + cancellationToken.ThrowIfCancellationRequested(); + clonedSymbols.Add(PostExtractionHookMutationMaterializer.CloneSymbol(symbol)); + } + + var artifact = new CSharpPrepassSymbolArtifact( + normalizedPath, + checksum, + clonedSymbols); + if (!_artifacts.TryAdd(normalizedPath, artifact)) + return false; + + _admittedFiles++; + _admittedSymbols += symbols.Count; + _admittedEstimatedBytes += estimatedBytes; + Report("admitted", normalizedPath); + return true; + } + } + + internal bool TryTake( + string path, + string checksum, + out CSharpPrepassSymbolArtifact artifact) + { + ArgumentNullException.ThrowIfNull(path); + ArgumentNullException.ThrowIfNull(checksum); + var normalizedPath = FileIndexer.NormalizeIndexPath(path); + if (!_artifacts.TryRemove(normalizedPath, out artifact!)) + return false; + + if (!string.Equals(artifact.Checksum, checksum, StringComparison.Ordinal)) + { + artifact = null!; + Report("checksum_mismatch", normalizedPath); + return false; + } + + Report("taken", normalizedPath); + return true; + } + + internal void Clear() + { + lock (_admissionLock) + { + _artifacts.Clear(); + _admittedFiles = 0; + _admittedSymbols = 0; + _admittedEstimatedBytes = 0; + } + Report("cleared", string.Empty); + } + + private static long EstimateBytes( + string path, + string checksum, + int symbolCount) + { + return EstimatedArtifactBytes + + (path.Length * sizeof(char)) + + (checksum.Length * sizeof(char)) + + (symbolCount * EstimatedSymbolBytes); + } + + private static void Report(string phase, string path) + => ScopedEventForTesting.Value?.Invoke( + new CSharpPrepassSymbolArtifactCacheEvent(phase, path)); +} + +internal sealed record CSharpPrepassSymbolArtifact( + string Path, + string Checksum, + List Symbols); + +internal readonly record struct CSharpPrepassSymbolArtifactCacheEvent( + string Phase, + string Path); diff --git a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs index 8fa90ebcf..aaedbcc22 100644 --- a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs +++ b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs @@ -26,7 +26,8 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( Func? isExistingSymbolPathExcluded = null, bool loadExistingSymbolsOnlyForPendingQualifiedMemberAccess = false, bool patternConfigsAlreadyLoaded = false, - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + CSharpPrepassSymbolArtifactCache? symbolArtifactCache = null) { var targetCount = fileTargets.TryGetNonEnumeratedCount(out var count) ? count : 0; var candidates = new List(targetCount); @@ -80,6 +81,12 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( } var extractedByCandidate = new List?[candidates.Count]; + var artifactChecksums = symbolArtifactCache == null + ? null + : new string?[candidates.Count]; + var artifactHadRegexTimeouts = symbolArtifactCache == null + ? null + : new bool[candidates.Count]; var sourceEvidenceComplete = 1; var hasPendingQualifiedMemberAccessCandidate = 0; string? firstIncompleteSourcePath = null; @@ -97,12 +104,29 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( reportCandidateFile?.Invoke(candidateIndex, target.DisplayRelativePath); try { - var content = indexer.LoadCSharpStaticInterfaceCandidateContentForPrepass( - target.FilePath, - target.RelativePath, - includeQualifiedMemberAccessCandidate: - loadExistingSymbolsOnlyForPendingQualifiedMemberAccess, - cancellationToken); + string? content; + string? checksum = null; + if (symbolArtifactCache == null) + { + content = indexer.LoadCSharpStaticInterfaceCandidateContentForPrepass( + target.FilePath, + target.RelativePath, + includeQualifiedMemberAccessCandidate: + loadExistingSymbolsOnlyForPendingQualifiedMemberAccess, + cancellationToken); + } + else + { + var loaded = indexer + .LoadCSharpStaticInterfaceCandidateContentWithChecksumForPrepass( + target.FilePath, + target.RelativePath, + includeQualifiedMemberAccessCandidate: + loadExistingSymbolsOnlyForPendingQualifiedMemberAccess, + cancellationToken); + content = loaded?.Content; + checksum = loaded?.Checksum; + } if (content is not null) { if (content.AsSpan().IndexOf('.') >= 0) @@ -114,20 +138,39 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( if (MayContainCSharpWorkspaceReferenceTargets(content)) { + var extractionFilePath = symbolArtifactCache == null + ? target.IndexPath + : target.FilePath; + var extractionProjectRoot = symbolArtifactCache == null + ? null + : indexer.ProjectRootForExtraction; + using var regexTimeouts = symbolArtifactCache == null + ? null + : BoundedRegex.CaptureTimeouts( + "csharp", + "symbol_extraction"); extractedByCandidate[candidateIndex] = patternConfigsAlreadyLoaded ? SymbolExtractor.ExtractWithPatternConfigsLoaded( 0, "csharp", content, - target.IndexPath, + extractionFilePath, + extractionProjectRoot, cancellationToken: cancellationToken) : SymbolExtractor.Extract( 0, "csharp", content, - target.IndexPath, + extractionFilePath, + extractionProjectRoot, cancellationToken: cancellationToken); + if (regexTimeouts != null) + { + artifactChecksums![candidateIndex] = checksum; + artifactHadRegexTimeouts![candidateIndex] = + regexTimeouts.HasTimeouts; + } } } } @@ -162,10 +205,23 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( pendingSymbolCount += extracted?.Count ?? 0; var pendingSymbols = new List(pendingSymbolCount); - foreach (var extracted in extractedByCandidate) + for (var candidateIndex = 0; candidateIndex < extractedByCandidate.Length; candidateIndex++) { + var extracted = extractedByCandidate[candidateIndex]; if (extracted != null) + { + var checksum = artifactChecksums?[candidateIndex]; + if (symbolArtifactCache != null && checksum != null) + { + symbolArtifactCache.TryAdmit( + candidates[candidateIndex].IndexPath, + checksum, + extracted, + artifactHadRegexTimeouts![candidateIndex], + cancellationToken); + } pendingSymbols.AddRange(extracted); + } } var hasSourceStaticInterfaceContracts = HasCSharpStaticInterfaceContractSymbol(pendingSymbols); @@ -225,7 +281,8 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( excludedExistingFileIds: null, loadExistingSymbolsOnlyForPendingQualifiedMemberAccess: false, patternConfigsAlreadyLoaded: false, - cancellationToken: cancellationToken); + cancellationToken: cancellationToken, + symbolArtifactCache: null); } internal static bool HasCSharpStaticInterfaceContractSymbol(IEnumerable symbols) diff --git a/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs b/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs index a1b33a84e..3162e5253 100644 --- a/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs +++ b/src/CodeIndex/Indexer/Hooks/PostExtractionHookMutationMaterializer.cs @@ -178,7 +178,7 @@ internal static void RefreshLanguageIdentity(string? language, IEnumerable new() { Id = symbol.Id, diff --git a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs index 704e43969..2c432e20a 100644 --- a/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs +++ b/src/CodeIndex/Indexer/Scanning/FileContentLoader.cs @@ -32,6 +32,10 @@ internal readonly record struct NormalizedIndexableContent( internal int ConflictMarkerLine => Facts.ConflictMarkerLine; } + internal readonly record struct CSharpPrepassCandidateContent( + string Content, + string Checksum); + internal LoadedFileContent Load( string absolutePath, string normalizedRelativePath, @@ -103,12 +107,14 @@ internal string LoadNormalizedContentForPrepass( return NormalizeContentForPrepass(content); } - internal (string? Content, bool RequiresRetry) LoadCSharpStaticInterfaceCandidateContentForPrepass( + internal (CSharpPrepassCandidateContent? Content, bool RequiresRetry) + LoadCSharpStaticInterfaceCandidateContentForPrepass( string absolutePath, string normalizedRelativePath, string relativePath, bool retryOnMutation, bool includeQualifiedMemberAccessCandidate, + bool includeChecksum, CancellationToken cancellationToken) { var readPath = _resolveFileReadPath(absolutePath); @@ -161,8 +167,21 @@ internal string LoadNormalizedContentForPrepass( if (bytes is null || IsGitLfsPointer(bytes)) return (null, RequiresRetry: false); - var (content, _, _, _) = DecodeIndexableContent(bytes, relativePath, inspectRawByteContent: false); - return (NormalizeContentForPrepass(content), RequiresRetry: false); + var (content, warning, inspection, _) = DecodeIndexableContent( + bytes, + relativePath, + inspectRawByteContent: false); + var normalized = NormalizeContentForPrepass(content); + var checksum = includeChecksum + ? warning is null + && !inspection.IsUtf16 + && ReferenceEquals(normalized, content) + ? ComputeRawChecksum(bytes) + : ComputeChecksumFromNormalizedContent(normalized) + : string.Empty; + return ( + new CSharpPrepassCandidateContent(normalized, checksum), + RequiresRetry: false); } internal static string NormalizeLineEndings(string content) diff --git a/src/CodeIndex/Indexer/Scanning/FileIndexer.RecordLoading.cs b/src/CodeIndex/Indexer/Scanning/FileIndexer.RecordLoading.cs index 334e13a65..7cf2c04e1 100644 --- a/src/CodeIndex/Indexer/Scanning/FileIndexer.RecordLoading.cs +++ b/src/CodeIndex/Indexer/Scanning/FileIndexer.RecordLoading.cs @@ -4,6 +4,8 @@ namespace CodeIndex.Indexer; public partial class FileIndexer { + internal string ProjectRootForExtraction => _projectRoot; + /// /// Build a FileRecord and return file content (avoids reading the file twice). /// FileRecordを構築しファイル内容も返す（二重読み込み防止）。 @@ -126,6 +128,33 @@ internal string LoadNormalizedContentForPrepass(string absolutePath, string rela string relativePath, bool includeQualifiedMemberAccessCandidate, CancellationToken cancellationToken = default) + => LoadCSharpStaticInterfaceCandidateContentForPrepassCore( + absolutePath, + relativePath, + includeQualifiedMemberAccessCandidate, + includeChecksum: false, + cancellationToken)?.Content; + + internal FileContentLoader.CSharpPrepassCandidateContent? + LoadCSharpStaticInterfaceCandidateContentWithChecksumForPrepass( + string absolutePath, + string relativePath, + bool includeQualifiedMemberAccessCandidate, + CancellationToken cancellationToken = default) + => LoadCSharpStaticInterfaceCandidateContentForPrepassCore( + absolutePath, + relativePath, + includeQualifiedMemberAccessCandidate, + includeChecksum: true, + cancellationToken); + + private FileContentLoader.CSharpPrepassCandidateContent? + LoadCSharpStaticInterfaceCandidateContentForPrepassCore( + string absolutePath, + string relativePath, + bool includeQualifiedMemberAccessCandidate, + bool includeChecksum, + CancellationToken cancellationToken) { var normalizedRelativePath = NormalizeIndexPath(relativePath); for (var attempt = 0; ; attempt++) @@ -145,6 +174,7 @@ internal string LoadNormalizedContentForPrepass(string absolutePath, string rela relativePath, retryOnMutation: attempt == 0, includeQualifiedMemberAccessCandidate, + includeChecksum, cancellationToken); if (!requiresRetry) return content; diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index de6f9cd70..dab9b3630 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -327,6 +327,9 @@ static string FormatDiagnosticPath(string projectRoot, string path) // Load current reference-language support before the deferred mutation phase. // deferred mutation phase の前に現在の reference-language support を読み込む。 ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(projectPath); + var csharpPrepassSymbolArtifacts = CSharpPrepassSymbolArtifactCache + .CreateForFreshBuiltInExtraction( + startedWithNoIndexedFilesBeforeRebuild && !rebuild); var purgedRefs = 0; // Scan and index / スキャン・インデックス @@ -717,7 +720,8 @@ CSharpStaticInterfaceWorkspaceSymbols BuildStableCSharpWorkspace( excludedExistingFileIds: staleFilePurgePlan.FileIds, isExistingSymbolPathExcluded: IsExistingCSharpSymbolPathNowNonCSharp, patternConfigsAlreadyLoaded: true, - cancellationToken: requestToken)); + cancellationToken: requestToken, + symbolArtifactCache: csharpPrepassSymbolArtifacts)); forceFullCSharpRefreshFromInvalidatedNoOp = indexSnapshot.CSharpStaticInterfaceSourceEvidence == true || csharpWorkspace.HasStaticInterfaceContracts @@ -725,6 +729,8 @@ CSharpStaticInterfaceWorkspaceSymbols BuildStableCSharpWorkspace( } if (!csharpWorkspace.SourceContractEvidenceComplete) { + csharpPrepassSymbolArtifacts?.Clear(); + csharpPrepassSymbolArtifacts = null; incompleteCSharpPrepassPaths = csharpWorkspace.IncompleteSourcePaths ?? []; deferCSharpMutationsForIncompleteScan = true; staleFilePurgePlan = FilePurgePlan.Empty; @@ -804,6 +810,8 @@ void RecordIncompleteCSharpPrepassFailures(IReadOnlyList paths) void DeferCSharpMutationsForLoadedSnapshotDrift(string path) { + csharpPrepassSymbolArtifacts?.Clear(); + csharpPrepassSymbolArtifacts = null; deferCSharpMutationsForIncompleteScan = true; preservePriorPositiveCSharpSourceNoOp = false; csharpSourceEvidenceForStamp = false; @@ -1224,6 +1232,8 @@ await EmitProgressNotificationAsync( } if (!stableFiles) { + csharpPrepassSymbolArtifacts?.Clear(); + csharpPrepassSymbolArtifacts = null; var driftPath = FormatCSharpWorkspaceSnapshotPath(changedFilePath); incompleteCSharpPrepassPaths = [driftPath]; deferCSharpMutationsForIncompleteScan = true; @@ -1514,8 +1524,23 @@ await EmitProgressNotificationAsync( } List symbols; FileIssue? symbolRegexTimeoutIssue; - using (var regexTimeouts = BoundedRegex.CaptureTimeouts(record.Lang, "symbol_extraction")) + if (string.Equals(record.Lang, "csharp", StringComparison.Ordinal) + && record.Checksum is { } checksum + && csharpPrepassSymbolArtifacts?.TryTake( + record.Path, + checksum, + out var symbolArtifact) == true) + { + symbols = symbolArtifact.Symbols; + foreach (var symbol in symbols) + symbol.FileId = fileId; + symbolRegexTimeoutIssue = null; + } + else { + using var regexTimeouts = BoundedRegex.CaptureTimeouts( + record.Lang, + "symbol_extraction"); symbols = SymbolExtractor.ExtractNormalized( fileId, record.Lang, @@ -1526,7 +1551,10 @@ await EmitProgressNotificationAsync( requestToken, loaded.ConflictMarkerLine, patternConfigsAlreadyLoaded: true); - symbolRegexTimeoutIssue = IndexCommandRunner.BuildRegexTimeoutIssue(record.Path, regexTimeouts); + symbolRegexTimeoutIssue = + IndexCommandRunner.BuildRegexTimeoutIssue( + record.Path, + regexTimeouts); } var familyScopeKey = indexer.GetFamilyScopeKey(filePath, record.Lang); SymbolExtractor.ApplyFamilyScope(symbols, familyScopeKey, record.Lang); @@ -1770,6 +1798,9 @@ await EmitProgressNotificationAsync( await EmitProgressNotificationAsync(progressToken, processed, files.Count).ConfigureAwait(false); } + csharpPrepassSymbolArtifacts?.Clear(); + csharpPrepassSymbolArtifacts = null; + var referenceIdentityReadyForMutualRecursionRefresh = !deferCSharpMutationsForIncompleteScan && mutualRecursionRefreshNeeded ? writer.CSharpFamilyTrustAllowsReferenceIdentityReady( diff --git a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs new file mode 100644 index 000000000..9cb2c4faf --- /dev/null +++ b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs @@ -0,0 +1,182 @@ +using System.Reflection; +using CodeIndex.Indexer; +using CodeIndex.Models; + +namespace CodeIndex.Tests; + +public class CSharpPrepassSymbolArtifactCacheTests +{ + [Fact] + public void TryTake_MatchingChecksumOwnsDeepCloneAndIsTakeOnce() + { + var source = CreatePopulatedSymbol(); + var expectedValues = GetSymbolPropertyValues(source); + var cache = new CSharpPrepassSymbolArtifactCache(); + + Assert.True(cache.TryAdmit("src/Cafe\u0301.cs", "checksum-a", [source], hadRegexTimeout: false)); + source.Name = "workspace-mutated"; + source.FileId = 91; + source.FamilyKey = "workspace-family"; + + Assert.True(cache.TryTake("src/Caf\u00E9.cs", "checksum-a", out var artifact)); + var clone = Assert.Single(artifact.Symbols); + Assert.NotSame(source, clone); + foreach (var (property, expected) in expectedValues) + Assert.Equal(expected, property.GetValue(clone)); + + clone.Name = "main-mutated"; + clone.FileId = 123; + clone.FamilyKey = "main-family"; + Assert.Equal("workspace-mutated", source.Name); + Assert.Equal(91, source.FileId); + Assert.Equal("workspace-family", source.FamilyKey); + Assert.False(cache.TryTake("src/Caf\u00E9.cs", "checksum-a", out _)); + } + + [Fact] + public void TryTake_ChecksumMismatchConsumesArtifactAndReportsFallback() + { + var events = new List(); + var previous = CSharpPrepassSymbolArtifactCache.EventForTesting; + try + { + CSharpPrepassSymbolArtifactCache.EventForTesting = events.Add; + var cache = new CSharpPrepassSymbolArtifactCache(); + Assert.True(cache.TryAdmit( + "src/Fixture.cs", + "checksum-a", + [CreatePopulatedSymbol()], + hadRegexTimeout: false)); + + Assert.False(cache.TryTake("src/Fixture.cs", "checksum-b", out _)); + Assert.False(cache.TryTake("src/Fixture.cs", "checksum-a", out _)); + Assert.Equal(0, cache.Count); + Assert.Contains(events, item => + item.Phase == "checksum_mismatch" + && item.Path == "src/Fixture.cs"); + } + finally + { + CSharpPrepassSymbolArtifactCache.EventForTesting = previous; + } + } + + [Fact] + public void TryAdmit_EnforcesFileSymbolAndEstimatedByteCapsWithoutPartialPublish() + { + var symbol = CreatePopulatedSymbol(); + var fileBound = new CSharpPrepassSymbolArtifactCache( + maxFiles: 1, + maxSymbols: 10, + maxEstimatedBytes: 1_000_000); + Assert.True(fileBound.TryAdmit("a.cs", "a", [symbol], hadRegexTimeout: false)); + Assert.False(fileBound.TryAdmit("b.cs", "b", [symbol], hadRegexTimeout: false)); + Assert.Equal(1, fileBound.Count); + + var symbolBound = new CSharpPrepassSymbolArtifactCache( + maxFiles: 10, + maxSymbols: 1, + maxEstimatedBytes: 1_000_000); + Assert.False(symbolBound.TryAdmit( + "symbols.cs", + "a", + [symbol, symbol], + hadRegexTimeout: false)); + Assert.Equal(0, symbolBound.Count); + Assert.Equal(0, symbolBound.AdmittedSymbolCount); + + var byteBound = new CSharpPrepassSymbolArtifactCache( + maxFiles: 10, + maxSymbols: 10, + maxEstimatedBytes: 1); + Assert.False(byteBound.TryAdmit("bytes.cs", "a", [], hadRegexTimeout: false)); + Assert.Equal(0, byteBound.Count); + Assert.Equal(0, byteBound.AdmittedEstimatedBytes); + } + + [Fact] + public void TryAdmit_CancellationDoesNotPublishPartialArtifact() + { + var cache = new CSharpPrepassSymbolArtifactCache(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.ThrowsAny(() => + cache.TryAdmit( + "src/Fixture.cs", + "checksum", + [CreatePopulatedSymbol()], + hadRegexTimeout: false, + cancellationToken: cancellation.Token)); + Assert.Equal(0, cache.Count); + Assert.Equal(0, cache.AdmittedFileCount); + } + + [Fact] + public void TryAdmit_RegexTimeoutDoesNotPublishPartialSymbols() + { + var events = new List(); + var previous = CSharpPrepassSymbolArtifactCache.EventForTesting; + try + { + CSharpPrepassSymbolArtifactCache.EventForTesting = events.Add; + var cache = new CSharpPrepassSymbolArtifactCache(); + + Assert.False(cache.TryAdmit( + "src/Fixture.cs", + "checksum", + [CreatePopulatedSymbol()], + hadRegexTimeout: true)); + + Assert.Equal(0, cache.Count); + Assert.Equal(0, cache.AdmittedSymbolCount); + Assert.Contains(events, item => + item.Phase == "regex_timeout_skipped" + && item.Path == "src/Fixture.cs"); + } + finally + { + CSharpPrepassSymbolArtifactCache.EventForTesting = previous; + } + } + + private static Dictionary GetSymbolPropertyValues( + SymbolRecord symbol) + => typeof(SymbolRecord) + .GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .ToDictionary(property => property, property => property.GetValue(symbol)); + + private static SymbolRecord CreatePopulatedSymbol() + => new() + { + Id = 7, + FileId = 11, + Kind = "function", + SubKind = "method", + Name = "Fixture", + IdentityNameFolded = "fixture-identity", + DisplayNameFolded = "fixture-display", + Line = 3, + StartLine = 2, + StartColumn = 4, + EndLine = 8, + BodyStartLine = 4, + BodyEndLine = 7, + Signature = "public static int Fixture()", + ContainerKind = "class", + ContainerName = "Host", + ContainerQualifiedName = "Demo.Host", + FamilyKey = "Demo+Host", + Visibility = "public", + ReturnType = "int", + IsPartialDeclaration = true, + IsFileLocalDeclaration = true, + IsExplicitFileLocalDeclaration = true, + DeclarationStructureMutatedByHook = true, + DeclarationSemanticScore = 5, + IdentifierStartColumn = 22, + IsMetadataTarget = false, + MetadataTargetSource = SymbolRecord.MetadataTargetSourceExtractor, + SameLineSignatureOccurrenceIndex = 2, + }; +} diff --git a/tests/CodeIndex.Tests/FileIndexerTests.cs b/tests/CodeIndex.Tests/FileIndexerTests.cs index 0af4a382a..f9b93dcb1 100644 --- a/tests/CodeIndex.Tests/FileIndexerTests.cs +++ b/tests/CodeIndex.Tests/FileIndexerTests.cs @@ -2221,6 +2221,64 @@ public void LoadCSharpStaticInterfaceCandidateContentForPrepass_ContractEncoding Assert.True(CSharpStaticInterfacePrepass.MayContainCSharpStaticInterfaceContract(candidateContent!)); } + [Theory] + [InlineData("utf8-normalized")] + [InlineData("utf16-le")] + [InlineData("utf16-be")] + [InlineData("invalid-utf8")] + public void LoadCSharpStaticInterfaceCandidateContentWithChecksumForPrepass_MatchesAuthoritativeLoad( + string encodingName) + { + using var project = TestProjectHelper.CreateTempProjectScope( + "cdidx_csharp_prepass_checksum_encoding"); + const string source = + "\uFEFFpublic interface I { static abstract int M(); }\r\n" + + "\u200Bpublic static class C { }\r\n"; + byte[] bytes = encodingName switch + { + "utf8-normalized" => Encoding.UTF8.GetBytes(source), + "utf16-le" => new UnicodeEncoding( + bigEndian: false, + byteOrderMark: true) + .GetPreamble() + .Concat(new UnicodeEncoding(false, true).GetBytes(source)) + .ToArray(), + "utf16-be" => new UnicodeEncoding( + bigEndian: true, + byteOrderMark: true) + .GetPreamble() + .Concat(new UnicodeEncoding(true, true).GetBytes(source)) + .ToArray(), + "invalid-utf8" => Encoding.UTF8 + .GetBytes(source) + .Concat([(byte)0xFF, (byte)'\n']) + .ToArray(), + _ => throw new ArgumentOutOfRangeException( + nameof(encodingName), + encodingName, + null), + }; + var path = TestProjectHelper.WriteBinaryFile( + project.Root, + "Fixture.cs", + bytes); + var indexer = new FileIndexer(project.Root, ignoreCase: false); + + var prepass = indexer + .LoadCSharpStaticInterfaceCandidateContentWithChecksumForPrepass( + path, + "Fixture.cs", + includeQualifiedMemberAccessCandidate: false); + var authoritative = indexer.BuildLoadedRecordWithRawBytes( + path, + "Fixture.cs", + knownLanguage: "csharp"); + + Assert.NotNull(prepass); + Assert.Equal(authoritative.Content, prepass.Value.Content); + Assert.Equal(authoritative.Record.Checksum, prepass.Value.Checksum); + } + [Fact] public void LoadCSharpStaticInterfaceCandidateContentForPrepass_IndexBlockingNullPreservesBinaryRejection() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 90a59b85d..24114304f 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -1122,6 +1122,10 @@ public void Run_FullScan_RechecksIndexabilityBeforeContentRead() public void Run_FullScan_IncludeSymbolKindKeepsOnlyMatchingSymbols() { var projectRoot = CreateTempProject(); + var previousArtifactHook = + CSharpPrepassSymbolArtifactCache.EventForTesting; + var artifactEvents = + new ConcurrentQueue(); try { File.WriteAllText(Path.Combine(projectRoot, "app.py"), """ @@ -1131,20 +1135,69 @@ class App: def helper(): return App() """); + const string csharpRelativePath = "Cafe\u0301.cs"; + var csharpIndexPath = FileIndexer.NormalizeIndexPath(csharpRelativePath); + var csharpPath = Path.Combine(projectRoot, csharpRelativePath); + const string csharpSource = """ + namespace Demo; + file partial class Fixture + { + public static int RemovedByFilter() => 1; + } + """; + File.WriteAllText(csharpPath, csharpSource); + var indexer = new FileIndexer(projectRoot, ignoreCase: false); + var expectedCSharpSymbols = SymbolExtractor.Extract( + 0, + "csharp", + csharpSource, + csharpPath, + projectRoot); + SymbolExtractor.ApplyFamilyScope( + expectedCSharpSymbols, + indexer.GetFamilyScopeKey(csharpPath, "csharp"), + "csharp"); + var expectedFamilyKey = Assert.Single( + expectedCSharpSymbols, + symbol => symbol.Kind == "class" + && symbol.Name == "Fixture").FamilyKey; + CSharpPrepassSymbolArtifactCache.EventForTesting = + artifactEvents.Enqueue; - var (exitCode, json) = RunAndCaptureJson([projectRoot, "--include-symbol-kind", "class", "--json"]); + var (exitCode, json) = RunAndCaptureJson( + [projectRoot, "--include-symbol-kind", "class", "--parallelism", "1", "--json"]); Assert.Equal(CommandExitCodes.Success, exitCode); Assert.Equal("success", json.GetProperty("status").GetString()); Assert.True(json.GetProperty("summary").GetProperty("symbols_dropped_by_kind_filter").GetInt32() > 0); + Assert.Contains( + artifactEvents, + item => item.Phase == "taken" + && item.Path == csharpIndexPath); - var counts = ReadSymbolKindCounts(Path.Combine(projectRoot, ".cdidx", "codeindex.db")); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + var counts = ReadSymbolKindCounts(dbPath); Assert.True(counts.GetValueOrDefault("class") > 0); Assert.DoesNotContain(counts.Keys, kind => !string.Equals(kind, "class", StringComparison.OrdinalIgnoreCase)); + using var connection = OpenNonPoolingConnection(dbPath); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT family_key + FROM symbols + WHERE file_id = (SELECT id FROM files WHERE path = $path) + AND kind = 'class' + AND name = 'Fixture' + """; + command.Parameters.AddWithValue("$path", csharpIndexPath); + Assert.Equal(expectedFamilyKey, command.ExecuteScalar()); } finally { + CSharpPrepassSymbolArtifactCache.EventForTesting = + previousArtifactHook; DeleteDirectory(projectRoot); + SqliteConnection.ClearAllPools(); } } @@ -2131,11 +2184,17 @@ public void Run_FullScan_ParallelCsharpStaticInterfacePrepass_IndexesImplicitImp var previousLookupHook = ReferenceExtractor.CSharpStaticInterfaceMemberLookupsBuiltForTesting; var previousPrepassHook = IndexCommandRunner.FullScanCSharpPrepassForTesting; var previousContentLoadHook = IndexCommandRunner.FullScanFileContentLoadForTesting; + var previousArtifactHook = + CSharpPrepassSymbolArtifactCache.EventForTesting; + var artifactEvents = + new ConcurrentQueue(); var matchingLookupBuilds = 0; var noOpPrepassCount = 0; var noOpContentLoadCount = 0; try { + CSharpPrepassSymbolArtifactCache.EventForTesting = + artifactEvents.Enqueue; ReferenceExtractor.CSharpStaticInterfaceMemberLookupsBuiltForTesting = symbols => { if (symbols.Any(symbol => @@ -2171,6 +2230,16 @@ public interface IPrecomputedLookupFixture [projectRoot, "--parallelism", "4", "--json", "--quiet"], _jsonOptions); Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal( + implementationFileCount + 1, + artifactEvents.Count(item => item.Phase == "admitted")); + Assert.Equal( + implementationFileCount + 1, + artifactEvents.Count(item => item.Phase == "taken")); + Assert.Contains( + artifactEvents, + item => item.Phase == "cleared"); + artifactEvents.Clear(); var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); InstallCSharpEvidenceWriteAudit(dbPath); @@ -2182,6 +2251,7 @@ public interface IPrecomputedLookupFixture Assert.Equal(CommandExitCodes.Success, noOpExitCode); Assert.Equal(0, noOpPrepassCount); Assert.Equal(0, noOpContentLoadCount); + Assert.Empty(artifactEvents); Assert.Equal(0L, CountCSharpEvidenceWrites(dbPath)); using var conn = OpenNonPoolingConnection(dbPath); @@ -2243,6 +2313,160 @@ static long CountCSharpEvidenceWrites(string path) ReferenceExtractor.CSharpStaticInterfaceMemberLookupsBuiltForTesting = previousLookupHook; IndexCommandRunner.FullScanCSharpPrepassForTesting = previousPrepassHook; IndexCommandRunner.FullScanFileContentLoadForTesting = previousContentLoadHook; + CSharpPrepassSymbolArtifactCache.EventForTesting = + previousArtifactHook; + DeleteDirectory(projectRoot); + SqliteConnection.ClearAllPools(); + } + } + + [Fact] + public void Run_FreshFullScan_CSharpPrepassArtifactChecksumMismatchFallsBackToAuthoritativeMainRead() + { + var projectRoot = CreateTempProject(); + var sourcePath = Path.Combine(projectRoot, "Fixture.cs"); + const string originalSource = + "public class Fixture { public static int Alpha() => 1; }\n"; + const string mutatedSource = + "public class Fixture { public static int Bravo() => 1; }\n"; + Assert.Equal( + Encoding.UTF8.GetByteCount(originalSource), + Encoding.UTF8.GetByteCount(mutatedSource)); + File.WriteAllText(sourcePath, originalSource); + var originalModified = File.GetLastWriteTimeUtc(sourcePath); + var previousContentLoadHook = + IndexCommandRunner.FullScanFileContentLoadForTesting; + var previousArtifactHook = + CSharpPrepassSymbolArtifactCache.EventForTesting; + var artifactEvents = + new ConcurrentQueue(); + var mutated = 0; + try + { + CSharpPrepassSymbolArtifactCache.EventForTesting = + artifactEvents.Enqueue; + IndexCommandRunner.FullScanFileContentLoadForTesting = path => + { + if (path != "Fixture.cs" + || Interlocked.Exchange(ref mutated, 1) != 0) + { + return; + } + + File.WriteAllText(sourcePath, mutatedSource); + File.SetLastWriteTimeUtc(sourcePath, originalModified); + }; + + var exitCode = IndexCommandRunner.Run( + [projectRoot, "--parallelism", "2", "--json", "--quiet"], + _jsonOptions); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(1, Volatile.Read(ref mutated)); + Assert.Contains( + artifactEvents, + item => item.Phase == "admitted" + && item.Path == "Fixture.cs"); + Assert.Contains( + artifactEvents, + item => item.Phase == "checksum_mismatch" + && item.Path == "Fixture.cs"); + Assert.DoesNotContain( + artifactEvents, + item => item.Phase == "taken" + && item.Path == "Fixture.cs"); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using var connection = OpenNonPoolingConnection(dbPath); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT name + FROM symbols + WHERE file_id = (SELECT id FROM files WHERE path = 'Fixture.cs') + """; + using var reader = command.ExecuteReader(); + var names = new List(); + while (reader.Read()) + names.Add(reader.GetString(0)); + Assert.Contains("Bravo", names); + Assert.DoesNotContain("Alpha", names); + } + finally + { + IndexCommandRunner.FullScanFileContentLoadForTesting = + previousContentLoadHook; + CSharpPrepassSymbolArtifactCache.EventForTesting = + previousArtifactHook; + DeleteDirectory(projectRoot); + SqliteConnection.ClearAllPools(); + } + } + + [Theory] + [InlineData("rebuild")] + [InlineData("symbols-only")] + [InlineData("stall-seam")] + public void Run_CSharpPrepassArtifactReuse_IsDisabledOutsideFreshFullIndex( + string mode) + { + var projectRoot = CreateTempProject(); + var sourcePath = Path.Combine(projectRoot, "Fixture.cs"); + File.WriteAllText( + sourcePath, + "public class Fixture { public static int Run() => 1; }\n"); + var previousArtifactHook = + CSharpPrepassSymbolArtifactCache.EventForTesting; + var previousStallTimeout = + IndexCommandRunner.IndexExtractionStallTimeoutForTesting; + var artifactEvents = + new ConcurrentQueue(); + try + { + CSharpPrepassSymbolArtifactCache.EventForTesting = + artifactEvents.Enqueue; + if (mode == "stall-seam") + { + IndexCommandRunner.IndexExtractionStallTimeoutForTesting = + () => TimeSpan.FromMinutes(1); + } + var args = new List + { + projectRoot, + "--json", + "--quiet", + }; + if (mode == "rebuild") + { + args.Add("--rebuild"); + args.Add("--yes"); + } + else if (mode == "symbols-only") + args.Add("--symbols-only"); + + var exitCode = IndexCommandRunner.Run(args.ToArray(), _jsonOptions); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.DoesNotContain( + artifactEvents, + item => item.Phase is "admitted" or "taken"); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using var connection = OpenNonPoolingConnection(dbPath); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT COUNT(*) + FROM symbols + WHERE name = 'Fixture' + """; + Assert.Equal(1L, command.ExecuteScalar()); + } + finally + { + CSharpPrepassSymbolArtifactCache.EventForTesting = + previousArtifactHook; + IndexCommandRunner.IndexExtractionStallTimeoutForTesting = + previousStallTimeout; DeleteDirectory(projectRoot); SqliteConnection.ClearAllPools(); } diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 76fa39197..31d216462 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Text; using System.Text.Json.Nodes; using System.Text.Json; @@ -10100,6 +10101,10 @@ public void ToolsCall_Index_DeletedCsharpStaticInterfaceContractDoesNotRegenerat var previousLookupHook = ReferenceExtractor.CSharpStaticInterfaceMemberLookupsBuiltForTesting; var previousCSharpPrepassHook = McpServer.McpIndexCSharpPrepassForTesting; var previousContentLoadHook = McpServer.McpIndexFileContentLoadForTesting; + var previousArtifactHook = + CSharpPrepassSymbolArtifactCache.EventForTesting; + var artifactEvents = + new ConcurrentQueue(); using var extensionProject = TestProjectHelper.CreateExecutableExtensionTestProjectScope( "cdidx_mcp_csharp_source_evidence_hook"); using var env = EnvironmentVariableScope.Capture( @@ -10110,6 +10115,8 @@ public void ToolsCall_Index_DeletedCsharpStaticInterfaceContractDoesNotRegenerat var noOpContentLoadCount = 0; try { + CSharpPrepassSymbolArtifactCache.EventForTesting = + artifactEvents.Enqueue; DbWriter.CSharpContractPreflightForTesting = () => { preflightCount++; @@ -10155,6 +10162,16 @@ public interface IParseable using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); var initialResponse = CallIndex(server, fixtureDir); Assert.False(initialResponse["result"]?["isError"]?.GetValue() ?? false, initialResponse.ToJsonString()); + Assert.Equal( + 2, + artifactEvents.Count(item => item.Phase == "admitted")); + Assert.Equal( + 2, + artifactEvents.Count(item => item.Phase == "taken")); + Assert.Contains( + artifactEvents, + item => item.Phase == "cleared"); + artifactEvents.Clear(); Assert.Equal(1, matchingLookupBuilds); Assert.Equal(0L, CountPersistedContractMembers()); Assert.True(ReadSourceEvidence()); @@ -10173,6 +10190,7 @@ public interface IParseable Assert.Equal(1, matchingLookupBuilds); Assert.Equal(0, noOpCSharpPrepassCount); Assert.Equal(0, noOpContentLoadCount); + Assert.Empty(artifactEvents); Assert.Equal(0L, CountCSharpEvidenceWrites()); var noOpSummary = noOpResponse["result"]!["structuredContent"]!["summary"]!; Assert.Equal(2L, noOpSummary["files"]!.GetValue()); @@ -10278,6 +10296,8 @@ long CountCSharpEvidenceWrites() ReferenceExtractor.CSharpStaticInterfaceMemberLookupsBuiltForTesting = previousLookupHook; McpServer.McpIndexCSharpPrepassForTesting = previousCSharpPrepassHook; McpServer.McpIndexFileContentLoadForTesting = previousContentLoadHook; + CSharpPrepassSymbolArtifactCache.EventForTesting = + previousArtifactHook; GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); @@ -13156,12 +13176,18 @@ public void ToolsCall_Index_Rebuild_SucceedsOnFreshDb() var fixtureDir = Path.Combine(Path.GetFullPath("."), $"mcp_index_rebuild_fresh_{Guid.NewGuid():N}"); Directory.CreateDirectory(fixtureDir); var dbPath = TestProjectHelper.CreateTempDbPath("cdidx_mcp_index_rebuild_fresh"); + var previousArtifactHook = + CSharpPrepassSymbolArtifactCache.EventForTesting; + var artifactEvents = + new ConcurrentQueue(); var statSnapshotReads = 0; try { File.WriteAllText(Path.Combine(fixtureDir, "app.cs"), "public class App { }"); using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); DbWriter.ReusableStatSnapshotReadForTesting = () => statSnapshotReads++; + CSharpPrepassSymbolArtifactCache.EventForTesting = + artifactEvents.Enqueue; var request = new JsonObject { @@ -13183,10 +13209,15 @@ public void ToolsCall_Index_Rebuild_SucceedsOnFreshDb() Assert.False(response["result"]!["isError"]?.GetValue() ?? false); Assert.True(response["result"]!["structuredContent"]!["summary"]!["files"]!.GetValue() >= 1L); Assert.Equal(0, statSnapshotReads); + Assert.DoesNotContain( + artifactEvents, + item => item.Phase is "admitted" or "taken"); } finally { DbWriter.ReusableStatSnapshotReadForTesting = null; + CSharpPrepassSymbolArtifactCache.EventForTesting = + previousArtifactHook; TestProjectHelper.DeleteSqliteDatabaseFiles(dbPath); TestProjectHelper.DeleteDirectory(fixtureDir); } From 3b2b398edaf93317c72ccffeb18b822c3ee9aed3 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 09:39:16 +0900 Subject: [PATCH 08/16] Transfer C# prepass symbol ownership --- DEVELOPER_GUIDE.md | 24 +- TESTING_GUIDE.md | 4 +- .../+large-codebase-initial-indexing.fixed.md | 4 +- .../CSharpPrepassSymbolArtifactCache.cs | 48 ++- .../Indexer/CSharpStaticInterfacePrepass.cs | 51 +++- .../CSharpPrepassSymbolArtifactCacheTests.cs | 279 ++++++++++++++++++ 6 files changed, 377 insertions(+), 33 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 781692420..b8d1c1498 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -421,10 +421,15 @@ rebuilds and symbols-only runs; MCP evaluates the same empty-database condition before any rebuild mutation. The cache never owns source text or bytes: the main pass still performs its authoritative content read and hook, stat-snapshot and TOCTOU validation, and checksum calculation. It consumes a normalized-path -artifact once only after that checksum matches. Cached symbols are deep clones. -Artifact-producing extraction receives the main pass's absolute file path and -project root so file-local family identities stay identical. File ID assignment, -family scope, source observation, post-extraction hooks, +artifact once only after that checksum matches. Generic cache admission keeps +deep-clone isolation for direct callers. The fresh-index production path instead +materializes both workspace lookup snapshots first, then transfers ownership of +each admitted per-file symbol list and releases the redundant workspace-symbol +fallback list. Main-pass mutation therefore remains isolated from the lookup +snapshots without retaining duplicate `SymbolRecord` objects. Artifact-producing +extraction receives the main pass's absolute file path and project root so +file-local family identities stay identical. File ID assignment, family scope, +source observation, post-extraction hooks, kind filtering, caps, line validation, persistence, reference extraction, and bounded-regex issue reporting remain on the normal main-pass path. Incomplete prepasses, extraction-stall test seams, checksum drift, regex timeouts, and cache @@ -4176,10 +4181,13 @@ static-interface prepass の raw built-in C# symbol artifact を再利用でき 0件の状態から開始した初回 full index だけです。CLI は rebuild と symbols-only を除外し、MCP は rebuild mutation より前の空 database 条件を使います。cache は source text / byte を保持せず、main pass は引き続き authoritative な content read と hook、stat snapshot / TOCTOU 検証、checksum 計算を -実行します。正規化 path の artifact は checksum 一致後に1回だけ取り出します。cached symbol は deep -clone とします。artifact を生成する extraction には main pass と同じ absolute file path / project root を -渡し、file-local family identity を一致させてください。FileId、family scope、source observation、 -post-extraction hook、kind filter、cap、line 検証、 +実行します。正規化 path の artifact は checksum 一致後に1回だけ取り出します。汎用 cache admission は +direct caller 向けの deep-clone isolation を維持します。fresh-index の production 経路では、先に2種類の +workspace lookup snapshot を materialize し、その後で admit した file ごとの symbol list の所有権を +cache へ移し、重複する workspace-symbol fallback list を解放します。これにより main-pass mutation と +lookup snapshot の分離を保ったまま、重複する `SymbolRecord` object を保持しません。artifact を生成する +extraction には main pass と同じ absolute file path / project root を渡し、file-local family identity を +一致させてください。FileId、family scope、source observation、post-extraction hook、kind filter、cap、line 検証、 persistence、reference extraction、bounded-regex issue は通常の main-pass 経路で処理してください。 不完全な prepass、extraction-stall test seam、checksum drift、regex timeout、cache 上限では通常 extraction へ fallback します。timeout した prepass 結果は partial であり、一過性の結果を diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index fe79ff5fd..a1135bbb5 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -427,7 +427,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result cover multi-frame remainder reuse, CRLF split across a 4 KiB read boundary, Unicode bytes, an unterminated final frame, stable EOF, and rejection at one byte over the negotiated cap. They protect direct worker-response deserialization without constructing decoded JSON strings. - `IndexCommandRunnerTests`, `FileIndexerTests`, and `PerformanceTests` also cover `CSharpStaticInterfacePrepass` text, raw-byte, chunked raw-token, and streaming file contract probes. Stable candidate reads must authorize and open each file once, keep raw-negative reads to one bounded pass, and rewind that same handle only for a raw-positive full decode. A detected in-place or atomic-replacement mutation must discard that snapshot and reauthorize/reopen once so the prepass cannot diverge from the main indexing pass. Preserve UTF-8 / UTF-16, NUL rejection, growth, cancellation, and lexical-boundary behavior. The 576 KiB semantic-negative/positive allocation guard runs each probe 12 times and stays below 4 KiB of current-thread allocation so a whole-content mask cannot return. - The parallel C# static-interface full-scan fixture uses 64 implementation files and treats one workspace lookup build as a performance contract. Keep the contract lookup attached to the immutable prepass snapshot across CLI full scan, scoped update, and MCP indexing; do not rebuild it once per C# file. -- `CSharpPrepassSymbolArtifactCacheTests`, `FileIndexerTests`, and the CLI/MCP fresh-index fixtures protect bounded prepass artifact reuse. Keep deep-clone independence, take-once checksum matching, mismatch consumption, atomic file/symbol/estimated-byte caps, cancellation without partial admission, and non-admission of partial symbols after any bounded-regex timeout. Encoding theories must compare UTF-8, UTF-16 LE/BE, and invalid-UTF-8 prepass checksums with the authoritative loader. Integration coverage must prove reuse only for an empty non-rebuild full index, ordinary extraction for rebuild/symbols-only/existing/incomplete-or-stall paths, authoritative main-read mutation fallback, unchanged post hooks and family/kind processing, and cache clearing before graph work. +- `CSharpPrepassSymbolArtifactCacheTests`, `FileIndexerTests`, and the CLI/MCP fresh-index fixtures protect bounded prepass artifact reuse. Keep deep-clone independence for generic admission; production owned-list admission must retain list/symbol identity only after successful atomic publication, leave rejected or cancelled input caller-owned, and release workspace fallback symbols only after both lookup snapshots are materialized. Preserve lookup parity and mutation isolation, take-once checksum matching, mismatch consumption, atomic file/symbol/estimated-byte caps, cancellation without partial admission, and non-admission of partial symbols after any bounded-regex timeout. Encoding theories must compare UTF-8, UTF-16 LE/BE, and invalid-UTF-8 prepass checksums with the authoritative loader. Integration coverage must prove reuse only for an empty non-rebuild full index, ordinary extraction for rebuild/symbols-only/existing/incomplete-or-stall paths, authoritative main-read mutation fallback, unchanged post hooks and family/kind processing, and cache clearing before graph work. - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` builds 20,000 unrelated generic interfaces around one contract and keeps lookup allocation below 64 KiB. Discover contract containers before parsing generic declarations, while the paired `ReferenceExtractorTests` case preserves declaration/member ordering, partial-interface last-write behavior, and contract member order. - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` keeps a 1,200-statement, regex-and-brace-heavy source at one emitted symbol and pins current-thread allocations below 6.0 MB for JavaScript, 8.2 MB for TypeScript, and 14.1 MB combined. This is a physical-pass contract: do not restore a second full-file sanitizer for private-scope analysis. Pair allocation changes with the JavaScript/TypeScript literal-and-comment scope theory so templates, comments, and regex braces cannot create or leak private classes. - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` keeps persisted C# contract-member candidates and the pending-path contract flag on one `files(lang)` → `symbols(file_id, kind)` row pass. Derive both results from that reader; do not restore a second all-contract query for excluded paths. Interface declarations may be loaded only for exact retained contract container names through bounded `symbols(name)` batches, and those dynamic tail shapes must stay out of the prepared cache. `DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` pins both index plans, while `LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` requires negative/LIKE-decoy-only reads to execute no interface phase and keeps unrelated plain interfaces out of managed materialization. `PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` also splits a 503-path language lookup at 500 parameters, returns only persisted C# paths plus sorted purge-plan IDs/bytes, keeps both SQL tail shapes out of the prepared cache, and stops before the second batch when cancellation is requested. @@ -1441,7 +1441,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" multi-frame remainder の再利用、4 KiB read 境界をまたぐ CRLF、Unicode byte、改行なし最終 frame、安定した EOF、合意済み上限を1 byte 超えた時点での拒否を検証します。decode 済み JSON string を作らず worker response を直接 deserialize する経路を固定します。 - `IndexCommandRunnerTests`、`FileIndexerTests`、`PerformanceTests` は `CSharpStaticInterfacePrepass` のテキスト判定、raw-byte、chunked raw-token、streaming file 契約 probe も扱います。安定した候補読み取りは各 file を1回だけ認可・openし、raw-negative は bounded pass 1回に留め、raw-positive の full decode だけ同じ handle を rewind してください。in-place mutation または atomic replacement を検知した場合は snapshot を破棄し、prepass と main indexing pass が乖離しないよう1回だけ再認可・再openします。UTF-8 / UTF-16、NUL 拒否、growth、cancellation、lexical boundary を維持してください。576 KiB の semantic-negative/positive allocation guard は各 probe を12回実行して current-thread allocation を4 KiB未満に保ち、content 全体 mask の再導入を防ぎます。 - parallel C# static-interface full-scan fixture は64個のimplementation fileを使い、workspace lookup buildが1回であることをperformance contractとします。CLI full scan、scoped update、MCP indexingを横断してcontract lookupをimmutable prepass snapshotに保持し、C# fileごとの再構築を戻さないでください。 -- `CSharpPrepassSymbolArtifactCacheTests`、`FileIndexerTests`、CLI/MCP の fresh-index fixture は bounded prepass artifact reuse を固定します。deep-clone の独立性、checksum 一致時の take-once、不一致時の消費、file / symbol / estimated-byte cap の原子性、partial admission を残さない cancellation、bounded-regex timeout 後の partial symbol をadmitしない契約を維持してください。encoding theory は UTF-8、UTF-16 LE/BE、不正 UTF-8 の prepass checksum を authoritative loader と比較します。integration coverage では空 database の非 rebuild full index だけが再利用し、rebuild / symbols-only / existing / incomplete-or-stall 経路は通常 extraction、main read 中の mutation は checksum fallback、post hook と family/kind 処理は従来どおり、graph 開始前に cache が clear されることを証明してください。 +- `CSharpPrepassSymbolArtifactCacheTests`、`FileIndexerTests`、CLI/MCP の fresh-index fixture は bounded prepass artifact reuse を固定します。汎用 admission の deep-clone 独立性を維持し、production の owned-list admission は原子的な publish 成功後だけ list / symbol identity を保持し、reject または cancel された input は caller-owned のままにしてください。2種類の lookup snapshot を materialize した後だけ workspace fallback symbol を解放し、lookup parity と mutation isolation を保ちます。checksum 一致時の take-once、不一致時の消費、file / symbol / estimated-byte cap の原子性、partial admission を残さない cancellation、bounded-regex timeout 後の partial symbol をadmitしない契約も維持してください。encoding theory は UTF-8、UTF-16 LE/BE、不正 UTF-8 の prepass checksum を authoritative loader と比較します。integration coverage では空 database の非 rebuild full index だけが再利用し、rebuild / symbols-only / existing / incomplete-or-stall 経路は通常 extraction、main read 中の mutation は checksum fallback、post hook と family/kind 処理は従来どおり、graph 開始前に cache が clear されることを証明してください。 - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` は、1件のcontractの周囲に20,000件の無関係なgeneric interfaceを構築し、lookup allocationを64 KiB未満に固定します。generic宣言を解析する前にcontract containerを検出し、対になる`ReferenceExtractorTests`で宣言/member順、partial interfaceの後勝ち、contract member順を維持してください。 - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` は、regex と brace が密な1,200 statement の source を出力 symbol 1件に保ち、current-thread allocation を JavaScript 6.0 MB未満、TypeScript 8.2 MB未満、合計14.1 MB未満に固定します。これは physical-pass 契約であり、private-scope 解析専用の2回目の全file sanitizerを戻さないでください。allocation変更時はJavaScript/TypeScriptのliteral/comment scope theoryも対にし、template、comment、regexのbraceがprivate classを生成またはscope外へ漏らさないことを維持してください。 - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` は、永続化済み C# contract member 候補と pending-path contract flag を1回の `files(lang)` → `symbols(file_id, kind)` row passで得ることを固定します。両方を同じ reader から導出し、除外 path 判定のための2回目の全 contract query を戻さないでください。interface 宣言は厳密検証後に保持した contract container 名だけを bounded な `symbols(name)` batch で取得でき、その dynamic tail shape は prepared cache に残してはいけません。`DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` は両方の index plan を固定し、`LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` は negative / LIKE decoy だけの読込で interface phase が0回となり、無関係な通常 interface を managed materialization へ入れないことを要求します。`PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` は503件のpath言語lookupを500 parameterで分割し、永続C# pathと昇順のpurge-plan ID/byteだけを返し、両方のSQL tail shapeをprepared cache外に保ち、1つ目のbatch後のcancellationで2つ目を実行しないことも固定します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index f8ec6332a..77ec71bf1 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -34,7 +34,7 @@ affected: - **First-time full indexes resolve only references that have candidates** — empty-database CLI scans persist canonical unresolved defaults during bulk insertion, aggregate the candidate table once, and update candidate-bearing references by primary key instead of probing every reference row. Rebuilds, updates, retained graphs, and MCP indexing keep their existing recovery contracts. - **Fresh graph planning uses post-load cardinalities** — truly empty CLI and MCP bulk loads now analyze the populated file, symbol, and reference tables immediately before candidate resolution, allowing SQLite to plan the expensive first graph build from current statistics. Rebuilds, updates, existing databases, and symbols-only runs keep their prior lifecycle; cancellation still aborts, while a statistics-only SQLite failure rolls back its savepoint and continues best-effort. - **Persistent symbol workers reuse bounded pattern-directory snapshots across languages** — each project-root reload discovers user and root configs once, while nested ancestor directories, including missing and rejected results, are observed once per worker command. The cache follows live filesystem casing and falls back to uncached discovery when full so configs are never skipped; direct registry callers keep dynamic discovery. -- **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once deep clones of built-in symbols already extracted for the static-interface workspace. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. +- **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once built-in symbols already extracted for the static-interface workspace. After materializing the immutable lookup snapshots, the prepass transfers ownership of admitted per-file symbol lists and releases the redundant workspace fallback objects instead of cloning the full symbol graph. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. ## 日本語 @@ -44,4 +44,4 @@ affected: - **初回full indexでcandidateを持つreferenceだけを解決するようにしました** — 空databaseからのCLI scanではbulk insert中にcanonicalな未解決値を保存し、candidate tableを1回集約して、全reference rowをprobeせずcandidateを持つrowだけをprimary keyで更新します。rebuild、update、retained graph、MCP indexingの既存recovery契約は変更しません。 - **新規graph planningでbulk load後のcardinalityを使うようにしました** — 真に空のdatabaseから始まるCLI / MCP bulk loadでは、candidate解決の直前に投入済みのfile、symbol、reference tableを解析し、SQLiteが初回の高コストなgraph構築を最新統計から計画できるようにします。rebuild、update、既存database、symbols-onlyのlifecycleは従来どおりです。cancellationは引き続き中断し、統計更新だけのSQLite failureはsavepointを戻してbest-effortで継続します。 - **persistent symbol worker が全言語で上限付き pattern-directory snapshot を再利用するようにしました** — project-root reload ごとに user / root config を1回だけ探索し、missing や reject を含む nested ancestor directory は worker command ごとに初回結果を再利用します。cache は実 filesystem の case policy に従い、飽和時は uncached discovery に fallback して config を skip しません。registry の direct caller は従来どおり動的に探索します。 -- **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once の deep clone として利用できます。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 +- **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once で利用できます。immutable な lookup snapshot を materialize した後、prepass は admit した file ごとの symbol list の所有権を移し、symbol graph 全体を clone せず重複する workspace fallback object を解放します。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 diff --git a/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs b/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs index 3d346517c..9df0f85df 100644 --- a/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs +++ b/src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs @@ -61,6 +61,39 @@ internal bool TryAdmit( IReadOnlyList symbols, bool hadRegexTimeout, CancellationToken cancellationToken = default) + => TryAdmitCore( + path, + checksum, + symbols, + ownedSymbols: null, + hadRegexTimeout, + cancellationToken); + + // A successful publish transfers the list and every mutable record to the cache. + // Publish failure leaves ownership with the caller. + // publish 成功時だけ list と mutable record の所有権を cache へ移す。 + // failure 時は caller が所有権を保持する。 + internal bool TryAdmitOwned( + string path, + string checksum, + List symbols, + bool hadRegexTimeout, + CancellationToken cancellationToken = default) + => TryAdmitCore( + path, + checksum, + symbols, + symbols, + hadRegexTimeout, + cancellationToken); + + private bool TryAdmitCore( + string path, + string checksum, + IReadOnlyList symbols, + List? ownedSymbols, + bool hadRegexTimeout, + CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(path); ArgumentNullException.ThrowIfNull(checksum); @@ -88,17 +121,22 @@ internal bool TryAdmit( return false; } - var clonedSymbols = new List(symbols.Count); - foreach (var symbol in symbols) + var artifactSymbols = ownedSymbols; + if (artifactSymbols == null) { - cancellationToken.ThrowIfCancellationRequested(); - clonedSymbols.Add(PostExtractionHookMutationMaterializer.CloneSymbol(symbol)); + artifactSymbols = new List(symbols.Count); + foreach (var symbol in symbols) + { + cancellationToken.ThrowIfCancellationRequested(); + artifactSymbols.Add( + PostExtractionHookMutationMaterializer.CloneSymbol(symbol)); + } } var artifact = new CSharpPrepassSymbolArtifact( normalizedPath, checksum, - clonedSymbols); + artifactSymbols); if (!_artifacts.TryAdd(normalizedPath, artifact)) return false; diff --git a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs index aaedbcc22..845d22ef9 100644 --- a/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs +++ b/src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs @@ -209,19 +209,7 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( { var extracted = extractedByCandidate[candidateIndex]; if (extracted != null) - { - var checksum = artifactChecksums?[candidateIndex]; - if (symbolArtifactCache != null && checksum != null) - { - symbolArtifactCache.TryAdmit( - candidates[candidateIndex].IndexPath, - checksum, - extracted, - artifactHadRegexTimeouts![candidateIndex], - cancellationToken); - } pendingSymbols.AddRange(extracted); - } } var hasSourceStaticInterfaceContracts = HasCSharpStaticInterfaceContractSymbol(pendingSymbols); @@ -248,14 +236,45 @@ internal static CSharpStaticInterfaceWorkspaceSymbols BuildWorkspaceSymbols( IReadOnlyList incompletePaths = firstIncompleteSourcePath == null ? [] : [firstIncompleteSourcePath]; + var isSourceEvidenceComplete = sourceEvidenceComplete != 0; + var staticInterfaceMemberLookups = + ReferenceExtractor.BuildCSharpStaticInterfaceMemberLookups(symbols); + var qualifiedPatternLookups = + ReferenceExtractor.BuildCSharpQualifiedPatternLookups(symbols); + // Materialize every immutable workspace lookup before transferring the raw + // per-file lists. The main pass may mutate owned symbols after take, while + // reference extraction must continue to observe this prepass snapshot. + // raw list の所有権移譲前に lookup を確定し、main pass の mutation から + // prepass snapshot を分離する。 + if (symbolArtifactCache != null) + { + for (var candidateIndex = 0; + candidateIndex < extractedByCandidate.Length; + candidateIndex++) + { + var extracted = extractedByCandidate[candidateIndex]; + var checksum = artifactChecksums![candidateIndex]; + if (extracted == null || checksum == null) + continue; + + symbolArtifactCache.TryAdmitOwned( + candidates[candidateIndex].IndexPath, + checksum, + extracted, + artifactHadRegexTimeouts![candidateIndex], + cancellationToken); + } + } + IReadOnlyList referenceFallbackSymbols = + symbolArtifactCache == null ? symbols : []; return new CSharpStaticInterfaceWorkspaceSymbols( - symbols, + referenceFallbackSymbols, hasStaticInterfaceContracts, - ReferenceExtractor.BuildCSharpStaticInterfaceMemberLookups(symbols), + staticInterfaceMemberLookups, hasSourceStaticInterfaceContracts, - sourceEvidenceComplete != 0, + isSourceEvidenceComplete, incompletePaths, - ReferenceExtractor.BuildCSharpQualifiedPatternLookups(symbols), + qualifiedPatternLookups, requiresMemberReadReferenceRefresh); } diff --git a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs index 9cb2c4faf..8bafe7620 100644 --- a/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs +++ b/tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs @@ -1,4 +1,5 @@ using System.Reflection; +using CodeIndex.Database; using CodeIndex.Indexer; using CodeIndex.Models; @@ -33,6 +34,216 @@ public void TryTake_MatchingChecksumOwnsDeepCloneAndIsTakeOnce() Assert.False(cache.TryTake("src/Caf\u00E9.cs", "checksum-a", out _)); } + [Fact] + public void TryAdmitOwned_TransfersIdentityOnlyAfterSuccessfulAtomicAdmission() + { + var source = CreatePopulatedSymbol(); + var ownedSymbols = new List { source }; + var cache = new CSharpPrepassSymbolArtifactCache(); + + Assert.True(cache.TryAdmitOwned( + "src/Fixture.cs", + "checksum", + ownedSymbols, + hadRegexTimeout: false)); + Assert.True(cache.TryTake("src/Fixture.cs", "checksum", out var artifact)); + Assert.Same(ownedSymbols, artifact.Symbols); + Assert.Same(source, Assert.Single(artifact.Symbols)); + Assert.False(cache.TryTake("src/Fixture.cs", "checksum", out _)); + + var rejectedSymbols = new List { CreatePopulatedSymbol() }; + var capped = new CSharpPrepassSymbolArtifactCache( + maxFiles: 1, + maxSymbols: 1, + maxEstimatedBytes: 1); + Assert.False(capped.TryAdmitOwned( + "src/Rejected.cs", + "checksum", + rejectedSymbols, + hadRegexTimeout: false)); + Assert.Equal(0, capped.Count); + Assert.False(capped.TryTake("src/Rejected.cs", "checksum", out _)); + Assert.Same(rejectedSymbols[0], Assert.Single(rejectedSymbols)); + + var timeoutSymbols = new List { CreatePopulatedSymbol() }; + Assert.False(cache.TryAdmitOwned( + "src/Timeout.cs", + "checksum", + timeoutSymbols, + hadRegexTimeout: true)); + Assert.Equal(0, cache.Count); + Assert.False(cache.TryTake("src/Timeout.cs", "checksum", out _)); + Assert.Same(timeoutSymbols[0], Assert.Single(timeoutSymbols)); + + var cancelledSymbols = new List { CreatePopulatedSymbol() }; + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + Assert.ThrowsAny(() => + cache.TryAdmitOwned( + "src/Cancelled.cs", + "checksum", + cancelledSymbols, + hadRegexTimeout: false, + cancellationToken: cancellation.Token)); + Assert.Equal(0, cache.Count); + Assert.False(cache.TryTake("src/Cancelled.cs", "checksum", out _)); + Assert.Same(cancelledSymbols[0], Assert.Single(cancelledSymbols)); + } + + [Fact] + public void TryAdmitOwned_AvoidsGenericCloneAllocationForLargeArtifact() + { + const int symbolCount = 4_096; + const long minimumAllocationSavings = 500L * 1024; + WarmUpArtifactAdmissionsForAllocationMeasurement(); + var genericSymbols = CreateMinimalSymbols(symbolCount); + var ownedSymbols = CreateMinimalSymbols(symbolCount); + var genericCache = new CSharpPrepassSymbolArtifactCache(); + var ownedCache = new CSharpPrepassSymbolArtifactCache(); + + var allocatedBeforeGenericAdmission = + GC.GetAllocatedBytesForCurrentThread(); + var genericAdmitted = genericCache.TryAdmit( + "src/Fixture.cs", + "checksum", + genericSymbols, + hadRegexTimeout: false); + var genericAllocatedBytes = + GC.GetAllocatedBytesForCurrentThread() + - allocatedBeforeGenericAdmission; + + var allocatedBeforeOwnedAdmission = + GC.GetAllocatedBytesForCurrentThread(); + var ownedAdmitted = ownedCache.TryAdmitOwned( + "src/Fixture.cs", + "checksum", + ownedSymbols, + hadRegexTimeout: false); + var ownedAllocatedBytes = + GC.GetAllocatedBytesForCurrentThread() + - allocatedBeforeOwnedAdmission; + + Assert.True(genericAdmitted); + Assert.True(ownedAdmitted); + Assert.True( + genericAllocatedBytes - ownedAllocatedBytes + >= minimumAllocationSavings, + $"Expected owned admission to save at least {minimumAllocationSavings} bytes; " + + $"generic={genericAllocatedBytes}, owned={ownedAllocatedBytes}."); + + Assert.True(genericCache.TryTake( + "src/Fixture.cs", + "checksum", + out var genericArtifact)); + Assert.NotSame(genericSymbols, genericArtifact.Symbols); + Assert.NotSame(genericSymbols[0], genericArtifact.Symbols[0]); + var cachedGenericName = genericArtifact.Symbols[0].Name; + genericSymbols[0].Name = "mutated-after-admission"; + Assert.Equal(cachedGenericName, genericArtifact.Symbols[0].Name); + + Assert.True(ownedCache.TryTake( + "src/Fixture.cs", + "checksum", + out var ownedArtifact)); + Assert.Same(ownedSymbols, ownedArtifact.Symbols); + Assert.Same(ownedSymbols[0], ownedArtifact.Symbols[0]); + } + + [Fact] + public void BuildWorkspaceSymbols_OwnedArtifactsKeepMaterializedLookupsIsolated() + { + using var project = TestProjectHelper.CreateTempProjectScope( + "csharp_prepass_owned_artifact"); + var sourcePath = TestProjectHelper.WriteTextFile( + project.Root, + "src/Contracts.cs", + """ + namespace Demo; + public interface IShape + { + static abstract T Create(T value); + } + public enum Shade + { + Red, + } + public static class Tokens + { + public const int Answer = 42; + } + """); + var dbPath = TestProjectHelper.CreateProjectDb(project.Root); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + var indexer = new FileIndexer(project.Root, ignoreCase: false); + var target = CSharpStaticInterfacePrepass.FileTarget.Create( + project.Root, + sourcePath, + "csharp"); + + var baseline = CSharpStaticInterfacePrepass.BuildWorkspaceSymbols( + writer, + indexer, + [target], + includeExistingSymbols: false); + var cache = new CSharpPrepassSymbolArtifactCache(); + var compact = CSharpStaticInterfacePrepass.BuildWorkspaceSymbols( + writer, + indexer, + [target], + includeExistingSymbols: false, + symbolArtifactCache: cache); + + Assert.NotEmpty(baseline.Symbols); + Assert.Empty(compact.Symbols); + Assert.Equal( + FlattenStaticInterfaceLookups(baseline.StaticInterfaceMemberLookups!), + FlattenStaticInterfaceLookups(compact.StaticInterfaceMemberLookups!)); + var compactQualifiedLookups = + FlattenQualifiedPatternLookups(compact.QualifiedPatternLookups!); + Assert.Equal( + FlattenQualifiedPatternLookups(baseline.QualifiedPatternLookups!), + compactQualifiedLookups); + var compactStaticInterfaceLookups = + FlattenStaticInterfaceLookups(compact.StaticInterfaceMemberLookups!); + Assert.Contains( + compactStaticInterfaceLookups, + item => item.Contains("IShape:Create:", StringComparison.Ordinal)); + Assert.Contains( + compactQualifiedLookups, + item => item.StartsWith("enum:Red:Shade:", StringComparison.Ordinal)); + Assert.Contains( + compactQualifiedLookups, + item => item.StartsWith("constant:Answer:Tokens:", StringComparison.Ordinal)); + + var checksum = indexer.BuildRecord(sourcePath).record.Checksum; + Assert.NotNull(checksum); + Assert.True(cache.TryTake(target.IndexPath, checksum, out var artifact)); + var contract = Assert.Single( + artifact.Symbols, + symbol => symbol.Name == "Create" && symbol.ContainerName == "IShape"); + contract.Name = "MutatedCreate"; + contract.ContainerName = "MutatedShape"; + contract.Signature = "mutated"; + var enumMember = Assert.Single( + artifact.Symbols, + symbol => symbol.Name == "Red" && symbol.ContainerName == "Shade"); + enumMember.Name = "MutatedRed"; + enumMember.ContainerName = "MutatedShade"; + var constant = Assert.Single( + artifact.Symbols, + symbol => symbol.Name == "Answer" && symbol.ContainerName == "Tokens"); + constant.Name = "MutatedAnswer"; + constant.ContainerName = "MutatedTokens"; + + Assert.Equal( + compactStaticInterfaceLookups, + FlattenStaticInterfaceLookups(compact.StaticInterfaceMemberLookups!)); + Assert.Equal( + compactQualifiedLookups, + FlattenQualifiedPatternLookups(compact.QualifiedPatternLookups!)); + } + [Fact] public void TryTake_ChecksumMismatchConsumesArtifactAndReportsFallback() { @@ -146,6 +357,74 @@ public void TryAdmit_RegexTimeoutDoesNotPublishPartialSymbols() .GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) .ToDictionary(property => property, property => property.GetValue(symbol)); + private static void WarmUpArtifactAdmissionsForAllocationMeasurement() + { + var genericCache = new CSharpPrepassSymbolArtifactCache(); + Assert.True(genericCache.TryAdmit( + "warmup-generic.cs", + "checksum", + CreateMinimalSymbols(1), + hadRegexTimeout: false)); + Assert.True(genericCache.TryTake( + "warmup-generic.cs", + "checksum", + out _)); + + var ownedCache = new CSharpPrepassSymbolArtifactCache(); + var ownedSymbols = CreateMinimalSymbols(1); + Assert.True(ownedCache.TryAdmitOwned( + "warmup-owned.cs", + "checksum", + ownedSymbols, + hadRegexTimeout: false)); + Assert.True(ownedCache.TryTake( + "warmup-owned.cs", + "checksum", + out _)); + } + + private static List CreateMinimalSymbols(int count) + { + var symbols = new List(count); + for (var index = 0; index < count; index++) + { + symbols.Add(new SymbolRecord + { + Kind = "function", + Name = "Fixture", + Line = index + 1, + StartLine = index + 1, + EndLine = index + 1, + }); + } + + return symbols; + } + + private static string[] FlattenStaticInterfaceLookups( + ReferenceExtractor.CSharpStaticInterfaceMemberLookups lookups) + => lookups.ContractsByType + .SelectMany(pair => pair.Value.Select(contract => + $"{pair.Key}:{contract.Name}:{contract.Kind}:{contract.ParameterShape}:{contract.ReturnTypeShape}")) + .Concat(lookups.InterfaceGenericParameters.SelectMany(pair => + pair.Value.Select(parameter => $"{pair.Key}:generic:{parameter}"))) + .Order(StringComparer.Ordinal) + .ToArray(); + + private static string[] FlattenQualifiedPatternLookups( + ReferenceExtractor.CSharpQualifiedPatternLookups lookups) + => lookups.EnumMemberLookup + .SelectMany(pair => pair.Value.Select(target => + $"enum:{pair.Key}:{target.EnumName}:{target.QualifiedEnumName}:{target.AllowShortNameFallback}")) + .Concat(lookups.ConstantPatternMemberLookup.SelectMany(pair => + pair.Value.Select(target => + $"constant:{pair.Key}:{target.ContainerName}:{target.QualifiedContainerName}:{target.AllowShortNameFallback}"))) + .Concat(lookups.TypePatternLookup.SelectMany(pair => + pair.Value.Select(target => + $"type:{pair.Key}:{target.ContainerName}:{target.QualifiedContainerName}:{target.AllowShortNameFallback}"))) + .Order(StringComparer.Ordinal) + .ToArray(); + private static SymbolRecord CreatePopulatedSymbol() => new() { From fdc6455a802a1cb3b86c673a26f52416f2ff4b3f Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 11:40:15 +0900 Subject: [PATCH 09/16] Skip impossible built-in symbol patterns --- DEVELOPER_GUIDE.md | 20 + TESTING_GUIDE.md | 18 + .../+large-codebase-initial-indexing.fixed.md | 9 + .../Symbols/SymbolExtractor.CSharpScanner.cs | 49 +- .../Indexer/Symbols/SymbolExtractor.Cpp.cs | 19 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 67 +- .../SymbolExtractor.ExtractionPhases.cs | 13 +- .../Symbols/SymbolExtractor.Patterns.cs | 861 ++++++++++-------- .../Indexer/Symbols/SymbolExtractor.cs | 37 + ...SymbolExtractorRequiredLiteralGateTests.cs | 247 +++++ 10 files changed, 906 insertions(+), 434 deletions(-) create mode 100644 tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b8d1c1498..670cc7961 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1880,6 +1880,16 @@ Extractor strategy by language surface: | Windows application manifests | Manifest element paths, assembly identities, execution levels, and supported-OS values remain structural symbols. Dependent assembly identities emit `dependency` references, while local `file`, `codeBase`, and probing paths emit `project_reference` edges. | | XML / NuGet.config | Generic XML emits bounded element and attribute paths. NuGet.config additionally promotes package sources, source mappings, signature validation mode, trusted signer names, certificate fingerprints, and `allowUntrustedRoot` values to semantic `property` symbols with `nuget.*` subkinds. | +Before the line-oriented regex loop, built-in case-sensitive patterns may opt into an explicit +`RequiredLiteral` Tier A gate. The literal must contain at least two characters and be an Ordinal +substring of every successful regex path. If it is absent from the normalized file content, that +pattern is skipped without changing the order of the remaining patterns. `IgnoreCase` patterns, +one-character literals, optional or alternative paths without a shared literal, project custom +patterns, and plugins are deliberately excluded. Supplemental scans that consult the pattern list, +including C# incomplete-attribute recovery and C++ same-line member recovery, must consume the same +ordered applicable set. The content-wide check may retain a pattern because its literal appears in a +comment or string; that only reduces the optimization and cannot suppress a real match. + JavaScript and TypeScript export/reference details: | Area | Behavior | @@ -5591,6 +5601,16 @@ LIMIT 20; | Windows application manifest | manifest element path、assembly identity、execution level、supported OS value を structural symbol として維持します。依存 assembly identity は `dependency` reference、local な `file` / `codeBase` / probing path は `project_reference` edge を出力します。 | | XML / NuGet.config | 汎用 XML は上限付きの element / attribute path を出力します。NuGet.config ではさらに package source、source mapping、署名検証モード、trusted signer 名、証明書 fingerprint、`allowUntrustedRoot` の値を `nuget.*` subkind 付きの semantic `property` symbol にします。 | +行指向の正規表現 loop に入る前に、built-in の case-sensitive pattern は明示的な +`RequiredLiteral` Tier A gate を opt-in できます。literal は2文字以上で、正規表現の全成功経路に +Ordinal の substring として必ず現れなければなりません。正規化済み file content に存在しない +場合だけその pattern を skip し、残る pattern の順序は変えません。`IgnoreCase` pattern、1文字の +literal、共通 literal を持たない optional / alternative path、project の custom pattern、plugin は +意図的に対象外です。C# の不完全 attribute recovery や C++ の same-line member recovery を含め、 +pattern list を参照する補助 scan は同じ順序の applicable set を使わなければなりません。comment や +string 内に literal があるため pattern を残すことはありますが、その場合は最適化量が減るだけで、 +本物の match を抑止しません。 + JavaScript / TypeScript の export / reference 詳細: | 項目 | 動作 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index a1135bbb5..3ad943b17 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -428,6 +428,15 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result - `IndexCommandRunnerTests`, `FileIndexerTests`, and `PerformanceTests` also cover `CSharpStaticInterfacePrepass` text, raw-byte, chunked raw-token, and streaming file contract probes. Stable candidate reads must authorize and open each file once, keep raw-negative reads to one bounded pass, and rewind that same handle only for a raw-positive full decode. A detected in-place or atomic-replacement mutation must discard that snapshot and reauthorize/reopen once so the prepass cannot diverge from the main indexing pass. Preserve UTF-8 / UTF-16, NUL rejection, growth, cancellation, and lexical-boundary behavior. The 576 KiB semantic-negative/positive allocation guard runs each probe 12 times and stays below 4 KiB of current-thread allocation so a whole-content mask cannot return. - The parallel C# static-interface full-scan fixture uses 64 implementation files and treats one workspace lookup build as a performance contract. Keep the contract lookup attached to the immutable prepass snapshot across CLI full scan, scoped update, and MCP indexing; do not rebuild it once per C# file. - `CSharpPrepassSymbolArtifactCacheTests`, `FileIndexerTests`, and the CLI/MCP fresh-index fixtures protect bounded prepass artifact reuse. Keep deep-clone independence for generic admission; production owned-list admission must retain list/symbol identity only after successful atomic publication, leave rejected or cancelled input caller-owned, and release workspace fallback symbols only after both lookup snapshots are materialized. Preserve lookup parity and mutation isolation, take-once checksum matching, mismatch consumption, atomic file/symbol/estimated-byte caps, cancellation without partial admission, and non-admission of partial symbols after any bounded-regex timeout. Encoding theories must compare UTF-8, UTF-16 LE/BE, and invalid-UTF-8 prepass checksums with the authoritative loader. Integration coverage must prove reuse only for an empty non-rebuild full index, ordinary extraction for rebuild/symbols-only/existing/incomplete-or-stall paths, authoritative main-read mutation fallback, unchanged post hooks and family/kind processing, and cache clearing before graph work. +- `SymbolExtractorRequiredLiteralGateTests` keeps built-in required-literal gating deterministic and + output-preserving. It pins 400 audited Tier A patterns across 51 case-sensitive languages, compares + all 29 readable `SymbolRecord` fields in emitted order, exercises representative positive fixtures + across Python, JavaScript, TypeScript, Go, Rust, Java, C/C++, Swift, F#, Scala, Terraform, Protobuf, + and Zig, and verifies literal absence for every annotated language. Keep the IgnoreCase and short- + literal fail-fast checks, C# incomplete-attribute recovery, and C++ same-line recovery in the same + bounded suite. Do not add current-worktree paths, `.cdidx` databases, allocation/wall-clock + assertions, or repository-wide extraction to this test; measure cold-index performance externally + and pair it with natural-key database parity. - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` builds 20,000 unrelated generic interfaces around one contract and keeps lookup allocation below 64 KiB. Discover contract containers before parsing generic declarations, while the paired `ReferenceExtractorTests` case preserves declaration/member ordering, partial-interface last-write behavior, and contract member order. - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` keeps a 1,200-statement, regex-and-brace-heavy source at one emitted symbol and pins current-thread allocations below 6.0 MB for JavaScript, 8.2 MB for TypeScript, and 14.1 MB combined. This is a physical-pass contract: do not restore a second full-file sanitizer for private-scope analysis. Pair allocation changes with the JavaScript/TypeScript literal-and-comment scope theory so templates, comments, and regex braces cannot create or leak private classes. - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` keeps persisted C# contract-member candidates and the pending-path contract flag on one `files(lang)` → `symbols(file_id, kind)` row pass. Derive both results from that reader; do not restore a second all-contract query for excluded paths. Interface declarations may be loaded only for exact retained contract container names through bounded `symbols(name)` batches, and those dynamic tail shapes must stay out of the prepared cache. `DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` pins both index plans, while `LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` requires negative/LIKE-decoy-only reads to execute no interface phase and keeps unrelated plain interfaces out of managed materialization. `PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` also splits a 503-path language lookup at 500 parameters, returns only persisted C# paths plus sorted purge-plan IDs/bytes, keeps both SQL tail shapes out of the prepared cache, and stops before the second batch when cancellation is requested. @@ -1442,6 +1451,15 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `IndexCommandRunnerTests`、`FileIndexerTests`、`PerformanceTests` は `CSharpStaticInterfacePrepass` のテキスト判定、raw-byte、chunked raw-token、streaming file 契約 probe も扱います。安定した候補読み取りは各 file を1回だけ認可・openし、raw-negative は bounded pass 1回に留め、raw-positive の full decode だけ同じ handle を rewind してください。in-place mutation または atomic replacement を検知した場合は snapshot を破棄し、prepass と main indexing pass が乖離しないよう1回だけ再認可・再openします。UTF-8 / UTF-16、NUL 拒否、growth、cancellation、lexical boundary を維持してください。576 KiB の semantic-negative/positive allocation guard は各 probe を12回実行して current-thread allocation を4 KiB未満に保ち、content 全体 mask の再導入を防ぎます。 - parallel C# static-interface full-scan fixture は64個のimplementation fileを使い、workspace lookup buildが1回であることをperformance contractとします。CLI full scan、scoped update、MCP indexingを横断してcontract lookupをimmutable prepass snapshotに保持し、C# fileごとの再構築を戻さないでください。 - `CSharpPrepassSymbolArtifactCacheTests`、`FileIndexerTests`、CLI/MCP の fresh-index fixture は bounded prepass artifact reuse を固定します。汎用 admission の deep-clone 独立性を維持し、production の owned-list admission は原子的な publish 成功後だけ list / symbol identity を保持し、reject または cancel された input は caller-owned のままにしてください。2種類の lookup snapshot を materialize した後だけ workspace fallback symbol を解放し、lookup parity と mutation isolation を保ちます。checksum 一致時の take-once、不一致時の消費、file / symbol / estimated-byte cap の原子性、partial admission を残さない cancellation、bounded-regex timeout 後の partial symbol をadmitしない契約も維持してください。encoding theory は UTF-8、UTF-16 LE/BE、不正 UTF-8 の prepass checksum を authoritative loader と比較します。integration coverage では空 database の非 rebuild full index だけが再利用し、rebuild / symbols-only / existing / incomplete-or-stall 経路は通常 extraction、main read 中の mutation は checksum fallback、post hook と family/kind 処理は従来どおり、graph 開始前に cache が clear されることを証明してください。 +- `SymbolExtractorRequiredLiteralGateTests` は built-in required-literal gate の決定性と output + 不変性を固定します。51 の case-sensitive 言語にまたがる監査済み Tier A pattern 400件、出力順を + 含む `SymbolRecord` の readable field 29個すべて、Python、JavaScript、TypeScript、Go、Rust、 + Java、C/C++、Swift、F#、Scala、Terraform、Protobuf、Zig の代表的な positive fixture、注釈済み + 全言語での literal 不在を検証します。同じ bounded suite に IgnoreCase / 短い literal の + fail-fast、C# の不完全 attribute recovery、C++ の same-line recovery を維持してください。 + current worktree path、`.cdidx` database、allocation / wall-clock assertion、repository 全体の + extraction をこの test に追加してはいけません。cold-index performance は外部で測定し、database + の natural-key parity と対にしてください。 - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` は、1件のcontractの周囲に20,000件の無関係なgeneric interfaceを構築し、lookup allocationを64 KiB未満に固定します。generic宣言を解析する前にcontract containerを検出し、対になる`ReferenceExtractorTests`で宣言/member順、partial interfaceの後勝ち、contract member順を維持してください。 - `PerformanceTests.SymbolExtraction_JavaScriptTypeScriptScopeLexing_ReusesSanitizedSnapshot` は、regex と brace が密な1,200 statement の source を出力 symbol 1件に保ち、current-thread allocation を JavaScript 6.0 MB未満、TypeScript 8.2 MB未満、合計14.1 MB未満に固定します。これは physical-pass 契約であり、private-scope 解析専用の2回目の全file sanitizerを戻さないでください。allocation変更時はJavaScript/TypeScriptのliteral/comment scope theoryも対にし、template、comment、regexのbraceがprivate classを生成またはscope外へ漏らさないことを維持してください。 - `PreparedCommandCacheTests.DbWriter_WithCache_CSharpStaticInterfaceContractQueriesReuseCacheAndOneWorkspaceRead` は、永続化済み C# contract member 候補と pending-path contract flag を1回の `files(lang)` → `symbols(file_id, kind)` row passで得ることを固定します。両方を同じ reader から導出し、除外 path 判定のための2回目の全 contract query を戻さないでください。interface 宣言は厳密検証後に保持した contract container 名だけを bounded な `symbols(name)` batch で取得でき、その dynamic tail shape は prepared cache に残してはいけません。`DatabaseTests.CSharpContractWorkspaceQueries_UseFileKindThenBoundedInterfaceNamePlans` は両方の index plan を固定し、`LoadCSharpContractWorkspace_MaterializesOnlyCandidatesAndMatchingInterfaces` は negative / LIKE decoy だけの読込で interface phase が0回となり、無関係な通常 interface を managed materialization へ入れないことを要求します。`PreparedCommandCacheTests.DbWriter_CSharpStaticInterfaceContractMemberPreflightsAreExactBatchedAndCancellable` は503件のpath言語lookupを500 parameterで分割し、永続C# pathと昇順のpurge-plan ID/byteだけを返し、両方のSQL tail shapeをprepared cache外に保ち、1つ目のbatch後のcancellationで2つ目を実行しないことも固定します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 77ec71bf1..fc7889aa0 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -11,6 +11,12 @@ affected: - src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Patterns.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs - tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs @@ -22,6 +28,7 @@ affected: - tests/CodeIndex.Tests/IndexCommandRunnerReferenceIndexBulkLoadTests.cs - tests/CodeIndex.Tests/McpServerToolsCallTests.cs - tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs + - tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md --- @@ -35,6 +42,7 @@ affected: - **Fresh graph planning uses post-load cardinalities** — truly empty CLI and MCP bulk loads now analyze the populated file, symbol, and reference tables immediately before candidate resolution, allowing SQLite to plan the expensive first graph build from current statistics. Rebuilds, updates, existing databases, and symbols-only runs keep their prior lifecycle; cancellation still aborts, while a statistics-only SQLite failure rolls back its savepoint and continues best-effort. - **Persistent symbol workers reuse bounded pattern-directory snapshots across languages** — each project-root reload discovers user and root configs once, while nested ancestor directories, including missing and rejected results, are observed once per worker command. The cache follows live filesystem casing and falls back to uncached discovery when full so configs are never skipped; direct registry callers keep dynamic discovery. - **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once built-in symbols already extracted for the static-interface workspace. After materializing the immutable lookup snapshots, the prepass transfers ownership of admitted per-file symbol lists and releases the redundant workspace fallback objects instead of cloning the full symbol graph. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. +- **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate before line-by-line matching. Pattern order and output stay unchanged, and C# incomplete-attribute plus C++ same-line recovery consume the same filtered set. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. ## 日本語 @@ -45,3 +53,4 @@ affected: - **新規graph planningでbulk load後のcardinalityを使うようにしました** — 真に空のdatabaseから始まるCLI / MCP bulk loadでは、candidate解決の直前に投入済みのfile、symbol、reference tableを解析し、SQLiteが初回の高コストなgraph構築を最新統計から計画できるようにします。rebuild、update、既存database、symbols-onlyのlifecycleは従来どおりです。cancellationは引き続き中断し、統計更新だけのSQLite failureはsavepointを戻してbest-effortで継続します。 - **persistent symbol worker が全言語で上限付き pattern-directory snapshot を再利用するようにしました** — project-root reload ごとに user / root config を1回だけ探索し、missing や reject を含む nested ancestor directory は worker command ごとに初回結果を再利用します。cache は実 filesystem の case policy に従い、飽和時は uncached discovery に fallback して config を skip しません。registry の direct caller は従来どおり動的に探索します。 - **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once で利用できます。immutable な lookup snapshot を materialize した後、prepass は admit した file ごとの symbol list の所有権を移し、symbol graph 全体を clone せず重複する workspace fallback object を解放します。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 +- **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、行単位の match 前に監査済みの2文字以上の literal を Ordinal で判定します。pattern 順と出力は変えず、C# の不完全 attribute recovery と C++ の same-line recovery も同じ filtered set を使います。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs index ec04364db..7df45d6e8 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs @@ -447,7 +447,8 @@ private static string StripLeadingCSharpAttributeLists( ref bool inLeadingAttributeBlock, ref int attributeBracketDepth, ref int attributeParenDepth, - bool insideEnumBody) + bool insideEnumBody, + IReadOnlyList applicablePatterns) { var index = 0; while (index < line.Length && char.IsWhiteSpace(line[index])) @@ -459,7 +460,12 @@ private static string StripLeadingCSharpAttributeLists( if (!inLeadingAttributeBlock && line[index] != '[') return line; - if (inLeadingAttributeBlock && ShouldRecoverFromIncompleteLeadingCSharpAttribute(line, index, insideEnumBody, attributeParenDepth)) + if (inLeadingAttributeBlock && ShouldRecoverFromIncompleteLeadingCSharpAttribute( + line, + index, + insideEnumBody, + attributeParenDepth, + applicablePatterns)) { inLeadingAttributeBlock = false; attributeBracketDepth = 0; @@ -528,26 +534,32 @@ private static bool ShouldRecoverFromIncompleteLeadingCSharpAttribute( string line, int firstNonWhitespaceIndex, bool insideEnumBody, - int attributeParenDepth) + int attributeParenDepth, + IReadOnlyList applicablePatterns) { if (firstNonWhitespaceIndex >= line.Length || line[firstNonWhitespaceIndex] == '[') return false; - return TryMatchAnyRecoverableCSharpPattern(line, insideEnumBody, attributeParenDepth); + return TryMatchAnyRecoverableCSharpPattern( + line, + insideEnumBody, + attributeParenDepth, + applicablePatterns); } - private static bool TryMatchAnyRecoverableCSharpPattern(string line, bool insideEnumBody, int attributeParenDepth) + private static bool TryMatchAnyRecoverableCSharpPattern( + string line, + bool insideEnumBody, + int attributeParenDepth, + IReadOnlyList applicablePatterns) { - if (PatternCache.TryGetValue("csharp", out var patterns)) + foreach (var pattern in applicablePatterns) { - foreach (var pattern in patterns) - { - if (ReferenceEquals(pattern.Regex, CSharpEnumMemberRegex)) - continue; + if (ReferenceEquals(pattern.Regex, CSharpEnumMemberRegex)) + continue; - if (pattern.Regex.IsMatch(line)) - return true; - } + if (pattern.Regex.IsMatch(line)) + return true; } return insideEnumBody @@ -3590,10 +3602,10 @@ private static bool ShouldSkipCSharpSwitchExpressionPropertyCandidate( && getCSharpSwitchExpressionLines?.Invoke() is { } csharpSwitchExpressionLines && csharpSwitchExpressionLines[lineIndex]; - private static string[] BuildCSharpMatchLines(string[] structuralLines) - => BuildCSharpMatchLines(structuralLines, out _); - - private static string[] BuildCSharpMatchLines(string[] structuralLines, out int[]?[] collapsedToRaw) + private static string[] BuildCSharpMatchLines( + string[] structuralLines, + IReadOnlyList applicablePatterns, + out int[]?[] collapsedToRaw) { var matchLines = new string[structuralLines.Length]; collapsedToRaw = new int[]?[structuralLines.Length]; @@ -3613,7 +3625,8 @@ private static string[] BuildCSharpMatchLines(string[] structuralLines, out int[ ref inLeadingAttributeBlock, ref attributeBracketDepth, ref attributeParenDepth, - activeEnumBodyDepth > 0), + activeEnumBodyDepth > 0, + applicablePatterns), out var lineCollapsedToRaw); collapsedToRaw[lineIndex] = lineCollapsedToRaw; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs index 3802d1f5b..fc4638e15 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs @@ -17,7 +17,11 @@ private readonly record struct CppSameLineClassMemberIdentity( CppSameLineClassKey ClassKey, string MemberName); - private static void ExtractCppSameLineClassBodyMembers(long fileId, string[] lines, List symbols) + private static void ExtractCppSameLineClassBodyMembers( + long fileId, + string[] lines, + IReadOnlyList applicablePatterns, + List symbols) { var classSymbols = BuildCppSameLineClassSymbolSnapshot(symbols); if (classSymbols is null) @@ -46,7 +50,15 @@ private static void ExtractCppSameLineClassBodyMembers(long fileId, string[] lin classSymbol.Kind, classSymbol.Name); foreach (var segment in EnumerateTrimmedCppSegments(body)) - TryAddCppSameLineClassMemberSymbol(fileId, classSymbol, classKey, segment, lineIndex + 1, symbols, ref existingMembers); + TryAddCppSameLineClassMemberSymbol( + fileId, + classSymbol, + classKey, + segment, + lineIndex + 1, + applicablePatterns, + symbols, + ref existingMembers); } } @@ -134,10 +146,11 @@ private static bool TryAddCppSameLineClassMemberSymbol( CppSameLineClassKey classKey, string segment, int lineNumber, + IReadOnlyList applicablePatterns, List symbols, ref HashSet? existingMembers) { - foreach (var pattern in PatternCache["cpp"]) + foreach (var pattern in applicablePatterns) { if (pattern.Kind != "function") continue; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index d96bc6f33..96da727a2 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -20,7 +20,9 @@ private static List ExtractCore( string? projectRoot = null, bool patternConfigsAlreadyLoaded = false, CancellationToken cancellationToken = default, - int? maxSymbols = null) + int? maxSymbols = null, + bool applyRequiredLiteralGate = true, + RequiredLiteralGateCounts? requiredLiteralGateCounts = null) { var originalLang = lang; if (TryPrepareSymbolExtraction( @@ -102,7 +104,17 @@ private static List ExtractCore( if (patterns == null || lang == null) return []; - var scanInputs = new PatternScanInputs(lang, filePath, lines); + var applicablePatterns = SelectApplicablePatterns( + patterns, + content, + applyRequiredLiteralGate); + if (requiredLiteralGateCounts != null) + { + requiredLiteralGateCounts.PatternCount = patterns.Count; + requiredLiteralGateCounts.ApplicablePatternCount = applicablePatterns.Count; + } + + var scanInputs = new PatternScanInputs(lang, filePath, lines, applicablePatterns); var pythonModulePrefix = scanInputs.PythonModulePrefix; var structuralLines = scanInputs.StructuralLines; var scientificBodyScannerLines = scanInputs.ScientificBodyScannerLines; @@ -177,7 +189,7 @@ private static List ExtractCore( bool? deferCSharpEventAtPatternStart = null; bool? deferCSharpDelegateAtPatternStart = null; bool? recoverableCSharpPatternAtPatternStart = null; - foreach (var pattern in patterns) + foreach (var pattern in applicablePatterns) { if (prologClauseContinuationLines?[i] == true && prologContinuationResumeOffset < 0 @@ -316,12 +328,14 @@ private static List ExtractCore( ? TryMatchAnyRecoverableCSharpPattern( matchLine[lineOffset..], insideEnumBody: false, - attributeParenDepth: 0) + attributeParenDepth: 0, + applicablePatterns) : recoverableCSharpPatternAtPatternStart ??= TryMatchAnyRecoverableCSharpPattern( matchLine[lineOffset..], insideEnumBody: false, - attributeParenDepth: 0)))) + attributeParenDepth: 0, + applicablePatterns)))) { lineOffset = FindNextSameLineBraceStatementStart(matchLine, lineOffset + 1, lang); continue; @@ -837,7 +851,7 @@ private static List ExtractCore( cssScannerLines, i, openingBraceIndex, - patterns, + applicablePatterns, symbols, cssSeenSymbols); } @@ -1263,7 +1277,7 @@ private static List ExtractCore( cssScannerLine, cssScannerLines!, i, - patterns, + applicablePatterns, symbols, cssSeenSymbols); } @@ -1285,7 +1299,8 @@ private static List ExtractCore( GetJavaScriptTypeScriptSanitizedLines, csharpMatchLines, pythonModulePrefix, - prologMultilineHeads); + prologMultilineHeads, + applicablePatterns); } if (lang == "csharp") { @@ -1307,6 +1322,42 @@ private static List ExtractCore( return symbols; } + private static IReadOnlyList SelectApplicablePatterns( + IReadOnlyList patterns, + string content, + bool applyRequiredLiteralGate) + { + if (!applyRequiredLiteralGate) + return patterns; + + // A content-wide Ordinal check can only remove a pattern when a match is impossible. + // Preserve the original order, return the original list when nothing is skipped, and pass + // this same applicable set to every supplemental recovery scan. + // content 全体の Ordinal 判定で match 不可能な pattern だけを除外する。元の順序を保ち、 + // skip がなければ元 list を返し、補助 recovery scan にも同じ applicable set を渡す。 + List? applicablePatterns = null; + for (var patternIndex = 0; patternIndex < patterns.Count; patternIndex++) + { + var pattern = patterns[patternIndex]; + if (pattern.RequiredLiteral is { } requiredLiteral + && !content.Contains(requiredLiteral, StringComparison.Ordinal)) + { + if (applicablePatterns == null) + { + applicablePatterns = new List(patterns.Count - 1); + for (var prefixIndex = 0; prefixIndex < patternIndex; prefixIndex++) + applicablePatterns.Add(patterns[prefixIndex]); + } + + continue; + } + + applicablePatterns?.Add(pattern); + } + + return applicablePatterns ?? patterns; + } + private static readonly Regex PrologOpenClauseRegex = new( @"^\s*(?:(?:[a-z][A-Za-z0-9_]*\s*(?:\([^\r\n]*\))?\s*(?::-|-->))|:-)", RegexOptions.Compiled | RegexOptions.CultureInvariant); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs index a6b1acd18..42fa75dac 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs @@ -19,7 +19,11 @@ private sealed class PatternScanInputs private bool[]? _cssQualifiedRuleAncestors; private string[]? _javaScriptTypeScriptSanitizedLines; - public PatternScanInputs(string lang, string? filePath, string[] lines) + public PatternScanInputs( + string lang, + string? filePath, + string[] lines, + IReadOnlyList applicablePatterns) { _lang = lang; _lines = lines; @@ -72,7 +76,7 @@ public PatternScanInputs(string lang, string? filePath, string[] lines) int[]?[] csharpMatchColumnToRaw = null!; CSharpMatchLines = lang == "csharp" - ? BuildCSharpMatchLines(lines, out csharpMatchColumnToRaw) + ? BuildCSharpMatchLines(lines, applicablePatterns, out csharpMatchColumnToRaw) : null; CSharpMatchColumnToRaw = csharpMatchColumnToRaw; GetCSharpLineStartStates = lang == "csharp" @@ -646,7 +650,8 @@ private static void AddSupplementalSymbols( Func getJavaScriptTypeScriptSanitizedLines, string[]? csharpMatchLines, string? pythonModulePrefix, - Dictionary? prologMultilineHeads) + Dictionary? prologMultilineHeads, + IReadOnlyList applicablePatterns) { if (lang == "javascript") ExtractJavaScriptBareMethods(fileId, lines, symbols, getPrivateScopeColumns!, getJavaScriptTypeScriptSanitizedLines); @@ -677,7 +682,7 @@ private static void AddSupplementalSymbols( ExtractGoGroupedDeclarations(fileId, lines, symbols, extractionState); if (lang == "cpp") { - ExtractCppSameLineClassBodyMembers(fileId, lines, symbols); + ExtractCppSameLineClassBodyMembers(fileId, lines, applicablePatterns, symbols); ExtractCppBalancedCallableSymbols(fileId, lines, structuralLines, symbols, extractionState); ExtractCppFriendDeclarationSymbols(fileId, lines, symbols, extractionState); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Patterns.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Patterns.cs index 5d6a81b5d..5aa1dfdfb 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Patterns.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Patterns.cs @@ -373,12 +373,19 @@ private enum BodyStyle SqlProcBody, } + // RequiredLiteral is an explicit Tier A opt-in for built-in, case-sensitive patterns only. + // It must be an Ordinal substring of every successful regex path and contain at least two + // characters. IgnoreCase, custom/plugin, one-character, and path-optional literals stay null. + // RequiredLiteral は built-in の case-sensitive pattern だけが明示的に opt-in する Tier A + // metadata。全成功経路に Ordinal で必ず現れる2文字以上の substring に限定し、IgnoreCase、 + // custom/plugin、1文字、optional path の literal には設定しない。 private sealed record SymbolPattern( string Kind, Regex Regex, BodyStyle BodyStyle, string? VisibilityGroup = null, - string? ReturnTypeGroup = null); + string? ReturnTypeGroup = null, + string? RequiredLiteral = null); private enum CssContextKind { @@ -700,38 +707,38 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult { ["python"] = [ - new("function", new Regex(@"^\s*(?:async\s+)?def\s+(?\w+)\s*(?:\[[^\]]*\])?\s*(?:\(|\[)", RegexOptions.Compiled), BodyStyle.Indent), - new("lambda", new Regex(@"^\s*(?\w+)\s*=\s*lambda\b", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Indent), + new("function", new Regex(@"^\s*(?:async\s+)?def\s+(?\w+)\s*(?:\[[^\]]*\])?\s*(?:\(|\[)", RegexOptions.Compiled), BodyStyle.Indent, RequiredLiteral: "def"), + new("lambda", new Regex(@"^\s*(?\w+)\s*=\s*lambda\b", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "lambda"), + new("class", new Regex(@"^\s*class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Indent, RequiredLiteral: "class"), new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|collections)\.)?(?:NamedTuple|namedtuple)\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:dataclasses\.)?make_dataclass\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?TypedDict\s*\(", RegexOptions.Compiled), BodyStyle.None), + new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:dataclasses\.)?make_dataclass\s*\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "make_dataclass"), + new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?TypedDict\s*\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "TypedDict"), new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:enum\.)?(?:Enum|IntEnum|Flag|IntFlag|StrEnum)\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:pydantic\.)?create_model\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("typealias", new Regex(@"^\s*type\s+(?\w+)\s*(?:\[[^\]]*\])?\s*=", RegexOptions.Compiled), BodyStyle.None), - new("typealias", new Regex(@"^\s*(?\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?TypeAlias\s*=", RegexOptions.Compiled), BodyStyle.None), - new("typealias", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?NewType\s*\(", RegexOptions.Compiled), BodyStyle.None), + new("class", new Regex(@"^\s*(?\w+)\s*=\s*(?:pydantic\.)?create_model\s*\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "create_model"), + new("typealias", new Regex(@"^\s*type\s+(?\w+)\s*(?:\[[^\]]*\])?\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("typealias", new Regex(@"^\s*(?\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?TypeAlias\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "TypeAlias"), + new("typealias", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?NewType\s*\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "NewType"), new("type_parameter", new Regex(@"^\s*(?\w+)\s*=\s*(?:(?:typing|typing_extensions)\.)?(?:TypeVar|ParamSpec|TypeVarTuple)\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("property", new Regex(@"^\s*(?\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?Final(?:\[[^\]]+\])?\s*=", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*(?:from\s+(?(?:\.+[\w.]*|[\w.]+))\s+import\b|import\s+(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*(?:[_\p{L}]\w*\s*=\s*)?(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), + new("property", new Regex(@"^\s*(?\w+)\s*:\s*(?:(?:typing|typing_extensions)\.)?Final(?:\[[^\]]+\])?\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "Final"), + new("import", new Regex(@"^\s*(?:from\s+(?(?:\.+[\w.]*|[\w.]+))\s+import\b|import\s+(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), + new("import", new Regex(@"^\s*(?:[_\p{L}]\w*\s*=\s*)?(?:importlib\.import_module|importlib\.util\.find_spec|__import__)\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["cython"] = [ - new("import", new Regex(@"^\s*from\s+(?" + CythonDottedIdentifierPattern + @")\s+cimport\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*cimport\s+(?" + CythonDottedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*import\s+(?" + CythonDottedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*include\s+(?:'(?[^']+)'|""(?[^""]+)"")", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*cdef\s+extern\s+from\s+(?:'(?[^']+)'|""(?[^""]+)"")", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("class", new Regex(@"^\s*cdef\s+class\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), - new("class", new Regex(@"^\s*class\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), - new("struct", new Regex(@"^\s*(?:ctypedef\s+)?cdef\s+struct\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), - new("enum", new Regex(@"^\s*(?:ctypedef\s+)?cdef\s+enum\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), - new("typealias", new Regex(@"^\s*ctypedef\s+(?!(?:struct|enum|union)\b)(?.+?)\s+(?" + CythonIdentifierPattern + @")\s*$", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), - new("function", new Regex(@"^\s*(?:cdef|cpdef)\s+(?!(?:class|struct|enum|extern)\b)" + CythonDeclarationPrefixPattern + @"(?:(?[\w.<>*,\[\]\s]+?)\s+)?(?" + CythonIdentifierPattern + @")\s*(?:\[[^\]]+\]\s*)?\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, ReturnTypeGroup: "returnType"), - new("function", new Regex(@"^\s*(?:async\s+)?def\s+(?" + CythonIdentifierPattern + @")\s*(?:\[[^\]]*\])?\s*(?:\(|\[)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), + new("import", new Regex(@"^\s*from\s+(?" + CythonDottedIdentifierPattern + @")\s+cimport\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "cimport"), + new("import", new Regex(@"^\s*cimport\s+(?" + CythonDottedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "cimport"), + new("import", new Regex(@"^\s*import\s+(?" + CythonDottedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "import"), + new("import", new Regex(@"^\s*include\s+(?:'(?[^']+)'|""(?[^""]+)"")", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "include"), + new("import", new Regex(@"^\s*cdef\s+extern\s+from\s+(?:'(?[^']+)'|""(?[^""]+)"")", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "extern"), + new("class", new Regex(@"^\s*cdef\s+class\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, RequiredLiteral: "class"), + new("class", new Regex(@"^\s*class\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, RequiredLiteral: "class"), + new("struct", new Regex(@"^\s*(?:ctypedef\s+)?cdef\s+struct\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, RequiredLiteral: "struct"), + new("enum", new Regex(@"^\s*(?:ctypedef\s+)?cdef\s+enum\s+(?" + CythonIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, RequiredLiteral: "enum"), + new("typealias", new Regex(@"^\s*ctypedef\s+(?!(?:struct|enum|union)\b)(?.+?)\s+(?" + CythonIdentifierPattern + @")\s*$", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType", RequiredLiteral: "ctypedef"), + new("function", new Regex(@"^\s*(?:cdef|cpdef)\s+(?!(?:class|struct|enum|extern)\b)" + CythonDeclarationPrefixPattern + @"(?:(?[\w.<>*,\[\]\s]+?)\s+)?(?" + CythonIdentifierPattern + @")\s*(?:\[[^\]]+\]\s*)?\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, ReturnTypeGroup: "returnType", RequiredLiteral: "def"), + new("function", new Regex(@"^\s*(?:async\s+)?def\s+(?" + CythonIdentifierPattern + @")\s*(?:\[[^\]]*\])?\s*(?:\(|\[)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, RequiredLiteral: "def"), new("function", new Regex(@"^\s*" + CythonNativeReturnTypePattern + @"(?" + CythonIdentifierPattern + @")\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), - new("property", new Regex(@"^\s*cdef\s+(?!(?:class|struct|enum|extern)\b)" + CythonDeclarationPrefixPattern + @"(?.+?)\s+(?" + CythonIdentifierPattern + @")\s*(?::|=|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), + new("property", new Regex(@"^\s*cdef\s+(?!(?:class|struct|enum|extern)\b)" + CythonDeclarationPrefixPattern + @"(?.+?)\s+(?" + CythonIdentifierPattern + @")\s*(?::|=|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType", RequiredLiteral: "cdef"), ], ["cobol"] = [ @@ -746,11 +753,11 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult [ // Include optional `*` between `function` and name for generator functions (e.g. `function* gen()`, `async function* asyncGen()`) // `function` と名前の間に任意の `*` を許容し、ジェネレータ関数 (`function* gen()`, `async function* asyncGen()`) にも対応 - new("function", new Regex(@"^\s*(?export)\s+(?default)\s+(?async\s+)?function(?:\s+|\s*(?\*)\s*)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*(?:(?export)\s+(?:default\s+)?)?(?async\s+)?function(?:\s+|\s*(?\*)\s*)(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function\s*(?\*)?\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function(?:\s+|\s*(?\*)\s*)\w+\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("function", new Regex(@"^\s*(?export)\s+(?default)\s+(?async\s+)?function(?:\s+|\s*(?\*)\s*)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), + new("function", new Regex(@"^\s*(?:(?export)\s+(?:default\s+)?)?(?async\s+)?function(?:\s+|\s*(?\*)\s*)(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), + new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "=>"), + new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function\s*(?\*)?\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), + new("function", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function(?:\s+|\s*(?\*)\s*)\w+\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), // HOC-wrapped / call-result component bindings such as // `const Wrapped = React.memo(...)`, `const Box = React.forwardRef(...)`, // `const Connected = connect(...)(Component)`, `const Styled = styled.div`...``, @@ -813,20 +820,20 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // stopAfterFirstPatternMatch が立ち、こちらで上書きされないようにする。 // Closes #240. new("function", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?[A-Z]\w*)\s*=\s*(?:React\.(?:memo|forwardRef|lazy)\s*\(|styled[.(`]|connect\s*\(|memo\s*\(|forwardRef\s*\(|lazy\s*\(|observer\s*\(|with[A-Z]\w*\s*\()", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("class", new Regex(@"^\s*(?:(?export)\s+)?(?:default\s+)?class\s+(?(?!extends\b)\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("import", new Regex(@"^\s*import\s+(?.+?)\s+from\s+", RegexOptions.Compiled), BodyStyle.None), + new("class", new Regex(@"^\s*(?:(?export)\s+)?(?:default\s+)?class\s+(?(?!extends\b)\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "class"), + new("import", new Regex(@"^\s*import\s+(?.+?)\s+from\s+", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["typescript"] = [ // Include optional `*` between `function` and name for generator functions (e.g. `function* gen()`, `async function* asyncGen()`) // `function` と名前の間に任意の `*` を許容し、ジェネレータ関数 (`function* gen()`, `async function* asyncGen()`) にも対応 - new("function", new Regex(@"^\s*(?export)\s+(?default)\s+(?async\s+)?function(?:\s+|\s*(?\*)\s*)" + TypeScriptOptionalTypeParameterListPattern + @"\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*(?:(?export)\s+(?:default\s+)?)?(?:declare\s+)?(?async\s+)?function(?:\s+|\s*(?\*)\s*)(?\w+)\s*[\(<]", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("import", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?type\s+(?\w+)" + TypeScriptOptionalTypeParameterListPattern + @"\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("property", new Regex(@"^\s*(?:(?export)\s+)?declare\s+(?:const|let|var)\s+(?\w+)(?::\s*[^;=]+)?\s*;", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function\s*(?\*)?\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function(?:\s+|\s*(?\*)\s*)\w+\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("function", new Regex(@"^\s*(?export)\s+(?default)\s+(?async\s+)?function(?:\s+|\s*(?\*)\s*)" + TypeScriptOptionalTypeParameterListPattern + @"\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), + new("function", new Regex(@"^\s*(?:(?export)\s+(?:default\s+)?)?(?:declare\s+)?(?async\s+)?function(?:\s+|\s*(?\*)\s*)(?\w+)\s*[\(<]", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), + new("import", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?type\s+(?\w+)" + TypeScriptOptionalTypeParameterListPattern + @"\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "type"), + new("property", new Regex(@"^\s*(?:(?export)\s+)?declare\s+(?:const|let|var)\s+(?\w+)(?::\s*[^;=]+)?\s*;", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "declare"), + new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?:async\s+)?(?:\([^)]*\)|[^=])\s*=>", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "=>"), + new("lambda", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function\s*(?\*)?\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), + new("function", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?\w+)\s*(?::\s*.+?)?\s*=\s*(?async\s+)?function(?:\s+|\s*(?\*)\s*)\w+\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), // HOC-wrapped / call-result component bindings — same narrow HOC-prefix set // as the JavaScript row above, extended with an optional TypeScript generic // type-argument token between the HOC call name and its `(` via the shared @@ -880,17 +887,17 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // Closes #240. new("function", new Regex(@"^\s*(?:(?export)\s+)?(?:const|let|var)\s+(?[A-Z]\w*)\s*(?::\s*.+?)?\s*=\s*(?:React\.(?:memo|forwardRef|lazy)\s*" + TypeScriptOptionalHocTypeArgsPattern + @"\(|styled[.(`]|connect\s*" + TypeScriptOptionalHocTypeArgsPattern + @"\(|memo\s*" + TypeScriptOptionalHocTypeArgsPattern + @"\(|forwardRef\s*" + TypeScriptOptionalHocTypeArgsPattern + @"\(|lazy\s*" + TypeScriptOptionalHocTypeArgsPattern + @"\(|observer\s*" + TypeScriptOptionalHocTypeArgsPattern + @"\(|with[A-Z]\w*\s*" + TypeScriptOptionalHocTypeArgsPattern + @"\()", RegexOptions.Compiled), BodyStyle.None, "visibility"), // Abstract class, declare class / 抽象クラス、declare クラス - new("class", new Regex(@"^\s*(?:(?export)\s+)?(?:default\s+)?(?:(?:abstract|declare)\s+)*class\s+(?(?!(?:extends|implements)\b)\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("class", new Regex(@"^\s*(?:(?export)\s+)?(?:default\s+)?(?:(?:abstract|declare)\s+)*class\s+(?(?!(?:extends|implements)\b)\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "class"), // UMD namespace export / UMD 名前空間エクスポート - new("namespace", new Regex($@"^\s*export\s+as\s+namespace\s+(?{JavaScriptTypeScriptIdentifierPattern})", RegexOptions.Compiled), BodyStyle.None), + new("namespace", new Regex($@"^\s*export\s+as\s+namespace\s+(?{JavaScriptTypeScriptIdentifierPattern})", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "namespace"), // namespace/module — supports both identifier (namespace Foo) and quoted ambient (declare module 'express') // 名前空間・モジュール — 識別子形式と引用符付きアンビエント形式の両方に対応 new("namespace", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?(?:namespace|module)\s+['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), new("namespace", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?(?:namespace|module)\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("interface", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?interface\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("import", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?type\s+(?\w+)(?:\s*<[^=]+>)?", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("enum", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?(?:const\s+)?enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("import", new Regex(@"^\s*import\s+(?.+?)\s+from\s+", RegexOptions.Compiled), BodyStyle.None), + new("interface", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?interface\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "interface"), + new("import", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?type\s+(?\w+)(?:\s*<[^=]+>)?", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "type"), + new("enum", new Regex(@"^\s*(?:(?export)\s+)?(?:declare\s+)?(?:const\s+)?enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), + new("import", new Regex(@"^\s*import\s+(?.+?)\s+from\s+", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["csharp"] = [ @@ -900,11 +907,11 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // verbatim / Unicode escape 識別子の各セグメントを `CSharpNamespacePattern` / // `CSharpIdentifierPattern` 経由で受け入れ、`CSharpSymbolNameNormalizer` で // canonical 化する。 - new("namespace", new Regex($@"^\s*namespace\s+(?{CSharpNamespacePattern})\s*;", RegexOptions.Compiled), BodyStyle.None), // file-scoped namespace (C# 10+) - new("namespace", new Regex($@"^\s*namespace\s+(?{CSharpNamespacePattern})", RegexOptions.Compiled), BodyStyle.Brace), // block-scoped namespace + new("namespace", new Regex($@"^\s*namespace\s+(?{CSharpNamespacePattern})\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "namespace"), // file-scoped namespace (C# 10+) + new("namespace", new Regex($@"^\s*namespace\s+(?{CSharpNamespacePattern})", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "namespace"), // block-scoped namespace // extern alias (must precede using directives per C# spec) — captures assembly-alias reconciliation // extern alias — C# 仕様上 using より前に置かれるファイル先頭宣言。アセンブリエイリアス用 - new("import", new Regex($@"^\s*extern\s+alias\s+(?{CSharpIdentifierPattern})\s*;", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex($@"^\s*extern\s+alias\s+(?{CSharpIdentifierPattern})\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "extern"), // using alias (using X = Y;) — must come before general using to capture alias name. // Verbatim alias identifiers like `using @AliasAttr = A.BaseAttr;` still surface as an // `import` row via `CSharpIdentifierPattern`; the DbWriter-side normalizer strips the @@ -912,8 +919,8 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // using エイリアス — 一般 using より前に配置しエイリアス名を取得。verbatim 識別子 // (`using @AliasAttr = A.BaseAttr;`) も `CSharpIdentifierPattern` 経由で import 行として // 拾える。 - new("import", new Regex($@"^\s*(?:global\s+)?using\s+(?{CSharpIdentifierPattern})\s*=\s*[^;]+;", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*(?:global\s+)?using\s+(?:static\s+)?(?[^;=]+);", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex($@"^\s*(?:global\s+)?using\s+(?{CSharpIdentifierPattern})\s*=\s*[^;]+;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "using"), + new("import", new Regex(@"^\s*(?:global\s+)?using\s+(?:static\s+)?(?[^;=]+);", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "using"), // Const field — must come before class/method patterns to avoid misclassification. // Modifier order is free: visibility may appear anywhere in the modifier sequence, // so `new public const` and `public new const` are both captured. Closes #355. @@ -932,7 +939,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // 取りこぼさない。従来の手書き文字クラスには `(` / `)` / `\s` が無く、 // `public const (int, int) Pair = (1, 2);` は returnType 群で失敗し、以降のどの行にも // マッチしなかった。Closes #346. - new("function", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:new|static)\s+)*const\s+(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"), + new("function", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:new|static)\s+)*const\s+(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType", RequiredLiteral: "const"), // Static readonly field / static readonly フィールド // Modifier order is free: `static` and `readonly` may appear in any order, and `new` // (member hiding) may appear anywhere in the modifier sequence. Visibility is also @@ -955,7 +962,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult + $@"(?=(?:(?:{CSharpVisibilityPattern}|new|static|readonly)\s+)*readonly\s+)" + $@"(?:(?{CSharpVisibilityPattern})\s+|(?:new|static|readonly)\s+)+" + $@"(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*[=;]", - RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"), + RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType", RequiredLiteral: "readonly"), // Plain field (instance, readonly, volatile, plain static, etc.). It keeps the // internal `property` tag used by scanner gating and normalizes public output to // `field`. Must come AFTER the `const` and `static readonly` patterns (which take priority @@ -991,16 +998,16 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // `new public interface` for nested types). Closes #355. // インターフェース — visibility 省略可。修飾子順序は自由 // （例: `partial public interface`、`file interface`、ネスト型向けの `new public interface`）。Closes #355. - new("interface", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:partial|unsafe|file|new)\s+)*interface\s+(?{CSharpIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("interface", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:partial|unsafe|file|new)\s+)*interface\s+(?{CSharpIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "interface"), // Enum — visibility optional / enum — visibility 省略可 - new("enum", CSharpEnumDeclarationRegex, BodyStyle.Brace, "visibility"), + new("enum", CSharpEnumDeclarationRegex, BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), // Struct (including record struct, ref struct, readonly struct) — visibility optional; // modifier order is free, so visibility may appear anywhere in the modifier sequence // (e.g. `readonly public struct`, `ref public struct`). Closes #355. // 構造体（record struct, ref struct, readonly struct を含む）— visibility 省略可。 // 修飾子順序は自由で、visibility は任意位置に置いてよい（例: `readonly public struct`、 // `ref public struct`）。Closes #355. - new("struct", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|partial|readonly|file|new|ref|unsafe)\s+)*(?:record\s+)?struct\s+(?{CSharpIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("struct", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|partial|readonly|file|new|ref|unsafe)\s+)*(?:record\s+)?struct\s+(?{CSharpIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "struct"), // Class (including record, record class) — visibility optional (defaults to internal // for top-level); modifier order is free, so visibility may appear anywhere in the // modifier sequence (e.g. `abstract public class`, `sealed public class`). Closes #355. @@ -1025,7 +1032,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult + $@"(?=(?:(?:{CSharpVisibilityPattern}|static|abstract|virtual|sealed|override|new|unsafe|extern)\s+)*static\s+)" + $@"(?:(?{CSharpVisibilityPattern})\s+|(?:static|abstract|virtual|sealed|override|new|unsafe|extern)\s+)+" + @"(?(?:implicit|explicit)\s+operator\s+.+?)\s*\(", - RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "operator"), // Operator overload (+ - * / == != < > etc.) — must come before method pattern. // Visibility may appear before or after `static`. Closes #355. // Modifier slot also accepts `abstract|virtual|sealed|override|new` so C# 11 @@ -1043,7 +1050,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult + $@"(?=(?:(?:{CSharpVisibilityPattern}|static|abstract|virtual|sealed|override|new|unsafe|extern)\s+)*static\s+)" + $@"(?:(?{CSharpVisibilityPattern})\s+|(?:static|abstract|virtual|sealed|override|new|unsafe|extern)\s+)+" + @".+?\s+(?operator\s+(?:checked\s+)?\S+)\s*\(", - RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "operator"), // Method with return type — visibility optional for explicit interface impl and nested members. // Negative lookahead excludes call-site lines (await/return/throw/yield/var/typeof/sizeof/nameof/default/if/for/while/switch/catch/lock/using) // and ternary continuation branches (`? Foo(...)` / `: Foo(...)`) that would otherwise resemble returnType + name. @@ -1066,9 +1073,9 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // コンストラクタ初期化子 (`: base(...)` / `: this(...)`) が phantom `function base` / `function this` // として漏れないよう二重化する。Closes #331. // 注意: `new` は除外しない。`new void Hidden()` は C# のメンバー隠蔽宣言として有効。 - new("function", new Regex($@"^\s*(?!\[\s*(?:assembly|module|type|return|param|field|property|event|method)\s*:)(?:(?{CSharpVisibilityPattern})\s+|(?:static|sealed|partial|readonly|unsafe|extern|virtual|override|abstract|new|file|ref(?:\s+readonly)?)\s+)*async\s+(?(?=[\w@?.<>\[\],:\s]*IAsync(?:Enumerable|Enumerator)\b){CSharpTypePattern})\s+(?!(?:base|this)\b)(?{CSharpIdentifierPattern})\s*{CSharpMethodTypeParameterListPattern}\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType"), + new("function", new Regex($@"^\s*(?!\[\s*(?:assembly|module|type|return|param|field|property|event|method)\s*:)(?:(?{CSharpVisibilityPattern})\s+|(?:static|sealed|partial|readonly|unsafe|extern|virtual|override|abstract|new|file|ref(?:\s+readonly)?)\s+)*async\s+(?(?=[\w@?.<>\[\],:\s]*IAsync(?:Enumerable|Enumerator)\b){CSharpTypePattern})\s+(?!(?:base|this)\b)(?{CSharpIdentifierPattern})\s*{CSharpMethodTypeParameterListPattern}\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType", RequiredLiteral: "IAsync"), new("function", new Regex($@"^\s*(?!\[\s*(?:assembly|module|type|return|param|field|property|event|method)\s*:)(?![?:])(?!(?:await|return|throw|yield|var|typeof|sizeof|nameof|default|if|for|foreach|while|switch|catch|lock|using|case|else|when|break|continue|goto|from|where|select|orderby|group|join|let|into|on|equals|ascending|descending|by)\b)(?!\s*(?:(?:{CSharpVisibilityPattern}|static|sealed|partial|readonly|unsafe|extern|virtual|override|abstract|async|new|file|ref(?:\s+readonly)?)\s+)*delegate\b(?!\s*\*))(?:(?{CSharpVisibilityPattern})\s+|(?:static|sealed|partial|readonly|unsafe|extern|virtual|override|abstract|async|new|file|ref(?:\s+readonly)?)\s+)*(?!{CSharpNonTypeKeywordPattern})(?{CSharpTypePattern})\s+(?!(?:base|this)\b)(?{CSharpIdentifierPattern})\s*{CSharpMethodTypeParameterListPattern}\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType"), - new("lambda", new Regex($@"^\s*(?:var|{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*=\s*(?:async\s+)?(?:\([^)]*\)|{CSharpIdentifierPattern})\s*=>", RegexOptions.Compiled), BodyStyle.None), + new("lambda", new Regex($@"^\s*(?:var|{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*=\s*(?:async\s+)?(?:\([^)]*\)|{CSharpIdentifierPattern})\s*=>", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "=>"), // Constructor (no return type, name followed by parenthesis) — needs visibility. // `unsafe` / `extern` can appear before or after visibility, and C# 14 partial // constructors place `partial` after visibility, so declarations like @@ -1127,7 +1134,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // 戻り値型を省略した partial method 宣言。旧来の partial method 構文では // accessibility を省略し、戻り値型は `void` とみなされる。`public partial Widget();` // は constructor のまま扱うため、この行は constructor 行の後ろに置く。 - new("function", new Regex($@"^\s*(?:(?:static|sealed|readonly|unsafe|extern|virtual|override|abstract|async|new|file|ref(?:\s+readonly)?)\s+)*(?partial)\s+(?{CSharpIdentifierPattern})\s*{CSharpMethodTypeParameterListPattern}\(", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), + new("function", new Regex($@"^\s*(?:(?:static|sealed|readonly|unsafe|extern|virtual|override|abstract|async|new|file|ref(?:\s+readonly)?)\s+)*(?partial)\s+(?{CSharpIdentifierPattern})\s*{CSharpMethodTypeParameterListPattern}\(", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType", RequiredLiteral: "partial"), // Static constructor / 静的コンストラクタ // Keep this ahead of the property rows so same-line compact bodies such as // `class C { static C() { } public int P { get; set; } }` emit the static ctor @@ -1139,7 +1146,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // pattern scan を打ち切る前に static ctor を先に拾う必要があるため、property 行より前に置く。 // この形は「戻り値型なし・引数なし・`static` 前後の任意 `unsafe`」に限定されるため、 // 通常メソッドとは重ならない。Closes #478. - new("function", new Regex($@"^\s*(?:unsafe\s+)?static\s+(?:unsafe\s+)?(?{CSharpIdentifierPattern})\s*\(\s*\)\s*\{{?", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex($@"^\s*(?:unsafe\s+)?static\s+(?:unsafe\s+)?(?{CSharpIdentifierPattern})\s*\(\s*\)\s*\{{?", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "static"), // Property with get/set/init — visibility optional // Reject statement keywords (return/throw/switch/...) as the return type so that // multi-line statement fragments merged by BuildCSharpPropertyMatchLine — e.g. @@ -1162,12 +1169,12 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // ReferenceExtractor.FindInnermostContainer が accessor 内呼び出しを外側 // クラスではなく property に帰属させるために必要。 // Closes #233. - new("property", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|virtual|override|abstract|sealed|new|required|partial|readonly|unsafe|extern|ref(?:\s+readonly)?)\s+)*(?!(?:class|struct|interface|enum|record|namespace|delegate|event|const|using|return|throw|yield|var|typeof|sizeof|nameof|default|if|for|foreach|while|switch|catch|lock|case|else|when|break|continue|goto|await)\b)(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*=>\s*", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType"), + new("property", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|virtual|override|abstract|sealed|new|required|partial|readonly|unsafe|extern|ref(?:\s+readonly)?)\s+)*(?!(?:class|struct|interface|enum|record|namespace|delegate|event|const|using|return|throw|yield|var|typeof|sizeof|nameof|default|if|for|foreach|while|switch|catch|lock|case|else|when|break|continue|goto|await)\b)(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*=>\s*", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType", RequiredLiteral: "=>"), // Delegate — visibility optional; modifier order is free. Accepts `static` / `unsafe` / // `file` (file-scoped delegate) / `new` (nested delegate hiding). Closes #355. // デリゲート — visibility 省略可。修飾子順序は自由。`static` / `unsafe` / // `file`（file スコープ delegate）/ `new`（ネスト delegate の隠蔽）を受け付ける。Closes #355. - new("delegate", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|unsafe|file|new)\s+)*delegate\s+(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*[\(<]", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"), + new("delegate", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|unsafe|file|new)\s+)*delegate\s+(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*[\(<]", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType", RequiredLiteral: "delegate"), // Event — visibility optional; modifier order is free. Accepts `static` / `unsafe` / // `extern` plus inheritance modifiers (`virtual` / `override` / `abstract` / `sealed` / `new`) // which are all legal on event declarations per the C# spec. `partial` is also legal on @@ -1179,7 +1186,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // も受け付ける。event には `partial` も合法 (C# 14 field-like partial event、およびアクセサ // ベースの partial member 拡張) なので、ここでも受け付けないと `partial event` 宣言が // symbols / definition / outline から無言で欠落する。Closes #350. - new("event", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|unsafe|extern|virtual|override|abstract|sealed|new|partial)\s+)*event\s+(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*(?:[;=]|\{{)", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"), + new("event", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|unsafe|extern|virtual|override|abstract|sealed|new|partial)\s+)*event\s+(?{CSharpTypePattern})\s+(?{CSharpIdentifierPattern})\s*(?:[;=]|\{{)", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType", RequiredLiteral: "event"), // Explicit interface event implementation (e.g. event EventHandler IFoo.Changed) // must capture the trailing member name rather than dropping the declaration or // inventing the qualifier as the event name. BodyStyle.Brace lets accessor blocks @@ -1188,7 +1195,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // qualifier 側ではなく末尾のメンバー名を event 名として捕捉しなければならない。 // BodyStyle.Brace を使い、同一行/次行どちらの accessor block も通常の brace-range // 経路で扱う。 - new("event", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|unsafe|extern|virtual|override|abstract|sealed|new|partial)\s+)*event\s+(?{CSharpTypePattern})\s+(?{CSharpExplicitInterfaceQualifierPattern})\s*\.\s*(?{CSharpIdentifierPattern})\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType"), + new("event", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|unsafe|extern|virtual|override|abstract|sealed|new|partial)\s+)*event\s+(?{CSharpTypePattern})\s+(?{CSharpExplicitInterfaceQualifierPattern})\s*\.\s*(?{CSharpIdentifierPattern})\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType", RequiredLiteral: "event"), // Explicit interface implementation (e.g. void IDisposable.Dispose()) // Requires a valid return type (not a statement keyword) and interface name before the dot. // Reject named-argument labels only when they are followed by a qualified call site, @@ -1233,12 +1240,12 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult new("property", new Regex($@"^\s*(?![?:])(?!(?:class|struct|interface|enum|record|namespace|delegate|event|const|using|return|throw|yield|var|typeof|sizeof|nameof|default|if|for|foreach|while|switch|catch|lock|case|else|when|break|continue|goto|await)\b)(?:(?ref(?:\s+readonly)?)\s+)?(?{CSharpTypePattern})\s+(?{CSharpExplicitInterfaceQualifierPattern})\s*\.\s*(?{CSharpIdentifierPattern})\s*\{{", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), // Explicit interface property implementation (expression body), e.g. string IThing.Name => "x"; // 明示的インターフェースプロパティ実装（式本体）。例: string IThing.Name => "x"; - new("property", new Regex($@"^\s*(?![?:])(?!(?:class|struct|interface|enum|record|namespace|delegate|event|const|using|return|throw|yield|var|typeof|sizeof|nameof|default|if|for|foreach|while|switch|catch|lock|case|else|when|break|continue|goto|await)\b)(?:(?ref(?:\s+readonly)?)\s+)?(?{CSharpTypePattern})\s+(?{CSharpExplicitInterfaceQualifierPattern})\s*\.\s*(?{CSharpIdentifierPattern})\s*=>\s*", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), + new("property", new Regex($@"^\s*(?![?:])(?!(?:class|struct|interface|enum|record|namespace|delegate|event|const|using|return|throw|yield|var|typeof|sizeof|nameof|default|if|for|foreach|while|switch|catch|lock|case|else|when|break|continue|goto|await)\b)(?:(?ref(?:\s+readonly)?)\s+)?(?{CSharpTypePattern})\s+(?{CSharpExplicitInterfaceQualifierPattern})\s*\.\s*(?{CSharpIdentifierPattern})\s*=>\s*", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType", RequiredLiteral: "=>"), // Explicit interface indexer implementation. The display name stays `Item`, while // the captured qualifier becomes part of the persisted exact-query identity. // 明示的インターフェース indexer 実装。表示名は `Item` のままにし、捕捉した // qualifier は永続化する完全一致 query identity に含める。 - new("function", new Regex($@"^\s*(?![?:])(?:(?ref(?:\s+readonly)?)\s+)?(?{CSharpTypePattern})\s+(?{CSharpExplicitInterfaceQualifierPattern})\s*\.\s*(?this)\s*\[", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), + new("function", new Regex($@"^\s*(?![?:])(?:(?ref(?:\s+readonly)?)\s+)?(?{CSharpTypePattern})\s+(?{CSharpExplicitInterfaceQualifierPattern})\s*\.\s*(?this)\s*\[", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType", RequiredLiteral: "this"), // Indexer (this[...]) — `partial` is legal on indexers since C# 13 (extended partial // member support), so accept it alongside the other modifiers. Otherwise every // `partial` indexer declaration would be silently dropped from symbols / definition / @@ -1246,7 +1253,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // インデクサ (this[...]) — C# 13 で indexer に対しても `partial` が使える (partial // member 拡張) ため、他の修飾子と並べて受け付ける。そうしないと `partial` indexer 宣言 // が symbols / definition / outline から無言で欠落する。Closes #350. - new("function", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|virtual|override|abstract|sealed|new|readonly|unsafe|extern|partial|ref(?:\s+readonly)?)\s+)*(?{CSharpTypePattern})\s+(?this)\s*\[", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType"), + new("function", new Regex($@"^\s*(?:(?{CSharpVisibilityPattern})\s+|(?:static|virtual|override|abstract|sealed|new|readonly|unsafe|extern|partial|ref(?:\s+readonly)?)\s+)*(?{CSharpTypePattern})\s+(?this)\s*\[", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType", RequiredLiteral: "this"), // Finalizer (destructor) / ファイナライザ（デストラクタ） new("function", new Regex($@"^\s*~(?{CSharpIdentifierPattern})\s*\(\s*\)", RegexOptions.Compiled), BodyStyle.Brace), // Enum member (e.g. Red, Green = 1,) — requires 4+ spaces indent, name only, @@ -1255,23 +1262,23 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // 数値/16進/識別子の値指定はオプション。文字列/オブジェクト代入にはマッチしない。 new("enum", CSharpEnumMemberRegex, BodyStyle.None), // #region for navigation / ナビゲーション用 #region - new("namespace", new Regex(@"^\s*#region\s+(?.+)$", RegexOptions.Compiled), BodyStyle.None), + new("namespace", new Regex(@"^\s*#region\s+(?.+)$", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "#region"), ], ["go"] = [ - new("namespace", new Regex(@"^\s*package\s+(?[A-Za-z_]\w*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^func\s+(?:\([^)]+\)\s+)?(?\w+)(?:\[[^\]\r\n]+\])?\s*[\(\[]", RegexOptions.Compiled), BodyStyle.Brace), - new("lambda", new Regex(@"^\s*(?\w+)\s*(?::=|=)\s*func\s*\(", RegexOptions.Compiled), BodyStyle.Brace), - new("struct", new Regex(@"^type\s+(?\w+)(?:\[[^\]]+\])?\s+struct\b", RegexOptions.Compiled), BodyStyle.Brace), - new("protocol", new Regex(@"^type\s+(?\w+)(?:\[[^\]]+\])?\s+interface\b", RegexOptions.Compiled), BodyStyle.Brace), + new("namespace", new Regex(@"^\s*package\s+(?[A-Za-z_]\w*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "package"), + new("function", new Regex(@"^func\s+(?:\([^)]+\)\s+)?(?\w+)(?:\[[^\]\r\n]+\])?\s*[\(\[]", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "func"), + new("lambda", new Regex(@"^\s*(?\w+)\s*(?::=|=)\s*func\s*\(", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "func"), + new("struct", new Regex(@"^type\s+(?\w+)(?:\[[^\]]+\])?\s+struct\b", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "struct"), + new("protocol", new Regex(@"^type\s+(?\w+)(?:\[[^\]]+\])?\s+interface\b", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "interface"), // Type alias (type Name = OtherType or type Name OtherType) / 型エイリアス - new("import", new Regex(@"^type\s+(?\w+)(?:\[[^\]]+\])?\s+[=\w]", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"^type\s+(?\w+)(?:\[[^\]]+\])?\s+[=\w]", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), // Top-level const declarations / トップレベル const 宣言 - new("property", new Regex(@"^const\s+(?\w+)(?:\s+\w[\w.*\[\]]*)?\s*=", RegexOptions.Compiled), BodyStyle.None), + new("property", new Regex(@"^const\s+(?\w+)(?:\s+\w[\w.*\[\]]*)?\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "const"), // Const declaration inside const block / const ブロック内の定数宣言 new("property", new Regex(@"^\s+(?[A-Z]\w*)\s*=\s*", RegexOptions.Compiled), BodyStyle.None), // Package-level var / パッケージレベル変数 - new("property", new Regex(@"^var\s+(?\w+)\s", RegexOptions.Compiled), BodyStyle.None), + new("property", new Regex(@"^var\s+(?\w+)\s", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "var"), ], ["fortran"] = [ @@ -1317,51 +1324,51 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ["rust"] = [ // macro_rules! / マクロ定義 - new("function", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?macro_rules!\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("function", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?macro_rules!\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "macro_rules!"), // const/static items / 定数・静的変数 new("function", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?(?:const|static)\s+(?(?:r#)?\w+)\s*:", RegexOptions.Compiled), BodyStyle.None, "visibility"), // fn with expanded modifiers: async, const, unsafe, default, extern (ABI optional) / // 拡張修飾子: async, const, unsafe, default, extern（ABI は省略可） - new("function", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?(?:(?:async|const|unsafe|default|extern(?:\s+""[^""]+"")?)\s+)*fn\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("class", new Regex(@"\b(?unsafe)\s*\{", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?(?:(?:async|const|unsafe|default|extern(?:\s+""[^""]+"")?)\s+)*fn\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "fn"), + new("class", new Regex(@"\b(?unsafe)\s*\{", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "unsafe"), new("struct", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?(?:struct|union)\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("enum", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?enum\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("enum", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?enum\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), // Enum variants / `Red`, `Ok(T)`, `Circle { radius: f64 }`, `Point` new("property", new Regex(@"^\s{4,}(?[A-Z][A-Za-z0-9_]*)\s*(?:\([^()\r\n]*\)|\{[^{}\r\n]*\})?\s*,?\s*$", RegexOptions.Compiled), BodyStyle.None), - new("protocol", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?trait\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("protocol", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?trait\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "trait"), // impl Trait for Type / `unsafe impl Trait for Type` should attach to the type being extended. // `impl Trait for Type` / `unsafe impl Trait for Type` は、拡張先の型に紐づける。 - new("class", new Regex(@"^\s*(?:unsafe\s+)?impl(?:<[^>]+>)?\s+.+?\s+for\s+(?:(?:r#)?\w+::)*(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*(?:unsafe\s+)?impl(?:<[^>]+>)?\s+(?:(?:r#)?\w+::)*(?(?:r#)?\w+)(?!\s+for\b)", RegexOptions.Compiled), BodyStyle.Brace), + new("class", new Regex(@"^\s*(?:unsafe\s+)?impl(?:<[^>]+>)?\s+.+?\s+for\s+(?:(?:r#)?\w+::)*(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "impl"), + new("class", new Regex(@"^\s*(?:unsafe\s+)?impl(?:<[^>]+>)?\s+(?:(?:r#)?\w+::)*(?(?:r#)?\w+)(?!\s+for\b)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "impl"), // file module declarations and inline modules / ファイルモジュール宣言とインラインモジュール - new("file_module", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?mod\s+(?(?:r#)?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("namespace", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?mod\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("file_module", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?mod\s+(?(?:r#)?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "mod"), + new("namespace", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?mod\s+(?(?:r#)?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "mod"), // Trait associated type defaults / trait 関連型のデフォルト - new("property", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?type\s+(?(?:r#)?\w+)(?:\s*<[^=>]+>)?(?:\s*:\s*[^=;]+)?\s*=\s*(?[^;]+)", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"), + new("property", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?type\s+(?(?:r#)?\w+)(?:\s*<[^=>]+>)?(?:\s*:\s*[^=;]+)?\s*=\s*(?[^;]+)", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType", RequiredLiteral: "type"), // type alias / 型エイリアス - new("import", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?type\s+(?(?:r#)?\w+)(?:\s*<[^=]+>)?", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("import", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?use\s+(?.+);", RegexOptions.Compiled), BodyStyle.None, "visibility"), + new("import", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?type\s+(?(?:r#)?\w+)(?:\s*<[^=]+>)?", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "type"), + new("import", new Regex(@"^\s*(?:(?pub(?:\([^)]*\))?)\s+)?use\s+(?.+);", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "use"), ], ["java"] = [ // Package declaration / package 宣言 - new("namespace", new Regex($@"^\s*package\s+(?{JavaQualifiedIdentifierPattern})\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex($@"^\s*package\s+(?{JavaQualifiedIdentifierPattern})\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "package"), // Module declaration (Java 9+ module-info.java) / モジュール宣言（Java 9+ の module-info.java） - new("namespace", new Regex($@"^\s*(?:open\s+)?module\s+(?{JavaQualifiedIdentifierPattern})\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("namespace", new Regex($@"^\s*(?:open\s+)?module\s+(?{JavaQualifiedIdentifierPattern})\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "module"), // Annotation type (@interface) / アノテーション型 - new("class", new Regex($@"^\s*(?public|private|protected)?\s*@interface\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility"), + new("class", new Regex($@"^\s*(?public|private|protected)?\s*@interface\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility", RequiredLiteral: "@interface"), // record (Java 16+) — must come before general class pattern / record は一般クラスパターンの前に配置 - new("class", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|final|abstract|sealed|non-sealed|strictfp)\s+)*record\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility"), + new("class", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|final|abstract|sealed|non-sealed|strictfp)\s+)*record\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility", RequiredLiteral: "record"), // Interface / インターフェース - new("interface", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|abstract|sealed|non-sealed|strictfp)\s+)*interface\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility"), + new("interface", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|abstract|sealed|non-sealed|strictfp)\s+)*interface\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility", RequiredLiteral: "interface"), // Enum / enum - new("enum", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|strictfp)\s+)*enum\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility"), + new("enum", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|strictfp)\s+)*enum\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), // Class — with extended modifiers (final, sealed, static, abstract, strictfp) // クラス — 拡張修飾子対応（final, sealed, static, abstract, strictfp） - new("class", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|final|abstract|sealed|non-sealed|strictfp)\s+)*class\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility"), + new("class", new Regex($@"^\s*(?public|private|protected)?\s*(?:(?:static|final|abstract|sealed|non-sealed|strictfp)\s+)*class\s+(?{JavaIdentifierPattern})", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility", RequiredLiteral: "class"), // Static final field (Java equivalent of C# const) — order-flexible and annotation-friendly. // static final フィールド — 語順柔軟かつアノテーション併用にも対応。 - new("function", new Regex($@"^\s*(?:@\w+(?:\([^)]*\))?\s+)*(?public|private|protected)?\s*(?=(?:(?:static|final|transient|volatile)\s+)*static\b)(?=(?:(?:static|final|transient|volatile)\s+)*final\b)(?:(?:static|final|transient|volatile)\s+)*(?{JavaReturnTypePattern})\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, "visibility", "returnType"), + new("function", new Regex($@"^\s*(?:@\w+(?:\([^)]*\))?\s+)*(?public|private|protected)?\s*(?=(?:(?:static|final|transient|volatile)\s+)*static\b)(?=(?:(?:static|final|transient|volatile)\s+)*final\b)(?:(?:static|final|transient|volatile)\s+)*(?{JavaReturnTypePattern})\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, "visibility", "returnType", RequiredLiteral: "static"), // Method with return type — expanded modifiers (default, native, synchronized, final) // 戻り値型付きメソッド — 拡張修飾子対応（default, native, synchronized, final） new("function", new Regex($@"^\s*(?!(?:return|throw|new|if|for|while|switch|do|case|else|try|catch|finally|synchronized|break|continue|yield|assert)\b)(?:@\w+(?:\([^)]*\))?\s+)*(?public|private|protected)?\s*(?:(?:static|abstract|synchronized|final|default|native|strictfp)\s+)*(?!(?:record)\b){JavaMethodTypeParameterPattern}(?{JavaReturnTypePattern})\s+(?{JavaIdentifierPattern})\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, "visibility", "returnType"), @@ -1371,100 +1378,100 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // enum メンバーは ExtractJavaEnumMembers の body-scoped scanner で抽出する。 // 任意のインデントスタイル（タブ、2スペース、4スペース）に対応しつつ、enum 本体外の // メンバー風の行（例: クラス本体内の `\tRED();` メソッド呼び出し）を誤検出しない。 - new("import", new Regex(@"^\s*import\s+(?.+);", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"^\s*import\s+(?.+);", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["kotlin"] = [ // Companion object / コンパニオンオブジェクト - new("class", new Regex($@"^\s*companion\s+object(?:\s+(?{KotlinIdentifierPattern}))?", RegexOptions.Compiled), BodyStyle.Brace), + new("class", new Regex($@"^\s*companion\s+object(?:\s+(?{KotlinIdentifierPattern}))?", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "companion"), // Interface / インターフェース // Kotlin fun interface / Kotlin の fun interface も interface として扱う。 - new("interface", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:sealed|expect|actual)\s+)*(?:fun\s+)?interface\s+(?{KotlinIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("interface", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:sealed|expect|actual)\s+)*(?:fun\s+)?interface\s+(?{KotlinIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "interface"), // Enum class / enum クラス - new("enum", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:expect|actual)\s+)*enum\s+class\s+(?{KotlinIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("enum", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:expect|actual)\s+)*enum\s+class\s+(?{KotlinIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), // Class/object with expanded modifiers: data, sealed, value, inline, inner, annotation, expect, actual // クラス/オブジェクト — 拡張修飾子対応: data, sealed, value, inline, inner, annotation, expect, actual new("class", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:abstract|data|sealed|open|inner|value|inline|annotation|expect|actual)\s+)*(?:class|object)\s+(?{KotlinIdentifierPattern})", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), // Function / 関数 (including extension, secondary constructor, override, and abstract forms) // 関数 — 拡張・セカンダリコンストラクタ・override・abstract 形を含む - new("function", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:suspend|inline|infix|operator|tailrec|external|expect|actual|abstract|override|open|final)\s+)*fun\s+(?:<[^>]+>\s+)?(?:{KotlinIdentifierPattern}(?:<[^>]+>)?\.)?(?{KotlinIdentifierPattern})\s*[\(<](?:.*?\))?(?::\s*(?[^ {{=]+))?", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType"), + new("function", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:suspend|inline|infix|operator|tailrec|external|expect|actual|abstract|override|open|final)\s+)*fun\s+(?:<[^>]+>\s+)?(?:{KotlinIdentifierPattern}(?:<[^>]+>)?\.)?(?{KotlinIdentifierPattern})\s*[\(<](?:.*?\))?(?::\s*(?[^ {{=]+))?", RegexOptions.Compiled), BodyStyle.Brace, "visibility", "returnType", RequiredLiteral: "fun"), // Secondary constructor / セカンダリコンストラクタ - new("function", new Regex(@"^\s*(?public|private|protected|internal)?\s*constructor\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("function", new Regex(@"^\s*(?public|private|protected|internal)?\s*constructor\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "constructor"), // Enum entry / enum エントリ new("property", new Regex($@"^\s{{2,}}(?(?:[A-Z][A-Z0-9_]*|`[^`\r\n]+`))\s*(?:\((?[^)]*)\))?\s*(?:,|\{{|;)?\s*$", RegexOptions.Compiled), BodyStyle.Brace, "returnType"), // Top-level val/var property / トップレベルプロパティ new("property", new Regex($@"^\s*(?public|private|protected|internal)?\s*(?:(?:const|lateinit|override)\s+)?(?:val|var)\s+(?{KotlinIdentifierPattern})\s*[=:]", RegexOptions.Compiled), BodyStyle.None, "visibility"), // Type alias / 型エイリアス - new("import", new Regex($@"^\s*(?public|private|protected|internal)?\s*typealias\s+(?{KotlinIdentifierPattern})(?:\s*<[^=]+>)?\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("import", new Regex(@"^\s*import\s+(?.+)", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex($@"^\s*(?public|private|protected|internal)?\s*typealias\s+(?{KotlinIdentifierPattern})(?:\s*<[^=]+>)?\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "typealias"), + new("import", new Regex(@"^\s*import\s+(?.+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["ruby"] = [ // attr_accessor/attr_reader/attr_writer as property declarations / プロパティ宣言 - new("property", new Regex(@"^\s*attr_(?:accessor|reader|writer)\s+:(?\w+)", RegexOptions.Compiled), BodyStyle.None), + new("property", new Regex(@"^\s*attr_(?:accessor|reader|writer)\s+:(?\w+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "attr_"), // alias_method / alias — capture the introduced method name for navigation - new("function", new Regex(@"^\s*alias_method\b\s+:?(?\w+[?!=]?)\s*,\s*:?\w+[?!=]?", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*alias\b\s+:?(?\w+[?!=]?)\s+:?\w+[?!=]?", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^\s*alias_method\b\s+:?(?\w+[?!=]?)\s*,\s*:?\w+[?!=]?", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "alias_method"), + new("function", new Regex(@"^\s*alias\b\s+:?(?\w+[?!=]?)\s+:?\w+[?!=]?", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "alias"), // scope/has_many/belongs_to (Rails DSL) — extracted as function for navigation new("function", new Regex(@"^\s*(?:scope|has_many|has_one|belongs_to)\s+:(?\w+)", RegexOptions.Compiled), BodyStyle.None), - new("property", new Regex(@"^\s*enum\s+:(?\w+)", RegexOptions.Compiled), BodyStyle.None), - new("property", new Regex(@"^\s*attribute\s+:(?\w+)", RegexOptions.Compiled), BodyStyle.None), - new("property", new Regex(@"^\s*store_accessor\s+:\w+\s*,\s*:(?\w+)", RegexOptions.Compiled), BodyStyle.None), - new("namespace", new Regex(@"^\s*namespace\s+:(?\w+)\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("function", new Regex(@"^\s*factory\s+:(?\w+)\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("function", new Regex(@"^\s*shared_examples(?:_for)?\s+(?['""])(?[^'""]+)\k\s*do\b", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("property", new Regex(@"^\s*subject\s*\(\s*:(?\w+)\s*\)\s*do\b", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("property", new Regex(@"^\s*let!?\s*\(\s*:(?\w+)\s*\)\s*do\b", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("function", new Regex(@"^\s*task\s+(?::(?\w+)|(?\w+)\s*:)", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*(?[A-Z][A-Za-z0-9_]*)\s*=\s*Class\.new\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("class", new Regex(@"^\s*(?[A-Z][A-Za-z0-9_]*)\s*=\s*Struct\.new\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd), + new("property", new Regex(@"^\s*enum\s+:(?\w+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "enum"), + new("property", new Regex(@"^\s*attribute\s+:(?\w+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "attribute"), + new("property", new Regex(@"^\s*store_accessor\s+:\w+\s*,\s*:(?\w+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "store_accessor"), + new("namespace", new Regex(@"^\s*namespace\s+:(?\w+)\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "namespace"), + new("function", new Regex(@"^\s*factory\s+:(?\w+)\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "factory"), + new("function", new Regex(@"^\s*shared_examples(?:_for)?\s+(?['""])(?[^'""]+)\k\s*do\b", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "shared_examples"), + new("property", new Regex(@"^\s*subject\s*\(\s*:(?\w+)\s*\)\s*do\b", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "subject"), + new("property", new Regex(@"^\s*let!?\s*\(\s*:(?\w+)\s*\)\s*do\b", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "let"), + new("function", new Regex(@"^\s*task\s+(?::(?\w+)|(?\w+)\s*:)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "task"), + new("class", new Regex(@"^\s*(?[A-Z][A-Za-z0-9_]*)\s*=\s*Class\.new\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "Class.new"), + new("class", new Regex(@"^\s*(?[A-Z][A-Za-z0-9_]*)\s*=\s*Struct\.new\b.*\bdo\b", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "Struct.new"), new("property", new Regex(@"^\s*(?[A-Z][A-Za-z0-9_]*)\s*=", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*(?:(?:private|protected|public)\s+)?def\s+(?:(?:self|[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)\.)?(?\[\]=?|\*\*|<<|>>|<=>|===|==|!=|!~|=~|<=|>=|[+\-*/%&|^~<>]=?|[+\-]@|!)", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("function", new Regex(@"^\s*(?:(?:private|protected|public)\s+)?def\s+(?:(?:self|[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)\.)?(?\w+[?!=]?)", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("class", new Regex(@"^\s*class\s+(?[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("class", new Regex(@"^\s*module\s+(?[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)", RegexOptions.Compiled), BodyStyle.RubyEnd), - new("import", new Regex(@"^\s*require(?:_relative)?\s+(?.+)", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^\s*(?:(?:private|protected|public)\s+)?def\s+(?:(?:self|[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)\.)?(?\[\]=?|\*\*|<<|>>|<=>|===|==|!=|!~|=~|<=|>=|[+\-*/%&|^~<>]=?|[+\-]@|!)", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "def"), + new("function", new Regex(@"^\s*(?:(?:private|protected|public)\s+)?def\s+(?:(?:self|[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)\.)?(?\w+[?!=]?)", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "def"), + new("class", new Regex(@"^\s*class\s+(?[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "class"), + new("class", new Regex(@"^\s*module\s+(?[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*)", RegexOptions.Compiled), BodyStyle.RubyEnd, RequiredLiteral: "module"), + new("import", new Regex(@"^\s*require(?:_relative)?\s+(?.+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "require"), ], ["crystal"] = [ - new("namespace", new Regex(@"^\s*module\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), - new("class", new Regex(@"^\s*(?:abstract\s+)?class\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), - new("struct", new Regex(@"^\s*struct\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), - new("enum", new Regex(@"^\s*enum\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), - new("function", new Regex(@"^\s*(?:(?:private|protected)\s+)*abstract\s+def\s+(?:self\.)?(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?:private\s+|protected\s+)?def\s+(?:self\.)?(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), - new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), - new("typealias", new Regex(@"^\s*alias\s+(?[A-Z]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*require\s+(?.+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*module\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd, RequiredLiteral: "module"), + new("class", new Regex(@"^\s*(?:abstract\s+)?class\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd, RequiredLiteral: "class"), + new("struct", new Regex(@"^\s*struct\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd, RequiredLiteral: "struct"), + new("enum", new Regex(@"^\s*enum\s+(?[A-Z]\w*(?:::[A-Z]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd, RequiredLiteral: "enum"), + new("function", new Regex(@"^\s*(?:(?:private|protected)\s+)*abstract\s+def\s+(?:self\.)?(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "abstract"), + new("function", new Regex(@"^\s*(?:private\s+|protected\s+)?def\s+(?:self\.)?(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd, RequiredLiteral: "def"), + new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*[?!=]?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd, RequiredLiteral: "macro"), + new("typealias", new Regex(@"^\s*alias\s+(?[A-Z]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "alias"), + new("import", new Regex(@"^\s*require\s+(?.+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "require"), ], ["groovy"] = [ - new("namespace", new Regex(@"^\s*package\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*package\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "package"), new("interface", new Regex(@"^\s*(?:(?:public|private|protected|static|abstract)\s+)*(?:interface|trait)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("enum", new Regex(@"^\s*(?:(?:public|private|protected|static)\s+)*enum\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("class", new Regex(@"^\s*(?:(?:public|private|protected|static|abstract|final)\s+)*class\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("enum", new Regex(@"^\s*(?:(?:public|private|protected|static)\s+)*enum\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "enum"), + new("class", new Regex(@"^\s*(?:(?:public|private|protected|static|abstract|final)\s+)*class\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "class"), new("function", new Regex(@"^\s*(?:@[A-Za-z_$][\w.$]*(?:\s*\([^)\r\n]*\))?\s+)*(?!(?:if|for|while|switch|catch|return|throw|new)\b)(?:(?:public|private|protected|static|final|abstract|synchronized|native|strictfp)\s+)*(?:<[^(){}\r\n]+>\s+)?(?def|void|boolean|byte|char|short|int|long|float|double|BigDecimal|BigInteger|String|[A-Za-z_$][\w.$]*(?:\s*<[^(){}\r\n]+>)?(?:\s*\[\])*)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), new("lambda", new Regex(@"^\s*(?:def\s+)?(?[A-Za-z_]\w*)\s*=\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("import", new Regex(@"^\s*import\s+(?:static\s+)?(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*(?:\.\*)?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*import\s+(?:static\s+)?(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*(?:\.\*)?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "import"), ], ["julia"] = [ - new("namespace", new Regex(@"^\s*(?:baremodule|module)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("struct", new Regex(@"^\s*(?:mutable\s+)?struct\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("type", new Regex(@"^\s*(?:abstract|primitive)\s+type\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("function", new Regex(@"^\s*function\s+(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), + new("namespace", new Regex(@"^\s*(?:baremodule|module)\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd, RequiredLiteral: "module"), + new("struct", new Regex(@"^\s*(?:mutable\s+)?struct\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd, RequiredLiteral: "struct"), + new("type", new Regex(@"^\s*(?:abstract|primitive)\s+type\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd, RequiredLiteral: "type"), + new("function", new Regex(@"^\s*function\s+(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*(?:\(|\{)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd, RequiredLiteral: "function"), + new("function", new Regex(@"^\s*macro\s+(?[A-Za-z_]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd, RequiredLiteral: "macro"), new("function", new Regex(@"^\s*(?" + JuliaQualifiedCallableIdentifierPattern + @")\s*\([^)\r\n]*\)\s*(?:where\s*(?:\{[^}\r\n]*\}|[A-Za-z_]\w*)\s*)?=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.JuliaShortFunction), - new("property", new Regex(@"^\s*const\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("property", new Regex(@"^\s*const\s+(?[A-Z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "const"), new("import", new Regex(@"^\s*(?:using|import)\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["tcl"] = [ - new("namespace", new Regex(@"^\s*namespace\s+eval\s+(?[A-Za-z_:][\w:.-]*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("class", new Regex(@"^\s*oo::class\s+create\s+(?[A-Za-z_:][\w:.-]*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*proc\s+(?[A-Za-z_:][\w:.-]*)\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*namespace\s+eval\s+(?[A-Za-z_:][\w:.-]*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "namespace"), + new("class", new Regex(@"^\s*oo::class\s+create\s+(?[A-Za-z_:][\w:.-]*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "oo::class"), + new("function", new Regex(@"^\s*proc\s+(?[A-Za-z_:][\w:.-]*)\s+", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "proc"), new("property", new Regex(@"^\s*(?:variable|set)\s+(?[A-Za-z_:][\w:.-]*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*package\s+(?:require|provide)\s+(?[A-Za-z_:][\w:.-]*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*package\s+(?:require|provide)\s+(?[A-Za-z_:][\w:.-]*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "package"), ], ["ada"] = [ @@ -1476,143 +1483,143 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["d"] = [ - new("namespace", new Regex(@"^\s*module\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("interface", new Regex(@"^\s*(?:(?:public|private|protected|package|static|abstract|extern)\s+)*interface\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("class", new Regex(@"^\s*(?:(?:public|private|protected|package|static|abstract|final|extern)\s+)*class\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("struct", new Regex(@"^\s*(?:(?:public|private|protected|package|static|extern)\s+)*struct\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("union", new Regex(@"^\s*(?:(?:public|private|protected|package|static|extern)\s+)*union\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("enum", new Regex(@"^\s*(?:(?:public|private|protected|package|static)\s+)*enum\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("namespace", new Regex(@"^\s*module\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "module"), + new("interface", new Regex(@"^\s*(?:(?:public|private|protected|package|static|abstract|extern)\s+)*interface\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "interface"), + new("class", new Regex(@"^\s*(?:(?:public|private|protected|package|static|abstract|final|extern)\s+)*class\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "class"), + new("struct", new Regex(@"^\s*(?:(?:public|private|protected|package|static|extern)\s+)*struct\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "struct"), + new("union", new Regex(@"^\s*(?:(?:public|private|protected|package|static|extern)\s+)*union\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "union"), + new("enum", new Regex(@"^\s*(?:(?:public|private|protected|package|static)\s+)*enum\s+(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "enum"), new("typealias", new Regex(@"^\s*(?:alias|typedef)\s+(?[A-Za-z_]\w*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("function", new Regex(@"^\s*(?!(?:if|for|while|switch|catch|return|throw|new|assert|version|debug)\b)(?:(?:public|private|protected|package|static|extern|export|final|abstract|override|synchronized|pure|nothrow|@safe|@trusted|@system)\s+)*(?(?:auto|void|bool|byte|ubyte|short|ushort|int|uint|long|ulong|cent|ucent|float|double|real|char|wchar|dchar|string|[A-Za-z_][\w.]*)(?:\s*[*\[\]])*)\s+(?[A-Za-z_]\w*)\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), - new("import", new Regex(@"^\s*import\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*import\s+(?[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "import"), ], ["nim"] = [ - new("type", new Regex(@"^\s*type\s+(?[A-Za-z_]\w*)\*?\s*=\s*(?:ref\s+)?(?:object|enum|tuple|distinct)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), + new("type", new Regex(@"^\s*type\s+(?[A-Za-z_]\w*)\*?\s*=\s*(?:ref\s+)?(?:object|enum|tuple|distinct)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent, RequiredLiteral: "type"), new("type", new Regex(@"^\s+(?[A-Za-z_]\w*)\*?\s*=\s*(?:ref\s+)?(?:object|enum|tuple|distinct)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), new("function", new Regex(@"^\s*(?:proc|func|method|iterator|template|macro|converter)\s+(?`[^`\r\n]+`|[A-Za-z_]\w*)\*?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Indent), new("property", new Regex(@"^\s*(?:const|let|var)\s+(?[A-Za-z_]\w*)\*?\s*(?::|=)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("import", new Regex(@"^\s*(?:import|include)\s+(?[A-Za-z_][\w./]*(?:\s*,\s*[A-Za-z_][\w./]*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*from\s+(?[A-Za-z_][\w./]*)\s+import\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*from\s+(?[A-Za-z_][\w./]*)\s+import\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "import"), ], ["perl"] = [ // Perl package declarations / Perl の package 宣言 - new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "package"), + new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "package"), // Perl class feature declarations / Perl class feature の宣言 - new("class", new Regex(@"^\s*class\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("interface", new Regex(@"^\s*role\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("class", new Regex(@"^\s*class\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "class"), + new("interface", new Regex(@"^\s*role\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "role"), // Perl constants are compile-time subroutines, so expose them as functions for navigation. // Perl constant はコンパイル時 subroutine なので、ナビゲーション用に function として出す。 - new("function", new Regex(@"^\s*use\s+constant\s+(?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*use\s+constant\s+(?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "constant"), // Perl module imports / Perl の module import - new("import", new Regex(@"^\s*use\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*require\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*use\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "use"), + new("import", new Regex(@"^\s*require\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "require"), // Moose/Moo attributes / Moose/Moo の属性 - new("property", new Regex(@"^\s*has\s+(?['""]?)\+?(?" + PerlIdentifierPattern + @")\k\s*=>", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("property", new Regex(@"^\s*has\s+(?['""]?)\+?(?" + PerlIdentifierPattern + @")\k\s*=>", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "has"), // Package variables / package 変数 - new("property", new Regex(@"^\s*our\s+[$@%](?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("property", new Regex(@"^\s*our\s+[$@%](?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "our"), // Perl class feature fields / Perl class feature の field - new("property", new Regex(@"^\s*field\s+[$@%](?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("property", new Regex(@"^\s*field\s+[$@%](?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "field"), // Perl subroutines / Perl の subroutine - new("function", new Regex(@"^\s*(?:(?:my|state)\s+)?sub\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("function", new Regex(@"^\s*(?:(?:my|state)\s+)?sub\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "sub"), new("function", new Regex(@"^\s*(?:method|fun)\s+(?" + PerlIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), ], ["matlab"] = [ - new("class", new Regex(@"^\s*classdef\s*(?:\([^)]*\)\s*)?(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("function", new Regex(@"^\s*function\s+(?:(?:\[[^\]]+\]|[A-Za-z]\w*)\s*=\s*)?(?[A-Za-z]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd), - new("import", new Regex(@"^\s*import\s+(?[A-Za-z]\w*(?:\.[A-Za-z*]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("class", new Regex(@"^\s*classdef\s*(?:\([^)]*\)\s*)?(?[A-Za-z]\w*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd, RequiredLiteral: "classdef"), + new("function", new Regex(@"^\s*function\s+(?:(?:\[[^\]]+\]|[A-Za-z]\w*)\s*=\s*)?(?[A-Za-z]\w*)\s*(?:\(|$)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.ScientificEnd, RequiredLiteral: "function"), + new("import", new Regex(@"^\s*import\s+(?[A-Za-z]\w*(?:\.[A-Za-z*]\w*)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "import"), ], ["prolog"] = [ - new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "module"), + new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "use_module"), new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|:-|[a-z][A-Za-z0-9_]*\s*\(\s*$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["ambiguous_pl"] = [ // Keep ambiguous .pl files structured without choosing Perl or Prolog prematurely. // .pl の判定が曖昧でも Perl / Prolog のどちらかへ早計に固定せず、構造を保持する。 - new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("class", new Regex(@"^\s*class\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("interface", new Regex(@"^\s*role\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("function", new Regex(@"^\s*use\s+constant\s+(?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*use\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*require\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("property", new Regex(@"^\s*our\s+[$@%](?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?:(?:my|state)\s+)?sub\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*\{", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "package"), + new("namespace", new Regex(@"^\s*package\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s+v?[\d._]+)?\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "package"), + new("class", new Regex(@"^\s*class\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "class"), + new("interface", new Regex(@"^\s*role\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "role"), + new("function", new Regex(@"^\s*use\s+constant\s+(?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "constant"), + new("import", new Regex(@"^\s*use\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "use"), + new("import", new Regex(@"^\s*require\s+(?" + PerlQualifiedIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "require"), + new("property", new Regex(@"^\s*our\s+[$@%](?" + PerlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "our"), + new("function", new Regex(@"^\s*(?:(?:my|state)\s+)?sub\s+(?" + PerlQualifiedIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "sub"), new("function", new Regex(@"^\s*(?:method|fun)\s+(?" + PerlIdentifierPattern + @")\b(?:\s*:[^{;]+)?", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*:-\s*module\s*\(\s*(?[a-z][A-Za-z0-9_]*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "module"), + new("import", new Regex(@"^\s*:-\s*use_module\s*\(\s*(?[a-z][A-Za-z0-9_]*(?:\([^)]*\))?)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "use_module"), new("function", new Regex(@"^\s*(?[a-z][A-Za-z0-9_]*)\s*(?:\([^\r\n]*\))?\s*(?::-|-->|\.(?=\s*(?:$|:-|[a-z][A-Za-z0-9_]*\s*\(\s*$|[a-z][A-Za-z0-9_]*(?:\s*\([^)]*\))?\s*(?::-|-->|\.))))", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["c"] = [ new("function", new Regex(CFunctionStartBlacklistPattern + CFunctionReturnTypePattern + CFunctionNameBlacklistPattern + @"(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), // #define macros / #define マクロ - new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)\(", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)(?=\s|$)", RegexOptions.Compiled), BodyStyle.None), - new("struct", new Regex(@"^\s*typedef\s+struct\s+(?:\w+\s*)?(?:\*+\s*)?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None), - new("struct", new Regex(@"^\s*(?:typedef\s+)?struct\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("union", new Regex(@"^\s*typedef\s+union\s+(?:\w+\s*)?(?:\*+\s*)?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None), - new("union", new Regex(@"^\s*(?:typedef\s+)?union\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*typedef\s+enum\s+(?:\w+\s+)?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None), - new("enum", new Regex(@"^\s*(?:typedef\s+)?enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "define"), + new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)(?=\s|$)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "define"), + new("struct", new Regex(@"^\s*typedef\s+struct\s+(?:\w+\s*)?(?:\*+\s*)?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "struct"), + new("struct", new Regex(@"^\s*(?:typedef\s+)?struct\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "struct"), + new("union", new Regex(@"^\s*typedef\s+union\s+(?:\w+\s*)?(?:\*+\s*)?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "union"), + new("union", new Regex(@"^\s*(?:typedef\s+)?union\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "union"), + new("enum", new Regex(@"^\s*typedef\s+enum\s+(?:\w+\s+)?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "enum"), + new("enum", new Regex(@"^\s*(?:typedef\s+)?enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "enum"), new("import", new Regex(@"^\s*#\s*(?:include(?:_next)?|import)\s+(?:<(?[^>]+)>|""(?[^""]+)""|(?[^\s]+))", RegexOptions.Compiled), BodyStyle.None), ], ["cpp"] = [ - new("namespace", new Regex(CppFunctionStartBlacklistPattern + @"(?:export\s+)?module\s+(?[\w.]+(?::[\w.]+)?)\b", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(CppFunctionStartBlacklistPattern + @"(?:export\s+)?import\s+(?:<(?[^>\r\n]+)>|""(?[^""\r\n]+)""|(?:?[A-Za-z_]\w*(?:[.:][A-Za-z_]\w*)*))\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("namespace", new Regex(CppFunctionStartBlacklistPattern + @"inline\s+namespace\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("interface", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?:export\s+)?concept\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.None), - new("specialization", new Regex(CppFunctionStartBlacklistPattern + @"\s*template\s*<[^>]*>\s*(?:class|struct|union)\s+(?(?:[A-Za-z_]\w*::)*[A-Za-z_]\w*)\s*<[^;{}]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("specialization", new Regex(CppFunctionStartBlacklistPattern + @"\s*template\s*<>\s*" + CppAttributePrefixPattern + @"(?:extern\s+""(?:C|C\+\+)""\s*)?" + CppAttributePrefixPattern + @"(?:(?(?:(?:" + CppFunctionReturnTypeAtomPattern + @")[\s*&]+)+))?(?:(?:[\w:<>]+\s*::\s*)+)?" + CFunctionNameBlacklistPattern + @"(?~?\w+|operator(?:\s*\(\)|\s*\[\]|\s*[^\s(]+(?:\s+[^\s(]+)?))\s*<[^>\r\n]+>\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), + new("namespace", new Regex(CppFunctionStartBlacklistPattern + @"(?:export\s+)?module\s+(?[\w.]+(?::[\w.]+)?)\b", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "module"), + new("import", new Regex(CppFunctionStartBlacklistPattern + @"(?:export\s+)?import\s+(?:<(?[^>\r\n]+)>|""(?[^""\r\n]+)""|(?:?[A-Za-z_]\w*(?:[.:][A-Za-z_]\w*)*))\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "import"), + new("namespace", new Regex(CppFunctionStartBlacklistPattern + @"inline\s+namespace\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "namespace"), + new("interface", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?:export\s+)?concept\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "concept"), + new("specialization", new Regex(CppFunctionStartBlacklistPattern + @"\s*template\s*<[^>]*>\s*(?:class|struct|union)\s+(?(?:[A-Za-z_]\w*::)*[A-Za-z_]\w*)\s*<[^;{}]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "template"), + new("specialization", new Regex(CppFunctionStartBlacklistPattern + @"\s*template\s*<>\s*" + CppAttributePrefixPattern + @"(?:extern\s+""(?:C|C\+\+)""\s*)?" + CppAttributePrefixPattern + @"(?:(?(?:(?:" + CppFunctionReturnTypeAtomPattern + @")[\s*&]+)+))?(?:(?:[\w:<>]+\s*::\s*)+)?" + CFunctionNameBlacklistPattern + @"(?~?\w+|operator(?:\s*\(\)|\s*\[\]|\s*[^\s(]+(?:\s+[^\s(]+)?))\s*<[^>\r\n]+>\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType", RequiredLiteral: "template"), new("function", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + CppAttributePrefixPattern + @"(?:extern\s+""(?:C|C\+\+)""\s*)?" + CppAttributePrefixPattern + @"(?:(?(?:(?:" + CppFunctionReturnTypeAtomPattern + @")[\s*&]+)+))?(?:(?:[\w:<>]+\s*::\s*)+)?" + CFunctionNameBlacklistPattern + @"(?~?\w+|operator(?:\s*\(\)|\s*\[\]|\s*[^\s(]+(?:\s+[^\s(]+)?))(?:\s*<[^>]+>)?\s*\(", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), // Type alias / 型エイリアス - new("import", new Regex(CppFunctionStartBlacklistPattern + @"using\s+enum\s+(?(?:[A-Za-z_]\w*::)*[A-Za-z_]\w*)\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(CppFunctionStartBlacklistPattern + @"template\s*<[^>]+>\s*(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(CppFunctionStartBlacklistPattern + @"template\s*<[^>]+>\s*(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.Brace), - new("import", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.Brace), - new("import", new Regex(@"^\s*typedef\s+(?![^;]*\().*\b(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*typedef\s+(?![^;]*\().*\b(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.Brace), + new("import", new Regex(CppFunctionStartBlacklistPattern + @"using\s+enum\s+(?(?:[A-Za-z_]\w*::)*[A-Za-z_]\w*)\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "using"), + new("import", new Regex(CppFunctionStartBlacklistPattern + @"template\s*<[^>]+>\s*(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "using"), + new("import", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "using"), + new("import", new Regex(CppFunctionStartBlacklistPattern + @"template\s*<[^>]+>\s*(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "using"), + new("import", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?:export\s+)?using\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "using"), + new("import", new Regex(@"^\s*typedef\s+(?![^;]*\().*\b(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "typedef"), + new("import", new Regex(@"^\s*typedef\s+(?![^;]*\().*\b(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "typedef"), // #define macros / #define マクロ - new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)\(", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)(?=\s|$)", RegexOptions.Compiled), BodyStyle.None), - new("property", new Regex(@"^(?:export\s+)?(?:(?:inline|static)\s+)*constexpr\s+(?(?:[\w:<>~]+(?:\s*[*&])?\s+)+)(?(?:[A-Z_]\w*|k[A-Z]\w*))\s*=", RegexOptions.Compiled), BodyStyle.None, ReturnTypeGroup: "returnType"), - new("property", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?(?:[\w:<>~]+[\s*&]+)+)(?:(?:[\w:<>]+\s*::\s*)+)(?\w+)\s*=\s*[^;]+;", RegexOptions.Compiled), BodyStyle.None, ReturnTypeGroup: "returnType"), - new("class", new Regex(CppFunctionStartBlacklistPattern + @"\s*(?:export\s+)?" + CppTemplatePrefixPattern + @"class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("struct", new Regex(CppFunctionStartBlacklistPattern + @"\s*(?:export\s+)?" + CppTemplatePrefixPattern + @"struct\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("union", new Regex(CppFunctionStartBlacklistPattern + @"\s*(?:export\s+)?" + CppTemplatePrefixPattern + @"union\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("namespace", new Regex(@"^\s*(?:export\s+)?namespace\s+(?!\w+\s*=)(?\w+(?:::\w+)*)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*(?:export\s+)?(?:typedef\s+)?enum\s+(?:class\s+)?(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "define"), + new("function", new Regex(@"^\s*#\s*define\s+(?[A-Za-z_]\w*)(?=\s|$)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "define"), + new("property", new Regex(@"^(?:export\s+)?(?:(?:inline|static)\s+)*constexpr\s+(?(?:[\w:<>~]+(?:\s*[*&])?\s+)+)(?(?:[A-Z_]\w*|k[A-Z]\w*))\s*=", RegexOptions.Compiled), BodyStyle.None, ReturnTypeGroup: "returnType", RequiredLiteral: "constexpr"), + new("property", new Regex(CppFunctionStartBlacklistPattern + CppTemplatePrefixPattern + @"(?(?:[\w:<>~]+[\s*&]+)+)(?:(?:[\w:<>]+\s*::\s*)+)(?\w+)\s*=\s*[^;]+;", RegexOptions.Compiled), BodyStyle.None, ReturnTypeGroup: "returnType", RequiredLiteral: "::"), + new("class", new Regex(CppFunctionStartBlacklistPattern + @"\s*(?:export\s+)?" + CppTemplatePrefixPattern + @"class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "class"), + new("struct", new Regex(CppFunctionStartBlacklistPattern + @"\s*(?:export\s+)?" + CppTemplatePrefixPattern + @"struct\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "struct"), + new("union", new Regex(CppFunctionStartBlacklistPattern + @"\s*(?:export\s+)?" + CppTemplatePrefixPattern + @"union\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "union"), + new("namespace", new Regex(@"^\s*(?:export\s+)?namespace\s+(?!\w+\s*=)(?\w+(?:::\w+)*)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "namespace"), + new("enum", new Regex(@"^\s*(?:export\s+)?(?:typedef\s+)?enum\s+(?:class\s+)?(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "enum"), new("import", new Regex(@"^\s*#\s*(?:include|import)\s+(?:<(?[^>]+)>|""(?[^""]+)""|(?[^\s]+))", RegexOptions.Compiled), BodyStyle.None), ], ["php"] = [ // Variable-bound closures / 変数に束縛されたクロージャ - new("function", new Regex(@"^\s*\$(?\w+)\s*=\s*(?:static\s+)?function\s*\(", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*\$(?\w+)\s*=\s*(?:static\s+)?fn\s*\(", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^\s*\$(?\w+)\s*=\s*(?:static\s+)?function\s*\(", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "function"), + new("function", new Regex(@"^\s*\$(?\w+)\s*=\s*(?:static\s+)?fn\s*\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "fn"), // Const declaration / 定数宣言 new("function", new Regex(@"^\s*define\s*\(\s*['""](?[A-Za-z_]\w*)['""]\s*,", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?:(?public|private|protected)\s+)?const\s+(?\??[A-Za-z_\\][\w\\]*(?:\s*[|&]\s*\??[A-Za-z_\\][\w\\]*)*)\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", ReturnTypeGroup: "returnType"), - new("function", new Regex(@"^\s*(?:(?public|private|protected)\s+)?const\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility"), + new("function", new Regex(@"^\s*(?:(?public|private|protected)\s+)?const\s+(?\??[A-Za-z_\\][\w\\]*(?:\s*[|&]\s*\??[A-Za-z_\\][\w\\]*)*)\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", ReturnTypeGroup: "returnType", RequiredLiteral: "const"), + new("function", new Regex(@"^\s*(?:(?public|private|protected)\s+)?const\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "const"), // Class property declarations / クラスプロパティ宣言 new("property", new Regex(@"^\s*(?:(?public|private|protected|var)\s+)(?:(?:static|readonly)\s+)*(?:(?\??[A-Za-z_\\][\w\\]*(?:\s*[|&]\s*\??[A-Za-z_\\][\w\\]*)*)\s+)?\$(?\w+)\b", RegexOptions.Compiled), BodyStyle.None, "visibility", ReturnTypeGroup: "returnType"), - new("function", new Regex(@"^\s*(?:(?:(?public|private|protected)|static|abstract|final)\s+)*function\s+(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("function", new Regex(@"^\s*(?:(?:(?public|private|protected)|static|abstract|final)\s+)*function\s+(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "function"), // Class with expanded modifiers: abstract, final, readonly (PHP 8.2+) // 拡張修飾子対応: abstract, final, readonly (PHP 8.2+) - new("class", new Regex(@"^\s*(?:(?:abstract|final|readonly)\s+)*class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("interface", new Regex(@"^\s*interface\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("trait", new Regex(@"^\s*trait\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*enum\s+(?\w+)(?:\s*:\s*(?[A-Za-z_\\][\w\\]*))?", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType"), - new("property", new Regex(@"^\s*case\s+(?\w+)(?:\s*=\s*(?[^;]+?))?\s*;", RegexOptions.Compiled), BodyStyle.None, ReturnTypeGroup: "returnType"), + new("class", new Regex(@"^\s*(?:(?:abstract|final|readonly)\s+)*class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "class"), + new("interface", new Regex(@"^\s*interface\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "interface"), + new("trait", new Regex(@"^\s*trait\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "trait"), + new("enum", new Regex(@"^\s*enum\s+(?\w+)(?:\s*:\s*(?[A-Za-z_\\][\w\\]*))?", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "returnType", RequiredLiteral: "enum"), + new("property", new Regex(@"^\s*case\s+(?\w+)(?:\s*=\s*(?[^;]+?))?\s*;", RegexOptions.Compiled), BodyStyle.None, ReturnTypeGroup: "returnType", RequiredLiteral: "case"), // Namespace / 名前空間 - new("namespace", new Regex(@"^\s*namespace\s+(?[\w\\]+)", RegexOptions.Compiled), BodyStyle.Brace), + new("namespace", new Regex(@"^\s*namespace\s+(?[\w\\]+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "namespace"), ], ["swift"] = [ @@ -1620,17 +1627,17 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // wrapped in backticks (e.g. `func `repeat`() {}`). // Swift の関数名は通常識別子に加えて、バッククォートでエスケープした識別子 // （例: `func `repeat`() {}`）も取りうる。 - new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:static|class|nonisolated|mutating|nonmutating|prefix|infix|postfix)\s+)*(?:override\s+)?func\s+(?`[^`]+`|\w+|[~!%^&*+\-=|/?<>.]+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:required|convenience|nonisolated|mutating|nonmutating|override)\s+)*(?init)(?:\?)?\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:nonisolated)\s+)*(?deinit)\s*(?:\{|$)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:static|class|nonisolated|mutating|nonmutating|override)\s+)*(?subscript)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("struct", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:final)\s+)*struct\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("enum", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:indirect\s+)?enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("property", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:indirect\s+)?case\s+(?\w+)(?(?:\s*(?:\([^:\r\n]*\))?(?:\s*=\s*(?(?:""(?:\\.|[^""\\])*""|[^,\r\n])+))?\s*(?:,\s*\w+(?:\s*\([^:\r\n]*\))?(?:\s*=\s*(?:""(?:\\.|[^""\\])*""|[^,\r\n])+)?)*)\s*)$", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType"), - new("property", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:indirect\s+)?case\s+(?\w+)(?:\s*\([^)]*\))?(?:\s*=\s*(?.+?))?\s*$", RegexOptions.Compiled), BodyStyle.None, "visibility", ReturnTypeGroup: "returnType"), - new("protocol", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"protocol\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("associatedtype", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"associatedtype\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("typealias", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"typealias\s+(?\w+)(?:\s*<[^=]+>)?\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility"), + new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:static|class|nonisolated|mutating|nonmutating|prefix|infix|postfix)\s+)*(?:override\s+)?func\s+(?`[^`]+`|\w+|[~!%^&*+\-=|/?<>.]+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "func"), + new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:required|convenience|nonisolated|mutating|nonmutating|override)\s+)*(?init)(?:\?)?\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "init"), + new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:nonisolated)\s+)*(?deinit)\s*(?:\{|$)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "deinit"), + new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:static|class|nonisolated|mutating|nonmutating|override)\s+)*(?subscript)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "subscript"), + new("struct", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:final)\s+)*struct\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "struct"), + new("enum", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:indirect\s+)?enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), + new("property", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:indirect\s+)?case\s+(?\w+)(?(?:\s*(?:\([^:\r\n]*\))?(?:\s*=\s*(?(?:""(?:\\.|[^""\\])*""|[^,\r\n])+))?\s*(?:,\s*\w+(?:\s*\([^:\r\n]*\))?(?:\s*=\s*(?:""(?:\\.|[^""\\])*""|[^,\r\n])+)?)*)\s*)$", RegexOptions.Compiled), BodyStyle.None, "visibility", "returnType", RequiredLiteral: "case"), + new("property", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:indirect\s+)?case\s+(?\w+)(?:\s*\([^)]*\))?(?:\s*=\s*(?.+?))?\s*$", RegexOptions.Compiled), BodyStyle.None, "visibility", ReturnTypeGroup: "returnType", RequiredLiteral: "case"), + new("protocol", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"protocol\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "protocol"), + new("associatedtype", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"associatedtype\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "associatedtype"), + new("typealias", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"typealias\s+(?\w+)(?:\s*<[^=]+>)?\s*=", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "typealias"), new("property", new Regex(@"^\s*" + SwiftAttributePattern + @"(?(?:public|private|internal|open|fileprivate|package)(?:\s*\(\s*set\s*\))?)?\s*" + SwiftAttributePattern + @"(?:(?:lazy|weak|unowned|final|static|class|nonisolated)\s+)*(?:let|var)\s+(?`[^`]+`|\w+)(?=\s*(?:[:=]|$))", RegexOptions.Compiled), BodyStyle.None, "visibility"), // Extension declarations are important search anchors in Swift-heavy codebases. // A dedicated parser keeps nested generic targets searchable even when the @@ -1638,65 +1645,65 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // extension 宣言は Swift コード検索における重要なアンカー。 // 専用パーサにより、protocol conformance や `where` 句が付く場合でも // ネストした generic target を検索対象として維持する。 - new("class", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:final)\s+)?extension\s+(?[^\r\n{]+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("class", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:final)\s+)?extension\s+(?[^\r\n{]+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "extension"), // actor (Swift 5.5+) / アクター new("class", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:(?:final|distributed)\s+)*(?:class|actor)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), // Type alias / 型エイリアス: backtick-escaped names and generic/where clauses. - new("typealias", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"typealias\s+(?`[^`]+`|\w+)(?=\s*(?:<|=|where\b|$))", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"macro\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("interface", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"precedencegroup\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:prefix|infix|postfix)\s+operator\s+(?\S+)", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("import", new Regex(@"^\s*" + SwiftAttributePattern + @"(?:(?:public|private|internal|open|fileprivate|package)\s+)?import\s+(?.+)", RegexOptions.Compiled), BodyStyle.None), + new("typealias", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"typealias\s+(?`[^`]+`|\w+)(?=\s*(?:<|=|where\b|$))", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "typealias"), + new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"macro\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "macro"), + new("interface", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"precedencegroup\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "precedencegroup"), + new("function", new Regex(@"^\s*" + SwiftAttributePattern + @"(?public|private|internal|open|fileprivate|package)?\s*" + SwiftAttributePattern + @"(?:prefix|infix|postfix)\s+operator\s+(?\S+)", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "operator"), + new("import", new Regex(@"^\s*" + SwiftAttributePattern + @"(?:(?:public|private|internal|open|fileprivate|package)\s+)?import\s+(?.+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["objc"] = [ - new("class", new Regex(@"^\s*@interface\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*@implementation\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*@(?:interface|implementation)\s+(?\w+\s*\(\s*[^)]+?\s*\))\b", RegexOptions.Compiled), BodyStyle.Brace), - new("interface", new Regex(@"^\s*@protocol\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.Brace), + new("class", new Regex(@"^\s*@interface\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "@interface"), + new("class", new Regex(@"^\s*@implementation\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "@implementation"), + new("class", new Regex(@"^\s*@(?:interface|implementation)\s+(?\w+\s*\(\s*[^)]+?\s*\))\b", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "@i"), + new("interface", new Regex(@"^\s*@protocol\s+(?\w+)\b", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "@protocol"), // Apple enum macros / Apple の enum マクロ - new("enum", new Regex(@"^\s*typedef\s+(?:NS_(?:CLOSED_)?ENUM|NS_EXTENSIBLE_ENUM)\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*typedef\s+NS_OPTIONS\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*typedef\s+NS_ERROR_ENUM\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*typedef\s+(?:CF_ENUM|CF_OPTIONS)\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace), - new("property", new Regex(@"^\s*@property\b(?:\s*\([^)]*\))?.*?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None), + new("enum", new Regex(@"^\s*typedef\s+(?:NS_(?:CLOSED_)?ENUM|NS_EXTENSIBLE_ENUM)\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "NS_"), + new("enum", new Regex(@"^\s*typedef\s+NS_OPTIONS\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "NS_OPTIONS"), + new("enum", new Regex(@"^\s*typedef\s+NS_ERROR_ENUM\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "NS_ERROR_ENUM"), + new("enum", new Regex(@"^\s*typedef\s+(?:CF_ENUM|CF_OPTIONS)\s*\([^,]+,\s*(?\w+)\s*\)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "CF_"), + new("property", new Regex(@"^\s*@property\b(?:\s*\([^)]*\))?.*?(?\w+)\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "@property"), new("function", new Regex(@"^\s*[+-]\s*\([^)]*\)\s*(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), new("import", new Regex(@"^\s*#(?:import|include)\s+[<""](?[^"">]+)[>""]", RegexOptions.Compiled), BodyStyle.None), ], ["fsharp"] = [ - new("function", new Regex(@"^\s*let!?\s+(?:(?:rec|mutable|inline|private|internal|public)\s+)*(?(?:``[^`]+``|\w+))(?:\s+(?:\w+|\())?", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*use!?\s+(?(?:``[^`]+``|\w+))\s*(?:=|:)", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*and\s+(?(?:``[^`]+``|\w+))\s+(?:``[^`]+``|\w+|\()", RegexOptions.Compiled), BodyStyle.None), - new("struct", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*\{", RegexOptions.Compiled), BodyStyle.Brace), - new("interface", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*interface\b", RegexOptions.Compiled), BodyStyle.None), - new("struct", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*struct\b", RegexOptions.Compiled), BodyStyle.None), - new("delegate", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*delegate\b", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^=]+?>)?(?:\s+when\b[^=]+)?\s*=\s*class\b", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^\s*let!?\s+(?:(?:rec|mutable|inline|private|internal|public)\s+)*(?(?:``[^`]+``|\w+))(?:\s+(?:\w+|\())?", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "let"), + new("function", new Regex(@"^\s*use!?\s+(?(?:``[^`]+``|\w+))\s*(?:=|:)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "use"), + new("function", new Regex(@"^\s*and\s+(?(?:``[^`]+``|\w+))\s+(?:``[^`]+``|\w+|\()", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "and"), + new("struct", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*\{", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "type"), + new("interface", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*interface\b", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("struct", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*struct\b", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("delegate", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*delegate\b", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("class", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^=]+?>)?(?:\s+when\b[^=]+)?\s*=\s*class\b", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), // Generic abbreviations such as `type Result<'T> = Choice<'T, string>` should not be // mistaken for union cases just because the right-hand side starts with a capitalized // type name. // `type Result<'T> = Choice<'T, string>` のような generic abbreviation は、 // 右辺が大文字始まりの型名でも union case と誤認しない。 - new("typealias", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*<[^=]+?>\s*(?:when\b[^=]+)?\s*=\s*(?![^\r\n]*\|)(?!(?:class|delegate|interface|struct|enum|exception)\b)(?!\{)(?!\|)(?!\()", RegexOptions.Compiled), BodyStyle.None), - new("enum", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*(?:\|?\s*[A-Z][\w']*\b(?:\s*\|[^=].*)?)", RegexOptions.Compiled), BodyStyle.Brace), + new("typealias", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*<[^=]+?>\s*(?:when\b[^=]+)?\s*=\s*(?![^\r\n]*\|)(?!(?:class|delegate|interface|struct|enum|exception)\b)(?!\{)(?!\|)(?!\()", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("enum", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?\s*=\s*(?:\|?\s*[A-Z][\w']*\b(?:\s*\|[^=].*)?)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "type"), // Simple aliases without generic parameters stay searchable as `typealias`. // generic 引数なしの単純な alias も `typealias` として検索可能にする。 - new("typealias", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*(?![^\r\n]*\|)(?!(?:class|delegate|interface|struct|enum|exception)\b)(?!\{)(?!\|)(?!\()", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?(?:\s+when\b[^=]+)?\s*(?:\([^)]*\))\s*=", RegexOptions.Compiled), BodyStyle.None), - new("struct", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*\{", RegexOptions.Compiled), BodyStyle.None), - new("enum", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*(?:\|\s*)?\w+(?:\s*\|\s*\w+)+", RegexOptions.Compiled), BodyStyle.None), - new("exception", new Regex(@"^\s*exception\s+(?:(?:private|internal)\s+)?(?(?:``[^`]+``|\w+))", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*(?!\{)(?!\|)(?!class\b)(?!delegate\b)(?!struct\b)(?!interface\b)(?!enum\b).+", RegexOptions.Compiled), BodyStyle.None), - new("namespace", new Regex(@"^\s*namespace\s+(?:(?:rec|global)\s+)*(?[\w.]+)", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*module\s+(?:(?:(?:private|internal)\s+|rec\s+))*(?:``[^`]+``|[\w.]+)\s*=\s*(?(?:``[^`]+``|[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("namespace", new Regex(@"^\s*module\s+(?:(?:(?:private|internal)\s+|rec\s+))*(?(?:``[^`]+``|[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*(?:(?private|internal|public)\s+)?override\s+(?:(?:this|_|\w+)\.)?(?(?:``[^`]+``|\w+))\s*(?:\(|=|:)", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("function", new Regex(@"^\s*(?:(?private|internal|public)\s+)?abstract\s+(?!member\b)(?(?:``[^`]+``|\w+))\s*:", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("property", new Regex(@"^\s*(?:(?private|internal|public)\s+)?(?:static\s+)?val\s+(?:mutable\s+)?(?(?:``[^`]+``|\w+))\s*:", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("property", new Regex(@"^\s*(?:(?private|internal|public)\s+)?(?:(?:static|abstract|override|default)\s+)*member\s+(?:(?:private|internal)\s+)?val\s+(?(?:``[^`]+``|\w+))(?=\s|\(|=|:|$)", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("function", new Regex(@"^\s*(?:(?private|internal|public)\s+)?(?:(?:static|abstract|override|default)\s+)*member\s+(?:(?:private|internal)\s+)?(?:(?:inline)\s+)?(?:(?:this|_|\w+)\.)?(?!val\b)(?(?:``[^`]+``|\w+))(?=\s|\(|=|:|$)", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("import", new Regex(@"^\s*open\s+(?:type\s+)?(?(?:``[^`]+``|[\w.]+))", RegexOptions.Compiled), BodyStyle.None), + new("typealias", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*(?![^\r\n]*\|)(?!(?:class|delegate|interface|struct|enum|exception)\b)(?!\{)(?!\|)(?!\()", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("class", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))(?:\s*<[^>]+>)?(?:\s+when\b[^=]+)?\s*(?:\([^)]*\))\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("struct", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*\{", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("enum", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*(?:\|\s*)?\w+(?:\s*\|\s*\w+)+", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("exception", new Regex(@"^\s*exception\s+(?:(?:private|internal)\s+)?(?(?:``[^`]+``|\w+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "exception"), + new("import", new Regex(@"^\s*type\s+(?:(?:rec|private|internal|public)\s+)?(?(?:``[^`]+``|\w+))\s*=\s*(?!\{)(?!\|)(?!class\b)(?!delegate\b)(?!struct\b)(?!interface\b)(?!enum\b).+", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("namespace", new Regex(@"^\s*namespace\s+(?:(?:rec|global)\s+)*(?[\w.]+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "namespace"), + new("import", new Regex(@"^\s*module\s+(?:(?:(?:private|internal)\s+|rec\s+))*(?:``[^`]+``|[\w.]+)\s*=\s*(?(?:``[^`]+``|[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "module"), + new("namespace", new Regex(@"^\s*module\s+(?:(?:(?:private|internal)\s+|rec\s+))*(?(?:``[^`]+``|[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "module"), + new("function", new Regex(@"^\s*(?:(?private|internal|public)\s+)?override\s+(?:(?:this|_|\w+)\.)?(?(?:``[^`]+``|\w+))\s*(?:\(|=|:)", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "override"), + new("function", new Regex(@"^\s*(?:(?private|internal|public)\s+)?abstract\s+(?!member\b)(?(?:``[^`]+``|\w+))\s*:", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "abstract"), + new("property", new Regex(@"^\s*(?:(?private|internal|public)\s+)?(?:static\s+)?val\s+(?:mutable\s+)?(?(?:``[^`]+``|\w+))\s*:", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "val"), + new("property", new Regex(@"^\s*(?:(?private|internal|public)\s+)?(?:(?:static|abstract|override|default)\s+)*member\s+(?:(?:private|internal)\s+)?val\s+(?(?:``[^`]+``|\w+))(?=\s|\(|=|:|$)", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "member"), + new("function", new Regex(@"^\s*(?:(?private|internal|public)\s+)?(?:(?:static|abstract|override|default)\s+)*member\s+(?:(?:private|internal)\s+)?(?:(?:inline)\s+)?(?:(?:this|_|\w+)\.)?(?!val\b)(?(?:``[^`]+``|\w+))(?=\s|\(|=|:|$)", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "member"), + new("import", new Regex(@"^\s*open\s+(?:type\s+)?(?(?:``[^`]+``|[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "open"), ], ["vb"] = [ @@ -1718,113 +1725,113 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["scala"] = [ - new("implicit", new Regex(@"^\s*(?private|protected)?\s*implicit\s+(?:override\s+)?(?:def|val|var|class)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("given", new Regex(@"^\s*(?private|protected)?\s*given\s+(?:(?\w+)\s*(?::|as)|(?[A-Z]\w*)\b)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("function", new Regex(@"^\s*(?private|protected)?\s*(?:override\s+)?def\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("interface", new Regex(@"^\s*(?private|protected)?\s*(?:sealed\s+)?trait\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("enum", new Regex(@"^\s*(?private|protected)?\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("class", new Regex(@"^\s*(?private|protected)?\s*(?:abstract\s+|sealed\s+|final\s+)?(?:case\s+)?class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("object", new Regex(@"^\s*(?private|protected)?\s*(?:sealed\s+|final\s+)?(?:case\s+)?object\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("import", new Regex(@"^\s*type\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*import\s+(?.+)", RegexOptions.Compiled), BodyStyle.None), + new("implicit", new Regex(@"^\s*(?private|protected)?\s*implicit\s+(?:override\s+)?(?:def|val|var|class)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "implicit"), + new("given", new Regex(@"^\s*(?private|protected)?\s*given\s+(?:(?\w+)\s*(?::|as)|(?[A-Z]\w*)\b)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "given"), + new("function", new Regex(@"^\s*(?private|protected)?\s*(?:override\s+)?def\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "def"), + new("interface", new Regex(@"^\s*(?private|protected)?\s*(?:sealed\s+)?trait\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "trait"), + new("enum", new Regex(@"^\s*(?private|protected)?\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), + new("class", new Regex(@"^\s*(?private|protected)?\s*(?:abstract\s+|sealed\s+|final\s+)?(?:case\s+)?class\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "class"), + new("object", new Regex(@"^\s*(?private|protected)?\s*(?:sealed\s+|final\s+)?(?:case\s+)?object\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "object"), + new("import", new Regex(@"^\s*type\s+(?\w+)\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "type"), + new("import", new Regex(@"^\s*import\s+(?.+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["haskell"] = [ - new("function", new Regex(@"^(?:>\s+|\s*)(?[a-z_]\w*)\s+::", RegexOptions.Compiled), BodyStyle.None), - new("interface", new Regex(@"^\s*class\s+(?[A-Z]\w*)", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^(?:>\s+|\s*)(?[a-z_]\w*)\s+::", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "::"), + new("interface", new Regex(@"^\s*class\s+(?[A-Z]\w*)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "class"), new("class", new Regex(@"^\s*(?:data|newtype|type)\s+(?[A-Z]\w*)", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*import\s+(?:qualified\s+)?(?[\w.]+)", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"^\s*import\s+(?:qualified\s+)?(?[\w.]+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["r"] = [ - new("function", new Regex(@"^\s*`(?[^`]+)`\s*<[\w.]+)\s*<[^`]+)`\s*<[\w.]+)\s*<[^`]+)`\s*=\s*(?:function\s*\(|\\\s*\()", RegexOptions.Compiled), BodyStyle.Brace), new("function", new Regex(@"^\s*(?[\w.]+)\s*=\s*(?:function\s*\(|\\\s*\()", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*assign\s*\(\s*(?:x\s*=\s*)?['""](?[^'""]+)['""]\s*,\s*(?:value\s*=\s*)?(?:function\s*\(|\\\s*\()", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*(?:function\s*\(|\\\s*\()[^\r\n#]*(?:->>|->)\s*`(?[^`]+)`", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*(?:function\s*\(|\\\s*\()[^\r\n#]*(?:->>|->)\s*(?[\w.]+)", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?test_that\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*assign\s*\(\s*(?:x\s*=\s*)?['""](?[^'""]+)['""]\s*,\s*(?:value\s*=\s*)?(?:function\s*\(|\\\s*\()", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "assign"), + new("function", new Regex(@"^\s*(?:function\s*\(|\\\s*\()[^\r\n#]*(?:->>|->)\s*`(?[^`]+)`", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "->"), + new("function", new Regex(@"^\s*(?:function\s*\(|\\\s*\()[^\r\n#]*(?:->>|->)\s*(?[\w.]+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "->"), + new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?test_that\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "test_that"), new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:describe|it)\s*\(\s*['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*output\$(?[\w.]+)\s*<[^'""]+)['""]\s*\]\s*\]\s*<[\w.]+)\s*<[^'""]+)['""]\s*\]\s*\]\s*<[^`]+)`\s*(?:<[\w.]+)\s*(?:<[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*(?:(?:[\w.]+)::)?setIs\s*\(.*?\b(?:class2|to)\s*=\s*(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*inherit\s*=\s*(?:c\(\s*)?(?:['""](?[^'""]+)['""]|(?[A-Z][\w.]*))", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?setValidity\s*\(\s*(?:(?:Class|class|classes|name)\s*=\s*)?(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:setGeneric|setGroupGeneric)\s*\(\s*(?:(?:f|generic|name)\s*=\s*)?['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?setMethod\s*\(\s*(?:(?:f|generic|name)\s*=\s*)?(?:['""](?[^'""]+)['""]|(?[\w.]+))\s*,", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*(?public|private|active)\s*=\s*list\(\s*(?[\w.]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.None, "visibility"), - new("import", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:library|require)\s*\(\s*help\s*=\s*(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), + new("class", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:setClass|setRefClass|setClassUnion|setOldClass|R6Class)\s*\(\s*(?:(?:Class|classes|className|classname|name)\s*=\s*)?(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "Class"), + new("class", new Regex(@"^\s*(?:(?:[\w.]+)::)?setIs\s*\(.*?\b(?:class2|to)\s*=\s*(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "setIs"), + new("class", new Regex(@"^\s*inherit\s*=\s*(?:c\(\s*)?(?:['""](?[^'""]+)['""]|(?[A-Z][\w.]*))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "inherit"), + new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?setValidity\s*\(\s*(?:(?:Class|class|classes|name)\s*=\s*)?(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "setValidity"), + new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:setGeneric|setGroupGeneric)\s*\(\s*(?:(?:f|generic|name)\s*=\s*)?['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "Generic"), + new("function", new Regex(@"^\s*(?:(?:[\w.]+)::)?setMethod\s*\(\s*(?:(?:f|generic|name)\s*=\s*)?(?:['""](?[^'""]+)['""]|(?[\w.]+))\s*,", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "setMethod"), + new("function", new Regex(@"^\s*(?public|private|active)\s*=\s*list\(\s*(?[\w.]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.None, "visibility", RequiredLiteral: "function"), + new("import", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:library|require)\s*\(\s*help\s*=\s*(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "help"), new("import", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:library|require|requireNamespace)\s*\(\s*(?:(?:package|pkg)\s*=\s*)?(?:['""](?[^'""]+)['""]|(?[\w.]+))", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:source|sys\.source)\s*\(\s*(?:file\s*=\s*)?['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"^\s*(?:(?:[\w.]+)::)?(?:source|sys\.source)\s*\(\s*(?:file\s*=\s*)?['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "source"), ], ["lua"] = [ - new("function", new Regex(@"^\s*(?:local\s+)?function\s+(?[\w.:]+)\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd), - new("function", new Regex(@"^\s*local\s+(?[\w]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd), - new("function", new Regex(@"^\s*(?[\w]+(?:[.:][\w]+)+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd), - new("import", new Regex(@"^\s*(?:local\s+\w+\s*=\s*)?require\s*\(?['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^\s*(?:local\s+)?function\s+(?[\w.:]+)\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd, RequiredLiteral: "function"), + new("function", new Regex(@"^\s*local\s+(?[\w]+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd, RequiredLiteral: "function"), + new("function", new Regex(@"^\s*(?[\w]+(?:[.:][\w]+)+)\s*=\s*function\s*\(", RegexOptions.Compiled), BodyStyle.ElixirEnd, RequiredLiteral: "function"), + new("import", new Regex(@"^\s*(?:local\s+\w+\s*=\s*)?require\s*\(?['""](?[^'""]+)['""]", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "require"), ], ["elixir"] = [ - new("function", new Regex(@"^\s*(?:def|defp|defmacro|defguardp?)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.ElixirEnd), - new("class", new Regex(@"^\s*defmodule\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd), - new("interface", new Regex(@"^\s*defprotocol\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd), - new("protocol_impl", new Regex(@"^\s*defimpl\s+(?[\w.]+(?:\s*,\s*for:\s*(?:\[[^\]]+\]|[\w.{}]+))?)", RegexOptions.Compiled), BodyStyle.ElixirEnd), + new("function", new Regex(@"^\s*(?:def|defp|defmacro|defguardp?)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.ElixirEnd, RequiredLiteral: "def"), + new("class", new Regex(@"^\s*defmodule\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd, RequiredLiteral: "defmodule"), + new("interface", new Regex(@"^\s*defprotocol\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.ElixirEnd, RequiredLiteral: "defprotocol"), + new("protocol_impl", new Regex(@"^\s*defimpl\s+(?[\w.]+(?:\s*,\s*for:\s*(?:\[[^\]]+\]|[\w.{}]+))?)", RegexOptions.Compiled), BodyStyle.ElixirEnd, RequiredLiteral: "defimpl"), new("import", new Regex(@"^\s*(?:import|alias|use|require)\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.None), ], ["clojure"] = [ // Clojure forms are parenthesized, so use conservative line anchors. // Clojure の form は括弧ベースなので、保守的な行アンカーだけを拾う。 - new("namespace", new Regex(@"^\s*\(\s*ns\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("class", new Regex(@"^\s*\(\s*(?:defrecord|deftype)\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("protocol", new Regex(@"^\s*\(\s*defprotocol\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*\(\s*(?:defn-?|defmacro|defmulti|defmethod)\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("property", new Regex(@"^\s*\(\s*(?:def|defonce)\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*\(\s*ns\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "ns"), + new("class", new Regex(@"^\s*\(\s*(?:defrecord|deftype)\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "def"), + new("protocol", new Regex(@"^\s*\(\s*defprotocol\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "def"), + new("function", new Regex(@"^\s*\(\s*(?:defn-?|defmacro|defmulti|defmethod)\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "def"), + new("property", new Regex(@"^\s*\(\s*(?:def|defonce)\s+(?[^\s\)\[\{]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "def"), ], ["erlang"] = [ - new("namespace", new Regex(@"^\s*-module\s*\(\s*(?[a-z][\w@]*|'[^'\r\n]+')\s*\)\s*\.", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("struct", new Regex(@"^\s*-record\s*\(\s*(?[a-z][\w@]*|'[^'\r\n]+')\s*,", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*-module\s*\(\s*(?[a-z][\w@]*|'[^'\r\n]+')\s*\)\s*\.", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "-module"), + new("struct", new Regex(@"^\s*-record\s*\(\s*(?[a-z][\w@]*|'[^'\r\n]+')\s*,", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "-record"), new("type", new Regex(@"^\s*-(?:type|opaque)\s+(?[a-z][\w@]*|'[^'\r\n]+')\s*(?:\(|::)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?[a-z][\w@]*|'[^'\r\n]+')\s*\([^)\r\n]*\)\s*(?:when\b[^-\r\n]*)?->", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*(?[a-z][\w@]*|'[^'\r\n]+')\s*\([^)\r\n]*\)\s*(?:when\b[^-\r\n]*)?->", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "->"), new("import", new Regex(@"^\s*-(?:import|include(?:_lib)?)\s*\(\s*(?[^)\r\n]+)\)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["ocaml"] = [ - new("namespace", new Regex(@"^\s*module\s+(?:type\s+)?(?[A-Z][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("class", new Regex(@"^\s*class(?:\s+type)?\s+(?[A-Za-z_][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("type", new Regex(@"^\s*type\s+(?:nonrec\s+)?(?:'[\w]+\s+)*(?[A-Za-z_][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*let\s+(?:rec\s+)?(?[A-Za-z_][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*val\s+(?[A-Za-z_][A-Za-z0-9_']*)\s*:", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*open\s+(?[A-Z][\w.']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("namespace", new Regex(@"^\s*module\s+(?:type\s+)?(?[A-Z][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "module"), + new("class", new Regex(@"^\s*class(?:\s+type)?\s+(?[A-Za-z_][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "class"), + new("type", new Regex(@"^\s*type\s+(?:nonrec\s+)?(?:'[\w]+\s+)*(?[A-Za-z_][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "type"), + new("function", new Regex(@"^\s*let\s+(?:rec\s+)?(?[A-Za-z_][A-Za-z0-9_']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "let"), + new("function", new Regex(@"^\s*val\s+(?[A-Za-z_][A-Za-z0-9_']*)\s*:", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "val"), + new("import", new Regex(@"^\s*open\s+(?[A-Z][\w.']*)\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "open"), ], ["raku"] = [ new("namespace", new Regex(@"^\s*(?:unit\s+)?(?:module|package)\s+(?[\w:.-]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("interface", new Regex(@"^\s*(?:unit\s+)?role\s+(?[\w:.-]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), + new("interface", new Regex(@"^\s*(?:unit\s+)?role\s+(?[\w:.-]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd, RequiredLiteral: "role"), new("class", new Regex(@"^\s*(?:unit\s+)?(?:class|grammar)\s+(?[\w:.-]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), - new("enum", new Regex(@"^\s*enum\s+(?[\w:.-]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("enum", new Regex(@"^\s*enum\s+(?[\w:.-]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "enum"), new("function", new Regex(@"^\s*(?:(?:my|our|multi|proto|only)\s+)*(?:sub|method|submethod|macro)\s+(?[\w:!?.-]+)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.RubyEnd), new("property", new Regex(@"^\s*(?:(?:my|our|state|constant)\s+)*(?[$@%&]\w[\w-]*)\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), ], ["dart"] = [ new("function", new Regex(@"^\s*(?!return\b|await\b|const\b|new\b|throw\b|yield\b|if\b|else\b|for\b|while\b|switch\b|case\b|catch\b|do\b|try\b|finally\b|class\b|enum\b|mixin\b|extension\b|typedef\b|library\b|part\b|import\b|export\b)(?:(?:static|abstract|override|external)\s+)*(?\w[\w<>,\s\?]*?)\s+(?(?!if\b|else\b|for\b|while\b|switch\b|case\b|class\b|enum\b|mixin\b|extension\b|typedef\b|library\b|part\b|import\b|export\b|abstract\b|void\b|var\b|final\b|late\b|const\b|new\b|return\b|throw\b|yield\b|await\b|extends\b|implements\b|with\b|on\b|is\b|as\b|in\b|of\b|super\b|this\b)\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, ReturnTypeGroup: "rt"), - new("function", new Regex(@"^\s*factory\s+(?[A-Z_]\w*(?:\.\w+)?)\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*const\s+(?[A-Z_]\w*(?:\.\w+)?)\s*\((?=[^)]*(?:\bthis\b|\bsuper\b))", RegexOptions.Compiled), BodyStyle.None), - new("function", DartBareConstConstructorRegex, BodyStyle.None), + new("function", new Regex(@"^\s*factory\s+(?[A-Z_]\w*(?:\.\w+)?)\s*\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "factory"), + new("function", new Regex(@"^\s*const\s+(?[A-Z_]\w*(?:\.\w+)?)\s*\((?=[^)]*(?:\bthis\b|\bsuper\b))", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "const"), + new("function", DartBareConstConstructorRegex, BodyStyle.None, RequiredLiteral: "const"), new("function", new Regex(@"^\s*(?[A-Z_]\w*(?:\.\w+)?)\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*typedef\s+(?\w+)(?:<[^>]*>)?\s*=", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*typedef\s+(?:[\w<>,\[\]\?\.\s]+\s+)+(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.None), - new("enum", new Regex(@"^\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), + new("class", new Regex(@"^\s*typedef\s+(?\w+)(?:<[^>]*>)?\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "typedef"), + new("class", new Regex(@"^\s*typedef\s+(?:[\w<>,\[\]\?\.\s]+\s+)+(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "typedef"), + new("enum", new Regex(@"^\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "enum"), new("class", new Regex(@"^\s*(?:abstract\s+)?(?:class|mixin)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*extension\s+(?\w+)\s+on\s+", RegexOptions.Compiled), BodyStyle.Brace), - new("import", new Regex(@"^\s*import\s+'(?[^']+)'", RegexOptions.Compiled), BodyStyle.None), + new("class", new Regex(@"^\s*extension\s+(?\w+)\s+on\s+", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "extension"), + new("import", new Regex(@"^\s*import\s+'(?[^']+)'", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["pascal"] = [ @@ -1839,21 +1846,21 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["smalltalk"] = [ - new("class", new Regex(@"^\s*(?:[A-Za-z_]\w*)\s+subclass:\s*#(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("class", new Regex(@"^\s*(?:[A-Za-z_]\w*)\s+subclass:\s*#(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "subclass:"), new("class", new Regex(@"^\s*(?:Class\s+named:|Object\s+subclass:)\s*#(?[A-Za-z_]\w*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?:[A-Za-z_]\w*)(?:\s+class)?\s*>>\s*(?[A-Za-z_]\w*:?(?:\s+[A-Za-z_]\w+\s+[A-Za-z_]\w*:)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.SmalltalkMethod), + new("function", new Regex(@"^\s*(?:[A-Za-z_]\w*)(?:\s+class)?\s*>>\s*(?[A-Za-z_]\w*:?(?:\s+[A-Za-z_]\w+\s+[A-Za-z_]\w*:)*)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.SmalltalkMethod, RequiredLiteral: ">>"), ], ["graphql"] = [ - new("interface", new Regex(@"^\s*interface\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), + new("interface", new Regex(@"^\s*interface\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "interface"), + new("enum", new Regex(@"^\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "enum"), new("class", new Regex(@"^\s*(?:type|union|scalar|input)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), new("function", new Regex(@"^\s*(?:query|mutation|subscription)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*fragment\s+(?\w+)\s+on\s+\w+", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*directive\s+@(?\w+)", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*extend\s+(?:type|interface|input|enum)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*extend\s+(?:union|scalar)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*schema\s*\{", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*fragment\s+(?\w+)\s+on\s+\w+", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "fragment"), + new("function", new Regex(@"^\s*directive\s+@(?\w+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "directive"), + new("class", new Regex(@"^\s*extend\s+(?:type|interface|input|enum)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "extend"), + new("class", new Regex(@"^\s*extend\s+(?:union|scalar)\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "extend"), + new("class", new Regex(@"^\s*schema\s*\{", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "schema"), ], ["gradle"] = [ @@ -1863,7 +1870,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ["makefile"] = [ new("property", new Regex(@"^(?[\w.-]+)\s*(?::=|::=|=|\?=|\+=)", RegexOptions.Compiled), BodyStyle.None), // Makefile variable assignments / Makefile変数代入 - new("rule", new Regex(@"^(?\.PHONY)\s*:(?!=|:=)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), // Makefile special-target metadata / Makefile特殊ターゲットメタデータ + new("rule", new Regex(@"^(?\.PHONY)\s*:(?!=|:=)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: ".PHONY"), // Makefile special-target metadata / Makefile特殊ターゲットメタデータ new("function", new Regex(@"^(?!\.PHONY\s*:)(?[\w.%-]+)\s*:(?!=|:=)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), // Makefile targets / Makefileターゲット ], ["cmake"] = @@ -1896,39 +1903,39 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["protobuf"] = [ - new("class", new Regex(@"^\s*message\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("enum", new Regex(@"^\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("namespace", new Regex(@"^\s*package\s+(?[\w.]+)\s*;", RegexOptions.Compiled), BodyStyle.None), - new("class", new Regex(@"^\s*oneof\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*extend\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*service\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*rpc\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None), - new("import", new Regex(@"^\s*import\s+""(?[^""]+)"";", RegexOptions.Compiled), BodyStyle.None), + new("class", new Regex(@"^\s*message\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "message"), + new("enum", new Regex(@"^\s*enum\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "enum"), + new("namespace", new Regex(@"^\s*package\s+(?[\w.]+)\s*;", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "package"), + new("class", new Regex(@"^\s*oneof\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "oneof"), + new("class", new Regex(@"^\s*extend\s+(?[\w.]+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "extend"), + new("class", new Regex(@"^\s*service\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "service"), + new("function", new Regex(@"^\s*rpc\s+(?\w+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "rpc"), + new("import", new Regex(@"^\s*import\s+""(?[^""]+)"";", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "import"), ], ["verilog"] = [ new("module", new Regex(@"^\s*(?:module|macromodule|primitive)\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*function\s+(?:automatic\s+|static\s+)?(?:(?[\w$:\[\]\s]+?)\s+)?(?" + HdlIdentifierPattern + @")\s*(?:\(|;)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), - new("function", new Regex(@"^\s*task\s+(?:automatic\s+|static\s+)?(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("property", new Regex(@"^\s*(?:parameter|localparam)\s+(?:type\s+)?" + HdlDeclaratorPrefixPattern + @"(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("function", new Regex(@"^\s*function\s+(?:automatic\s+|static\s+)?(?:(?[\w$:\[\]\s]+?)\s+)?(?" + HdlIdentifierPattern + @")\s*(?:\(|;)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType", RequiredLiteral: "function"), + new("function", new Regex(@"^\s*task\s+(?:automatic\s+|static\s+)?(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "task"), + new("property", new Regex(@"^\s*(?:parameter|localparam)\s+(?:type\s+)?" + HdlDeclaratorPrefixPattern + @"(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "param"), new("property", new Regex(@"^\s*(?:input|output|inout|wire|reg|logic)\s+" + HdlDeclaratorPrefixPattern + @"(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*`include\s+""(?[^""]+)""", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*`include\s+""(?[^""]+)""", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "`include"), ], ["systemverilog"] = [ new("module", new Regex(@"^\s*(?:module|macromodule|primitive|program)\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("interface", new Regex(@"^\s*interface\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("package", new Regex(@"^\s*package\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("class", new Regex(@"^\s*(?:virtual\s+)?class\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("enum", new Regex(@"^\s*typedef\s+enum\b[^\r\n]*\s+(?" + HdlIdentifierPattern + @")\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("struct", new Regex(@"^\s*typedef\s+struct\b[^\r\n]*\s+(?" + HdlIdentifierPattern + @")\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("typealias", new Regex(@"^\s*typedef\s+(?!(?:enum|struct|union)\b)[^\r\n]*\s+(?" + HdlIdentifierPattern + @")\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*function\s+(?:automatic\s+|static\s+|virtual\s+)?(?:(?[\w$:\[\]\s]+?)\s+)?(?" + HdlIdentifierPattern + @")\s*(?:\(|;)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), - new("function", new Regex(@"^\s*task\s+(?:automatic\s+|static\s+|virtual\s+)?(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("property", new Regex(@"^\s*(?:parameter|localparam)\s+(?:type\s+)?" + HdlDeclaratorPrefixPattern + @"(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("interface", new Regex(@"^\s*interface\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "interface"), + new("package", new Regex(@"^\s*package\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "package"), + new("class", new Regex(@"^\s*(?:virtual\s+)?class\s+(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "class"), + new("enum", new Regex(@"^\s*typedef\s+enum\b[^\r\n]*\s+(?" + HdlIdentifierPattern + @")\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "enum"), + new("struct", new Regex(@"^\s*typedef\s+struct\b[^\r\n]*\s+(?" + HdlIdentifierPattern + @")\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "struct"), + new("typealias", new Regex(@"^\s*typedef\s+(?!(?:enum|struct|union)\b)[^\r\n]*\s+(?" + HdlIdentifierPattern + @")\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "typedef"), + new("function", new Regex(@"^\s*function\s+(?:automatic\s+|static\s+|virtual\s+)?(?:(?[\w$:\[\]\s]+?)\s+)?(?" + HdlIdentifierPattern + @")\s*(?:\(|;)", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType", RequiredLiteral: "function"), + new("function", new Regex(@"^\s*task\s+(?:automatic\s+|static\s+|virtual\s+)?(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "task"), + new("property", new Regex(@"^\s*(?:parameter|localparam)\s+(?:type\s+)?" + HdlDeclaratorPrefixPattern + @"(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "param"), new("property", new Regex(@"^\s*(?:input|output|inout|wire|reg|logic|rand|randc)\s+" + HdlDeclaratorPrefixPattern + @"(?" + HdlIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*import\s+(?" + HdlIdentifierPattern + @"(?:::(?:" + HdlIdentifierPattern + @"|\*))?)\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("import", new Regex(@"^\s*`include\s+""(?[^""]+)""", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("import", new Regex(@"^\s*import\s+(?" + HdlIdentifierPattern + @"(?:::(?:" + HdlIdentifierPattern + @"|\*))?)\s*;", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "import"), + new("import", new Regex(@"^\s*`include\s+""(?[^""]+)""", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "`include"), ], ["vhdl"] = [ @@ -1948,40 +1955,40 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ], ["glsl"] = [ - new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "struct"), new("property", new Regex(@"^\s*" + ShaderAttributePrefixPattern + @"(?:uniform|buffer)\s+(?:(?" + ShaderTypePattern + @")\s+)?(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), new("property", new Regex(@"^\s*" + ShaderAttributePrefixPattern + @"(?:in|out|attribute|varying)\s+(?" + ShaderTypePattern + @")\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), new("function", new Regex(ShaderFunctionStartBlacklistPattern + @"\s*" + ShaderAttributePrefixPattern + @"(?" + ShaderTypePattern + @")\s+(?" + ShaderIdentifierPattern + @")\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), ], ["hlsl"] = [ - new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("property", new Regex(@"^\s*(?:cbuffer|tbuffer)\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "struct"), + new("property", new Regex(@"^\s*(?:cbuffer|tbuffer)\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "buffer"), new("property", new Regex(@"^\s*(?:globallycoherent\s+)?(?:RW)?(?:Texture\w*|Buffer|StructuredBuffer|RWStructuredBuffer|ByteAddressBuffer|RWByteAddressBuffer|Sampler\w*)\s*(?:<[^>\r\n]+>)?\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), new("property", new Regex(@"^\s*(?:groupshared|static|uniform)\s+(?" + ShaderTypePattern + @")\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), new("function", new Regex(ShaderFunctionStartBlacklistPattern + @"\s*" + ShaderAttributePrefixPattern + @"(?" + ShaderTypePattern + @")\s+(?" + ShaderIdentifierPattern + @")\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), ], ["metal"] = [ - new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "struct"), new("property", new Regex(@"^\s*(?:constant|device|threadgroup)\s+(?" + ShaderTypePattern + @")\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, ReturnTypeGroup: "returnType"), new("function", new Regex(ShaderFunctionStartBlacklistPattern + @"\s*(?:kernel|vertex|fragment)\s+(?" + ShaderTypePattern + @")\s+(?" + ShaderIdentifierPattern + @")\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), new("function", new Regex(ShaderFunctionStartBlacklistPattern + @"\s*(?" + ShaderTypePattern + @")\s+(?" + ShaderIdentifierPattern + @")\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, ReturnTypeGroup: "returnType"), ], ["wgsl"] = [ - new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), - new("typealias", new Regex(@"^\s*alias\s+(?" + ShaderIdentifierPattern + @")\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), + new("struct", new Regex(@"^\s*struct\s+(?" + ShaderIdentifierPattern + @")\b", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "struct"), + new("typealias", new Regex(@"^\s*alias\s+(?" + ShaderIdentifierPattern + @")\s*=", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None, RequiredLiteral: "alias"), new("property", new Regex(@"^\s*(?:@\w+(?:\([^)]*\))?\s*)*(?:var(?:<[^>\r\n]+>)?|let|const|override)\s+(?" + ShaderIdentifierPattern + @")\s*:", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.None), - new("function", new Regex(@"^\s*(?:@\w+(?:\([^)]*\))?\s*)*fn\s+(?" + ShaderIdentifierPattern + @")\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace), + new("function", new Regex(@"^\s*(?:@\w+(?:\([^)]*\))?\s*)*fn\s+(?" + ShaderIdentifierPattern + @")\s*\(", RegexOptions.Compiled | RegexOptions.CultureInvariant), BodyStyle.Brace, RequiredLiteral: "fn"), ], ["shell"] = [ // Bash/Zsh function declarations / Bash/Zsh 関数宣言 new("function", new Regex(@"^\s*(?:function\s+)?(?\w+)\s*\(\s*\)\s*\{?", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*function\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*function\s+(?\w+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "function"), // Alias definitions / エイリアス定義 - new("alias", new Regex(@"^\s*alias(?:\s+-[^\s=]+)*\s+(?[A-Za-z_][A-Za-z0-9_-]*)\s*=", RegexOptions.Compiled), BodyStyle.None), + new("alias", new Regex(@"^\s*alias(?:\s+-[^\s=]+)*\s+(?[A-Za-z_][A-Za-z0-9_-]*)\s*=", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "alias"), ], ["sql"] = [ @@ -2078,16 +2085,16 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult [ // Terraform resource/data: capture the logical name (second quoted token), not the type // Terraform resource/data: 型ではなく論理名（第2引用トークン）をキャプチャ - new("class", new Regex(@"^\s*resource\s+""[^""]+""\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*data\s+""[^""]+""\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*module\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*provider\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*(?terraform)\s*\{", RegexOptions.Compiled), BodyStyle.Brace), + new("class", new Regex(@"^\s*resource\s+""[^""]+""\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "resource"), + new("class", new Regex(@"^\s*data\s+""[^""]+""\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "data"), + new("class", new Regex(@"^\s*module\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "module"), + new("class", new Regex(@"^\s*provider\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "provider"), + new("class", new Regex(@"^\s*(?terraform)\s*\{", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "terraform"), new("class", new Regex(@"^\s*(?import|moved|removed)\s*\{", RegexOptions.Compiled), BodyStyle.Brace), - new("class", new Regex(@"^\s*check\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*variable\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*output\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), - new("function", new Regex(@"^\s*(?locals)\s*\{", RegexOptions.Compiled), BodyStyle.Brace), + new("class", new Regex(@"^\s*check\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "check"), + new("function", new Regex(@"^\s*variable\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "variable"), + new("function", new Regex(@"^\s*output\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "output"), + new("function", new Regex(@"^\s*(?locals)\s*\{", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "locals"), ], ["css"] = [ @@ -2096,11 +2103,11 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // @counter-style / カウンタースタイル new("function", new Regex(@"^\s*@counter-style\s+(?[\w-]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.Brace), // @function (SCSS) / 関数 - new("function", new Regex(@"^\s*@function\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*@function\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "@function"), // @mixin (SCSS) / ミックスイン - new("function", new Regex(@"^\s*@mixin\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*@mixin\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "@mixin"), // @keyframes / キーフレーム - new("function", new Regex(@"^\s*@keyframes\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*@keyframes\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "@keyframes"), // @font-face / フォントフェイス new("function", new Regex(@"^\s*@font-face\b", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.Brace), // @property / カスタムプロパティ登録 @@ -2114,7 +2121,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // Grouping at-rules / grouping at-rule new("namespace", new Regex(@"^\s*@(?layer|container|supports|media)\b[^{]*\{", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.Brace), // :root selector / :root セレクタ - new("class", new Regex(@"^\s*(?:root)\s*[,{]", RegexOptions.Compiled), BodyStyle.Brace), + new("class", new Regex(@"^\s*(?:root)\s*[,{]", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: ":root"), // Standalone attribute selector / 単独属性セレクタ new("class", new Regex(@"^\s*(?\[[^\]]+\](?:(?:::?[\w-]+)|(?:\[[^\]]+\]))*)\s*[,{]", RegexOptions.Compiled), BodyStyle.Brace), // Pseudo-class / pseudo-element / attribute selectors / 疑似クラス・疑似要素・属性セレクタ @@ -2126,7 +2133,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // Native CSS nesting selectors / ネイティブ CSS nesting セレクタ new("property", new Regex(@"^\s*&(?:(?::(?[\w-]+))|(?:\s*(?:[>+~]\s*)?(?:\.|#)?(?[\w-]+)))(?:(?:::?[\w-]+)|(?:\[[^\]]+\]))*\s*[,{]", RegexOptions.Compiled), BodyStyle.Brace), // CSS custom property declaration / CSS カスタムプロパティ宣言 - new("property", new Regex(@"^\s*(?--[\w-]+)\s*:", RegexOptions.Compiled), BodyStyle.None), + new("property", new Regex(@"^\s*(?--[\w-]+)\s*:", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "--"), // SCSS $variable declaration / SCSS 変数宣言 new("property", new Regex(@"^\$(?[\w-]+)\s*:", RegexOptions.Compiled), BodyStyle.None), // SCSS placeholder selector / SCSS プレースホルダーセレクタ @@ -2138,7 +2145,7 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult // Sass インデント構文は波括弧を持たないため、行単位のアンカーとして扱う。 new("import", new Regex(@"^\s*@(?:import|use|forward)\s+(?.+?)(?:\s*!default)?\s*$", RegexOptions.Compiled), BodyStyle.None), new("function", new Regex(@"^\s*(?:@mixin\s+|=)(?[\w-]+)", RegexOptions.Compiled), BodyStyle.None), - new("function", new Regex(@"^\s*@function\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.None), + new("function", new Regex(@"^\s*@function\s+(?[\w-]+)", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "@function"), new("function", new Regex(@"^\s*@keyframes\s+(?[\w-]+)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant), BodyStyle.None), new("property", new Regex(@"^\s*\$(?[\w-]+)\s*:", RegexOptions.Compiled), BodyStyle.None), new("class", new Regex(@"^\s*(?[.#%][\w-]+)(?=[\s\.,:>+~\[]|$)", RegexOptions.Compiled), BodyStyle.None), @@ -2247,19 +2254,71 @@ private enum JavaScriptTypeScriptFunctionHeaderConsumeResult ["zig"] = [ // Public and private function declarations / 公開・非公開の関数宣言 - new("function", new Regex(@"^\s*(?:(?pub)\s+)?(?:inline\s+)?fn\s+(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("function", new Regex(@"^\s*(?:(?pub)\s+)?(?:inline\s+)?fn\s+(?\w+)\s*\(", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "fn"), // Struct/union/enum defined via const / const による struct/union/enum 定義 - new("struct", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*(?:extern\s+|packed\s+)?struct\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("enum", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*(?:extern\s+)?enum\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), - new("class", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*(?:extern\s+|packed\s+)?union\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("struct", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*(?:extern\s+|packed\s+)?struct\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "struct"), + new("enum", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*(?:extern\s+)?enum\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "enum"), + new("class", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*(?:extern\s+|packed\s+)?union\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "union"), // Error set / エラーセット - new("class", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*error\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility"), + new("class", new Regex(@"^\s*(?:(?pub)\s+)?const\s+(?\w+)\s*=\s*error\b", RegexOptions.Compiled), BodyStyle.Brace, "visibility", RequiredLiteral: "error"), // Test declarations / テスト宣言 - new("function", new Regex(@"^\s*test\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace), + new("function", new Regex(@"^\s*test\s+""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.Brace, RequiredLiteral: "test"), // @import / インポート - new("import", new Regex(@"^\s*(?:(?:pub)\s+)?const\s+\w+\s*=\s*@import\s*\(\s*""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.None), + new("import", new Regex(@"^\s*(?:(?:pub)\s+)?const\s+\w+\s*=\s*@import\s*\(\s*""(?[^""]+)""", RegexOptions.Compiled), BodyStyle.None, RequiredLiteral: "@import"), ], }; + static SymbolExtractor() + { + foreach (var (language, patterns) in PatternCache) + { + foreach (var pattern in patterns) + ValidateRequiredLiteralGate(language, pattern.Kind, pattern.Regex.Options, pattern.RequiredLiteral); + } + } + + private static void ValidateRequiredLiteralGate( + string language, + string kind, + RegexOptions regexOptions, + string? requiredLiteral) + { + if (requiredLiteral is null) + return; + + if (requiredLiteral.Length < 2) + { + throw new InvalidOperationException( + $"Required literal gates must contain at least two characters ({language}/{kind})."); + } + + if ((regexOptions & RegexOptions.IgnoreCase) != 0) + { + throw new InvalidOperationException( + $"Required literal gates are not valid for case-insensitive patterns ({language}/{kind})."); + } + } + + internal static void ValidateRequiredLiteralGateForTesting( + RegexOptions regexOptions, + string? requiredLiteral) => + ValidateRequiredLiteralGate("test", "test", regexOptions, requiredLiteral); + + internal static IReadOnlyList<(string Language, string Kind, string Literal, RegexOptions Options)> + GetRequiredLiteralGateMetadataForTesting() + { + var metadata = new List<(string Language, string Kind, string Literal, RegexOptions Options)>(); + foreach (var (language, patterns) in PatternCache) + { + foreach (var pattern in patterns) + { + if (pattern.RequiredLiteral is { } literal) + metadata.Add((language, pattern.Kind, literal, pattern.Regex.Options)); + } + } + + return metadata; + } + private static readonly string[] BuiltInSymbolLanguages = PatternCache.Keys.ToArray(); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index fc0ed3852..0e59952c4 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -340,6 +340,43 @@ public static List Extract(long fileId, string? lang, string conte patternConfigsAlreadyLoaded: false, cancellationToken: cancellationToken); + private sealed class RequiredLiteralGateCounts + { + public int PatternCount { get; set; } + public int ApplicablePatternCount { get; set; } + } + + internal static List ExtractForRequiredLiteralGateTesting( + long fileId, + string lang, + string content, + bool applyRequiredLiteralGate, + out int patternCount, + out int applicablePatternCount, + string? filePath = null, + string? projectRoot = null, + CancellationToken cancellationToken = default) + { + var counts = new RequiredLiteralGateCounts(); + var symbols = ExtractCore( + fileId, + lang, + content, + contentIsNormalized: false, + hasOversizeLine: null, + conflictMarkerLine: null, + filePath, + projectRoot, + patternConfigsAlreadyLoaded: false, + cancellationToken: cancellationToken, + maxSymbols: null, + applyRequiredLiteralGate: applyRequiredLiteralGate, + requiredLiteralGateCounts: counts); + patternCount = counts.PatternCount; + applicablePatternCount = counts.ApplicablePatternCount; + return symbols; + } + internal static bool TryExtractBounded( long fileId, string? lang, diff --git a/tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs b/tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs new file mode 100644 index 000000000..4e5c661d1 --- /dev/null +++ b/tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs @@ -0,0 +1,247 @@ +using System.Reflection; +using System.Text.RegularExpressions; +using CodeIndex.Indexer; +using CodeIndex.Models; + +namespace CodeIndex.Tests; + +public sealed class SymbolExtractorRequiredLiteralGateTests +{ + private static readonly PropertyInfo[] SymbolProperties = typeof(SymbolRecord) + .GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .Where(property => property.CanRead && property.GetIndexParameters().Length == 0) + .OrderBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + public static TheoryData PositiveLanguageFixtures => new() + { + { + "python", + "class Widget:\n marker = \"ＣＬＡＳＳ DEF\"\n", + "Widget" + }, + { + "javascript", + "export function run() {}\nconst marker = \"ＦＵＮＣＴＩＯＮ CLASS\";\n", + "run" + }, + { + "typescript", + "export interface Item {}\nconst marker = \"ＩＮＴＥＲＦＡＣＥ CLASS\";\n", + "Item" + }, + { + "go", + "package sample\nfunc Run() {}\n// ＦＵＮＣ\n", + "Run" + }, + { + "rust", + "pub fn run() {}\n// ＦＮ\n", + "run" + }, + { + "java", + "public class Widget { String marker = \"ＣＬＡＳＳ\"; }\n", + "Widget" + }, + { + "c", + "struct Widget { int value; }; /* ＳＴＲＵＣＴ */\n", + "Widget" + }, + { + "cpp", + "class Widget { public: int Run(); }; // ＣＬＡＳＳ\n", + "Widget" + }, + { + "swift", + "struct Widget { let marker = \"ＳＴＲＵＣＴ\" }\n", + "Widget" + }, + { + "fsharp", + "module Sample\nlet run value = value\n// ＬＥＴ\n", + "run" + }, + { + "scala", + "object Widget { val marker = \"ＯＢＪＥＣＴ\" }\n", + "Widget" + }, + { + "terraform", + "resource \"kind\" \"main\" {}\n# ＲＥＳＯＵＲＣＥ\n", + "main" + }, + { + "protobuf", + "message Widget {}\n// ＭＥＳＳＡＧＥ\n", + "Widget" + }, + { + "zig", + "pub fn run() void {}\n// ＦＮ\n", + "run" + }, + }; + + [Fact] + public void RequiredLiteralMetadata_UsesOnlyAuditedCaseSensitiveTierAValues() + { + var metadata = SymbolExtractor.GetRequiredLiteralGateMetadataForTesting(); + + Assert.Equal(29, SymbolProperties.Length); + Assert.Equal(400, metadata.Count); + Assert.Equal(51, metadata.Select(entry => entry.Language).Distinct(StringComparer.Ordinal).Count()); + Assert.All(metadata, entry => + { + Assert.True( + entry.Literal.Length >= 2, + $"{entry.Language}/{entry.Kind} uses a one-character required literal."); + Assert.Equal(RegexOptions.None, entry.Options & RegexOptions.IgnoreCase); + }); + } + + [Fact] + public void RequiredLiteralMetadata_RejectsIgnoreCaseAndShortLiterals() + { + Assert.Throws( + () => SymbolExtractor.ValidateRequiredLiteralGateForTesting(RegexOptions.IgnoreCase, "class")); + Assert.Throws( + () => SymbolExtractor.ValidateRequiredLiteralGateForTesting(RegexOptions.None, "x")); + Assert.Throws( + () => SymbolExtractor.ValidateRequiredLiteralGateForTesting(RegexOptions.None, "")); + + SymbolExtractor.ValidateRequiredLiteralGateForTesting(RegexOptions.None, null); + } + + [Theory] + [MemberData(nameof(PositiveLanguageFixtures))] + public void Extract_RequiredLiteralGatePreservesRepresentativeLanguageOutput( + string language, + string content, + string expectedSymbolName) + { + var baseline = Extract(language, content, applyRequiredLiteralGate: false, out var patternCount, out _); + var gated = Extract(language, content, applyRequiredLiteralGate: true, out _, out var applicablePatternCount); + + AssertSymbolsEqual(baseline, gated, language); + Assert.Contains(gated, symbol => symbol.Name == expectedSymbolName); + Assert.True( + applicablePatternCount < patternCount, + $"{language} fixture did not skip any impossible patterns."); + } + + [Fact] + public void Extract_RequiredLiteralGateSkipsAbsentLiteralsForEveryAnnotatedLanguage() + { + var languages = SymbolExtractor.GetRequiredLiteralGateMetadataForTesting() + .Select(entry => entry.Language) + .Distinct(StringComparer.Ordinal) + .OrderBy(language => language, StringComparer.Ordinal); + + foreach (var language in languages) + { + const string content = "Ω １２３ ＣＬＡＳＳ ＦＵＮＣＴＩＯＮ\n"; + var baseline = Extract(language, content, applyRequiredLiteralGate: false, out var patternCount, out _); + var gated = Extract(language, content, applyRequiredLiteralGate: true, out _, out var applicablePatternCount); + + AssertSymbolsEqual(baseline, gated, language); + Assert.True( + applicablePatternCount < patternCount, + $"{language} did not skip an absent Ordinal required literal."); + } + } + + [Fact] + public void Extract_CSharp_RequiredLiteralGatePreservesAdversarialOutput() + { + const string content = """ + // namespace interface enum struct operator event delegate partial readonly extern using + internal class Ωmega + { + private const string Words = "namespace interface enum struct operator event delegate"; + private const string FullWidth = "ｎａｍｅｓｐａｃｅ ｉｎｔｅｒｆａｃｅ"; + private static readonly string Arrow = "=>"; + public int Value => 1; + public static implicit operator int(Ωmega value) => value.Value; + public event Action? Changed; + public int this[int index] => index; + } + """; + + var baseline = Extract("csharp", content, applyRequiredLiteralGate: false, out var patternCount, out _); + var gated = Extract("csharp", content, applyRequiredLiteralGate: true, out _, out var applicablePatternCount); + + AssertSymbolsEqual(baseline, gated); + Assert.Contains(gated, symbol => symbol.Kind == "class" && symbol.Name == "Ωmega"); + Assert.True(applicablePatternCount < patternCount); + } + + [Fact] + public void Extract_CSharpIncompleteAttributeRecovery_UsesApplicablePatterns() + { + const string content = """ + [Broken( + public class Recovered + { + } + """; + + var baseline = Extract("csharp", content, applyRequiredLiteralGate: false, out _, out _); + var gated = Extract("csharp", content, applyRequiredLiteralGate: true, out var patternCount, out var applicablePatternCount); + + AssertSymbolsEqual(baseline, gated); + Assert.Contains(gated, symbol => symbol.Kind == "class" && symbol.Name == "Recovered"); + Assert.True(applicablePatternCount < patternCount); + } + + [Fact] + public void Extract_CppSameLineRecovery_UsesApplicablePatterns() + { + const string content = "class Box { public: int Run(); };"; + + var baseline = Extract("cpp", content, applyRequiredLiteralGate: false, out _, out _); + var gated = Extract("cpp", content, applyRequiredLiteralGate: true, out var patternCount, out var applicablePatternCount); + + AssertSymbolsEqual(baseline, gated); + Assert.Contains(gated, symbol => symbol.Kind == "class" && symbol.Name == "Box"); + Assert.Contains(gated, symbol => symbol.Kind == "function" && symbol.Name == "Run"); + Assert.True(applicablePatternCount < patternCount); + } + + private static List Extract( + string language, + string content, + bool applyRequiredLiteralGate, + out int patternCount, + out int applicablePatternCount) => + SymbolExtractor.ExtractForRequiredLiteralGateTesting( + 1, + language, + content, + applyRequiredLiteralGate, + out patternCount, + out applicablePatternCount); + + private static void AssertSymbolsEqual( + IReadOnlyList expected, + IReadOnlyList actual, + string? context = null) + { + Assert.True( + expected.Count == actual.Count, + $"Symbol count differs for {context ?? "adversarial input"}: {expected.Count} != {actual.Count}"); + for (var symbolIndex = 0; symbolIndex < expected.Count; symbolIndex++) + { + foreach (var property in SymbolProperties) + { + Assert.True( + Equals(property.GetValue(expected[symbolIndex]), property.GetValue(actual[symbolIndex])), + $"{context ?? "adversarial input"} symbol {symbolIndex} property {property.Name} differs"); + } + } + } +} From f681eb9721d5e0ccb56f27fab0f5f6ef040f777b Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 13:26:09 +0900 Subject: [PATCH 10/16] Read symbol worker requests as raw UTF-8 --- DEVELOPER_GUIDE.md | 27 ++ TESTING_GUIDE.md | 4 + .../+large-codebase-initial-indexing.fixed.md | 5 + src/CodeIndex/Cli/ProgramRunner.cs | 3 +- .../Indexer/Symbols/SymbolExtractionWorker.cs | 116 +++++++- src/CodeIndex/WorkerProtocolJsonValidator.cs | 42 +++ .../IndexCommandRunnerTests.cs | 7 +- ...SymbolExtractionWorkerUtf8ProtocolTests.cs | 247 ++++++++++++++++++ 8 files changed, 439 insertions(+), 12 deletions(-) create mode 100644 tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 670cc7961..cecdfe775 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -282,6 +282,21 @@ array, short files skip impossible FTS-token tracking, and high-ratio invalid UTF-8 decode replacements retain only the aggregate count used by `non_utf8_likely` rather than a line number for every damaged line. +Isolated symbol extraction uses one byte-oriented newline protocol for every +language. The parent still serializes each request once with +`JsonSerializer.SerializeToUtf8Bytes`, writes those bytes followed by LF, and +reads the bounded UTF-8 response frame. The child opens standard input as a raw +stream, reads it with `BoundedLineReader.ReadUtf8LineAsync`, validates the byte +payload, and deserializes directly from that span; the `TextReader` overload is +diagnostic-only. Do not reintroduce a `Console.In` / decoded-string copy in the +production path. Preserve the negotiated frame/byte bound, JSON depth/property/ +string bounds, CRLF stripping (including buffer boundaries), an unterminated +final frame, stable EOF, and cancellation of a pending read. Invalid UTF-8 and +malformed JSON must keep returning the sanitized exception category without +echoing request content or secrets. This contract is shared by every language +request routed through the symbol worker, including built-in and custom-pattern +configurations. + Parallel full scans use shared dynamic work claiming for the main extraction body. To keep a large file near the input tail from starting only in the final worker wave, they probe at most the last `min(4 * workers, 64)` work items and @@ -4077,6 +4092,18 @@ content 再走査を避けます。normalized facts を持たない caller 用 短い file は発生し得ない FTS token 追跡を省き、高比率の invalid UTF-8 decode replacement は 破損行ごとの番号ではなく `non_utf8_likely` に必要な集約件数だけを保持します。 +isolated symbol extraction は全言語で1つの byte-oriented newline protocol を共有します。 +parent は各 request を引き続き `JsonSerializer.SerializeToUtf8Bytes` で1回だけ serialize し、 +その byte 列と LF を書き込み、上限付き UTF-8 response frame を読みます。child は標準入力を +raw stream として開き、`BoundedLineReader.ReadUtf8LineAsync` で読み取り、byte payload を検証して +その span から直接 deserialize します。`TextReader` overload は診断専用です。本番経路へ +`Console.In` や decoded string の copy を戻さないでください。negotiated frame / byte 上限、 +JSON の depth / property / string 上限、buffer 境界をまたぐ場合を含む CRLF の除去、終端改行の +ない最後の frame、安定した EOF、pending read の cancellation を維持します。不正 UTF-8 と +malformed JSON は request content や secret を反射せず、sanitization 済みの exception category +だけを返してください。この契約は built-in / custom pattern configuration を含め、symbol worker +へ routing されるすべての language request で共有されます。 + parallel full scan は extraction 本体を共有dynamic claimで配分します。入力末尾の大きなfileが 最後のworker waveまで開始されないことを防ぐため、末尾の `min(4 * workers, 64)` work itemだけをprobeし、size取得済みかつ上限内のfileを大きい順に diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 3ad943b17..91efc72ac 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -93,6 +93,8 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result Search snippet origin-priority coverage keeps PascalCase, snake_case, and phrase queries in coordinated mixed comment/string/code fixtures so identifier focus, same-line code-column clamping, over-1-MiB valid chunks, final-window dropped counts, filtered-origin refocusing, and the phrase control share one contract. Recovery-command coverage keeps resolved execution arguments separate from support-safe display arguments. Assert structured argv, current `dotnet`/apphost prefix preservation, replay of option-like paths under CLI `--show-paths`, default CLI/MCP redaction metadata, and correct quoting for both POSIX sh and PowerShell. Include paths with spaces, quotes, dollar signs, shell metacharacters, POSIX home/temp roots, Windows drives, UNC roots, option-like source names such as `--db`, and file-URI database query parameters containing raw/encoded paths, percent-encoded sensitive keys, or path values with embedded sensitive assignments. Default-output assertions must reject the fixture's full absolute paths and secrets while preserving safe URI controls. Pair this with `status --config` coverage for default DB/data/log path and URI-query redaction, always-redacted secrets, and explicit `--show-paths`. Console writer synchronization coverage yields between character writes instead of sleeping per character; use enough whole-line iterations to expose interleaving without adding wall-clock delay. +- `SymbolExtractionWorkerUtf8ProtocolTests.cs` + The deterministic, cross-target symbol-worker protocol suite owns the production raw-UTF-8 stdin boundary shared by every language. Keep Unicode multi-frame input, CRLF, an unterminated final frame and stable EOF together; separately pin byte and JSON payload/property/depth/string bounds, invalid-UTF-8 and malformed-JSON sanitization without secret reflection, BOM-free output, and cancellation of a pending stream read. Keep the legacy `TextReader` tests as diagnostic-path coverage, and use temporary benchmarks only for adoption decisions—remove them before committing. - `WorkspaceCommandRunnerTests.cs` Workspace status coverage keeps missing manifests, empty and malformed manifests, missing project directories, all-missing databases, mixed healthy/degraded members, and shared-database layouts independently observable. Assert the compatibility `exists` alias beside unambiguous `project_exists` and `db_exists` fields, structured repair command names and argv (including paths with spaces), human labels, aggregate reasons/actions, and the stable `--check` exit policy: ready `0`, missing `2`, degraded `5`, and invalid input `1`. - `SymbolExtractor*Tests.cs` and `ReferenceExtractor*Tests.cs` @@ -1112,6 +1114,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" search snippet の origin 優先順位 coverage は PascalCase、snake_case、phrase query を連携した comment / string / code 混在 fixture にまとめ、identifier focus、同一行の code 列への clamping、1 MiB を超える有効 chunk、最終 window の dropped count、filter 後 origin への再 focus、phrase の control を一つの contract として検証します。 recovery command の coverage では、解決済みの実行引数とサポート共有向けの表示引数を分離して検証します。構造化 argv、現在の `dotnet` / apphost prefix の維持、CLI `--show-paths` による option と紛らわしい path の再実行、既定の CLI/MCP redaction metadata、POSIX sh と PowerShell 双方の正しい quoting を確認してください。空白、quote、dollar sign、shell metacharacter、POSIX の home/temp root、Windows drive、UNC root、`--db` のように option と紛らわしい source 名、raw / encoded path、percent-encoded な機密 key、機密 assignment を内包する path 値を持つ file-URI database query parameter を含めます。既定出力に fixture の完全な絶対パスや secret が残らず、安全な URI control は維持されることを assertion にします。`status --config` の DB/data/log path と URI query の既定 redaction、mode に関係なく維持される secret redaction、明示的 `--show-paths` も対で検証してください。 console writer synchronization coverageは文字writeごとのsleepではなくyieldを使い、wall-clock delayを追加せずinterleavingを露出できる十分なwhole-line iterationを維持してください。 +- `SymbolExtractionWorkerUtf8ProtocolTests.cs` + 全言語で共有する本番 symbol worker の raw UTF-8 stdin 境界は、この deterministic な cross-target protocol suite で検証します。Unicode の複数 frame、CRLF、終端改行のない最終 frame、安定した EOF を1つの fixture に保ち、byte および JSON payload / property / depth / string の各上限、不正 UTF-8 と malformed JSON が secret を反射しない sanitization、BOM のない出力、pending stream read の cancellation をそれぞれ固定してください。従来の `TextReader` test は診断経路の coverage として残し、採用判断用の一時 benchmark は commit 前に削除します。 - `WorkspaceCommandRunnerTests.cs` workspace status の coverage では、manifest 不在、空 / malformed manifest、project directory 不在、全 database 不在、healthy / degraded member の混在、shared-database layout をそれぞれ独立して観測可能にします。曖昧さのない `project_exists` / `db_exists` と互換用 `exists` alias、構造化された修復 command 名と argv（空白を含む path を含む）、human-readable label、集約 reason / action、ならびに ready `0`、missing `2`、degraded `5`、invalid input `1` の安定した `--check` exit policy を検証してください。 - `SymbolExtractor*Tests.cs` と `ReferenceExtractor*Tests.cs` diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index fc7889aa0..876d296e1 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -1,6 +1,7 @@ --- category: fixed affected: + - src/CodeIndex/Cli/ProgramRunner.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs - src/CodeIndex/Database/DbWriter.cs - src/CodeIndex/Database/DbWriter.References.cs @@ -18,6 +19,7 @@ affected: - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Patterns.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs + - src/CodeIndex/WorkerProtocolJsonValidator.cs - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs - tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs - tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs @@ -29,6 +31,7 @@ affected: - tests/CodeIndex.Tests/McpServerToolsCallTests.cs - tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs - tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs + - tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md --- @@ -43,6 +46,7 @@ affected: - **Persistent symbol workers reuse bounded pattern-directory snapshots across languages** — each project-root reload discovers user and root configs once, while nested ancestor directories, including missing and rejected results, are observed once per worker command. The cache follows live filesystem casing and falls back to uncached discovery when full so configs are never skipped; direct registry callers keep dynamic discovery. - **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once built-in symbols already extracted for the static-interface workspace. After materializing the immutable lookup snapshots, the prepass transfers ownership of admitted per-file symbol lists and releases the redundant workspace fallback objects instead of cloning the full symbol graph. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. - **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate before line-by-line matching. Pattern order and output stay unchanged, and C# incomplete-attribute plus C++ same-line recovery consume the same filtered set. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. +- **Symbol workers consume the existing UTF-8 request frames without decoding them twice** — the parent keeps its single `SerializeToUtf8Bytes` write, while the all-language child path now performs bounded newline framing, validation, and deserialization directly from raw standard-input bytes. CRLF/final-EOF framing, protocol and JSON bounds, cancellation, Unicode behavior, and sanitized invalid-UTF-8/JSON errors remain unchanged; the decoded `TextReader` path stays available for diagnostics. ## 日本語 @@ -54,3 +58,4 @@ affected: - **persistent symbol worker が全言語で上限付き pattern-directory snapshot を再利用するようにしました** — project-root reload ごとに user / root config を1回だけ探索し、missing や reject を含む nested ancestor directory は worker command ごとに初回結果を再利用します。cache は実 filesystem の case policy に従い、飽和時は uncached discovery に fallback して config を skip しません。registry の direct caller は従来どおり動的に探索します。 - **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once で利用できます。immutable な lookup snapshot を materialize した後、prepass は admit した file ごとの symbol list の所有権を移し、symbol graph 全体を clone せず重複する workspace fallback object を解放します。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 - **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、行単位の match 前に監査済みの2文字以上の literal を Ordinal で判定します。pattern 順と出力は変えず、C# の不完全 attribute recovery と C++ の same-line recovery も同じ filtered set を使います。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 +- **symbol worker が既存の UTF-8 request frame を二重 decode せず処理するようにしました** — parent 側の `SerializeToUtf8Bytes` による1回の書き込みは変えず、全言語共通の child 経路で標準入力の raw byte から上限付き newline framing、validation、deserialize を直接行います。CRLF / final EOF の framing、protocol / JSON 上限、cancellation、Unicode の挙動、不正 UTF-8 / JSON の sanitization 済み error は従来どおりで、decoded `TextReader` 経路も診断用に維持します。 diff --git a/src/CodeIndex/Cli/ProgramRunner.cs b/src/CodeIndex/Cli/ProgramRunner.cs index 383fe4308..e33d6b18e 100644 --- a/src/CodeIndex/Cli/ProgramRunner.cs +++ b/src/CodeIndex/Cli/ProgramRunner.cs @@ -129,10 +129,11 @@ internal static int Run( { if (args.Length > 0 && StringComparer.Ordinal.Equals(args[0], SymbolExtractionWorker.CommandName)) { + using var symbolWorkerInput = Console.OpenStandardInput(); using var symbolWorkerOutput = Console.OpenStandardOutput(); _ = SymbolExtractionWorker.TryRunCommand( args, - Console.In, + symbolWorkerInput, symbolWorkerOutput, Console.Error, out var symbolWorkerExitCode, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs index 2610eccb8..dc53ac429 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Globalization; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using CodeIndex.Cli; @@ -556,7 +557,7 @@ internal static bool TryRunCommand( exitCode = RunCommand( args, - input, + (maxCharacters, maxUtf8Bytes, _) => ReadTextRequestFrame(input, maxCharacters, maxUtf8Bytes), response => WriteResponse(output, response), error, maxProtocolLineCharacters, @@ -567,7 +568,7 @@ internal static bool TryRunCommand( internal static bool TryRunCommand( string[] args, - TextReader input, + Stream input, Stream output, TextWriter error, out int exitCode, @@ -583,7 +584,11 @@ internal static bool TryRunCommand( exitCode = RunCommand( args, - input, + (maxCharacters, maxUtf8Bytes, token) => ReadUtf8RequestFrame( + input, + maxCharacters, + maxUtf8Bytes, + token), response => WriteResponse(output, response), error, maxProtocolLineCharacters, @@ -669,7 +674,7 @@ internal static bool TryCreateStartInfo( private static int RunCommand( string[] args, - TextReader input, + ReadRequestFrame readRequestFrame, Action writeResponse, TextWriter error, int maxProtocolLineCharacters, @@ -698,10 +703,13 @@ private static int RunCommand( cancellationToken.ThrowIfCancellationRequested(); WorkerResponse response; WorkerRequest request; - string? requestJson; + WorkerRequestFrame? requestFrame; try { - requestJson = BoundedLineReader.ReadLine(input, maxProtocolLineCharacters, maxProtocolLineUtf8Bytes); + requestFrame = readRequestFrame( + maxProtocolLineCharacters, + maxProtocolLineUtf8Bytes, + cancellationToken); } catch (BoundedLineLengthException ex) { @@ -710,10 +718,14 @@ private static int RunCommand( return 1; } - if (requestJson is null) + if (requestFrame is null) break; - if (!WorkerProtocolJsonValidator.TryValidate(requestJson, maxProtocolLineCharacters, out var validationError)) + if (!TryValidateRequestFrame( + requestFrame.Value, + maxProtocolLineCharacters, + maxProtocolLineUtf8Bytes, + out var validationError)) { response = new WorkerResponse(null, validationError, null); writeResponse(response); @@ -722,7 +734,7 @@ private static int RunCommand( try { - request = BoundedJson.Deserialize(requestJson, maxProtocolLineUtf8Bytes, JsonOptions) + request = DeserializeRequestFrame(requestFrame.Value, maxProtocolLineUtf8Bytes) ?? throw new InvalidOperationException("worker request was empty."); } catch (Exception ex) @@ -761,6 +773,75 @@ private static int RunCommand( } } + private static WorkerRequestFrame? ReadTextRequestFrame( + TextReader input, + int maxProtocolLineCharacters, + int maxProtocolLineUtf8Bytes) + { + var requestJson = BoundedLineReader.ReadLine( + input, + maxProtocolLineCharacters, + maxProtocolLineUtf8Bytes); + return requestJson is null + ? null + : WorkerRequestFrame.FromText(requestJson); + } + + private static WorkerRequestFrame? ReadUtf8RequestFrame( + Stream input, + int maxProtocolLineCharacters, + int maxProtocolLineUtf8Bytes, + CancellationToken cancellationToken) + { + var requestUtf8 = BoundedLineReader.ReadUtf8LineAsync( + input, + maxProtocolLineUtf8Bytes, + cancellationToken) + .GetAwaiter() + .GetResult(); + if (requestUtf8 is null) + return null; + + var utf8ByteCount = requestUtf8.Value.Length; + if (utf8ByteCount > maxProtocolLineCharacters) + { + var characterCount = Encoding.UTF8.GetCharCount(requestUtf8.Value.Span); + if (characterCount > maxProtocolLineCharacters) + { + throw new BoundedLineLengthException( + characterCount, + utf8ByteCount, + maxProtocolLineCharacters, + maxProtocolLineUtf8Bytes); + } + } + + return WorkerRequestFrame.FromUtf8(requestUtf8.Value); + } + + private static bool TryValidateRequestFrame( + WorkerRequestFrame requestFrame, + int maxProtocolLineCharacters, + int maxProtocolLineUtf8Bytes, + out string validationError) + => requestFrame.IsUtf8 + ? WorkerProtocolJsonValidator.TryValidate( + requestFrame.Utf8Json, + maxProtocolLineCharacters, + maxProtocolLineUtf8Bytes, + out validationError) + : WorkerProtocolJsonValidator.TryValidate( + requestFrame.Json!, + maxProtocolLineCharacters, + out validationError); + + private static WorkerRequest? DeserializeRequestFrame( + WorkerRequestFrame requestFrame, + int maxProtocolLineUtf8Bytes) + => requestFrame.IsUtf8 + ? BoundedJson.Deserialize(requestFrame.Utf8Json.Span, maxProtocolLineUtf8Bytes, JsonOptions) + : BoundedJson.Deserialize(requestFrame.Json!, maxProtocolLineUtf8Bytes, JsonOptions); + private static void WriteResponse(TextWriter output, WorkerResponse response) { output.WriteLine(JsonSerializer.Serialize(response, JsonOptions)); @@ -774,6 +855,23 @@ private static void WriteResponse(Stream output, WorkerResponse response) output.Flush(); } + private delegate WorkerRequestFrame? ReadRequestFrame( + int maxProtocolLineCharacters, + int maxProtocolLineUtf8Bytes, + CancellationToken cancellationToken); + + private readonly record struct WorkerRequestFrame( + string? Json, + ReadOnlyMemory Utf8Json, + bool IsUtf8) + { + internal static WorkerRequestFrame FromText(string json) + => new(json, ReadOnlyMemory.Empty, IsUtf8: false); + + internal static WorkerRequestFrame FromUtf8(ReadOnlyMemory utf8Json) + => new(null, utf8Json, IsUtf8: true); + } + private static WorkerResponse InvokeInsideWorker(WorkerRequest request, WorkerOptions options, CancellationToken cancellationToken) { using var capturedOut = new BoundedTextWriter(CapturedConsoleMaxChars); diff --git a/src/CodeIndex/WorkerProtocolJsonValidator.cs b/src/CodeIndex/WorkerProtocolJsonValidator.cs index 5f5add5df..129949d40 100644 --- a/src/CodeIndex/WorkerProtocolJsonValidator.cs +++ b/src/CodeIndex/WorkerProtocolJsonValidator.cs @@ -40,6 +40,35 @@ internal static bool TryValidate(string json, int maxStringCharacters, out strin } } + internal static bool TryValidate( + ReadOnlyMemory utf8Json, + int maxPayloadCharacters, + int maxUtf8Bytes, + out string error) + { + var maxDepth = ResolveMaxJsonDepth(); + var maxProperties = MaxJsonPropertiesForTesting ?? DefaultMaxJsonProperties; + var effectiveMaxStringCharacters = MaxStringCharactersForTesting ?? maxPayloadCharacters; + var propertyCount = 0; + if (IsPayloadOverLimit(utf8Json.Span, maxPayloadCharacters, maxUtf8Bytes)) + { + error = SafeDiagnosticFormatter.FormatCategoryType("worker_protocol_error", "json_payload_length_exceeded"); + return false; + } + + try + { + using var document = BoundedJson.ParseDocument(utf8Json, maxUtf8Bytes, maxDepth); + ValidateElement(document.RootElement, maxProperties, effectiveMaxStringCharacters, ref propertyCount, out error); + return error.Length == 0; + } + catch (Exception ex) when (ex is JsonException or InvalidDataException) + { + error = SafeDiagnosticFormatter.FormatCategoryType("worker_protocol_error", nameof(JsonException)); + return false; + } + } + private static int ResolveMaxJsonDepth() { var maxDepth = MaxJsonDepthForTesting ?? DefaultMaxJsonDepth; @@ -56,6 +85,19 @@ private static bool IsPayloadOverLimit(string json, int maxCharactersAndUtf8Byte return Encoding.UTF8.GetByteCount(json) > maxCharactersAndUtf8Bytes; } + private static bool IsPayloadOverLimit( + ReadOnlySpan utf8Json, + int maxCharacters, + int maxUtf8Bytes) + { + if (maxCharacters <= 0 || maxUtf8Bytes <= 0 || utf8Json.Length > maxUtf8Bytes) + return true; + if (utf8Json.Length <= maxCharacters) + return false; + + return Encoding.UTF8.GetCharCount(utf8Json) > maxCharacters; + } + private static void ValidateElement( JsonElement element, int maxProperties, diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index d5d9db587..3b0d7dcc5 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -579,8 +579,11 @@ public void SymbolExtractionWorker_StreamResponseWritesBomlessUtf8Frame() "public class 顧客 { }\n", Path.Combine(projectRoot, "顧客.cs"), projectRoot); - using var input = new StringReader( - JsonSerializer.Serialize(request, SymbolExtractionWorker.JsonOptions) + "\n"); + var requestUtf8 = JsonSerializer.SerializeToUtf8Bytes(request, SymbolExtractionWorker.JsonOptions); + using var input = new MemoryStream(); + input.Write(requestUtf8); + input.WriteByte((byte)'\n'); + input.Position = 0; using var output = new MemoryStream(); using var error = new StringWriter(); diff --git a/tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs b/tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs new file mode 100644 index 000000000..3caa9735c --- /dev/null +++ b/tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs @@ -0,0 +1,247 @@ +using System.Text; +using System.Text.Json; +using CodeIndex.Cli; +using CodeIndex.Indexer; + +namespace CodeIndex.Tests; + +public sealed class SymbolExtractionWorkerUtf8ProtocolTests +{ + [Fact] + public void RawUtf8Input_PreservesUnicodeCrLfAndFinalEofFrames() + { + var projectRoot = Directory.CreateTempSubdirectory("cdidx-symbol-worker-utf8-").FullName; + try + { + var firstRequest = new SymbolExtractionWorker.WorkerRequest( + 1, + "csharp", + "public class 顧客 { }\n", + Path.Combine(projectRoot, "顧客.cs"), + projectRoot); + var secondRequest = new SymbolExtractionWorker.WorkerRequest( + 2, + "python", + "class Invoice:\n pass\n", + Path.Combine(projectRoot, "invoice.py"), + projectRoot); + using var input = new MemoryStream(); + JsonSerializer.Serialize(input, firstRequest, SymbolExtractionWorker.JsonOptions); + input.WriteByte((byte)'\r'); + input.WriteByte((byte)'\n'); + JsonSerializer.Serialize(input, secondRequest, SymbolExtractionWorker.JsonOptions); + input.Position = 0; + using var output = new MemoryStream(); + using var error = new StringWriter(); + + var handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out var exitCode); + + Assert.True(handled); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, error.ToString()); + var responseUtf8 = output.ToArray(); + Assert.Equal((byte)'{', responseUtf8[0]); + var responses = Encoding.UTF8.GetString(responseUtf8) + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(line => JsonSerializer.Deserialize( + line, + SymbolExtractionWorker.JsonOptions)!) + .ToArray(); + Assert.Collection( + responses, + response => Assert.Contains(response.Symbols!, symbol => symbol.Name == "顧客"), + response => Assert.Contains(response.Symbols!, symbol => symbol.Name == "Invoice")); + } + finally + { + Directory.Delete(projectRoot, recursive: true); + } + } + + [Fact] + public void RawUtf8Input_InvalidUtf8AndMalformedJsonDoNotEchoPayload() + { + const string secret = "SECRET_RAW_SYMBOL_WORKER_UTF8"; + using var input = new MemoryStream(); + input.Write(Encoding.UTF8.GetBytes("{\"Content\":\"" + secret + "-JSON\n")); + input.Write(Encoding.UTF8.GetBytes("{\"Content\":\"" + secret + "-UTF8")); + input.WriteByte(0xff); + input.WriteByte((byte)'\n'); + input.Position = 0; + using var output = new MemoryStream(); + using var error = new StringWriter(); + + var handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out var exitCode); + + Assert.True(handled); + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal(string.Empty, error.ToString()); + var responseText = Encoding.UTF8.GetString(output.ToArray()); + var responses = responseText.Split('\n', StringSplitOptions.RemoveEmptyEntries); + Assert.Equal(2, responses.Length); + foreach (var responseTextLine in responses) + { + using var response = JsonDocument.Parse(responseTextLine); + Assert.Equal( + "worker_protocol_error: JsonException", + response.RootElement.GetProperty("WorkerError").GetString()); + } + + Assert.DoesNotContain(secret, responseText, StringComparison.Ordinal); + } + + [Fact] + public void RawUtf8Input_EnforcesByteFrameLimit() + { + using var input = new MemoryStream(Encoding.UTF8.GetBytes("abcdef\n")); + using var output = new MemoryStream(); + using var error = new StringWriter(); + + var handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out var exitCode, + maxProtocolLineCharacters: 5, + maxProtocolLineUtf8Bytes: 5); + + Assert.True(handled); + Assert.Equal(1, exitCode); + Assert.Equal(string.Empty, error.ToString()); + using var response = JsonDocument.Parse(output.ToArray().AsMemory(0, checked((int)output.Length - 1))); + Assert.Equal( + "worker_protocol_error: BoundedLineLengthException", + response.RootElement.GetProperty("WorkerError").GetString()); + } + + [Fact] + public void RawUtf8Validator_PreservesPayloadPropertyDepthAndStringLimits() + { + lock (TestConsoleLock.Gate) + { + try + { + var multibyteJson = Encoding.UTF8.GetBytes("{\"x\":\"あ\"}"); + Assert.False(WorkerProtocolJsonValidator.TryValidate( + multibyteJson, + maxPayloadCharacters: 8, + maxUtf8Bytes: multibyteJson.Length, + out var payloadError)); + Assert.Equal("worker_protocol_error: json_payload_length_exceeded", payloadError); + + WorkerProtocolJsonValidator.MaxJsonPropertiesForTesting = 1; + AssertValidationError( + "{\"FileId\":0,\"Lang\":\"csharp\"}", + "worker_protocol_error: json_property_limit_exceeded"); + WorkerProtocolJsonValidator.MaxJsonPropertiesForTesting = null; + + WorkerProtocolJsonValidator.MaxJsonDepthForTesting = 4; + AssertValidationError( + "{\"FileId\":0,\"Lang\":{\"nested\":{\"too\":{\"deep\":{\"overflow\":\"csharp\"}}}}}", + "worker_protocol_error: JsonException"); + WorkerProtocolJsonValidator.MaxJsonDepthForTesting = null; + + WorkerProtocolJsonValidator.MaxStringCharactersForTesting = 4; + AssertValidationError( + "{\"Content\":\"too long\"}", + "worker_protocol_error: json_string_length_exceeded"); + } + finally + { + WorkerProtocolJsonValidator.MaxJsonPropertiesForTesting = null; + WorkerProtocolJsonValidator.MaxJsonDepthForTesting = null; + WorkerProtocolJsonValidator.MaxStringCharactersForTesting = null; + } + } + } + + [Fact] + public void RawUtf8Input_CancellationInterruptsPendingRead() + { + using var cts = new CancellationTokenSource(); + using var input = new StalledReadStream(cts.Cancel); + using var output = new MemoryStream(); + using var error = new StringWriter(); + + var handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out var exitCode, + cancellationToken: cts.Token); + + Assert.True(handled); + Assert.True(input.ReadStarted); + Assert.Equal(CommandExitCodes.CancelledBySignal, exitCode); + Assert.Equal(0, output.Length); + Assert.Equal(string.Empty, error.ToString()); + } + + private static void AssertValidationError(string json, string expectedError) + { + var utf8Json = Encoding.UTF8.GetBytes(json); + Assert.False(WorkerProtocolJsonValidator.TryValidate( + utf8Json, + maxPayloadCharacters: utf8Json.Length, + maxUtf8Bytes: utf8Json.Length, + out var error)); + Assert.Equal(expectedError, error); + } + + private sealed class StalledReadStream(Action onReadStarted) : Stream + { + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + internal bool ReadStarted { get; private set; } + public override long Length => throw new NotSupportedException(); + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + } + + public override int Read(byte[] buffer, int offset, int count) + => throw new NotSupportedException(); + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + ReadStarted = true; + onReadStarted(); + return new ValueTask(WaitForCancellationAsync(cancellationToken)); + } + + public override long Seek(long offset, SeekOrigin origin) + => throw new NotSupportedException(); + + public override void SetLength(long value) + => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) + => throw new NotSupportedException(); + + private static async Task WaitForCancellationAsync(CancellationToken cancellationToken) + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return 0; + } + } +} From f5f8236d635164fa4bf36e2f20843890d50b9f6c Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 15:29:02 +0900 Subject: [PATCH 11/16] Gate symbol regexes on exact match inputs --- DEVELOPER_GUIDE.md | 21 +- TESTING_GUIDE.md | 18 +- .../+large-codebase-initial-indexing.fixed.md | 6 +- .../Symbols/SymbolExtractor.CSharpScanner.cs | 50 +++- .../Indexer/Symbols/SymbolExtractor.Cpp.cs | 17 +- .../Indexer/Symbols/SymbolExtractor.Css.cs | 21 +- .../Symbols/SymbolExtractor.ExtractCore.cs | 112 +++++++- .../SymbolExtractor.ExtractionPhases.cs | 23 +- .../Indexer/Symbols/SymbolExtractor.Java.cs | 49 ++++ .../Indexer/Symbols/SymbolExtractor.cs | 12 +- ...SymbolExtractorRequiredLiteralGateTests.cs | 262 ++++++++++++++++-- 11 files changed, 523 insertions(+), 68 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index cecdfe775..069b8eb84 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1902,8 +1902,15 @@ pattern is skipped without changing the order of the remaining patterns. `Ignore one-character literals, optional or alternative paths without a shared literal, project custom patterns, and plugins are deliberately excluded. Supplemental scans that consult the pattern list, including C# incomplete-attribute recovery and C++ same-line member recovery, must consume the same -ordered applicable set. The content-wide check may retain a pattern because its literal appears in a -comment or string; that only reduces the optimization and cannot suppress a real match. +ordered applicable set. Immediately before each regex call, the same Ordinal proof is applied to the +exact input presented to that call, after transformations such as C# property-header merging, Fortran +continuation joining, Java/Kotlin annotation stripping, C# wrapped-modifier synthesis, C++ same-line +segmentation, or CSS selector-brace reconstruction. A miss behaves like a failed regex attempt rather +than terminating language-specific recovery: notably, a C# static-constructor pattern rejected on the +bare identifier line must still try each synthesized `static ...` wrapper. The content-wide check may +retain a pattern because its literal appears in a comment, string, annotation, or another declaration; +the exact-input check then recovers that lost optimization without changing matches. Patterns without +`RequiredLiteral`, including custom and plugin patterns, still run unchanged. JavaScript and TypeScript export/reference details: @@ -5635,8 +5642,14 @@ Ordinal の substring として必ず現れなければなりません。正規 literal、共通 literal を持たない optional / alternative path、project の custom pattern、plugin は 意図的に対象外です。C# の不完全 attribute recovery や C++ の same-line member recovery を含め、 pattern list を参照する補助 scan は同じ順序の applicable set を使わなければなりません。comment や -string 内に literal があるため pattern を残すことはありますが、その場合は最適化量が減るだけで、 -本物の match を抑止しません。 +string 内に literal があるため pattern を残すことはあります。各 regex call の直前には、C# property +header の結合、Fortran continuation の連結、Java / Kotlin annotation の除去、C# wrapped modifier の +合成、C++ same-line segment、CSS selector の brace 再構成などを反映した、実際に regex へ渡す input +そのものに同じ Ordinal 判定を適用します。miss は言語固有 recovery を終了せず、regex failure と同様に +扱います。特に C# static constructor の bare identifier 行が gate miss しても、合成した各 `static ...` +wrapper は引き続き試さなければなりません。comment、string、annotation、別 declaration にだけ literal +がある場合は exact-input 判定が失われた最適化を回収し、match は変えません。custom / plugin pattern を +含む `RequiredLiteral` のない pattern は従来どおり実行します。 JavaScript / TypeScript の export / reference 詳細: diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 91efc72ac..fe77b29b4 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -435,8 +435,14 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result all 29 readable `SymbolRecord` fields in emitted order, exercises representative positive fixtures across Python, JavaScript, TypeScript, Go, Rust, Java, C/C++, Swift, F#, Scala, Terraform, Protobuf, and Zig, and verifies literal absence for every annotated language. Keep the IgnoreCase and short- - literal fail-fast checks, C# incomplete-attribute recovery, and C++ same-line recovery in the same - bounded suite. Do not add current-worktree paths, `.cdidx` databases, allocation/wall-clock + literal fail-fast checks in the same bounded suite. Its seam holds the file-level gate on while + switching only the exact-input gate off/on, counts actual pattern-regex calls and exact-input literal + skips, and compares complete ordered output. A bounded mixed-input fixture must prove at least a 30% + attempt reduction without a wall-clock assertion. Pin C# merged properties + and wrapped static constructors, Fortran continuations, Java/Kotlin annotation stripping, C# + incomplete-attribute recovery, C++ same-line members, and CSS reconstructed selector segments; an + initial bare static-constructor input gate miss must still reach wrapped-modifier recovery. Do not add + current-worktree paths, `.cdidx` databases, allocation/wall-clock assertions, or repository-wide extraction to this test; measure cold-index performance externally and pair it with natural-key database parity. - `PerformanceTests.CSharpStaticInterfaceLookup_UnrelatedInterfacesStayWithinAllocationBudget` builds 20,000 unrelated generic interfaces around one contract and keeps lookup allocation below 64 KiB. Discover contract containers before parsing generic declarations, while the paired `ReferenceExtractorTests` case preserves declaration/member ordering, partial-interface last-write behavior, and contract member order. @@ -1460,7 +1466,13 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 含む `SymbolRecord` の readable field 29個すべて、Python、JavaScript、TypeScript、Go、Rust、 Java、C/C++、Swift、F#、Scala、Terraform、Protobuf、Zig の代表的な positive fixture、注釈済み 全言語での literal 不在を検証します。同じ bounded suite に IgnoreCase / 短い literal の - fail-fast、C# の不完全 attribute recovery、C++ の same-line recovery を維持してください。 + fail-fast を維持してください。seam は file-level gate を有効なまま exact-input gate だけを off / on + し、実際の pattern-regex call 数と exact-input literal skip 数を記録して、順序を含む完全な output を + 比較します。bounded mixed-input fixture では wall-clock assertion を使わず、attempt が30%以上減る + ことを検証してください。C# の merged property / wrapped static + constructor、Fortran continuation、Java / Kotlin annotation 除去、C# の不完全 attribute recovery、 + C++ same-line member、CSS の再構成済み selector segment を固定し、bare static-constructor input の + 初回 gate miss 後も wrapped-modifier recovery へ進むことを検証してください。 current worktree path、`.cdidx` database、allocation / wall-clock assertion、repository 全体の extraction をこの test に追加してはいけません。cold-index performance は外部で測定し、database の natural-key parity と対にしてください。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 876d296e1..20a23a5dc 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -14,8 +14,10 @@ affected: - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Css.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs + - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Java.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Patterns.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs @@ -45,7 +47,7 @@ affected: - **Fresh graph planning uses post-load cardinalities** — truly empty CLI and MCP bulk loads now analyze the populated file, symbol, and reference tables immediately before candidate resolution, allowing SQLite to plan the expensive first graph build from current statistics. Rebuilds, updates, existing databases, and symbols-only runs keep their prior lifecycle; cancellation still aborts, while a statistics-only SQLite failure rolls back its savepoint and continues best-effort. - **Persistent symbol workers reuse bounded pattern-directory snapshots across languages** — each project-root reload discovers user and root configs once, while nested ancestor directories, including missing and rejected results, are observed once per worker command. The cache follows live filesystem casing and falls back to uncached discovery when full so configs are never skipped; direct registry callers keep dynamic discovery. - **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once built-in symbols already extracted for the static-interface workspace. After materializing the immutable lookup snapshots, the prepass transfers ownership of admitted per-file symbol lists and releases the redundant workspace fallback objects instead of cloning the full symbol graph. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. -- **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate before line-by-line matching. Pattern order and output stay unchanged, and C# incomplete-attribute plus C++ same-line recovery consume the same filtered set. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. +- **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate both at file selection and immediately before each regex call against its exact transformed input. Pattern order and output stay unchanged across C#/Fortran merges, Java/Kotlin annotation stripping, C# wrapped-modifier and incomplete-attribute recovery, C++ same-line members, and CSS reconstructed selector segments; a bare C# static-constructor gate miss still reaches the synthesized `static ...` retry. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. - **Symbol workers consume the existing UTF-8 request frames without decoding them twice** — the parent keeps its single `SerializeToUtf8Bytes` write, while the all-language child path now performs bounded newline framing, validation, and deserialization directly from raw standard-input bytes. CRLF/final-EOF framing, protocol and JSON bounds, cancellation, Unicode behavior, and sanitized invalid-UTF-8/JSON errors remain unchanged; the decoded `TextReader` path stays available for diagnostics. ## 日本語 @@ -57,5 +59,5 @@ affected: - **新規graph planningでbulk load後のcardinalityを使うようにしました** — 真に空のdatabaseから始まるCLI / MCP bulk loadでは、candidate解決の直前に投入済みのfile、symbol、reference tableを解析し、SQLiteが初回の高コストなgraph構築を最新統計から計画できるようにします。rebuild、update、既存database、symbols-onlyのlifecycleは従来どおりです。cancellationは引き続き中断し、統計更新だけのSQLite failureはsavepointを戻してbest-effortで継続します。 - **persistent symbol worker が全言語で上限付き pattern-directory snapshot を再利用するようにしました** — project-root reload ごとに user / root config を1回だけ探索し、missing や reject を含む nested ancestor directory は worker command ごとに初回結果を再利用します。cache は実 filesystem の case policy に従い、飽和時は uncached discovery に fallback して config を skip しません。registry の direct caller は従来どおり動的に探索します。 - **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once で利用できます。immutable な lookup snapshot を materialize した後、prepass は admit した file ごとの symbol list の所有権を移し、symbol graph 全体を clone せず重複する workspace fallback object を解放します。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 -- **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、行単位の match 前に監査済みの2文字以上の literal を Ordinal で判定します。pattern 順と出力は変えず、C# の不完全 attribute recovery と C++ の same-line recovery も同じ filtered set を使います。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 +- **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、file 選択時と各 regex call の直前に、実際の変換済み input に対して監査済みの2文字以上の literal を Ordinal で判定します。C# / Fortran の結合、Java / Kotlin annotation 除去、C# wrapped-modifier / 不完全 attribute recovery、C++ same-line member、CSS の再構成済み selector segment でも pattern 順と出力を変えず、bare C# static constructor の初回 gate miss 後も合成した `static ...` を再試行します。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 - **symbol worker が既存の UTF-8 request frame を二重 decode せず処理するようにしました** — parent 側の `SerializeToUtf8Bytes` による1回の書き込みは変えず、全言語共通の child 経路で標準入力の raw byte から上限付き newline framing、validation、deserialize を直接行います。CRLF / final EOF の framing、protocol / JSON 上限、cancellation、Unicode の挙動、不正 UTF-8 / JSON の sanitization 済み error は従来どおりで、decoded `TextReader` 経路も診断用に維持します。 diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs index 7df45d6e8..7c88ee404 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs @@ -448,7 +448,9 @@ private static string StripLeadingCSharpAttributeLists( ref int attributeBracketDepth, ref int attributeParenDepth, bool insideEnumBody, - IReadOnlyList applicablePatterns) + IReadOnlyList applicablePatterns, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { var index = 0; while (index < line.Length && char.IsWhiteSpace(line[index])) @@ -465,7 +467,9 @@ private static string StripLeadingCSharpAttributeLists( index, insideEnumBody, attributeParenDepth, - applicablePatterns)) + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) { inLeadingAttributeBlock = false; attributeBracketDepth = 0; @@ -535,36 +539,58 @@ private static bool ShouldRecoverFromIncompleteLeadingCSharpAttribute( int firstNonWhitespaceIndex, bool insideEnumBody, int attributeParenDepth, - IReadOnlyList applicablePatterns) + IReadOnlyList applicablePatterns, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { if (firstNonWhitespaceIndex >= line.Length || line[firstNonWhitespaceIndex] == '[') return false; return TryMatchAnyRecoverableCSharpPattern( line, + 0, insideEnumBody, attributeParenDepth, - applicablePatterns); + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts); } private static bool TryMatchAnyRecoverableCSharpPattern( string line, + int lineOffset, bool insideEnumBody, int attributeParenDepth, - IReadOnlyList applicablePatterns) + IReadOnlyList applicablePatterns, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { + var matchInputSpan = line.AsSpan(lineOffset); + string? matchInput = null; foreach (var pattern in applicablePatterns) { if (ReferenceEquals(pattern.Regex, CSharpEnumMemberRegex)) continue; - if (pattern.Regex.IsMatch(line)) + if (!ShouldAttemptPatternRegex( + pattern, + matchInputSpan, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) + { + continue; + } + + matchInput ??= lineOffset == 0 ? line : line[lineOffset..]; + if (pattern.Regex.IsMatch(matchInput)) return true; } - return insideEnumBody - && attributeParenDepth == 0 - && CSharpEnumMemberNameRegex.IsMatch(line); + if (!insideEnumBody || attributeParenDepth != 0) + return false; + + matchInput ??= lineOffset == 0 ? line : line[lineOffset..]; + return CSharpEnumMemberNameRegex.IsMatch(matchInput); } /// @@ -3605,6 +3631,8 @@ private static bool ShouldSkipCSharpSwitchExpressionPropertyCandidate( private static string[] BuildCSharpMatchLines( string[] structuralLines, IReadOnlyList applicablePatterns, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts, out int[]?[] collapsedToRaw) { var matchLines = new string[structuralLines.Length]; @@ -3626,7 +3654,9 @@ private static string[] BuildCSharpMatchLines( ref attributeBracketDepth, ref attributeParenDepth, activeEnumBodyDepth > 0, - applicablePatterns), + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts), out var lineCollapsedToRaw); collapsedToRaw[lineIndex] = lineCollapsedToRaw; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs index fc4638e15..ff27b3a02 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs @@ -21,7 +21,9 @@ private static void ExtractCppSameLineClassBodyMembers( long fileId, string[] lines, IReadOnlyList applicablePatterns, - List symbols) + List symbols, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { var classSymbols = BuildCppSameLineClassSymbolSnapshot(symbols); if (classSymbols is null) @@ -58,6 +60,8 @@ private static void ExtractCppSameLineClassBodyMembers( lineIndex + 1, applicablePatterns, symbols, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts, ref existingMembers); } } @@ -148,6 +152,8 @@ private static bool TryAddCppSameLineClassMemberSymbol( int lineNumber, IReadOnlyList applicablePatterns, List symbols, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts, ref HashSet? existingMembers) { foreach (var pattern in applicablePatterns) @@ -155,6 +161,15 @@ private static bool TryAddCppSameLineClassMemberSymbol( if (pattern.Kind != "function") continue; + if (!ShouldAttemptPatternRegex( + pattern, + segment.AsSpan(), + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) + { + continue; + } + var match = pattern.Regex.Match(segment); if (!match.Success) continue; diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Css.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Css.cs index 747c9a903..4dee8e254 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Css.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Css.cs @@ -89,7 +89,9 @@ private static void TryAddCssInlineSelectorSegment( int openingBraceIndex, IReadOnlyList patterns, List symbols, - HashSet? cssSeenSymbols) + HashSet? cssSeenSymbols, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { if (string.IsNullOrWhiteSpace(maskedSegment)) return; @@ -100,6 +102,15 @@ private static void TryAddCssInlineSelectorSegment( if (pattern.BodyStyle != BodyStyle.Brace) continue; + if (!ShouldAttemptPatternRegex( + pattern, + matchLine.AsSpan(), + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) + { + continue; + } + var match = pattern.Regex.Match(matchLine); if (!match.Success) continue; @@ -141,7 +152,9 @@ private static void TryAddCssSelectorListSegments( int openingBraceIndex, IReadOnlyList patterns, List symbols, - HashSet? cssSeenSymbols) + HashSet? cssSeenSymbols, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { foreach (var (rawPart, maskedPart) in EnumerateCssCommaSeparatedSegments(rawSegment, maskedSegment)) { @@ -154,7 +167,9 @@ private static void TryAddCssSelectorListSegments( openingBraceIndex, patterns, symbols, - cssSeenSymbols); + cssSeenSymbols, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts); } } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs index 96da727a2..b9bb4fae9 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractCore.cs @@ -21,7 +21,8 @@ private static List ExtractCore( bool patternConfigsAlreadyLoaded = false, CancellationToken cancellationToken = default, int? maxSymbols = null, - bool applyRequiredLiteralGate = true, + bool applyRequiredLiteralFileGate = true, + bool applyRequiredLiteralMatchInputGate = true, RequiredLiteralGateCounts? requiredLiteralGateCounts = null) { var originalLang = lang; @@ -107,14 +108,20 @@ private static List ExtractCore( var applicablePatterns = SelectApplicablePatterns( patterns, content, - applyRequiredLiteralGate); + applyRequiredLiteralFileGate); if (requiredLiteralGateCounts != null) { requiredLiteralGateCounts.PatternCount = patterns.Count; requiredLiteralGateCounts.ApplicablePatternCount = applicablePatterns.Count; } - var scanInputs = new PatternScanInputs(lang, filePath, lines, applicablePatterns); + var scanInputs = new PatternScanInputs( + lang, + filePath, + lines, + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts); var pythonModulePrefix = scanInputs.PythonModulePrefix; var structuralLines = scanInputs.StructuralLines; var scientificBodyScannerLines = scanInputs.ScientificBodyScannerLines; @@ -226,11 +233,45 @@ private static List ExtractCore( while (lineOffset >= 0 && lineOffset < patternMatchLine.Length) { var javaLeadingAnnotationOffset = 0; - var match = lang is "java" or "kotlin" - ? (TryMatchJavaDeclarationSegment(pattern.Regex, patternMatchLine[lineOffset..], lang == "kotlin", out var javaMatch, out javaLeadingAnnotationOffset) - ? javaMatch - : pattern.Regex.Match(patternMatchLine[lineOffset..])) - : pattern.Regex.Match(patternMatchLine[lineOffset..]); + Match match; + if (lang is "java" or "kotlin") + { + var javaPatternMatched = TryMatchJavaDeclarationPatternSegment( + pattern, + patternMatchLine, + lineOffset, + lang == "kotlin", + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts, + out match, + out javaLeadingAnnotationOffset, + out var initialJavaInputAttempted); + if (!javaPatternMatched + && initialJavaInputAttempted + && ShouldAttemptPatternRegex( + pattern, + patternMatchLine.AsSpan(lineOffset), + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) + { + // Preserve the existing failed-helper fallback attempt. This is + // intentionally gated again because it is a distinct regex call. + match = pattern.Regex.Match(patternMatchLine[lineOffset..]); + } + } + else if (ShouldAttemptPatternRegex( + pattern, + patternMatchLine.AsSpan(lineOffset), + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) + { + match = pattern.Regex.Match(patternMatchLine[lineOffset..]); + } + else + { + match = Match.Empty; + } + if (!match.Success && lang == "csharp" && pattern.Kind == "function" @@ -259,6 +300,15 @@ private static List ExtractCore( foreach (var candidatePrefix in EnumerateCSharpWrappedModifierCandidates(wrappedInfo.Value.Prefix)) { var wrappedMatchLine = candidatePrefix + " " + patternMatchLine.TrimStart(); + if (!ShouldAttemptPatternRegex( + pattern, + wrappedMatchLine.AsSpan(), + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) + { + continue; + } + var wrappedMatch = pattern.Regex.Match(wrappedMatchLine); if (wrappedMatch.Success) { @@ -326,16 +376,22 @@ private static List ExtractCore( && pattern.BodyStyle == BodyStyle.None && !(lineOffset != patternStartOffset ? TryMatchAnyRecoverableCSharpPattern( - matchLine[lineOffset..], + matchLine, + lineOffset, insideEnumBody: false, attributeParenDepth: 0, - applicablePatterns) + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts) : recoverableCSharpPatternAtPatternStart ??= TryMatchAnyRecoverableCSharpPattern( - matchLine[lineOffset..], + matchLine, + lineOffset, insideEnumBody: false, attributeParenDepth: 0, - applicablePatterns)))) + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)))) { lineOffset = FindNextSameLineBraceStatementStart(matchLine, lineOffset + 1, lang); continue; @@ -853,7 +909,9 @@ private static List ExtractCore( openingBraceIndex, applicablePatterns, symbols, - cssSeenSymbols); + cssSeenSymbols, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts); } } @@ -1300,7 +1358,9 @@ private static List ExtractCore( csharpMatchLines, pythonModulePrefix, prologMultilineHeads, - applicablePatterns); + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts); } if (lang == "csharp") { @@ -1358,6 +1418,30 @@ private static IReadOnlyList SelectApplicablePatterns( return applicablePatterns ?? patterns; } + private static bool ShouldAttemptPatternRegex( + SymbolPattern pattern, + ReadOnlySpan matchInput, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) + { + // This second-stage proof must inspect the exact transformed input for one regex call. + // Callers treat false as a failed match and must still run language-specific recovery. + // 第2段の proof は1回の regex call に渡す変換済み input そのものを調べる。 + // false は match failure と同様に扱い、言語固有 recovery は引き続き実行する。 + if (applyRequiredLiteralMatchInputGate + && pattern.RequiredLiteral is { } requiredLiteral + && matchInput.IndexOf(requiredLiteral.AsSpan(), StringComparison.Ordinal) < 0) + { + if (requiredLiteralGateCounts != null) + requiredLiteralGateCounts.MatchInputLiteralSkipCount++; + return false; + } + + if (requiredLiteralGateCounts != null) + requiredLiteralGateCounts.RegexAttemptCount++; + return true; + } + private static readonly Regex PrologOpenClauseRegex = new( @"^\s*(?:(?:[a-z][A-Za-z0-9_]*\s*(?:\([^\r\n]*\))?\s*(?::-|-->))|:-)", RegexOptions.Compiled | RegexOptions.CultureInvariant); diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs index 42fa75dac..acd4ab60d 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.ExtractionPhases.cs @@ -23,7 +23,9 @@ public PatternScanInputs( string lang, string? filePath, string[] lines, - IReadOnlyList applicablePatterns) + IReadOnlyList applicablePatterns, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { _lang = lang; _lines = lines; @@ -76,7 +78,12 @@ public PatternScanInputs( int[]?[] csharpMatchColumnToRaw = null!; CSharpMatchLines = lang == "csharp" - ? BuildCSharpMatchLines(lines, applicablePatterns, out csharpMatchColumnToRaw) + ? BuildCSharpMatchLines( + lines, + applicablePatterns, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts, + out csharpMatchColumnToRaw) : null; CSharpMatchColumnToRaw = csharpMatchColumnToRaw; GetCSharpLineStartStates = lang == "csharp" @@ -651,7 +658,9 @@ private static void AddSupplementalSymbols( string[]? csharpMatchLines, string? pythonModulePrefix, Dictionary? prologMultilineHeads, - IReadOnlyList applicablePatterns) + IReadOnlyList applicablePatterns, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts) { if (lang == "javascript") ExtractJavaScriptBareMethods(fileId, lines, symbols, getPrivateScopeColumns!, getJavaScriptTypeScriptSanitizedLines); @@ -682,7 +691,13 @@ private static void AddSupplementalSymbols( ExtractGoGroupedDeclarations(fileId, lines, symbols, extractionState); if (lang == "cpp") { - ExtractCppSameLineClassBodyMembers(fileId, lines, applicablePatterns, symbols); + ExtractCppSameLineClassBodyMembers( + fileId, + lines, + applicablePatterns, + symbols, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts); ExtractCppBalancedCallableSymbols(fileId, lines, structuralLines, symbols, extractionState); ExtractCppFriendDeclarationSymbols(fileId, lines, symbols, extractionState); } diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Java.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Java.cs index a8dd452c5..3862b761d 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Java.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.Java.cs @@ -1284,6 +1284,55 @@ private static bool TryConsumeKotlinAnnotationTarget(string span, ref int index) return true; } + private static bool TryMatchJavaDeclarationPatternSegment( + SymbolPattern pattern, + string matchLine, + int segmentOffset, + bool allowKotlinUseSiteTargets, + bool applyRequiredLiteralMatchInputGate, + RequiredLiteralGateCounts? requiredLiteralGateCounts, + out Match match, + out int leadingAnnotationOffset, + out bool initialInputAttempted) + { + var segmentSpan = matchLine.AsSpan(segmentOffset); + initialInputAttempted = ShouldAttemptPatternRegex( + pattern, + segmentSpan, + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts); + leadingAnnotationOffset = 0; + if (!initialInputAttempted) + { + match = Match.Empty; + return false; + } + + var segment = segmentOffset == 0 ? matchLine : matchLine[segmentOffset..]; + match = pattern.Regex.Match(segment); + if (match.Success) + return true; + + var skippedOffset = SkipLeadingJavaAnnotations(segment, allowKotlinUseSiteTargets); + if (skippedOffset <= 0 || skippedOffset >= segment.Length + || !ShouldAttemptPatternRegex( + pattern, + segment.AsSpan(skippedOffset), + applyRequiredLiteralMatchInputGate, + requiredLiteralGateCounts)) + { + return false; + } + + var strippedMatch = pattern.Regex.Match(segment[skippedOffset..]); + if (!strippedMatch.Success) + return false; + + match = strippedMatch; + leadingAnnotationOffset = skippedOffset; + return true; + } + private static bool TryMatchJavaDeclarationSegment( Regex regex, string segment, diff --git a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs index 0e59952c4..64c8e812e 100644 --- a/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs +++ b/src/CodeIndex/Indexer/Symbols/SymbolExtractor.cs @@ -344,15 +344,20 @@ private sealed class RequiredLiteralGateCounts { public int PatternCount { get; set; } public int ApplicablePatternCount { get; set; } + public int RegexAttemptCount { get; set; } + public int MatchInputLiteralSkipCount { get; set; } } internal static List ExtractForRequiredLiteralGateTesting( long fileId, string lang, string content, - bool applyRequiredLiteralGate, + bool applyRequiredLiteralFileGate, + bool applyRequiredLiteralMatchInputGate, out int patternCount, out int applicablePatternCount, + out int regexAttemptCount, + out int matchInputLiteralSkipCount, string? filePath = null, string? projectRoot = null, CancellationToken cancellationToken = default) @@ -370,10 +375,13 @@ internal static List ExtractForRequiredLiteralGateTesting( patternConfigsAlreadyLoaded: false, cancellationToken: cancellationToken, maxSymbols: null, - applyRequiredLiteralGate: applyRequiredLiteralGate, + applyRequiredLiteralFileGate: applyRequiredLiteralFileGate, + applyRequiredLiteralMatchInputGate: applyRequiredLiteralMatchInputGate, requiredLiteralGateCounts: counts); patternCount = counts.PatternCount; applicablePatternCount = counts.ApplicablePatternCount; + regexAttemptCount = counts.RegexAttemptCount; + matchInputLiteralSkipCount = counts.MatchInputLiteralSkipCount; return symbols; } diff --git a/tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs b/tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs index 4e5c661d1..76d2acee6 100644 --- a/tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs @@ -7,6 +7,12 @@ namespace CodeIndex.Tests; public sealed class SymbolExtractorRequiredLiteralGateTests { + private readonly record struct GateMetrics( + int PatternCount, + int ApplicablePatternCount, + int RegexAttemptCount, + int MatchInputLiteralSkipCount); + private static readonly PropertyInfo[] SymbolProperties = typeof(SymbolRecord) .GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) .Where(property => property.CanRead && property.GetIndexParameters().Length == 0) @@ -87,6 +93,68 @@ public sealed class SymbolExtractorRequiredLiteralGateTests }, }; + public static TheoryData TransformedInputFixtures => new() + { + { + "csharp", + """ + internal class Cache + { + static + Cache() { } + } + """, + "function", + "Cache", + true + }, + { + "csharp", + """ + internal class Box + { + public int Value + => 42; + } + """, + "property", + "Value", + true + }, + { + "fortran", + """ + subroutine & + & Run() + end subroutine Run + """, + "function", + "Run", + false + }, + { + "java", + "@classMarker public interface Annotated {}\n", + "interface", + "Annotated", + true + }, + { + "kotlin", + "@file:funMarker public interface Annotated {}\n", + "interface", + "Annotated", + false + }, + { + "css", + ":root, .theme { color: red; }\n", + "class", + ".theme", + true + }, + }; + [Fact] public void RequiredLiteralMetadata_UsesOnlyAuditedCaseSensitiveTierAValues() { @@ -124,13 +192,23 @@ public void Extract_RequiredLiteralGatePreservesRepresentativeLanguageOutput( string content, string expectedSymbolName) { - var baseline = Extract(language, content, applyRequiredLiteralGate: false, out var patternCount, out _); - var gated = Extract(language, content, applyRequiredLiteralGate: true, out _, out var applicablePatternCount); + var baseline = Extract( + language, + content, + applyRequiredLiteralFileGate: false, + applyRequiredLiteralMatchInputGate: false, + out var baselineMetrics); + var gated = Extract( + language, + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: true, + out var gatedMetrics); AssertSymbolsEqual(baseline, gated, language); Assert.Contains(gated, symbol => symbol.Name == expectedSymbolName); Assert.True( - applicablePatternCount < patternCount, + gatedMetrics.ApplicablePatternCount < baselineMetrics.PatternCount, $"{language} fixture did not skip any impossible patterns."); } @@ -145,18 +223,28 @@ public void Extract_RequiredLiteralGateSkipsAbsentLiteralsForEveryAnnotatedLangu foreach (var language in languages) { const string content = "Ω １２３ ＣＬＡＳＳ ＦＵＮＣＴＩＯＮ\n"; - var baseline = Extract(language, content, applyRequiredLiteralGate: false, out var patternCount, out _); - var gated = Extract(language, content, applyRequiredLiteralGate: true, out _, out var applicablePatternCount); + var baseline = Extract( + language, + content, + applyRequiredLiteralFileGate: false, + applyRequiredLiteralMatchInputGate: false, + out var baselineMetrics); + var gated = Extract( + language, + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: true, + out var gatedMetrics); AssertSymbolsEqual(baseline, gated, language); Assert.True( - applicablePatternCount < patternCount, + gatedMetrics.ApplicablePatternCount < baselineMetrics.PatternCount, $"{language} did not skip an absent Ordinal required literal."); } } [Fact] - public void Extract_CSharp_RequiredLiteralGatePreservesAdversarialOutput() + public void Extract_CSharp_RequiredLiteralMatchInputGatePreservesAdversarialOutput() { const string content = """ // namespace interface enum struct operator event delegate partial readonly extern using @@ -172,59 +260,183 @@ internal class Ωmega } """; - var baseline = Extract("csharp", content, applyRequiredLiteralGate: false, out var patternCount, out _); - var gated = Extract("csharp", content, applyRequiredLiteralGate: true, out _, out var applicablePatternCount); + var baseline = Extract( + "csharp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: false, + out var baselineMetrics); + var gated = Extract( + "csharp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: true, + out var gatedMetrics); AssertSymbolsEqual(baseline, gated); Assert.Contains(gated, symbol => symbol.Kind == "class" && symbol.Name == "Ωmega"); - Assert.True(applicablePatternCount < patternCount); + Assert.Equal(baselineMetrics.ApplicablePatternCount, gatedMetrics.ApplicablePatternCount); + Assert.Equal(0, baselineMetrics.MatchInputLiteralSkipCount); + Assert.True(gatedMetrics.MatchInputLiteralSkipCount > 0); + Assert.True(gatedMetrics.RegexAttemptCount < baselineMetrics.RegexAttemptCount); + } + + [Theory] + [MemberData(nameof(TransformedInputFixtures))] + public void Extract_RequiredLiteralMatchInputGatePreservesTransformedAndSupplementalInputs( + string language, + string content, + string expectedKind, + string expectedName, + bool expectMatchInputLiteralSkip) + { + var baseline = Extract( + language, + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: false, + out var baselineMetrics); + var gated = Extract( + language, + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: true, + out var gatedMetrics); + + AssertSymbolsEqual(baseline, gated, language); + Assert.Contains(gated, symbol => symbol.Kind == expectedKind && symbol.Name == expectedName); + Assert.Equal(baselineMetrics.ApplicablePatternCount, gatedMetrics.ApplicablePatternCount); + Assert.Equal(0, baselineMetrics.MatchInputLiteralSkipCount); + Assert.Equal(expectMatchInputLiteralSkip, gatedMetrics.MatchInputLiteralSkipCount > 0); + if (expectMatchInputLiteralSkip) + { + Assert.True( + gatedMetrics.RegexAttemptCount < baselineMetrics.RegexAttemptCount, + $"{language} exact-input gate did not reduce regex attempts."); + } + else + { + Assert.Equal(baselineMetrics.RegexAttemptCount, gatedMetrics.RegexAttemptCount); + } } [Fact] public void Extract_CSharpIncompleteAttributeRecovery_UsesApplicablePatterns() { const string content = """ - [Broken( + [operatorMarker( public class Recovered { } """; - var baseline = Extract("csharp", content, applyRequiredLiteralGate: false, out _, out _); - var gated = Extract("csharp", content, applyRequiredLiteralGate: true, out var patternCount, out var applicablePatternCount); + var baseline = Extract( + "csharp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: false, + out var baselineMetrics); + var gated = Extract( + "csharp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: true, + out var gatedMetrics); AssertSymbolsEqual(baseline, gated); Assert.Contains(gated, symbol => symbol.Kind == "class" && symbol.Name == "Recovered"); - Assert.True(applicablePatternCount < patternCount); + Assert.Equal(baselineMetrics.ApplicablePatternCount, gatedMetrics.ApplicablePatternCount); + Assert.Equal(0, baselineMetrics.MatchInputLiteralSkipCount); + Assert.True(gatedMetrics.MatchInputLiteralSkipCount > 0); + Assert.True(gatedMetrics.RegexAttemptCount < baselineMetrics.RegexAttemptCount); } [Fact] public void Extract_CppSameLineRecovery_UsesApplicablePatterns() { - const string content = "class Box { public: int Run(); };"; + const string content = "class Box { public: int Run(); };\n// operator\n"; - var baseline = Extract("cpp", content, applyRequiredLiteralGate: false, out _, out _); - var gated = Extract("cpp", content, applyRequiredLiteralGate: true, out var patternCount, out var applicablePatternCount); + var baseline = Extract( + "cpp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: false, + out var baselineMetrics); + var gated = Extract( + "cpp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: true, + out var gatedMetrics); AssertSymbolsEqual(baseline, gated); Assert.Contains(gated, symbol => symbol.Kind == "class" && symbol.Name == "Box"); Assert.Contains(gated, symbol => symbol.Kind == "function" && symbol.Name == "Run"); - Assert.True(applicablePatternCount < patternCount); + Assert.Equal(baselineMetrics.ApplicablePatternCount, gatedMetrics.ApplicablePatternCount); + Assert.Equal(0, baselineMetrics.MatchInputLiteralSkipCount); + Assert.True(gatedMetrics.MatchInputLiteralSkipCount > 0); + Assert.True(gatedMetrics.RegexAttemptCount < baselineMetrics.RegexAttemptCount); + } + + [Fact] + public void Extract_RequiredLiteralMatchInputGateReducesRegexAttemptsAtLeastThirtyPercent() + { + var requiredLiterals = SymbolExtractor.GetRequiredLiteralGateMetadataForTesting() + .Where(entry => entry.Language == "csharp") + .Select(entry => entry.Literal) + .Distinct(StringComparer.Ordinal) + .OrderBy(literal => literal, StringComparer.Ordinal); + var content = "// " + string.Join(" ", requiredLiterals) + "\n" + + string.Join("\n", Enumerable.Range(0, 64).Select(index => $"unrelated_token_{index};")); + + var baseline = Extract( + "csharp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: false, + out var baselineMetrics); + var gated = Extract( + "csharp", + content, + applyRequiredLiteralFileGate: true, + applyRequiredLiteralMatchInputGate: true, + out var gatedMetrics); + + AssertSymbolsEqual(baseline, gated); + Assert.Equal(baselineMetrics.PatternCount, baselineMetrics.ApplicablePatternCount); + Assert.Equal(baselineMetrics.ApplicablePatternCount, gatedMetrics.ApplicablePatternCount); + Assert.Equal(0, baselineMetrics.MatchInputLiteralSkipCount); + Assert.True(gatedMetrics.MatchInputLiteralSkipCount > 0); + Assert.True( + (long)gatedMetrics.RegexAttemptCount * 10 <= (long)baselineMetrics.RegexAttemptCount * 7, + $"Exact-input gate reduced attempts from {baselineMetrics.RegexAttemptCount} " + + $"to {gatedMetrics.RegexAttemptCount}, less than the required 30% reduction."); } private static List Extract( string language, string content, - bool applyRequiredLiteralGate, - out int patternCount, - out int applicablePatternCount) => - SymbolExtractor.ExtractForRequiredLiteralGateTesting( + bool applyRequiredLiteralFileGate, + bool applyRequiredLiteralMatchInputGate, + out GateMetrics metrics) + { + var symbols = SymbolExtractor.ExtractForRequiredLiteralGateTesting( 1, language, content, - applyRequiredLiteralGate, - out patternCount, - out applicablePatternCount); + applyRequiredLiteralFileGate, + applyRequiredLiteralMatchInputGate, + out var patternCount, + out var applicablePatternCount, + out var regexAttemptCount, + out var matchInputLiteralSkipCount); + metrics = new GateMetrics( + patternCount, + applicablePatternCount, + regexAttemptCount, + matchInputLiteralSkipCount); + return symbols; + } private static void AssertSymbolsEqual( IReadOnlyList expected, From d8a71cf4d5f550e43be257206f374eb17bf996a5 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 16:23:41 +0900 Subject: [PATCH 12/16] Skip redundant fresh fold verification --- DEVELOPER_GUIDE.md | 39 ++++ TESTING_GUIDE.md | 4 + .../+large-codebase-initial-indexing.fixed.md | 10 + ...mmandRunner.FullScan.ExtractionPipeline.cs | 7 + .../IndexCommandRunner.FullScan.Readiness.cs | 24 +- .../Cli/IndexCommandRunner.FullScan.cs | 10 + .../Database/DbWriter.FoldBackfill.cs | 8 + .../Database/DbWriter.FreshFoldReadiness.cs | 117 ++++++++++ src/CodeIndex/Database/DbWriter.ReadyFlags.cs | 16 +- ...xtractorPluginRegistry.PatternWorkspace.cs | 107 +++++++-- .../ExtractorPluginRegistry.PluginLoading.cs | 1 + .../Extensibility/ExtractorPluginRegistry.cs | 59 +++++ .../Mcp/McpToolHandlers.Indexing.Execution.cs | 19 +- tests/CodeIndex.Tests/DatabaseTests.cs | 216 ++++++++++++++++++ .../ExtractorPluginRegistryTests.cs | 40 ++++ .../IndexCommandRunnerFullScanTests.cs | 130 +++++++++++ .../IndexCommandRunnerTests.cs | 6 + .../McpServerToolsCallTests.cs | 16 ++ 18 files changed, 809 insertions(+), 20 deletions(-) create mode 100644 src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 069b8eb84..2ae20aae4 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1574,6 +1574,25 @@ updates, rebuilds, retained-graph rebuilds, and MCP indexing keep the establishe particular, MCP can durably commit per-file batches before graph finalization, so it must retain its existing retry semantics until a separately designed recovery contract can cover that state. +Fold readiness has a narrower authoritative-fresh optimization shared by ordinary CLI and MCP +full indexing. Before the first write, `DbWriter` may issue an opaque, one-shot claim only from a +`BEGIN IMMEDIATE` snapshot in which `files`, `symbols`, and `symbol_references` are all empty. The +claim is bound to its writer/connection and captured `PRAGMA data_version`; only that writer may +consume it once, and an intervening commit from another connection invalidates it. When the +built-in extractor pipeline completes successfully, finalization may use the claim to omit the +allocation-heavy read and re-fold of every persisted symbol/reference value, but it still runs +the SQL NULL-completeness check before stamping FoldReady. +The run also captures the registry's monotonic accepted-producer mutation generation. Any +generation change invalidates the claim even if a transient custom producer was later removed and +the final registry is built-in-only again; staged workspace replacement commits participate in +that history, while diagnostic-only publication and unchanged missing-directory discovery do not. + +This shortcut is deliberately fail-closed. Rebuilds, updates, legacy or existing indexes, public +`DbWriter` readiness APIs, custom plugins or pattern configurations, post-extraction hooks, incorrectly +owned or reused claims, and any externally committed database change use the established full +value validation. The shortcut changes neither the folded values produced during insertion nor +the readiness transaction and rollback semantics. + For C# explicit-interface members, `symbols.name` remains the short display/discovery alias, while `symbols.name_folded` stores the normalized interface qualifier plus terminal method generic arity. When that identity differs, `symbols.display_name_folded` stores the short @@ -5319,6 +5338,26 @@ rebuild、MCP indexingは従来経路を維持します。特にMCPはgraph fina commitできるため、その状態を扱う独立したrecovery契約が設計されるまでは既存の再試行semanticsを 変更してはいけません。 +fold readiness には、通常の CLI / MCP full indexing が共有する、より限定的な authoritative-fresh +最適化があります。最初の書き込み前に、`DbWriter` は `files`、`symbols`、 +`symbol_references` がすべて空である同一の `BEGIN IMMEDIATE` snapshot からのみ、opaque で +一回限りの claim を発行できます。claim は writer / connection と取得時の +`PRAGMA data_version` に束縛され、同じ writer が一度だけ consume できます。stamp 決定前に +別 connection が commit した場合は無効になります。built-in extractor pipeline が正常完了した +場合、finalization は claim を使って、永続化済みの全 symbol / reference value を読み出して +再 fold する allocation-heavy な処理を省けますが、FoldReady を stamp する前の SQL による +NULL completeness check は引き続き実行します。 +run は registry の accepted-producer mutation generation も取得します。一時的な custom producer +が後で削除され、最終 registry が再び built-in-only になっていても、generation が変化していれば +claim を無効化します。staged workspace replacement の commit もこの履歴に含めますが、 +diagnostic-only publication と状態が変わらない missing-directory discovery は generation を進めません。 + +この shortcut は意図的に fail closed です。rebuild、update、legacy または既存 index、public な +`DbWriter` readiness API、custom plugin / pattern config、post-extraction hook、owner が異なるか +再利用された claim、外部 connection が commit した database では、従来の full value validation +を使います。shortcut は insert 時に生成する folded value も、readiness transaction / rollback の +semantics も変更しません。 + C# の明示的 interface member では、`symbols.name` は短い表示用 / discovery alias のままにし、 `symbols.name_folded` に正規化した interface qualifier と末尾 method の generic arity を 保存します。identity が異なる場合は `symbols.display_name_folded` に短い Unicode-folded diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index fe77b29b4..fd6177cf8 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -696,6 +696,8 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result Concurrent read and read-during-write scenarios (WAL mode validation), including the issue #180 bug-catching snapshot-isolation regressions for all three multi-statement reader entry points: (1) `GetStatus` seeds `refs == files * refsPerFile` and asserts every concurrent observation preserves that invariant; (2) `AnalyzeSymbol` seeds one symbol `S` plus matching reference/caller pairs, toggles a second file symmetrically, and asserts `references.Count == callers.Count` across every `inspect`/`analyze_symbol` bundle; (3) `GetRepoMap` seeds a baseline modified timestamp and toggles a newer file, asserting `latest_modified == workspace_latest_modified` across every map call. Each test fails without the DEFERRED-transaction wrap on the matching reader and passes with it. - `PerformanceTests.cs` Bounded CI smoke coverage plus large-scale data benchmarks. `CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` and the allocation budget guards run in the default `net8.0` suite, so they are blocking PR/CI checks on the production target, but their broad budgets are intended to catch only severe indexing/search or allocation regressions rather than act as benchmarks. `ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` uses dense C# private-property receivers and Python imported-type calls to prevent per-candidate full-symbol rescans from returning. `ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` covers dense C# declaration containers and GitHub Actions jobs so name/range ownership resolution stays indexed. `Extraction_DenseDelimitedLists_StayWithinAllocationBudget` covers Python imports, YAML needs, JSON paths, and Fortran procedure lists without temporary split-array growth. `ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` keeps all-language dedupe identities value-based when qualified names are long. The 10K+ large-scale tests remain skip-by-default; run them manually with `--filter`. + Focused authoritative-fresh fold-readiness coverage spans `DatabaseTests`, `IndexCommandRunnerTests`, and `McpServerToolsCallTests`: the built-in empty-database CLI/MCP path must consume its claim once, retain the NULL-column verification, skip the stored-value re-fold scan, and produce the same readiness/version/fingerprint/language stamps and Unicode, Markdown, C#, Nim, and TypeScript query results as full validation. Pair it with fail-closed cases for each initially nonempty ownership table (`files`, `symbols`, or `symbol_references`), a wrong or reused claim, an intervening external commit observed through `PRAGMA data_version`, rebuild/update/legacy/public-writer paths, and custom plugins, patterns, or post-extraction hooks; full validation must still reject NULL and stale non-NULL folds. A run-barrier regression must also activate a custom producer and then reload back to built-in-only before readiness: the current producer count returns to zero, but the monotonic mutation generation changes and forces full validation. Unchanged missing-directory and diagnostic-only publications must not change that generation. + For performance audits, alternate identical repository-scale fresh fixtures, isolate the readiness-finalization interval, and report elapsed time plus `GC.GetAllocatedBytesForCurrentThread`; adoption requires removing row-count-proportional managed allocation without changing rows, stamps, or query results. Keep wall-clock measurements out of blocking CI assertions and remove temporary instrumentation after recording the result. `ReferenceExtraction_MaskedMultilinePayloads_StayWithinAllocationBudget` keeps C# raw strings, Java text blocks, and TypeScript template literals from materializing trimmed reference contexts after structural masking has made a line empty. `CppHeaderDetection_LargeSample_DoesNotMaterializeLineArrays` keeps bounded C / C++ header-disambiguation samples on span-based line walks instead of allocating a string and array for every sampled line. `DelimitedSpanWalking_DenseExtractorLists_DoesNotAllocate` locks the shared single-delimiter walker to allocation-free trim/remove-empty semantics used by repository metadata, application manifests, VHDL, and CUDA extraction. @@ -1725,6 +1727,8 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" 並行読み取りと書き込み中読み取りシナリオ（WALモード検証）。issue #180 の bug-catching な snapshot 隔離回帰テストを 3 つの multi-statement reader 経路について含む。(1) `GetStatus` は `refs == files * refsPerFile` の seed 不変条件を立て、並行観測が常にこの条件を維持することを要求する。(2) `AnalyzeSymbol` はシンボル `S` に対して reference/caller を対称に 1 対 1 で seed し、もう 1 ファイルを対称に toggle することで `inspect` / `analyze_symbol` bundle の `references.Count == callers.Count` を常に保証する。(3) `GetRepoMap` はベースラインの modified と新しい toggle 対象ファイルを用意し、`latest_modified == workspace_latest_modified` が常に一致することを要求する。各テストは対応する reader の DEFERRED transaction を外すと落ち、戻すと通ることを確認済み。 - `PerformanceTests.cs` bounded な CI smoke と大規模データベンチマークを扱います。`CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` と allocation budget guard は通常の `net8.0` suite で実行されるため production target 上の PR / CI blocking check ですが、benchmark ではなく重大な indexing/search または allocation 退行だけを拾う広めの budget を使います。`ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` は密な C# private-property receiver と Python imported-type call を使い、candidate ごとの full-symbol 再走査が戻るのを防ぎます。`ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` は密な C# declaration container と GitHub Actions job を扱い、name / range ownership 解決の索引化を維持します。`Extraction_DenseDelimitedLists_StayWithinAllocationBudget` は Python import、YAML needs、JSON path、Fortran procedure list を使い、一時 split-array の増加を防ぎます。`ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` は長い qualified name でも全言語共通 dedupe identity を value-based に維持します。10K+ の大規模テストは引き続きデフォルト Skip で、`--filter` で手動実行します。 + authoritative-fresh fold readiness の focused coverage は `DatabaseTests`、`IndexCommandRunnerTests`、`McpServerToolsCallTests` で分担します。built-in の empty-database CLI / MCP 経路が claim を一度だけ consume し、NULL column の検証を維持しつつ、保存 value の再 fold scan を省き、full validation と同じ readiness / version / fingerprint / language stamp、および Unicode、Markdown、C#、Nim、TypeScript の query result を生成することを固定します。初期状態で ownership table（`files`、`symbols`、`symbol_references`）のいずれかが非空の場合、owner が異なるか再利用された claim、`PRAGMA data_version` で観測される外部 commit、rebuild / update / legacy / public-writer 経路、custom plugin / pattern / post-extraction hook は fail closed であることも対にし、full validation が NULL と stale な非 NULL fold を引き続き拒否することを確認します。run barrier では custom producer を一度 active にしてから readiness 前に built-in-only へ reload し、最終 producer count が zero に戻っていても monotonic mutation generation の変化で full validation へ戻ることを固定します。状態不変の missing-directory と diagnostic-only publication では generation が変わらないことも確認します。 + 性能監査では、同一の repository-scale fresh fixture を交互に実行し、readiness finalization 区間を分離して、経過時間と `GC.GetAllocatedBytesForCurrentThread` を報告します。row 数に比例する managed allocation を取り除きつつ、row、stamp、query result が変わらないことを採用条件にします。wall-clock 計測は blocking CI assertion にせず、結果を記録したら一時 instrumentation を削除してください。 `ReferenceExtraction_MaskedMultilinePayloads_StayWithinAllocationBudget` は、構造マスク後に空行となった C# raw string、Java text block、TypeScript template literal から trim 済み reference context を実体化しないことを固定します。 `CppHeaderDetection_LargeSample_DoesNotMaterializeLineArrays` は、bounded な C / C++ header 判定 sample を span ベースで行走査し、sampled line ごとの string と array を割り当てないことを固定します。 `DelimitedSpanWalking_DenseExtractorLists_DoesNotAllocate` は、repository metadata、application manifest、VHDL、CUDA extraction が共有する single-delimiter walker の trim / remove-empty semantics を allocation-free に固定します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 20a23a5dc..4c0682805 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -2,8 +2,13 @@ category: fixed affected: - src/CodeIndex/Cli/ProgramRunner.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs - src/CodeIndex/Database/DbWriter.cs + - src/CodeIndex/Database/DbWriter.FoldBackfill.cs + - src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs + - src/CodeIndex/Database/DbWriter.ReadyFlags.cs - src/CodeIndex/Database/DbWriter.References.cs - src/CodeIndex/Database/DbWriter.ReferenceSql.cs - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -11,6 +16,8 @@ affected: - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs - src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs + - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PatternWorkspace.cs + - src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PluginLoading.cs - src/CodeIndex/Indexer/Scanning/FileContentLoader.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.CSharpScanner.cs - src/CodeIndex/Indexer/Symbols/SymbolExtractor.Cpp.cs @@ -25,6 +32,7 @@ affected: - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs - tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs - tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs + - tests/CodeIndex.Tests/DatabaseTests.cs - tests/CodeIndex.Tests/FileIndexerTests.cs - tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -49,6 +57,7 @@ affected: - **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once built-in symbols already extracted for the static-interface workspace. After materializing the immutable lookup snapshots, the prepass transfers ownership of admitted per-file symbol lists and releases the redundant workspace fallback objects instead of cloning the full symbol graph. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. - **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate both at file selection and immediately before each regex call against its exact transformed input. Pattern order and output stay unchanged across C#/Fortran merges, Java/Kotlin annotation stripping, C# wrapped-modifier and incomplete-attribute recovery, C++ same-line members, and CSS reconstructed selector segments; a bare C# static-constructor gate miss still reaches the synthesized `static ...` retry. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. - **Symbol workers consume the existing UTF-8 request frames without decoding them twice** — the parent keeps its single `SerializeToUtf8Bytes` write, while the all-language child path now performs bounded newline framing, validation, and deserialization directly from raw standard-input bytes. CRLF/final-EOF framing, protocol and JSON bounds, cancellation, Unicode behavior, and sanitized invalid-UTF-8/JSON errors remain unchanged; the decoded `TextReader` path stays available for diagnostics. +- **Fresh built-in indexes finalize fold readiness without re-folding every stored name** — when ordinary CLI or MCP indexing owns a database proven empty across `files`, `symbols`, and `symbol_references`, an opaque one-use claim guarded by SQLite `data_version` keeps the final SQL NULL-completeness check while avoiding materializing and re-folding every symbol/reference string. A monotonic accepted-producer generation also invalidates the claim when custom plugins or patterns were transiently active and later removed before readiness. This removes row-count-proportional finalization work and hundreds of MiB of managed allocation on large first indexes; rebuilds, updates, legacy or existing indexes, public writer calls, custom plugins, patterns, post-extraction hooks, reused claims, and externally changed databases fail closed to full value validation. ## 日本語 @@ -61,3 +70,4 @@ affected: - **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once で利用できます。immutable な lookup snapshot を materialize した後、prepass は admit した file ごとの symbol list の所有権を移し、symbol graph 全体を clone せず重複する workspace fallback object を解放します。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 - **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、file 選択時と各 regex call の直前に、実際の変換済み input に対して監査済みの2文字以上の literal を Ordinal で判定します。C# / Fortran の結合、Java / Kotlin annotation 除去、C# wrapped-modifier / 不完全 attribute recovery、C++ same-line member、CSS の再構成済み selector segment でも pattern 順と出力を変えず、bare C# static constructor の初回 gate miss 後も合成した `static ...` を再試行します。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 - **symbol worker が既存の UTF-8 request frame を二重 decode せず処理するようにしました** — parent 側の `SerializeToUtf8Bytes` による1回の書き込みは変えず、全言語共通の child 経路で標準入力の raw byte から上限付き newline framing、validation、deserialize を直接行います。CRLF / final EOF の framing、protocol / JSON 上限、cancellation、Unicode の挙動、不正 UTF-8 / JSON の sanitization 済み error は従来どおりで、decoded `TextReader` 経路も診断用に維持します。 +- **新規 built-in index の fold readiness 確定で、保存済みの全名前を再 fold しないようにしました** — 通常の CLI / MCP indexing が `files`、`symbols`、`symbol_references` のすべてが空であると証明された database を所有する場合、SQLite `data_version` で保護された opaque で一回限りの claim により、最後の SQL NULL completeness check を維持しながら、全 symbol / reference string の materialize と再 fold を省きます。単調増加する accepted-producer generation により、custom plugin / pattern が一時的に active になり readiness 前に削除された場合も claim を無効化します。巨大な初回 index で row 数に比例する finalization work と数百 MiB の managed allocation を取り除きます。rebuild、update、legacy または既存 index、public writer 呼び出し、custom plugin / pattern、post-extraction hook、再利用 claim、外部変更された database は fail closed で full value validation に戻ります。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs index 7caab71f4..fdce89410 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs @@ -23,6 +23,11 @@ private sealed class FullScanExtractionPipelineContext internal required int FilesCount { get; init; } internal required bool ForceExtractorRefresh { get; init; } internal required bool StartedWithNoIndexedFiles { get; init; } + internal DbWriter.AuthoritativeFreshFoldRowsClaim? AuthoritativeFreshFoldRowsClaim + { + get; + init; + } internal required bool PriorSymbolsOnlyGraphOmitted { get; init; } internal required bool SymbolKindFilterMatchesPrior { get; init; } internal required bool CSharpIndexedProjectRootCompatible @@ -138,6 +143,8 @@ private static FullScanExtractionPipelineResult context.Options.MaxFileSizeBytes, maxSymbolCount: context.Options.MaxSymbolsPerFile + 1, maxReferenceCount: context.Options.MaxReferencesPerFile + 1); + if (postExtractionHooks.HasHooks) + context.AuthoritativeFreshFoldRowsClaim?.Invalidate(); var scheduling = ResolveFullScanExtractionScheduling( context, postExtractionHooks); diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs index 21b09a66e..751b58dc7 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs @@ -2,6 +2,7 @@ using System.Text.Json; using CodeIndex.Database; using CodeIndex.Indexer; +using CodeIndex.Indexer.Extensibility; namespace CodeIndex.Cli; @@ -25,6 +26,14 @@ private sealed class FullScanReadinessContext internal required int Purged { get; init; } internal required bool ScanHadErrors { get; init; } internal required bool StartedWithNoIndexedFiles { get; init; } + internal DbWriter.AuthoritativeFreshFoldRowsClaim? AuthoritativeFreshFoldRowsClaim + { + get; + init; + } + internal required ExtractorPluginRegistry.FoldProducerReadinessSnapshot + FreshFoldProducerSnapshot + { get; init; } internal required bool HasCSharpFilesAfter { get; init; } internal required bool CSharpSourceEvidenceComplete { get; init; } internal required bool CSharpSourceEvidenceForStamp { get; init; } @@ -202,10 +211,23 @@ context.SkippedSymbolExtractorLanguages is null && writer.SymbolExtractorVersionsMatchCurrent(skippedSymbolExtractorLanguageSet); if (context.Skipped == 0 || canRestampExistingFoldTrust) { + // Re-check the mutable process/workspace registry at the consumption boundary. + // A producer registered after extraction began must fail closed too. + // mutable registry を消費直前にも再確認し、途中登録された producer も拒否する。 + var currentFoldProducerSnapshot = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot( + context.ProjectRoot); + if (!currentFoldProducerSnapshot.UsesOnlyBuiltInProducers + || currentFoldProducerSnapshot.MutationGeneration + != context.FreshFoldProducerSnapshot.MutationGeneration) + { + context.AuthoritativeFreshFoldRowsClaim?.Invalidate(); + } var foldStampResult = writer.MarkFoldReadyWithResult( stampCurrentSymbolExtractorVersions: context.Skipped == 0, symbolExtractorLanguagesToStamp: - context.Skipped == 0 ? context.IndexedSymbolExtractorLanguages : null); + context.Skipped == 0 ? context.IndexedSymbolExtractorLanguages : null, + authoritativeFreshRowsClaim: context.AuthoritativeFreshFoldRowsClaim); foldReadyAfter = foldStampResult == FoldReadyStampResult.Ready; if (foldStampResult == FoldReadyStampResult.MissingBackfill) { diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 10303b646..00b7386fe 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -235,6 +235,13 @@ void ThrowIfFullScanCancelled(int filesProcessed, int? filesTotal) ThrowIfFullScanCancelled(0, files.Count); ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(projectRoot); + var freshFoldProducerSnapshot = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectRoot); + var authoritativeFreshFoldRowsClaim = !options.Rebuild + && startedWithNoIndexedFiles + && freshFoldProducerSnapshot.UsesOnlyBuiltInProducers + ? writer.TryClaimAuthoritativeFreshFoldRows(cancellationToken) + : null; var purgedRefs = 0; int processed = 0, skipped = 0, warnings = warningList.Count, errors = errorList.Count; @@ -942,6 +949,7 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis FilesCount = files.Count, ForceExtractorRefresh = forceExtractorRefresh, StartedWithNoIndexedFiles = startedWithNoIndexedFiles, + AuthoritativeFreshFoldRowsClaim = authoritativeFreshFoldRowsClaim, PriorSymbolsOnlyGraphOmitted = priorSymbolsOnlyGraphOmitted, SymbolKindFilterMatchesPrior = @@ -1313,6 +1321,8 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis Purged = purged, ScanHadErrors = scanHadErrors, StartedWithNoIndexedFiles = startedWithNoIndexedFiles, + AuthoritativeFreshFoldRowsClaim = authoritativeFreshFoldRowsClaim, + FreshFoldProducerSnapshot = freshFoldProducerSnapshot, HasCSharpFilesAfter = hasCSharpFilesAfter, CSharpSourceEvidenceComplete = csharpSourceEvidenceComplete, CSharpSourceEvidenceForStamp = csharpSourceEvidenceForStamp, diff --git a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs index f25156e0f..508183ecf 100644 --- a/src/CodeIndex/Database/DbWriter.FoldBackfill.cs +++ b/src/CodeIndex/Database/DbWriter.FoldBackfill.cs @@ -12,6 +12,7 @@ public partial class DbWriter private static readonly AsyncLocal ScopedFoldBackfillRowUpdatedForTesting = new(); private static readonly AsyncLocal ScopedFoldBackfillVerificationForTesting = new(); + private static readonly AsyncLocal ScopedFoldValueVerificationForTesting = new(); internal static Action? FoldBackfillRowUpdatedForTesting { @@ -25,6 +26,12 @@ internal static Action? FoldBackfillVerificationForTesting set => ScopedFoldBackfillVerificationForTesting.Value = value; } + internal static Action? FoldValueVerificationForTesting + { + get => ScopedFoldValueVerificationForTesting.Value; + set => ScopedFoldValueVerificationForTesting.Value = value; + } + /// /// A pre-v3 C# naming contract can be upgraded without reparsing only when every symbol kind /// that may represent an explicit-interface member still has its declaration signature. @@ -134,6 +141,7 @@ internal bool AllPresentFoldedColumnValuesMatchCurrentFold() private bool AllFoldedColumnValuesMatchCurrentFoldCore(bool allowMissingValues) { + FoldValueVerificationForTesting?.Invoke(); var markdownSymbolIdentityFolds = BuildMarkdownSymbolIdentityFoldMap(); var hasDisplayNameFolded = DbSchemaCache.LoadColumns(_conn, "symbols").Contains("display_name_folded"); diff --git a/src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs b/src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs new file mode 100644 index 000000000..b35a96036 --- /dev/null +++ b/src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs @@ -0,0 +1,117 @@ +using Microsoft.Data.Sqlite; + +namespace CodeIndex.Database; + +public partial class DbWriter +{ + /// + /// One-shot proof that all fold-bearing tables were empty before an authoritative + /// fresh-index run began. The proof is bound to one writer/connection and is invalidated + /// when another connection commits before the readiness stamp is decided. + /// authoritative fresh index 開始前に fold 対象3 table が空だったことを示す一回限りの証明。 + /// writer/connection に束縛し、stamp 決定前に別 connection が commit した場合は無効化する。 + /// + internal sealed class AuthoritativeFreshFoldRowsClaim + { + private readonly DbWriter _owner; + private readonly long _dataVersion; + private int _consumedOrInvalidated; + + internal AuthoritativeFreshFoldRowsClaim(DbWriter owner, long dataVersion) + { + _owner = owner; + _dataVersion = dataVersion; + } + + internal void Invalidate() + => Interlocked.Exchange(ref _consumedOrInvalidated, 1); + + internal bool TryConsume(DbWriter owner, long dataVersion) + { + if (Interlocked.CompareExchange(ref _consumedOrInvalidated, 1, 0) != 0) + return false; + + return ReferenceEquals(_owner, owner) && _dataVersion == dataVersion; + } + } + + /// + /// Claims the authoritative fresh-row shortcut only when files, symbols, and references + /// are all empty in one BEGIN IMMEDIATE snapshot. PRAGMA data_version lets the eventual + /// consumer distinguish this writer's own intervening commits from commits made by any + /// other connection. Existing transactions fail closed because production callers claim + /// before their first write scope. + /// files/symbols/references が同じ BEGIN IMMEDIATE snapshot で全て空の場合だけ claim する。 + /// data_version により同じ writer 自身の commit は許可し、別 connection の commit は拒否する。 + /// + internal AuthoritativeFreshFoldRowsClaim? TryClaimAuthoritativeFreshFoldRows( + CancellationToken cancellationToken = default) + { + var gateLease = EnterTransactionGate( + cancellationToken, + "claim authoritative fresh fold rows"); + try + { + if (IsInTransaction()) + return null; + + var beganTransaction = false; + try + { + Execute("BEGIN IMMEDIATE", cancellationToken); + beganTransaction = true; + + cancellationToken.ThrowIfCancellationRequested(); + using var emptyCheck = _conn.CreateCommand(); + emptyCheck.CommandText = + """ + SELECT CASE + WHEN EXISTS(SELECT 1 FROM files LIMIT 1) + OR EXISTS(SELECT 1 FROM symbols LIMIT 1) + OR EXISTS(SELECT 1 FROM symbol_references LIMIT 1) + THEN 0 + ELSE 1 + END + """; + var allFoldRowTablesEmpty = Convert.ToInt64( + emptyCheck.ExecuteScalar(), + System.Globalization.CultureInfo.InvariantCulture) == 1; + var dataVersion = ReadDataVersion(); + + Execute("COMMIT"); + beganTransaction = false; + return allFoldRowTablesEmpty + ? new AuthoritativeFreshFoldRowsClaim(this, dataVersion) + : null; + } + catch + { + if (beganTransaction) + { + try { Execute("ROLLBACK"); } + catch (SqliteException) { /* best effort */ } + } + + throw; + } + } + finally + { + gateLease.Dispose(); + } + } + + private bool TryConsumeAuthoritativeFreshFoldRowsClaim( + AuthoritativeFreshFoldRowsClaim? claim) + => claim?.TryConsume(this, ReadDataVersion()) == true; + + private long ReadDataVersion() + { + using var command = _conn.CreateCommand(); + command.Transaction = _activeTransaction; + command.CommandText = "PRAGMA data_version"; + return Convert.ToInt64( + command.ExecuteScalar(), + System.Globalization.CultureInfo.InvariantCulture); + } +} diff --git a/src/CodeIndex/Database/DbWriter.ReadyFlags.cs b/src/CodeIndex/Database/DbWriter.ReadyFlags.cs index ba873e160..10eef0fea 100644 --- a/src/CodeIndex/Database/DbWriter.ReadyFlags.cs +++ b/src/CodeIndex/Database/DbWriter.ReadyFlags.cs @@ -46,7 +46,8 @@ public bool MarkFoldReady( internal FoldReadyStampResult MarkFoldReadyWithResult( bool stampCurrentSymbolExtractorVersions = false, - IReadOnlyCollection? symbolExtractorLanguagesToStamp = null) + IReadOnlyCollection? symbolExtractorLanguagesToStamp = null, + AuthoritativeFreshFoldRowsClaim? authoritativeFreshRowsClaim = null) { var gateLease = EnterTransactionGate(); try @@ -59,7 +60,14 @@ internal FoldReadyStampResult MarkFoldReadyWithResult( if (stampCurrentSymbolExtractorVersions) StampSymbolExtractorVersions(symbolExtractorLanguagesToStamp); - var validationResult = ValidateFoldRowsForReadyStamp(); + // A fresh-run claim skips only the expensive value-by-value re-fold. The + // cheap NULL check below always remains authoritative, and the claim is + // consumed once even when that check fails. + // fresh-run claim が省略するのは高コストな全行 re-fold だけであり、NULL + // 検証は常に実行する。NULL 検証失敗時も claim は一回で消費する。 + var verifyCurrentFoldValues = + !TryConsumeAuthoritativeFreshFoldRowsClaim(authoritativeFreshRowsClaim); + var validationResult = ValidateFoldRowsForReadyStamp(verifyCurrentFoldValues); if (validationResult != FoldReadyStampResult.Ready) { if (ownTransaction) @@ -99,7 +107,7 @@ internal FoldReadyStampResult MarkFoldReadyWithResult( } } - private FoldReadyStampResult ValidateFoldRowsForReadyStamp() + private FoldReadyStampResult ValidateFoldRowsForReadyStamp(bool verifyCurrentFoldValues = true) { if (!AllFoldedColumnsBackfilledCore( requireCurrentSymbolExtractorVersions: false, @@ -108,7 +116,7 @@ private FoldReadyStampResult ValidateFoldRowsForReadyStamp() return FoldReadyStampResult.MissingBackfill; } - return AllFoldedColumnValuesMatchCurrentFold() + return !verifyCurrentFoldValues || AllFoldedColumnValuesMatchCurrentFold() ? FoldReadyStampResult.Ready : FoldReadyStampResult.NonCurrentFoldValues; } diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PatternWorkspace.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PatternWorkspace.cs index 4eaef3624..1f41252a9 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PatternWorkspace.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PatternWorkspace.cs @@ -8,6 +8,7 @@ public static partial class ExtractorPluginRegistry private sealed class PatternWorkspaceState(string? workspaceRoot, bool includeUserConfiguration = true) { private ExtractorWorkspaceSnapshot snapshot = ExtractorWorkspaceSnapshot.Empty; + private bool active = workspaceRoot is null; internal object Gate { get; } = new(); internal string? WorkspaceRoot { get; } = workspaceRoot; @@ -31,6 +32,11 @@ private sealed class PatternWorkspaceState(string? workspaceRoot, bool includeUs internal long LastAccessSequence { get; set; } internal long ReloadSequence { get; set; } internal long WorkspaceGeneration { get; set; } + internal bool Active + { + get => Volatile.Read(ref active); + set => Volatile.Write(ref active, value); + } internal ExtractorWorkspaceSnapshot GetSnapshot() => Volatile.Read(ref snapshot); @@ -67,20 +73,24 @@ internal void PublishSnapshot() SetPatternExtensions("user"); SetLanguageExtensions(extensions, user.SymbolExtractors.Values.Select(extractor => (extractor.Language, extractor.FileExtensions))); SetLanguageExtensions(extensions, user.ReferenceExtractors.Values.Select(extractor => (extractor.Language, extractor.FileExtensions))); - Volatile.Write( - ref snapshot, - new ExtractorWorkspaceSnapshot( - new ReadOnlyDictionary(symbolExtractors), - new ReadOnlyDictionary(referenceExtractors), - new ReadOnlyDictionary(extensions), - Configs.ToArray(), - Diagnostics.ToArray(), - ConfigCount, - SkippedFileCount, - DiagnosticTotalCount, - RuleCount, - PluginAssemblyCount, - 0)); + var nextSnapshot = new ExtractorWorkspaceSnapshot( + new ReadOnlyDictionary(symbolExtractors), + new ReadOnlyDictionary(referenceExtractors), + new ReadOnlyDictionary(extensions), + Configs.ToArray(), + Diagnostics.ToArray(), + ConfigCount, + SkippedFileCount, + DiagnosticTotalCount, + RuleCount, + PluginAssemblyCount, + 0); + if (Active + && !AcceptedFoldProducerSnapshotsEqual(GetSnapshot(), nextSnapshot)) + { + AdvanceAcceptedFoldProducerMutationGeneration(); + } + Volatile.Write(ref snapshot, nextSnapshot); void CopyPatternExtractors(string source, Dictionary target) { @@ -110,6 +120,7 @@ internal void Reset() lock (Gate) { Retired = false; + Active = WorkspaceRoot is null; ClearState(); PublishSnapshot(); } @@ -119,6 +130,7 @@ internal void Retire() { lock (Gate) { + Active = false; Retired = true; ClearState(); Volatile.Write(ref snapshot, ExtractorWorkspaceSnapshot.Empty); @@ -189,6 +201,7 @@ private sealed record UserExtractorSnapshot( private static long workspaceAccessSequence; private static long workspaceReloadSequence; private static long workspaceGeneration; + private static long acceptedFoldProducerMutationGeneration; private static PatternWorkspaceState CreatePatternWorkspace( string workspaceRoot, @@ -256,13 +269,21 @@ private static bool TryReplacePatternWorkspace(PatternWorkspaceState state) return false; } + var previousSnapshot = index >= 0 + ? PatternWorkspaces[index].GetSnapshot() + : GetFallbackPatternSnapshot(state.IncludeUserConfiguration); + if (!AcceptedFoldProducerSnapshotsEqual(previousSnapshot, state.GetSnapshot())) + AdvanceAcceptedFoldProducerMutationGeneration(); + if (index >= 0) { replaced = PatternWorkspaces[index]; + replaced.Active = false; PatternWorkspaces[index] = state; } else PatternWorkspaces.Add(state); + state.Active = true; TouchPatternWorkspace(state); evicted = TrimPatternWorkspaces(state); @@ -298,6 +319,10 @@ private static PatternWorkspaceState GetOrCreatePatternWorkspace(string workspac } state = CreatePatternWorkspace(workspaceRoot); + var previousSnapshot = GetFallbackPatternSnapshot(includeUserConfiguration: true); + if (!AcceptedFoldProducerSnapshotsEqual(previousSnapshot, state.GetSnapshot())) + AdvanceAcceptedFoldProducerMutationGeneration(); + state.Active = true; PatternWorkspaces.Add(state); TouchPatternWorkspace(state); evicted = TrimPatternWorkspaces(state); @@ -373,6 +398,13 @@ private static void TouchPatternWorkspace(PatternWorkspaceState state) .Where(state => !ReferenceEquals(state, retainedState)) .OrderBy(state => state.LastAccessSequence) .First(); + if (!AcceptedFoldProducerSnapshotsEqual( + evicted.GetSnapshot(), + GetFallbackPatternSnapshot(evicted.IncludeUserConfiguration))) + { + AdvanceAcceptedFoldProducerMutationGeneration(); + } + evicted.Active = false; PatternWorkspaces.Remove(evicted); return evicted; } @@ -392,6 +424,16 @@ internal static void ReleaseWorkspaceSnapshots() .Concat(PendingPatternWorkspaces) .Distinct() .ToArray(); + if (workspaces.Any(workspace => + workspace.Active + && !AcceptedFoldProducerSnapshotsEqual( + workspace.GetSnapshot(), + GetFallbackPatternSnapshot(workspace.IncludeUserConfiguration)))) + { + AdvanceAcceptedFoldProducerMutationGeneration(); + } + foreach (var workspace in workspaces) + workspace.Active = false; PatternWorkspaces.Clear(); PendingPatternWorkspaces.Clear(); workspaceAccessSequence = 0; @@ -414,6 +456,43 @@ private static void PublishUserExtractorSnapshot() new ReadOnlyDictionary(new Dictionary(ReferenceExtractors, StringComparer.Ordinal)))); } + private static ExtractorWorkspaceSnapshot GetFallbackPatternSnapshot( + bool includeUserConfiguration) + => includeUserConfiguration + ? DefaultPatternWorkspace.GetSnapshot() + : ExtractorWorkspaceSnapshot.Empty; + + private static bool AcceptedFoldProducerSnapshotsEqual( + ExtractorWorkspaceSnapshot left, + ExtractorWorkspaceSnapshot right) + => left.ConfigCount == right.ConfigCount + && left.PluginAssemblyCount == right.PluginAssemblyCount + && ExtractorMapsEqual(left.SymbolExtractors, right.SymbolExtractors) + && ExtractorMapsEqual(left.ReferenceExtractors, right.ReferenceExtractors); + + private static bool ExtractorMapsEqual( + IReadOnlyDictionary left, + IReadOnlyDictionary right) + where TExtractor : class + { + if (left.Count != right.Count) + return false; + + foreach (var (language, extractor) in left) + { + if (!right.TryGetValue(language, out var other) + || !ReferenceEquals(extractor, other)) + { + return false; + } + } + + return true; + } + + private static long AdvanceAcceptedFoldProducerMutationGeneration() + => Interlocked.Increment(ref acceptedFoldProducerMutationGeneration); + private static void SetLanguageExtensions( Dictionary target, IEnumerable<(string Language, IReadOnlyCollection FileExtensions)> extractors) diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PluginLoading.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PluginLoading.cs index f9ce5cb19..f7ddb04b3 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PluginLoading.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.PluginLoading.cs @@ -176,6 +176,7 @@ private static void TryLoadPlugin(string pluginPath, PatternWorkspaceState? work worker, fullPath, workspaceState: null); + AdvanceAcceptedFoldProducerMutationGeneration(); pluginAssemblyCount++; LoadedPluginWorkers.Add(worker); LoadedPluginStagingHandles.Add(staging!); diff --git a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs index 831258af7..2d022c44e 100644 --- a/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs +++ b/src/CodeIndex/Indexer/Extensibility/ExtractorPluginRegistry.cs @@ -52,6 +52,10 @@ public static partial class ExtractorPluginRegistry private static bool suppressDefaultPluginDiscoveryForTesting; private static readonly AsyncLocal AuthorizedConfigurationScope = new(); + internal readonly record struct FoldProducerReadinessSnapshot( + bool UsesOnlyBuiltInProducers, + long MutationGeneration); + internal static IDisposable BeginAuthorizedConfigurationScope() { var previous = AuthorizedConfigurationScope.Value; @@ -196,6 +200,10 @@ public static void Register(ISymbolExtractor extractor) var language = NormalizePluginLanguage(extractor.Language); lock (Gate) { + var changed = !SymbolExtractors.TryGetValue(language, out var previous) + || !ReferenceEquals(previous, extractor); + if (changed) + AdvanceAcceptedFoldProducerMutationGeneration(); SymbolExtractors[language] = extractor; PublishUserExtractorSnapshot(); } @@ -209,6 +217,10 @@ public static void Register(IReferenceExtractor extractor) var language = NormalizePluginLanguage(extractor.Language); lock (Gate) { + var changed = !ReferenceExtractors.TryGetValue(language, out var previous) + || !ReferenceEquals(previous, extractor); + if (changed) + AdvanceAcceptedFoldProducerMutationGeneration(); ReferenceExtractors[language] = extractor; PublishUserExtractorSnapshot(); } @@ -220,6 +232,12 @@ internal static void ResetForTests() { lock (Gate) { + if (SymbolExtractors.Count > 0 + || ReferenceExtractors.Count > 0 + || pluginAssemblyCount > 0) + { + AdvanceAcceptedFoldProducerMutationGeneration(); + } SymbolExtractors.Clear(); ReferenceExtractors.Clear(); PublishUserExtractorSnapshot(); @@ -250,6 +268,12 @@ internal static void ReloadForTests() { lock (Gate) { + if (SymbolExtractors.Count > 0 + || ReferenceExtractors.Count > 0 + || pluginAssemblyCount > 0) + { + AdvanceAcceptedFoldProducerMutationGeneration(); + } SymbolExtractors.Clear(); ReferenceExtractors.Clear(); PublishUserExtractorSnapshot(); @@ -386,6 +410,41 @@ internal static void LoadPatternConfigsForProjectRoot(string? projectRoot) LoadPatternConfigsForProjectRoot(state, fullRoot); } + /// + /// Captures a stable view of both current fold-row producer ownership and its monotonic + /// mutation history. Diagnostic-only publications do not advance the generation. + /// 現在の fold-row producer ownership と単調増加する変更履歴を同じ安定 snapshot で返す。 + /// diagnostic-only publication では generation を進めない。 + /// + internal static FoldProducerReadinessSnapshot CaptureFoldProducerReadinessSnapshot( + string? workspaceRoot) + { + EnsurePluginsLoaded(); + while (true) + { + var generationBefore = Volatile.Read(ref acceptedFoldProducerMutationGeneration); + var patternSnapshot = GetPatternSnapshot(workspaceRoot); + int globalPluginAssemblyCount; + lock (Gate) + globalPluginAssemblyCount = pluginAssemblyCount; + var generationAfter = Volatile.Read(ref acceptedFoldProducerMutationGeneration); + if (generationBefore != generationAfter) + continue; + + return new FoldProducerReadinessSnapshot( + UsesOnlyBuiltInProducers: + globalPluginAssemblyCount == 0 + && patternSnapshot.PluginAssemblyCount == 0 + && patternSnapshot.ConfigCount == 0 + && patternSnapshot.SymbolExtractors.Count == 0 + && patternSnapshot.ReferenceExtractors.Count == 0, + MutationGeneration: generationAfter); + } + } + + internal static bool UsesOnlyBuiltInFoldProducers(string? workspaceRoot) + => CaptureFoldProducerReadinessSnapshot(workspaceRoot).UsesOnlyBuiltInProducers; + internal static void ReloadPatternConfigsForProjectRoot( string? projectRoot, Func? openFile = null, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index dab9b3630..4c2a7bb80 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -327,6 +327,13 @@ static string FormatDiagnosticPath(string projectRoot, string path) // Load current reference-language support before the deferred mutation phase. // deferred mutation phase の前に現在の reference-language support を読み込む。 ExtractorPluginRegistry.LoadPatternConfigsForProjectRoot(projectPath); + var freshFoldProducerSnapshot = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectPath); + var authoritativeFreshFoldRowsClaim = startedWithNoIndexedFilesBeforeRebuild + && !rebuild + && freshFoldProducerSnapshot.UsesOnlyBuiltInProducers + ? writer.TryClaimAuthoritativeFreshFoldRows(requestToken) + : null; var csharpPrepassSymbolArtifacts = CSharpPrepassSymbolArtifactCache .CreateForFreshBuiltInExtraction( startedWithNoIndexedFilesBeforeRebuild && !rebuild); @@ -2113,13 +2120,23 @@ await EmitProgressNotificationAsync( var canRestampExistingFoldTrust = foldVersionMatchesCurrent && foldFingerprintMatchesCurrent; if (skipped == 0 || canRestampExistingFoldTrust) { + var currentFoldProducerSnapshot = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectPath); + if (!currentFoldProducerSnapshot.UsesOnlyBuiltInProducers + || currentFoldProducerSnapshot.MutationGeneration + != freshFoldProducerSnapshot.MutationGeneration + || postExtractionHooks.ValueIfCreated?.HasHooks == true) + { + authoritativeFreshFoldRowsClaim?.Invalidate(); + } // The stamp transaction performs the only row verification for the common // current-metadata path and reports whether NULL or stale values blocked it. // current metadata 経路の row 検証は stamp transaction 内の一度だけにまとめ、 // NULL と stale value のどちらが妨げたかも保持する。 var foldStampResult = writer.MarkFoldReadyWithResult( stampCurrentSymbolExtractorVersions: skipped == 0, - symbolExtractorLanguagesToStamp: skipped == 0 ? indexedSymbolExtractorLanguages : null); + symbolExtractorLanguagesToStamp: skipped == 0 ? indexedSymbolExtractorLanguages : null, + authoritativeFreshRowsClaim: authoritativeFreshFoldRowsClaim); foldReadyAfter = foldStampResult == FoldReadyStampResult.Ready; if (foldStampResult == FoldReadyStampResult.MissingBackfill) foldReadyReason = DegradationReasonCodes.MissingFoldBackfill; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 355d2c114..4a713ed3a 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -10596,6 +10596,222 @@ public void MarkFoldReady_StampsFoldReadyWhenAllRowsBackfilled() Assert.Equal(DbContext.FoldReadyFlag, _db.GetUserVersion() & DbContext.FoldReadyFlag); } + [Fact] + public void MarkFoldReady_AuthoritativeFreshClaimSkipsValueVerificationOnceForOwnerWrites() + { + var claim = _writer.TryClaimAuthoritativeFreshFoldRows(); + Assert.NotNull(claim); + + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/fresh.py", + Lang = "python", + Size = 30, + Lines = 3, + Modified = new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc), + }); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "Straße", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + + var nullChecks = 0; + var valueChecks = 0; + try + { + DbWriter.FoldBackfillVerificationForTesting = () => nullChecks++; + DbWriter.FoldValueVerificationForTesting = () => valueChecks++; + + Assert.Equal( + FoldReadyStampResult.Ready, + _writer.MarkFoldReadyWithResult(authoritativeFreshRowsClaim: claim)); + Assert.Equal(1, nullChecks); + Assert.Equal(0, valueChecks); + + Assert.Equal( + FoldReadyStampResult.Ready, + _writer.MarkFoldReadyWithResult(authoritativeFreshRowsClaim: claim)); + Assert.Equal(2, nullChecks); + Assert.Equal(1, valueChecks); + Assert.Null(_writer.TryClaimAuthoritativeFreshFoldRows()); + } + finally + { + DbWriter.FoldBackfillVerificationForTesting = null; + DbWriter.FoldValueVerificationForTesting = null; + } + } + + [Fact] + public void MarkFoldReady_AuthoritativeFreshClaimStillRejectsNullFoldValues() + { + var claim = _writer.TryClaimAuthoritativeFreshFoldRows(); + Assert.NotNull(claim); + + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/fresh_null.py", + Lang = "python", + Size = 30, + Lines = 3, + Modified = new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc), + }); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "Straße", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + using (var command = _db.Connection.CreateCommand()) + { + command.CommandText = "UPDATE symbols SET name_folded = NULL"; + Assert.Equal(1, command.ExecuteNonQuery()); + } + + var nullChecks = 0; + var valueChecks = 0; + try + { + DbWriter.FoldBackfillVerificationForTesting = () => nullChecks++; + DbWriter.FoldValueVerificationForTesting = () => valueChecks++; + + Assert.Equal( + FoldReadyStampResult.MissingBackfill, + _writer.MarkFoldReadyWithResult(authoritativeFreshRowsClaim: claim)); + Assert.Equal(1, nullChecks); + Assert.Equal(0, valueChecks); + Assert.Equal(0, _db.GetUserVersion() & DbContext.FoldReadyFlag); + } + finally + { + DbWriter.FoldBackfillVerificationForTesting = null; + DbWriter.FoldValueVerificationForTesting = null; + } + } + + [Fact] + public void MarkFoldReady_AuthoritativeFreshClaimFailsClosedAfterExternalCommit() + { + var claim = _writer.TryClaimAuthoritativeFreshFoldRows(); + Assert.NotNull(claim); + + using (var externalDb = new DbContext(DbOpenIntent.WriteIndex, _dbPath)) + { + externalDb.InitializeSchema(); + var externalWriter = new DbWriter(externalDb.Connection); + var fileId = externalWriter.UpsertFile(new FileRecord + { + Path = "src/external.py", + Lang = "python", + Size = 30, + Lines = 3, + Modified = new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc), + }); + externalWriter.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "Straße", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + using var corrupt = externalDb.Connection.CreateCommand(); + corrupt.CommandText = "UPDATE symbols SET name_folded = 'not-current'"; + Assert.Equal(1, corrupt.ExecuteNonQuery()); + } + + var valueChecks = 0; + try + { + DbWriter.FoldValueVerificationForTesting = () => valueChecks++; + + Assert.Equal( + FoldReadyStampResult.NonCurrentFoldValues, + _writer.MarkFoldReadyWithResult(authoritativeFreshRowsClaim: claim)); + Assert.Equal(1, valueChecks); + Assert.Equal(0, _db.GetUserVersion() & DbContext.FoldReadyFlag); + } + finally + { + DbWriter.FoldValueVerificationForTesting = null; + } + } + + [Fact] + public void MarkFoldReady_AuthoritativeFreshClaimFailsClosedForDifferentWriter() + { + var claim = _writer.TryClaimAuthoritativeFreshFoldRows(); + Assert.NotNull(claim); + + var fileId = _writer.UpsertFile(new FileRecord + { + Path = "src/wrong_owner.py", + Lang = "python", + Size = 30, + Lines = 3, + Modified = new DateTime(2025, 6, 1, 0, 0, 0, DateTimeKind.Utc), + }); + _writer.InsertSymbols([ + new SymbolRecord + { + FileId = fileId, + Kind = "function", + Name = "Straße", + Line = 1, + StartLine = 1, + EndLine = 1, + }, + ]); + using (var corrupt = _db.Connection.CreateCommand()) + { + corrupt.CommandText = "UPDATE symbols SET name_folded = 'not-current'"; + Assert.Equal(1, corrupt.ExecuteNonQuery()); + } + + var otherWriter = new DbWriter(_db.Connection); + var valueChecks = 0; + try + { + DbWriter.FoldValueVerificationForTesting = () => valueChecks++; + + Assert.Equal( + FoldReadyStampResult.NonCurrentFoldValues, + otherWriter.MarkFoldReadyWithResult(authoritativeFreshRowsClaim: claim)); + Assert.Equal(1, valueChecks); + Assert.Equal(0, _db.GetUserVersion() & DbContext.FoldReadyFlag); + } + finally + { + DbWriter.FoldValueVerificationForTesting = null; + } + } + + [Fact] + public void TryClaimAuthoritativeFreshFoldRows_PreCancelledRequestLeavesWriterUsable() + { + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + Assert.Throws(() => + _writer.TryClaimAuthoritativeFreshFoldRows(cancellation.Token)); + Assert.NotNull(_writer.TryClaimAuthoritativeFreshFoldRows()); + } + [Fact] public void MarkFoldReady_LeavesFoldReadyUnsetWhenNullFoldedRowExists() { diff --git a/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs index e3e15b2f0..0a389daba 100644 --- a/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs +++ b/tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs @@ -1090,6 +1090,7 @@ public void LoadPatternConfigsForProjectRoot_UsesExplicitRootInsteadOfCurrentDir Assert.Equal("projectdsl", extensions[".projecttoy"]); Assert.False(extensions.ContainsKey(".cwdtoy")); + Assert.False(ExtractorPluginRegistry.UsesOnlyBuiltInFoldProducers(projectRoot)); } finally { @@ -1101,6 +1102,45 @@ public void LoadPatternConfigsForProjectRoot_UsesExplicitRootInsteadOfCurrentDir } } + [Fact] + public void FoldProducerReadinessSnapshot_DiagnosticAndMissingReloadDoNotAdvanceGeneration() + { + var projectRoot = TestProjectHelper.CreateTempProject( + "extractor_registry_fold_generation_diagnostics"); + lock (TestConsoleLock.Gate) + { + try + { + ExtractorPluginRegistry.ResetForTests(); + var initial = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectRoot); + + ExtractorPluginRegistry.ReloadPatternConfigsForProjectRoot( + projectRoot, + directoryExists: static (_, _) => false); + var afterMissing = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectRoot); + Assert.True(afterMissing.UsesOnlyBuiltInProducers); + Assert.Equal(initial.MutationGeneration, afterMissing.MutationGeneration); + + WritePatternConfig( + projectRoot, + "broken.yaml", + "language: \"broken\"\nextensions:\n - extension: \".broken\"\n"); + ExtractorPluginRegistry.ReloadPatternConfigsForProjectRoot(projectRoot); + var afterDiagnostic = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectRoot); + Assert.True(afterDiagnostic.UsesOnlyBuiltInProducers); + Assert.Equal(initial.MutationGeneration, afterDiagnostic.MutationGeneration); + } + finally + { + ExtractorPluginRegistry.ResetForTests(); + TestProjectHelper.DeleteDirectory(projectRoot); + } + } + } + [Fact] public void LoadPatternConfigsForPath_StopsAtWorkspaceRootAndReportsProvenance_Issue4597() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 24114304f..c000cf7a9 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -1955,6 +1955,126 @@ public void Run_FullScanWithoutCSharp_DoesNotRunCSharpPrepass() } } + [Fact] + public void Run_FreshBuiltInSpecialFoldIdentitiesMatchFullValidator() + { + var projectRoot = TestProjectHelper.CreateTempProject("cdidx_fresh_special_fold_identities"); + var foldValueVerifications = 0; + try + { + File.WriteAllText( + Path.Combine(projectRoot, "guide.md"), + "# Café Guide\n\n## Shared Heading\n\n## Shared Heading\n\n[again](#shared-heading-1)\n"); + File.WriteAllText( + Path.Combine(projectRoot, "service.cs"), + "public interface IFoo { void Run(); }\n" + + "public class Service : IFoo { void IFoo.Run() { } }\n"); + File.WriteAllText( + Path.Combine(projectRoot, "style.nim"), + "proc My_Proc*() = discard\nproc call*() = myProc()\n"); + File.WriteAllText( + Path.Combine(projectRoot, "worker.ts"), + "interface Worker { runTask(): void; }\n" + + "class Impl implements Worker { runTask(): void {} }\n"); + File.WriteAllText( + Path.Combine(projectRoot, "unicode.py"), + "def Straße():\n return 1\n"); + DbWriter.FoldValueVerificationForTesting = () => foldValueVerifications++; + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(0, foldValueVerifications); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + var writer = new DbWriter(db.Connection); + Assert.True(writer.AllFoldedColumnValuesMatchCurrentFold()); + Assert.Equal(1, foldValueVerifications); + Assert.Equal( + DbContext.FoldReadyFlag, + db.GetUserVersion() & DbContext.FoldReadyFlag); + } + finally + { + DbWriter.FoldValueVerificationForTesting = null; + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_FreshFoldClaim_TransientCustomProducerHistoryFailsClosed() + { + lock (TestConsoleLock.Gate) + { + var projectRoot = TestProjectHelper.CreateTempProject( + "cdidx_fresh_fold_transient_custom_producer"); + var foldValueVerifications = 0; + var reloaded = 0; + ExtractorPluginRegistry.FoldProducerReadinessSnapshot? customSnapshot = null; + ExtractorPluginRegistry.FoldProducerReadinessSnapshot? restoredSnapshot = null; + try + { + ExtractorPluginRegistry.ResetForTests(); + File.WriteAllText( + Path.Combine(projectRoot, "app.py"), + "def Straße():\n return 1\n"); + var initialSnapshot = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectRoot); + Assert.True(initialSnapshot.UsesOnlyBuiltInProducers); + + DbWriter.FoldValueVerificationForTesting = () => foldValueVerifications++; + IndexCommandRunner.FullScanInputSnapshotBarrierForTesting = phase => + { + if (!string.Equals(phase, "before_readiness", StringComparison.Ordinal) + || Interlocked.Exchange(ref reloaded, 1) != 0) + { + return; + } + + var patternsDirectory = Path.Combine(projectRoot, ".cdidx", "patterns"); + Directory.CreateDirectory(patternsDirectory); + var patternPath = Path.Combine(patternsDirectory, "transient.yaml"); + File.WriteAllText( + patternPath, + "language: \"transientdsl\"\n" + + "extensions:\n - extension: \".transient\"\n" + + "patterns:\n - kind: \"class\"\n" + + " regex: \"^entity (?\\\\w+)\"\n"); + ExtractorPluginRegistry.ReloadPatternConfigsForProjectRoot(projectRoot); + customSnapshot = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectRoot); + + File.Delete(patternPath); + Directory.Delete(patternsDirectory); + ExtractorPluginRegistry.ReloadPatternConfigsForProjectRoot(projectRoot); + restoredSnapshot = + ExtractorPluginRegistry.CaptureFoldProducerReadinessSnapshot(projectRoot); + }; + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(1, reloaded); + Assert.False(customSnapshot!.Value.UsesOnlyBuiltInProducers); + Assert.True(restoredSnapshot!.Value.UsesOnlyBuiltInProducers); + Assert.NotEqual( + initialSnapshot.MutationGeneration, + restoredSnapshot.Value.MutationGeneration); + Assert.Equal(1, foldValueVerifications); + } + finally + { + IndexCommandRunner.FullScanInputSnapshotBarrierForTesting = null; + DbWriter.FoldValueVerificationForTesting = null; + ExtractorPluginRegistry.ResetForTests(); + DeleteDirectory(projectRoot); + } + } + } + [Fact] public void Run_FullScanAfterHeadChange_WithPostExtractionHooksKeepsSequentialReferences() { @@ -1963,6 +2083,8 @@ public void Run_FullScanAfterHeadChange_WithPostExtractionHooksKeepsSequentialRe "cdidx_head_changed_hook_extensions_sequential"); bool? parallelized = null; var originalHooksDir = Environment.GetEnvironmentVariable("CDIDX_HOOKS_DIR"); + var previousFoldValueVerification = DbWriter.FoldValueVerificationForTesting; + var foldValueVerifications = 0; try { var hooksDir = Path.Combine(extensionProject.Root, "hooks"); @@ -1973,6 +2095,11 @@ public void Run_FullScanAfterHeadChange_WithPostExtractionHooksKeepsSequentialRe hookAssemblyPath, Path.Combine(hooksDir, Path.GetFileName(hookAssemblyPath))); Environment.SetEnvironmentVariable("CDIDX_HOOKS_DIR", hooksDir); + DbWriter.FoldValueVerificationForTesting = () => + { + foldValueVerifications++; + previousFoldValueVerification?.Invoke(); + }; RunGit(projectRoot, "init"); File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "public class App { public void Run() { } }\n"); @@ -1985,6 +2112,8 @@ public void Run_FullScanAfterHeadChange_WithPostExtractionHooksKeepsSequentialRe Assert.Equal( initialStatus == "partial" ? CommandExitCodes.PartialResult : CommandExitCodes.Success, initialExitCode); + Assert.Equal(initialStatus == "success" ? 1 : 0, foldValueVerifications); + DbWriter.FoldValueVerificationForTesting = previousFoldValueVerification; File.AppendAllText(Path.Combine(projectRoot, "app.cs"), "public class Next { public void Run() { } }\n"); RunGit(projectRoot, "add", "app.cs"); @@ -2004,6 +2133,7 @@ public void Run_FullScanAfterHeadChange_WithPostExtractionHooksKeepsSequentialRe finally { Environment.SetEnvironmentVariable("CDIDX_HOOKS_DIR", originalHooksDir); + DbWriter.FoldValueVerificationForTesting = previousFoldValueVerification; IndexCommandRunner.FullScanExtractionSchedulingForTesting = null; SqliteConnection.ClearAllPools(); GC.Collect(); diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs index 3b0d7dcc5..8996cfcda 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerTests.cs @@ -2922,6 +2922,7 @@ public void Run_FreshAndRebuildWithoutTypeScript_SkipTypeScriptAugmentationRebui var rebuiltTypeScriptAugmentation = false; var refreshCount = 0; var foldBackfillVerifications = 0; + var foldValueVerifications = 0; var languagePresenceChecks = 0; var indexedLanguageReads = 0; var statReuseLookups = 0; @@ -2939,6 +2940,7 @@ public void Run_FreshAndRebuildWithoutTypeScript_SkipTypeScriptAugmentationRebui previousRefreshHook?.Invoke(); }; DbWriter.FoldBackfillVerificationForTesting = () => foldBackfillVerifications++; + DbWriter.FoldValueVerificationForTesting = () => foldValueVerifications++; DbWriter.LanguagePresenceCheckForTesting = _ => languagePresenceChecks++; DbWriter.IndexedLanguagesReadForTesting = () => indexedLanguageReads++; DbWriter.ReusableUnchangedFileLookupForTesting = _ => reusableLookups++; @@ -2952,6 +2954,7 @@ public void Run_FreshAndRebuildWithoutTypeScript_SkipTypeScriptAugmentationRebui Assert.False(rebuiltTypeScriptAugmentation); Assert.Equal(1, refreshCount); Assert.Equal(1, foldBackfillVerifications); + Assert.Equal(0, foldValueVerifications); Assert.Equal(0, languagePresenceChecks); Assert.Equal(0, indexedLanguageReads); Assert.Equal(0, statReuseLookups); @@ -2960,6 +2963,7 @@ public void Run_FreshAndRebuildWithoutTypeScript_SkipTypeScriptAugmentationRebui Assert.Equal(2, json.GetProperty("summary").GetProperty("files_total").GetInt64()); refreshCount = 0; + foldValueVerifications = 0; var (rebuildExitCode, rebuildJson) = RunAndCaptureJson([ projectRoot, "--db", @@ -2972,6 +2976,7 @@ public void Run_FreshAndRebuildWithoutTypeScript_SkipTypeScriptAugmentationRebui Assert.Equal("success", rebuildJson.GetProperty("status").GetString()); Assert.False(rebuiltTypeScriptAugmentation); Assert.Equal(1, refreshCount); + Assert.Equal(1, foldValueVerifications); using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); Assert.Equal( DbContext.TypeScriptAugmentationVersion.ToString(CultureInfo.InvariantCulture), @@ -2982,6 +2987,7 @@ public void Run_FreshAndRebuildWithoutTypeScript_SkipTypeScriptAugmentationRebui IndexCommandRunner.FullScanTypeScriptAugmentationRebuildForTesting = null; DbWriter.MutualRecursionRefreshForTesting = previousRefreshHook; DbWriter.FoldBackfillVerificationForTesting = null; + DbWriter.FoldValueVerificationForTesting = null; DbWriter.LanguagePresenceCheckForTesting = null; DbWriter.IndexedLanguagesReadForTesting = null; DbWriter.ReusableUnchangedFileLookupForTesting = null; diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index 31d216462..d45e399b6 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -8069,6 +8069,7 @@ public void ToolsCall_Index_FreshAndRebuildWithoutTypeScriptSkipTypeScriptAugmen var rebuiltTypeScriptAugmentation = false; var refreshCount = 0; var foldBackfillVerifications = 0; + var foldValueVerifications = 0; var languagePresenceChecks = 0; var indexedLanguageReads = 0; var statReuseLookups = 0; @@ -8087,6 +8088,7 @@ public void ToolsCall_Index_FreshAndRebuildWithoutTypeScriptSkipTypeScriptAugmen previousRefreshHook?.Invoke(); }; DbWriter.FoldBackfillVerificationForTesting = () => foldBackfillVerifications++; + DbWriter.FoldValueVerificationForTesting = () => foldValueVerifications++; DbWriter.LanguagePresenceCheckForTesting = _ => languagePresenceChecks++; DbWriter.IndexedLanguagesReadForTesting = () => indexedLanguageReads++; DbWriter.ReusableUnchangedFileLookupForTesting = _ => reusableLookups++; @@ -8099,6 +8101,7 @@ public void ToolsCall_Index_FreshAndRebuildWithoutTypeScriptSkipTypeScriptAugmen Assert.False(rebuiltTypeScriptAugmentation); Assert.Equal(1, refreshCount); Assert.Equal(1, foldBackfillVerifications); + Assert.Equal(0, foldValueVerifications); Assert.Equal(0, languagePresenceChecks); Assert.Equal(0, indexedLanguageReads); Assert.Equal(0, statReuseLookups); @@ -8107,12 +8110,14 @@ public void ToolsCall_Index_FreshAndRebuildWithoutTypeScriptSkipTypeScriptAugmen Assert.Equal(2, response["result"]!["structuredContent"]!["summary"]!["files"]!.GetValue()); refreshCount = 0; + foldValueVerifications = 0; var rebuildResponse = CallIndex(server, fixtureDir, args => args["rebuild"] = true); Assert.False( rebuildResponse["result"]?["isError"]?.GetValue() ?? false, rebuildResponse.ToJsonString()); Assert.False(rebuiltTypeScriptAugmentation); Assert.Equal(1, refreshCount); + Assert.Equal(1, foldValueVerifications); using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); db.TryMigrateForRead(); Assert.Equal( @@ -8124,6 +8129,7 @@ public void ToolsCall_Index_FreshAndRebuildWithoutTypeScriptSkipTypeScriptAugmen McpServer.McpIndexTypeScriptAugmentationRebuildForTesting = null; DbWriter.MutualRecursionRefreshForTesting = previousRefreshHook; DbWriter.FoldBackfillVerificationForTesting = null; + DbWriter.FoldValueVerificationForTesting = null; DbWriter.LanguagePresenceCheckForTesting = null; DbWriter.IndexedLanguagesReadForTesting = null; DbWriter.ReusableUnchangedFileLookupForTesting = null; @@ -10103,6 +10109,7 @@ public void ToolsCall_Index_DeletedCsharpStaticInterfaceContractDoesNotRegenerat var previousContentLoadHook = McpServer.McpIndexFileContentLoadForTesting; var previousArtifactHook = CSharpPrepassSymbolArtifactCache.EventForTesting; + var previousFoldValueVerification = DbWriter.FoldValueVerificationForTesting; var artifactEvents = new ConcurrentQueue(); using var extensionProject = TestProjectHelper.CreateExecutableExtensionTestProjectScope( @@ -10113,10 +10120,16 @@ public void ToolsCall_Index_DeletedCsharpStaticInterfaceContractDoesNotRegenerat var matchingLookupBuilds = 0; var noOpCSharpPrepassCount = 0; var noOpContentLoadCount = 0; + var foldValueVerifications = 0; try { CSharpPrepassSymbolArtifactCache.EventForTesting = artifactEvents.Enqueue; + DbWriter.FoldValueVerificationForTesting = () => + { + foldValueVerifications++; + previousFoldValueVerification?.Invoke(); + }; DbWriter.CSharpContractPreflightForTesting = () => { preflightCount++; @@ -10162,6 +10175,8 @@ public interface IParseable using var server = new McpServer(dbPath, ConsoleUi.LoadVersion()); var initialResponse = CallIndex(server, fixtureDir); Assert.False(initialResponse["result"]?["isError"]?.GetValue() ?? false, initialResponse.ToJsonString()); + Assert.Equal(1, foldValueVerifications); + DbWriter.FoldValueVerificationForTesting = previousFoldValueVerification; Assert.Equal( 2, artifactEvents.Count(item => item.Phase == "admitted")); @@ -10298,6 +10313,7 @@ long CountCSharpEvidenceWrites() McpServer.McpIndexFileContentLoadForTesting = previousContentLoadHook; CSharpPrepassSymbolArtifactCache.EventForTesting = previousArtifactHook; + DbWriter.FoldValueVerificationForTesting = previousFoldValueVerification; GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); From 7b68c798127cf10f9f30bf566b94ec3993929e8e Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 17:34:06 +0900 Subject: [PATCH 13/16] Optimize persisted kind validation --- DEVELOPER_GUIDE.md | 13 ++++ TESTING_GUIDE.md | 6 ++ .../+large-codebase-initial-indexing.fixed.md | 4 ++ src/CodeIndex/Models/SymbolKindCatalog.cs | 23 +++++-- tests/CodeIndex.Tests/DatabaseTests.cs | 64 +++++++++++++++++++ .../CodeIndex.Tests/SymbolKindCatalogTests.cs | 56 ++++++++++++++++ 6 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 tests/CodeIndex.Tests/SymbolKindCatalogTests.cs diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 2ae20aae4..68fef7450 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1059,6 +1059,13 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc `symbols.kind`, `symbols.container_kind`, and `symbol_references.container_kind` use the public symbol kind taxonomy below. New extractors must register new kind values in `SymbolKindCatalog` before writing them so schema checks, writer validation, CLI filters, and downstream JSON consumers stay aligned. +The ordered `SymbolKinds` and `ReferenceKinds` arrays are process-static schema and +enumeration contracts; treat their elements as immutable after type initialization. +`IsValidSymbolKind` and `IsValidReferenceKind` use immutable Ordinal lookup sets so +per-row writer validation stays constant-time across every indexed language. Add new +values in the catalog source and update the exhaustive catalog and schema-parity tests; +do not mutate the public arrays at runtime. + | Kind | Current producers / meaning | Graph behavior | |---|---|---| | `accessor` | Accessor declarations when extracted separately from their owning property | Search/filter symbol | @@ -4782,6 +4789,12 @@ regression には、scope rule の focused correctness test と、ユーザー 書き込み前に `SymbolKindCatalog` へ登録し、schema check、writer validation、CLI filter、downstream JSON consumer が同じ値を理解できるようにしてください。 +順序付きの `SymbolKinds` / `ReferenceKinds` array は process-static な schema・列挙契約であり、 +型初期化後は要素を immutable として扱います。`IsValidSymbolKind` と +`IsValidReferenceKind` は immutable な Ordinal lookup set を使い、全インデックス対象言語の +行ごとの writer validation を定数時間に保ちます。値を追加する場合は catalog source を変更し、 +catalog 全件と schema parity の test も更新してください。公開 array を実行時に変更してはいけません。 + | Kind | 現在の producer / 意味 | Graph behavior | |---|---|---| | `accessor` | owning property から別 symbol として抽出される accessor declaration | Search/filter symbol | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index fd6177cf8..a35fa67ca 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -181,6 +181,9 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result Normalized-content facts coverage compares normalization, all derived facts, chunk payloads, and validation issue order against an independent fixed-seed oracle. Keep exact UTF-16 line, unicode61 rune, normalized UTF-8 conflict-budget, replacement-line, trailing-newline, and 80/10 chunk boundaries explicit. The blocking `net8.0` allocation checks use 100,000-line inputs to prevent per-line boundary arrays from returning and require high-ratio invalid UTF-8 loads to discard replacement-line details while preserving aggregate/fallback issue parity. - `PathCompatibilityMatrixTests.cs` Cross-platform path compatibility matrix coverage for path casing, boundary-prefix comparisons, private-child case probes (including numeric root basenames), filesystem-aware exact/prefix filename language detection, Windows long-path prefixing, POSIX sensitive-file permissions, symlink/dangling-entry scan behavior, submodule passthrough under default skip directories, and git skip-worktree path normalization. Keep new platform/path fixture scenarios here when the same assumption needs to be visible across indexing, Git helper, DB/query, installer, or status surfaces. +- `SymbolKindCatalogTests.cs` + Cross-language taxonomy coverage requires every declared symbol and reference kind to be unique and accepted by the exact Ordinal lookup, while null, empty, whitespace-only, case variants, trailing-space variants, and unknown values remain rejected. Keep the writer's unknown symbol/reference and container-kind diagnostics, schema/catalog parity, and pattern-sidecar invalid-kind rejection in the same focused validation set. + Do not put a timing assertion in this test class. When a static lookup replay is useful, keep its Release harness temporary, feed the same persisted kind/count distribution to the legacy and candidate lookups, alternate execution order, and remove it before committing. The normal empty-database full-index A/B remains authoritative for the user-visible performance decision. - `DatabaseTests.cs`, `DatabasePermissionPolicyTests.cs`, `DbReader*Tests.cs` SQLite schema, write paths, migrations, and query behavior. DbReader coverage is split by query family, including search, SQL qualified-name handling, file dependencies, impact, and symbol-query suites, while shared seeded fixture state remains on the root `DbReaderTests` part. Maintenance-lookup coverage keeps the `files(checksum)`, `files(path COLLATE NOCASE)`, and `file_issues(file_id, kind)` indexes aligned with their predicates and requires `EXPLAIN QUERY PLAN` index `SEARCH` operations for checksum purge, ASCII case-alias lookup, reusable-stat issue probes, and directory/stem rename candidates. The case-alias fixture uses changed content so checksum cannot hide a missing path lookup, and managed validation remains authoritative because SQLite `NOCASE` is only an ASCII prefilter, not a Unicode casing contract. Keep wildcard-bearing, extensionless, and near-match stem semantics in one fixture. Scoped-cleanup planning coverage unions checksum and exact same-directory/stem candidates into ascending deduplicated ID snapshots, merges overlapping plans, proves apply does not absorb matching rows added after planning, and rereads planned IDs immediately before apply so a deleted C# contract that reappears is deferred until a clean retry. The grouped C# fixture gives many targets one common checksum/stem and requires candidate-reader work to grow with unique keys plus returned rows rather than target-count squared; it also asserts that C# pre-workspace planning never deletes matching non-C# rows. @@ -1215,6 +1218,9 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" normalized-content facts の coverage は、正規化、全 derived facts、chunk payload、validation issue 順を独立した固定 seed oracle と比較します。UTF-16 line、unicode61 rune、正規化後 UTF-8 の conflict budget、replacement line、末尾改行、80/10 chunk の正確な境界を明示的に維持してください。blocking な `net8.0` allocation check は100,000行 input で行単位の境界 array が戻ることを防ぎ、高比率 invalid UTF-8 load が aggregate / fallback issue parity を保ったまま replacement-line detail を破棄することを必須とします。 - `PathCompatibilityMatrixTests.cs` path casing、boundary-prefix 比較、数字だけの root basename を含む private-child case probe、filesystem-aware な完全一致/prefix ファイル名言語判定、Windows long-path prefix、POSIX の sensitive file 権限、symlink / dangling entry の scan 挙動、既定 skip directory 配下の submodule passthrough、git skip-worktree path 正規化を横断する compatibility matrix カバレッジです。同じ platform/path 前提を indexing、Git helper、DB/query、installer、status の各 surface で見える形にしたい場合は、新しい fixture シナリオをここに追加してください。 +- `SymbolKindCatalogTests.cs` + 全言語共通の taxonomy coverage では、宣言済みの全 symbol / reference kind が重複せず、完全一致の Ordinal lookup で受理されることを必須とします。null、空文字、空白のみ、case 違い、末尾空白、未知の値は引き続き拒否してください。writer の未知 symbol/reference kind と container kind の診断、schema/catalog parity、pattern sidecar の invalid-kind 拒否も同じ focused validation set で維持します。 + この test class に timing assertion を追加してはいけません。static lookup replay が有用な場合は Release harness を一時的なものに限定し、同一の保存済み kind/count 分布を旧来経路と候補経路へ流し、実行順を交互にしたうえで commit 前に削除します。ユーザーに見える性能の採否は、通常の空 database full-index A/B を authoritative としてください。 - `DatabaseTests.cs`、`DatabasePermissionPolicyTests.cs`、`DbReader*Tests.cs` SQLite スキーマ、書き込み経路、マイグレーション、クエリ挙動のテスト。DbReader のカバレッジは search、SQL qualified name、file dependency、impact、symbol query などの query family ごとの partial suite に分割し、共有の seed 済み fixture 状態は root 側の `DbReaderTests` に残します。 maintenance lookup の coverage では `files(checksum)`、`files(path COLLATE NOCASE)`、`file_issues(file_id, kind)` の index を predicate と同期させ、checksum purge、ASCII case-alias lookup、再利用 stat の issue probe、directory/stem rename 候補が `EXPLAIN QUERY PLAN` で index `SEARCH` を使うことを必須とします。case-alias fixture は checksum で path lookup の欠落が隠れないよう content も変更し、SQLite `NOCASE` は Unicode casing contract ではなく ASCII prefilter にすぎないため managed 検証を authoritative に保ちます。wildcard を含む stem、拡張子なし、近似 stem の意味論は1つの fixture にまとめます。scoped cleanup planning の coverage では checksum と正確な同一 directory/stem の候補を昇順・重複排除済み ID snapshot に統合し、重複 plan の merge、plan 後に追加された一致 row を apply が取り込まないこと、apply 直前に planned ID を再読込して再出現した C# contract を clean retry まで延期することを固定します。grouped C# fixture は多数の target に共通 checksum/stem を与え、candidate reader の処理量が target 数の二乗ではなく unique key 数と返却 row 数に比例すること、および C# workspace 前の plan が一致する non-C# row を削除しないことを要求します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 4c0682805..8c2c3ac0c 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -30,6 +30,7 @@ affected: - src/CodeIndex/Indexer/Symbols/SymbolExtractionWorker.cs - src/CodeIndex/WorkerProtocolJsonValidator.cs - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs + - src/CodeIndex/Models/SymbolKindCatalog.cs - tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs - tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs - tests/CodeIndex.Tests/DatabaseTests.cs @@ -42,6 +43,7 @@ affected: - tests/CodeIndex.Tests/ReferenceSecondaryIndexBulkLoadGuardTests.cs - tests/CodeIndex.Tests/SymbolExtractorRequiredLiteralGateTests.cs - tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs + - tests/CodeIndex.Tests/SymbolKindCatalogTests.cs - DEVELOPER_GUIDE.md - TESTING_GUIDE.md --- @@ -58,6 +60,7 @@ affected: - **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate both at file selection and immediately before each regex call against its exact transformed input. Pattern order and output stay unchanged across C#/Fortran merges, Java/Kotlin annotation stripping, C# wrapped-modifier and incomplete-attribute recovery, C++ same-line members, and CSS reconstructed selector segments; a bare C# static-constructor gate miss still reaches the synthesized `static ...` retry. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. - **Symbol workers consume the existing UTF-8 request frames without decoding them twice** — the parent keeps its single `SerializeToUtf8Bytes` write, while the all-language child path now performs bounded newline framing, validation, and deserialization directly from raw standard-input bytes. CRLF/final-EOF framing, protocol and JSON bounds, cancellation, Unicode behavior, and sanitized invalid-UTF-8/JSON errors remain unchanged; the decoded `TextReader` path stays available for diagnostics. - **Fresh built-in indexes finalize fold readiness without re-folding every stored name** — when ordinary CLI or MCP indexing owns a database proven empty across `files`, `symbols`, and `symbol_references`, an opaque one-use claim guarded by SQLite `data_version` keeps the final SQL NULL-completeness check while avoiding materializing and re-folding every symbol/reference string. A monotonic accepted-producer generation also invalidates the claim when custom plugins or patterns were transiently active and later removed before readiness. This removes row-count-proportional finalization work and hundreds of MiB of managed allocation on large first indexes; rebuilds, updates, legacy or existing indexes, public writer calls, custom plugins, patterns, post-extraction hooks, reused claims, and externally changed databases fail closed to full value validation. +- **All-language persistence validates kind taxonomies through immutable lookups** — symbol, reference, and container-kind validation now uses process-static Ordinal sets instead of rescanning the ordered public taxonomy arrays for every persisted row. Schema generation, public taxonomy enumeration, exact case-sensitive membership, invalid-kind diagnostics, and CLI/MCP behavior remain unchanged. ## 日本語 @@ -71,3 +74,4 @@ affected: - **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、file 選択時と各 regex call の直前に、実際の変換済み input に対して監査済みの2文字以上の literal を Ordinal で判定します。C# / Fortran の結合、Java / Kotlin annotation 除去、C# wrapped-modifier / 不完全 attribute recovery、C++ same-line member、CSS の再構成済み selector segment でも pattern 順と出力を変えず、bare C# static constructor の初回 gate miss 後も合成した `static ...` を再試行します。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 - **symbol worker が既存の UTF-8 request frame を二重 decode せず処理するようにしました** — parent 側の `SerializeToUtf8Bytes` による1回の書き込みは変えず、全言語共通の child 経路で標準入力の raw byte から上限付き newline framing、validation、deserialize を直接行います。CRLF / final EOF の framing、protocol / JSON 上限、cancellation、Unicode の挙動、不正 UTF-8 / JSON の sanitization 済み error は従来どおりで、decoded `TextReader` 経路も診断用に維持します。 - **新規 built-in index の fold readiness 確定で、保存済みの全名前を再 fold しないようにしました** — 通常の CLI / MCP indexing が `files`、`symbols`、`symbol_references` のすべてが空であると証明された database を所有する場合、SQLite `data_version` で保護された opaque で一回限りの claim により、最後の SQL NULL completeness check を維持しながら、全 symbol / reference string の materialize と再 fold を省きます。単調増加する accepted-producer generation により、custom plugin / pattern が一時的に active になり readiness 前に削除された場合も claim を無効化します。巨大な初回 index で row 数に比例する finalization work と数百 MiB の managed allocation を取り除きます。rebuild、update、legacy または既存 index、public writer 呼び出し、custom plugin / pattern、post-extraction hook、再利用 claim、外部変更された database は fail closed で full value validation に戻ります。 +- **全言語の永続化で kind taxonomy を immutable lookup により検証するようにしました** — symbol、reference、container kind の検証は、永続化する各行で順序付き公開 taxonomy array を再走査せず、process-static な Ordinal set を使います。schema 生成、公開 taxonomy の列挙、case-sensitive な完全一致、invalid-kind 診断、CLI / MCP の挙動は変わりません。 diff --git a/src/CodeIndex/Models/SymbolKindCatalog.cs b/src/CodeIndex/Models/SymbolKindCatalog.cs index f7db928f8..4fc201bb6 100644 --- a/src/CodeIndex/Models/SymbolKindCatalog.cs +++ b/src/CodeIndex/Models/SymbolKindCatalog.cs @@ -1,3 +1,5 @@ +using System.Collections.Frozen; + namespace CodeIndex.Models; /// @@ -135,16 +137,25 @@ public static class SymbolKindCatalog "use", ]; + // The schema, extractors, and writer share one process-static taxonomy. Keep the public + // arrays for ordered enumeration and schema generation, but validate hot persistence rows + // through immutable ordinal lookups instead of scanning the arrays for every symbol and + // reference. Taxonomy tables are immutable after type initialization by contract. + // schema・extractor・writer は process-static な taxonomy を共有する。順序付き列挙と + // schema 生成には公開 array を維持し、hot な永続化行の検証は行ごとの array 走査ではなく + // immutable な ordinal lookup を使う。taxonomy table は型初期化後 immutable という契約である。 + private static readonly FrozenSet ValidSymbolKinds = + SymbolKinds.ToFrozenSet(StringComparer.Ordinal); + + private static readonly FrozenSet ValidReferenceKinds = + ReferenceKinds.ToFrozenSet(StringComparer.Ordinal); + public static bool IsValidSymbolKind(string? kind) - => Contains(SymbolKinds, kind); + => kind != null && ValidSymbolKinds.Contains(kind); public static bool IsValidReferenceKind(string? kind) - => Contains(ReferenceKinds, kind); + => kind != null && ValidReferenceKinds.Contains(kind); public static string ToSqlCheckInList(IEnumerable values) => string.Join(", ", values.Select(value => $"'{value.Replace("'", "''", StringComparison.Ordinal)}'")); - - private static bool Contains(IEnumerable values, string? value) - => !string.IsNullOrWhiteSpace(value) - && values.Contains(value, StringComparer.Ordinal); } diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 4a713ed3a..5e6cae855 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -2963,6 +2963,70 @@ public void InsertSymbols_UnknownKind_ThrowsBeforePersisting() Assert.Contains("Unknown symbol kind", ex.Message); } + [Fact] + public void InsertSymbols_UnknownContainerKind_ThrowsBeforePersisting() + { + var ex = Assert.Throws(() => _writer.InsertSymbols( + [ + new SymbolRecord + { + FileId = 1, + Kind = "class", + Name = "Run", + Line = 1, + ContainerKind = "metohd", + }, + ])); + + Assert.Equal("symbol", ex.ParamName); + Assert.Contains("Unknown symbol container kind 'metohd'", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void InsertReferences_UnknownKind_ThrowsBeforePersisting() + { + var fileId = UpsertTestFile("src/unknown-reference-kind.cs", "unknown-reference-kind"); + var ex = Assert.Throws(() => _writer.InsertReferences( + [ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "Run", + ReferenceKind = "cal", + Line = 1, + Column = 1, + Context = "Run();", + }, + ], refreshMutualRecursionFlags: false)); + + Assert.Equal("reference", ex.ParamName); + Assert.Contains("Unknown reference kind 'cal'", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, _writer.GetCounts().references); + } + + [Fact] + public void InsertReferences_UnknownContainerKind_ThrowsBeforePersisting() + { + var fileId = UpsertTestFile("src/unknown-reference-container.cs", "unknown-reference-container"); + var ex = Assert.Throws(() => _writer.InsertReferences( + [ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "Run", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "Run();", + ContainerKind = "metohd", + }, + ], refreshMutualRecursionFlags: false)); + + Assert.Equal("reference", ex.ParamName); + Assert.Contains("Unknown reference container kind 'metohd'", ex.Message, StringComparison.Ordinal); + Assert.Equal(0, _writer.GetCounts().references); + } + [Fact] public void InsertChunks_CancelledBeforeBatch_ThrowsOperationCanceled_Issue3738() { diff --git a/tests/CodeIndex.Tests/SymbolKindCatalogTests.cs b/tests/CodeIndex.Tests/SymbolKindCatalogTests.cs new file mode 100644 index 000000000..f122f8de3 --- /dev/null +++ b/tests/CodeIndex.Tests/SymbolKindCatalogTests.cs @@ -0,0 +1,56 @@ +using CodeIndex.Models; + +namespace CodeIndex.Tests; + +public class SymbolKindCatalogTests +{ + [Fact] + public void Validation_AllDeclaredKindsUseExactSharedTaxonomy() + { + Assert.Equal( + SymbolKindCatalog.SymbolKinds.Length, + SymbolKindCatalog.SymbolKinds.Distinct(StringComparer.Ordinal).Count()); + Assert.Equal( + SymbolKindCatalog.ReferenceKinds.Length, + SymbolKindCatalog.ReferenceKinds.Distinct(StringComparer.Ordinal).Count()); + + Assert.All( + SymbolKindCatalog.SymbolKinds, + kind => Assert.True(SymbolKindCatalog.IsValidSymbolKind(kind), kind)); + Assert.All( + SymbolKindCatalog.ReferenceKinds, + kind => Assert.True(SymbolKindCatalog.IsValidReferenceKind(kind), kind)); + } + + [Fact] + public void Validation_NullEmptyWhitespaceCaseAndUnknownRemainRejected() + { + string?[] invalidSymbolKinds = + [ + null, + string.Empty, + " ", + "\t\r\n", + "Class", + "class ", + "unknown_symbol_kind", + ]; + string?[] invalidReferenceKinds = + [ + null, + string.Empty, + " ", + "\t\r\n", + "Call", + "call ", + "unknown_reference_kind", + ]; + + Assert.All( + invalidSymbolKinds, + kind => Assert.False(SymbolKindCatalog.IsValidSymbolKind(kind), kind)); + Assert.All( + invalidReferenceKinds, + kind => Assert.False(SymbolKindCatalog.IsValidReferenceKind(kind), kind)); + } +} From fcb1429df43eb7f10ed1a9daa9deb3675bfb56ac Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 20:34:55 +0900 Subject: [PATCH 14/16] Serialize in-process symbol worker protocol tests --- TESTING_GUIDE.md | 4 +- ...SymbolExtractionWorkerUtf8ProtocolTests.cs | 74 ++++++++++++------- 2 files changed, 49 insertions(+), 29 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index a35fa67ca..959944c0e 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -94,7 +94,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result Recovery-command coverage keeps resolved execution arguments separate from support-safe display arguments. Assert structured argv, current `dotnet`/apphost prefix preservation, replay of option-like paths under CLI `--show-paths`, default CLI/MCP redaction metadata, and correct quoting for both POSIX sh and PowerShell. Include paths with spaces, quotes, dollar signs, shell metacharacters, POSIX home/temp roots, Windows drives, UNC roots, option-like source names such as `--db`, and file-URI database query parameters containing raw/encoded paths, percent-encoded sensitive keys, or path values with embedded sensitive assignments. Default-output assertions must reject the fixture's full absolute paths and secrets while preserving safe URI controls. Pair this with `status --config` coverage for default DB/data/log path and URI-query redaction, always-redacted secrets, and explicit `--show-paths`. Console writer synchronization coverage yields between character writes instead of sleeping per character; use enough whole-line iterations to expose interleaving without adding wall-clock delay. - `SymbolExtractionWorkerUtf8ProtocolTests.cs` - The deterministic, cross-target symbol-worker protocol suite owns the production raw-UTF-8 stdin boundary shared by every language. Keep Unicode multi-frame input, CRLF, an unterminated final frame and stable EOF together; separately pin byte and JSON payload/property/depth/string bounds, invalid-UTF-8 and malformed-JSON sanitization without secret reflection, BOM-free output, and cancellation of a pending stream read. Keep the legacy `TextReader` tests as diagnostic-path coverage, and use temporary benchmarks only for adoption decisions—remove them before committing. + The deterministic, cross-target symbol-worker protocol suite owns the production raw-UTF-8 stdin boundary shared by every language. Keep Unicode multi-frame input, CRLF, an unterminated final frame and stable EOF together; separately pin byte and JSON payload/property/depth/string bounds, invalid-UTF-8 and malformed-JSON sanitization without secret reflection, BOM-free output, and cancellation of a pending stream read. In-process worker startup resets process-static pattern-discovery state, so every `TryRunCommand` call in this suite must hold `TestConsoleLock.Gate`, shared with the worker pattern-cache tests. Keep the legacy `TextReader` tests as diagnostic-path coverage, and use temporary benchmarks only for adoption decisions—remove them before committing. - `WorkspaceCommandRunnerTests.cs` Workspace status coverage keeps missing manifests, empty and malformed manifests, missing project directories, all-missing databases, mixed healthy/degraded members, and shared-database layouts independently observable. Assert the compatibility `exists` alias beside unambiguous `project_exists` and `db_exists` fields, structured repair command names and argv (including paths with spaces), human labels, aggregate reasons/actions, and the stable `--check` exit policy: ready `0`, missing `2`, degraded `5`, and invalid input `1`. - `SymbolExtractor*Tests.cs` and `ReferenceExtractor*Tests.cs` @@ -1126,7 +1126,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" recovery command の coverage では、解決済みの実行引数とサポート共有向けの表示引数を分離して検証します。構造化 argv、現在の `dotnet` / apphost prefix の維持、CLI `--show-paths` による option と紛らわしい path の再実行、既定の CLI/MCP redaction metadata、POSIX sh と PowerShell 双方の正しい quoting を確認してください。空白、quote、dollar sign、shell metacharacter、POSIX の home/temp root、Windows drive、UNC root、`--db` のように option と紛らわしい source 名、raw / encoded path、percent-encoded な機密 key、機密 assignment を内包する path 値を持つ file-URI database query parameter を含めます。既定出力に fixture の完全な絶対パスや secret が残らず、安全な URI control は維持されることを assertion にします。`status --config` の DB/data/log path と URI query の既定 redaction、mode に関係なく維持される secret redaction、明示的 `--show-paths` も対で検証してください。 console writer synchronization coverageは文字writeごとのsleepではなくyieldを使い、wall-clock delayを追加せずinterleavingを露出できる十分なwhole-line iterationを維持してください。 - `SymbolExtractionWorkerUtf8ProtocolTests.cs` - 全言語で共有する本番 symbol worker の raw UTF-8 stdin 境界は、この deterministic な cross-target protocol suite で検証します。Unicode の複数 frame、CRLF、終端改行のない最終 frame、安定した EOF を1つの fixture に保ち、byte および JSON payload / property / depth / string の各上限、不正 UTF-8 と malformed JSON が secret を反射しない sanitization、BOM のない出力、pending stream read の cancellation をそれぞれ固定してください。従来の `TextReader` test は診断経路の coverage として残し、採用判断用の一時 benchmark は commit 前に削除します。 + 全言語で共有する本番 symbol worker の raw UTF-8 stdin 境界は、この deterministic な cross-target protocol suite で検証します。Unicode の複数 frame、CRLF、終端改行のない最終 frame、安定した EOF を1つの fixture に保ち、byte および JSON payload / property / depth / string の各上限、不正 UTF-8 と malformed JSON が secret を反射しない sanitization、BOM のない出力、pending stream read の cancellation をそれぞれ固定してください。in-process worker の起動は process-static な pattern discovery state を reset するため、この suite の全 `TryRunCommand` 呼び出しは worker pattern-cache test と共有する `TestConsoleLock.Gate` を保持してください。従来の `TextReader` test は診断経路の coverage として残し、採用判断用の一時 benchmark は commit 前に削除します。 - `WorkspaceCommandRunnerTests.cs` workspace status の coverage では、manifest 不在、空 / malformed manifest、project directory 不在、全 database 不在、healthy / degraded member の混在、shared-database layout をそれぞれ独立して観測可能にします。曖昧さのない `project_exists` / `db_exists` と互換用 `exists` alias、構造化された修復 command 名と argv（空白を含む path を含む）、human-readable label、集約 reason / action、ならびに ready `0`、missing `2`、degraded `5`、invalid input `1` の安定した `--check` exit policy を検証してください。 - `SymbolExtractor*Tests.cs` と `ReferenceExtractor*Tests.cs` diff --git a/tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs b/tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs index 3caa9735c..d4b7cd77b 100644 --- a/tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs +++ b/tests/CodeIndex.Tests/SymbolExtractionWorkerUtf8ProtocolTests.cs @@ -34,12 +34,17 @@ public void RawUtf8Input_PreservesUnicodeCrLfAndFinalEofFrames() using var output = new MemoryStream(); using var error = new StringWriter(); - var handled = SymbolExtractionWorker.TryRunCommand( - [SymbolExtractionWorker.CommandName], - input, - output, - error, - out var exitCode); + bool handled; + int exitCode; + lock (TestConsoleLock.Gate) + { + handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out exitCode); + } Assert.True(handled); Assert.Equal(CommandExitCodes.Success, exitCode); @@ -76,12 +81,17 @@ public void RawUtf8Input_InvalidUtf8AndMalformedJsonDoNotEchoPayload() using var output = new MemoryStream(); using var error = new StringWriter(); - var handled = SymbolExtractionWorker.TryRunCommand( - [SymbolExtractionWorker.CommandName], - input, - output, - error, - out var exitCode); + bool handled; + int exitCode; + lock (TestConsoleLock.Gate) + { + handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out exitCode); + } Assert.True(handled); Assert.Equal(CommandExitCodes.Success, exitCode); @@ -107,14 +117,19 @@ public void RawUtf8Input_EnforcesByteFrameLimit() using var output = new MemoryStream(); using var error = new StringWriter(); - var handled = SymbolExtractionWorker.TryRunCommand( - [SymbolExtractionWorker.CommandName], - input, - output, - error, - out var exitCode, - maxProtocolLineCharacters: 5, - maxProtocolLineUtf8Bytes: 5); + bool handled; + int exitCode; + lock (TestConsoleLock.Gate) + { + handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out exitCode, + maxProtocolLineCharacters: 5, + maxProtocolLineUtf8Bytes: 5); + } Assert.True(handled); Assert.Equal(1, exitCode); @@ -174,13 +189,18 @@ public void RawUtf8Input_CancellationInterruptsPendingRead() using var output = new MemoryStream(); using var error = new StringWriter(); - var handled = SymbolExtractionWorker.TryRunCommand( - [SymbolExtractionWorker.CommandName], - input, - output, - error, - out var exitCode, - cancellationToken: cts.Token); + bool handled; + int exitCode; + lock (TestConsoleLock.Gate) + { + handled = SymbolExtractionWorker.TryRunCommand( + [SymbolExtractionWorker.CommandName], + input, + output, + error, + out exitCode, + cancellationToken: cts.Token); + } Assert.True(handled); Assert.True(input.ReadStarted); From 343a6db682b7891323adbedf02a405ffefc877b8 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 20:49:00 +0900 Subject: [PATCH 15/16] Keep persisted kind taxonomy canonical --- DEVELOPER_GUIDE.md | 25 +++++----- TESTING_GUIDE.md | 2 + .../+large-codebase-initial-indexing.fixed.md | 7 ++- .../Cli/ExportImportCommandRunner.Ctags.cs | 4 +- .../DbContext.SchemaInitialization.cs | 26 +++++----- src/CodeIndex/Models/SymbolKindCatalog.cs | 42 +++++++++++----- .../DbSchemaConstraintTests.cs | 48 +++++++++++++++++++ 7 files changed, 115 insertions(+), 39 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 68fef7450..0221a777a 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1059,12 +1059,14 @@ Do not add mutable static caches, shared `StringBuilder` instances, reused `Matc `symbols.kind`, `symbols.container_kind`, and `symbol_references.container_kind` use the public symbol kind taxonomy below. New extractors must register new kind values in `SymbolKindCatalog` before writing them so schema checks, writer validation, CLI filters, and downstream JSON consumers stay aligned. -The ordered `SymbolKinds` and `ReferenceKinds` arrays are process-static schema and -enumeration contracts; treat their elements as immutable after type initialization. -`IsValidSymbolKind` and `IsValidReferenceKind` use immutable Ordinal lookup sets so -per-row writer validation stays constant-time across every indexed language. Add new -values in the catalog source and update the exhaustive catalog and schema-parity tests; -do not mutate the public arrays at runtime. +The ordered `SymbolKinds` and `ReferenceKinds` arrays remain public compatibility +snapshots; callers must treat their elements as immutable after type initialization. +A private canonical ordered taxonomy is the sole source for immutable Ordinal writer +lookups, SQLite schema checks and migrations, and ctags filters. This keeps those +internal contracts aligned even if legacy consumer code accidentally replaces an +element in a public array. Add new values in the catalog source and update the +exhaustive catalog, schema-parity, and public-mutation isolation tests; do not mutate +the public arrays at runtime. | Kind | Current producers / meaning | Graph behavior | |---|---|---| @@ -4789,11 +4791,12 @@ regression には、scope rule の focused correctness test と、ユーザー 書き込み前に `SymbolKindCatalog` へ登録し、schema check、writer validation、CLI filter、downstream JSON consumer が同じ値を理解できるようにしてください。 -順序付きの `SymbolKinds` / `ReferenceKinds` array は process-static な schema・列挙契約であり、 -型初期化後は要素を immutable として扱います。`IsValidSymbolKind` と -`IsValidReferenceKind` は immutable な Ordinal lookup set を使い、全インデックス対象言語の -行ごとの writer validation を定数時間に保ちます。値を追加する場合は catalog source を変更し、 -catalog 全件と schema parity の test も更新してください。公開 array を実行時に変更してはいけません。 +順序付きの `SymbolKinds` / `ReferenceKinds` array は公開互換 snapshot として維持し、caller は +型初期化後の要素を immutable として扱います。private な canonical 順序付き taxonomy だけを、 +immutable な Ordinal writer lookup、SQLite schema check / migration、ctags filter の source にします。 +そのため legacy consumer が公開 array の要素を誤って置換しても、内部契約は同期したままです。 +値を追加する場合は catalog source を変更し、catalog 全件、schema parity、公開 mutation 隔離の +test も更新してください。公開 array を実行時に変更してはいけません。 | Kind | 現在の producer / 意味 | Graph behavior | |---|---|---| diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 959944c0e..e30702805 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -183,6 +183,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result Cross-platform path compatibility matrix coverage for path casing, boundary-prefix comparisons, private-child case probes (including numeric root basenames), filesystem-aware exact/prefix filename language detection, Windows long-path prefixing, POSIX sensitive-file permissions, symlink/dangling-entry scan behavior, submodule passthrough under default skip directories, and git skip-worktree path normalization. Keep new platform/path fixture scenarios here when the same assumption needs to be visible across indexing, Git helper, DB/query, installer, or status surfaces. - `SymbolKindCatalogTests.cs` Cross-language taxonomy coverage requires every declared symbol and reference kind to be unique and accepted by the exact Ordinal lookup, while null, empty, whitespace-only, case variants, trailing-space variants, and unknown values remain rejected. Keep the writer's unknown symbol/reference and container-kind diagnostics, schema/catalog parity, and pattern-sidecar invalid-kind rejection in the same focused validation set. + `DbSchemaConstraintTests` runs the public-array mutation regression in the non-parallel SQLite-sensitive collection and restores both arrays in `finally`. It must prove that writer validation and newly generated symbol/reference CHECK clauses continue to use the same private canonical taxonomy while the compatibility arrays are visibly mutated. Do not put a timing assertion in this test class. When a static lookup replay is useful, keep its Release harness temporary, feed the same persisted kind/count distribution to the legacy and candidate lookups, alternate execution order, and remove it before committing. The normal empty-database full-index A/B remains authoritative for the user-visible performance decision. - `DatabaseTests.cs`, `DatabasePermissionPolicyTests.cs`, `DbReader*Tests.cs` SQLite schema, write paths, migrations, and query behavior. DbReader coverage is split by query family, including search, SQL qualified-name handling, file dependencies, impact, and symbol-query suites, while shared seeded fixture state remains on the root `DbReaderTests` part. @@ -1220,6 +1221,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" path casing、boundary-prefix 比較、数字だけの root basename を含む private-child case probe、filesystem-aware な完全一致/prefix ファイル名言語判定、Windows long-path prefix、POSIX の sensitive file 権限、symlink / dangling entry の scan 挙動、既定 skip directory 配下の submodule passthrough、git skip-worktree path 正規化を横断する compatibility matrix カバレッジです。同じ platform/path 前提を indexing、Git helper、DB/query、installer、status の各 surface で見える形にしたい場合は、新しい fixture シナリオをここに追加してください。 - `SymbolKindCatalogTests.cs` 全言語共通の taxonomy coverage では、宣言済みの全 symbol / reference kind が重複せず、完全一致の Ordinal lookup で受理されることを必須とします。null、空文字、空白のみ、case 違い、末尾空白、未知の値は引き続き拒否してください。writer の未知 symbol/reference kind と container kind の診断、schema/catalog parity、pattern sidecar の invalid-kind 拒否も同じ focused validation set で維持します。 + `DbSchemaConstraintTests` の公開 array mutation 回帰は、並列実行しない SQLite-sensitive collection で実行し、`finally` で両 array を復元します。互換 array の変更が実際に見える間も、writer validation と新規生成する symbol / reference CHECK 句が同じ private canonical taxonomy を使い続けることを必須とします。 この test class に timing assertion を追加してはいけません。static lookup replay が有用な場合は Release harness を一時的なものに限定し、同一の保存済み kind/count 分布を旧来経路と候補経路へ流し、実行順を交互にしたうえで commit 前に削除します。ユーザーに見える性能の採否は、通常の空 database full-index A/B を authoritative としてください。 - `DatabaseTests.cs`、`DatabasePermissionPolicyTests.cs`、`DbReader*Tests.cs` SQLite スキーマ、書き込み経路、マイグレーション、クエリ挙動のテスト。DbReader のカバレッジは search、SQL qualified name、file dependency、impact、symbol query などの query family ごとの partial suite に分割し、共有の seed 済み fixture 状態は root 側の `DbReaderTests` に残します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 8c2c3ac0c..46271e8d8 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -2,6 +2,7 @@ category: fixed affected: - src/CodeIndex/Cli/ProgramRunner.cs + - src/CodeIndex/Cli/ExportImportCommandRunner.Ctags.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.ExtractionPipeline.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Readiness.cs @@ -12,6 +13,7 @@ affected: - src/CodeIndex/Database/DbWriter.References.cs - src/CodeIndex/Database/DbWriter.ReferenceSql.cs - src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs + - src/CodeIndex/Database/DbContext.SchemaInitialization.cs - src/CodeIndex/Database/ReferenceSecondaryIndexBulkLoadGuard.cs - src/CodeIndex/Indexer/CSharpStaticInterfacePrepass.cs - src/CodeIndex/Indexer/CSharpPrepassSymbolArtifactCache.cs @@ -34,6 +36,7 @@ affected: - tests/CodeIndex.Tests/ExtractorPluginRegistryTests.cs - tests/CodeIndex.Tests/CSharpPrepassSymbolArtifactCacheTests.cs - tests/CodeIndex.Tests/DatabaseTests.cs + - tests/CodeIndex.Tests/DbSchemaConstraintTests.cs - tests/CodeIndex.Tests/FileIndexerTests.cs - tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -60,7 +63,7 @@ affected: - **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate both at file selection and immediately before each regex call against its exact transformed input. Pattern order and output stay unchanged across C#/Fortran merges, Java/Kotlin annotation stripping, C# wrapped-modifier and incomplete-attribute recovery, C++ same-line members, and CSS reconstructed selector segments; a bare C# static-constructor gate miss still reaches the synthesized `static ...` retry. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. - **Symbol workers consume the existing UTF-8 request frames without decoding them twice** — the parent keeps its single `SerializeToUtf8Bytes` write, while the all-language child path now performs bounded newline framing, validation, and deserialization directly from raw standard-input bytes. CRLF/final-EOF framing, protocol and JSON bounds, cancellation, Unicode behavior, and sanitized invalid-UTF-8/JSON errors remain unchanged; the decoded `TextReader` path stays available for diagnostics. - **Fresh built-in indexes finalize fold readiness without re-folding every stored name** — when ordinary CLI or MCP indexing owns a database proven empty across `files`, `symbols`, and `symbol_references`, an opaque one-use claim guarded by SQLite `data_version` keeps the final SQL NULL-completeness check while avoiding materializing and re-folding every symbol/reference string. A monotonic accepted-producer generation also invalidates the claim when custom plugins or patterns were transiently active and later removed before readiness. This removes row-count-proportional finalization work and hundreds of MiB of managed allocation on large first indexes; rebuilds, updates, legacy or existing indexes, public writer calls, custom plugins, patterns, post-extraction hooks, reused claims, and externally changed databases fail closed to full value validation. -- **All-language persistence validates kind taxonomies through immutable lookups** — symbol, reference, and container-kind validation now uses process-static Ordinal sets instead of rescanning the ordered public taxonomy arrays for every persisted row. Schema generation, public taxonomy enumeration, exact case-sensitive membership, invalid-kind diagnostics, and CLI/MCP behavior remain unchanged. +- **All-language persistence validates kind taxonomies through immutable lookups** — symbol, reference, and container-kind validation now uses process-static Ordinal sets instead of rescanning the ordered public taxonomy arrays for every persisted row. A private canonical ordered taxonomy also drives schema checks, migrations, and ctags filters, so accidental mutation of a public compatibility-array element cannot split validation from persisted constraints. Public taxonomy enumeration, exact case-sensitive membership, invalid-kind diagnostics, and CLI/MCP behavior remain unchanged. ## 日本語 @@ -74,4 +77,4 @@ affected: - **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、file 選択時と各 regex call の直前に、実際の変換済み input に対して監査済みの2文字以上の literal を Ordinal で判定します。C# / Fortran の結合、Java / Kotlin annotation 除去、C# wrapped-modifier / 不完全 attribute recovery、C++ same-line member、CSS の再構成済み selector segment でも pattern 順と出力を変えず、bare C# static constructor の初回 gate miss 後も合成した `static ...` を再試行します。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 - **symbol worker が既存の UTF-8 request frame を二重 decode せず処理するようにしました** — parent 側の `SerializeToUtf8Bytes` による1回の書き込みは変えず、全言語共通の child 経路で標準入力の raw byte から上限付き newline framing、validation、deserialize を直接行います。CRLF / final EOF の framing、protocol / JSON 上限、cancellation、Unicode の挙動、不正 UTF-8 / JSON の sanitization 済み error は従来どおりで、decoded `TextReader` 経路も診断用に維持します。 - **新規 built-in index の fold readiness 確定で、保存済みの全名前を再 fold しないようにしました** — 通常の CLI / MCP indexing が `files`、`symbols`、`symbol_references` のすべてが空であると証明された database を所有する場合、SQLite `data_version` で保護された opaque で一回限りの claim により、最後の SQL NULL completeness check を維持しながら、全 symbol / reference string の materialize と再 fold を省きます。単調増加する accepted-producer generation により、custom plugin / pattern が一時的に active になり readiness 前に削除された場合も claim を無効化します。巨大な初回 index で row 数に比例する finalization work と数百 MiB の managed allocation を取り除きます。rebuild、update、legacy または既存 index、public writer 呼び出し、custom plugin / pattern、post-extraction hook、再利用 claim、外部変更された database は fail closed で full value validation に戻ります。 -- **全言語の永続化で kind taxonomy を immutable lookup により検証するようにしました** — symbol、reference、container kind の検証は、永続化する各行で順序付き公開 taxonomy array を再走査せず、process-static な Ordinal set を使います。schema 生成、公開 taxonomy の列挙、case-sensitive な完全一致、invalid-kind 診断、CLI / MCP の挙動は変わりません。 +- **全言語の永続化で kind taxonomy を immutable lookup により検証するようにしました** — symbol、reference、container kind の検証は、永続化する各行で順序付き公開 taxonomy array を再走査せず、process-static な Ordinal set を使います。private な canonical 順序付き taxonomy を schema check、migration、ctags filter にも使うため、公開互換 array の要素が誤って変更されても validation と永続 constraint は分裂しません。公開 taxonomy の列挙、case-sensitive な完全一致、invalid-kind 診断、CLI / MCP の挙動は変わりません。 diff --git a/src/CodeIndex/Cli/ExportImportCommandRunner.Ctags.cs b/src/CodeIndex/Cli/ExportImportCommandRunner.Ctags.cs index ecf07c3e2..079935b02 100644 --- a/src/CodeIndex/Cli/ExportImportCommandRunner.Ctags.cs +++ b/src/CodeIndex/Cli/ExportImportCommandRunner.Ctags.cs @@ -33,7 +33,7 @@ WHERE s.name IS NOT NULL AND trim(s.name) != '' AND s.kind IS NOT NULL AND trim(s.kind) != '' - AND s.kind IN ({SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds)}) + AND s.kind IN ({SymbolKindCatalog.PersistedSymbolKindSqlCheckInList}) """; AppendCtagsFilters(ref sql, filters); sql += " ORDER BY s.name COLLATE NOCASE, f.path, COALESCE(s.start_line, s.line, 1)"; @@ -48,7 +48,7 @@ private static SqliteCommand CreateCtagsSkipReasonCommand(SqliteConnection conne var skipReasonCases = new List { $"WHEN s.name IS NULL OR trim(s.name) = '' THEN '{CtagsSkipInvalidName}'", - $"WHEN s.kind IS NULL OR trim(s.kind) = '' OR s.kind NOT IN ({SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds)}) THEN '{CtagsSkipUnsupportedKind}'", + $"WHEN s.kind IS NULL OR trim(s.kind) = '' OR s.kind NOT IN ({SymbolKindCatalog.PersistedSymbolKindSqlCheckInList}) THEN '{CtagsSkipUnsupportedKind}'", }; if (filters.GeneratedFileFilterAvailable && !filters.IncludeGenerated) skipReasonCases.Add($"WHEN COALESCE(f.generated, 0) != 0 THEN '{CtagsSkipGeneratedCode}'"); diff --git a/src/CodeIndex/Database/DbContext.SchemaInitialization.cs b/src/CodeIndex/Database/DbContext.SchemaInitialization.cs index 3ce9490f2..8f52fb934 100644 --- a/src/CodeIndex/Database/DbContext.SchemaInitialization.cs +++ b/src/CodeIndex/Database/DbContext.SchemaInitialization.cs @@ -112,8 +112,8 @@ file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE, UNIQUE(file_id, line, context) )"); - var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); - var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); + var symbolKindCheck = SymbolKindCatalog.PersistedSymbolKindSqlCheckInList; + var referenceKindCheck = SymbolKindCatalog.PersistedReferenceKindSqlCheckInList; // Symbols table / シンボルテーブル Execute(@" @@ -372,7 +372,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS fts_chunks USING fts5( private void EnforceRequiredFileIdConstraints() { - var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); + var symbolKindCheck = SymbolKindCatalog.PersistedSymbolKindSqlCheckInList; var legacyAlterTable = ExecuteScalar("PRAGMA legacy_alter_table"); RunWithForeignKeysDisabledForMigration( "EnforceRequiredFileIdConstraints", @@ -461,8 +461,8 @@ private void RebuildReferenceLineTablesWithRequiredFileId() return; } - var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); - var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); + var symbolKindCheck = SymbolKindCatalog.PersistedSymbolKindSqlCheckInList; + var referenceKindCheck = SymbolKindCatalog.PersistedReferenceKindSqlCheckInList; const string referenceLinesCreateSql = """ CREATE TABLE reference_lines ( @@ -525,8 +525,8 @@ private void EnforceReferenceLineSetNullConstraint() if (SymbolReferencesReferenceLineDeletesSetNull()) return; - var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); - var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); + var symbolKindCheck = SymbolKindCatalog.PersistedSymbolKindSqlCheckInList; + var referenceKindCheck = SymbolKindCatalog.PersistedReferenceKindSqlCheckInList; var symbolReferencesCreateSql = $""" CREATE TABLE symbol_references ( @@ -601,8 +601,8 @@ private void EnsureReferenceLinesContextKey() if (ReferenceLinesHasContextUniqueKey()) return; - var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); - var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); + var symbolKindCheck = SymbolKindCatalog.PersistedSymbolKindSqlCheckInList; + var referenceKindCheck = SymbolKindCatalog.PersistedReferenceKindSqlCheckInList; const string referenceLinesCreateSql = """ CREATE TABLE reference_lines ( @@ -696,8 +696,8 @@ private bool ReferenceLinesHasContextUniqueKey() private void EnsureKindCheckConstraintsCurrent() { - var symbolKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.SymbolKinds); - var referenceKindCheck = SymbolKindCatalog.ToSqlCheckInList(SymbolKindCatalog.ReferenceKinds); + var symbolKindCheck = SymbolKindCatalog.PersistedSymbolKindSqlCheckInList; + var referenceKindCheck = SymbolKindCatalog.PersistedReferenceKindSqlCheckInList; var symbolsCreateSql = $""" CREATE TABLE symbols ( @@ -761,13 +761,13 @@ resolution_candidate_count INTEGER NOT NULL DEFAULT 0 var rebuilt = false; RunWithForeignKeysDisabledForMigration("EnsureKindCheckConstraintsCurrent", () => { - if (!TableCheckContainsAll("symbols", SymbolKindCatalog.SymbolKinds)) + if (!TableCheckContainsAll("symbols", SymbolKindCatalog.PersistedSymbolKinds)) { RebuildTableWithCurrentKindChecks("symbols", "_symbols_kind_check", symbolsCreateSql, symbolsColumns); rebuilt = true; } - if (!TableCheckContainsAll("symbol_references", SymbolKindCatalog.SymbolKinds.Concat(SymbolKindCatalog.ReferenceKinds))) + if (!TableCheckContainsAll("symbol_references", SymbolKindCatalog.PersistedSymbolKinds.Concat(SymbolKindCatalog.PersistedReferenceKinds))) { RebuildTableWithCurrentKindChecks("symbol_references", "_symbol_references_kind_check", symbolReferencesCreateSql, symbolReferencesColumns); rebuilt = true; diff --git a/src/CodeIndex/Models/SymbolKindCatalog.cs b/src/CodeIndex/Models/SymbolKindCatalog.cs index 4fc201bb6..15769d7bc 100644 --- a/src/CodeIndex/Models/SymbolKindCatalog.cs +++ b/src/CodeIndex/Models/SymbolKindCatalog.cs @@ -8,7 +8,7 @@ namespace CodeIndex.Models; /// public static class SymbolKindCatalog { - public static readonly string[] SymbolKinds = + private static readonly string[] CanonicalSymbolKinds = [ "accessor", "add", @@ -83,6 +83,13 @@ public static class SymbolKindCatalog "workdir", ]; + // Preserve the public field and ordered array surface for compatibility, but do not use + // this mutable array as an internal source of truth. Callers have historically been able + // to replace individual elements even though the field itself is readonly. + // 公開 field と順序付き array の互換 surface は維持するが、この mutable array を内部の + // source of truth にはしない。field 自体は readonly でも caller は従来から要素を置換できる。 + public static readonly string[] SymbolKinds = [.. CanonicalSymbolKinds]; + /// /// Broad compatibility families for consumers that do not recognize newer semantic kinds. /// 新しい semantic kind を認識しない consumer 向けの広い互換 family。 @@ -95,7 +102,7 @@ public static class SymbolKindCatalog ["typealias"] = "type", }); - public static readonly string[] ReferenceKinds = + private static readonly string[] CanonicalReferenceKinds = [ "annotation", "attribute", @@ -137,18 +144,31 @@ public static class SymbolKindCatalog "use", ]; - // The schema, extractors, and writer share one process-static taxonomy. Keep the public - // arrays for ordered enumeration and schema generation, but validate hot persistence rows - // through immutable ordinal lookups instead of scanning the arrays for every symbol and - // reference. Taxonomy tables are immutable after type initialization by contract. - // schema・extractor・writer は process-static な taxonomy を共有する。順序付き列挙と - // schema 生成には公開 array を維持し、hot な永続化行の検証は行ごとの array 走査ではなく - // immutable な ordinal lookup を使う。taxonomy table は型初期化後 immutable という契約である。 + public static readonly string[] ReferenceKinds = [.. CanonicalReferenceKinds]; + + // The private ordered snapshots are the sole source for persistence validation, schema + // checks/migrations, and ctags filters. Public arrays remain compatibility copies, so an + // accidental element mutation cannot split those internal contracts. + // private な順序付き snapshot だけを persistence validation、schema check / migration、 + // ctags filter の source とする。公開 array は互換用 copy のため、誤った要素変更でも + // これらの内部契約が分裂しない。 + internal static IReadOnlyList PersistedSymbolKinds { get; } = + Array.AsReadOnly(CanonicalSymbolKinds); + + internal static IReadOnlyList PersistedReferenceKinds { get; } = + Array.AsReadOnly(CanonicalReferenceKinds); + + internal static string PersistedSymbolKindSqlCheckInList { get; } = + ToSqlCheckInList(CanonicalSymbolKinds); + + internal static string PersistedReferenceKindSqlCheckInList { get; } = + ToSqlCheckInList(CanonicalReferenceKinds); + private static readonly FrozenSet ValidSymbolKinds = - SymbolKinds.ToFrozenSet(StringComparer.Ordinal); + CanonicalSymbolKinds.ToFrozenSet(StringComparer.Ordinal); private static readonly FrozenSet ValidReferenceKinds = - ReferenceKinds.ToFrozenSet(StringComparer.Ordinal); + CanonicalReferenceKinds.ToFrozenSet(StringComparer.Ordinal); public static bool IsValidSymbolKind(string? kind) => kind != null && ValidSymbolKinds.Contains(kind); diff --git a/tests/CodeIndex.Tests/DbSchemaConstraintTests.cs b/tests/CodeIndex.Tests/DbSchemaConstraintTests.cs index 4069bcd78..b512db2f0 100644 --- a/tests/CodeIndex.Tests/DbSchemaConstraintTests.cs +++ b/tests/CodeIndex.Tests/DbSchemaConstraintTests.cs @@ -5,6 +5,7 @@ namespace CodeIndex.Tests; +[Collection("SQLite pool sensitive")] public class DbSchemaConstraintTests { [Theory] @@ -92,6 +93,53 @@ public void InitializeSchema_KindCheckConstraintsMatchCatalog_Issue4178() } } + [Fact] + public void InitializeSchema_PublicTaxonomyMutationCannotSplitCanonicalContracts() + { + var canonicalSymbolKinds = SymbolKindCatalog.PersistedSymbolKinds.ToArray(); + var canonicalReferenceKinds = SymbolKindCatalog.PersistedReferenceKinds.ToArray(); + var originalPublicSymbolKinds = SymbolKindCatalog.SymbolKinds.ToArray(); + var originalPublicReferenceKinds = SymbolKindCatalog.ReferenceKinds.ToArray(); + const string mutatedSymbolKind = "mutated_public_symbol_kind"; + const string mutatedReferenceKind = "mutated_public_reference_kind"; + var dbDir = TestProjectHelper.CreateTempProject("codeindex_schema_public_taxonomy_mutation"); + var dbPath = Path.Combine(dbDir, "codeindex.db"); + + try + { + SymbolKindCatalog.SymbolKinds[0] = mutatedSymbolKind; + SymbolKindCatalog.ReferenceKinds[0] = mutatedReferenceKind; + + Assert.Equal(mutatedSymbolKind, SymbolKindCatalog.SymbolKinds[0]); + Assert.Equal(mutatedReferenceKind, SymbolKindCatalog.ReferenceKinds[0]); + Assert.True(SymbolKindCatalog.IsValidSymbolKind(canonicalSymbolKinds[0])); + Assert.True(SymbolKindCatalog.IsValidReferenceKind(canonicalReferenceKinds[0])); + Assert.False(SymbolKindCatalog.IsValidSymbolKind(mutatedSymbolKind)); + Assert.False(SymbolKindCatalog.IsValidReferenceKind(mutatedReferenceKind)); + + using (var db = new DbContext(DbOpenIntent.WriteIndex, dbPath)) + db.InitializeSchema(); + + using var conn = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString); + conn.Open(); + var symbolsSql = ReadCreateSql(conn, "symbols"); + var referencesSql = ReadCreateSql(conn, "symbol_references"); + + AssertSameSet(canonicalSymbolKinds, ExtractCheckValues(symbolsSql, "kind")); + AssertSameSet(canonicalSymbolKinds, ExtractCheckValues(symbolsSql, "container_kind")); + AssertSameSet(canonicalReferenceKinds, ExtractCheckValues(referencesSql, "reference_kind")); + AssertSameSet(canonicalSymbolKinds, ExtractCheckValues(referencesSql, "container_kind")); + Assert.DoesNotContain(mutatedSymbolKind, symbolsSql, StringComparison.Ordinal); + Assert.DoesNotContain(mutatedReferenceKind, referencesSql, StringComparison.Ordinal); + } + finally + { + originalPublicSymbolKinds.CopyTo(SymbolKindCatalog.SymbolKinds, 0); + originalPublicReferenceKinds.CopyTo(SymbolKindCatalog.ReferenceKinds, 0); + TestProjectHelper.DeleteDirectory(dbDir); + } + } + private static void SeedLegacyNullableFileIdSchema(string dbPath) { using var conn = new SqliteConnection(new SqliteConnectionStringBuilder { DataSource = dbPath }.ConnectionString); From 54c717b43d3aad908cc5b9f0e2c0e01423acb986 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Tue, 11 Aug 2026 21:03:58 +0900 Subject: [PATCH 16/16] Harden authoritative fresh indexing state --- DEVELOPER_GUIDE.md | 16 +++ TESTING_GUIDE.md | 4 + .../+large-codebase-initial-indexing.fixed.md | 8 +- .../Cli/IndexCommandRunner.FullScan.cs | 10 ++ .../Database/DbWriter.FreshFoldReadiness.cs | 99 ++++++++++++++++--- .../DbWriter.ReferenceGraphRefreshScope.cs | 3 + tests/CodeIndex.Tests/DatabaseTests.cs | 27 +++++ .../FreshReferenceResolutionTests.cs | 14 +++ .../IndexCommandRunnerFullScanTests.cs | 88 +++++++++++++++++ 9 files changed, 254 insertions(+), 15 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 0221a777a..035292d21 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1575,6 +1575,11 @@ SQLite resolves referenced tables while preparing every statement in a command b A true empty-database ordinary CLI full scan (not `--rebuild` or `--symbols-only`) opts into a separate fresh-resolution contract. Reference inserts persist canonical provisional values (`unresolved`, candidate count zero, and zero self/mutual flags) without adding bind parameters. +The early empty observation is advisory: immediately after the authoritative outer write +transaction begins, the CLI rechecks `files`, `symbols`, and `symbol_references` in that +transaction. If another connection committed any row during the pre-write gap, the graph scope +disables fresh insert defaults before the first persisted row and finalization uses the ordinary +full-resolution SQL, including candidate-free references. Finalization scans `symbol_reference_candidates` once into materialized per-reference facts and updates only candidate-bearing references by primary key; candidate-free references retain their provisional values, and the self flag is derived in that same sparse update. The opt-in remains @@ -1591,6 +1596,10 @@ consume it once, and an intervening commit from another connection invalidates i built-in extractor pipeline completes successfully, finalization may use the claim to omit the allocation-heavy read and re-fold of every persisted symbol/reference value, but it still runs the SQL NULL-completeness check before stamping FoldReady. +The raw `BEGIN IMMEDIATE` helper deliberately performs its post-success cancellation check only +after the caller has recorded rollback ownership. Cancellation at that boundary therefore rolls +back the raw transaction before releasing the writer gate, leaving a warm CLI/MCP connection +usable by the next request. The run also captures the registry's monotonic accepted-producer mutation generation. Any generation change invalidates the claim even if a transient custom producer was later removed and the final registry is built-in-only again; staged workspace replacement commits participate in @@ -5346,6 +5355,10 @@ prepared command で作成してください。SQLite は command batch の全st 真に空のdatabaseから始める通常のCLI full scan（`--rebuild` と `--symbols-only` を除く）だけは、 fresh resolution専用の契約をopt-inします。reference insertはbind parameterを増やさず、 `unresolved`、candidate count 0、self/mutual flag 0というcanonicalな暫定値を永続化します。 +早期のempty確認はadvisoryです。authoritativeなouter write transaction開始直後に、CLIは同じ +transaction内で`files`、`symbols`、`symbol_references`を再確認します。write前のgapで別connectionが +1行でもcommitしていた場合は、最初のrowを永続化する前にgraph scopeのfresh insert defaultを無効化し、 +candidateを持たないreferenceも含めて通常のfull-resolution SQLでfinalizeします。 finalizationは`symbol_reference_candidates`をreferenceごとのmaterialized factsへ1回走査し、 candidateを持つreferenceだけをprimary keyで更新します。candidateを持たないreferenceは暫定値を 維持し、self flagも同じsparse update内で導出します。このopt-inはgraph transaction失敗後も @@ -5363,6 +5376,9 @@ fold readiness には、通常の CLI / MCP full indexing が共有する、よ 場合、finalization は claim を使って、永続化済みの全 symbol / reference value を読み出して 再 fold する allocation-heavy な処理を省けますが、FoldReady を stamp する前の SQL による NULL completeness check は引き続き実行します。 +raw `BEGIN IMMEDIATE` helperは、成功後のcancellation checkを、callerがrollback ownershipを記録した +後にだけ行います。その境界でcancelされてもwriter gateを解放する前にraw transactionをrollbackし、 +warmなCLI / MCP connectionを次のrequestで引き続き利用できます。 run は registry の accepted-producer mutation generation も取得します。一時的な custom producer が後で削除され、最終 registry が再び built-in-only になっていても、generation が変化していれば claim を無効化します。staged workspace replacement の commit もこの履歴に含めますが、 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index e30702805..498d5a986 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -39,6 +39,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Lowering or raising the file-size policy must reprocess unchanged files in CLI and MCP indexing so a prior `file_too_large` issue cannot be reused. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps and must not label an incomplete index as fold-only. Human output must identify incomplete generations instead of printing a complete summary. - Reference-identity refresh coverage treats a stable graph rebuild as a physical-write performance contract. Keep NULL-safe changed-row predicates for source identity, the four-column target-resolution tuple, self-reference, and mutual-recursion updates; trigger audits must remain at zero on a stable rerun, repair each corrupted phase once, and prove a later-phase failure rolls back earlier identity writes. SQLite `changes()` must continue to report the final mutual-recursion phase. - Fresh-reference resolution coverage belongs in `FreshReferenceResolutionTests`. Keep the empty/rebuild/symbols-only policy truth table, the unchanged 14-parameter insert shape with distinct fresh/ordinary SQL cache entries, canonical provisional values, the materialized candidate-side aggregation shape, and exact fresh-versus-full semantic parity for `unresolved`, `resolved`, `resolved_group`, `ambiguous`, and self-reference rows across C# and a non-C# language. Failure tests must prove that fresh defaults remain pending after graph rollback and clear only after a successful commit. Pair these database tests with the full-scan bulk-load theory that observes `unresolved` provisional rows on a fresh CLI scan and NULL resolution state on rebuild. MCP remains outside this opt-in because its per-file durable transactions require the existing graph-failure retry contract. + A full-scan barrier regression must commit a candidate-free row from a second connection after the early empty observation but before the outer write transaction, then prove transaction-local revalidation switches to ordinary full resolution and normalizes that row. - C# reference-graph fact coverage is an SQL-shape performance contract. Each of the six managed arity/receiver/value-type functions must occur once only in its reference or symbol materialization statement, and full, scoped, and retained refresh chains must populate reference, symbol, type-identity, and constructor-identity facts in that order before normalization and candidate consumers. Candidate SQL must use the WITHOUT ROWID fact primary keys without retaining project/file-local identity construction or constructor-owner range scans. Keep full/scoped/retained semantic snapshots across partial generic and file-local constructors, primary constructors with same-leaf generic arities, scoped target-definition mutations, fallback owners, legacy NULL identities, stale-row cleanup, and refresh rollback/retry alongside the focused property-receiver regressions. - C# metadata-target resolver coverage treats propagation work and stable reruns as performance contracts. Keep the reverse-ordered 8,000-class chain at exactly `n - 1` dependency edges and `n` queue visits instead of using a wall-clock threshold; retain cross-file partial fan-in, an unseeded cycle, pre-cancellation, rollback of an earlier row after an injected later update failure, zero trigger-audited writes on a stable rerun, and exactly one write when repairing a corrupted derived row. - Reference-insert transaction coverage keeps the public APIs' #1518 transaction/SAVEPOINT per 71-row batch, while the explicit atomic-file APIs must reject calls without a live caller-owned transaction and open zero reference-batch scopes. Atomic-file reference-line materialization may group only complete 71-row reference batches, stopping before the union of `(file_id, line, context)` keys would exceed 333 rows or after 32 batches; reference INSERT executions and their progress/cancellation checkpoints remain on the original 71-row boundaries. Preserve exact public/atomic statement counts, the 333-row and 32-batch stops, the unique `reference_lines` autoindex lookup plan, batch-two/three failure rollback for both normal and new-file reference-line paths, same/different contexts across a batch boundary, cancellation and empty-input ordering, and guarded multi-language integration coverage for full scan, scoped update, MCP indexing, and TypeScript augmentation rebuild. The controlled 321,352-reference/856-file performance contract retains the repository snapshot's five/six-batch large-file distribution and compares 5,009 public batch scopes with zero atomic-file batch scopes without using a wall-clock threshold. @@ -701,6 +702,7 @@ Candidate-ordered parallel-index recovery tests must prove that the fatal result - `PerformanceTests.cs` Bounded CI smoke coverage plus large-scale data benchmarks. `CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` and the allocation budget guards run in the default `net8.0` suite, so they are blocking PR/CI checks on the production target, but their broad budgets are intended to catch only severe indexing/search or allocation regressions rather than act as benchmarks. `ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` uses dense C# private-property receivers and Python imported-type calls to prevent per-candidate full-symbol rescans from returning. `ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` covers dense C# declaration containers and GitHub Actions jobs so name/range ownership resolution stays indexed. `Extraction_DenseDelimitedLists_StayWithinAllocationBudget` covers Python imports, YAML needs, JSON paths, and Fortran procedure lists without temporary split-array growth. `ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` keeps all-language dedupe identities value-based when qualified names are long. The 10K+ large-scale tests remain skip-by-default; run them manually with `--filter`. Focused authoritative-fresh fold-readiness coverage spans `DatabaseTests`, `IndexCommandRunnerTests`, and `McpServerToolsCallTests`: the built-in empty-database CLI/MCP path must consume its claim once, retain the NULL-column verification, skip the stored-value re-fold scan, and produce the same readiness/version/fingerprint/language stamps and Unicode, Markdown, C#, Nim, and TypeScript query results as full validation. Pair it with fail-closed cases for each initially nonempty ownership table (`files`, `symbols`, or `symbol_references`), a wrong or reused claim, an intervening external commit observed through `PRAGMA data_version`, rebuild/update/legacy/public-writer paths, and custom plugins, patterns, or post-extraction hooks; full validation must still reject NULL and stale non-NULL folds. A run-barrier regression must also activate a custom producer and then reload back to built-in-only before readiness: the current producer count returns to zero, but the monotonic mutation generation changes and forces full validation. Unchanged missing-directory and diagnostic-only publications must not change that generation. + A deterministic cancel-after-`BEGIN IMMEDIATE` test must prove that the raw transaction is rolled back and the same writer can immediately start and commit another transaction. For performance audits, alternate identical repository-scale fresh fixtures, isolate the readiness-finalization interval, and report elapsed time plus `GC.GetAllocatedBytesForCurrentThread`; adoption requires removing row-count-proportional managed allocation without changing rows, stamps, or query results. Keep wall-clock measurements out of blocking CI assertions and remove temporary instrumentation after recording the result. `ReferenceExtraction_MaskedMultilinePayloads_StayWithinAllocationBudget` keeps C# raw strings, Java text blocks, and TypeScript template literals from materializing trimmed reference contexts after structural masking has made a line empty. `CppHeaderDetection_LargeSample_DoesNotMaterializeLineArrays` keeps bounded C / C++ header-disambiguation samples on span-based line walks instead of allocating a string and array for every sampled line. @@ -1072,6 +1074,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。file-size policy を下げた場合も上げた場合も、CLI / MCP indexing は unchanged file を再処理し、以前の `file_too_large` issue を再利用してはいけません。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別し、incomplete index を fold-only と表示しない必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 - reference identity refresh coverage は、安定graphの再構築を物理writeのperformance contractとします。source identity、target resolutionの4列tuple、self reference、mutual recursionの更新にはNULL-safeなchanged-row predicateを維持し、安定rerunのtrigger auditは0、各corrupt phaseのrepairは1回、後段phaseの失敗で先行identity writeもrollbackされることを検証してください。SQLite `changes()` は引き続き最後のmutual-recursion phaseを表します。 - fresh reference resolutionのcoverageは`FreshReferenceResolutionTests`が担当します。empty/rebuild/symbols-onlyのpolicy truth table、fresh/ordinaryでSQL cache entryを分けても14 parameterのinsert shapeが不変であること、canonicalな暫定値、candidate側materialized aggregationのSQL shape、およびC#と非C#言語を横断する`unresolved`、`resolved`、`resolved_group`、`ambiguous`、self-referenceのfresh/full完全同値を維持してください。failure testはgraph rollback後もfresh defaultsがpendingのままで、成功commit後にだけ解除されることを証明します。database testは、fresh CLI scanで暫定`unresolved`、rebuildでNULL resolution stateを観測するfull-scan bulk-load theoryと対にしてください。MCPはfile単位のdurable transactionが既存graph失敗再試行契約を必要とするため、このopt-inの対象外です。 + full-scan barrier回帰では、早期empty確認後からouter write transaction開始前の間に別connectionからcandidate-free rowをcommitし、transaction内の再検証が通常のfull resolutionへ切り替わってそのrowを正規化することを必須とします。 - C# reference-graph fact coverage は SQL shape の performance contract です。arity / receiver / value-type を求める6つの managed function は reference または symbol の materialization statement 内にそれぞれ1回だけ置き、full / scoped / retained refresh chain は reference、symbol、type identity、constructor identity の fact をこの順で normalization と candidate consumer より前に投入してください。candidate SQL は WITHOUT ROWID fact の主キーを使い、project / file-local identity の再構築や constructor-owner range の再走査を残してはいけません。partial generic / file-local constructor、同名別 generic arity を持つ primary constructor、scoped target 定義変更、fallback owner、legacy NULL identity、stale row cleanup、refresh rollback / retry の full / scoped / retained semantic snapshot と、focused property-receiver regression を維持します。 - C# metadata-target resolver coverage は propagation work と安定 rerun を performance contract とします。逆順に保存した 8,000 class の chain では wall-clock threshold を使わず、dependency edge が厳密に `n - 1`、queue visit が `n` であることを維持してください。cross-file partial fan-in、seed を持たない cycle、事前 cancel、後段 update の注入失敗時に先行 row も rollback されること、安定 rerun の trigger audit が write 0 回、破損した derived row の修復が厳密に 1 write であることも残します。 - reference insert の transaction coverage は、public API の #1518 契約として71 row batchごとの transaction/SAVEPOINTを維持し、明示atomic-file APIは呼出元所有のlive transactionなしでは拒否され、reference batch scopeを0回に保つことを検証します。atomic-fileのreference-line materializationは完全な71 row reference batchだけをまとめ、`(file_id, line, context)` keyの和集合が333行を超える直前、または32 batchで停止します。reference INSERTの実行回数とprogress/cancellation checkpointは元の71 row境界に保ってください。public/atomicの正確なstatement数、333行/32 batch停止、`reference_lines` unique autoindexのlookup plan、通常/new-file両方のreference-line pathでbatch 2/3失敗時の全rollback、batch境界をまたぐ同一/異なるcontext、cancelとempty入力の順序、full scan・scoped update・MCP indexing・TypeScript augmentation rebuildのmulti-language guard付きintegrationを維持してください。321,352 refs / 856 filesの制御performance契約は自己snapshotの5/6 batch巨大file分布を保ち、wall-clock閾値を使わずpublic 5,009 scopeとatomic-file 0 scopeを比較します。 @@ -1736,6 +1739,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - `PerformanceTests.cs` bounded な CI smoke と大規模データベンチマークを扱います。`CiPerformanceSmoke_IndexAndSearchSmallFixture_StaysWithinBudget` と allocation budget guard は通常の `net8.0` suite で実行されるため production target 上の PR / CI blocking check ですが、benchmark ではなく重大な indexing/search または allocation 退行だけを拾う広めの budget を使います。`ReferenceExtraction_RepeatedSymbolMembership_StaysWithinAllocationBudget` は密な C# private-property receiver と Python imported-type call を使い、candidate ごとの full-symbol 再走査が戻るのを防ぎます。`ReferenceExtraction_RepeatedContainerLookup_StaysWithinAllocationBudget` は密な C# declaration container と GitHub Actions job を扱い、name / range ownership 解決の索引化を維持します。`Extraction_DenseDelimitedLists_StayWithinAllocationBudget` は Python import、YAML needs、JSON path、Fortran procedure list を使い、一時 split-array の増加を防ぎます。`ReferenceDedupe_DenseLongIdentities_StayWithinAllocationBudget` は長い qualified name でも全言語共通 dedupe identity を value-based に維持します。10K+ の大規模テストは引き続きデフォルト Skip で、`--filter` で手動実行します。 authoritative-fresh fold readiness の focused coverage は `DatabaseTests`、`IndexCommandRunnerTests`、`McpServerToolsCallTests` で分担します。built-in の empty-database CLI / MCP 経路が claim を一度だけ consume し、NULL column の検証を維持しつつ、保存 value の再 fold scan を省き、full validation と同じ readiness / version / fingerprint / language stamp、および Unicode、Markdown、C#、Nim、TypeScript の query result を生成することを固定します。初期状態で ownership table（`files`、`symbols`、`symbol_references`）のいずれかが非空の場合、owner が異なるか再利用された claim、`PRAGMA data_version` で観測される外部 commit、rebuild / update / legacy / public-writer 経路、custom plugin / pattern / post-extraction hook は fail closed であることも対にし、full validation が NULL と stale な非 NULL fold を引き続き拒否することを確認します。run barrier では custom producer を一度 active にしてから readiness 前に built-in-only へ reload し、最終 producer count が zero に戻っていても monotonic mutation generation の変化で full validation へ戻ることを固定します。状態不変の missing-directory と diagnostic-only publication では generation が変わらないことも確認します。 + `BEGIN IMMEDIATE`成功直後のdeterministicなcancel testでは、raw transactionがrollbackされ、同じwriterが直後に別transactionを開始・commitできることを必須とします。 性能監査では、同一の repository-scale fresh fixture を交互に実行し、readiness finalization 区間を分離して、経過時間と `GC.GetAllocatedBytesForCurrentThread` を報告します。row 数に比例する managed allocation を取り除きつつ、row、stamp、query result が変わらないことを採用条件にします。wall-clock 計測は blocking CI assertion にせず、結果を記録したら一時 instrumentation を削除してください。 `ReferenceExtraction_MaskedMultilinePayloads_StayWithinAllocationBudget` は、構造マスク後に空行となった C# raw string、Java text block、TypeScript template literal から trim 済み reference context を実体化しないことを固定します。 `CppHeaderDetection_LargeSample_DoesNotMaterializeLineArrays` は、bounded な C / C++ header 判定 sample を span ベースで行走査し、sampled line ごとの string と array を割り当てないことを固定します。 diff --git a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md index 46271e8d8..c83b436a7 100644 --- a/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md +++ b/changelog.d/unreleased/+large-codebase-initial-indexing.fixed.md @@ -56,13 +56,13 @@ affected: - **Fresh large-codebase indexes finalize mutual-recursion edges without duplicate reverse lookups** — reference graph finalization now materializes each candidate edge's desired recursion flag once, avoiding a costly second set of random B-tree probes when only a small number of flags change. - **Fresh and rebuilt indexes skip unused incremental graph bookkeeping** — once a full reference-graph refresh is known, symbol and reference batches no longer populate dirty-scope tables that the full plan never reads, removing repeated set construction across all indexed languages. - **Initial C# workspace prepasses reuse the loaded extractor configuration** — CLI and MCP indexing no longer rediscover default plugins under a shared lock for every static/enum/const candidate after the workspace pattern snapshot has already been loaded. -- **First-time full indexes resolve only references that have candidates** — empty-database CLI scans persist canonical unresolved defaults during bulk insertion, aggregate the candidate table once, and update candidate-bearing references by primary key instead of probing every reference row. Rebuilds, updates, retained graphs, and MCP indexing keep their existing recovery contracts. +- **First-time full indexes resolve only references that have candidates** — empty-database CLI scans persist canonical unresolved defaults during bulk insertion, aggregate the candidate table once, and update candidate-bearing references by primary key instead of probing every reference row. The CLI revalidates all ownership tables inside its outer write transaction, falling back to ordinary full resolution if another connection committed during the pre-write gap. Rebuilds, updates, retained graphs, and MCP indexing keep their existing recovery contracts. - **Fresh graph planning uses post-load cardinalities** — truly empty CLI and MCP bulk loads now analyze the populated file, symbol, and reference tables immediately before candidate resolution, allowing SQLite to plan the expensive first graph build from current statistics. Rebuilds, updates, existing databases, and symbols-only runs keep their prior lifecycle; cancellation still aborts, while a statistics-only SQLite failure rolls back its savepoint and continues best-effort. - **Persistent symbol workers reuse bounded pattern-directory snapshots across languages** — each project-root reload discovers user and root configs once, while nested ancestor directories, including missing and rejected results, are observed once per worker command. The cache follows live filesystem casing and falls back to uncached discovery when full so configs are never skipped; direct registry callers keep dynamic discovery. - **First-time C# extraction reuses checksum-verified prepass symbols** — empty-database CLI and MCP full indexes can consume bounded, take-once built-in symbols already extracted for the static-interface workspace. After materializing the immutable lookup snapshots, the prepass transfers ownership of admitted per-file symbol lists and releases the redundant workspace fallback objects instead of cloning the full symbol graph. The main pass still rereads and validates every file and falls back on checksum drift, incomplete prepasses, regex timeouts, or cache limits; rebuilds, updates, and symbols-only runs remain unchanged. - **Initial indexes skip regex patterns whose mandatory literals are absent** — built-in case-sensitive symbol patterns now opt into an audited, Ordinal two-or-more-character literal gate both at file selection and immediately before each regex call against its exact transformed input. Pattern order and output stay unchanged across C#/Fortran merges, Java/Kotlin annotation stripping, C# wrapped-modifier and incomplete-attribute recovery, C++ same-line members, and CSS reconstructed selector segments; a bare C# static-constructor gate miss still reaches the synthesized `static ...` retry. IgnoreCase, custom/plugin, one-character, and no-common-literal patterns remain ungated. - **Symbol workers consume the existing UTF-8 request frames without decoding them twice** — the parent keeps its single `SerializeToUtf8Bytes` write, while the all-language child path now performs bounded newline framing, validation, and deserialization directly from raw standard-input bytes. CRLF/final-EOF framing, protocol and JSON bounds, cancellation, Unicode behavior, and sanitized invalid-UTF-8/JSON errors remain unchanged; the decoded `TextReader` path stays available for diagnostics. -- **Fresh built-in indexes finalize fold readiness without re-folding every stored name** — when ordinary CLI or MCP indexing owns a database proven empty across `files`, `symbols`, and `symbol_references`, an opaque one-use claim guarded by SQLite `data_version` keeps the final SQL NULL-completeness check while avoiding materializing and re-folding every symbol/reference string. A monotonic accepted-producer generation also invalidates the claim when custom plugins or patterns were transiently active and later removed before readiness. This removes row-count-proportional finalization work and hundreds of MiB of managed allocation on large first indexes; rebuilds, updates, legacy or existing indexes, public writer calls, custom plugins, patterns, post-extraction hooks, reused claims, and externally changed databases fail closed to full value validation. +- **Fresh built-in indexes finalize fold readiness without re-folding every stored name** — when ordinary CLI or MCP indexing owns a database proven empty across `files`, `symbols`, and `symbol_references`, an opaque one-use claim guarded by SQLite `data_version` keeps the final SQL NULL-completeness check while avoiding materializing and re-folding every symbol/reference string. A monotonic accepted-producer generation also invalidates the claim when custom plugins or patterns were transiently active and later removed before readiness. Cancellation immediately after the claim's `BEGIN IMMEDIATE` now rolls back the raw transaction before releasing the warm writer. This removes row-count-proportional finalization work and hundreds of MiB of managed allocation on large first indexes; rebuilds, updates, legacy or existing indexes, public writer calls, custom plugins, patterns, post-extraction hooks, reused claims, and externally changed databases fail closed to full value validation. - **All-language persistence validates kind taxonomies through immutable lookups** — symbol, reference, and container-kind validation now uses process-static Ordinal sets instead of rescanning the ordered public taxonomy arrays for every persisted row. A private canonical ordered taxonomy also drives schema checks, migrations, and ctags filters, so accidental mutation of a public compatibility-array element cannot split validation from persisted constraints. Public taxonomy enumeration, exact case-sensitive membership, invalid-kind diagnostics, and CLI/MCP behavior remain unchanged. ## 日本語 @@ -70,11 +70,11 @@ affected: - **巨大コードベースの新規インデックスで相互再帰 edge の reverse lookup を重複実行しないようにしました** — reference graph の確定時に各候補 edge の望ましい recursion flag を一度だけ materialize し、変更対象の flag が少数の場合に発生していた高コストな2回目のランダム B-tree probe を避けます。 - **新規作成および rebuild 時に未使用の差分 graph bookkeeping を省くようにしました** — reference graph の full refresh が確定した後は、その plan が参照しない dirty scope table を symbol / reference batch ごとに投入せず、全インデックス対象言語にまたがる反復的な set 構築を取り除きます。 - **初回 C# workspace prepass で読込済み extractor config を再利用するようにしました** — workspace pattern snapshot の読込後に、static / enum / const の候補ごとに共有lock下でdefault pluginを再探索しないよう、CLIとMCP indexingを既読込経路へ接続します。 -- **初回full indexでcandidateを持つreferenceだけを解決するようにしました** — 空databaseからのCLI scanではbulk insert中にcanonicalな未解決値を保存し、candidate tableを1回集約して、全reference rowをprobeせずcandidateを持つrowだけをprimary keyで更新します。rebuild、update、retained graph、MCP indexingの既存recovery契約は変更しません。 +- **初回full indexでcandidateを持つreferenceだけを解決するようにしました** — 空databaseからのCLI scanではbulk insert中にcanonicalな未解決値を保存し、candidate tableを1回集約して、全reference rowをprobeせずcandidateを持つrowだけをprimary keyで更新します。CLIはouter write transaction内で全ownership tableを再検証し、write前のgapで別connectionがcommitしていた場合は通常のfull resolutionへfallbackします。rebuild、update、retained graph、MCP indexingの既存recovery契約は変更しません。 - **新規graph planningでbulk load後のcardinalityを使うようにしました** — 真に空のdatabaseから始まるCLI / MCP bulk loadでは、candidate解決の直前に投入済みのfile、symbol、reference tableを解析し、SQLiteが初回の高コストなgraph構築を最新統計から計画できるようにします。rebuild、update、既存database、symbols-onlyのlifecycleは従来どおりです。cancellationは引き続き中断し、統計更新だけのSQLite failureはsavepointを戻してbest-effortで継続します。 - **persistent symbol worker が全言語で上限付き pattern-directory snapshot を再利用するようにしました** — project-root reload ごとに user / root config を1回だけ探索し、missing や reject を含む nested ancestor directory は worker command ごとに初回結果を再利用します。cache は実 filesystem の case policy に従い、飽和時は uncached discovery に fallback して config を skip しません。registry の direct caller は従来どおり動的に探索します。 - **初回 C# extraction で checksum 検証済み prepass symbol を再利用するようにしました** — 空 database からの CLI / MCP full index は、static-interface workspace 用に抽出済みの built-in symbol を上限付き・take-once で利用できます。immutable な lookup snapshot を materialize した後、prepass は admit した file ごとの symbol list の所有権を移し、symbol graph 全体を clone せず重複する workspace fallback object を解放します。main pass は各 file を引き続き再読込・検証し、checksum drift、不完全な prepass、regex timeout、cache 上限では通常 extraction へ fallback します。rebuild、update、symbols-only は従来どおりです。 - **初回 index で必須 literal がない正規表現 pattern を skip するようにしました** — built-in の case-sensitive symbol pattern は、file 選択時と各 regex call の直前に、実際の変換済み input に対して監査済みの2文字以上の literal を Ordinal で判定します。C# / Fortran の結合、Java / Kotlin annotation 除去、C# wrapped-modifier / 不完全 attribute recovery、C++ same-line member、CSS の再構成済み selector segment でも pattern 順と出力を変えず、bare C# static constructor の初回 gate miss 後も合成した `static ...` を再試行します。IgnoreCase、custom/plugin、1文字、共通 literal を持たない pattern は gate 対象外です。 - **symbol worker が既存の UTF-8 request frame を二重 decode せず処理するようにしました** — parent 側の `SerializeToUtf8Bytes` による1回の書き込みは変えず、全言語共通の child 経路で標準入力の raw byte から上限付き newline framing、validation、deserialize を直接行います。CRLF / final EOF の framing、protocol / JSON 上限、cancellation、Unicode の挙動、不正 UTF-8 / JSON の sanitization 済み error は従来どおりで、decoded `TextReader` 経路も診断用に維持します。 -- **新規 built-in index の fold readiness 確定で、保存済みの全名前を再 fold しないようにしました** — 通常の CLI / MCP indexing が `files`、`symbols`、`symbol_references` のすべてが空であると証明された database を所有する場合、SQLite `data_version` で保護された opaque で一回限りの claim により、最後の SQL NULL completeness check を維持しながら、全 symbol / reference string の materialize と再 fold を省きます。単調増加する accepted-producer generation により、custom plugin / pattern が一時的に active になり readiness 前に削除された場合も claim を無効化します。巨大な初回 index で row 数に比例する finalization work と数百 MiB の managed allocation を取り除きます。rebuild、update、legacy または既存 index、public writer 呼び出し、custom plugin / pattern、post-extraction hook、再利用 claim、外部変更された database は fail closed で full value validation に戻ります。 +- **新規 built-in index の fold readiness 確定で、保存済みの全名前を再 fold しないようにしました** — 通常の CLI / MCP indexing が `files`、`symbols`、`symbol_references` のすべてが空であると証明された database を所有する場合、SQLite `data_version` で保護された opaque で一回限りの claim により、最後の SQL NULL completeness check を維持しながら、全 symbol / reference string の materialize と再 fold を省きます。単調増加する accepted-producer generation により、custom plugin / pattern が一時的に active になり readiness 前に削除された場合も claim を無効化します。claimの`BEGIN IMMEDIATE`直後にcancelされても、warm writerを解放する前にraw transactionをrollbackします。巨大な初回 index で row 数に比例する finalization work と数百 MiB の managed allocation を取り除きます。rebuild、update、legacy または既存 index、public writer 呼び出し、custom plugin / pattern、post-extraction hook、再利用 claim、外部変更された database は fail closed で full value validation に戻ります。 - **全言語の永続化で kind taxonomy を immutable lookup により検証するようにしました** — symbol、reference、container kind の検証は、永続化する各行で順序付き公開 taxonomy array を再走査せず、process-static な Ordinal set を使います。private な canonical 順序付き taxonomy を schema check、migration、ctags filter にも使うため、公開互換 array の要素が誤って変更されても validation と永続 constraint は分裂しません。公開 taxonomy の列挙、case-sensitive な完全一致、invalid-kind 診断、CLI / MCP の挙動は変わりません。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index 00b7386fe..b10f8b52b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -845,6 +845,16 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis using var hotspotAggregateRefresh = writer.BeginDeferredHotspotReferenceAggregateRefresh( deferSecondaryIndexes: !options.SymbolsOnly && useFtsBulkLoad); using var fullScanTxn = writer.BeginTransaction(cancellationToken, "full scan write phase"); + if (referenceGraphRefresh.FreshReferenceResolutionDefaultsPending + && !writer.CanUseFreshReferenceResolutionDefaultsInCurrentTransaction(cancellationToken)) + { + // Another connection committed after the early empty-DB observation. All file, + // symbol, and reference writes below remain in the authoritative full refresh, but + // existing candidate-free references must be normalized by the ordinary full SQL. + // 早期のempty-DB確認後に別connectionがcommitした。以降のfile/symbol/reference writeは + // authoritative full refreshのまま維持し、既存candidate-free referenceは通常のfull SQLで正規化する。 + referenceGraphRefresh.DisableFreshReferenceResolutionDefaults(); + } fullScanWritePhaseStarted = true; writer.SetMeta( DbContext.WorkspaceVerificationPendingPathsCompleteMetaKey, diff --git a/src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs b/src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs index b35a96036..5594bb5d9 100644 --- a/src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs +++ b/src/CodeIndex/Database/DbWriter.FreshFoldReadiness.cs @@ -4,6 +4,26 @@ namespace CodeIndex.Database; public partial class DbWriter { + private const string AuthoritativeFreshRowsEmptySql = + """ + SELECT CASE + WHEN EXISTS(SELECT 1 FROM files LIMIT 1) + OR EXISTS(SELECT 1 FROM symbols LIMIT 1) + OR EXISTS(SELECT 1 FROM symbol_references LIMIT 1) + THEN 0 + ELSE 1 + END + """; + + private static readonly AsyncLocal + ScopedFreshFoldBeginImmediateCompletedForTesting = new(); + + internal static Action? FreshFoldBeginImmediateCompletedForTesting + { + get => ScopedFreshFoldBeginImmediateCompletedForTesting.Value; + set => ScopedFreshFoldBeginImmediateCompletedForTesting.Value = value; + } + /// /// One-shot proof that all fold-bearing tables were empty before an authoritative /// fresh-index run began. The proof is bound to one writer/connection and is invalidated @@ -58,21 +78,12 @@ internal bool TryConsume(DbWriter owner, long dataVersion) var beganTransaction = false; try { - Execute("BEGIN IMMEDIATE", cancellationToken); + BeginImmediateForAuthoritativeFreshClaim(cancellationToken); beganTransaction = true; cancellationToken.ThrowIfCancellationRequested(); using var emptyCheck = _conn.CreateCommand(); - emptyCheck.CommandText = - """ - SELECT CASE - WHEN EXISTS(SELECT 1 FROM files LIMIT 1) - OR EXISTS(SELECT 1 FROM symbols LIMIT 1) - OR EXISTS(SELECT 1 FROM symbol_references LIMIT 1) - THEN 0 - ELSE 1 - END - """; + emptyCheck.CommandText = AuthoritativeFreshRowsEmptySql; var allFoldRowTablesEmpty = Convert.ToInt64( emptyCheck.ExecuteScalar(), System.Globalization.CultureInfo.InvariantCulture) == 1; @@ -101,6 +112,72 @@ ELSE 1 } } + /// + /// Execute BEGIN with cancellation-aware SQLite interruption but without a post-success + /// token check. The caller records cleanup ownership immediately after this method returns, + /// then performs the post-BEGIN cancellation check inside its guarded try/catch. + /// cancellation-aware な SQLite interrupt 付きで BEGIN を実行するが、成功後の token check は + /// caller が cleanup ownership を記録した直後、guarded try/catch 内で行う。 + /// + private void BeginImmediateForAuthoritativeFreshClaim(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using var command = _conn.CreateCommand(); + command.CommandText = "BEGIN IMMEDIATE"; + using var cancellationRegistration = RegisterSqliteInterrupt(cancellationToken); + try + { + command.ExecuteNonQuery(); + FreshFoldBeginImmediateCompletedForTesting?.Invoke(); + } + catch (SqliteException exception) when (IsSqliteInterruptCancellation(exception, cancellationToken)) + { + throw new OperationCanceledException( + "SQLite authoritative-fresh claim was interrupted.", + exception, + cancellationToken); + } + } + + /// + /// Revalidate the fresh-reference shortcut after the authoritative CLI write transaction + /// has begun. The transaction snapshot/lock closes the pre-write gap; a false result makes + /// reference insertion and final resolution use their ordinary full-refresh defaults. + /// authoritative CLI write transaction 開始後に fresh-reference shortcut を再検証する。 + /// transaction snapshot / lock で write 前の gap を閉じ、false なら通常の full-refresh + /// default で reference insert と最終 resolution を行う。 + /// + internal bool CanUseFreshReferenceResolutionDefaultsInCurrentTransaction( + CancellationToken cancellationToken = default) + { + if (!IsInTransaction()) + { + throw new InvalidOperationException( + "Fresh reference resolution defaults must be revalidated inside the active write transaction."); + } + + cancellationToken.ThrowIfCancellationRequested(); + using var command = _conn.CreateCommand(); + command.Transaction = _activeTransaction; + command.CommandText = AuthoritativeFreshRowsEmptySql; + using var cancellationRegistration = RegisterSqliteInterrupt(cancellationToken); + try + { + var tablesAreEmpty = Convert.ToInt64( + command.ExecuteScalar(), + System.Globalization.CultureInfo.InvariantCulture) == 1; + cancellationToken.ThrowIfCancellationRequested(); + return tablesAreEmpty; + } + catch (SqliteException exception) when (IsSqliteInterruptCancellation(exception, cancellationToken)) + { + throw new OperationCanceledException( + "SQLite fresh reference resolution revalidation was interrupted.", + exception, + cancellationToken); + } + } + private bool TryConsumeAuthoritativeFreshFoldRowsClaim( AuthoritativeFreshFoldRowsClaim? claim) => claim?.TryConsume(this, ReadDataVersion()) == true; diff --git a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs index 9a39014cf..d0a9059ab 100644 --- a/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs +++ b/src/CodeIndex/Database/DbWriter.ReferenceGraphRefreshScope.cs @@ -853,6 +853,9 @@ internal bool FreshReferenceResolutionDefaultsPending internal void RequireFullRefresh() => _forceFullRefresh = true; + internal void DisableFreshReferenceResolutionDefaults() + => _freshReferenceResolutionDefaultsPending = false; + internal void MarkRefreshCompleted() { _forceFullRefresh = false; diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index 5e6cae855..1d8fe08d2 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -10876,6 +10876,33 @@ public void TryClaimAuthoritativeFreshFoldRows_PreCancelledRequestLeavesWriterUs Assert.NotNull(_writer.TryClaimAuthoritativeFreshFoldRows()); } + [Fact] + public void TryClaimAuthoritativeFreshFoldRows_CancelAfterBeginRollsBackAndLeavesWriterUsable() + { + using var cancellation = new CancellationTokenSource(); + var previousHook = DbWriter.FreshFoldBeginImmediateCompletedForTesting; + try + { + DbWriter.FreshFoldBeginImmediateCompletedForTesting = () => + { + previousHook?.Invoke(); + cancellation.Cancel(); + }; + + Assert.Throws(() => + _writer.TryClaimAuthoritativeFreshFoldRows(cancellation.Token)); + Assert.True(cancellation.IsCancellationRequested); + } + finally + { + DbWriter.FreshFoldBeginImmediateCompletedForTesting = previousHook; + } + + using (var nextTransaction = _writer.BeginTransaction()) + nextTransaction.Commit(); + Assert.NotNull(_writer.TryClaimAuthoritativeFreshFoldRows()); + } + [Fact] public void MarkFoldReady_LeavesFoldReadyUnsetWhenNullFoldedRowExists() { diff --git a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs index 91535b571..f118239a5 100644 --- a/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs +++ b/tests/CodeIndex.Tests/FreshReferenceResolutionTests.cs @@ -57,6 +57,20 @@ public void BeginReferenceGraphRefreshScope_RejectsFreshDefaultsWithoutForcedFul Assert.Equal("useFreshReferenceResolutionDefaults", exception.ParamName); } + [Fact] + public void FreshDefaultsRevalidation_RequiresTransactionAndRejectsPersistedRows() + { + Assert.Throws(() => + _writer.CanUseFreshReferenceResolutionDefaultsInCurrentTransaction()); + + using var transaction = _writer.BeginTransaction(); + Assert.True(_writer.CanUseFreshReferenceResolutionDefaultsInCurrentTransaction()); + + InsertFile("src/concurrent.py", "python"); + + Assert.False(_writer.CanUseFreshReferenceResolutionDefaultsInCurrentTransaction()); + } + [Fact] public void InsertReferences_FreshDefaultsKeepParameterShapeAndUseSeparateCachedSql() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index c000cf7a9..2642055c9 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -3209,6 +3209,94 @@ public void Run_FullScan_ValidatesScanInputExactlyBeforeWriteAndReadiness( } } + [Fact] + public void Run_FreshFullScan_ExternalPreTransactionWriteFallsBackToFullResolution() + { + var projectRoot = CreateTempProject(); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + var previousBarrierHook = IndexCommandRunner.FullScanInputSnapshotBarrierForTesting; + var injected = 0; + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.py"), "def run():\n return 1\n"); + IndexCommandRunner.FullScanInputSnapshotBarrierForTesting = phase => + { + previousBarrierHook?.Invoke(phase); + if (!string.Equals(phase, "before_write", StringComparison.Ordinal) + || Interlocked.Exchange(ref injected, 1) != 0) + { + return; + } + + using var externalDb = new DbContext(DbOpenIntent.WriteIndex, dbPath); + externalDb.InitializeSchema(); + var externalWriter = new DbWriter(externalDb.Connection); + var fileId = externalWriter.UpsertFile(new FileRecord + { + Path = "external/concurrent.py", + Lang = "python", + Size = 20, + Lines = 1, + Checksum = "external-concurrent", + Modified = new DateTime(2026, 8, 11, 0, 0, 0, DateTimeKind.Utc), + }); + externalWriter.InsertReferences( + [ + new ReferenceRecord + { + FileId = fileId, + SymbolName = "NoCandidate", + ReferenceKind = "call", + Line = 1, + Column = 1, + Context = "NoCandidate()", + ContainerKind = "function", + ContainerName = "external", + IsSelfReference = true, + IsMutualRecursion = true, + }, + ], + refreshMutualRecursionFlags: false); + }; + + var (exitCode, json) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, exitCode); + Assert.Equal("success", json.GetProperty("status").GetString()); + Assert.Equal(1, injected); + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var command = db.Connection.CreateCommand(); + command.CommandText = + """ + SELECT r.resolution_state, + r.resolution_candidate_count, + r.target_symbol_id, + r.target_symbol_key, + r.is_self_reference, + r.is_mutual_recursion + FROM symbol_references AS r + JOIN files AS f ON f.id = r.file_id + WHERE f.path = 'external/concurrent.py' + AND r.symbol_name = 'NoCandidate' + """; + using var reader = command.ExecuteReader(); + Assert.True(reader.Read()); + Assert.Equal("unresolved", reader.GetString(0)); + Assert.Equal(0, reader.GetInt32(1)); + Assert.True(reader.IsDBNull(2)); + Assert.True(reader.IsDBNull(3)); + Assert.Equal(0, reader.GetInt32(4)); + Assert.Equal(0, reader.GetInt32(5)); + Assert.False(reader.Read()); + } + finally + { + IndexCommandRunner.FullScanInputSnapshotBarrierForTesting = previousBarrierHook; + DeleteDirectory(projectRoot); + SqliteConnection.ClearAllPools(); + } + } + [Fact] public void Run_FullScan_FreshSnapshotAbortRetainsDiscoveredLanguageFailuresWithoutRows() {