diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index 977a46ac6..5b9492d99 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -138,6 +138,7 @@ CI watching must be bounded. Do not loop indefinitely. - `status --json` and related JSON/MCP payloads currently expose the trust fields documented in `README.md` and `DEVELOPER_GUIDE.md`, including `fold_ready`, `fold_ready_reason`, `graph_table_available`, `graph_data_current`, `index_complete`, `index_incomplete_reasons`, `issues_table_available`, `file_issues_data_current`, `migration_in_progress`, `sql_graph_contract_ready`, `sql_graph_contract_degraded_reason`, `hotspot_family_ready`, `hotspot_family_degraded_reason`, `language_readiness`, `csharp_symbol_name_ready`, `csharp_metadata_target_ready`, `csharp_metadata_target_degraded_reason`, `indexed_head_commit`, `workspace_verified_head_sha`, `worktree_head_changed`, `indexed_head_sha`, `indexed_head_branch`, `indexed_head_timestamp`, `commits_ahead_of_indexed_head`, `head_freshness`, `index_writer_version`, `index_newer_than_reader`, `index_newer_than_reader_reason`, `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `git_executable`, `path_case_sensitive`, `data_dir`, `data_dir_source`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`), `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`), `maintenance_guidance`, WAL checkpoint diagnostics (`read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`), `symbol_kinds`, `symbols_by_language`, status kind cap metadata (`symbol_kind_limit`, `symbol_kind_name_limit`, `symbol_kind_total_count`, `symbol_kind_omitted_count`, `symbol_kind_names_truncated`, `symbols_by_language_kind_total_counts`, `symbols_by_language_kind_omitted_counts`, `symbols_by_language_kind_names_truncated`), `process`, `last_index_run`, `last_failed_or_partial_index_run`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`, `last_workspace_freshened_at`, `hooks`, `hook_diagnostics`, `trust_overrides`, MCP-only `mcp_session`, `mcp.rate_limit.bucket_limit`, `mcp.rate_limit.bucket_limit_rejection_count`, and the `status --check`-only `stale_after_seconds` / `index_age_seconds` threshold audit fields and `repair_commands`. - `status --check` repair actions are structured by `name`, `action`, `args`, `mutation_class`, `safety_class`, and `safety_notes`. Preserve compatibility `reason` as the first trigger and ordered `reasons` as the complete trigger set. Deduplicate only exact structured identities; different targets, options, actions, mutation classes, or safety semantics must remain distinct. JSON and human output must use the same deduplicated order. Human output must preserve platform-aware shell quoting, visibly escape control characters to keep each repair action on one diagnostic line, and leave structured JSON `args` unchanged. - `maintenance_guidance.fts_optimization` is the shared, read-only recommendation contract for status, explain, optimize preview, and optimize execution. Keep `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` synchronized; stale or unavailable snapshots must not recommend mutation. +- A successful CLI or MCP `index --rebuild` evaluates the shared freelist warning threshold after the index transaction commits. Incremental-auto-vacuum databases may run bounded `PRAGMA incremental_vacuum`; legacy databases must skip automatic reclaim instead of running a full `VACUUM`. Preserve the immediate index-result and persisted `last_index_run.rebuild_reclaim` telemetry, stable states/reasons, before/after ratios and byte/page counts, and the rule that reclaim failure never reclassifies an already committed index run as failed. - A valid CLI `status --stale-after ` implies the workspace check. Check-mode JSON includes `query_context.check_mode` (`explicit` or `implied_by_stale_after`) and `query_context.stale_after_seconds`; ordinary status JSON omits `query_context`. - Every bounded `workspace_check` path list (`changed_files`, `missing_files`, `outside_sparse_cone_files`, `unindexed_files`, `unverifiable_files`, and `scan_errors`) carries an authoritative count plus matching `*_truncated`, `*_path_limit`, and `*_omitted_count` fields. List-only `--fields` projections must retain those signals automatically; compact output retains the signals without path arrays; and `--max-json-bytes` may remove only trailing paths while updating both the per-list omitted count and the envelope byte-limit signal. Human check diagnostics must label each displayed list as a sample or complete. - `status --explain` derives accepted top-level keys from the same source-generated `StatusResult` serializer metadata as `status --json`, excludes ignored properties, and supports bounded dot-separated member paths without reading runtime values. Major readiness, trust, extension, maintenance, and cap-hit sections return structured meaning, source, dependencies, interpretation, and repair guidance; unknown input is sanitized and returns bounded valid candidates. Bounded status explain envelopes also omit database paths, timings, indexed HEADs, and stable-at timestamps. diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 47bb429c1..69ef976db 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -1217,6 +1217,7 @@ Current stable codes and triggers: | Read-only opens and fallback | Query-only commands open with SQLite `Mode=ReadOnly` from the first attempt, retain WAL visibility, and never use writable setup or opportunistic migrations. A write-capable intent may still fall back to read-only when writable journal/WAL setup fails; an explicitly supplied `immutable=1` URI is the opt-in stale-snapshot escape hatch. If a WAL is present and must be observed from storage that cannot expose its sidecars, copy `.db`, `.db-wal`, and `.db-shm` together to a readable location or use a SQLite backup from an environment that can open the full WAL set. | | Status pragma diagnostics | `status --json` exposes the selected read-only connection under `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) and resolved connection values under `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`). It also exposes prepared-command cache counters under `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) for automation and support diagnostics. `maintenance_guidance` derives `wal_state`, `freelist_ratio`, `freelist_state`, `estimated_*_reclaimable`, `auto_vacuum_mode(_name)`, `recommended_command`, and `post_maintenance_follow_up` from those raw metrics without changing the raw values. Its nested `fts_optimization` uses the same pure evaluator as optimize preview and execution, exposing `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state` without writing to the database. `status --check --json` adds structured `repair_commands[]` entries with `name`, `action`, `args`, `mutation_class`, `safety_class`, `safety_notes`, compatibility `reason`, and ordered `reasons`. Exact structured identities are deduplicated and aggregate reasons in check priority order; differences in target, options, action, mutation class, or safety semantics prevent merging. Human check output uses the same command set, preserves platform-aware shell quoting, visibly escapes control characters to keep each `[repair]` action on one diagnostic line, and leaves structured JSON `args` unchanged. `last_failed_or_partial_index_run` exposes bounded failed/partial index context (`status`, `mode`, timings, counts, stable error code, reason, `progress_persisted`, and bounded `recovery_hint`) and must not include raw exception text or file paths. | | Maintenance thresholds | WAL guidance flips to `checkpoint_recommended` at `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (default 64 MiB). Freelist guidance flips to `vacuum_recommended` at `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (default `0.20`). Invalid or out-of-range env values fall back to defaults. | +| Post-rebuild reclaim | After a successful CLI or MCP rebuild commits, cdidx evaluates the shared freelist threshold and runs `PRAGMA incremental_vacuum` only when reclaim is recommended and `auto_vacuum=INCREMENTAL`. It never performs an automatic full `VACUUM`; legacy databases report `skipped/auto_vacuum_not_incremental` and retain the explicit `cdidx vacuum` conversion path. Reclaim runs after the index transaction, emits bounded progress/log phases, and persists `last_index_run.rebuild_reclaim` with stable state/reason, duration, before/after logical database sizes and physical main-file samples, page/free-page counts and ratios, and reclaimed page/byte counts. Physical samples can lag while WAL-backed pages await checkpoint; logical sizes and page metrics are the immediate attribution contract. Cancellation, busy/read-only I/O, or another reclaim failure cannot roll back or reclassify the already committed index generation. | | Maintenance command precedence | `maintenance_guidance.recommended_command` preserves the existing vacuum-then-checkpoint precedence. It returns `cdidx optimize --db ` only when WAL and freelist states are both exactly `ok` and the trusted FTS write snapshot reaches its threshold; an `unknown` higher-priority state or a stale/unavailable FTS snapshot never selects an optimize command. | | Page attribution | `status --json` reads SQLite page ownership without mutating the source. It prefers `dbstat` page bytes and otherwise traverses a bounded b-tree/WAL snapshot (at most 1,000,000 pages and 100,000 schema objects); when a live WAL connection is not already backed by a stable detached file set, the fallback first makes a cancellation-aware private backup of that connection's active read snapshot so a concurrent commit cannot mix generations. `allocated_object_bytes + freelist_bytes + unexplained_residual_bytes` equals `logical_database_bytes`; table/index and internal/leaf/overflow/other page subtotals each reconcile to `allocated_object_bytes`. Payload, unused space, and structural overhead form a second reconciliation. Physical main/WAL/SHM bytes are reported separately. Output is capped at 20 object names, each support-sanitized to at most 128 characters. A failed or inconsistent probe returns `available=false`, a stable `unavailable_reason`, and null/omitted attribution values rather than zeros. | | Vacuum | `cdidx vacuum` runs `PRAGMA incremental_vacuum` against writable incremental-auto-vacuum DBs, and performs a one-time `PRAGMA auto_vacuum=INCREMENTAL` plus full `VACUUM` conversion for legacy no-autovacuum DBs. `cdidx vacuum --dry-run --json` estimates reclaimable pages/bytes and returns the same maintenance guidance without executing vacuum pragmas. Real `cdidx vacuum --json` also reports before/after DB and WAL byte samples; `wal_checkpoint_timing_note` explains that `wal_size_bytes_after` is measured before connection cleanup, so later `status --json` output may show a smaller WAL after checkpoint/truncation. | @@ -4850,6 +4851,7 @@ apply 時は `PRAGMA optimize` を実行します。 | read-only open / fallback | query-only command は最初の試行から SQLite `Mode=ReadOnly` で開き、WAL の可視性を保ちながら writable setup と opportunistic migration を実行しません。write-capable intent は journal/WAL setup に失敗した場合に read-only へ fallback することがあります。明示的な `immutable=1` URI は stale snapshot を許容する opt-in escape hatch です。sidecar を公開できない storage 上の WAL を観測する必要がある場合は、`.db` / `.db-wal` / `.db-shm` をまとめて readable location に copy するか、full WAL set を open できる環境で SQLite backup を使います。 | | status pragma diagnostics | `status --json` は選択された read-only connection を `sqlite_connection_policy` (`active_mode=read_only`, `open_mode=read_only`) で、解決済みの接続値を `db_pragma_settings` (`journal_mode`, `synchronous`, `wal_autocheckpoint`, `busy_timeout_ms`, `page_count`, `freelist_count`, `page_size`, `auto_vacuum`) で公開します。また、prepared command cache counter を `prepared_command_cache` (`count`, `capacity`, `hit_count`, `miss_count`, `eviction_count`) で公開します。`maintenance_guidance` は raw 値を変えずに `wal_state`、`freelist_ratio`、`freelist_state`、`estimated_*_reclaimable`、`auto_vacuum_mode(_name)`、`recommended_command`、`post_maintenance_follow_up` を派生します。nested な `fts_optimization` は optimize preview / execution と同じ純粋 evaluator を使い、database に書き込まず `recommended`、`action`、`reason`、`threshold_writes`、`observed_writes`、`state` を公開します。`status --check --json` は `repair_commands[]` に `name`、`action`、`args`、`mutation_class`、`safety_class`、`safety_notes`、互換用の `reason`、順序付きの `reasons` を返します。完全に同一の構造化 identity は deduplicate して check の優先順に reason を集約し、target、option、action、mutation class、安全性 semantics が異なる場合は merge しません。human check output も同じ command set を使い、platform-aware な shell quote を維持し、control character を可視 escape して各 `[repair]` action を1行に保ちます。構造化 JSON の `args` は変更しません。`last_failed_or_partial_index_run` は bounded な failed / partial index context (`status`、`mode`、timing、count、stable error code、reason、`progress_persisted`、bounded な `recovery_hint`) のみを公開し、raw exception text や file path を含めてはいけません。 | | maintenance threshold | WAL guidance は `CDIDX_MAINTENANCE_WAL_WARN_BYTES` (既定 64 MiB) 以上で `checkpoint_recommended` になります。freelist guidance は `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (既定 `0.20`) 以上で `vacuum_recommended` になります。不正・範囲外の環境変数値は既定値へ戻します。 | +| rebuild 後の reclaim | CLI / MCP rebuild が正常に commit された後、cdidx は共通の freelist threshold を評価し、reclaim が推奨され、かつ `auto_vacuum=INCREMENTAL` の場合だけ `PRAGMA incremental_vacuum` を実行します。自動の full `VACUUM` は実行せず、legacy database は `skipped/auto_vacuum_not_incremental` を報告して明示的な `cdidx vacuum` conversion path を維持します。reclaim は index transaction の後に実行し、上限付き progress / log phase を出力して、stable な state / reason、duration、before / after の logical database size と物理 main-file sample、page / free-page count と ratio、回収 page / byte 数を `last_index_run.rebuild_reclaim` に保存します。WAL-backed page が checkpoint 待ちの間は物理 sample が遅れて変化する場合があり、即時 attribution の contract は logical size と page metrics です。cancellation、busy / read-only I/O、その他の reclaim failure は commit 済み index generation を rollback したり failure に再分類したりしません。 | | maintenance command の優先順位 | `maintenance_guidance.recommended_command` は既存の vacuum、checkpoint の順序を維持します。WAL と freelist の state が両方とも厳密に `ok` で、信頼できる FTS write snapshot が threshold に達した場合だけ `cdidx optimize --db ` を返します。上位 state が `unknown` の場合や FTS snapshot が stale / unavailable の場合は optimize command を選択しません。 | | page attribution | `status --json` は source を変更せずに SQLite page ownership を読み取ります。`dbstat` page byte を優先し、利用できない場合は件数上限付きの b-tree / WAL snapshot traversal（最大1,000,000 page、100,000 schema object）へ fallback します。live WAL connection が安定した detached file set に基づいていない場合、fallback は先にその connection の active read snapshot を cancellation 対応の private backup に固定し、並行 commit による世代混在を防ぎます。`allocated_object_bytes + freelist_bytes + unexplained_residual_bytes` は `logical_database_bytes` と一致し、table/index と internal/leaf/overflow/other page の小計はそれぞれ `allocated_object_bytes` と一致します。payload、unused space、structural overhead も別に再照合されます。物理 main/WAL/SHM byte は分離して報告します。出力する object 名は最大20件で、各名称は support-safe sanitizer により最大128文字になります。probe の失敗・不整合時は `available=false`、安定した `unavailable_reason`、null / 省略された attribution 値を返し、ゼロとして偽装しません。 | | vacuum | `cdidx vacuum` は incremental-auto-vacuum DB では `PRAGMA incremental_vacuum` を実行し、legacy no-autovacuum DB では初回のみ `PRAGMA auto_vacuum=INCREMENTAL` と full `VACUUM` で変換します。`cdidx vacuum --dry-run --json` は vacuum pragma を実行せず、回収可能 page/byte の推定と同じ maintenance guidance を返します。実行系 `cdidx vacuum --json` は DB / WAL byte の before / after sample も返します。`wal_checkpoint_timing_note` は `wal_size_bytes_after` が connection cleanup 前の計測であり、checkpoint / truncation 後の `status --json` では WAL が小さく見える場合があることを示します。 | diff --git a/README.md b/README.md index abeb6a1ce..aa0bf2bd4 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ visible here as a compact compatibility index. | Extension and extractor diagnostics | `unknown_extension_file_count`, `unknown_extension_files`, `unknown_extension_files_truncated`, `unknown_extension_file_path_limit`, `unknown_extension_extension_counts`, `unknown_extension_category_counts`, `unknown_extension_groups`, `extractors`, `hooks`, `hook_diagnostics`. | | Runtime trust and permissions | `trust_overrides`, `git_executable`, `path_case_sensitive`, `data_dir_mode`, `db_file_mode`, `database_permission_policy`, `database_permission_diagnostics`, `mac_profile`, `mac_profile_diagnostics`. | | Check context and run diagnostics | `stale_after_seconds`, `index_age_seconds`, `query_context.check_mode`, `query_context.stale_after_seconds`, `process`, `last_index_run`, `last_workspace_freshened_at`, `last_failed_or_partial_index_run`. | -| Last-run detail | `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | +| Last-run detail | `last_index_run.bytes_read_skipped_file_count`, `last_index_run.bytes_read_incomplete`, `last_index_run.diagnostics`, `last_index_run.diagnostic_count`, `last_index_run.diagnostics_truncated`, `last_index_run.reference_extraction_cap_hits`, `last_index_run.rebuild_reclaim`, `last_failed_or_partial_index_run.progress_persisted`, `last_failed_or_partial_index_run.recovery_hint`, `last_failed_or_partial_index_run.file_errors`. | | SQLite and maintenance | `sqlite_connection_policy`, `db_size_bytes`, `wal_size_bytes`, `db_pragma_settings`, `prepared_command_cache`, `maintenance_guidance`, `maintenance_guidance.fts_optimization`, `threshold_writes`, `observed_writes`. | | WAL checkpoint diagnostics | `read_only_fallback`, `wal_checkpoint_attempted`, `wal_checkpoint_succeeded`, `wal_checkpoint_skipped_reason`, `wal_checkpoint_failure_reason`, `wal_checkpoint_busy`, `wal_checkpoint_log_page_count`, `wal_checkpoint_checkpointed_page_count`, `wal_checkpoint_remaining_page_count`, `read_only_immutable_fallback`, `wal_stale_snapshot_risk`, `wal_stale_snapshot_reason`. | | Database size attribution | `database_size_attribution`. | @@ -376,7 +376,7 @@ field group を表に残します。 | extension / extractor diagnostics | `unknown_extension_file_count`、`unknown_extension_files`、`unknown_extension_files_truncated`、`unknown_extension_file_path_limit`、`unknown_extension_extension_counts`、`unknown_extension_category_counts`、`unknown_extension_groups`、`extractors`、`hooks`、`hook_diagnostics`。 | | runtime trust / permissions | `trust_overrides`、`git_executable`、`path_case_sensitive`、`data_dir_mode`、`db_file_mode`、`database_permission_policy`、`database_permission_diagnostics`、`mac_profile`、`mac_profile_diagnostics`。 | | check context / run diagnostics | `stale_after_seconds`、`index_age_seconds`、`query_context.check_mode`、`query_context.stale_after_seconds`、`process`、`last_index_run`、`last_workspace_freshened_at`、`last_failed_or_partial_index_run`。 | -| last-run detail | `last_index_run.bytes_read_skipped_file_count`、`last_index_run.bytes_read_incomplete`、`last_index_run.diagnostics`、`last_index_run.diagnostic_count`、`last_index_run.diagnostics_truncated`、`last_index_run.reference_extraction_cap_hits`、`last_failed_or_partial_index_run.progress_persisted`、`last_failed_or_partial_index_run.recovery_hint`、`last_failed_or_partial_index_run.file_errors`。 | +| last-run detail | `last_index_run.bytes_read_skipped_file_count`、`last_index_run.bytes_read_incomplete`、`last_index_run.diagnostics`、`last_index_run.diagnostic_count`、`last_index_run.diagnostics_truncated`、`last_index_run.reference_extraction_cap_hits`、`last_index_run.rebuild_reclaim`、`last_failed_or_partial_index_run.progress_persisted`、`last_failed_or_partial_index_run.recovery_hint`、`last_failed_or_partial_index_run.file_errors`。 | | SQLite / maintenance | `sqlite_connection_policy`、`db_size_bytes`、`wal_size_bytes`、`db_pragma_settings`、`prepared_command_cache`、`maintenance_guidance`、`maintenance_guidance.fts_optimization`、`threshold_writes`、`observed_writes`。 | | WAL checkpoint diagnostics | `read_only_fallback`、`wal_checkpoint_attempted`、`wal_checkpoint_succeeded`、`wal_checkpoint_skipped_reason`、`wal_checkpoint_failure_reason`、`wal_checkpoint_busy`、`wal_checkpoint_log_page_count`、`wal_checkpoint_checkpointed_page_count`、`wal_checkpoint_remaining_page_count`、`read_only_immutable_fallback`、`wal_stale_snapshot_risk`、`wal_stale_snapshot_reason`。 | | database size attribution | `database_size_attribution`。 | diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index da1f9715c..a23afe060 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -32,6 +32,7 @@ Use the full suite by default. Use targeted filters only while iterating locally - Markdown unused-audit coverage indexes one real Markdown fixture containing common backtick and tilde fence-language markers. Keep default suppression, `documentation_surface` totals, reason tags, and `--all` recovery in that shared fixture. - Unused by-bucket JSON coverage keeps `symbols` as the only full-row collection and checks every lightweight bucket membership against its canonical `symbol_index`. Preserve empty, single-row, large multi-bucket-set, CLI full/compact, MCP, and bounded UTF-8 cursor fixtures; byte-limited pages must stay within the requested budget and reassemble without duplicate or skipped canonical rows. - FTS optimization recommendation coverage keeps the shared evaluator exact at one write below, at, and one write above the 25-write threshold. Status, explain, optimize dry-run, optimize execution, and vacuum maintenance guidance must expose the same `recommended`, `action`, `reason`, `threshold_writes`, `observed_writes`, and `state`; stale batches, known WAL-stale snapshots, forward-incompatible schema stamps, and unavailable legacy counters/page snapshots suppress the recommendation, query-only status performs no source writes, and execution uses the focused counter/page/forward-contract/freshness snapshot instead of full status scans. A hot-WAL fixture opened through an explicit `immutable=1` URI must prove that status, standalone optimize dry-run, and the `index --optimize` dry-run alias preserve the same stale recommendation. A WAL or freelist state of `unknown` cannot select the optimize command, and a successful optimize reports the reset counter afterward. +- Rebuild reclaim coverage creates a real high-freelist incremental-auto-vacuum database, runs CLI rebuild with a concurrently open reader, and verifies row integrity, before/after logical database-size reduction, ratio reduction below the shared threshold, reclaimed pages/bytes, immediate JSON, persisted `last_index_run.rebuild_reclaim`, and truthful explicit-vacuum metrics. Keep focused cases for below-threshold no-op, injected reclaim failure after commit, and interrupted rebuild recovery; a maintenance failure must preserve a usable committed database and stable bounded telemetry. - Full-scan CLI and MCP no-op coverage treats one repository-wide reusable-stat snapshot read and one folded-readiness verification as performance contracts. Keep assertions for one snapshot read, one stat lookup per candidate, one folded verification, and no content load for unchanged files when changing incremental indexing. - Reference-graph refresh coverage treats graph-neutral indexing as a performance contract across incremental full scan, scoped `--files` update, and MCP indexing. Keep zero-refresh assertions for new and modified source files without symbols/references, plus a single batched refresh assertion when existing or new graph identity rows change. A healthy incremental generation must restrict identity/candidate/recursion work to transaction-committed dirty files, old and new `(language, folded name)` dependencies, and their old/new reciprocal edges; retain C#/Python language-transition and unchanged-target parity with a subsequent full refresh, rolled-back file batches, cancellation/retry, orphan-candidate cleanup, and the controlled 4,100-of-4,100 broad-scope fallback. Fresh/rebuild runs, missing identity contracts, and dirty sets of at least 4,096 references covering at least 50% of the graph must keep the full-refresh path. Query-plan coverage must keep all four scoped update phases and all ten candidate inserts on dirty-table-driven reference primary-key seeks, keep C# instantiate grouping on lookup names plus `idx_symbols_name_folded`, and prove that a sub-4,096 dirty set does not count the whole reference table without an explicit diagnostic hook. - Index-generation completeness coverage uses a table-driven full, symbols-only, max-file-byte, max-symbol, and max-reference matrix, with extractor failure kept separate because it uses a mutable hook. Assert that index-command JSON, immediate status, and workspace health expose identical index/graph booleans and reason arrays where available; MCP cap coverage must match the persisted status snapshot as well. Remove the additive completeness metadata in healthy and capped fixtures to preserve legacy fallback coverage, and clear issue readiness before a scoped capped update to prove current omission evidence survives degraded prior metadata. Lowering or raising the file-size policy must reprocess unchanged files in CLI and MCP indexing so a prior `file_too_large` issue cannot be reused. Structured remediation must distinguish symbols-only / missing-graph causes from reference safety caps and must not label an incomplete index as fold-only. Human output must identify incomplete generations instead of printing a complete summary. @@ -1028,6 +1029,7 @@ dotnet test --filter "FullyQualifiedName~GitHelperTests" - Markdown の unused audit coverage は、一般的な backtick / tilde fence の language marker を含む実 Markdown fixture を1回 index します。同じ fixture で既定抑制、`documentation_surface` totals、reason tag、`--all` による復元を維持してください。 - unused の bucket 別 JSON coverage では、完全な行集合を `symbols` だけに保ち、各 lightweight bucket membership を正規の `symbol_index` と照合します。空集合、1行、大規模な複数 bucket 集合、CLI の full / compact、MCP、UTF-8 byte 上限付き cursor の fixture を維持してください。byte 制限された各 page は要求 budget 内に収まり、連結時に正規行の重複や欠落が発生してはいけません。 - FTS optimization recommendation coverage は、25 write threshold の1つ下、ちょうど、1つ上で shared evaluator の境界を固定します。status、explain、optimize dry-run、optimize execution、vacuum maintenance guidance は同じ `recommended`、`action`、`reason`、`threshold_writes`、`observed_writes`、`state` を公開し、stale batch、既知の WAL-stale snapshot、forward-incompatible な schema stamp、利用できない legacy counter / page snapshot は recommendation を抑止します。query-only status は source に書き込まず、execution は full status scan ではなく counter / page / forward-contract / freshness に限定した snapshot を使います。hot WAL fixture を明示的な `immutable=1` URI で開き、status、standalone optimize dry-run、`index --optimize` dry-run alias が同じ stale recommendation を保持することも証明します。WAL または freelist の state が `unknown` の場合は optimize command を選択せず、成功した optimize は reset 後の counter を返す必要があります。 +- rebuild reclaim coverage は、実際に high-freelist となった incremental-auto-vacuum database を作成し、reader connection を開いたまま CLI rebuild を実行して、row integrity、before / after の logical database size 縮小、共通 threshold 未満への ratio 低下、回収 page / byte、即時 JSON、保存された `last_index_run.rebuild_reclaim`、明示的 vacuum の正確な metrics を検証します。below-threshold no-op、commit 後の reclaim failure 注入、interrupted rebuild recovery の focused case も維持してください。maintenance failure が発生しても利用可能な commit 済み database と stable で上限付きの telemetry を保持する必要があります。 - full-scan CLI と MCP の no-op coverage は、リポジトリ全体の reusable-stat snapshot read と folded-readiness verification がそれぞれ 1 回であることを performance contract とします。incremental indexing を変更するときは、snapshot read が 1 回、候補ごとの stat lookup が 1 回、folded verification が 1 回、unchanged file の content load が 0 回という assertion を維持してください。 - reference-graph refresh coverage は、incremental full scan、scoped `--files` update、MCP indexing を横断する graph-neutral indexing を performance contract とします。symbol/reference を持たない新規・変更 source file では refresh 0 回を維持し、既存または新規の graph identity 行が変化する場合は batch 全体で refresh 1 回を assertion してください。健全な incremental generation では identity / candidate / recursion 処理を transaction commit 済みの dirty file、旧・新の `(language, folded name)` 依存、旧・新の逆辺に限定します。C# / Python の言語遷移、未変更targetを参照する新規callerと後続full refreshのparity、rollback file batch、cancel後retry、孤立candidate cleanup、4,100件中4,100件をdirtyにする制御broad-scope fallbackを維持してください。fresh/rebuild、identity契約欠落、または4,096件以上かつgraphの50%以上を占めるdirty集合ではfull-refresh経路を維持します。query-plan coverageでは、scoped updateの4 phaseとcandidate INSERT 10本をdirty table起点のreference主キーseekに保ち、C# instantiate groupingをlookup nameと`idx_symbols_name_folded`起点にし、明示的なdiagnostic hookがない4,096件未満のdirty集合ではreference table全件COUNTを行わないことを検証してください。 - index generation の completeness coverage は full、symbols-only、max-file-byte、max-symbol、max-reference を table-driven matrix で検証し、mutable hook を使う extractor failure は別 case に保ちます。index command JSON、直後の status、workspace health で、利用可能な index/graph の boolean と reason array が完全に一致すること、MCP の cap case も persisted status snapshot と一致することを assertion してください。healthy / capped fixture から additive completeness metadata を削除して legacy fallback coverage を維持し、scoped capped update の前に issue readiness を clear して、prior metadata が degraded でも今回の omission evidence が失われないことを検証します。file-size policy を下げた場合も上げた場合も、CLI / MCP indexing は unchanged file を再処理し、以前の `file_too_large` issue を再利用してはいけません。structured remediation は symbols-only / missing-graph 原因と reference safety cap を区別し、incomplete index を fold-only と表示しない必要があります。human output は complete summary ではなく incomplete generation を明示する必要があります。 diff --git a/USER_GUIDE.md b/USER_GUIDE.md index aeaa8b022..9e3740c35 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -861,6 +861,8 @@ Use the smallest change that reduces the expensive part of your run. `index --dry-run --rebuild` previews a full replacement scan but does not delete the existing index, so it never prompts for `--yes`. Add `--json --memory-trace` to receive a `memory_timeline` with `start`, `snapshot`, `scan`, and `finalize` samples from the preview itself. Dry-run reads its database snapshot and source files without changing the workspace or DB/WAL/SHM set. +After a successful real `index --rebuild` commit, cdidx checks the same freelist ratio used by `status --json` maintenance guidance. When the ratio is at least `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO` (default `0.20`) and the database already uses incremental auto-vacuum, cdidx runs bounded incremental reclaim; it never runs an automatic full `VACUUM`. Progress/log output identifies the reclaim phase, CLI JSON returns `rebuild_reclaim`, and `status --json` retains the same object under `last_index_run.rebuild_reclaim`, including before/after logical database sizes, physical main-file samples, page/free-page counts and ratios, and reclaimed page/byte counts. The physical samples can remain unchanged until SQLite checkpoints WAL-backed pages; the logical sizes and page metrics are the immediate reclaim attribution. A skipped or failed reclaim leaves the committed index usable and reports a stable reason; use explicit `cdidx vacuum` for legacy databases or later retry. + On an actual full scan, the same timeline also separates `csharp_prepass`, `extraction`, `reference_graph`, `text_index`, `finalize`, and `commit`; file/commit-scoped updates report the shared extraction, graph, text-index, and finalization boundaries. Sample `elapsed_ms` values are cumulative, so subtract adjacent samples to attribute elapsed time without enabling a profiler. Index finalization reads reference-completeness metadata through the active writer transaction while preserving the issue-readiness state already established for that run, and validates cross-file hotspot-family readiness in one grouped pass over only the languages present in the index. Large mixed-language indexes therefore avoid repeated reader bootstraps and per-language correlated symbol scans before returning CLI or MCP completion output without treating degraded issue data as authoritative. @@ -4340,6 +4342,8 @@ cdidx index . --duration-format seconds `index --dry-run --rebuild` は full replacement scan を preview しますが既存 index を削除しないため、`--yes` の確認を要求しません。`--json --memory-trace` を追加すると、preview 自身から取得した `start`、`snapshot`、`scan`、`finalize` sample を含む `memory_timeline` を返します。dry-run は database snapshot と source file を読み取るだけで、workspace や DB/WAL/SHM set を変更しません。 +実際の `index --rebuild` が正常に commit された後、cdidx は `status --json` の maintenance guidance と同じ freelist ratio を確認します。ratio が `CDIDX_MAINTENANCE_FREELIST_WARN_RATIO`（既定 `0.20`）以上で、database が既に incremental auto-vacuum を使用している場合に限り、上限付きの incremental reclaim を実行し、自動の full `VACUUM` は実行しません。progress / log output は reclaim phase を示し、CLI JSON は `rebuild_reclaim`、`status --json` は同じ object を `last_index_run.rebuild_reclaim` に保持し、before / after の logical database size、物理 main-file sample、page / free-page count と ratio、回収 page / byte 数を含めます。WAL-backed page が SQLite により checkpoint されるまでは物理 sample が変わらないことがあり、即時の reclaim attribution には logical size と page metrics を使用します。reclaim が skip または失敗しても commit 済み index は利用可能なままで stable reason を報告します。legacy database や後からの再試行には明示的な `cdidx vacuum` を使用してください。 + 実際の full scan では、同じ timeline が `csharp_prepass`、`extraction`、`reference_graph`、`text_index`、`finalize`、`commit` も分離します。file/commit-scoped update は共通の extraction、graph、text-index、finalize 境界を返します。sample の `elapsed_ms` は累積値なので、隣接 sample の差分から profiler なしで所要時間を帰属できます。 index finalize は、その run で既に確定した issue-readiness state を維持しながら active writer transaction から reference completeness metadata を読み、cross-file hotspot-family readiness は index に実在する言語だけを対象に1回の grouped scan で検証します。これにより、degraded な issue data を authoritative と誤認せず、大規模な mixed-language index でも CLI / MCP の完了出力前に reader bootstrap や言語ごとの correlated symbol scan を繰り返しません。 diff --git a/changelog.d/unreleased/5057.fixed.md b/changelog.d/unreleased/5057.fixed.md new file mode 100644 index 000000000..7119014e1 --- /dev/null +++ b/changelog.d/unreleased/5057.fixed.md @@ -0,0 +1,33 @@ +--- +category: fixed +issues: + - 5057 +affected: + - README.md + - src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs + - src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs + - src/CodeIndex/Cli/JsonOutputContracts.cs + - src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs + - src/CodeIndex/Database/DbContext.SchemaMetadata.cs + - src/CodeIndex/Database/DbReader.FilesStatus.cs + - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs + - src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs + - src/CodeIndex/Mcp/McpToolOutputSchemas.cs + - src/CodeIndex/Models/QueryResults.cs + - tests/CodeIndex.Tests/DatabaseTests.cs + - tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs + - tests/CodeIndex.Tests/McpServerToolsCallTests.cs + - USER_GUIDE.md + - DEVELOPER_GUIDE.md + - TESTING_GUIDE.md + - AGENT_GUIDE.md +--- + +## English + +- **Successful rebuilds now reclaim excessive SQLite free pages without unconditional full vacuum (#5057, related to #1631)** — CLI and MCP rebuilds evaluate the existing maintenance threshold after commit, use bounded incremental vacuum only for compatible databases, preserve a usable index on skip/failure/interruption, and expose before/after logical database-size, page, ratio, duration, and reclaimed-byte telemetry in the index result and `last_index_run`. + +## 日本語 + +- **成功した rebuild が無条件の full vacuum なしで過剰な SQLite free page を回収するようになりました (#5057、#1631 関連)** — CLI / MCP rebuild は commit 後に既存の maintenance threshold を評価し、対応 database に限って上限付き incremental vacuum を使用します。skip / failure / interruption 時も利用可能な index を維持し、index result と `last_index_run` に before / after の logical database size、page、ratio、duration、回収 byte の telemetry を公開します。 diff --git a/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs b/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs index 5a8d4d4de..6344e0bbe 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.Diagnostics.cs @@ -150,6 +150,7 @@ private static void StampLastIndexRunMetadata( (DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, referenceExtractionCapHits == null ? null : JsonSerializer.Serialize(referenceExtractionCapHits, StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary)), + (DbContext.LastIndexRunRebuildReclaimMetaKey, null), (DbContext.LastIndexRunPeakMemoryMbMetaKey, memoryTimeline == null ? null : (memoryTimeline.PeakWorkingSetBytes / (1024 * 1024)).ToString(System.Globalization.CultureInfo.InvariantCulture))); @@ -158,6 +159,31 @@ private static void StampLastIndexRunMetadata( writer.ClearLastFailedIndexRunMetadata(); } + internal static bool TryStampRebuildReclaimMetadata( + DbWriter writer, + StatusRebuildReclaim rebuildReclaim, + long durationMs, + IndexMemoryTimelineJsonResult? memoryTimeline) + { + try + { + writer.SetMetaValues( + (DbContext.LastIndexRunDurationMsMetaKey, + durationMs.ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.LastIndexRunPeakMemoryMbMetaKey, memoryTimeline == null + ? null + : (memoryTimeline.PeakWorkingSetBytes / (1024 * 1024)).ToString(System.Globalization.CultureInfo.InvariantCulture)), + (DbContext.LastIndexRunRebuildReclaimMetaKey, + JsonSerializer.Serialize(rebuildReclaim, StatusMetadataJsonContext.Default.StatusRebuildReclaim))); + return true; + } + catch (Exception ex) + { + GlobalToolLog.Error("rebuild_reclaim_metadata_write_failed", ex, includeStacks: false); + return false; + } + } + internal static void StampLastIndexRunDiagnostics(DbWriter writer, IReadOnlyList? diagnostics) { var total = diagnostics?.Count ?? 0; diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs index abee04d94..2f835000b 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.Output.cs @@ -55,12 +55,11 @@ private sealed class FullScanFinalOutputContext internal string? PriorIndexedHeadCommit { get; init; } internal string? CurrentHeadCommit { get; init; } internal bool ShowNextSteps { get; init; } + internal StatusRebuildReclaim? RebuildReclaim { get; init; } } private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) { - if (output.Options.MemoryTrace) - output.MemorySamples.Add(CaptureMemorySample("commit", output.Stopwatch)); output.Stopwatch.Stop(); var memoryTimeline = BuildMemoryTimeline(output.MemorySamples); WarnIfMemoryThresholdExceeded(memoryTimeline); @@ -196,6 +195,7 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) FileErrors = output.FileErrorList.Count > 0 ? output.FileErrorList : null, Warnings = output.WarningList.Count > 0 ? output.WarningList : null, MemoryTimeline = memoryTimeline, + RebuildReclaim = output.RebuildReclaim, ElapsedMs = output.Stopwatch.ElapsedMilliseconds, }, output.JsonContext.IndexFullScanJsonResult)); } @@ -236,6 +236,13 @@ private static int WriteFullScanFinalOutput(FullScanFinalOutputContext output) CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("C# names", output.CSharpSymbolNameReadyAfter ? "ready" : "degraded", indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("C# meta", output.CSharpMetadataTargetReadyAfter ? "ready" : "degraded", indent: " ")); CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Fold", output.FoldReadyAfter ? "ready" : "degraded", indent: " ")); + if (output.RebuildReclaim is { State: "completed", BytesReclaimed: long reclaimedBytes }) + { + CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine( + "Reclaimed", + ConsoleUi.FormatBytes(reclaimedBytes), + indent: " ")); + } CommandOutputWriter.WriteLine(ConsoleUi.FormatSummaryLine("Elapsed", ConsoleUi.FormatDuration(output.Stopwatch.Elapsed, output.Options.DurationFormat), indent: " ")); CommandOutputWriter.WriteLine(); if (output.Errors > 0) diff --git a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs index a5bb01dda..9804120e8 100644 --- a/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs +++ b/src/CodeIndex/Cli/IndexCommandRunner.FullScan.cs @@ -1338,9 +1338,34 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis hotspotAggregateRefresh.Complete(cancellationToken); writer.ClearBatchInProgress(); fullScanTxn.Commit(); + if (options.MemoryTrace) + memorySamples.Add(CaptureMemorySample("commit", stopwatch)); if (referenceSecondaryIndexBulkLoad != null && willRebuildTypeScriptAugmentationAfterReadinessValidation) writer.ReportReferenceSecondaryIndexBulkLoadState("full_scan_committed"); + StatusRebuildReclaim? rebuildReclaim = null; + if (options.Rebuild && errors == 0) + { + WriteFullScanJsonLiveness(options, "evaluating rebuild free-page reclaim..."); + CancellationTokenSource? reclaimCts = null; + if (!options.Json && !options.Quiet) + reclaimCts = ConsoleUi.StartSpinner("Reclaiming rebuild free space...", spinnerFrames); + try + { + rebuildReclaim = db.RunRebuildReclaimIfRecommended(cancellationToken); + } + finally + { + ConsoleUi.StopSpinner(reclaimCts); + } + if (options.MemoryTrace) + memorySamples.Add(CaptureMemorySample("rebuild_reclaim", stopwatch)); + TryStampRebuildReclaimMetadata( + writer, + rebuildReclaim, + stopwatch.ElapsedMilliseconds, + BuildMemoryTimeline(memorySamples)); + } return WriteFullScanFinalOutput(new FullScanFinalOutputContext { Writer = writer, @@ -1388,6 +1413,7 @@ void RecordFullScanTargetStatSkip(int fileIndex, IndexedFileStatReuseResult exis PriorIndexedHeadCommit = priorIndexedHeadCommit, CurrentHeadCommit = currentHeadCommit, ShowNextSteps = showNextSteps, + RebuildReclaim = rebuildReclaim, }); } } diff --git a/src/CodeIndex/Cli/JsonOutputContracts.cs b/src/CodeIndex/Cli/JsonOutputContracts.cs index f76348c03..c7740beb5 100644 --- a/src/CodeIndex/Cli/JsonOutputContracts.cs +++ b/src/CodeIndex/Cli/JsonOutputContracts.cs @@ -1079,6 +1079,7 @@ internal sealed class IndexFullScanJsonResult : IVersionedJsonResult public List? FileErrors { get; init; } public List? Warnings { get; init; } public IndexMemoryTimelineJsonResult? MemoryTimeline { get; init; } + public StatusRebuildReclaim? RebuildReclaim { get; init; } public long ElapsedMs { get; init; } } diff --git a/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs b/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs index 0a85bc969..6acbc0f3e 100644 --- a/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs +++ b/src/CodeIndex/Database/DbContext.ConnectionLifecycle.cs @@ -3,6 +3,7 @@ using CodeIndex.Indexer; using CodeIndex.Models; using Microsoft.Data.Sqlite; +using System.Diagnostics; using System.Globalization; using System.Runtime.ExceptionServices; @@ -523,6 +524,145 @@ public VacuumResult RunIncrementalVacuum(bool dryRun, CancellationToken cancella MaintenanceGuidance: guidance); } + internal StatusRebuildReclaim RunRebuildReclaimIfRecommended(CancellationToken cancellationToken) + { + var started = Stopwatch.GetTimestamp(); + VacuumMetrics? before = null; + StatusMaintenanceGuidance? guidanceBefore = null; + try + { + cancellationToken.ThrowIfCancellationRequested(); + ReportMaintenanceProgress("rebuild_reclaim", "metrics_before", _connection.DataSource); + var beforeMetrics = ReadVacuumMetrics(); + before = beforeMetrics; + guidanceBefore = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( + beforeMetrics.PageCount, + beforeMetrics.FreelistCount, + beforeMetrics.PageSize, + beforeMetrics.WalSizeBytes, + beforeMetrics.DbSizeBytes, + beforeMetrics.AutoVacuumMode)); + + if (guidanceBefore.FreelistState != "vacuum_recommended") + { + return BuildRebuildReclaimResult( + state: "not_needed", + reason: "freelist_below_threshold", + beforeMetrics, + beforeMetrics, + guidanceBefore, + started); + } + + // Automatic rebuild maintenance must stay bounded to incremental auto-vacuum. + // Legacy databases continue to use the explicit `cdidx vacuum` full-VACUUM path. + // rebuild の自動 maintenance は incremental auto-vacuum に限定する。 + // legacy DB の full VACUUM は明示的な `cdidx vacuum` に残す。 + if (beforeMetrics.AutoVacuumMode != 2) + { + return BuildRebuildReclaimResult( + state: "skipped", + reason: "auto_vacuum_not_incremental", + beforeMetrics, + beforeMetrics, + guidanceBefore, + started); + } + + cancellationToken.ThrowIfCancellationRequested(); + ReportMaintenanceProgress("rebuild_reclaim", "incremental_vacuum", _connection.DataSource); + Execute(DbPragmaPolicy.IncrementalVacuumPragmaSql(beforeMetrics.FreelistCount)); + cancellationToken.ThrowIfCancellationRequested(); + ReportMaintenanceProgress("rebuild_reclaim", "metrics_after", _connection.DataSource); + var after = ReadVacuumMetrics(); + var guidanceAfter = MaintenanceGuidanceBuilder.Build(new MaintenanceMetrics( + after.PageCount, + after.FreelistCount, + after.PageSize, + after.WalSizeBytes, + after.DbSizeBytes, + after.AutoVacuumMode)); + var completed = guidanceAfter.FreelistState != "vacuum_recommended"; + return BuildRebuildReclaimResult( + state: completed ? "completed" : "incomplete", + reason: completed ? "threshold_exceeded" : "freelist_still_above_threshold", + beforeMetrics, + after, + guidanceBefore, + started, + guidanceAfter.FreelistRatio); + } + catch (Exception ex) + { + GlobalToolLog.Error("rebuild_reclaim_failed", ex, includeStacks: false); + return BuildRebuildReclaimResult( + state: ex is OperationCanceledException ? "cancelled" : "failed", + reason: ClassifyRebuildReclaimFailure(ex), + before, + after: null, + guidanceBefore, + started); + } + } + + private static StatusRebuildReclaim BuildRebuildReclaimResult( + string state, + string reason, + VacuumMetrics? before, + VacuumMetrics? after, + StatusMaintenanceGuidance? guidanceBefore, + long startedTimestamp, + double? freelistRatioAfter = null) + { + long? pagesReclaimed = before.HasValue && after.HasValue + ? Math.Max(0, before.Value.PageCount - after.Value.PageCount) + : null; + var pageSize = after?.PageSize ?? before?.PageSize; + return new StatusRebuildReclaim + { + State = state, + Reason = reason, + DurationMs = (long)Stopwatch.GetElapsedTime(startedTimestamp).TotalMilliseconds, + PageSizeBytes = pageSize, + PageCountBefore = before?.PageCount, + FreelistCountBefore = before?.FreelistCount, + FreelistRatioBefore = guidanceBefore?.FreelistRatio, + FreelistThresholdRatio = guidanceBefore?.FreelistThresholdRatio, + EstimatedBytesReclaimableBefore = guidanceBefore?.EstimatedBytesReclaimable, + PageCountAfter = after?.PageCount, + FreelistCountAfter = after?.FreelistCount, + FreelistRatioAfter = after.HasValue + ? freelistRatioAfter ?? guidanceBefore?.FreelistRatio + : null, + PagesReclaimed = pagesReclaimed, + BytesReclaimed = pagesReclaimed.HasValue && pageSize.HasValue + ? pagesReclaimed.Value * pageSize.Value + : null, + LogicalDatabaseBytesBefore = before.HasValue + ? before.Value.PageCount * before.Value.PageSize + : null, + LogicalDatabaseBytesAfter = after.HasValue + ? after.Value.PageCount * after.Value.PageSize + : null, + DbSizeBytesBefore = before?.DbSizeBytes, + DbSizeBytesAfter = after?.DbSizeBytes, + AutoVacuumMode = before?.AutoVacuumMode, + }; + } + + private static string ClassifyRebuildReclaimFailure(Exception exception) + => exception switch + { + OperationCanceledException => "cancelled", + SqliteException { SqliteErrorCode: 5 } => "sqlite_busy", + SqliteException { SqliteErrorCode: 6 } => "sqlite_locked", + SqliteException { SqliteErrorCode: 8 } => "sqlite_read_only", + SqliteException => "sqlite_error", + UnauthorizedAccessException => "access_denied", + IOException => "io_error", + _ => "unexpected_error", + }; + private static string? BuildWalCheckpointTimingNote(bool dryRun) => dryRun ? null diff --git a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs index df6203160..03600e29f 100644 --- a/src/CodeIndex/Database/DbContext.SchemaMetadata.cs +++ b/src/CodeIndex/Database/DbContext.SchemaMetadata.cs @@ -156,6 +156,7 @@ public static string GetDynamicReferenceGraphContractVersionMetaKey(string lang) public const string LastIndexRunDiagnosticCountMetaKey = "last_index_run_diagnostic_count"; public const string LastIndexRunDiagnosticsTruncatedMetaKey = "last_index_run_diagnostics_truncated"; public const string LastIndexRunReferenceExtractionCapHitsMetaKey = "last_index_run_reference_extraction_cap_hits_json"; + public const string LastIndexRunRebuildReclaimMetaKey = "last_index_run_rebuild_reclaim_json"; public const int LastIndexRunDiagnosticSampleLimit = 50; public const string LastFailedIndexRunStatusMetaKey = "last_failed_index_run_status"; public const string LastFailedIndexRunModeMetaKey = "last_failed_index_run_mode"; diff --git a/src/CodeIndex/Database/DbReader.FilesStatus.cs b/src/CodeIndex/Database/DbReader.FilesStatus.cs index 7300ffddd..22afb7ab8 100644 --- a/src/CodeIndex/Database/DbReader.FilesStatus.cs +++ b/src/CodeIndex/Database/DbReader.FilesStatus.cs @@ -2092,11 +2092,13 @@ private long ExecuteScalar(string sql) var diagnosticsTruncated = ParseMetaBool(TryGetMetaStringInternal(DbContext.LastIndexRunDiagnosticsTruncatedMetaKey)); var referenceExtractionCapHits = ParseReferenceExtractionCapHits( TryGetMetaStringInternal(DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey)); + var rebuildReclaim = ParseRebuildReclaim( + TryGetMetaStringInternal(DbContext.LastIndexRunRebuildReclaimMetaKey)); if (mode == null && startedAt == null && durationMs == null && filesScanned == null && filesSkipped == null && parseErrors == null && bytesRead == null && bytesReadSkippedFileCount == null && bytesReadIncomplete == null && rowsUpserted == null && rowsDeleted == null && peakMemoryMb == null && diagnostics == null && diagnosticCount == null && diagnosticsTruncated == null - && referenceExtractionCapHits == null) + && referenceExtractionCapHits == null && rebuildReclaim == null) { return null; } @@ -2119,9 +2121,24 @@ private long ExecuteScalar(string sql) DiagnosticCount = diagnosticCount, DiagnosticsTruncated = diagnosticsTruncated, ReferenceExtractionCapHits = referenceExtractionCapHits, + RebuildReclaim = rebuildReclaim, }; } + private static StatusRebuildReclaim? ParseRebuildReclaim(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + return null; + try + { + return JsonSerializer.Deserialize(json, StatusMetadataJsonContext.Default.StatusRebuildReclaim); + } + catch (JsonException) + { + return null; + } + } + private static ReferenceExtractionCapHitSummary? ParseReferenceExtractionCapHits(string? json) { if (string.IsNullOrWhiteSpace(json)) diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs index fb11337aa..d19328090 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Execution.cs @@ -2135,7 +2135,8 @@ await EmitProgressNotificationAsync( (DbContext.LastIndexRunRowsDeletedMetaKey, purged.ToString(System.Globalization.CultureInfo.InvariantCulture)), (DbContext.LastIndexRunReferenceExtractionCapHitsMetaKey, JsonSerializer.Serialize( referenceExtractionCapHits, - StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary))); + StatusMetadataJsonContext.Default.ReferenceExtractionCapHitSummary)), + (DbContext.LastIndexRunRebuildReclaimMetaKey, null)); writer.MarkIndexCompleteness(writer.GetPersistedIndexOmissionReasons()); writer.ClearLastFailedIndexRunMetadata(); // Persist the current HEAD only after the run is fully successful (errors == 0). @@ -2223,6 +2224,21 @@ await EmitProgressNotificationAsync( writer.ReportReferenceSecondaryIndexBulkLoadState("readiness_committed"); referenceSecondaryIndexBulkLoad?.Complete(requestToken); hotspotAggregateRefresh.Complete(requestToken); + StatusRebuildReclaim? rebuildReclaim = null; + if (rebuild && !scanResult.HadErrors && errors == 0) + { + await EmitProgressNotificationAsync( + progressToken, + files.Count, + files.Count, + "Evaluating rebuild free-page reclaim.").ConfigureAwait(false); + rebuildReclaim = db.RunRebuildReclaimIfRecommended(requestToken); + IndexCommandRunner.TryStampRebuildReclaimMetadata( + writer, + rebuildReclaim, + runStopwatch.ElapsedMilliseconds, + memoryTimeline: null); + } if (!scanResult.HadErrors && errors == 0) { var plannerMaintenanceFailure = db.RunPlannerStatisticsMaintenance( @@ -2267,6 +2283,7 @@ await EmitProgressNotificationAsync( csharpMetadataTargetReadyAfter, foldReadyAfter, foldReadyReason, + rebuildReclaim, memorySamples, failures, mcpIndexDiagnostics, diff --git a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs index 7f69ca393..c858a82be 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.Indexing.Results.cs @@ -35,6 +35,7 @@ private sealed record IndexCompletionDetails( bool CSharpMetadataTargetReady, bool FoldReady, string? FoldReadyReason, + StatusRebuildReclaim? RebuildReclaim, JsonArray? MemoryTrace, IReadOnlyList Failures, IReadOnlyList Diagnostics, @@ -145,6 +146,12 @@ private JsonNode BuildIndexCompletionResult(JsonNode? id, IndexCompletionDetails ["fold_ready"] = details.FoldReady, ["fold_ready_reason"] = details.FoldReadyReason }; + if (details.RebuildReclaim != null) + { + structured["rebuild_reclaim"] = System.Text.Json.JsonSerializer.SerializeToNode( + details.RebuildReclaim, + _jsonOptions); + } if (details.MemoryTrace != null) structured["memory_trace"] = details.MemoryTrace; if (details.Failures.Count > 0) diff --git a/src/CodeIndex/Mcp/McpToolOutputSchemas.cs b/src/CodeIndex/Mcp/McpToolOutputSchemas.cs index 20309755c..161e74a05 100644 --- a/src/CodeIndex/Mcp/McpToolOutputSchemas.cs +++ b/src/CodeIndex/Mcp/McpToolOutputSchemas.cs @@ -440,6 +440,7 @@ private static JsonObject IndexProperties() ["mode"] = StringSchema(), ["summary"] = ObjectSchema(), ["dry_run"] = BooleanSchema(), + ["rebuild_reclaim"] = ObjectSchema(), ["readiness"] = Reference("readiness"), }; diff --git a/src/CodeIndex/Models/QueryResults.cs b/src/CodeIndex/Models/QueryResults.cs index adebf036f..a0003c041 100644 --- a/src/CodeIndex/Models/QueryResults.cs +++ b/src/CodeIndex/Models/QueryResults.cs @@ -2107,6 +2107,69 @@ public sealed class StatusLastIndexRun public bool? DiagnosticsTruncated { get; set; } [JsonPropertyName("reference_extraction_cap_hits")] public ReferenceExtractionCapHitSummary? ReferenceExtractionCapHits { get; set; } + [JsonPropertyName("rebuild_reclaim")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public StatusRebuildReclaim? RebuildReclaim { get; set; } +} + +/// +/// Bounded telemetry for the thresholded free-page reclaim that follows a successful rebuild. +/// 成功した rebuild 後にしきい値付きで行う free-page 回収の bounded telemetry。 +/// +public sealed class StatusRebuildReclaim +{ + public string State { get; set; } = "not_needed"; + public string Reason { get; set; } = "freelist_below_threshold"; + [JsonPropertyName("duration_ms")] + public long DurationMs { get; set; } + [JsonPropertyName("page_size_bytes")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PageSizeBytes { get; set; } + [JsonPropertyName("page_count_before")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PageCountBefore { get; set; } + [JsonPropertyName("freelist_count_before")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? FreelistCountBefore { get; set; } + [JsonPropertyName("freelist_ratio_before")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? FreelistRatioBefore { get; set; } + [JsonPropertyName("freelist_threshold_ratio")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? FreelistThresholdRatio { get; set; } + [JsonPropertyName("estimated_bytes_reclaimable_before")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? EstimatedBytesReclaimableBefore { get; set; } + [JsonPropertyName("page_count_after")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PageCountAfter { get; set; } + [JsonPropertyName("freelist_count_after")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? FreelistCountAfter { get; set; } + [JsonPropertyName("freelist_ratio_after")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public double? FreelistRatioAfter { get; set; } + [JsonPropertyName("pages_reclaimed")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? PagesReclaimed { get; set; } + [JsonPropertyName("bytes_reclaimed")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? BytesReclaimed { get; set; } + [JsonPropertyName("logical_database_bytes_before")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? LogicalDatabaseBytesBefore { get; set; } + [JsonPropertyName("logical_database_bytes_after")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? LogicalDatabaseBytesAfter { get; set; } + [JsonPropertyName("db_size_bytes_before")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? DbSizeBytesBefore { get; set; } + [JsonPropertyName("db_size_bytes_after")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? DbSizeBytesAfter { get; set; } + [JsonPropertyName("auto_vacuum_mode")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public long? AutoVacuumMode { get; set; } } public sealed class StatusFailedOrPartialIndexRun @@ -2149,6 +2212,7 @@ public sealed class StatusIndexFileError [JsonSerializable(typeof(List))] [JsonSerializable(typeof(ReferenceExtractionCapHitSummary))] +[JsonSerializable(typeof(StatusRebuildReclaim))] internal sealed partial class StatusMetadataJsonContext : JsonSerializerContext { } diff --git a/tests/CodeIndex.Tests/DatabaseTests.cs b/tests/CodeIndex.Tests/DatabaseTests.cs index c4053e36c..d58a37fae 100644 --- a/tests/CodeIndex.Tests/DatabaseTests.cs +++ b/tests/CodeIndex.Tests/DatabaseTests.cs @@ -4768,6 +4768,183 @@ INSERT INTO vacuum_payload (payload) } } + [Fact] + public void RunRebuildReclaimIfRecommended_ReclaimsHighFreelistAndReportsBoundaries_Issue5057() + { + var dbDir = TestProjectHelper.CreateTempProject("codeindex_rebuild_reclaim"); + var dbPath = Path.Combine(dbDir, "codeindex.db"); + var progress = new List(); + try + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + db.InitializeSchema(); + using (var cmd = db.Connection.CreateCommand()) + { + cmd.CommandText = @" + CREATE TABLE rebuild_payload (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE n(value) AS ( + SELECT 1 + UNION ALL + SELECT value + 1 FROM n WHERE value < 256 + ) + INSERT INTO rebuild_payload (payload) + SELECT randomblob(4096) FROM n; + DELETE FROM rebuild_payload;"; + cmd.ExecuteNonQuery(); + } + DbContext.MaintenanceProgressForTesting = + (operation, phase) => progress.Add($"{operation}:{phase}"); + + var result = db.RunRebuildReclaimIfRecommended(CancellationToken.None); + + Assert.Equal("completed", result.State); + Assert.Equal("threshold_exceeded", result.Reason); + Assert.Equal(2, result.AutoVacuumMode); + Assert.NotNull(result.FreelistRatioBefore); + Assert.NotNull(result.FreelistThresholdRatio); + Assert.True(result.FreelistRatioBefore >= result.FreelistThresholdRatio); + Assert.NotNull(result.FreelistRatioAfter); + Assert.True(result.FreelistRatioAfter < result.FreelistThresholdRatio); + Assert.True(result.PagesReclaimed > 0); + Assert.True(result.BytesReclaimed > 0); + Assert.True(result.LogicalDatabaseBytesBefore > result.LogicalDatabaseBytesAfter); + Assert.Contains("rebuild_reclaim:metrics_before", progress); + Assert.Contains("rebuild_reclaim:incremental_vacuum", progress); + Assert.Contains("rebuild_reclaim:metrics_after", progress); + } + finally + { + DbContext.MaintenanceProgressForTesting = null; + SqliteConnection.ClearAllPools(); + TestProjectHelper.DeleteDirectory(dbDir); + } + } + + [Fact] + public void RunRebuildReclaimIfRecommended_PostReclaimMetricsFailureDoesNotFabricateAfterValues_Issue5057() + { + var dbDir = TestProjectHelper.CreateTempProject("codeindex_rebuild_reclaim_metrics_failure"); + var dbPath = Path.Combine(dbDir, "codeindex.db"); + try + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + db.InitializeSchema(); + using (var cmd = db.Connection.CreateCommand()) + { + cmd.CommandText = @" + CREATE TABLE rebuild_metrics_failure_payload (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE n(value) AS ( + SELECT 1 + UNION ALL + SELECT value + 1 FROM n WHERE value < 256 + ) + INSERT INTO rebuild_metrics_failure_payload (payload) + SELECT randomblob(4096) FROM n; + DELETE FROM rebuild_metrics_failure_payload;"; + cmd.ExecuteNonQuery(); + } + DbContext.MaintenanceProgressForTesting = (operation, phase) => + { + if (operation == "rebuild_reclaim" && phase == "metrics_after") + throw new IOException("injected post-reclaim metrics failure"); + }; + + var result = db.RunRebuildReclaimIfRecommended(CancellationToken.None); + + Assert.Equal("failed", result.State); + Assert.Equal("io_error", result.Reason); + Assert.NotNull(result.PageCountBefore); + Assert.Null(result.PageCountAfter); + Assert.Null(result.FreelistCountAfter); + Assert.Null(result.FreelistRatioAfter); + Assert.Null(result.PagesReclaimed); + Assert.Null(result.BytesReclaimed); + Assert.Null(result.LogicalDatabaseBytesAfter); + Assert.Null(result.DbSizeBytesAfter); + using var pageCountCommand = db.Connection.CreateCommand(); + pageCountCommand.CommandText = "PRAGMA page_count"; + Assert.True((long)pageCountCommand.ExecuteScalar()! < result.PageCountBefore); + } + finally + { + DbContext.MaintenanceProgressForTesting = null; + SqliteConnection.ClearAllPools(); + TestProjectHelper.DeleteDirectory(dbDir); + } + } + + [Fact] + public void RunRebuildReclaimIfRecommended_BelowThresholdDoesNotVacuum_Issue5057() + { + var dbDir = TestProjectHelper.CreateTempProject("codeindex_rebuild_reclaim_not_needed"); + var dbPath = Path.Combine(dbDir, "codeindex.db"); + try + { + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + db.InitializeSchema(); + + var result = db.RunRebuildReclaimIfRecommended(CancellationToken.None); + + Assert.Equal("not_needed", result.State); + Assert.Equal("freelist_below_threshold", result.Reason); + Assert.Equal(0, result.PagesReclaimed); + Assert.Equal(result.FreelistCountBefore, result.FreelistCountAfter); + } + finally + { + SqliteConnection.ClearAllPools(); + TestProjectHelper.DeleteDirectory(dbDir); + } + } + + [Fact] + public void RunRebuildReclaimIfRecommended_LegacyDatabaseSkipsAutomaticFullVacuum_Issue5057() + { + var dbDir = TestProjectHelper.CreateTempProject("codeindex_rebuild_reclaim_legacy"); + var dbPath = Path.Combine(dbDir, "codeindex.db"); + try + { + using (var legacyConnection = new SqliteConnection($"Data Source={dbPath};Pooling=False")) + { + legacyConnection.Open(); + using var legacyCommand = legacyConnection.CreateCommand(); + legacyCommand.CommandText = "CREATE TABLE legacy_marker (id INTEGER PRIMARY KEY)"; + legacyCommand.ExecuteNonQuery(); + } + + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + db.InitializeSchema(); + using (var cmd = db.Connection.CreateCommand()) + { + cmd.CommandText = @" + CREATE TABLE rebuild_legacy_payload (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE n(value) AS ( + SELECT 1 + UNION ALL + SELECT value + 1 FROM n WHERE value < 256 + ) + INSERT INTO rebuild_legacy_payload (payload) + SELECT randomblob(4096) FROM n; + DELETE FROM rebuild_legacy_payload;"; + cmd.ExecuteNonQuery(); + } + + var result = db.RunRebuildReclaimIfRecommended(CancellationToken.None); + + Assert.Equal("skipped", result.State); + Assert.Equal("auto_vacuum_not_incremental", result.Reason); + Assert.Equal(0, result.AutoVacuumMode); + Assert.Equal(result.PageCountBefore, result.PageCountAfter); + Assert.Equal(result.FreelistCountBefore, result.FreelistCountAfter); + Assert.Equal(0, result.PagesReclaimed); + } + finally + { + SqliteConnection.ClearAllPools(); + TestProjectHelper.DeleteDirectory(dbDir); + } + } + [Fact] public void RunIncrementalVacuum_CancellationBeforeMetrics_ThrowsOperationCanceled_Issue3811() { diff --git a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs index 4b88767ed..90a59b85d 100644 --- a/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs +++ b/tests/CodeIndex.Tests/IndexCommandRunnerFullScanTests.cs @@ -18,6 +18,206 @@ namespace CodeIndex.Tests; public partial class IndexCommandRunnerTests { + [Fact] + public void Run_RebuildReclaimsHighFreelistWithConcurrentReaderAndPersistsTelemetry_Issue5057() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText( + Path.Combine(projectRoot, "app.cs"), + "public class App { public string Run() => \"ready\"; }\n"); + var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var connection = OpenNonPoolingConnection(dbPath)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = @" + CREATE TABLE rebuild_payload (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE n(value) AS ( + SELECT 1 + UNION ALL + SELECT value + 1 FROM n WHERE value < 256 + ) + INSERT INTO rebuild_payload (payload) + SELECT randomblob(4096) FROM n; + DELETE FROM rebuild_payload;"; + command.ExecuteNonQuery(); + } + + using var readerConnection = OpenNonPoolingConnection(dbPath); + readerConnection.Open(); + using var readerCommand = readerConnection.CreateCommand(); + readerCommand.CommandText = "SELECT COUNT(*) FROM files"; + Assert.Equal(1L, (long)readerCommand.ExecuteScalar()!); + + var (rebuildExitCode, rebuildJson) = RunAndCaptureJson( + [projectRoot, "--rebuild", "--yes", "--json", "--quiet", "--memory-trace"]); + + Assert.Equal(CommandExitCodes.Success, rebuildExitCode); + Assert.Equal(1L, (long)readerCommand.ExecuteScalar()!); + var memoryPhases = rebuildJson + .GetProperty("memory_timeline") + .GetProperty("samples") + .EnumerateArray() + .Select(sample => sample.GetProperty("phase").GetString()) + .ToArray(); + Assert.Equal(["commit", "rebuild_reclaim"], memoryPhases[^2..]); + var reclaim = rebuildJson.GetProperty("rebuild_reclaim"); + Assert.Equal("completed", reclaim.GetProperty("state").GetString()); + Assert.Equal("threshold_exceeded", reclaim.GetProperty("reason").GetString()); + Assert.True(reclaim.GetProperty("freelist_ratio_before").GetDouble() + >= reclaim.GetProperty("freelist_threshold_ratio").GetDouble()); + Assert.True(reclaim.GetProperty("freelist_ratio_after").GetDouble() + < reclaim.GetProperty("freelist_threshold_ratio").GetDouble()); + Assert.True(reclaim.GetProperty("pages_reclaimed").GetInt64() > 0); + Assert.True(reclaim.GetProperty("bytes_reclaimed").GetInt64() > 0); + Assert.True( + reclaim.GetProperty("logical_database_bytes_before").GetInt64() + > reclaim.GetProperty("logical_database_bytes_after").GetInt64()); + + var (statusExitCode, statusJson) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, statusExitCode); + var statusReclaim = statusJson.GetProperty("last_index_run").GetProperty("rebuild_reclaim"); + Assert.Equal("completed", statusReclaim.GetProperty("state").GetString()); + Assert.Equal( + reclaim.GetProperty("pages_reclaimed").GetInt64(), + statusReclaim.GetProperty("pages_reclaimed").GetInt64()); + Assert.Equal( + reclaim.GetProperty("logical_database_bytes_after").GetInt64(), + statusReclaim.GetProperty("logical_database_bytes_after").GetInt64()); + Assert.NotEqual( + "vacuum_recommended", + statusJson.GetProperty("maintenance_guidance").GetProperty("freelist_state").GetString()); + + using var db = new DbContext(DbOpenIntent.WriteIndex, dbPath); + using var integrityCommand = db.Connection.CreateCommand(); + integrityCommand.CommandText = "PRAGMA integrity_check"; + Assert.Equal("ok", integrityCommand.ExecuteScalar()); + var explicitDryRun = db.RunIncrementalVacuum(dryRun: true); + Assert.Equal("dry_run", explicitDryRun.Status); + Assert.Equal( + statusReclaim.GetProperty("freelist_count_after").GetInt64(), + explicitDryRun.FreelistCountBefore); + } + finally + { + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_RebuildReclaimFailureKeepsCommittedDatabaseUsable_Issue5057() + { + var projectRoot = CreateTempProject(); + try + { + File.WriteAllText(Path.Combine(projectRoot, "app.cs"), "public class App { }\n"); + var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + using (var connection = OpenNonPoolingConnection(dbPath)) + { + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = @" + CREATE TABLE rebuild_failure_payload (id INTEGER PRIMARY KEY, payload BLOB); + WITH RECURSIVE n(value) AS ( + SELECT 1 + UNION ALL + SELECT value + 1 FROM n WHERE value < 256 + ) + INSERT INTO rebuild_failure_payload (payload) + SELECT randomblob(4096) FROM n; + DELETE FROM rebuild_failure_payload;"; + command.ExecuteNonQuery(); + } + DbContext.MaintenanceProgressForTesting = (operation, phase) => + { + if (operation == "rebuild_reclaim" && phase == "incremental_vacuum") + throw new InvalidOperationException("injected rebuild reclaim failure"); + }; + + var (rebuildExitCode, rebuildJson) = RunAndCaptureJson( + [projectRoot, "--rebuild", "--yes", "--json", "--quiet"]); + + Assert.Equal(CommandExitCodes.Success, rebuildExitCode); + var reclaim = rebuildJson.GetProperty("rebuild_reclaim"); + Assert.Equal("failed", reclaim.GetProperty("state").GetString()); + Assert.Equal("unexpected_error", reclaim.GetProperty("reason").GetString()); + using var connectionAfter = OpenNonPoolingConnection(dbPath); + connectionAfter.Open(); + using var integrityCommand = connectionAfter.CreateCommand(); + integrityCommand.CommandText = "PRAGMA integrity_check"; + Assert.Equal("ok", integrityCommand.ExecuteScalar()); + using var countCommand = connectionAfter.CreateCommand(); + countCommand.CommandText = "SELECT COUNT(*) FROM files"; + Assert.Equal(1L, (long)countCommand.ExecuteScalar()!); + + var (statusExitCode, statusJson) = RunStatusAndCaptureJson(["--db", dbPath, "--json"]); + Assert.Equal(CommandExitCodes.Success, statusExitCode); + Assert.Equal( + "failed", + statusJson.GetProperty("last_index_run") + .GetProperty("rebuild_reclaim") + .GetProperty("state") + .GetString()); + Assert.Equal( + "vacuum_recommended", + statusJson.GetProperty("maintenance_guidance").GetProperty("freelist_state").GetString()); + } + finally + { + DbContext.MaintenanceProgressForTesting = null; + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + + [Fact] + public void Run_InterruptedRebuildPreservesPreviouslyCommittedDatabase_Issue5057() + { + var projectRoot = CreateTempProject(); + using var cancellation = new CancellationTokenSource(); + try + { + var sourcePath = Path.Combine(projectRoot, "app.cs"); + File.WriteAllText(sourcePath, "public class App { public int Value => 1; }\n"); + var (initialExitCode, _) = RunAndCaptureJson([projectRoot, "--json", "--quiet"]); + Assert.Equal(CommandExitCodes.Success, initialExitCode); + var dbPath = Path.Combine(projectRoot, ".cdidx", "codeindex.db"); + var checksumBefore = ReadIndexedChecksum(dbPath, "app.cs"); + Assert.NotNull(checksumBefore); + + File.WriteAllText(sourcePath, "public class App { public int Value => 2; }\n"); + IndexCommandRunner.FullScanFtsOptimizeForTesting = cancellation.Cancel; + + var (rebuildExitCode, rebuildJson) = RunAndCaptureJson( + [projectRoot, "--rebuild", "--yes", "--json", "--quiet"], + cancellation); + + Assert.Equal(CommandExitCodes.Interrupted, rebuildExitCode); + Assert.Equal(CommandErrorCodes.Interrupted, rebuildJson.GetProperty("error_code").GetString()); + Assert.Equal(checksumBefore, ReadIndexedChecksum(dbPath, "app.cs")); + using var connection = OpenNonPoolingConnection(dbPath); + connection.Open(); + using var integrityCommand = connection.CreateCommand(); + integrityCommand.CommandText = "PRAGMA integrity_check"; + Assert.Equal("ok", integrityCommand.ExecuteScalar()); + } + finally + { + IndexCommandRunner.FullScanFtsOptimizeForTesting = null; + SqliteConnection.ClearAllPools(); + DeleteDirectory(projectRoot); + } + } + [Fact] public void Run_FullScanAndScopedUpdateUseGuardedAtomicFileReferenceScope() { diff --git a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs index c5aac201c..ba7fb4536 100644 --- a/tests/CodeIndex.Tests/McpServerToolsCallTests.cs +++ b/tests/CodeIndex.Tests/McpServerToolsCallTests.cs @@ -8320,6 +8320,11 @@ public void ToolsCall_Index_RebuildUsesFullTypeScriptAugmentationPath() var rebuildResponse = CallIndex(server, fixtureDir, args => args["rebuild"] = true); Assert.False(rebuildResponse["result"]?["isError"]?.GetValue() ?? false, rebuildResponse.ToJsonString()); + var rebuildReclaim = rebuildResponse["result"]?["structuredContent"]?["rebuild_reclaim"]; + Assert.NotNull(rebuildReclaim); + Assert.Contains( + rebuildReclaim!["state"]!.GetValue(), + new[] { "completed", "not_needed" }); Assert.NotNull(groupingStats); Assert.Equal(2, groupingStats!.DeclarationCount); Assert.Null(groupingStats.ScopedNameCount);