From 6e855b93970622717422f544dbe61e92c045d22b Mon Sep 17 00:00:00 2001 From: 404-Page-Found Date: Tue, 28 Jul 2026 18:12:28 +1000 Subject: [PATCH 1/6] perf(history): limit parsing to recent entries Read JSONL history from the end and stop once enough valid entries are found. Closes #206 --- src/history/store.ts | 74 ++++++++++++++++++++++--------- tests/history-corruption.test.mjs | 16 +++++++ 2 files changed, 70 insertions(+), 20 deletions(-) diff --git a/src/history/store.ts b/src/history/store.ts index a4d0f2f..91acf23 100644 --- a/src/history/store.ts +++ b/src/history/store.ts @@ -12,6 +12,7 @@ const HISTORY_LOCK_STALE_MS = 60_000; const HISTORY_LOCK_TIMEOUT_MS = HISTORY_LOCK_STALE_MS + 10_000; const HISTORY_LOCK_HEARTBEAT_MS = 15_000; const HISTORY_LOCK_TAKEOVER_SUFFIX = '.takeover'; +const HISTORY_READ_CHUNK_SIZE = 64 * 1024; function hasErrorCode(error: unknown, code: string): boolean { return typeof error === 'object' && error !== null && 'code' in error && error.code === code; @@ -210,25 +211,58 @@ export async function loadEntries(limit = 200): Promise { const historyPath = getHistoryPath(); if (!existsSync(historyPath)) return []; - const raw = await readFile(historyPath, 'utf-8'); - const lines = raw - .split('\n') - .map((line, index) => ({ line, lineNumber: index + 1 })) - .filter(({ line }) => line.trim().length > 0) - .reverse(); - const entries: CommitEntry[] = []; - const corruptedLineNumbers: number[] = []; + const corruptedLineNumbers: Array = []; + const history = await open(historyPath, 'r'); - for (const { line, lineNumber } of lines) { - try { - const entry = JSON.parse(line) as CommitEntry; - if (entries.length < limit) { - entries.push(entry); + try { + const { size } = await history.stat(); + let position = size; + let remainder = ''; + let processedLineCount = 0; + let totalLineCount: number | undefined; + const corruptedLineIndexesFromEnd: number[] = []; + + const parseLine = (line: string) => { + const lineIndexFromEnd = processedLineCount++; + if (line.trim().length === 0) return; + + try { + entries.push(JSON.parse(line) as CommitEntry); + } catch { + corruptedLineIndexesFromEnd.push(lineIndexFromEnd); + } + }; + + while (position > 0 && entries.length < limit) { + const bytesToRead = Math.min(HISTORY_READ_CHUNK_SIZE, position); + position -= bytesToRead; + const buffer = Buffer.allocUnsafe(bytesToRead); + const { bytesRead } = await history.read(buffer, 0, bytesToRead, position); + const lines = (buffer.toString('utf-8', 0, bytesRead) + remainder).split('\n'); + remainder = lines.shift()!; + const processedBeforeChunk = processedLineCount; + + if (position === 0) { + totalLineCount = processedBeforeChunk + lines.length + 1; + } + + for (let index = lines.length - 1; index >= 0 && entries.length < limit; index--) { + parseLine(lines[index]!); } - } catch { - corruptedLineNumbers.push(lineNumber); } + + if (position === 0 && entries.length < limit) { + parseLine(remainder); + } + + for (const lineIndexFromEnd of corruptedLineIndexesFromEnd) { + corruptedLineNumbers.push( + totalLineCount === undefined ? undefined : totalLineCount - lineIndexFromEnd, + ); + } + } finally { + await history.close(); } warnCorruptedHistory(historyPath, corruptedLineNumbers); @@ -237,20 +271,20 @@ export async function loadEntries(limit = 200): Promise { } /** Emit a compact warning that identifies corrupted history line numbers. */ -function warnCorruptedHistory(historyPath: string, corruptedLineNumbers: number[]): void { +function warnCorruptedHistory(historyPath: string, corruptedLineNumbers: Array): void { if (corruptedLineNumbers.length === 0) return; const count = corruptedLineNumbers.length; const noun = count === 1 ? 'entry' : 'entries'; - const lines = corruptedLineNumbers + const knownLineNumbers = corruptedLineNumbers.filter((lineNumber): lineNumber is number => lineNumber !== undefined); + const lines = knownLineNumbers .slice(0, 5) .sort((a, b) => a - b) .join(', '); const suffix = count > 5 ? `, +${count - 5} more` : ''; + const location = knownLineNumbers.length === count ? `line ${lines}${suffix}` : 'recently scanned rows'; - console.warn( - `Warning: ignored ${count} corrupted commit history ${noun} in ${historyPath} (line ${lines}${suffix}).`, - ); + console.warn(`Warning: ignored ${count} corrupted commit history ${noun} in ${historyPath} (${location}).`); } /** Count raw history rows in the configured history file. */ diff --git a/tests/history-corruption.test.mjs b/tests/history-corruption.test.mjs index 08ab8c7..1da53fd 100644 --- a/tests/history-corruption.test.mjs +++ b/tests/history-corruption.test.mjs @@ -137,6 +137,22 @@ test('loadEntries warns and returns empty entries when all lines are corrupted', }); }); +test('loadEntries stops parsing once it has enough recent valid entries', async () => { + await withIsolatedHistory( + [ + '{invalid old line', + validEntry('fix: keep older valid entry', '2026-06-01T00:00:00Z'), + validEntry('feat: keep newest valid entry', '2026-06-01T00:00:01Z'), + ], + async ({ warnings }) => { + const entries = await loadEntries(1); + + assert.deepEqual(entries.map((entry) => entry.message), ['feat: keep newest valid entry']); + assert.deepEqual(warnings, []); + }, + ); +}); + test('countEntries counts raw non-empty history rows, including malformed JSON lines', async () => { await withIsolatedHistory( [ From 6c4e6a44ed5868fe3185fb5d46783cdafef0eade Mon Sep 17 00:00:00 2001 From: 404-Page-Found Date: Tue, 28 Jul 2026 18:18:36 +1000 Subject: [PATCH 2/6] fix(history): preserve UTF-8 at chunk boundaries --- src/history/store.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/history/store.ts b/src/history/store.ts index 91acf23..1ea0b8e 100644 --- a/src/history/store.ts +++ b/src/history/store.ts @@ -235,8 +235,10 @@ export async function loadEntries(limit = 200): Promise { }; while (position > 0 && entries.length < limit) { - const bytesToRead = Math.min(HISTORY_READ_CHUNK_SIZE, position); - position -= bytesToRead; + // Read a few extra bytes so the next chunk begins before any UTF-8 code point it intersects. + const chunkEnd = position; + position = Math.max(0, chunkEnd - HISTORY_READ_CHUNK_SIZE - 3); + const bytesToRead = chunkEnd - position; const buffer = Buffer.allocUnsafe(bytesToRead); const { bytesRead } = await history.read(buffer, 0, bytesToRead, position); const lines = (buffer.toString('utf-8', 0, bytesRead) + remainder).split('\n'); From 4d4f04ffa6634a97f0c384330969bcc91b7eef21 Mon Sep 17 00:00:00 2001 From: 404-Page-Found Date: Tue, 28 Jul 2026 18:28:38 +1000 Subject: [PATCH 3/6] test: verify loadEntries handles corrupted rows during partial scan Add test to ensure that when loadEntries performs a partial scan and encounters a corrupted entry at the end of the history, it correctly returns only valid entries, reports a single warning, and uses a generic location message referencing "recently scanned rows" instead of a specific file location. --- tests/history-corruption.test.mjs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/history-corruption.test.mjs b/tests/history-corruption.test.mjs index 1da53fd..d0431d4 100644 --- a/tests/history-corruption.test.mjs +++ b/tests/history-corruption.test.mjs @@ -153,6 +153,24 @@ test('loadEntries stops parsing once it has enough recent valid entries', async ); }); +test('loadEntries uses a generic location for corrupted rows in a partial scan', async () => { + const olderEntries = Array.from({ length: 700 }, (_, index) => + validEntry(`chore: keep older entry ${index}`, `2026-06-01T00:00:${String(index).padStart(2, '0')}Z`), + ); + + await withIsolatedHistory( + [...olderEntries, validEntry('fix: keep newest valid entry', '2026-06-01T00:01:00Z'), '{invalid newest line'], + async ({ warnings }) => { + const entries = await loadEntries(1); + + assert.deepEqual(entries.map((entry) => entry.message), ['fix: keep newest valid entry']); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /ignored 1 corrupted commit history entry/); + assert.match(warnings[0], /recently scanned rows/); + }, + ); +}); + test('countEntries counts raw non-empty history rows, including malformed JSON lines', async () => { await withIsolatedHistory( [ From 2e3378cac250f1ee5886ca83468ac1fac5296472 Mon Sep 17 00:00:00 2001 From: 404-Page-Found Date: Tue, 28 Jul 2026 19:05:18 +1000 Subject: [PATCH 4/6] fix: preserve UTF-8 characters when reading history entries backwards The previous implementation split chunks by converting to string and concatenating a remainder, which could break multi-byte UTF-8 code points. This caused corrupted characters (e.g., replacement character U+FFFD) in commit messages. Changed to operate on Buffer objects, splitting on newline bytes (0x0a) and keeping the remainder as a Buffer. This ensures multi-byte sequences are never split across chunk boundaries. Added a test that verifies a long history row containing multi-byte characters is correctly loaded without corruption. --- src/history/store.ts | 28 ++++++++++++++++++++-------- tests/history-corruption.test.mjs | 15 +++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/history/store.ts b/src/history/store.ts index 1ea0b8e..9760a51 100644 --- a/src/history/store.ts +++ b/src/history/store.ts @@ -218,13 +218,14 @@ export async function loadEntries(limit = 200): Promise { try { const { size } = await history.stat(); let position = size; - let remainder = ''; + let remainder = Buffer.alloc(0); let processedLineCount = 0; let totalLineCount: number | undefined; const corruptedLineIndexesFromEnd: number[] = []; - const parseLine = (line: string) => { + const parseLine = (lineBytes: Buffer) => { const lineIndexFromEnd = processedLineCount++; + const line = lineBytes.toString('utf-8'); if (line.trim().length === 0) return; try { @@ -235,14 +236,27 @@ export async function loadEntries(limit = 200): Promise { }; while (position > 0 && entries.length < limit) { - // Read a few extra bytes so the next chunk begins before any UTF-8 code point it intersects. const chunkEnd = position; position = Math.max(0, chunkEnd - HISTORY_READ_CHUNK_SIZE - 3); const bytesToRead = chunkEnd - position; const buffer = Buffer.allocUnsafe(bytesToRead); const { bytesRead } = await history.read(buffer, 0, bytesToRead, position); - const lines = (buffer.toString('utf-8', 0, bytesRead) + remainder).split('\n'); - remainder = lines.shift()!; + const chunk = Buffer.concat([buffer.subarray(0, bytesRead), remainder]); + const lines: Buffer[] = []; + const firstLineEnd = chunk.indexOf(0x0a); + if (firstLineEnd === -1) { + remainder = chunk; + } else { + remainder = chunk.subarray(0, firstLineEnd); + let lineStart = firstLineEnd + 1; + for (let index = lineStart; index < chunk.length; index++) { + if (chunk[index] === 0x0a) { + lines.push(chunk.subarray(lineStart, index)); + lineStart = index + 1; + } + } + lines.push(chunk.subarray(lineStart)); + } const processedBeforeChunk = processedLineCount; if (position === 0) { @@ -259,9 +273,7 @@ export async function loadEntries(limit = 200): Promise { } for (const lineIndexFromEnd of corruptedLineIndexesFromEnd) { - corruptedLineNumbers.push( - totalLineCount === undefined ? undefined : totalLineCount - lineIndexFromEnd, - ); + corruptedLineNumbers.push(totalLineCount === undefined ? undefined : totalLineCount - lineIndexFromEnd); } } finally { await history.close(); diff --git a/tests/history-corruption.test.mjs b/tests/history-corruption.test.mjs index d0431d4..29b0352 100644 --- a/tests/history-corruption.test.mjs +++ b/tests/history-corruption.test.mjs @@ -185,3 +185,18 @@ test('countEntries counts raw non-empty history rows, including malformed JSON l }, ); }); + +test('loadEntries preserves UTF-8 characters split across backward-read chunks', async () => { + const prefix = '{"timestamp":"2026-06-01T00:00:00Z","message":"a'; + const suffix = '","diff":"","model":"test-model","provider":"local"}'; + const messageTailLength = 65535 - Buffer.byteLength(suffix) - 1; + const row = `${prefix}\u00e9x\u00e9${'b'.repeat(messageTailLength)}${suffix}`; + + await withIsolatedHistory([row], async () => { + const entries = await loadEntries(1); + + assert.equal(entries.length, 1); + assert.equal(entries[0].message, `a\u00e9x\u00e9${'b'.repeat(messageTailLength)}`); + assert.doesNotMatch(entries[0].message, /\uFFFD/); + }); +}); From 0a971b6318fe6d96312dafbbf0f4009525c9dfec Mon Sep 17 00:00:00 2001 From: 404-Page-Found Date: Tue, 28 Jul 2026 19:48:04 +1000 Subject: [PATCH 5/6] perf: defer Buffer.concat in loadEntries to reduce memory allocations Refactor the backward reading logic in `loadEntries` to collect remainder chunks in an array (`remainderChunks`) instead of concatenating them with each new chunk. This avoids repeated `Buffer.concat` calls and reduces memory allocations, improving performance when reading large history files. Update the UTF-8 split test to use a 2MB message and verify that only one `Buffer.concat` call is made during the entire read operation. --- src/history/store.ts | 29 ++++++++++++++++------------- tests/history-corruption.test.mjs | 23 ++++++++++++++++++----- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/history/store.ts b/src/history/store.ts index 9760a51..c6490b3 100644 --- a/src/history/store.ts +++ b/src/history/store.ts @@ -218,7 +218,7 @@ export async function loadEntries(limit = 200): Promise { try { const { size } = await history.stat(); let position = size; - let remainder = Buffer.alloc(0); + let remainderChunks: Buffer[] = []; let processedLineCount = 0; let totalLineCount: number | undefined; const corruptedLineIndexesFromEnd: number[] = []; @@ -241,22 +241,24 @@ export async function loadEntries(limit = 200): Promise { const bytesToRead = chunkEnd - position; const buffer = Buffer.allocUnsafe(bytesToRead); const { bytesRead } = await history.read(buffer, 0, bytesToRead, position); - const chunk = Buffer.concat([buffer.subarray(0, bytesRead), remainder]); - const lines: Buffer[] = []; + const chunk = buffer.subarray(0, bytesRead); const firstLineEnd = chunk.indexOf(0x0a); if (firstLineEnd === -1) { - remainder = chunk; - } else { - remainder = chunk.subarray(0, firstLineEnd); - let lineStart = firstLineEnd + 1; - for (let index = lineStart; index < chunk.length; index++) { - if (chunk[index] === 0x0a) { - lines.push(chunk.subarray(lineStart, index)); - lineStart = index + 1; - } + remainderChunks.unshift(chunk); + continue; + } + + const combined = remainderChunks.length === 0 ? chunk : Buffer.concat([chunk, ...remainderChunks]); + remainderChunks = [combined.subarray(0, firstLineEnd)]; + const lines: Buffer[] = []; + let lineStart = firstLineEnd + 1; + for (let index = lineStart; index < combined.length; index++) { + if (combined[index] === 0x0a) { + lines.push(combined.subarray(lineStart, index)); + lineStart = index + 1; } - lines.push(chunk.subarray(lineStart)); } + lines.push(combined.subarray(lineStart)); const processedBeforeChunk = processedLineCount; if (position === 0) { @@ -269,6 +271,7 @@ export async function loadEntries(limit = 200): Promise { } if (position === 0 && entries.length < limit) { + const remainder = remainderChunks.length === 1 ? remainderChunks[0]! : Buffer.concat(remainderChunks); parseLine(remainder); } diff --git a/tests/history-corruption.test.mjs b/tests/history-corruption.test.mjs index 29b0352..ee30f80 100644 --- a/tests/history-corruption.test.mjs +++ b/tests/history-corruption.test.mjs @@ -189,14 +189,27 @@ test('countEntries counts raw non-empty history rows, including malformed JSON l test('loadEntries preserves UTF-8 characters split across backward-read chunks', async () => { const prefix = '{"timestamp":"2026-06-01T00:00:00Z","message":"a'; const suffix = '","diff":"","model":"test-model","provider":"local"}'; - const messageTailLength = 65535 - Buffer.byteLength(suffix) - 1; + const messageTailLength = 2 * 1024 * 1024 - Buffer.byteLength(suffix) - 1; const row = `${prefix}\u00e9x\u00e9${'b'.repeat(messageTailLength)}${suffix}`; + const originalConcat = Buffer.concat; + let concatCalls = 0; + Buffer.concat = (...args) => { + concatCalls += 1; + return originalConcat(...args); + }; + await withIsolatedHistory([row], async () => { - const entries = await loadEntries(1); + try { + const entries = await loadEntries(1); - assert.equal(entries.length, 1); - assert.equal(entries[0].message, `a\u00e9x\u00e9${'b'.repeat(messageTailLength)}`); - assert.doesNotMatch(entries[0].message, /\uFFFD/); + assert.equal(entries.length, 1); + assert.equal(entries[0].message, `a\u00e9x\u00e9${'b'.repeat(messageTailLength)}`); + assert.doesNotMatch(entries[0].message, /\uFFFD/); + } finally { + Buffer.concat = originalConcat; + } }); + + assert.equal(concatCalls, 1); }); From 4b1d68ea79e67d9e38c146ac062e161e392b25f0 Mon Sep 17 00:00:00 2001 From: 404-Page-Found Date: Tue, 28 Jul 2026 20:30:27 +1000 Subject: [PATCH 6/6] feat(history): add readHistoryChunk to handle short reads from filesystem The previous `history.read` call in `loadEntries` could return fewer bytes than requested, potentially causing incomplete chunk reads. Added `readHistoryChunk` which retries reading from the unread offset until all requested bytes are read or EOF is reached. Updated `loadEntries` to use the new function and exported it for testing. Added a unit test to verify retry behavior. --- src/history/store.ts | 20 +++++++++++++++++++- tests/history-corruption.test.mjs | 31 ++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/history/store.ts b/src/history/store.ts index c6490b3..6f8a1d8 100644 --- a/src/history/store.ts +++ b/src/history/store.ts @@ -14,6 +14,24 @@ const HISTORY_LOCK_HEARTBEAT_MS = 15_000; const HISTORY_LOCK_TAKEOVER_SUFFIX = '.takeover'; const HISTORY_READ_CHUNK_SIZE = 64 * 1024; +/** Read a complete history chunk, retrying when the filesystem returns a short read. */ +export async function readHistoryChunk( + history: Pick, + buffer: Buffer, + position: number, + bytesToRead: number, +): Promise { + let bytesRead = 0; + + while (bytesRead < bytesToRead) { + const result = await history.read(buffer, bytesRead, bytesToRead - bytesRead, position + bytesRead); + if (result.bytesRead === 0) break; + bytesRead += result.bytesRead; + } + + return bytesRead; +} + function hasErrorCode(error: unknown, code: string): boolean { return typeof error === 'object' && error !== null && 'code' in error && error.code === code; } @@ -240,7 +258,7 @@ export async function loadEntries(limit = 200): Promise { position = Math.max(0, chunkEnd - HISTORY_READ_CHUNK_SIZE - 3); const bytesToRead = chunkEnd - position; const buffer = Buffer.allocUnsafe(bytesToRead); - const { bytesRead } = await history.read(buffer, 0, bytesToRead, position); + const bytesRead = await readHistoryChunk(history, buffer, position, bytesToRead); const chunk = buffer.subarray(0, bytesRead); const firstLineEnd = chunk.indexOf(0x0a); if (firstLineEnd === -1) { diff --git a/tests/history-corruption.test.mjs b/tests/history-corruption.test.mjs index ee30f80..6d2f234 100644 --- a/tests/history-corruption.test.mjs +++ b/tests/history-corruption.test.mjs @@ -5,7 +5,7 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { getConfigDir } from '../dist/config/store.js'; -import { countEntries, loadEntries } from '../dist/history/store.js'; +import { countEntries, loadEntries, readHistoryChunk } from '../dist/history/store.js'; function writeHistory(lines) { const configDir = getConfigDir(); @@ -31,6 +31,35 @@ function validEntry(message, timestamp) { }); } +test('readHistoryChunk retries short reads from the unread offset', async () => { + const source = Buffer.from('history entry with a short read'); + const destination = Buffer.alloc(source.length); + const reads = []; + const history = { + async read(buffer, offset, length, position) { + reads.push({ offset, length, position }); + const bytesToCopy = Math.min(5, length); + source.copy(buffer, offset, position, position + bytesToCopy); + return { bytesRead: bytesToCopy, buffer }; + }, + }; + + assert.equal(await readHistoryChunk(history, destination, 0, source.length), source.length); + assert.deepEqual(destination, source); + assert.deepEqual( + reads.map(({ offset, position }) => ({ offset, position })), + [ + { offset: 0, position: 0 }, + { offset: 5, position: 5 }, + { offset: 10, position: 10 }, + { offset: 15, position: 15 }, + { offset: 20, position: 20 }, + { offset: 25, position: 25 }, + { offset: 30, position: 30 }, + ], + ); +}); + async function withIsolatedHistory(lines, assertion) { const originalHome = process.env.HOME; const originalAppData = process.env.APPDATA;