diff --git a/fixtures/tooltip-blip.reference.expected.jsonl b/fixtures/tooltip-blip.reference.expected.jsonl new file mode 100644 index 0000000..ef215b8 --- /dev/null +++ b/fixtures/tooltip-blip.reference.expected.jsonl @@ -0,0 +1,10 @@ +{"decision":"emit","elapsedMs":0,"reason":"prime","score":144.000000,"seq":1} +{"decision":"skip","elapsedMs":500,"score":0.000000,"seq":2} +{"decision":"skip","elapsedMs":1000,"score":0.000000,"seq":3} +{"decision":"debounced","elapsedMs":1500,"score":12.350000,"seq":4} +{"decision":"debounced","elapsedMs":2000,"score":11.700000,"seq":5} +{"decision":"debounced","elapsedMs":2500,"score":11.700000,"seq":6} +{"decision":"skip","elapsedMs":3000,"score":0.000000,"seq":7} +{"decision":"skip","elapsedMs":3500,"score":0.000000,"seq":8} +{"decision":"skip","elapsedMs":4000,"score":0.000000,"seq":9} +{"decision":"skip","elapsedMs":4500,"score":0.000000,"seq":10} diff --git a/fixtures/tooltip-blip/000001.png b/fixtures/tooltip-blip/000001.png new file mode 100644 index 0000000..5ce9ba9 Binary files /dev/null and b/fixtures/tooltip-blip/000001.png differ diff --git a/fixtures/tooltip-blip/000002.png b/fixtures/tooltip-blip/000002.png new file mode 100644 index 0000000..5ce9ba9 Binary files /dev/null and b/fixtures/tooltip-blip/000002.png differ diff --git a/fixtures/tooltip-blip/000003.png b/fixtures/tooltip-blip/000003.png new file mode 100644 index 0000000..5ce9ba9 Binary files /dev/null and b/fixtures/tooltip-blip/000003.png differ diff --git a/fixtures/tooltip-blip/000004.png b/fixtures/tooltip-blip/000004.png new file mode 100644 index 0000000..e46a2c8 Binary files /dev/null and b/fixtures/tooltip-blip/000004.png differ diff --git a/fixtures/tooltip-blip/000005.png b/fixtures/tooltip-blip/000005.png new file mode 100644 index 0000000..e46a2c8 Binary files /dev/null and b/fixtures/tooltip-blip/000005.png differ diff --git a/fixtures/tooltip-blip/000006.png b/fixtures/tooltip-blip/000006.png new file mode 100644 index 0000000..e46a2c8 Binary files /dev/null and b/fixtures/tooltip-blip/000006.png differ diff --git a/fixtures/tooltip-blip/000007.png b/fixtures/tooltip-blip/000007.png new file mode 100644 index 0000000..5ce9ba9 Binary files /dev/null and b/fixtures/tooltip-blip/000007.png differ diff --git a/fixtures/tooltip-blip/000008.png b/fixtures/tooltip-blip/000008.png new file mode 100644 index 0000000..5ce9ba9 Binary files /dev/null and b/fixtures/tooltip-blip/000008.png differ diff --git a/fixtures/tooltip-blip/000009.png b/fixtures/tooltip-blip/000009.png new file mode 100644 index 0000000..5ce9ba9 Binary files /dev/null and b/fixtures/tooltip-blip/000009.png differ diff --git a/fixtures/tooltip-blip/000010.png b/fixtures/tooltip-blip/000010.png new file mode 100644 index 0000000..5ce9ba9 Binary files /dev/null and b/fixtures/tooltip-blip/000010.png differ diff --git a/package-lock.json b/package-lock.json index 3868121..3d5adcd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -68,19 +68,19 @@ }, "packages/adapters": { "name": "@framesieve/adapters", - "version": "0.2.0", + "version": "0.3.0", "license": "Apache-2.0", "dependencies": { - "framesieve": "^0.3.0" + "framesieve": "^0.4.0" } }, "packages/cli": { "name": "@framesieve/cli", - "version": "0.1.1", + "version": "0.2.0", "license": "Apache-2.0", "dependencies": { - "@framesieve/adapters": "^0.2.0", - "framesieve": "^0.3.0" + "@framesieve/adapters": "^0.3.0", + "framesieve": "^0.4.0" }, "bin": { "fsieve": "dist/main.js" @@ -88,7 +88,7 @@ }, "packages/core": { "name": "framesieve", - "version": "0.3.0", + "version": "0.4.0", "license": "Apache-2.0" }, "packages/redact": { @@ -96,10 +96,10 @@ "version": "0.1.0", "license": "Apache-2.0", "devDependencies": { - "framesieve": "^0.3.0" + "framesieve": "^0.4.0" }, "peerDependencies": { - "framesieve": "^0.3.0", + "framesieve": "^0.4.0", "tesseract.js": ">=5" }, "peerDependenciesMeta": { diff --git a/packages/adapters/CHANGELOG.md b/packages/adapters/CHANGELOG.md index a4ed20e..021b9dd 100644 --- a/packages/adapters/CHANGELOG.md +++ b/packages/adapters/CHANGELOG.md @@ -3,6 +3,12 @@ Notable changes to `@framesieve/adapters`. Pre-1.0 the API may change between minor versions. +## 0.3.0 + +- Requires `framesieve` `^0.4.0` (adds the opt-in reference diff mode in + core). No adapter API changes; recorder and replay pass through + `diff.mode` like any other gate option. + ## 0.2.0 - The node recorder now observes the gate via `gate.tap` instead of diff --git a/packages/adapters/package.json b/packages/adapters/package.json index 2a47ae6..b57d1a5 100644 --- a/packages/adapters/package.json +++ b/packages/adapters/package.json @@ -1,6 +1,6 @@ { "name": "@framesieve/adapters", - "version": "0.2.0", + "version": "0.3.0", "description": "Capture sources (electron, browser, node) plus recorder and replay for framesieve.", "license": "Apache-2.0", "type": "module", @@ -54,6 +54,6 @@ "access": "public" }, "dependencies": { - "framesieve": "^0.3.0" + "framesieve": "^0.4.0" } } diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index e2d559e..24cc4f5 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -3,6 +3,12 @@ Notable changes to `@framesieve/cli`. Pre-1.0 the CLI surface may change between minor versions. +## 0.2.0 + +- Requires `framesieve` `^0.4.0` and `@framesieve/adapters` `^0.3.0` + (core adds the opt-in reference diff mode). `fsieve replay --sweep` + can vary `diff.mode` like any other gate option; no new CLI flags. + ## 0.1.1 - `--algorithm` now accepts `edge` (Sobel edge diff) in addition to diff --git a/packages/cli/package.json b/packages/cli/package.json index 590dd24..adff617 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@framesieve/cli", - "version": "0.1.1", + "version": "0.2.0", "description": "fsieve command: replay recordings and sweep gate parameters offline.", "license": "Apache-2.0", "type": "module", @@ -23,7 +23,7 @@ "access": "public" }, "dependencies": { - "@framesieve/adapters": "^0.2.0", - "framesieve": "^0.3.0" + "@framesieve/adapters": "^0.3.0", + "framesieve": "^0.4.0" } } diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 6a368c0..0d6d191 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -3,6 +3,22 @@ Notable changes to `framesieve`. Pre-1.0 the API may change between minor versions; breaking changes are called out here. +## 0.4.0 + +- New opt-in `diff.mode: "reference"` (default stays `"previous"`): + compares each frame against the last emitted frame instead of the + previous one, so a transient change that reverts to that baseline is + dropped and only a change that persists for `policy.referencePersistMs` + is emitted. The adaptive mask keeps down-weighting chronically moving + regions (it now keys off frame-to-frame motion), so reference mode + composes with busy-region suppression. It is a temporal filter, not a + semantic one: whether a persistent change matters is the caller's call. + The default `"previous"` mode decides identically to 0.3.0 - existing + recordings and fixtures are byte-for-byte unchanged. +- New `policy.referencePersistMs` (default 3000): the minimum time a + divergence from the last emitted frame must persist before it emits in + reference mode; ignored in the default previous mode. + ## 0.3.0 - New `gate.tap(observer)`: a synchronous observer of (frame, decision) diff --git a/packages/core/README.md b/packages/core/README.md index fc2b530..66334eb 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -164,11 +164,37 @@ during long silence, and static ignore regions. Set (reason "prime") when an observer needs the current state right away instead of waiting for the stream to settle. -By default the gate compares downsampled luma. Set `diff.algorithm: -"edge"` to compare Sobel edge maps instead: a theme or brightness color -shift (unchanged gradients) is ignored, while text and contour changes -still register. Compare it against your own recording with `fsieve -replay --algorithm edge`. +Stage 1 exposes two independent choices - *what to measure* and *what to +compare it against* - and they combine freely (any algorithm with either +mode): + +**What to measure (`diff.algorithm`).** By default the gate compares +downsampled luma. Set `diff.algorithm: "edge"` to compare Sobel edge maps +instead: a theme or brightness color shift (unchanged gradients) is +ignored, while text and contour changes still register. Compare it +against your own recording with `fsieve replay --algorithm edge`. + +**What to compare against (`diff.mode`).** By default the gate compares +each frame against the immediately previous one (`diff.mode: "previous"`). +Set `diff.mode: "reference"` to compare +against the last emitted frame instead: a change that appears and then +reverts to that baseline before it persists is dropped as transient (a +hover tooltip that comes and goes), while a change that stays for +`policy.referencePersistMs` (default 3000) is emitted and becomes the new +baseline (text typed, a panel that opens and stays). In `"previous"` mode +a transient registers twice - once when it appears and once when it +disappears; `"reference"` mode registers neither. The adaptive mask still +down-weights chronically moving regions in either mode (it keys off +frame-to-frame motion), so a playing video does not defeat the +persistence check. + +`"reference"` mode is a temporal filter, not a semantic one: it decides +whether a change stuck, never whether it matters. Judging importance (is +this persistent tooltip worth a model call?) is the caller's or the +downstream model's job - framesieve never makes that call. In +`"reference"` mode `policy.referencePersistMs` is the persistence window; +`policy.debounceMs` keeps its "settle after motion" meaning in the +default `"previous"` mode. ## Tuning without guesswork: record and replay diff --git a/packages/core/package.json b/packages/core/package.json index 98e5814..a6fc855 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "framesieve", - "version": "0.3.0", + "version": "0.4.0", "description": "Deterministic frame gating for vision AI. Decides which captured frames are worth sending to a VLM.", "license": "Apache-2.0", "type": "module", diff --git a/packages/core/src/blocks.ts b/packages/core/src/blocks.ts index 4021a17..6c0f513 100644 --- a/packages/core/src/blocks.ts +++ b/packages/core/src/blocks.ts @@ -47,10 +47,20 @@ export class BlockGrid { this.historySum = new Uint32Array(this.cols * this.rows); } - step(cm: ChangeMask): BlockJudgment { - const { mask, w, h } = cm; + /** + * `score` gates the emit (changed vs the comparison baseline); + * `motion` (changed vs the previous frame) drives the adaptive + * weight, so a chronically moving region is down-weighted even in + * reference mode where a held overlay stays diverged from the + * baseline. In "previous" mode the two masks are the same buffer and + * this reduces exactly to the single-mask behavior. + */ + step(score: ChangeMask, motion: ChangeMask = score): BlockJudgment { + const { mask: sMask, w, h } = score; + const mMask = motion.mask; + const sameMask = mMask === sMask; const changedBlocks: BlockChange[] = []; - let score = 0; + let scoreSum = 0; for (let row = 0; row < this.rows; row++) { const y0 = Math.floor((row * h) / this.rows); const y1 = Math.floor(((row + 1) * h) / this.rows); @@ -58,32 +68,40 @@ export class BlockGrid { const x0 = Math.floor((col * w) / this.cols); const x1 = Math.floor(((col + 1) * w) / this.cols); const total = (x1 - x0) * (y1 - y0); - let changed = 0; + let sChanged = 0; + let mChanged = 0; for (let y = y0; y < y1; y++) { const base = y * w; for (let x = x0; x < x1; x++) { - changed += mask[base + x]!; + sChanged += sMask[base + x]!; + if (!sameMask) mChanged += mMask[base + x]!; } } - const ratio = total === 0 ? 0 : changed / total; - const isChanged = changed > 0 && ratio >= this.ratioThreshold; + if (sameMask) mChanged = sChanged; + const sRatio = total === 0 ? 0 : sChanged / total; + const mRatio = total === 0 ? 0 : mChanged / total; + const isChangedScore = sChanged > 0 && sRatio >= this.ratioThreshold; + const isChangedMotion = mChanged > 0 && mRatio >= this.ratioThreshold; const block = row * this.cols + col; - if (isChanged) { + if (isChangedScore) { const weight = this.adaptive ? 1 - this.historySum[block]! / this.windowSize : 1; - score += weight; - changedBlocks.push({ col, row, ratio, weight }); + scoreSum += weight; + changedBlocks.push({ col, row, ratio: sRatio, weight }); } - // Update history after the weight for this frame is taken. + // Update history (drives the weight) from MOTION, after the + // weight for this frame is taken. const idx = block * this.windowSize + this.slot; this.historySum[block] = - this.historySum[block]! - this.history[idx]! + (isChanged ? 1 : 0); - this.history[idx] = isChanged ? 1 : 0; + this.historySum[block]! - + this.history[idx]! + + (isChangedMotion ? 1 : 0); + this.history[idx] = isChangedMotion ? 1 : 0; } } this.slot = (this.slot + 1) % this.windowSize; - return { score, changedBlocks }; + return { score: scoreSum, changedBlocks }; } reset(): void { diff --git a/packages/core/src/config.ts b/packages/core/src/config.ts index 1d1d2d3..aab2c52 100644 --- a/packages/core/src/config.ts +++ b/packages/core/src/config.ts @@ -3,6 +3,7 @@ import type { BlockOptions, CropOptions, DiffAlgorithm, + DiffMode, DiffOptions, EmitTransform, FrameGateOptions, @@ -28,6 +29,8 @@ const DIFF_ALGORITHMS: readonly DiffAlgorithm[] = [ "edge", ]; +const DIFF_MODES: readonly DiffMode[] = ["previous", "reference"]; + function checkInt(name: string, value: number, min: number): number { if (!Number.isInteger(value) || value < min) { throw new RangeError(`${name} must be an integer >= ${min}, got ${value}`); @@ -62,6 +65,10 @@ export function resolveOptions(options: FrameGateOptions = {}): ResolvedOptions if (!DIFF_ALGORITHMS.includes(algorithm)) { throw new RangeError(`unknown diff algorithm: ${String(algorithm)}`); } + const mode = options.diff?.mode ?? "previous"; + if (!DIFF_MODES.includes(mode)) { + throw new RangeError(`unknown diff mode: ${String(mode)}`); + } const onNonMonotonic = options.policy?.onNonMonotonic ?? "throw"; if (onNonMonotonic !== "throw" && onNonMonotonic !== "clamp") { throw new RangeError( @@ -71,6 +78,7 @@ export function resolveOptions(options: FrameGateOptions = {}): ResolvedOptions return { diff: { algorithm, + mode, downsampleFactor: checkInt( "diff.downsampleFactor", options.diff?.downsampleFactor ?? 8, @@ -127,6 +135,12 @@ export function resolveOptions(options: FrameGateOptions = {}): ResolvedOptions Number.MAX_SAFE_INTEGER, ), primeOnFirstFrame: options.policy?.primeOnFirstFrame ?? false, + referencePersistMs: checkNumber( + "policy.referencePersistMs", + options.policy?.referencePersistMs ?? 3000, + 0, + Number.MAX_SAFE_INTEGER, + ), onNonMonotonic, ignoreRegions: (options.policy?.ignoreRegions ?? []).map((r, i) => checkRegion(`policy.ignoreRegions[${i}]`, r), diff --git a/packages/core/src/diff.ts b/packages/core/src/diff.ts index e68677c..ea55f6f 100644 --- a/packages/core/src/diff.ts +++ b/packages/core/src/diff.ts @@ -1,4 +1,4 @@ -import type { DiffAlgorithm, FrameInput, Region } from "./types.ts"; +import type { DiffAlgorithm, DiffMode, FrameInput, Region } from "./types.ts"; import type { ResolvedOptions } from "./config.ts"; /** @@ -17,6 +17,18 @@ export interface ChangeMask { h: number; } +/** + * A step's result. `mask/w/h` is the SCORE mask (vs the comparison + * baseline: the previous frame in "previous" mode, the last committed + * frame in "reference" mode). `motion` is always vs the immediately + * previous frame; the block grid uses it to drive the adaptive mask so + * chronically moving regions are down-weighted regardless of mode. In + * "previous" mode the two are the same buffer. + */ +export interface DiffResult extends ChangeMask { + motion: ChangeMask; +} + export function validateFrame(frame: FrameInput): void { if (!Number.isInteger(frame.width) || frame.width < 1) { throw new RangeError(`frame.width must be a positive integer`); @@ -138,24 +150,35 @@ export function buildIgnoreMask( } /** - * Stateful stage-1 engine. Holds the previous working buffer. The - * baseline starts as an all-zero (black) buffer, so the first frame - * registers as "everything with content changed" without any special - * casing. A change in frame dimensions resets the baseline the same - * way. + * Stateful stage-1 engine. Holds two working buffers: + * - prevFrame: the immediately previous frame; advances every step. + * The motion mask is computed against it. + * - baseline: the comparison target for the score mask. In "previous" + * mode it tracks prevFrame (advances every step), so score == motion. + * In "reference" mode it only advances when the gate calls commit() + * (i.e. on emit), so a change that reverts before commit produces a + * zero score mask and is dropped as transient. + * Both start as all-zero (black) buffers, so the first frame registers + * as "everything with content changed" without special casing. A change + * in frame dimensions resets both the same way. */ export class DiffEngine { - private prev: Uint8Array | null = null; + private prevFrame: Uint8Array | null = null; + private baseline: Uint8Array | null = null; + /** Current step's working buffer, promoted to baseline by commit(). */ + private lastWork: Uint8Array | null = null; private prevW = 0; private prevH = 0; private ignoreMask: Uint8Array | null = null; private readonly algorithm: DiffAlgorithm; + private readonly mode: DiffMode; private readonly factor: number; private readonly threshold: number; private readonly ignoreRegions: Region[]; constructor(options: ResolvedOptions) { this.algorithm = options.diff.algorithm; + this.mode = options.diff.mode; this.factor = options.diff.algorithm === "pixel" ? 1 : options.diff.downsampleFactor; // A threshold of 0 would mark identical pixels as changed @@ -164,14 +187,31 @@ export class DiffEngine { this.ignoreRegions = options.policy.ignoreRegions; } - step(frame: FrameInput): ChangeMask { + /** Changed-pixel mask of work vs ref, honoring the ignore mask. */ + private diffMask( + work: Uint8Array, + ref: Uint8Array, + ignore: Uint8Array | null, + ): Uint8Array { + const mask = new Uint8Array(work.length); + const threshold = this.threshold; + for (let i = 0; i < mask.length; i++) { + if (ignore !== null && ignore[i] === 1) continue; + const d = work[i]! - ref[i]!; + if ((d >= 0 ? d : -d) >= threshold) mask[i] = 1; + } + return mask; + } + + step(frame: FrameInput): DiffResult { // "edge" downsamples like "downsample", then compares Sobel maps. const grayAlg: "downsample" | "pixel" = this.algorithm === "pixel" ? "pixel" : "downsample"; const { gray, w, h } = toGray(frame, grayAlg, this.factor); const work = this.algorithm === "edge" ? sobel(gray, w, h) : gray; - if (this.prev === null || this.prevW !== w || this.prevH !== h) { - this.prev = new Uint8Array(w * h); + if (this.prevFrame === null || this.prevW !== w || this.prevH !== h) { + this.prevFrame = new Uint8Array(w * h); + this.baseline = new Uint8Array(w * h); this.prevW = w; this.prevH = h; this.ignoreMask = buildIgnoreMask( @@ -182,20 +222,38 @@ export class DiffEngine { this.ignoreRegions, ); } - const prev = this.prev; const ignore = this.ignoreMask; - const mask = new Uint8Array(w * h); - for (let i = 0; i < mask.length; i++) { - if (ignore !== null && ignore[i] === 1) continue; - const d = work[i]! - prev[i]!; - if ((d >= 0 ? d : -d) >= this.threshold) mask[i] = 1; + if (this.mode === "previous") { + // Single comparison vs the previous frame; motion is the same + // buffer as the score mask, so the grid behaves exactly as before. + const mask = this.diffMask(work, this.prevFrame, ignore); + this.prevFrame = work; + const cm: ChangeMask = { mask, w, h }; + return { mask, w, h, motion: cm }; + } + // reference: score vs the committed baseline, motion vs prev frame. + const scoreMask = this.diffMask(work, this.baseline!, ignore); + const motionMask = this.diffMask(work, this.prevFrame, ignore); + this.prevFrame = work; // motion baseline advances every step + this.lastWork = work; // score baseline advances only on commit() + return { mask: scoreMask, w, h, motion: { mask: motionMask, w, h } }; + } + + /** + * Promote the current frame to the comparison baseline. The gate + * calls this on emit in "reference" mode; in "previous" mode the + * baseline already tracks every frame, so this is a no-op. + */ + commit(): void { + if (this.mode === "reference" && this.lastWork !== null) { + this.baseline = this.lastWork; } - this.prev = work; - return { mask, w, h }; } reset(): void { - this.prev = null; + this.prevFrame = null; + this.baseline = null; + this.lastWork = null; this.prevW = 0; this.prevH = 0; this.ignoreMask = null; diff --git a/packages/core/src/gate.ts b/packages/core/src/gate.ts index 3e03db4..f0ac707 100644 --- a/packages/core/src/gate.ts +++ b/packages/core/src/gate.ts @@ -116,11 +116,29 @@ class Gate implements FrameGate { this.lastEmitOut = null; if (this.silenceBaseMs === null) this.silenceBaseMs = frame.elapsedMs; - const judgment = this.grid.step(this.diff.step(frame)); + const diffResult = this.diff.step(frame); + const judgment = this.grid.step(diffResult, diffResult.motion); const { score, changedBlocks } = judgment; const crossed = changedBlocks.length > 0 && score >= this.options.blocks.minChangedBlocks; - if (crossed) { + if (this.options.diff.mode === "reference") { + // Persistence semantics: sinceMs marks when the divergence from + // the last emitted frame first appeared and is NOT refreshed while + // it persists, so stableFor measures how long it has lasted. A + // frame that no longer crosses (reverted to the baseline, or a + // chronically moving region the adaptive weight decayed) drops the + // pending change as transient. + if (crossed) { + if (this.pending === null) { + this.pending = { score, changedBlocks, sinceMs: frame.elapsedMs }; + } else { + this.pending.score = score; + this.pending.changedBlocks = changedBlocks; + } + } else { + this.pending = null; + } + } else if (crossed) { this.pending = { score, changedBlocks, sinceMs: frame.elapsedMs }; } @@ -151,7 +169,13 @@ class Gate implements FrameGate { if (this.pending !== null) { const stableFor = frame.elapsedMs - this.pending.sinceMs; - const debounced = stableFor < this.options.policy.debounceMs; + // In reference mode the wait is the persistence window; in the + // default previous mode it is the settle-after-motion debounce. + const settleWindow = + this.options.diff.mode === "reference" + ? this.options.policy.referencePersistMs + : this.options.policy.debounceMs; + const debounced = stableFor < settleWindow; const throttled = this.lastEmitMs !== null && frame.elapsedMs - this.lastEmitMs < this.options.policy.minIntervalMs; @@ -216,6 +240,13 @@ class Gate implements FrameGate { private recordEmit(frame: FrameInput, meta: EmitMeta): void { this.lastEmitMs = meta.elapsedMs; this.silenceBaseMs = meta.elapsedMs; + // Advance the reference-mode baseline to this frame (no-op in + // previous mode). Must run synchronously here, not in the async + // delivery below: diff.step() overwrites its lastWork on every push, + // so a deferred commit could promote the wrong frame. Keepalive does + // NOT commit - moving the baseline mid-pending would misread the next + // (unchanged) frame as a revert and cancel a real persistence check. + if (meta.reason !== "keepalive") this.diff.commit(); const { transform } = this.options; const { crop } = this.options; const { gridCols, gridRows } = this.options.blocks; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f781b70..33a10e2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -5,6 +5,7 @@ export { serializeDecision, parseDecisionLine } from "./serialize.ts"; export type { FrameInput, DiffAlgorithm, + DiffMode, Region, DiffOptions, BlockOptions, diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 8074409..c3693a5 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -52,6 +52,26 @@ export type DiffAlgorithm = */ | "edge"; +export type DiffMode = + /** + * Compare each frame against the immediately preceding frame. + * Default. Any frame-to-frame delta counts, so a transient overlay + * registers both when it appears and again when it disappears. + */ + | "previous" + /** + * Compare each frame against the last emitted frame (the committed + * baseline). A change that appears and then reverts to the baseline + * before it persists is dropped as transient; a change that persists + * for policy.referencePersistMs is emitted and becomes the new + * baseline. adaptiveMask still down-weights chronically moving + * regions (it keys off frame-to-frame motion, not baseline + * divergence). This is a temporal filter, not a semantic one: + * whether a persistent change matters is the caller's/VLM's call. + * Opt-in. + */ + | "reference"; + export interface Region { x: number; y: number; @@ -62,6 +82,14 @@ export interface Region { export interface DiffOptions { /** Diff algorithm. Default: "downsample". */ algorithm?: DiffAlgorithm; + /** + * Diff comparison baseline. "previous" (default) compares consecutive + * frames; "reference" compares against the last emitted frame, so a + * transient change that reverts is dropped and only a change that + * persists (see policy.referencePersistMs) is emitted. Default: + * "previous". + */ + mode?: DiffMode; /** * Downsample factor for "downsample" algorithm. The frame is reduced * to (width/factor) x (height/factor) grayscale. Default: 8. @@ -124,6 +152,16 @@ export interface PolicyOptions { * only to the first frame after construction/reset. Default: false. */ primeOnFirstFrame?: boolean; + /** + * Only used when diff.mode is "reference": the minimum time a + * divergence from the last emitted frame must persist before it is + * emitted. A change that reverts to the baseline sooner is dropped as + * transient. This is the reference-mode analog of debounceMs (which + * keeps its "settle after motion" meaning in the default "previous" + * mode). Ignored in "previous" mode. Evaluated against elapsedMs. + * Default: 3000. + */ + referencePersistMs?: number; /** * What to do when a frame's elapsedMs is less than the previous * frame's. "throw" (default) rejects it with a RangeError; "clamp" diff --git a/packages/core/test/diff.test.ts b/packages/core/test/diff.test.ts index 3d50b33..02d4cff 100644 --- a/packages/core/test/diff.test.ts +++ b/packages/core/test/diff.test.ts @@ -176,3 +176,72 @@ test("resolveOptions validates ranges", () => { RangeError, ); }); + +test("resolveOptions rejects an unknown diff mode and defaults to previous", () => { + assert.throws( + () => resolveOptions({ diff: { mode: "sideways" as unknown as "previous" } }), + /unknown diff mode: sideways/, + ); + assert.equal(resolveOptions({}).diff.mode, "previous"); + assert.equal(resolveOptions({ policy: {} }).policy.referencePersistMs, 3000); +}); + +test("reference mode holds the comparison baseline until commit()", () => { + const e = engine({ + diff: { downsampleFactor: 1, mode: "reference", luminanceThreshold: 10 }, + }); + const a = solidFrame(4, 4, 100, 0); + e.step(a); // vs black baseline: all changed + e.commit(); // baseline = a + const same = e.step({ ...a, elapsedMs: 500 }); + assert.equal(maskSum(same.mask), 0, "identical to the committed baseline"); + // Paint a change and step it twice WITHOUT commit: the baseline does + // not advance, so both score masks are identical (unlike previous mode). + const b = paintRect(a, 0, 0, 2, 2, 220); + const s1 = e.step({ ...b, elapsedMs: 1000 }); + const s2 = e.step({ ...b, elapsedMs: 1500 }); + assert.ok(maskSum(s1.mask) > 0, "diverges from the held baseline"); + assert.deepEqual([...s2.mask], [...s1.mask], "baseline held: same score mask"); + e.commit(); // baseline = b + const back = e.step({ ...b, elapsedMs: 2000 }); + assert.equal(maskSum(back.mask), 0, "now matches the new baseline"); +}); + +test("previous mode commit() is a no-op: same mask sequence with or without it", () => { + const mk = () => engine({ diff: { downsampleFactor: 1, luminanceThreshold: 10 } }); + const flat = solidFrame(4, 4, 50, 0); + const frames = [flat, paintRect(flat, 0, 0, 2, 2, 200), { ...flat }].map( + (f, i) => ({ ...f, elapsedMs: i * 500 }), + ); + const withCommit = mk(); + const without = mk(); + const a = frames.map((f) => { + const r = withCommit.step(f); + withCommit.commit(); + return [...r.mask]; + }); + const b = frames.map((f) => [...without.step(f).mask]); + assert.deepEqual(a, b); +}); + +test("reference mode: motion mask keys off the previous frame, score off the baseline", () => { + const e = engine({ + diff: { downsampleFactor: 1, mode: "reference", luminanceThreshold: 10 }, + }); + const bg = solidFrame(4, 4, 40, 0); + e.step(bg); + e.commit(); // baseline = bg + const overlay = paintRect(bg, 0, 0, 2, 2, 220); + const appear = e.step({ ...overlay, elapsedMs: 500 }); + assert.ok( + maskSum(appear.mask) > 0 && maskSum(appear.motion.mask) > 0, + "appears: both score and motion flag it", + ); + const hold = e.step({ ...overlay, elapsedMs: 1000 }); + assert.ok(maskSum(hold.mask) > 0, "score still diverges from the held baseline"); + assert.equal( + maskSum(hold.motion.mask), + 0, + "motion is zero: nothing moved since the previous frame", + ); +}); diff --git a/packages/core/test/fixtures.test.ts b/packages/core/test/fixtures.test.ts index b1df347..ae65f94 100644 --- a/packages/core/test/fixtures.test.ts +++ b/packages/core/test/fixtures.test.ts @@ -140,3 +140,50 @@ test("fixture decisions are identical across repeated runs", () => { const b = run("video-noise", {}); assert.deepEqual(a, b); }); + +// Default mode is "previous"; passing it explicitly must reproduce the +// checked-in goldens byte-for-byte on the same PNG sequences. Guards the +// "reference mode is fully backward compatible" claim for both the +// omitted-default and explicit-value code paths. +const PREV_CASES: Array<[string, FrameGateOptions]> = [ + ["slide-flip", {}], + ["cursor", {}], + ["cursor", { policy: { maxSilenceMs: 2000 } }], + ["video-noise", {}], +]; + +test('explicit diff.mode "previous" reproduces the default decisions', () => { + for (const [name, opts] of PREV_CASES) { + const dflt = run(name, opts); + const explicit = run(name, { ...opts, diff: { ...opts.diff, mode: "previous" } }); + assert.deepEqual(explicit, dflt, name); + } +}); + +test("tooltip-blip: reference mode drops the transient overlay (no threshold emit)", () => { + const decisions = run("tooltip-blip", { + diff: { mode: "reference" }, + policy: { primeOnFirstFrame: true }, + }); + const p = pattern(decisions); + assert.equal(p[0], "emit:prime"); // primeOnFirstFrame gives the baseline + // The tooltip opens a pending divergence (frames 3-5) but reverts + // before referencePersistMs, so it is never emitted as a threshold. + assert.ok( + p.slice(3, 6).includes("debounced"), + `tooltip should open a pending change: ${p.join(",")}`, + ); + assert.ok( + !p.includes("emit:threshold"), + `transient must not emit a threshold: ${p.join(",")}`, + ); + checkExpected("tooltip-blip", "reference", decisions); +}); + +test("tooltip-blip: previous mode registers the transient the reference mode drops", () => { + const p = pattern(run("tooltip-blip", { policy: { primeOnFirstFrame: true } })); + assert.ok( + p.includes("emit:threshold"), + `previous mode should emit the transition: ${p.join(",")}`, + ); +}); diff --git a/packages/core/test/gate.test.ts b/packages/core/test/gate.test.ts index 50ff829..d70eae7 100644 --- a/packages/core/test/gate.test.ts +++ b/packages/core/test/gate.test.ts @@ -633,3 +633,141 @@ test("copyFrameOnEmit false aliases the caller buffer (opt-in, no copy)", async assert.ok(out); assert.equal(out!.data, f.data); // same backing buffer, no copy }); + +// ---- reference diff mode ---- + +// Same policy for both modes (previous uses debounceMs, reference uses +// referencePersistMs), so the only variable is diff.mode. +const REF_POLICY = { + debounceMs: 0, + referencePersistMs: 800, + minIntervalMs: 0, + maxSilenceMs: 0, + primeOnFirstFrame: true, +}; + +test("reference mode drops a transient overlay that previous mode emits twice", () => { + const frames = [ + at(base, 0), // prime / baseline + blocksChanged(base, 2, 220, 500), // overlay appears (2 blocks) + blocksChanged(base, 2, 220, 1000), // overlay holds briefly + at(base, 1500), // overlay gone (back to baseline) + at(base, 2000), // stays gone + ]; + const prev = decisions( + createFrameGate({ ...GRID, diff: { downsampleFactor: 8, mode: "previous" }, policy: REF_POLICY }), + frames, + ).map((d) => [d.decision, d.reason ?? null]); + const ref = decisions( + createFrameGate({ ...GRID, diff: { downsampleFactor: 8, mode: "reference" }, policy: REF_POLICY }), + frames, + ).map((d) => [d.decision, d.reason ?? null]); + + assert.deepEqual(prev, [ + ["emit", "prime"], + ["emit", "threshold"], // appear settles (debounce 0) + ["skip", null], + ["emit", "threshold"], // disappear settles + ["skip", null], + ]); + assert.deepEqual(ref, [ + ["emit", "prime"], + ["debounced", null], // divergence from baseline opened, waiting to persist + ["debounced", null], + ["skip", null], // reverted before referencePersistMs -> dropped + ["skip", null], + ]); +}); + +test("reference mode emits a change that persists past referencePersistMs", () => { + const gate = createFrameGate({ + ...GRID, + diff: { downsampleFactor: 8, mode: "reference" }, + policy: { referencePersistMs: 800, minIntervalMs: 0, maxSilenceMs: 0, primeOnFirstFrame: true }, + }); + const d = decisions(gate, [ + at(base, 0), + blocksChanged(base, 2, 220, 500), + blocksChanged(base, 2, 220, 1000), + blocksChanged(base, 2, 220, 1500), // 1000ms >= 800ms: persisted + blocksChanged(base, 2, 220, 2000), // now the committed baseline + ]).map((x) => [x.decision, x.reason ?? null]); + assert.deepEqual(d, [ + ["emit", "prime"], + ["debounced", null], + ["debounced", null], + ["emit", "threshold"], + ["skip", null], + ]); +}); + +test("reference mode: a keepalive does not cancel an in-flight persistence check", () => { + // maxSilenceMs (1000) < referencePersistMs (1500): a keepalive fires + // while a real divergence is still accumulating persistence. It must + // NOT commit the baseline (which would misread the next held frame as + // a revert); the persistent change still emits a threshold afterward. + const gate = createFrameGate({ + ...GRID, + diff: { downsampleFactor: 8, mode: "reference" }, + policy: { referencePersistMs: 1500, minIntervalMs: 0, maxSilenceMs: 1000, primeOnFirstFrame: true }, + }); + const d = decisions(gate, [ + at(base, 0), // prime, baseline = base + blocksChanged(base, 2, 220, 500), // overlay appears, pending sinceMs=500 + blocksChanged(base, 2, 220, 1000), // keepalive due (1000ms silence); pending survives + blocksChanged(base, 2, 220, 1500), // still held + blocksChanged(base, 2, 220, 2000), // 1500ms persisted -> threshold + ]).map((x) => [x.decision, x.reason ?? null]); + assert.deepEqual(d, [ + ["emit", "prime"], + ["debounced", null], + ["emit", "keepalive"], + ["debounced", null], + ["emit", "threshold"], // pending was NOT cancelled by the keepalive + ]); +}); + +test("reference-mode commit happens synchronously in push(), independent of transform timing", () => { + const gate = createFrameGate({ + ...GRID, + diff: { downsampleFactor: 8, mode: "reference" }, + policy: { referencePersistMs: 0, minIntervalMs: 0, maxSilenceMs: 0, primeOnFirstFrame: true }, + transform: () => new Promise(() => {}), // never resolves + }); + gate.push(at(base, 0)); // prime: must commit baseline = base synchronously + const d = gate.push(at(base, 500)); // identical content + // If commit() were deferred into the (hung) async delivery, the + // baseline would still be black and this frame would cross. + assert.equal(d.score, 0); + assert.equal(d.decision, "skip"); +}); + +test("reference mode with adaptiveMask on still emits a static persistent overlay", () => { + // The adaptive weight is driven by frame-to-frame motion, not by + // baseline divergence, so a held overlay (motion 0 after it appears) + // keeps a high score and emits, instead of decaying below the gate. + const gate = createFrameGate({ + diff: { downsampleFactor: 8, mode: "reference" }, + blocks: { gridCols: 4, gridRows: 4, minChangedBlocks: 2 }, + adaptiveMask: { enabled: true, windowSize: 20 }, + policy: { referencePersistMs: 800, minIntervalMs: 0, maxSilenceMs: 0, primeOnFirstFrame: true }, + }); + const d = decisions(gate, [ + at(base, 0), + blocksChanged(base, 4, 220, 500), + blocksChanged(base, 4, 220, 1000), + blocksChanged(base, 4, 220, 1500), // held 1000ms >= 800ms + ]); + assert.deepEqual( + d.map((x) => [x.decision, x.reason ?? null]), + [ + ["emit", "prime"], + ["debounced", null], + ["debounced", null], + ["emit", "threshold"], + ], + ); + // adaptiveMask is active (score is discounted below the raw 4 blocks) + // yet still clears minChangedBlocks (2). + assert.ok(d[3]!.score >= 2 && d[3]!.score < 4, `score ${d[3]!.score}`); +}); diff --git a/packages/redact/package.json b/packages/redact/package.json index d557b10..7fd4751 100644 --- a/packages/redact/package.json +++ b/packages/redact/package.json @@ -32,7 +32,7 @@ "LICENSE" ], "peerDependencies": { - "framesieve": "^0.3.0", + "framesieve": "^0.4.0", "tesseract.js": ">=5" }, "peerDependenciesMeta": { @@ -41,6 +41,6 @@ } }, "devDependencies": { - "framesieve": "^0.3.0" + "framesieve": "^0.4.0" } } diff --git a/scripts/generate-fixtures.ts b/scripts/generate-fixtures.ts index 99d761f..a318228 100644 --- a/scripts/generate-fixtures.ts +++ b/scripts/generate-fixtures.ts @@ -129,3 +129,18 @@ function writeSequence(name: string, frames: Canvas[]): void { } writeSequence("video-noise", frames); } + +// tooltip-blip: a static slide with a tooltip overlay that appears for a +// few frames, then reverts completely to the slide. Exercises reference +// mode (the transient is dropped) vs previous mode (it registers on +// appear and again on disappear). +{ + const bg = slide(4); + const withTip = new Uint8ClampedArray(bg); + fillRect(withTip, 192, 56, 72, 48, 20); // opaque tooltip box + const seq = [bg, bg, bg, withTip, withTip, withTip, bg, bg, bg, bg]; + writeSequence( + "tooltip-blip", + seq.map((c) => new Uint8ClampedArray(c)), + ); +}