diff --git a/src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts b/src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts index 88fb1ca93..0adb03ae0 100644 --- a/src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts +++ b/src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts @@ -72,6 +72,16 @@ export function walkNonRawAndroidFixture(rawNodes: RawSnapshotNode[]): RawSnapsh return buildUiHierarchySnapshot(tree, undefined, { raw: false }).nodes; } +/** + * Runs a fixture through the REAL interactive-only walk — the shape `--settle` + * and `wait stable` consume, since their loop captures with + * `interactiveOnly: true` (`stable-capture.ts`). + */ +export function walkInteractiveOnlyAndroidFixture(rawNodes: RawSnapshotNode[]): RawSnapshotNode[] { + const tree = rawFixtureToAndroidTree(rawNodes); + return buildUiHierarchySnapshot(tree, undefined, { raw: false, interactiveOnly: true }).nodes; +} + /** * Real device capture (checkout-form fixture app, Gboard open, status bar * visible) archived at `~/.agent-device-bench/replay-runs/android-ime/raw-ime2.json` diff --git a/src/commands/interaction/runtime/settle.test.ts b/src/commands/interaction/runtime/settle.test.ts index 6f9e113ac..7ff2978d8 100644 --- a/src/commands/interaction/runtime/settle.test.ts +++ b/src/commands/interaction/runtime/settle.test.ts @@ -700,15 +700,15 @@ function androidStatusBarNodes(startIndex: number, clockLabel = '12:23') { const root = startIndex; return [ { + // The outermost status-bar node a real capture starts with. index: root, depth: 0, type: 'android.widget.FrameLayout', + identifier: 'com.android.systemui:id/status_bar_launch_animation_container', bundleId: ANDROID_SYSTEM_UI_BUNDLE_ID, rect: { x: 0, y: 0, width: 1344, height: 159 }, }, { - // The marker every real status-bar capture carries; the window-run - // drops as a whole because this member matches the status_bar* prefix. index: root + 1, depth: 1, parentIndex: root, diff --git a/src/contracts/android-system-chrome.ts b/src/contracts/android-system-chrome.ts index 224227ddb..b8ce37e23 100644 --- a/src/contracts/android-system-chrome.ts +++ b/src/contracts/android-system-chrome.ts @@ -1,69 +1,45 @@ +import type { SnapshotNode } from '../kernel/snapshot.ts'; + /** - * Android status-bar/navigation-bar chrome markers, shared between the settle-chrome - * classifier (`core/snapshot-chrome.ts`, #1198) and the helper content classifier - * (`platforms/android/snapshot-content-recovery.ts`). SystemUI hosts BOTH persistent chrome - * and actionable overlays (volume panel, media/output pickers, notification shade, quick - * settings), so chrome is never a package-level fact: only these status/nav-bar marker - * resource-ids classify as chrome; every other systemui surface is real content. + * Android status-bar/navigation-bar chrome identity, shared by the settle-chrome + * classifier (`core/snapshot-chrome.ts`) and the helper content classifier + * (`platforms/android/snapshot-content-recovery.ts`). SystemUI also hosts actionable + * overlays — volume panel, media pickers, the shade itself — so chrome is never a + * package-level fact: only the status/nav-bar container's subtree is chrome. */ export const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui'; -const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [ - 'com.android.systemui:id/status_bar', - 'com.android.systemui:id/navigation_bar', -] as const; - /** - * Surviving status-bar/nav-bar LEAF ids (#1251). The non-raw Android walk - * (`walkUiHierarchyNode` in `platforms/android/ui-hierarchy.ts`) drops - * unlabeled/unidentified structural nodes via `shouldIncludeStructuralAndroidNode`, - * re-parenting their children upward — and that silently swallows every - * `status_bar*`/`navigation_bar*` WRAPPER node, i.e. the only nodes the prefix - * check above matches. A non-raw capture is left with just their labeled/ - * identified LEAVES (clock, battery, wifi/mobile icons, nav buttons), whose - * own resource-ids carry no `status_bar`/`navigation_bar` prefix, so the run - * loses its marker and `collectAndroidSystemChromeRunIndexes` stops dropping - * it (verified against a real `--raw` vs. default capture pair of the same - * screen). Recognize those leaves directly, by EXACT id — not prefix, to stay - * tight: nothing here should ever swallow an actionable systemui overlay like - * the volume dialog or a media/output picker, which live under unrelated ids. - * `--raw` keeps the wrapper markers, so the prefix check above stays - * load-bearing there. - * - * A more robust fix would thread the AOSP window-type constants - * (`TYPE_STATUS_BAR` = 2000, `TYPE_NAVIGATION_BAR` = 2019) already parsed in - * `readNodeAttributes` (ui-hierarchy.ts) through to the output `SnapshotNode` - * and key off that instead of resource-ids — deferred until the id-based - * approach here proves insufficient. + * Resource-ids of the status-bar / navigation-bar WINDOW containers. Matched as an id + * segment, not a prefix, so an expanded shade's `split_shade_status_bar` counts. */ -const ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS: ReadonlySet = new Set( - [ - // status bar - 'clock', - 'battery', - 'statusIcons', - 'notificationIcons', - 'notification_icon_area', - 'system_icons', - 'cutout_space_view', - 'mobile_signal', - 'mobile_combo', - 'mobile_group', - 'wifi_signal', - 'wifi_combo', - 'wifi_group', - 'start_side_notif_and_chip_container', - // nav bar - 'back', - 'home', - 'recent_apps', - 'home_handle', - ].map((leaf) => `${ANDROID_SYSTEM_CHROME_PACKAGE}:id/${leaf}`), -); - -/** True when the resource-id marks Android status-bar or navigation-bar chrome. */ -export function isAndroidSystemChromeResourceId(resourceId: string | null | undefined): boolean { +export function isAndroidSystemChromeWindowResourceId( + resourceId: string | null | undefined, +): boolean { const identifier = resourceId ?? ''; - if (ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS.has(identifier)) return true; - return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix)); + if (!identifier.startsWith(`${ANDROID_SYSTEM_CHROME_PACKAGE}:id/`)) return false; + const leaf = identifier.slice(`${ANDROID_SYSTEM_CHROME_PACKAGE}:id/`.length); + return /(^|_)(status_bar|navigation_bar)(_|$)/.test(leaf); +} + +/** Android-internal provenance retained in daemon session snapshots only. */ +export type AndroidSystemChromeProvenance = { + systemChrome?: true; +}; + +export function hasAndroidSystemChromeProvenance(value: object): value is { systemChrome: true } { + return 'systemChrome' in value && value.systemChrome === true; +} + +/** Drops Android-internal provenance from a node embedded in a public response. */ +export function stripAndroidSystemChromeProvenanceFromNode(node: SnapshotNode): SnapshotNode { + if (!hasAndroidSystemChromeProvenance(node)) return node; + const { systemChrome: _systemChrome, ...published } = node; + return published; +} + +/** Drops Android-internal provenance at the daemon's public snapshot seam. */ +export function stripAndroidSystemChromeProvenance(nodes: SnapshotNode[]): SnapshotNode[] { + if (!nodes.some(hasAndroidSystemChromeProvenance)) return nodes; + return nodes.map(stripAndroidSystemChromeProvenanceFromNode); } diff --git a/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts b/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts index 9a5a568ba..0aa127e96 100644 --- a/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts +++ b/src/core/__tests__/snapshot-chrome-android-statusbar.test.ts @@ -1,28 +1,19 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '../../kernel/snapshot.ts'; -import { collectSettleChromeRefs } from '../snapshot-chrome.ts'; +import { collectSettleChromeRefs, withoutSettleChrome } from '../snapshot-chrome.ts'; import { ANDROID_IME_CAPTURE_RAW_NODES, + ANDROID_QS_SHADE_CAPTURE_RAW_NODES, + walkInteractiveOnlyAndroidFixture, walkNonRawAndroidFixture, } from '../../__tests__/test-utils/android-ui-hierarchy-fixtures.ts'; +import { isAndroidSystemChromeWindowResourceId } from '../../contracts/android-system-chrome.ts'; /** - * `ANDROID_IME_CAPTURE_RAW_NODES` (shared test util, see its own doc comment) - * is a real device `--raw` capture: `--raw` keeps every structural wrapper - * node (`status_bar_container`, `status_bar_contents`, ...) so the OLD - * prefix-only marker check finds them and drops the whole run. - * - * The default (non-raw) walk drops unlabeled/unidentified structural nodes - * (`shouldIncludeStructuralAndroidNode` in `platforms/android/ui-hierarchy.ts`), - * re-parenting their children upward — which silently removes every one of - * those marker-bearing wrappers AND several other anonymous structural nodes - * that have neither a hittable descendant nor meaningful text/id (a real - * `android.view.View` example is `com.android.systemui:id/home_handle`, see - * the nav-bar test below). `walkNonRawAndroidFixture` (shared test util) runs - * the fixture through the REAL `buildUiHierarchySnapshot({ raw: false })` - * walk instead of hand-simulating a subset of its drops, so every inclusion - * decision below is production's, not a guess at which nodes matter. + * The `walk*AndroidFixture` helpers run a real `--raw` device capture through + * production's own walk, so every inclusion decision below is production's + * rather than a hand-simulation of it. */ function refForIdentifier(nodes: SnapshotNode[], identifier: string): string { @@ -31,21 +22,7 @@ function refForIdentifier(nodes: SnapshotNode[], identifier: string): string { return node.ref; } -/** - * Status-bar leaf identifiers that SURVIVE the real non-raw walk for this - * fixture (verified against `buildUiHierarchySnapshot({ raw: false })` - * directly). Several ids the naive prefix-only simulation used to assert - * (`notification_icon_area`, `notificationIcons`, `cutout_space_view`, - * `system_icons`, `statusIcons`, `mobile_group`, `wifi_combo`, `wifi_group`, - * `start_side_notif_and_chip_container`, `battery`) are unlabeled structural - * wrappers with a generic resource id and no hittable descendant: production - * `shouldIncludeStructuralAndroidNode` drops every one of them too, same as - * the `status_bar*`/`navigation_bar*` wrappers. Their content isn't lost — - * `battery`'s labeled child ("Battery 100 percent.") re-parents upward and is - * covered by the "every systemui-owned node is chrome" assertion below — but - * the WRAPPER identifier itself is gone from the walked tree, so it can't be - * looked up by id. - */ +/** Status-bar leaves that survive the non-raw walk for this fixture; none carries a chrome id. */ const STATUS_BAR_LEAF_IDENTIFIERS = [ 'com.android.systemui:id/clock', 'com.android.systemui:id/mobile_combo', @@ -53,7 +30,25 @@ const STATUS_BAR_LEAF_IDENTIFIERS = [ 'com.android.systemui:id/wifi_signal', ]; -test('Android non-raw capture: status-bar leaves are recognized as chrome once their status_bar*/navigation_bar* marker wrapper is dropped by the walk (#1251)', () => { +test('Android chrome container ids match complete status/nav-bar segments only', () => { + assert.equal( + isAndroidSystemChromeWindowResourceId( + 'com.android.systemui:id/status_bar_launch_animation_container', + ), + true, + ); + assert.equal( + isAndroidSystemChromeWindowResourceId('com.android.systemui:id/split_shade_status_bar'), + true, + ); + assert.equal( + isAndroidSystemChromeWindowResourceId('com.android.systemui:id/status_barometer'), + false, + ); + assert.equal(isAndroidSystemChromeWindowResourceId('com.example:id/status_bar'), false); +}); + +test('Android non-raw capture: status-bar leaves stay chrome after the walk drops the container that identifies them (#1251)', () => { const walkedNodes = walkNonRawAndroidFixture(ANDROID_IME_CAPTURE_RAW_NODES); const nodes = attachRefs(walkedNodes); const chromeRefs = collectSettleChromeRefs(nodes, 'com.callstack.agentdevicelab'); @@ -105,15 +100,9 @@ test('Android non-raw capture: status-bar leaves are recognized as chrome once t }); test('Android actionable systemui overlay (volume dialog) still survives the chrome filter (#1251)', () => { - // Filter-logic unit test, NOT a live-capture-path claim (#1264 finding 2): this - // exercises `collectSettleChromeRefs` in isolation over a synthetic systemui - // run appended to an unrelated capture fixture. The leaf ids ARE real, - // live-verified volume-dialog ids (`volume_dialog_container`, - // `volume_new_ringer_active_icon_container`), but appending them to this - // fixture does not reproduce how a live capture reaches the filter — that is - // covered separately by the full-capture invariant test below. The point - // here is narrower: the status-bar/nav-bar leaf-id set (#1251) must NOT - // broaden to "any systemui id" and drop an actionable overlay it is handed. + // Real volume-dialog ids grafted onto an unrelated capture (#1264 finding 2: + // synthetic placement, real ids). Chrome must stay a status/nav-bar fact and + // never widen to "any systemui id", which would swallow this overlay. const rawWithVolumeDialog: RawSnapshotNode[] = [ ...ANDROID_IME_CAPTURE_RAW_NODES, { @@ -235,3 +224,53 @@ test('Android nav-bar leaves are recognized as chrome once their navigation_bar* 'expected home_handle to be dropped by the real non-raw walk, not retained', ); }); + +/** + * A fully expanded quick-settings shade hosts the status-bar icons and the + * quick-settings tiles in ONE window. Chrome classification must split them the + * same way whatever shape the capture arrives in — that is what #1318 (raw: + * every tile condemned) and #1319 (interactive-only: tiles kept) each hit from + * one side. + * + * Live-verified on emulator-5554 (Pixel 9 Pro XL API 37, deskclock): raising + * the shade mid-`--settle` diffs `+28 -25`, closing it `+25 -28`, and the + * shade's own status bar is absent from both while `snapshot -i` of the same + * screen still lists it. + */ +test('Android expanded quick-settings shade: tiles survive and the status bar drops, identically in every capture shape (#1318/#1319)', () => { + const appBundleId = 'com.google.android.deskclock'; + + for (const [shape, walk] of [ + ['interactive-only (--settle, wait stable)', walkInteractiveOnlyAndroidFixture], + ['non-raw (snapshot, replay divergence)', walkNonRawAndroidFixture], + ] as const) { + const nodes = attachRefs(walk(ANDROID_QS_SHADE_CAPTURE_RAW_NODES)); + const kept = withoutSettleChrome(nodes, appBundleId); + const keptIdentifiers = new Set(kept.map((node) => node.identifier)); + + // The shade's actionable content survives, so opening or closing it can + // never read as "nothing changed". + assert.equal(keptIdentifiers.has('com.android.systemui:id/slider'), true, shape); + assert.equal( + kept.some((node) => node.identifier?.startsWith('com.android.systemui:id/qs_tile_')), + true, + `expected the quick-settings tiles to survive chrome stripping — ${shape}`, + ); + + // ...while the status-bar subtree sharing that window still drops: clock, + // date and wifi tick constantly and are the canonical churn settle ignores. + const chromeRefs = collectSettleChromeRefs(nodes, appBundleId); + for (const identifier of [ + 'com.android.systemui:id/split_shade_status_bar', + 'com.android.systemui:id/clock', + 'com.android.systemui:id/wifi_signal', + ]) { + assert.equal( + chromeRefs.has(refForIdentifier(nodes, identifier)), + true, + `${identifier} ${shape}`, + ); + assert.equal(keptIdentifiers.has(identifier), false, `${identifier} ${shape}`); + } + } +}); diff --git a/src/core/snapshot-chrome.ts b/src/core/snapshot-chrome.ts index ff0909ba3..669b4314e 100644 --- a/src/core/snapshot-chrome.ts +++ b/src/core/snapshot-chrome.ts @@ -1,6 +1,7 @@ import { ANDROID_SYSTEM_CHROME_PACKAGE, - isAndroidSystemChromeResourceId, + hasAndroidSystemChromeProvenance, + isAndroidSystemChromeWindowResourceId, } from '../contracts/android-system-chrome.ts'; import type { SnapshotNode } from '../kernel/snapshot.ts'; import { isAndroidInputMethodSnapshotNode } from '../snapshot/android-input-method-overlays.ts'; @@ -202,18 +203,23 @@ function collectSubtreeIndexes( return indexes; } -// SystemUI hosts BOTH persistent chrome and actionable overlays (volume -// panel, media/output pickers), so chrome is never a package-level fact. -// Within `com.android.systemui`, only window-runs carrying a status-bar or -// navigation-bar marker resource-id drop; every other systemui surface is -// kept. Marker set live-verified on the emulator: the status-bar window -// carries `status_bar*` ids throughout while the VolumeDialog window carries -// only `volume_dialog*` ids (`input_method_nav*` bars are IME-owned and -// handled by the IME tier). The marker constants and the resource-id-level -// predicate live in `contracts/android-system-chrome.ts` so the Android -// helper content classifier reuses the same classification. -function hasAndroidSystemChromeMarker(node: SnapshotNode): boolean { - return isAndroidSystemChromeResourceId(node.identifier ?? ''); +// A node is chrome when it sits in the status-bar/nav-bar container's subtree, +// read from whichever source still has the container: the tree when it survived +// the walk, else the `systemChrome` the walk stamped before dropping it. +function isAndroidSystemChromeNode( + node: SnapshotNode, + byIndex: Map, +): boolean { + if (node.bundleId !== ANDROID_SYSTEM_CHROME_PACKAGE) return false; + if (hasAndroidSystemChromeProvenance(node)) return true; + const seen = new Set(); + let current: SnapshotNode | undefined = node; + while (current?.bundleId === ANDROID_SYSTEM_CHROME_PACKAGE && !seen.has(current.index)) { + if (isAndroidSystemChromeWindowResourceId(current.identifier)) return true; + seen.add(current.index); + current = current.parentIndex === undefined ? undefined : byIndex.get(current.parentIndex); + } + return false; } /** @@ -249,7 +255,7 @@ function collectAndroidSettleChrome( const systemChromeIndexes = appBundleId === ANDROID_SYSTEM_CHROME_PACKAGE ? new Set() - : collectAndroidSystemChromeRunIndexes(nodes, byIndex, imeIndexes); + : collectAndroidSystemChromeIndexes(nodes, byIndex, imeIndexes); // The one surviving container line per IME run; the rest of the run and all // status/nav-bar chrome never spend diff/tail budget. const strippedIndexes = new Set( @@ -268,47 +274,21 @@ function collectAndroidSettleChrome( } /** - * Systemui window-runs (contiguous same-package parent chains) that contain a - * status/nav-bar marker anywhere in the run. The whole marked run drops — - * unmarked wrappers above `status_bar_container` churn with the bar itself — - * while unmarked runs (volume panel, media pickers) are kept whole. + * Systemui nodes belonging to the status-bar / navigation-bar window. Chrome + * identity is per node and intrinsic, so an actionable systemui surface sharing + * a window with the status bar (an expanded quick-settings shade hosts both, + * #1318/#1319) keeps its own nodes. */ -function collectAndroidSystemChromeRunIndexes( +function collectAndroidSystemChromeIndexes( nodes: SnapshotNode[], byIndex: Map, imeIndexes: ReadonlySet, ): Set { - const systemUiIndexes = new Set( + return new Set( nodes - .filter( - (node) => node.bundleId === ANDROID_SYSTEM_CHROME_PACKAGE && !imeIndexes.has(node.index), - ) + .filter((node) => !imeIndexes.has(node.index) && isAndroidSystemChromeNode(node, byIndex)) .map((node) => node.index), ); - if (systemUiIndexes.size === 0) return new Set(); - // Union-find-lite: each systemui node resolves to its run root (the nearest - // ancestor chain member whose parent is absent or not systemui). - const runRootByIndex = new Map(); - const resolveRunRoot = (index: number): number => { - const cached = runRootByIndex.get(index); - if (cached !== undefined) return cached; - const parentIndex = byIndex.get(index)?.parentIndex; - const root = - parentIndex !== undefined && systemUiIndexes.has(parentIndex) - ? resolveRunRoot(parentIndex) - : index; - runRootByIndex.set(index, root); - return root; - }; - const markedRunRoots = new Set( - [...systemUiIndexes] - .filter((index) => { - const node = byIndex.get(index); - return node !== undefined && hasAndroidSystemChromeMarker(node); - }) - .map((index) => resolveRunRoot(index)), - ); - return new Set([...systemUiIndexes].filter((index) => markedRunRoots.has(resolveRunRoot(index)))); } /** iOS keyboard-window chrome unioned with Android IME/system chrome. */ diff --git a/src/daemon/handlers/__tests__/session-replay-divergence.test.ts b/src/daemon/handlers/__tests__/session-replay-divergence.test.ts index 3156c155a..825253edf 100644 --- a/src/daemon/handlers/__tests__/session-replay-divergence.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-divergence.test.ts @@ -21,6 +21,7 @@ vi.mock('../../../utils/timeouts.ts', async (importOriginal) => { }); import { dispatchCommand } from '../../../core/dispatch.ts'; +import type { RawSnapshotNode } from '../../../kernel/snapshot.ts'; import { makeAndroidSession, makeIosSession, @@ -1060,17 +1061,22 @@ test('buildReplayFailureDivergence: divergence capture drops the action snapshot expect(context?.snapshotInteractiveOnly).toBe(true); }); -// Wave-3 leg E (real walked fixture): a FULL-COVER quick-settings shade. Unlike -// the partial-cover overlay above, the shade owns the whole screen — every node -// is systemui and the status-bar icons share the shade's own window, so the -// run-level chrome rule (`collectAndroidSystemChromeRunIndexes`) condemns the -// ENTIRE capture, tiles included. The chrome filter must stay a FILTER and never -// become a narrower scoping (`captureDivergenceObservation`): a plain `snapshot` -// of this exact surface shows the tiles (#1301 `systemSurfaceOnly` carve-out), so -// a divergence publishing zero refs would be strictly NARROWER than `snapshot` — -// the invariant that amendment forbids. Live-verified 2026-07-17 on emulator-5556: -// pre-fix this screen came back with 0 refs. -test('buildReplayFailureDivergence: a full-cover quick-settings shade still publishes its hittable tiles (wave-3 leg E)', async () => { +// Real walked FULL-COVER quick-settings shade: every node is systemui and the +// status-bar icons share the shade's window. The second scenario pessimistically +// marks the whole tree as chrome to model older/OEM layouts whose status-bar +// container also owns real controls; the last-resort hittable fallback must keep +// replay repair actionable without promoting non-hittable status residue. +test.each([ + { + name: 'classifies the shade precisely', + prepare: (nodes: RawSnapshotNode[]) => nodes, + }, + { + name: 'survives a whole-tree chrome false positive', + prepare: (nodes: RawSnapshotNode[]) => + nodes.map((node) => ({ ...node, systemChrome: true as const })), + }, +])('buildReplayFailureDivergence: a full-cover quick-settings shade $name', async ({ prepare }) => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-divergence-qsshade-')); const sessionStore = new SessionStore(path.join(root, 'sessions')); const sessionName = 'default'; @@ -1079,7 +1085,7 @@ test('buildReplayFailureDivergence: a full-cover quick-settings shade still publ makeAndroidSession(sessionName, { appBundleId: 'com.google.android.deskclock' }), ); - const walked = walkNonRawAndroidFixture(ANDROID_QS_SHADE_CAPTURE_RAW_NODES); + const walked = prepare(walkNonRawAndroidFixture(ANDROID_QS_SHADE_CAPTURE_RAW_NODES)); mockDispatchCommand.mockResolvedValue({ nodes: walked, truncated: false, backend: 'android' }); const action = { @@ -1120,9 +1126,8 @@ test('buildReplayFailureDivergence: a full-cover quick-settings shade still publ expect(screen.refs.length).toBe(20); // SCREEN_REF_CAPTURE_LIMIT, from 23 hittable tiles expect(screen.truncated).toBe(true); - // Still a filter, not a blanket chrome dump: EVERY published ref is a node an - // agent can actually act on, so the non-hittable status residue in the SAME - // condemned run (battery/wifi/mobile icons) never rides along. + // Every published ref is actionable; non-hittable status residue in the same + // systemui window (battery/wifi/mobile icons) never rides along. const published = new Map( (sessionStore.get(sessionName)?.snapshot?.nodes ?? []).map((node) => [node.ref, node]), ); @@ -1130,22 +1135,3 @@ test('buildReplayFailureDivergence: a full-cover quick-settings shade still publ expect(screen.refs.some((ref) => ref.label === 'Battery charging, 100 percent.')).toBe(false); expect(screen.refs.some((ref) => ref.label === 'Wifi signal full.')).toBe(false); }); - -// The hittable gate on the chrome fallback, pinned against the OTHER real -// capture: an ordinary COLLAPSED status bar (with app content present) must not -// be promoted. Its clock is labeled but NOT hittable in the real capture, while -// the expanded shade's big clock IS — which is exactly why hittability, not -// label presence, is the gate. Without this the fallback would publish a bare -// "7:03" as the screen whenever an app tree yielded no meaningful targets. -test('buildReplayFailureDivergence: the chrome fallback never promotes a non-hittable collapsed status bar', async () => { - const collapsedClock = ANDROID_IME_CAPTURE_RAW_NODES.find( - (node) => node.identifier === 'com.android.systemui:id/clock', - ); - expect(collapsedClock?.label).toBe('7:03'); - expect(collapsedClock?.hittable).not.toBe(true); - - const shadeClock = ANDROID_QS_SHADE_CAPTURE_RAW_NODES.find( - (node) => node.identifier === 'com.android.systemui:id/clock', - ); - expect(shadeClock?.hittable).toBe(true); -}); diff --git a/src/daemon/handlers/session-replay-divergence.ts b/src/daemon/handlers/session-replay-divergence.ts index 06f541fbe..eb51475d1 100644 --- a/src/daemon/handlers/session-replay-divergence.ts +++ b/src/daemon/handlers/session-replay-divergence.ts @@ -381,12 +381,9 @@ function selectDivergenceScreenRefNodes( const nonChrome = nodes.filter( (node) => node.ref && !chromeRefs.has(node.ref) && isMeaningfulDivergenceTarget(node), ); - // Chrome fallback: an expanded shade is a single systemui run, so its own status - // icons condemn its tiles with it. Excluding chrome is a FILTER, never a narrower - // scoping (`captureDivergenceObservation`), so when the chrome runs ARE the - // screen, publish what an agent can still act on rather than a divergence - // narrower than a plain `snapshot`. Hittable, not labelled: a status clock is not - // a target, so a chrome-only screen still publishes nothing. + // Keep replay repair actionable when an older Android/OEM hierarchy puts a + // status/nav id on a container that also owns real controls. This activates + // only when the chrome classifier would otherwise erase the entire screen. const meaningful = nonChrome.length > 0 ? nonChrome : nodes.filter((node) => node.ref && node.hittable === true); // Occlusion fallback (#1264): a `covered` node is normally dropped — an agent diff --git a/src/daemon/selector-recording.ts b/src/daemon/selector-recording.ts index 4abe008e9..628fecc0d 100644 --- a/src/daemon/selector-recording.ts +++ b/src/daemon/selector-recording.ts @@ -1,4 +1,6 @@ import type { DaemonRequest } from './types.ts'; +import type { SnapshotNode } from '../kernel/snapshot.ts'; +import { stripAndroidSystemChromeProvenanceFromNode } from '../contracts/android-system-chrome.ts'; import { SessionStore } from './session-store.ts'; import { isInteractiveObservation } from './session-action-recorder.ts'; import { computeTargetEvidence, type RecordedTargetCapture } from './session-target-evidence.ts'; @@ -20,7 +22,12 @@ export function buildFindRecordResult( }; } -export function toDaemonFindData(result: Record): Record { +type DaemonFindResult = + | { kind: 'found'; waitedMs?: number } + | { kind: 'text'; ref: string; text: string; node: SnapshotNode } + | { kind: 'attrs'; ref: string; node: SnapshotNode }; + +export function toDaemonFindData(result: DaemonFindResult): Record { if (result.kind === 'found') { return { found: true, @@ -28,9 +35,9 @@ export function toDaemonFindData(result: Record): Record): Record { - const target = getResolvedTarget(result); +type DaemonGetResult = { + target: { kind: 'ref'; ref: string } | { kind: 'selector'; selector: string }; + text?: string; + node: SnapshotNode; +}; + +export function toDaemonGetData(result: DaemonGetResult): Record { + const { target } = result; return { - ...(target?.kind === 'ref' ? { ref: normalizeDaemonRef(target.ref) } : {}), - ...(target?.kind === 'selector' ? { selector: target.selector } : {}), + ...(target.kind === 'ref' ? { ref: normalizeDaemonRef(target.ref) } : {}), + ...(target.kind === 'selector' ? { selector: target.selector } : {}), ...(typeof result.text === 'string' ? { text: result.text } : {}), - ...(result.node && typeof result.node === 'object' ? { node: result.node } : {}), + node: stripAndroidSystemChromeProvenanceFromNode(result.node), }; } diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 9ddea48b2..2c00a5c11 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -492,16 +492,16 @@ function buildDirectIosGetResult( property: 'text' | 'attrs', selector: string, result: DirectIosSelectorQueryResult, -): Record | null { +) { if (!result.found || !result.node) return null; const base = { target: { kind: 'selector' as const, selector }, node: result.node, selectorChain: [selector], }; - if (property === 'attrs') return { kind: 'attrs', ...base }; + if (property === 'attrs') return { kind: 'attrs' as const, ...base }; if (typeof result.text !== 'string') return null; - return { kind: 'text', ...base, text: result.text }; + return { kind: 'text' as const, ...base, text: result.text }; } function buildDirectIosIsResult( diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index 11d8027a3..be399afeb 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -21,6 +21,7 @@ import { maybeBuildAndroidSnapshotTimeoutFailure } from './android-snapshot-time import { summarizeSnapshotDiagnostics } from '../snapshot-diagnostics.ts'; import { nextSnapshotGeneration } from './session-snapshot.ts'; import { activateCompleteRefFrame } from './ref-frame.ts'; +import { stripAndroidSystemChromeProvenance } from '../contracts/android-system-chrome.ts'; export async function dispatchSnapshotViaRuntime(params: { req: DaemonRequest; @@ -47,8 +48,13 @@ export async function dispatchSnapshotViaRuntime(params: { // capture above already stored the next session via setRecord, so the // store holds the generation these refs were minted from. const refsGeneration = params.sessionStore.get(sessionName)?.snapshotGeneration; + // ADR 0014: retain provenance in the immutable operational/ref-frame tree; + // project only the published copy so settle and replay keep the full evidence. + const publicNodes = stripAndroidSystemChromeProvenance(result.nodes); + const publicResult = + publicNodes === result.nodes ? result : { ...result, nodes: publicNodes }; return { - data: refsGeneration === undefined ? result : { ...result, refsGeneration }, + data: refsGeneration === undefined ? publicResult : { ...publicResult, refsGeneration }, record: { kind: 'snapshot', nodes: result.nodes.length, diff --git a/src/platforms/android/__tests__/snapshot-content-recovery.test.ts b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts index 781be22d4..e87b5566e 100644 --- a/src/platforms/android/__tests__/snapshot-content-recovery.test.ts +++ b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts @@ -320,24 +320,29 @@ test('rejects an active navigation-bar window with three-button chrome', () => { windowActive: true, packageName: 'com.android.systemui', className: 'android.widget.FrameLayout', - }), - node({ - text: 'Back', - resourceId: 'com.android.systemui:id/back', - packageName: 'com.android.systemui', - className: 'android.widget.ImageView', - }), - node({ - text: 'Home', - resourceId: 'com.android.systemui:id/home', - packageName: 'com.android.systemui', - className: 'android.widget.ImageView', - }), - node({ - text: 'Overview', - resourceId: 'com.android.systemui:id/recent_apps', - packageName: 'com.android.systemui', - className: 'android.widget.ImageView', + // Chrome identity lives on the container, as on a device: the leaves + // below carry no marker of their own. + resourceId: 'com.android.systemui:id/navigation_bar_frame', + children: [ + node({ + text: 'Back', + resourceId: 'com.android.systemui:id/back', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + node({ + text: 'Home', + resourceId: 'com.android.systemui:id/home', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + node({ + text: 'Overview', + resourceId: 'com.android.systemui:id/recent_apps', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + ], }), ]), { @@ -361,24 +366,27 @@ test('rejects an active status-chrome window with clock, battery, and signal ico windowActive: true, packageName: 'com.android.systemui', className: 'android.widget.FrameLayout', - }), - node({ - text: '7:52', - resourceId: 'com.android.systemui:id/clock', - packageName: 'com.android.systemui', - className: 'android.widget.TextView', - }), - node({ - text: 'Battery 100 percent', - resourceId: 'com.android.systemui:id/battery', - packageName: 'com.android.systemui', - className: 'android.widget.LinearLayout', - }), - node({ - text: 'Wifi signal full', - resourceId: 'com.android.systemui:id/wifi_signal', - packageName: 'com.android.systemui', - className: 'android.widget.ImageView', + resourceId: 'com.android.systemui:id/status_bar_container', + children: [ + node({ + text: '7:52', + resourceId: 'com.android.systemui:id/clock', + packageName: 'com.android.systemui', + className: 'android.widget.TextView', + }), + node({ + text: 'Battery 100 percent', + resourceId: 'com.android.systemui:id/battery', + packageName: 'com.android.systemui', + className: 'android.widget.LinearLayout', + }), + node({ + text: 'Wifi signal full', + resourceId: 'com.android.systemui:id/wifi_signal', + packageName: 'com.android.systemui', + className: 'android.widget.ImageView', + }), + ], }), ]), { @@ -473,9 +481,14 @@ function node(options: { className: string; windowType?: number; windowActive?: boolean; + /** Nested children, so chrome leaves can sit inside their status/nav-bar container as on a device. */ + children?: string[]; }): string { const windowAttrs = (options.windowType === undefined ? '' : ` window-type="${options.windowType}"`) + (options.windowActive === undefined ? '' : ` window-active="${options.windowActive}"`); - return ``; + const open = `` + : `${open}>${options.children.join('')}`; } diff --git a/src/platforms/android/snapshot-content-recovery.ts b/src/platforms/android/snapshot-content-recovery.ts index c33ddbf0d..8dd59dbc9 100644 --- a/src/platforms/android/snapshot-content-recovery.ts +++ b/src/platforms/android/snapshot-content-recovery.ts @@ -1,6 +1,6 @@ import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; import { isAndroidInputMethodOwnedNode } from '../../contracts/android-input-ownership.ts'; -import { isAndroidSystemChromeResourceId } from '../../contracts/android-system-chrome.ts'; +import { hasAndroidSystemChromeProvenance } from '../../contracts/android-system-chrome.ts'; import { classifyAndroidAlertIdentifier } from './alert-detection.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; @@ -299,7 +299,7 @@ function recordMeaningfulWindowOwnership( // Status/nav-bar chrome must not satisfy the system-surface floor: an active navigation bar // (Back + Home + Recents) or status chrome (clock, battery, signal icons) is missing-app-content // residue, not a usable shade/quick-settings surface. Only non-chrome nodes count. - if (isInActiveSystemSurfaceWindow(summary) && !isAndroidSystemChromeResourceId(node.resourceId)) { + if (isInActiveSystemSurfaceWindow(summary) && !hasAndroidSystemChromeProvenance(node)) { summary.activeSystemSurfaceMeaningfulNodeCount += 1; } } diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index a9d44eea2..7d5b448f5 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -2,12 +2,19 @@ import type { RawSnapshotNode, Rect, SnapshotOptions } from '../../kernel/snapsh import { parseBounds } from '../../utils/bounds.ts'; import { isScrollableType } from '../../utils/scrollable.ts'; import { intersectArea } from '../../utils/screenshot-geometry.ts'; +import { + type AndroidSystemChromeProvenance, + isAndroidSystemChromeWindowResourceId, +} from '../../contracts/android-system-chrome.ts'; + +type AndroidRawSnapshotNode = RawSnapshotNode & AndroidSystemChromeProvenance; export type AndroidSnapshotAnalysis = { rawNodeCount: number; maxDepth: number; }; +/** Parsed node metadata plus status/navigation subtree provenance when present. */ export type AndroidUiNodeMetadata = { text: string | null; desc: string | null; @@ -32,14 +39,45 @@ export type AndroidUiNodeMetadata = { windowActive?: boolean; windowFocused?: boolean; windowRect?: Rect; -}; +} & AndroidSystemChromeProvenance; + +/** + * Membership of the status-bar/nav-bar subtree over a `` token stream: the + * container id is the only chrome signal in the tree, its clock/battery/wifi + * leaves carrying no marker of their own. + */ +function createAndroidChromeSubtreeTracker() { + const openElements: boolean[] = []; + const inChromeNow = (): boolean => openElements[openElements.length - 1] === true; + return { + /** Enters an opening tag; returns whether that node is inside the chrome subtree. */ + open(resourceId: string | null | undefined, selfClosing: boolean): boolean { + const inChrome = inChromeNow() || isAndroidSystemChromeWindowResourceId(resourceId); + if (!selfClosing) openElements.push(inChrome); + return inChrome; + }, + /** Handles ``. */ + close(): void { + openElements.pop(); + }, + }; +} +/** Streams `` metadata in document order, carrying status-bar/nav-bar provenance. */ export function* androidUiNodes(xml: string): IterableIterator { - const nodeRegex = /]*>/g; - let match = nodeRegex.exec(xml); + const tokenRegex = /]*>|<\/node>/g; + const chrome = createAndroidChromeSubtreeTracker(); + let match = tokenRegex.exec(xml); while (match) { - yield readAndroidUiNodeMetadata(match[0]); - match = nodeRegex.exec(xml); + const token = match[0]; + if (token.startsWith('')); + yield inChrome ? { ...metadata, systemChrome: true } : metadata; + } + match = tokenRegex.exec(xml); } } @@ -67,14 +105,14 @@ export function parseUiHierarchy( } export type AndroidBuiltSnapshot = { - nodes: RawSnapshotNode[]; + nodes: AndroidRawSnapshotNode[]; sourceNodes: AndroidUiHierarchy[]; truncated?: boolean; analysis: AndroidSnapshotAnalysis; }; type AndroidSnapshotBuildState = { - nodes: RawSnapshotNode[]; + nodes: AndroidRawSnapshotNode[]; sourceNodes: AndroidUiHierarchy[]; maxNodes?: number; maxDepth: number; @@ -115,6 +153,16 @@ export function buildUiHierarchySnapshot( return state.truncated ? { ...snapshot, truncated: true } : snapshot; } +/** + * Chrome provenance is stamped HERE, while the tree still has the wrapper that + * carries it. `shouldIncludeAndroidNode` drops `status_bar*`/`navigation_bar*` + * wrappers and re-parents their children upward, so downstream classifiers see + * a clock/battery/wifi leaf sitting next to real content with nothing left to + * say which region it came from. Recording it on the way down (like + * `ancestorHittable`) means the answer is identical in every capture shape — + * `--raw` keeps the wrapper, a default capture drops it, both stamp the same + * descendants. + */ function walkUiHierarchyNode( state: AndroidSnapshotBuildState, node: AndroidNode, @@ -122,6 +170,7 @@ function walkUiHierarchyNode( parentIndex?: number, ancestorHittable: boolean = false, ancestorCollection: boolean = false, + ancestorSystemChrome: boolean = false, ): void { if (state.maxNodes !== undefined && state.nodes.length >= state.maxNodes) { state.truncated = true; @@ -138,8 +187,10 @@ function walkUiHierarchyNode( hasInteractiveDescendant(state, node), ancestorCollection, ); + const systemChrome = + ancestorSystemChrome || isAndroidSystemChromeWindowResourceId(node.identifier); const currentIndex = include - ? appendAndroidSnapshotNode(state, node, depth, parentIndex) + ? appendAndroidSnapshotNode(state, node, depth, parentIndex, systemChrome) : parentIndex; const nextAncestorHittable = ancestorHittable || Boolean(node.hittable); const nextAncestorCollection = ancestorCollection || isCollectionContainerType(node.type); @@ -151,6 +202,7 @@ function walkUiHierarchyNode( currentIndex, nextAncestorHittable, nextAncestorCollection, + systemChrome, ); if (state.truncated) return; } @@ -160,7 +212,8 @@ function appendAndroidSnapshotNode( state: AndroidSnapshotBuildState, node: AndroidNode, depth: number, - parentIndex?: number, + parentIndex: number | undefined, + systemChrome: boolean, ): number { const currentIndex = state.nodes.length; state.sourceNodes.push(node); @@ -180,6 +233,7 @@ function appendAndroidSnapshotNode( parentIndex, ...(node.hiddenContentAbove ? { hiddenContentAbove: true } : {}), ...(node.hiddenContentBelow ? { hiddenContentBelow: true } : {}), + ...(systemChrome ? { systemChrome: true } : {}), }); return currentIndex; } diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 209f5ab4c..15b148639 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -16,6 +16,17 @@ import { createProviderScenarioTempPath, withProviderScenarioResource } from './ type AndroidSettingsWorld = Awaited>; +const ANDROID_SYSTEM_SURFACE_XML = ` + + + + + + + + +`; + test('Provider-backed integration Android Settings flow uses scripted ADB provider', async () => { await withProviderScenarioResource(createAndroidSettingsWorld, async (world) => { const client = world.daemon.client(); @@ -26,6 +37,33 @@ test('Provider-backed integration Android Settings flow uses scripted ADB provid }); }, 15_000); +test('Provider-backed Android reads keep chrome provenance internal across public node payloads', async () => { + await withProviderScenarioResource( + async () => await createAndroidSettingsWorld({ snapshotXml: () => ANDROID_SYSTEM_SURFACE_XML }), + async (world) => { + const client = world.daemon.client(); + await client.apps.open({ app: 'settings', ...world.selection }); + + const response = await world.daemon.callCommand('snapshot', [], world.selection); + const snapshot = assertRpcOk<{ nodes: Array<{ ref: string; label?: string }> }>(response); + + assert.equal( + snapshot.nodes.some((node) => node.label === 'Display brightness'), + true, + ); + + const clock = snapshot.nodes.find((node) => node.label === '7:03'); + assert.ok(clock?.ref); + assertRpcOk( + await world.daemon.callCommand('get', ['attrs', `@${clock.ref}`], world.selection), + ); + assertRpcOk( + await world.daemon.callCommand('find', ['label', '7:03', 'get', 'attrs'], world.selection), + ); + }, + ); +}); + test('Provider-backed integration Android text provider handles Unicode without shell input text', async () => { await withProviderScenarioResource( async () => await createAndroidSettingsWorld({ nativeTextInjection: true }), diff --git a/test/integration/provider-scenarios/harness.ts b/test/integration/provider-scenarios/harness.ts index 196b73a05..b8a2e5e18 100644 --- a/test/integration/provider-scenarios/harness.ts +++ b/test/integration/provider-scenarios/harness.ts @@ -1,3 +1,4 @@ +import assert from 'node:assert/strict'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -54,7 +55,7 @@ export async function createProviderScenarioHarness( path.join(os.tmpdir(), 'agent-device-provider-scenario-session-'), ); const sessionStore = new SessionStore(sessionDir); - const handleRequest = createRequestHandler({ + const requestHandler = createRequestHandler({ logPath: path.join(os.tmpdir(), 'agent-device-provider-scenario-daemon.log'), token: PROVIDER_SCENARIO_TOKEN, sessionStore, @@ -62,6 +63,11 @@ export async function createProviderScenarioHarness( trackDownloadableArtifact, ...deps, }); + const handleRequest: typeof requestHandler = async (request) => { + const response = await requestHandler(request); + assertNoInternalChromeProvenance(response); + return response; + }; const transport: AgentDeviceDaemonTransport = async (req) => await handleRequest({ @@ -186,3 +192,11 @@ function responseToRpcResult(response: DaemonResponse, id: string): ProviderScen }, }; } + +function assertNoInternalChromeProvenance(response: DaemonResponse): void { + assert.equal( + JSON.stringify(response).includes('"systemChrome"'), + false, + 'Public daemon responses must not expose Android-internal systemChrome provenance', + ); +}