Skip to content
Merged
10 changes: 10 additions & 0 deletions src/__tests__/test-utils/android-ui-hierarchy-fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,16 @@ export function walkNonRawAndroidFixture(rawNodes: RawSnapshotNode[]): RawSnapsh
return buildUiHierarchySnapshot(tree, undefined, { raw: false }).nodes;
}

/**
* Runs a fixture through the REAL interactive-only walk — the shape `--settle`
* and `wait stable` consume, since their loop captures with
* `interactiveOnly: true` (`stable-capture.ts`).
*/
export function walkInteractiveOnlyAndroidFixture(rawNodes: RawSnapshotNode[]): RawSnapshotNode[] {
const tree = rawFixtureToAndroidTree(rawNodes);
return buildUiHierarchySnapshot(tree, undefined, { raw: false, interactiveOnly: true }).nodes;
}

/**
* Real device capture (checkout-form fixture app, Gboard open, status bar
* visible) archived at `~/.agent-device-bench/replay-runs/android-ime/raw-ime2.json`
Expand Down
4 changes: 2 additions & 2 deletions src/commands/interaction/runtime/settle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -700,15 +700,15 @@ function androidStatusBarNodes(startIndex: number, clockLabel = '12:23') {
const root = startIndex;
return [
{
// The outermost status-bar node a real capture starts with.
index: root,
depth: 0,
type: 'android.widget.FrameLayout',
identifier: 'com.android.systemui:id/status_bar_launch_animation_container',
bundleId: ANDROID_SYSTEM_UI_BUNDLE_ID,
rect: { x: 0, y: 0, width: 1344, height: 159 },
},
{
// The marker every real status-bar capture carries; the window-run
// drops as a whole because this member matches the status_bar* prefix.
index: root + 1,
depth: 1,
parentIndex: root,
Expand Down
98 changes: 37 additions & 61 deletions src/contracts/android-system-chrome.ts
Original file line number Diff line number Diff line change
@@ -1,69 +1,45 @@
import type { SnapshotNode } from '../kernel/snapshot.ts';

/**
* Android status-bar/navigation-bar chrome markers, shared between the settle-chrome
* classifier (`core/snapshot-chrome.ts`, #1198) and the helper content classifier
* (`platforms/android/snapshot-content-recovery.ts`). SystemUI hosts BOTH persistent chrome
* and actionable overlays (volume panel, media/output pickers, notification shade, quick
* settings), so chrome is never a package-level fact: only these status/nav-bar marker
* resource-ids classify as chrome; every other systemui surface is real content.
* Android status-bar/navigation-bar chrome identity, shared by the settle-chrome
* classifier (`core/snapshot-chrome.ts`) and the helper content classifier
* (`platforms/android/snapshot-content-recovery.ts`). SystemUI also hosts actionable
* overlays — volume panel, media pickers, the shade itself — so chrome is never a
* package-level fact: only the status/nav-bar container's subtree is chrome.
*/
export const ANDROID_SYSTEM_CHROME_PACKAGE = 'com.android.systemui';

const ANDROID_SYSTEM_CHROME_MARKER_PREFIXES = [
'com.android.systemui:id/status_bar',
'com.android.systemui:id/navigation_bar',
] as const;

/**
* Surviving status-bar/nav-bar LEAF ids (#1251). The non-raw Android walk
* (`walkUiHierarchyNode` in `platforms/android/ui-hierarchy.ts`) drops
* unlabeled/unidentified structural nodes via `shouldIncludeStructuralAndroidNode`,
* re-parenting their children upward — and that silently swallows every
* `status_bar*`/`navigation_bar*` WRAPPER node, i.e. the only nodes the prefix
* check above matches. A non-raw capture is left with just their labeled/
* identified LEAVES (clock, battery, wifi/mobile icons, nav buttons), whose
* own resource-ids carry no `status_bar`/`navigation_bar` prefix, so the run
* loses its marker and `collectAndroidSystemChromeRunIndexes` stops dropping
* it (verified against a real `--raw` vs. default capture pair of the same
* screen). Recognize those leaves directly, by EXACT id — not prefix, to stay
* tight: nothing here should ever swallow an actionable systemui overlay like
* the volume dialog or a media/output picker, which live under unrelated ids.
* `--raw` keeps the wrapper markers, so the prefix check above stays
* load-bearing there.
*
* A more robust fix would thread the AOSP window-type constants
* (`TYPE_STATUS_BAR` = 2000, `TYPE_NAVIGATION_BAR` = 2019) already parsed in
* `readNodeAttributes` (ui-hierarchy.ts) through to the output `SnapshotNode`
* and key off that instead of resource-ids — deferred until the id-based
* approach here proves insufficient.
* Resource-ids of the status-bar / navigation-bar WINDOW containers. Matched as an id
* segment, not a prefix, so an expanded shade's `split_shade_status_bar` counts.
*/
const ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS: ReadonlySet<string> = new Set(
[
// status bar
'clock',
'battery',
'statusIcons',
'notificationIcons',
'notification_icon_area',
'system_icons',
'cutout_space_view',
'mobile_signal',
'mobile_combo',
'mobile_group',
'wifi_signal',
'wifi_combo',
'wifi_group',
'start_side_notif_and_chip_container',
// nav bar
'back',
'home',
'recent_apps',
'home_handle',
].map((leaf) => `${ANDROID_SYSTEM_CHROME_PACKAGE}:id/${leaf}`),
);

/** True when the resource-id marks Android status-bar or navigation-bar chrome. */
export function isAndroidSystemChromeResourceId(resourceId: string | null | undefined): boolean {
export function isAndroidSystemChromeWindowResourceId(
resourceId: string | null | undefined,
): boolean {
const identifier = resourceId ?? '';
if (ANDROID_SYSTEM_CHROME_MARKER_LEAF_IDS.has(identifier)) return true;
return ANDROID_SYSTEM_CHROME_MARKER_PREFIXES.some((prefix) => identifier.startsWith(prefix));
if (!identifier.startsWith(`${ANDROID_SYSTEM_CHROME_PACKAGE}:id/`)) return false;
const leaf = identifier.slice(`${ANDROID_SYSTEM_CHROME_PACKAGE}:id/`.length);
return /(^|_)(status_bar|navigation_bar)(_|$)/.test(leaf);
}

/** Android-internal provenance retained in daemon session snapshots only. */
export type AndroidSystemChromeProvenance = {
systemChrome?: true;
};

export function hasAndroidSystemChromeProvenance(value: object): value is { systemChrome: true } {
return 'systemChrome' in value && value.systemChrome === true;
}

/** Drops Android-internal provenance from a node embedded in a public response. */
export function stripAndroidSystemChromeProvenanceFromNode(node: SnapshotNode): SnapshotNode {
if (!hasAndroidSystemChromeProvenance(node)) return node;
const { systemChrome: _systemChrome, ...published } = node;
return published;
}

/** Drops Android-internal provenance at the daemon's public snapshot seam. */
export function stripAndroidSystemChromeProvenance(nodes: SnapshotNode[]): SnapshotNode[] {
if (!nodes.some(hasAndroidSystemChromeProvenance)) return nodes;
return nodes.map(stripAndroidSystemChromeProvenanceFromNode);
}
121 changes: 80 additions & 41 deletions src/core/__tests__/snapshot-chrome-android-statusbar.test.ts
Original file line number Diff line number Diff line change
@@ -1,28 +1,19 @@
import { test } from 'vitest';
import assert from 'node:assert/strict';
import { attachRefs, type RawSnapshotNode, type SnapshotNode } from '../../kernel/snapshot.ts';
import { collectSettleChromeRefs } from '../snapshot-chrome.ts';
import { collectSettleChromeRefs, withoutSettleChrome } from '../snapshot-chrome.ts';
import {
ANDROID_IME_CAPTURE_RAW_NODES,
ANDROID_QS_SHADE_CAPTURE_RAW_NODES,
walkInteractiveOnlyAndroidFixture,
walkNonRawAndroidFixture,
} from '../../__tests__/test-utils/android-ui-hierarchy-fixtures.ts';
import { isAndroidSystemChromeWindowResourceId } from '../../contracts/android-system-chrome.ts';

/**
* `ANDROID_IME_CAPTURE_RAW_NODES` (shared test util, see its own doc comment)
* is a real device `--raw` capture: `--raw` keeps every structural wrapper
* node (`status_bar_container`, `status_bar_contents`, ...) so the OLD
* prefix-only marker check finds them and drops the whole run.
*
* The default (non-raw) walk drops unlabeled/unidentified structural nodes
* (`shouldIncludeStructuralAndroidNode` in `platforms/android/ui-hierarchy.ts`),
* re-parenting their children upward — which silently removes every one of
* those marker-bearing wrappers AND several other anonymous structural nodes
* that have neither a hittable descendant nor meaningful text/id (a real
* `android.view.View` example is `com.android.systemui:id/home_handle`, see
* the nav-bar test below). `walkNonRawAndroidFixture` (shared test util) runs
* the fixture through the REAL `buildUiHierarchySnapshot({ raw: false })`
* walk instead of hand-simulating a subset of its drops, so every inclusion
* decision below is production's, not a guess at which nodes matter.
* The `walk*AndroidFixture` helpers run a real `--raw` device capture through
* production's own walk, so every inclusion decision below is production's
* rather than a hand-simulation of it.
*/

function refForIdentifier(nodes: SnapshotNode[], identifier: string): string {
Expand All @@ -31,29 +22,33 @@ function refForIdentifier(nodes: SnapshotNode[], identifier: string): string {
return node.ref;
}

/**
* Status-bar leaf identifiers that SURVIVE the real non-raw walk for this
* fixture (verified against `buildUiHierarchySnapshot({ raw: false })`
* directly). Several ids the naive prefix-only simulation used to assert
* (`notification_icon_area`, `notificationIcons`, `cutout_space_view`,
* `system_icons`, `statusIcons`, `mobile_group`, `wifi_combo`, `wifi_group`,
* `start_side_notif_and_chip_container`, `battery`) are unlabeled structural
* wrappers with a generic resource id and no hittable descendant: production
* `shouldIncludeStructuralAndroidNode` drops every one of them too, same as
* the `status_bar*`/`navigation_bar*` wrappers. Their content isn't lost —
* `battery`'s labeled child ("Battery 100 percent.") re-parents upward and is
* covered by the "every systemui-owned node is chrome" assertion below — but
* the WRAPPER identifier itself is gone from the walked tree, so it can't be
* looked up by id.
*/
/** Status-bar leaves that survive the non-raw walk for this fixture; none carries a chrome id. */
const STATUS_BAR_LEAF_IDENTIFIERS = [
'com.android.systemui:id/clock',
'com.android.systemui:id/mobile_combo',
'com.android.systemui:id/mobile_signal',
'com.android.systemui:id/wifi_signal',
];

test('Android non-raw capture: status-bar leaves are recognized as chrome once their status_bar*/navigation_bar* marker wrapper is dropped by the walk (#1251)', () => {
test('Android chrome container ids match complete status/nav-bar segments only', () => {
assert.equal(
isAndroidSystemChromeWindowResourceId(
'com.android.systemui:id/status_bar_launch_animation_container',
),
true,
);
assert.equal(
isAndroidSystemChromeWindowResourceId('com.android.systemui:id/split_shade_status_bar'),
true,
);
assert.equal(
isAndroidSystemChromeWindowResourceId('com.android.systemui:id/status_barometer'),
false,
);
assert.equal(isAndroidSystemChromeWindowResourceId('com.example:id/status_bar'), false);
});

test('Android non-raw capture: status-bar leaves stay chrome after the walk drops the container that identifies them (#1251)', () => {
const walkedNodes = walkNonRawAndroidFixture(ANDROID_IME_CAPTURE_RAW_NODES);
const nodes = attachRefs(walkedNodes);
const chromeRefs = collectSettleChromeRefs(nodes, 'com.callstack.agentdevicelab');
Expand Down Expand Up @@ -105,15 +100,9 @@ test('Android non-raw capture: status-bar leaves are recognized as chrome once t
});

test('Android actionable systemui overlay (volume dialog) still survives the chrome filter (#1251)', () => {
// Filter-logic unit test, NOT a live-capture-path claim (#1264 finding 2): this
// exercises `collectSettleChromeRefs` in isolation over a synthetic systemui
// run appended to an unrelated capture fixture. The leaf ids ARE real,
// live-verified volume-dialog ids (`volume_dialog_container`,
// `volume_new_ringer_active_icon_container`), but appending them to this
// fixture does not reproduce how a live capture reaches the filter — that is
// covered separately by the full-capture invariant test below. The point
// here is narrower: the status-bar/nav-bar leaf-id set (#1251) must NOT
// broaden to "any systemui id" and drop an actionable overlay it is handed.
// Real volume-dialog ids grafted onto an unrelated capture (#1264 finding 2:
// synthetic placement, real ids). Chrome must stay a status/nav-bar fact and
// never widen to "any systemui id", which would swallow this overlay.
const rawWithVolumeDialog: RawSnapshotNode[] = [
...ANDROID_IME_CAPTURE_RAW_NODES,
{
Expand Down Expand Up @@ -235,3 +224,53 @@ test('Android nav-bar leaves are recognized as chrome once their navigation_bar*
'expected home_handle to be dropped by the real non-raw walk, not retained',
);
});

/**
* A fully expanded quick-settings shade hosts the status-bar icons and the
* quick-settings tiles in ONE window. Chrome classification must split them the
* same way whatever shape the capture arrives in — that is what #1318 (raw:
* every tile condemned) and #1319 (interactive-only: tiles kept) each hit from
* one side.
*
* Live-verified on emulator-5554 (Pixel 9 Pro XL API 37, deskclock): raising
* the shade mid-`--settle` diffs `+28 -25`, closing it `+25 -28`, and the
* shade's own status bar is absent from both while `snapshot -i` of the same
* screen still lists it.
*/
test('Android expanded quick-settings shade: tiles survive and the status bar drops, identically in every capture shape (#1318/#1319)', () => {
const appBundleId = 'com.google.android.deskclock';

for (const [shape, walk] of [
['interactive-only (--settle, wait stable)', walkInteractiveOnlyAndroidFixture],
['non-raw (snapshot, replay divergence)', walkNonRawAndroidFixture],
] as const) {
const nodes = attachRefs(walk(ANDROID_QS_SHADE_CAPTURE_RAW_NODES));
const kept = withoutSettleChrome(nodes, appBundleId);
const keptIdentifiers = new Set(kept.map((node) => node.identifier));

// The shade's actionable content survives, so opening or closing it can
// never read as "nothing changed".
assert.equal(keptIdentifiers.has('com.android.systemui:id/slider'), true, shape);
assert.equal(
kept.some((node) => node.identifier?.startsWith('com.android.systemui:id/qs_tile_')),
true,
`expected the quick-settings tiles to survive chrome stripping — ${shape}`,
);

// ...while the status-bar subtree sharing that window still drops: clock,
// date and wifi tick constantly and are the canonical churn settle ignores.
const chromeRefs = collectSettleChromeRefs(nodes, appBundleId);
for (const identifier of [
'com.android.systemui:id/split_shade_status_bar',
'com.android.systemui:id/clock',
'com.android.systemui:id/wifi_signal',
]) {
assert.equal(
chromeRefs.has(refForIdentifier(nodes, identifier)),
true,
`${identifier} ${shape}`,
);
assert.equal(keptIdentifiers.has(identifier), false, `${identifier} ${shape}`);
}
}
});
Loading
Loading