From 2da93f91e10aa175975e58a57dee9226a9f213f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 29 Jul 2026 12:53:43 +0200 Subject: [PATCH 01/20] test(android): add catalog emulator smoke coverage --- .github/workflows/android.yml | 12 +- .../src/screens/AutomationLabScreen.tsx | 2 +- .../dispatch-trigger-app-event.test.ts | 2 +- src/platforms/android/__tests__/alert.test.ts | 32 +++ .../__tests__/app-lifecycle-open.test.ts | 14 +- src/platforms/android/alert.ts | 1 - src/platforms/android/app-lifecycle.ts | 10 +- test/ci/trusted-fixture-artifact.test.mjs | 6 +- .../android-emulator-e2e/behavior-coverage.ts | 42 ++++ .../android-emulator-e2e/coverage-manifest.ts | 234 ++++++++++++++++++ .../android-emulator-e2e/live-assertions.ts | 70 ++++++ .../live-automation-scenario.ts | 207 ++++++++++++++++ .../live-coverage-report.ts | 64 +++++ .../live-form-scenario.ts | 73 ++++++ .../android-emulator-e2e/live-harness.ts | 222 +++++++++++++++++ .../live-inventory-scenario.ts | 42 ++++ .../android-emulator-e2e/live-runner.ts | 87 +++++++ .../android-emulator-e2e/scenarios.ts | 44 ++++ .../android-lifecycle.test.ts | 4 +- .../smoke-android-emulator-coverage.test.ts | 134 ++++++++++ .../smoke-android-emulator.test.ts | 15 ++ 21 files changed, 1303 insertions(+), 14 deletions(-) create mode 100644 src/platforms/android/__tests__/alert.test.ts create mode 100644 test/integration/android-emulator-e2e/behavior-coverage.ts create mode 100644 test/integration/android-emulator-e2e/coverage-manifest.ts create mode 100644 test/integration/android-emulator-e2e/live-assertions.ts create mode 100644 test/integration/android-emulator-e2e/live-automation-scenario.ts create mode 100644 test/integration/android-emulator-e2e/live-coverage-report.ts create mode 100644 test/integration/android-emulator-e2e/live-form-scenario.ts create mode 100644 test/integration/android-emulator-e2e/live-harness.ts create mode 100644 test/integration/android-emulator-e2e/live-inventory-scenario.ts create mode 100644 test/integration/android-emulator-e2e/live-runner.ts create mode 100644 test/integration/android-emulator-e2e/scenarios.ts create mode 100644 test/integration/smoke-android-emulator-coverage.test.ts create mode 100644 test/integration/smoke-android-emulator.test.ts diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 155aff259..475578526 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -51,6 +51,9 @@ jobs: - name: Report fixture cache source run: echo "Android fixture APK source=${{ steps.fixture-app.outputs.source }}" + - name: Run Android emulator catalog coverage contract + run: node --test test/integration/smoke-android-emulator-coverage.test.ts + - name: Run Android smoke checks uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 with: @@ -60,7 +63,14 @@ jobs: target: google_apis_playstore emulator-options: -no-window -gpu swiftshader_indirect -no-snapshot -noaudio -no-boot-anim -no-metrics script: | - sh ./test/scripts/android-fixture-cache-smoke.sh "${{ steps.fixture-app.outputs.apk-path }}" "${{ steps.fixture-app.outputs.app-id }}" + set -eu + ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" + test -n "$ANDROID_SERIAL" + pnpm build + pnpm clean:daemon + AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --test test/integration/smoke-android-emulator.test.ts + COVERAGE_REPORT="$(find test/artifacts/android-emulator -name coverage-report.json -print | tail -n 1)" + node --input-type=module -e 'import fs from "node:fs"; const report = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); console.log(`Android emulator scenario body: ${report.timings.scenarioBodyDurationMs}ms`); for (const scenario of report.timings.scenarios) console.log(` ${scenario.id}: ${scenario.durationMs}ms`);' "$COVERAGE_REPORT" node --experimental-strip-types src/bin.ts test test/integration/replays/android/01-settings.ad --retries 2 --report-junit test/artifacts/replays-android-smoke.junit.xml sh ./test/scripts/android-snapshot-helper-release-smoke.sh diff --git a/examples/test-app/src/screens/AutomationLabScreen.tsx b/examples/test-app/src/screens/AutomationLabScreen.tsx index e95022926..e0a138c4e 100644 --- a/examples/test-app/src/screens/AutomationLabScreen.tsx +++ b/examples/test-app/src/screens/AutomationLabScreen.tsx @@ -103,6 +103,7 @@ export function AutomationLabScreen(props: { testID="automation-title" /> + - diff --git a/src/core/__tests__/dispatch-trigger-app-event.test.ts b/src/core/__tests__/dispatch-trigger-app-event.test.ts index 98f8cc1a6..fe0b51e81 100644 --- a/src/core/__tests__/dispatch-trigger-app-event.test.ts +++ b/src/core/__tests__/dispatch-trigger-app-event.test.ts @@ -94,7 +94,7 @@ test('trigger-app-event opens deep link with encoded event payload', async () => const args = (await fs.readFile(argsLogPath, 'utf8')).trim().split('\n').filter(Boolean); assert.equal(args.includes('-d'), true); - assert.equal(args.includes(expectedUrl), true); + assert.equal(args.includes(`'${expectedUrl}'`), true); } finally { process.env.PATH = previousPath; if (previousArgsFile === undefined) delete process.env.AGENT_DEVICE_TEST_ARGS_FILE; diff --git a/src/platforms/android/__tests__/alert.test.ts b/src/platforms/android/__tests__/alert.test.ts new file mode 100644 index 000000000..c4a74494d --- /dev/null +++ b/src/platforms/android/__tests__/alert.test.ts @@ -0,0 +1,32 @@ +import { afterEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; + +vi.mock('../snapshot.ts', () => ({ snapshotAndroid: vi.fn() })); + +import { handleAndroidAlert } from '../alert.ts'; +import { snapshotAndroid } from '../snapshot.ts'; +import type { DeviceInfo } from '../../../kernel/device.ts'; + +const device: DeviceInfo = { + booted: true, + id: 'emulator-5554', + kind: 'emulator', + name: 'Pixel', + platform: 'android', +}; + +afterEach(() => vi.restoreAllMocks()); + +test('Android alert get retains the normal bounded helper settle policy', async () => { + vi.mocked(snapshotAndroid).mockResolvedValue({ + analysis: { maxDepth: 0, rawNodeCount: 0 }, + androidSnapshot: { backend: 'android-helper' }, + nodes: [], + }); + + await handleAndroidAlert(device, 'get'); + + assert.deepEqual(vi.mocked(snapshotAndroid).mock.calls[0]?.[1], { + includeHiddenContentHints: false, + }); +}); diff --git a/src/platforms/android/__tests__/app-lifecycle-open.test.ts b/src/platforms/android/__tests__/app-lifecycle-open.test.ts index e11074b05..748a66638 100644 --- a/src/platforms/android/__tests__/app-lifecycle-open.test.ts +++ b/src/platforms/android/__tests__/app-lifecycle-open.test.ts @@ -310,7 +310,7 @@ test('openAndroidApp ensures Android reverse before IPv6 localhost deep link lau '-a', 'android.intent.action.VIEW', '-d', - 'http://[::1]:8081/status', + "'http://[::1]:8081/status'", ], }, ]); @@ -526,6 +526,18 @@ test('openAndroidApp appends launchArgs to am start for deep link URL opens', as ); }); +test('openAndroidApp quotes deep link URL shell characters', async () => { + await withScriptedAdb( + 'agent-device-android-open-deep-link-shell-characters-', + androidOpenAdbScript(), + async ({ argsLogPath, device }) => { + await openAndroidApp(device, 'myapp://item/42?event=cold.start&source=smoke'); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /-d\n'myapp:\/\/item\/42\?event=cold\.start&source=smoke'/); + }, + ); +}); + test('openAndroidApp appends launchArgs to am start for app-bound URL opens', async () => { await withScriptedAdb( 'agent-device-android-open-launch-args-app-bound-url-', diff --git a/src/platforms/android/alert.ts b/src/platforms/android/alert.ts index 837848402..97f8c7cf8 100644 --- a/src/platforms/android/alert.ts +++ b/src/platforms/android/alert.ts @@ -126,7 +126,6 @@ async function readAndroidAlertCandidate( 'snapshot_capture', async () => await snapshotAndroid(device, { - helperWaitForIdleTimeoutMs: 0, includeHiddenContentHints: false, }), { backend: 'android', purpose: 'alert' }, diff --git a/src/platforms/android/app-lifecycle.ts b/src/platforms/android/app-lifecycle.ts index bdfaaf23c..24af6ee5f 100644 --- a/src/platforms/android/app-lifecycle.ts +++ b/src/platforms/android/app-lifecycle.ts @@ -303,9 +303,9 @@ export type OpenAndroidAppOptions = { // `adb shell` joins its argv with spaces and feeds the result to a device // shell, which re-tokenises. The other `am start` arguments (action, category, // component, etc.) are well-known and never contain shell-significant -// characters, so they round-trip untouched. Launch arguments are user-supplied -// and may contain JSON, spaces, `#`, etc.; each is single-quoted unless it -// consists entirely of safe shell characters. +// characters, so they round-trip untouched. URLs and launch arguments are +// user-supplied and may contain JSON, spaces, `#`, or `&`; each is single-quoted +// unless it consists entirely of safe shell characters. function quoteAndroidShellArg(arg: string): string { if (/^[A-Za-z0-9_@%+=:,./-]+$/.test(arg)) return arg; return `'${arg.replace(/'/g, `'\\''`)}'`; @@ -367,7 +367,7 @@ async function openAndroidDeepLink( '-a', 'android.intent.action.VIEW', '-d', - target, + quoteAndroidShellArg(target), ...androidDeepLinkPackageArgs(options.appBundleId), ...androidLaunchArgs(options), ]); @@ -398,7 +398,7 @@ async function openAndroidAppBoundDeepLink( '-a', 'android.intent.action.VIEW', '-d', - deepLinkUrl, + quoteAndroidShellArg(deepLinkUrl), '-p', resolved, ...androidLaunchArgs(options), diff --git a/test/ci/trusted-fixture-artifact.test.mjs b/test/ci/trusted-fixture-artifact.test.mjs index ee653590d..a6a537af8 100644 --- a/test/ci/trusted-fixture-artifact.test.mjs +++ b/test/ci/trusted-fixture-artifact.test.mjs @@ -64,7 +64,7 @@ test('producer, consumers, upload, and concurrency use the canonical platform-sc ); }); -test('Android smoke keeps its install/open/snapshot evidence in a checked-in script', (t) => { +test('Android smoke consumes the restored APK through catalog fixture E2E', (t) => { const workflow = parse(fs.readFileSync('.github/workflows/android.yml', 'utf8')); const smokeStep = workflow.jobs['smoke-android'].steps.find( (step) => step.name === 'Run Android smoke checks', @@ -79,9 +79,11 @@ test('Android smoke keeps its install/open/snapshot evidence in a checked-in scr const smokeScript = fs.readFileSync('test/scripts/android-fixture-cache-smoke.sh', 'utf8'); const packageVersion = JSON.parse(fs.readFileSync('package.json', 'utf8')).version; - assert.match(smokeStep.with.script, /android-fixture-cache-smoke\.sh/); + assert.match(smokeStep.with.script, /AGENT_DEVICE_ANDROID_E2E=1/); assert.match(smokeStep.with.script, /steps\.fixture-app\.outputs\.apk-path/); assert.match(smokeStep.with.script, /steps\.fixture-app\.outputs\.app-id/); + assert.match(smokeStep.with.script, /smoke-android-emulator\.test\.ts/); + assert.match(smokeStep.with.script, /android-snapshot-helper-release-smoke\.sh/); assert.equal(restoreStep.with['wait-for-artifact-seconds'], '600'); assert.match(sourceStep.run, /steps\.fixture-app\.outputs\.source/); assert.match(smokeScript, /snapshot -i .*--json > "\$SNAPSHOT_PATH"/); diff --git a/test/integration/android-emulator-e2e/behavior-coverage.ts b/test/integration/android-emulator-e2e/behavior-coverage.ts new file mode 100644 index 000000000..91664c918 --- /dev/null +++ b/test/integration/android-emulator-e2e/behavior-coverage.ts @@ -0,0 +1,42 @@ +export type AndroidEmulatorBehaviorId = + | 'android-resource-id-selectors' + | 'cold-start-deep-link-navigation' + | 'home-recents-restoration' + | 'orientation-fixture-state' + | 'safe-keyboard-dismissal'; + +type BehaviorCoverageEntry = { + assertion: string; + level: 'live'; + owner: string; +}; + +export const ANDROID_EMULATOR_BEHAVIOR_COVERAGE = { + 'android-resource-id-selectors': { + assertion: 'fixture test IDs are observed as Android resource IDs before their selectors act', + level: 'live', + owner: 'smoke:automation-system', + }, + 'cold-start-deep-link-navigation': { + assertion: 'cold deep link renders payload and normal navigation returns through Back', + level: 'live', + owner: 'smoke:automation-system', + }, + 'home-recents-restoration': { + assertion: + 'Home and Recents produce distinct Android system evidence before fixture restoration', + level: 'live', + owner: 'smoke:automation-system', + }, + 'orientation-fixture-state': { + assertion: + 'fixture window state observes landscape and portrait after Android rotation commands', + level: 'live', + owner: 'smoke:automation-system', + }, + 'safe-keyboard-dismissal': { + assertion: 'keyboard dismiss hides the IME while Checkout form remains on screen', + level: 'live', + owner: 'smoke:form-input', + }, +} as const satisfies Record; diff --git a/test/integration/android-emulator-e2e/coverage-manifest.ts b/test/integration/android-emulator-e2e/coverage-manifest.ts new file mode 100644 index 000000000..416dc6c70 --- /dev/null +++ b/test/integration/android-emulator-e2e/coverage-manifest.ts @@ -0,0 +1,234 @@ +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; + +type PublicCommand = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]; + +type RepositoryEvidence = { + path: string; + test: string; +}; + +export type AndroidEmulatorCoverageEntry = + | { assertion: string; level: 'live'; owner: string } + | { + assertion: string; + level: 'command-contract' | 'workflow-live' | 'capability-denial'; + owner: RepositoryEvidence; + } + | { assertion: string; level: 'known-gap'; owner: string; trackingIssue: string }; + +const C = PUBLIC_COMMANDS; +const live = (owner: string, assertion: string): AndroidEmulatorCoverageEntry => ({ + assertion, + level: 'live', + owner, +}); +const contract = (path: string, test: string, assertion: string): AndroidEmulatorCoverageEntry => ({ + assertion, + level: 'command-contract', + owner: { path, test }, +}); + +const ANDROID_LIFECYCLE_CONTRACT = { + path: 'test/integration/provider-scenarios/android-lifecycle.test.ts', + test: 'Provider-backed integration Android Settings flow uses scripted ADB provider', +} as const; + +/** One primary, observable owner for every public command on an Android emulator. */ +export const ANDROID_EMULATOR_E2E_COVERAGE = { + [C.alert]: live('smoke:automation-system', 'native alert actions update fixture-visible results'), + [C.appSwitcher]: live( + 'smoke:automation-system', + 'Recents pixels differ from Home and the fixture restores through Android app state', + ), + [C.apps]: live('smoke:inventory-install', 'installed fixture package appears in app inventory'), + [C.appState]: live( + 'smoke:automation-system', + 'Android foreground package changes on Home and returns to the fixture after restoration', + ), + [C.artifacts]: contract( + 'src/daemon/__tests__/http-server-artifacts.test.ts', + 'daemon artifact inventory lists artifacts and downloads consume them', + 'daemon inventory exposes typed downloadable artifact bytes', + ), + [C.audio]: contract( + 'src/daemon/handlers/__tests__/session-audio.test.ts', + 'audio probe starts host helper for Android emulator audio', + 'host audio probing has an Android-emulator session contract', + ), + [C.back]: live('smoke:automation-system', 'back returns from automation to the Settings tab'), + [C.batch]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario asserts typed nested Android batch outcomes', + ), + [C.boot]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario asserts typed Android boot result', + ), + [C.capabilities]: live( + 'smoke:inventory-install', + 'typed capability response includes fixture-driving Android commands', + ), + [C.click]: live('smoke:automation-system', 'resource-id selector opens fixture controls'), + [C.clipboard]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario round-trips Android clipboard text', + ), + [C.close]: live('smoke:capture-close', 'session inventory proves fixture lease removal'), + [C.devices]: live('smoke:inventory-install', 'selected emulator serial appears in inventory'), + [C.diff]: live('smoke:form-input', 'snapshot diff observes a form mutation'), + [C.doctor]: live('smoke:inventory-install', 'doctor discovers the installed fixture package'), + [C.events]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario records Android session command events', + ), + [C.fill]: live('smoke:form-input', 'replacement form text is read back from Android UI'), + [C.find]: live('smoke:automation-system', 'find observes the automation landmark'), + [C.focus]: live('smoke:form-input', 'snapshot-derived Android field point receives typed text'), + [C.gesture]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario verifies Android single- and multi-touch plans', + ), + [C.get]: live('smoke:automation-system', 'get returns fixture automation canary text'), + [C.home]: live( + 'smoke:automation-system', + 'Home changes Android foreground evidence before the fixture is restored', + ), + [C.install]: live('smoke:inventory-install', 'public CLI installs cached/repacked fixture APK'), + [C.installFromSource]: contract( + 'src/platforms/__tests__/install-source.test.ts', + 'prepareAndroidInstallArtifact resolves package identity for direct APK URL sources', + 'Android install-source resolves an installable artifact with typed identity', + ), + [C.is]: live('smoke:automation-system', 'visible predicate passes for Android fixture node'), + [C.keyboard]: live('smoke:form-input', 'safe dismissal hides keyboard without navigating Back'), + [C.logs]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario starts, inspects, restarts, and stops Android logcat', + ), + [C.longPress]: live('smoke:automation-system', '800ms hold increments durable fixture counter'), + [C.network]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario returns typed Android network entries', + ), + [C.open]: live( + 'smoke:automation-system', + 'cold deep link and normal fixture launch render landmarks', + ), + [C.orientation]: live( + 'smoke:automation-system', + 'fixture window state observes landscape then portrait Android rotation', + ), + [C.perf]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario validates typed Android process metrics', + ), + [C.prepare]: { + assertion: 'Android emulator capability model rejects Apple runner preparation', + level: 'capability-denial', + owner: { + path: 'test/integration/smoke-android-emulator-coverage.test.ts', + test: 'Android emulator capability denial matches the public catalog', + }, + }, + [C.press]: live('smoke:automation-system', 'semantic press updates durable fixture input state'), + [C.push]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario validates Android intent action and extras delivery', + ), + [C.reactNative]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario returns Android overlay dismissal state', + ), + [C.record]: contract( + 'test/integration/provider-scenarios/android-recording.test.ts', + 'Provider-backed integration Android recording flow uses scripted ADB provider pull capability', + 'Android recording finalizes through its durable manifest and pull contract', + ), + [C.reinstall]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario validates APK and bundle reinstall identities', + ), + [C.replay]: { + assertion: 'narrow Android Settings replay remains an additive live workflow check', + level: 'workflow-live', + owner: { path: '.github/workflows/android.yml', test: 'Run Android smoke checks' }, + }, + [C.screenshot]: live('smoke:capture-close', 'captured fixture file has a valid PNG signature'), + [C.scroll]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario validates Android scroll plans and resulting actions', + ), + [C.settings]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario validates Android device setting mutations', + ), + [C.shutdown]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario asserts typed Android shutdown result', + ), + [C.snapshot]: live( + 'smoke:automation-system', + 'interactive tree exposes Android resource-id fixture nodes', + ), + [C.swipe]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario validates Android swipe execution', + ), + [C.test]: contract( + 'test/integration/provider-scenarios/android-test-suite.test.ts', + 'Provider-backed integration Android replay test suite covers retries and fail-fast flags', + 'Android replay suite reports attempt outcomes and JUnit evidence', + ), + [C.trace]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario verifies Android trace lifecycle output', + ), + [C.triggerAppEvent]: contract( + ANDROID_LIFECYCLE_CONTRACT.path, + ANDROID_LIFECYCLE_CONTRACT.test, + 'provider scenario validates Android deep-link event delivery', + ), + [C.tvRemote]: { + assertion: 'Android mobile emulator capability model rejects TV remote input', + level: 'capability-denial', + owner: { + path: 'test/integration/smoke-android-emulator-coverage.test.ts', + test: 'Android emulator capability denial matches the public catalog', + }, + }, + [C.type]: live('smoke:form-input', 'typed suffix is read back from focused Android field'), + [C.viewport]: { + assertion: 'Android emulator capability model rejects standalone viewport control', + level: 'capability-denial', + owner: { + path: 'test/integration/smoke-android-emulator-coverage.test.ts', + test: 'Android emulator capability denial matches the public catalog', + }, + }, + [C.wait]: live('smoke:automation-system', 'wait observes durable fixture landmarks'), +} satisfies Record; + +export function liveCommandsForScenario(scenarioId: string): PublicCommand[] { + return Object.entries(ANDROID_EMULATOR_E2E_COVERAGE) + .filter( + ([, entry]) => + (entry.level === 'live' || entry.level === 'known-gap') && entry.owner === scenarioId, + ) + .map(([command]) => command as PublicCommand); +} diff --git a/test/integration/android-emulator-e2e/live-assertions.ts b/test/integration/android-emulator-e2e/live-assertions.ts new file mode 100644 index 000000000..99f8ea2d7 --- /dev/null +++ b/test/integration/android-emulator-e2e/live-assertions.ts @@ -0,0 +1,70 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +import { assertPngFile } from '../provider-scenarios/assertions.ts'; +import type { CliJsonResult } from '../cli-json.ts'; +import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; + +export function assertJsonContains(result: CliJsonResult, expected: string, message: string): void { + const serialized = JSON.stringify(result.json?.data ?? result.json); + assert.ok(serialized.includes(expected), `${message}\nreceived: ${serialized}`); +} + +export async function assertWaitText(context: LiveContext, expected: string): Promise { + const result = await runStep(context, `wait for ${expected}`, [ + 'wait', + 'text', + expected, + '10000', + ]); + assertJsonContains(result, expected, `wait should observe ${expected}`); + verifyCommand(context, 'wait', `wait observes durable text: ${expected}`); +} + +export async function assertElementText( + context: LiveContext, + selector: string, + expected: string, +): Promise { + const result = await runStep(context, `read ${selector}`, ['get', 'text', selector]); + assert.equal(result.json?.data?.text, expected, `${selector}: ${JSON.stringify(result.json)}`); +} + +export async function capturePng( + context: LiveContext, + step: string, + outputPath: string, +): Promise { + await runStep(context, step, ['screenshot', outputPath, '--max-size', '900']); + assertPngFile(outputPath); +} + +export function assertFilesDiffer(first: string, second: string, message: string): void { + assert.notDeepEqual(fs.readFileSync(first), fs.readFileSync(second), message); +} + +export function requireAndroidResourceId( + result: CliJsonResult, + suffix: string, +): { + identifier: string; + rect: { height: number; width: number; x: number; y: number }; +} { + const nodes = Array.isArray(result.json?.data?.nodes) ? result.json.data.nodes : []; + const node = nodes.find( + (candidate: { identifier?: unknown }) => + typeof candidate.identifier === 'string' && + (candidate.identifier === suffix || candidate.identifier.endsWith(`:id/${suffix}`)), + ); + assert.ok( + node, + `snapshot missing Android resource-id for ${suffix}: ${JSON.stringify(result.json)}`, + ); + const rect = (node as { rect?: unknown }).rect; + assert.ok(rect && typeof rect === 'object', `resource-id ${suffix} has no rect`); + const typedRect = rect as { height: number; width: number; x: number; y: number }; + for (const value of [typedRect.x, typedRect.y, typedRect.width, typedRect.height]) { + assert.ok(Number.isFinite(value), `resource-id ${suffix} has invalid rect`); + } + return { identifier: (node as { identifier: string }).identifier, rect: typedRect }; +} diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts new file mode 100644 index 000000000..0d3a7b0ee --- /dev/null +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -0,0 +1,207 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { + assertElementText, + assertFilesDiffer, + assertJsonContains, + assertWaitText, + capturePng, + requireAndroidResourceId, +} from './live-assertions.ts'; +import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; + +const C = PUBLIC_COMMANDS; +const FIXTURE_HOME_TITLE = 'Agent Device Tester'; +const AUTOMATION_DEEP_LINK = + 'agent-device-test-app:///automation?event=cold.start&payload=%7B%22source%22%3A%22android-deep-link%22%7D'; + +export async function assertAutomationSystem(context: LiveContext): Promise { + await runStep(context, 'load fixture bundle before cold deep link', [ + 'open', + context.appId, + '--relaunch', + ]); + await assertWaitText(context, FIXTURE_HOME_TITLE); + + await runStep(context, 'cold launch fixture through Android deep link', [ + 'open', + context.appId, + '--relaunch', + AUTOMATION_DEEP_LINK, + ]); + await assertWaitText(context, 'Automation lab'); + await assertElementText(context, 'id="automation-event-name"', 'cold.start'); + await assertElementText( + context, + 'id="automation-event-payload"', + '{"source":"android-deep-link"}', + ); + verifyCommand(context, C.open, 'cold Android deep link renders decoded fixture event payload'); + verifyBehavior( + context, + 'cold-start-deep-link-navigation', + 'cold deep link rendered exact event and payload before normal navigation resumed', + ); + + await runStep(context, 'continue from cold deep link to fixture catalog', [ + 'click', + 'id="automation-continue-catalog"', + ]); + await assertWaitText(context, 'Catalog'); + const catalogSnapshot = await runStep(context, 'capture Android catalog navigation', [ + 'snapshot', + '-i', + ]); + const settingsTab = ( + catalogSnapshot.json?.data?.nodes as { label?: unknown; ref?: unknown }[] | undefined + )?.find((node) => typeof node.label === 'string' && node.label.startsWith('Settings')); + const settingsRef = settingsTab?.ref; + assert.ok(typeof settingsRef === 'string', JSON.stringify(catalogSnapshot.json)); + await runStep(context, 'open Settings tab', ['click', `@${settingsRef}`]); + await assertWaitText(context, 'Settings'); + await runStep(context, 'open automation lab', ['click', 'id="open-automation-lab"']); + await assertWaitText(context, 'Automation lab'); + + const snapshot = await runStep(context, 'capture Android automation tree', ['snapshot', '-i']); + const eventName = requireAndroidResourceId(snapshot, 'automation-event-name'); + assert.match(eventName.identifier, /(^|:id\/)automation-event-name$/, eventName.identifier); + verifyCommand( + context, + C.snapshot, + `interactive tree exposes Android resource-id ${eventName.identifier}`, + ); + verifyBehavior( + context, + 'android-resource-id-selectors', + `observed ${eventName.identifier} before id selector-driven fixture actions`, + ); + + const heading = await runStep(context, 'read Android automation heading', [ + 'get', + 'text', + 'text="Automation lab"', + ]); + assertJsonContains(heading, 'Automation lab', 'get text should return automation heading'); + verifyCommand(context, C.get, 'get returns exact automation heading text'); + + const appState = await runStep(context, 'read Android fixture foreground app state', [ + 'appstate', + ]); + assert.equal(appState.json?.data?.package, context.appId, JSON.stringify(appState.json)); + verifyCommand( + context, + C.appState, + 'Android foreground package names the fixture before system UI', + ); + + const visible = await runStep(context, 'assert automation title visible', [ + 'is', + 'visible', + 'id="automation-title"', + ]); + assert.equal(visible.json?.data?.pass, true, JSON.stringify(visible.json)); + verifyCommand(context, C.is, 'visible predicate passes for automation resource-id'); + + const found = await runStep(context, 'find automation heading', [ + 'find', + 'text', + 'Automation lab', + 'exists', + ]); + assert.equal(found.json?.data?.found, true, JSON.stringify(found.json)); + verifyCommand(context, C.find, 'find observes the automation landmark'); + + await runStep(context, 'open Android fixture modal', ['click', 'id="automation-open-sheet"']); + await assertWaitText(context, 'Automation sheet'); + await runStep(context, 'close Android fixture modal', ['click', 'id="automation-close-sheet"']); + await assertWaitText(context, 'Automation lab'); + verifyCommand(context, C.click, 'resource-id selectors open and close fixture modal'); + + await assertOrientationFixtureState(context, 'landscape-left', 'landscape'); + await assertOrientationFixtureState(context, 'portrait', 'portrait'); + verifyCommand( + context, + C.orientation, + 'fixture window state observed landscape then portrait Android rotation', + ); + verifyBehavior( + context, + 'orientation-fixture-state', + 'fixture automation-window value changed to landscape and back to portrait', + ); + + await runStep(context, 'reveal input canaries', ['scroll', 'down', '0.7']); + await runStep(context, 'press semantic canary', ['press', 'id="automation-press"']); + await assertWaitText(context, 'Last input: press'); + verifyCommand(context, C.press, 'semantic press updates durable fixture input state'); + + await runStep(context, 'long press semantic canary', [ + 'longpress', + 'id="automation-longpress"', + '800', + ]); + await assertWaitText(context, 'Long presses: 1'); + verifyCommand(context, C.longPress, '800ms hold increments durable fixture long-press counter'); + + await runStep(context, 'open Android native alert', ['click', 'id="automation-open-alert"']); + await assertWaitText(context, 'Automation confirmation'); + const alert = await runStep(context, 'inspect Android native alert', ['alert', 'get']); + assertJsonContains(alert, 'Automation confirmation', 'alert get should expose fixture dialog'); + await runStep(context, 'dismiss Android native alert', ['alert', 'dismiss']); + await assertWaitText(context, 'Alert result: cancelled'); + await runStep(context, 'reopen Android native alert', ['click', 'id="automation-open-alert"']); + await runStep(context, 'accept Android native alert', ['alert', 'accept']); + await assertWaitText(context, 'Alert result: accepted'); + verifyCommand(context, C.alert, 'alert wait/get/dismiss/accept produce fixture-visible results'); + + await assertHomeAndRecentsRestoration(context); + await runStep(context, 'return from automation route with Back', ['back']); + await assertWaitText(context, 'Settings'); + verifyCommand(context, C.back, 'Android Back returns from automation route to Settings'); +} + +async function assertOrientationFixtureState( + context: LiveContext, + orientation: 'landscape-left' | 'portrait', + expectedWindow: 'landscape' | 'portrait', +): Promise { + await runStep(context, `set Android ${orientation} orientation`, ['orientation', orientation]); + await assertWaitText(context, expectedWindow); + await assertElementText(context, 'id="automation-window"', expectedWindow); +} + +async function assertHomeAndRecentsRestoration(context: LiveContext): Promise { + const foreground = path.join(context.artifactDir, 'foreground.png'); + await capturePng(context, 'capture Android fixture foreground', foreground); + await runStep(context, 'send Android fixture Home', ['home']); + const homeState = await runStep(context, 'read Android Home foreground state', ['appstate']); + assert.notEqual(homeState.json?.data?.package, context.appId, JSON.stringify(homeState.json)); + const home = path.join(context.artifactDir, 'home.png'); + await capturePng(context, 'capture Android Home surface', home); + assertFilesDiffer(foreground, home, 'Android Home should replace fixture pixels'); + verifyCommand(context, C.home, 'Home changes Android foreground package and system pixels'); + + await runStep(context, 'open Android Recents', ['app-switcher']); + const recents = path.join(context.artifactDir, 'recents.png'); + await capturePng(context, 'capture Android Recents surface', recents); + assertFilesDiffer(home, recents, 'Android Recents should differ from Home pixels'); + verifyCommand(context, C.appSwitcher, 'Recents pixels differ from Home system surface'); + + await runStep(context, 'restore fixture after Android system UI', ['open', context.appId]); + await runStep(context, 'wait for restored fixture automation control', [ + 'wait', + 'id="automation-open-sheet"', + '10000', + ]); + const restored = await runStep(context, 'verify restored Android fixture foreground state', [ + 'appstate', + ]); + assert.equal(restored.json?.data?.package, context.appId, JSON.stringify(restored.json)); + verifyBehavior( + context, + 'home-recents-restoration', + 'Android foreground package and distinct Home/Recents screenshots prove restoration path', + ); +} diff --git a/test/integration/android-emulator-e2e/live-coverage-report.ts b/test/integration/android-emulator-e2e/live-coverage-report.ts new file mode 100644 index 000000000..65c28ca1e --- /dev/null +++ b/test/integration/android-emulator-e2e/live-coverage-report.ts @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { + ANDROID_EMULATOR_BEHAVIOR_COVERAGE, + type AndroidEmulatorBehaviorId, +} from './behavior-coverage.ts'; +import { liveCommandsForScenario } from './coverage-manifest.ts'; +import type { LiveContext } from './live-harness.ts'; +import { ANDROID_EMULATOR_LIVE_SCENARIOS } from './scenarios.ts'; + +const MAX_ANDROID_EMULATOR_SCENARIO_BODY_MS = 4 * 60 * 1000; + +export function assertCoverageComplete(context: LiveContext): void { + const missingCommands = ANDROID_EMULATOR_LIVE_SCENARIOS.flatMap((scenario) => + liveCommandsForScenario(scenario.id).filter( + (command) => !context.commandEvidence[command]?.length, + ), + ); + const missingBehaviors = Object.entries(ANDROID_EMULATOR_BEHAVIOR_COVERAGE) + .filter(([, entry]) => entry.level === 'live') + .map(([behavior]) => behavior as AndroidEmulatorBehaviorId) + .filter((behavior) => !context.behaviorEvidence[behavior]?.length); + const missingScenarios = ANDROID_EMULATOR_LIVE_SCENARIOS.map((scenario) => scenario.id).filter( + (id) => !context.completedScenarios.includes(id), + ); + assert.deepEqual( + { missingBehaviors, missingCommands, missingScenarios }, + { missingBehaviors: [], missingCommands: [], missingScenarios: [] }, + 'Android emulator E2E coverage is incomplete', + ); + const totalDurationMs = Date.now() - context.startedAtMs; + assert.ok( + totalDurationMs <= MAX_ANDROID_EMULATOR_SCENARIO_BODY_MS, + `Android emulator scenario body took ${totalDurationMs}ms; limit is ${MAX_ANDROID_EMULATOR_SCENARIO_BODY_MS}ms`, + ); +} + +export function writeCoverageReport(context: LiveContext): void { + fs.writeFileSync( + path.join(context.artifactDir, 'coverage-report.json'), + JSON.stringify( + { + behaviorEvidence: context.behaviorEvidence, + commandEvidence: context.commandEvidence, + completedScenarios: context.completedScenarios, + stepHistory: context.stepHistory, + timings: { + scenarioBodyDurationMs: Date.now() - context.startedAtMs, + scenarios: context.timings, + }, + }, + null, + 2, + ), + ); +} + +export function liveBehaviorsForScenario(scenarioId: string): AndroidEmulatorBehaviorId[] { + return Object.entries(ANDROID_EMULATOR_BEHAVIOR_COVERAGE) + .filter(([, entry]) => entry.level === 'live' && entry.owner === scenarioId) + .map(([behavior]) => behavior as AndroidEmulatorBehaviorId); +} diff --git a/test/integration/android-emulator-e2e/live-form-scenario.ts b/test/integration/android-emulator-e2e/live-form-scenario.ts new file mode 100644 index 000000000..1dbcb457a --- /dev/null +++ b/test/integration/android-emulator-e2e/live-form-scenario.ts @@ -0,0 +1,73 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { + assertElementText, + assertFilesDiffer, + assertJsonContains, + assertWaitText, + capturePng, + requireAndroidResourceId, +} from './live-assertions.ts'; +import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live-harness.ts'; + +const C = PUBLIC_COMMANDS; + +export async function assertFormInput(context: LiveContext): Promise { + await runStep(context, 'open form tab', ['click', 'label="Form"']); + await assertWaitText(context, 'Checkout form'); + + const baseline = await runStep(context, 'establish form diff baseline', ['snapshot', '-i']); + requireAndroidResourceId(baseline, 'field-name'); + await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); + const diff = await runStep(context, 'observe form diff', ['diff', 'snapshot', '-i']); + const additions = Number(diff.json?.data?.summary?.additions ?? 0); + const removals = Number(diff.json?.data?.summary?.removals ?? 0); + assert.ok(additions + removals > 0, `expected non-empty diff: ${JSON.stringify(diff.json)}`); + verifyCommand(context, C.diff, 'snapshot diff reports a non-empty form mutation'); + + const name = await runStep(context, 'read filled full name', ['get', 'text', 'id="field-name"']); + assertJsonContains(name, 'Ada Lovelace', 'filled name should be observable in Android UI'); + verifyCommand(context, C.fill, 'replacement name text is read back from Android UI'); + + await runStep(context, 'fill email field', ['fill', 'id="field-email"', 'ada@example']); + const keyboardVisiblePath = path.join(context.artifactDir, 'keyboard-visible.png'); + await capturePng(context, 'capture visible Android keyboard', keyboardVisiblePath); + const dismiss = await runStep(context, 'dismiss Android keyboard safely', [ + 'keyboard', + 'dismiss', + ]); + assert.equal(dismiss.json?.data?.dismissed, true, JSON.stringify(dismiss.json)); + assert.equal(dismiss.json?.data?.visible, false, JSON.stringify(dismiss.json)); + const keyboardHiddenPath = path.join(context.artifactDir, 'keyboard-hidden.png'); + await capturePng(context, 'capture dismissed Android keyboard', keyboardHiddenPath); + assertFilesDiffer( + keyboardVisiblePath, + keyboardHiddenPath, + 'keyboard dismissal should change emulator pixels', + ); + await assertWaitText(context, 'Checkout form'); + verifyCommand( + context, + C.keyboard, + 'dismiss reports hidden keyboard while the Checkout form remains on screen', + ); + verifyBehavior( + context, + 'safe-keyboard-dismissal', + 'before/after pixels changed and Checkout form remained visible after non-Back dismissal', + ); + + const snapshot = await runStep(context, 'locate email Android resource-id', ['snapshot', '-i']); + const email = requireAndroidResourceId(snapshot, 'field-email'); + await runStep(context, 'focus email by snapshot-derived point', [ + 'focus', + String(email.rect.x + email.rect.width / 2), + String(email.rect.y + email.rect.height / 2), + ]); + await runStep(context, 'append email suffix', ['type', '.test']); + await assertElementText(context, 'id="field-email"', 'ada@example.test'); + verifyCommand(context, C.focus, `snapshot resource-id ${email.identifier} directs focus`); + verifyCommand(context, C.type, 'typed suffix is read back from focused Android field'); +} diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts new file mode 100644 index 000000000..64490129e --- /dev/null +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -0,0 +1,222 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { type CliJsonResult, formatResultDebug, runBuiltCliJson } from '../cli-json.ts'; +import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; +import { liveCommandsForScenario } from './coverage-manifest.ts'; +import { liveBehaviorsForScenario, writeCoverageReport } from './live-coverage-report.ts'; + +export { assertCoverageComplete, writeCoverageReport } from './live-coverage-report.ts'; + +type StepRecord = { + accepted: boolean; + command: string; + commandName?: string; + durationMs: number; + errorCode?: string; + errorMessage?: string; + scenario: string; + status: number; + step: string; +}; + +export type ScenarioTiming = { + durationMs: number; + id: string; +}; + +export type LiveContext = { + appId: string; + appPath: string; + artifactDir: string; + behaviorEvidence: Partial>; + commandEvidence: Record; + completedScenarios: string[]; + currentScenario: string; + env: NodeJS.ProcessEnv; + serial: string; + session: string; + sessionOpen: boolean; + startedAtMs: number; + stepHistory: StepRecord[]; + timings: ScenarioTiming[]; +}; + +export function createContext(): LiveContext { + const runId = `${Date.now()}-${process.pid}`; + const artifactDir = path.resolve('test/artifacts/android-emulator', runId); + fs.mkdirSync(artifactDir, { recursive: true }); + return { + appId: requiredEnv('AGENT_DEVICE_FIXTURE_APP_ID'), + appPath: requiredEnv('AGENT_DEVICE_FIXTURE_APP_PATH'), + artifactDir, + behaviorEvidence: {}, + commandEvidence: {}, + completedScenarios: [], + currentScenario: 'bootstrap', + env: process.env, + serial: requiredEnv('AGENT_DEVICE_ANDROID_SERIAL'), + session: `android-e2e-${process.pid.toString(36)}`, + sessionOpen: false, + startedAtMs: Date.now(), + stepHistory: [], + timings: [], + }; +} + +export async function runScenario( + context: LiveContext, + scenario: { id: string; run: (context: LiveContext) => Promise }, +): Promise { + context.currentScenario = scenario.id; + const commandCounts = new Map( + liveCommandsForScenario(scenario.id).map((command) => [ + command, + context.commandEvidence[command]?.length ?? 0, + ]), + ); + const behaviorCounts = new Map( + liveBehaviorsForScenario(scenario.id).map((behavior) => [ + behavior, + context.behaviorEvidence[behavior]?.length ?? 0, + ]), + ); + const startedAt = Date.now(); + try { + await scenario.run(context); + assertNewEvidence(context, scenario.id, commandCounts, behaviorCounts); + context.completedScenarios.push(scenario.id); + } finally { + context.timings.push({ durationMs: Date.now() - startedAt, id: scenario.id }); + writeCoverageReport(context); + } +} + +export async function runStep( + context: LiveContext, + step: string, + args: string[], + options: { commonFlags?: boolean; timeoutMs?: number } = {}, +): Promise { + const fullArgs = options.commonFlags === false ? withJson(args) : withCommonFlags(context, args); + const startedAt = Date.now(); + if (args[0] === 'open') context.sessionOpen = true; + const result = await runBuiltCliJson(fullArgs, context.env, { timeoutMs: options.timeoutMs }); + context.stepHistory.push({ + accepted: result.status === 0, + command: `agent-device ${fullArgs.join(' ')}`, + commandName: args[0], + durationMs: Date.now() - startedAt, + errorCode: stringValue(result.json?.error?.code), + errorMessage: stringValue(result.json?.error?.message), + scenario: context.currentScenario, + status: result.status, + step, + }); + writeStepHistory(context); + if (result.status !== 0) { + const message = [ + formatResultDebug(step, fullArgs, result), + `scenario: ${context.currentScenario}`, + `artifacts: ${context.artifactDir}`, + ].join('\n'); + fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); + assert.fail(message); + } + if (args[0] === 'close') context.sessionOpen = false; + return result; +} + +export function verifyCommand(context: LiveContext, command: string, evidence: string): void { + assert.ok( + context.stepHistory.some( + (step) => + step.scenario === context.currentScenario && step.commandName === command && step.accepted, + ), + `${context.currentScenario} credited ${command} without successful command execution`, + ); + context.commandEvidence[command] = [...(context.commandEvidence[command] ?? []), evidence]; +} + +export function verifyBehavior( + context: LiveContext, + behavior: AndroidEmulatorBehaviorId, + evidence: string, +): void { + context.behaviorEvidence[behavior] = [...(context.behaviorEvidence[behavior] ?? []), evidence]; +} + +export async function cleanupSession(context: LiveContext): Promise { + const failures: unknown[] = []; + if (context.sessionOpen) { + try { + await runStep(context, 'cleanup: restore portrait orientation', ['orientation', 'portrait']); + } catch (error) { + failures.push(error); + } + try { + await runStep(context, 'cleanup: close fixture session', ['close']); + } catch (error) { + failures.push(error); + } + } + if (failures.length === 0) return; + const errorPath = path.join(context.artifactDir, 'cleanup-error.txt'); + fs.writeFileSync(errorPath, failures.map(String).join('\n\n')); + throw new AggregateError(failures, `Android E2E cleanup failed; details: ${errorPath}`); +} + +function assertNewEvidence( + context: LiveContext, + scenarioId: string, + commandCounts: ReadonlyMap, + behaviorCounts: ReadonlyMap, +): void { + for (const command of liveCommandsForScenario(scenarioId)) { + assert.ok( + (context.commandEvidence[command]?.length ?? 0) > (commandCounts.get(command) ?? 0), + `${scenarioId} produced no command-specific evidence for ${command}`, + ); + } + for (const behavior of liveBehaviorsForScenario(scenarioId)) { + assert.ok( + (context.behaviorEvidence[behavior]?.length ?? 0) > (behaviorCounts.get(behavior) ?? 0), + `${scenarioId} produced no behavior evidence for ${behavior}`, + ); + } +} + +function withCommonFlags(context: LiveContext, args: string[]): string[] { + return [ + ...args, + '--platform', + 'android', + '--serial', + context.serial, + '--session', + context.session, + ...(args.includes('--json') ? [] : ['--json']), + ]; +} + +function withJson(args: string[]): string[] { + return args.includes('--json') ? args : [...args, '--json']; +} + +function writeStepHistory(context: LiveContext): void { + fs.writeFileSync( + path.join(context.artifactDir, 'step-history.json'), + JSON.stringify(context.stepHistory, null, 2), + ); +} + +function requiredEnv(name: string): string { + const value = process.env[name]?.trim(); + assert.ok(value, `${name} is required when AGENT_DEVICE_ANDROID_E2E=1`); + return value; +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} diff --git a/test/integration/android-emulator-e2e/live-inventory-scenario.ts b/test/integration/android-emulator-e2e/live-inventory-scenario.ts new file mode 100644 index 000000000..5b269066c --- /dev/null +++ b/test/integration/android-emulator-e2e/live-inventory-scenario.ts @@ -0,0 +1,42 @@ +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { assertJsonContains } from './live-assertions.ts'; +import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; + +const C = PUBLIC_COMMANDS; + +export async function assertInventoryAndInstall(context: LiveContext): Promise { + const devices = await runStep(context, 'list Android devices', ['devices']); + assertJsonContains( + devices, + context.serial, + 'device inventory should include selected emulator serial', + ); + verifyCommand(context, C.devices, 'selected emulator serial appears in typed inventory'); + + const capabilities = await runStep(context, 'read Android capabilities', ['capabilities']); + for (const command of ['click', 'fill', 'snapshot', 'type']) { + assertJsonContains(capabilities, command, `capabilities should include ${command}`); + } + verifyCommand( + context, + C.capabilities, + 'typed capability response includes fixture-driving commands', + ); + + await runStep(context, 'install cached fixture APK through public CLI', [ + 'install', + context.appPath, + ]); + const apps = await runStep(context, 'list installed user apps', ['apps']); + assertJsonContains(apps, context.appId, 'app inventory should include fixture package'); + verifyCommand(context, C.install, 'cached fixture APK installs through public CLI'); + verifyCommand(context, C.apps, 'installed fixture package appears in app inventory'); + + const doctor = await runStep(context, 'doctor fixture package discovery', [ + 'doctor', + '--app', + context.appId, + ]); + assertJsonContains(doctor, context.appId, 'doctor should discover fixture package'); + verifyCommand(context, C.doctor, 'doctor discovers the installed fixture package'); +} diff --git a/test/integration/android-emulator-e2e/live-runner.ts b/test/integration/android-emulator-e2e/live-runner.ts new file mode 100644 index 000000000..73ab59aac --- /dev/null +++ b/test/integration/android-emulator-e2e/live-runner.ts @@ -0,0 +1,87 @@ +import assert from 'node:assert/strict'; +import path from 'node:path'; + +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import { assertPngFile } from '../provider-scenarios/assertions.ts'; +import { assertAutomationSystem } from './live-automation-scenario.ts'; +import { assertFormInput } from './live-form-scenario.ts'; +import { assertInventoryAndInstall } from './live-inventory-scenario.ts'; +import { + assertCoverageComplete, + cleanupSession, + createContext, + runScenario, + runStep, + verifyCommand, + writeCoverageReport, + type LiveContext, +} from './live-harness.ts'; +import { bindAndroidEmulatorScenarios } from './scenarios.ts'; + +const C = PUBLIC_COMMANDS; + +const LIVE_SCENARIOS = bindAndroidEmulatorScenarios({ + automationSystem: assertAutomationSystem, + captureClose: assertCaptureAndClose, + formInput: assertFormInput, + inventoryInstall: assertInventoryAndInstall, +}); + +export async function runAndroidEmulatorE2E(): Promise { + const context = createContext(); + let primaryError: unknown; + try { + for (const scenario of LIVE_SCENARIOS) await runScenario(context, scenario); + assertCoverageComplete(context); + } catch (error) { + primaryError = error; + } + + let cleanupError: unknown; + try { + await cleanupSession(context); + } catch (error) { + cleanupError = error; + } + try { + writeCoverageReport(context); + } catch (error) { + cleanupError = cleanupError ?? error; + } + if (primaryError !== undefined && cleanupError !== undefined) { + throw new AggregateError( + [primaryError, cleanupError], + 'Android E2E failed and cleanup also failed', + ); + } + if (primaryError !== undefined) throw primaryError; + if (cleanupError !== undefined) throw cleanupError; +} + +async function assertCaptureAndClose(context: LiveContext): Promise { + const screenshotPath = path.join(context.artifactDir, 'fixture-smoke.png'); + const screenshot = await runStep(context, 'capture fixture screenshot', [ + 'screenshot', + screenshotPath, + '--max-size', + '900', + ]); + assert.ok( + JSON.stringify(screenshot.json?.data).includes(screenshotPath), + JSON.stringify(screenshot.json), + ); + assertPngFile(screenshotPath); + verifyCommand(context, C.screenshot, 'captured Android fixture file has a valid PNG signature'); + + await runStep(context, 'close fixture session', ['close']); + const sessions = await runStep(context, 'verify fixture session released', ['session', 'list'], { + commonFlags: false, + }); + const inventory = Array.isArray(sessions.json?.data?.sessions) ? sessions.json.data.sessions : []; + assert.equal( + inventory.some((session: { name?: unknown }) => session.name === context.session), + false, + JSON.stringify(sessions.json), + ); + verifyCommand(context, C.close, 'session inventory proves Android fixture lease was removed'); +} diff --git a/test/integration/android-emulator-e2e/scenarios.ts b/test/integration/android-emulator-e2e/scenarios.ts new file mode 100644 index 000000000..8c09d2272 --- /dev/null +++ b/test/integration/android-emulator-e2e/scenarios.ts @@ -0,0 +1,44 @@ +export type AndroidEmulatorScenario = { + id: string; + source: string; +}; + +type ScenarioRunnerKey = 'automationSystem' | 'captureClose' | 'formInput' | 'inventoryInstall'; + +type ScenarioDefinition = AndroidEmulatorScenario & { runner: ScenarioRunnerKey }; + +const SCENARIO_DEFINITIONS: readonly ScenarioDefinition[] = [ + { + id: 'smoke:inventory-install', + runner: 'inventoryInstall', + source: 'test/integration/android-emulator-e2e/live-inventory-scenario.ts', + }, + { + id: 'smoke:automation-system', + runner: 'automationSystem', + source: 'test/integration/android-emulator-e2e/live-automation-scenario.ts', + }, + { + id: 'smoke:form-input', + runner: 'formInput', + source: 'test/integration/android-emulator-e2e/live-form-scenario.ts', + }, + { + id: 'smoke:capture-close', + runner: 'captureClose', + source: 'test/integration/android-emulator-e2e/live-runner.ts', + }, +] as const; + +export const ANDROID_EMULATOR_LIVE_SCENARIOS: readonly AndroidEmulatorScenario[] = + SCENARIO_DEFINITIONS; + +export function bindAndroidEmulatorScenarios( + runners: Record Promise>, +): readonly (AndroidEmulatorScenario & { run: (context: Context) => Promise })[] { + return SCENARIO_DEFINITIONS.map(({ id, source, runner }) => ({ + id, + run: runners[runner], + source, + })); +} diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index 15b148639..4447234a7 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -1428,7 +1428,7 @@ function assertAndroidPushAndEventContract(world: AndroidSettingsWorld): void { '-a', 'android.intent.action.VIEW', '-d', - 'demo://agent-device/event?name=pre_open_ping&payload=%7B%22stage%22%3A%22explicit-selector%22%7D&platform=android', + "'demo://agent-device/event?name=pre_open_ping&payload=%7B%22stage%22%3A%22explicit-selector%22%7D&platform=android'", ]); assertCommandCall(adbCalls, [ 'shell', @@ -1438,7 +1438,7 @@ function assertAndroidPushAndEventContract(world: AndroidSettingsWorld): void { '-a', 'android.intent.action.VIEW', '-d', - 'demo://agent-device/event?name=screenshot_taken&payload=%7B%22source%22%3A%22provider-scenario%22%2C%22foreground%22%3Atrue%7D&platform=android', + "'demo://agent-device/event?name=screenshot_taken&payload=%7B%22source%22%3A%22provider-scenario%22%2C%22foreground%22%3Atrue%7D&platform=android'", '-p', 'com.example.demo', ]); diff --git a/test/integration/smoke-android-emulator-coverage.test.ts b/test/integration/smoke-android-emulator-coverage.test.ts new file mode 100644 index 000000000..e6dc5218a --- /dev/null +++ b/test/integration/smoke-android-emulator-coverage.test.ts @@ -0,0 +1,134 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; + +import { PUBLIC_COMMANDS } from '../../src/command-catalog.ts'; +import { isCommandSupportedOnDevice } from '../../src/core/capabilities.ts'; +import { ANDROID_EMULATOR_BEHAVIOR_COVERAGE } from './android-emulator-e2e/behavior-coverage.ts'; +import { + ANDROID_EMULATOR_E2E_COVERAGE, + liveCommandsForScenario, +} from './android-emulator-e2e/coverage-manifest.ts'; +import { ANDROID_EMULATOR_LIVE_SCENARIOS } from './android-emulator-e2e/scenarios.ts'; + +const ANDROID_EMULATOR = { + id: 'ci-android-emulator', + kind: 'emulator' as const, + name: 'CI Android Emulator', + platform: 'android' as const, + target: 'mobile' as const, +}; + +test('Android emulator coverage exhaustively classifies the public catalog', () => { + const publicCommands = Object.values(PUBLIC_COMMANDS).sort(); + assert.deepEqual(Object.keys(ANDROID_EMULATOR_E2E_COVERAGE).sort(), publicCommands); + for (const command of publicCommands) { + const entry = ANDROID_EMULATOR_E2E_COVERAGE[command]; + assert.ok(entry.assertion.trim().length > 0, `${command} needs an observable assertion`); + if (typeof entry.owner === 'string') { + assert.ok(entry.owner.trim().length > 0, `${command} needs a scenario owner`); + } else { + assert.ok(entry.owner.path.trim().length > 0, `${command} needs an evidence path`); + assert.ok(entry.owner.test.trim().length > 0, `${command} needs named evidence`); + } + if (entry.level === 'known-gap') { + assert.match(entry.trackingIssue, /^#\d+$/, `${command} gap needs a tracking issue`); + } + } +}); + +test('Android live claims execute in exactly one scenario with command-specific evidence', () => { + const scenarios = new Map( + ANDROID_EMULATOR_LIVE_SCENARIOS.map((scenario) => [scenario.id, scenario]), + ); + for (const [command, entry] of Object.entries(ANDROID_EMULATOR_E2E_COVERAGE)) { + if (entry.level !== 'live' && entry.level !== 'known-gap') continue; + assert.ok(scenarios.has(entry.owner), `${command} references missing scenario ${entry.owner}`); + assert.ok( + liveCommandsForScenario(entry.owner).includes( + command as (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS], + ), + `${entry.owner} does not execute ${command}`, + ); + const source = [ + fs.readFileSync(scenarios.get(entry.owner)?.source ?? '', 'utf8'), + fs.readFileSync('test/integration/android-emulator-e2e/live-assertions.ts', 'utf8'), + ].join('\n'); + const commandKey = Object.entries(PUBLIC_COMMANDS).find(([, value]) => value === command)?.[0]; + assert.ok(commandKey, `public command key missing for ${command}`); + assert.match( + source, + new RegExp(`verifyCommand\\(\\s*context,\\s*(?:C\\.${commandKey}|['"]${command}['"])`), + `${entry.owner} has no runtime command-specific evidence assertion for ${command}`, + ); + } + const claimed = ANDROID_EMULATOR_LIVE_SCENARIOS.flatMap((scenario) => + liveCommandsForScenario(scenario.id), + ); + assert.equal( + new Set(claimed).size, + claimed.length, + 'live commands need exactly one primary owner', + ); +}); + +test('Android emulator non-live owners name executable repository evidence', () => { + for (const [command, entry] of Object.entries(ANDROID_EMULATOR_E2E_COVERAGE)) { + if (entry.level === 'live' || entry.level === 'known-gap') continue; + const ownerPath = path.resolve(entry.owner.path); + assert.ok(fs.existsSync(ownerPath), `${command} owner does not exist: ${entry.owner.path}`); + assert.ok( + fs.readFileSync(ownerPath, 'utf8').includes(entry.owner.test), + `${command} owner does not contain named evidence: ${entry.owner.test}`, + ); + } +}); + +test('Android behavior patterns are owned by live fixture journeys', () => { + const scenarioIds = new Set(ANDROID_EMULATOR_LIVE_SCENARIOS.map((scenario) => scenario.id)); + for (const [behavior, entry] of Object.entries(ANDROID_EMULATOR_BEHAVIOR_COVERAGE)) { + assert.ok(entry.assertion.trim().length > 0, `${behavior} needs observable assertion`); + assert.equal(entry.level, 'live'); + assert.ok( + scenarioIds.has(entry.owner), + `${behavior} references missing scenario ${entry.owner}`, + ); + const source = fs.readFileSync( + ANDROID_EMULATOR_LIVE_SCENARIOS.find((scenario) => scenario.id === entry.owner)?.source ?? '', + 'utf8', + ); + assert.match( + source, + new RegExp(`verifyBehavior\\(\\s*context,\\s*['"]${behavior}['"]`), + `${entry.owner} has no runtime behavior assertion for ${behavior}`, + ); + } +}); + +test('Android emulator capability denial matches the public catalog', () => { + for (const [command, entry] of Object.entries(ANDROID_EMULATOR_E2E_COVERAGE)) { + const supported = isCommandSupportedOnDevice(command, ANDROID_EMULATOR); + if (command === PUBLIC_COMMANDS.audio) { + assert.equal( + supported, + process.platform === 'darwin', + 'Android emulator audio admission follows host audio-probe availability', + ); + continue; + } + assert.equal( + supported, + entry.level !== 'capability-denial', + `${command} ownership must match Android emulator capability admission`, + ); + } + for (const command of [ + PUBLIC_COMMANDS.prepare, + PUBLIC_COMMANDS.tvRemote, + PUBLIC_COMMANDS.viewport, + ]) { + assert.equal(isCommandSupportedOnDevice(command, ANDROID_EMULATOR), false, command); + assert.equal(ANDROID_EMULATOR_E2E_COVERAGE[command].level, 'capability-denial', command); + } +}); diff --git a/test/integration/smoke-android-emulator.test.ts b/test/integration/smoke-android-emulator.test.ts new file mode 100644 index 000000000..f652ab5f4 --- /dev/null +++ b/test/integration/smoke-android-emulator.test.ts @@ -0,0 +1,15 @@ +import test from 'node:test'; + +import { runAndroidEmulatorE2E } from './android-emulator-e2e/live-runner.ts'; + +const enabled = process.env.AGENT_DEVICE_ANDROID_E2E === '1'; + +test( + 'live Android emulator fixture E2E', + { + skip: enabled + ? false + : 'Set AGENT_DEVICE_ANDROID_E2E=1 with fixture APK path/id and emulator serial to run.', + }, + runAndroidEmulatorE2E, +); From 125fb4e6b797ba0feae019c285c477addb7d1fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 29 Jul 2026 13:57:11 +0200 Subject: [PATCH 02/20] test(android): use stable snapshot diff mutation --- .../android-emulator-e2e/coverage-manifest.ts | 2 +- .../live-form-scenario.ts | 34 ++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/test/integration/android-emulator-e2e/coverage-manifest.ts b/test/integration/android-emulator-e2e/coverage-manifest.ts index 416dc6c70..f7cad75b2 100644 --- a/test/integration/android-emulator-e2e/coverage-manifest.ts +++ b/test/integration/android-emulator-e2e/coverage-manifest.ts @@ -78,7 +78,7 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { ), [C.close]: live('smoke:capture-close', 'session inventory proves fixture lease removal'), [C.devices]: live('smoke:inventory-install', 'selected emulator serial appears in inventory'), - [C.diff]: live('smoke:form-input', 'snapshot diff observes a form mutation'), + [C.diff]: live('smoke:form-input', 'snapshot diff observes the Form-to-Settings transition'), [C.doctor]: live('smoke:inventory-install', 'doctor discovers the installed fixture package'), [C.events]: contract( ANDROID_LIFECYCLE_CONTRACT.path, diff --git a/test/integration/android-emulator-e2e/live-form-scenario.ts b/test/integration/android-emulator-e2e/live-form-scenario.ts index 1dbcb457a..3ad5ebc35 100644 --- a/test/integration/android-emulator-e2e/live-form-scenario.ts +++ b/test/integration/android-emulator-e2e/live-form-scenario.ts @@ -20,12 +20,38 @@ export async function assertFormInput(context: LiveContext): Promise { const baseline = await runStep(context, 'establish form diff baseline', ['snapshot', '-i']); requireAndroidResourceId(baseline, 'field-name'); - await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); - const diff = await runStep(context, 'observe form diff', ['diff', 'snapshot', '-i']); + await runStep(context, 'leave form before snapshot diff', ['back']); + const diff = await runStep(context, 'observe form navigation diff', ['diff', 'snapshot', '-i']); const additions = Number(diff.json?.data?.summary?.additions ?? 0); const removals = Number(diff.json?.data?.summary?.removals ?? 0); - assert.ok(additions + removals > 0, `expected non-empty diff: ${JSON.stringify(diff.json)}`); - verifyCommand(context, C.diff, 'snapshot diff reports a non-empty form mutation'); + assert.ok( + additions + removals > 0, + `expected non-empty form navigation diff: ${JSON.stringify(diff.json)}`, + ); + const lines = Array.isArray(diff.json?.data?.lines) + ? (diff.json.data.lines as { kind?: unknown; text?: unknown }[]) + : []; + assert.ok( + lines.some( + (line) => + line.kind === 'removed' && + typeof line.text === 'string' && + line.text.includes('Checkout form'), + ), + `expected removed Checkout form evidence: ${JSON.stringify(diff.json)}`, + ); + assert.ok( + lines.some( + (line) => + line.kind === 'added' && typeof line.text === 'string' && line.text.includes('Settings'), + ), + `expected added Settings evidence: ${JSON.stringify(diff.json)}`, + ); + verifyCommand(context, C.diff, 'snapshot diff reports the Form-to-Settings transition'); + + await runStep(context, 'reopen form tab after diff', ['click', 'label="Form"']); + await assertWaitText(context, 'Checkout form'); + await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); const name = await runStep(context, 'read filled full name', ['get', 'text', 'id="field-name"']); assertJsonContains(name, 'Ada Lovelace', 'filled name should be observable in Android UI'); From 56c78f260d4b3f9befe36d6242e1e81141b1b766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 29 Jul 2026 14:06:06 +0200 Subject: [PATCH 03/20] test(android): assert actual back destination --- .../integration/android-emulator-e2e/coverage-manifest.ts | 2 +- .../android-emulator-e2e/live-form-scenario.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/integration/android-emulator-e2e/coverage-manifest.ts b/test/integration/android-emulator-e2e/coverage-manifest.ts index f7cad75b2..fe6b24906 100644 --- a/test/integration/android-emulator-e2e/coverage-manifest.ts +++ b/test/integration/android-emulator-e2e/coverage-manifest.ts @@ -78,7 +78,7 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { ), [C.close]: live('smoke:capture-close', 'session inventory proves fixture lease removal'), [C.devices]: live('smoke:inventory-install', 'selected emulator serial appears in inventory'), - [C.diff]: live('smoke:form-input', 'snapshot diff observes the Form-to-Settings transition'), + [C.diff]: live('smoke:form-input', 'snapshot diff observes the Form-to-Home transition'), [C.doctor]: live('smoke:inventory-install', 'doctor discovers the installed fixture package'), [C.events]: contract( ANDROID_LIFECYCLE_CONTRACT.path, diff --git a/test/integration/android-emulator-e2e/live-form-scenario.ts b/test/integration/android-emulator-e2e/live-form-scenario.ts index 3ad5ebc35..e41dfe8b3 100644 --- a/test/integration/android-emulator-e2e/live-form-scenario.ts +++ b/test/integration/android-emulator-e2e/live-form-scenario.ts @@ -43,11 +43,13 @@ export async function assertFormInput(context: LiveContext): Promise { assert.ok( lines.some( (line) => - line.kind === 'added' && typeof line.text === 'string' && line.text.includes('Settings'), + line.kind === 'added' && + typeof line.text === 'string' && + line.text.includes('Agent Device Tester'), ), - `expected added Settings evidence: ${JSON.stringify(diff.json)}`, + `expected added Home evidence: ${JSON.stringify(diff.json)}`, ); - verifyCommand(context, C.diff, 'snapshot diff reports the Form-to-Settings transition'); + verifyCommand(context, C.diff, 'snapshot diff reports the Form-to-Home transition'); await runStep(context, 'reopen form tab after diff', ['click', 'label="Form"']); await assertWaitText(context, 'Checkout form'); From 7b42d7c55581ce3b23cb69f89d042938ee15f076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 29 Jul 2026 14:15:33 +0200 Subject: [PATCH 04/20] test(android): separate keyboard and fill IMEs --- .../live-form-scenario.ts | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/test/integration/android-emulator-e2e/live-form-scenario.ts b/test/integration/android-emulator-e2e/live-form-scenario.ts index e41dfe8b3..f956782cb 100644 --- a/test/integration/android-emulator-e2e/live-form-scenario.ts +++ b/test/integration/android-emulator-e2e/live-form-scenario.ts @@ -51,15 +51,25 @@ export async function assertFormInput(context: LiveContext): Promise { ); verifyCommand(context, C.diff, 'snapshot diff reports the Form-to-Home transition'); - await runStep(context, 'reopen form tab after diff', ['click', 'label="Form"']); + await runStep(context, 'close helper-IME session before keyboard check', ['close']); + await runStep(context, 'open fixture with the emulator keyboard', [ + 'open', + context.appId, + '--no-test-ime', + ]); + await assertWaitText(context, 'Agent Device Tester'); + await runStep(context, 'open form tab for keyboard check', ['click', 'label="Form"']); await assertWaitText(context, 'Checkout form'); - await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); - - const name = await runStep(context, 'read filled full name', ['get', 'text', 'id="field-name"']); - assertJsonContains(name, 'Ada Lovelace', 'filled name should be observable in Android UI'); - verifyCommand(context, C.fill, 'replacement name text is read back from Android UI'); - - await runStep(context, 'fill email field', ['fill', 'id="field-email"', 'ada@example']); + await runStep(context, 'focus email with the emulator keyboard', [ + 'click', + 'id="field-email"', + '--settle', + ]); + const keyboardStatus = await runStep(context, 'verify Android keyboard visible', [ + 'keyboard', + 'status', + ]); + assert.equal(keyboardStatus.json?.data?.visible, true, JSON.stringify(keyboardStatus.json)); const keyboardVisiblePath = path.join(context.artifactDir, 'keyboard-visible.png'); await capturePng(context, 'capture visible Android keyboard', keyboardVisiblePath); const dismiss = await runStep(context, 'dismiss Android keyboard safely', [ @@ -87,6 +97,22 @@ export async function assertFormInput(context: LiveContext): Promise { 'before/after pixels changed and Checkout form remained visible after non-Back dismissal', ); + await runStep(context, 'close emulator-keyboard session before text input', ['close']); + await runStep(context, 'reopen fixture with deterministic test IME', [ + 'open', + context.appId, + '--relaunch', + ]); + await assertWaitText(context, 'Agent Device Tester'); + await runStep(context, 'open form tab for text input', ['click', 'label="Form"']); + await assertWaitText(context, 'Checkout form'); + await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); + + const name = await runStep(context, 'read filled full name', ['get', 'text', 'id="field-name"']); + assertJsonContains(name, 'Ada Lovelace', 'filled name should be observable in Android UI'); + verifyCommand(context, C.fill, 'replacement name text is read back from Android UI'); + + await runStep(context, 'fill email field', ['fill', 'id="field-email"', 'ada@example']); const snapshot = await runStep(context, 'locate email Android resource-id', ['snapshot', '-i']); const email = requireAndroidResourceId(snapshot, 'field-email'); await runStep(context, 'focus email by snapshot-derived point', [ From 7b1f2c15cd157e535006031771022e760c590435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 29 Jul 2026 14:26:46 +0200 Subject: [PATCH 05/20] fix(ci): keep Android timing report in one shell --- .github/workflows/android.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 475578526..b2f9afb3d 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -69,8 +69,7 @@ jobs: pnpm build pnpm clean:daemon AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --test test/integration/smoke-android-emulator.test.ts - COVERAGE_REPORT="$(find test/artifacts/android-emulator -name coverage-report.json -print | tail -n 1)" - node --input-type=module -e 'import fs from "node:fs"; const report = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); console.log(`Android emulator scenario body: ${report.timings.scenarioBodyDurationMs}ms`); for (const scenario of report.timings.scenarios) console.log(` ${scenario.id}: ${scenario.durationMs}ms`);' "$COVERAGE_REPORT" + node --input-type=module -e 'import fs from "node:fs"; const report = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); console.log(`Android emulator scenario body: ${report.timings.scenarioBodyDurationMs}ms`); for (const scenario of report.timings.scenarios) console.log(` ${scenario.id}: ${scenario.durationMs}ms`);' "$(find test/artifacts/android-emulator -name coverage-report.json -print -quit)" node --experimental-strip-types src/bin.ts test test/integration/replays/android/01-settings.ad --retries 2 --report-junit test/artifacts/replays-android-smoke.junit.xml sh ./test/scripts/android-snapshot-helper-release-smoke.sh From 5f2d48ad343a2a1aa2e46a7262468ef1911fb4f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 29 Jul 2026 15:41:54 +0200 Subject: [PATCH 06/20] refactor(test): simplify simulator e2e coverage --- .github/workflows/android.yml | 10 +- src/platforms/android/__tests__/alert.test.ts | 32 -- .../android/__tests__/snapshot.test.ts | 29 -- src/platforms/android/snapshot.ts | 8 +- test/ci/trusted-fixture-artifact.test.mjs | 5 - .../android-emulator-e2e/behavior-coverage.ts | 8 +- .../android-emulator-e2e/coverage-manifest.ts | 106 +++---- .../android-emulator-e2e/live-assertions.ts | 95 +++--- .../live-automation-scenario.ts | 17 +- .../live-capture-scenario.ts | 36 +++ .../live-coverage-report.ts | 67 ++-- .../live-form-scenario.ts | 89 ++---- .../android-emulator-e2e/live-harness.ts | 230 +++----------- .../android-emulator-e2e/live-runner.ts | 63 +--- .../android-emulator-e2e/scenarios.ts | 83 +++-- .../ios-simulator-e2e/live-assertions.ts | 52 +--- .../ios-simulator-e2e/live-coverage-report.ts | 56 ++-- .../ios-simulator-e2e/live-harness.ts | 253 +++------------- .../integration/live-device-e2e/assertions.ts | 67 ++++ test/integration/live-device-e2e/coverage.ts | 63 ++++ test/integration/live-device-e2e/runtime.ts | 286 ++++++++++++++++++ .../smoke-android-emulator-coverage.test.ts | 95 +++--- test/scripts/android-fixture-cache-smoke.sh | 24 -- .../android-snapshot-helper-release-smoke.sh | 165 ---------- 24 files changed, 821 insertions(+), 1118 deletions(-) delete mode 100644 src/platforms/android/__tests__/alert.test.ts create mode 100644 test/integration/android-emulator-e2e/live-capture-scenario.ts create mode 100644 test/integration/live-device-e2e/assertions.ts create mode 100644 test/integration/live-device-e2e/coverage.ts create mode 100644 test/integration/live-device-e2e/runtime.ts delete mode 100644 test/scripts/android-fixture-cache-smoke.sh delete mode 100755 test/scripts/android-snapshot-helper-release-smoke.sh diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index b2f9afb3d..506a26957 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -54,6 +54,12 @@ jobs: - name: Run Android emulator catalog coverage contract run: node --test test/integration/smoke-android-emulator-coverage.test.ts + - name: Build agent-device CLI + run: pnpm build + + - name: Reset daemon state + run: pnpm clean:daemon + - name: Run Android smoke checks uses: reactivecircus/android-emulator-runner@b530d96654c385303d652368551fb075bc2f0b6b # v2.35.0 with: @@ -66,12 +72,8 @@ jobs: set -eu ANDROID_SERIAL="$(adb devices | awk 'NR > 1 && $2 == "device" { print $1; exit }')" test -n "$ANDROID_SERIAL" - pnpm build - pnpm clean:daemon AGENT_DEVICE_ANDROID_E2E=1 AGENT_DEVICE_ANDROID_SERIAL="$ANDROID_SERIAL" AGENT_DEVICE_FIXTURE_APP_PATH="${{ steps.fixture-app.outputs.apk-path }}" AGENT_DEVICE_FIXTURE_APP_ID="${{ steps.fixture-app.outputs.app-id }}" node --test test/integration/smoke-android-emulator.test.ts - node --input-type=module -e 'import fs from "node:fs"; const report = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); console.log(`Android emulator scenario body: ${report.timings.scenarioBodyDurationMs}ms`); for (const scenario of report.timings.scenarios) console.log(` ${scenario.id}: ${scenario.durationMs}ms`);' "$(find test/artifacts/android-emulator -name coverage-report.json -print -quit)" node --experimental-strip-types src/bin.ts test test/integration/replays/android/01-settings.ad --retries 2 --report-junit test/artifacts/replays-android-smoke.junit.xml - sh ./test/scripts/android-snapshot-helper-release-smoke.sh - name: Upload Android artifacts if: always() diff --git a/src/platforms/android/__tests__/alert.test.ts b/src/platforms/android/__tests__/alert.test.ts deleted file mode 100644 index c4a74494d..000000000 --- a/src/platforms/android/__tests__/alert.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { afterEach, test, vi } from 'vitest'; -import assert from 'node:assert/strict'; - -vi.mock('../snapshot.ts', () => ({ snapshotAndroid: vi.fn() })); - -import { handleAndroidAlert } from '../alert.ts'; -import { snapshotAndroid } from '../snapshot.ts'; -import type { DeviceInfo } from '../../../kernel/device.ts'; - -const device: DeviceInfo = { - booted: true, - id: 'emulator-5554', - kind: 'emulator', - name: 'Pixel', - platform: 'android', -}; - -afterEach(() => vi.restoreAllMocks()); - -test('Android alert get retains the normal bounded helper settle policy', async () => { - vi.mocked(snapshotAndroid).mockResolvedValue({ - analysis: { maxDepth: 0, rawNodeCount: 0 }, - androidSnapshot: { backend: 'android-helper' }, - nodes: [], - }); - - await handleAndroidAlert(device, 'get'); - - assert.deepEqual(vi.mocked(snapshotAndroid).mock.calls[0]?.[1], { - includeHiddenContentHints: false, - }); -}); diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index dddd49f35..9a42fe96c 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -515,35 +515,6 @@ test('snapshotAndroid reports helper-side truncation on the public snapshot resu assert.equal(result.androidSnapshot.helperTruncated, true); }); -test('snapshotAndroid forwards alert-style helper idle timeout override', async () => { - let instrumentArgs: string[] | undefined; - const helperAdb: AndroidAdbExecutor = async (args) => { - if (args.includes('--show-versioncode')) { - return installedHelperProbe; - } - if (args.includes('instrument')) { - instrumentArgs = args; - return { - exitCode: 0, - stdout: helperOutput(''), - stderr: '', - }; - } - throw new Error(`unexpected helper adb args: ${args.join(' ')}`); - }; - - await snapshotAndroid(device, { - helperAdb, - helperArtifact, - helperWaitForIdleTimeoutMs: 0, - }); - - assert.ok(instrumentArgs); - assert.equal(instrumentArgs[instrumentArgs.indexOf('waitForIdleTimeoutMs') + 1], '0'); - assert.equal(instrumentArgs.includes('outputPath'), false); - assert.equal(instrumentArgs.includes('emitChunks'), false); -}); - test('snapshotAndroid emits helper phase diagnostics', async () => { const helperAdb: AndroidAdbExecutor = async (args) => { if (args.includes('--show-versioncode')) { diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index be0dd3598..ba5235a77 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -58,7 +58,6 @@ export type AndroidSnapshotOptions = SnapshotOptions & { helperInstallPolicy?: AndroidSnapshotHelperInstallPolicy; helperSessionScope?: 'command' | 'daemon-session'; helperAdb?: AndroidAdbExecutor | AndroidAdbProvider; - helperWaitForIdleTimeoutMs?: number; includeHiddenContentHints?: boolean; }; @@ -192,7 +191,6 @@ async function captureAndroidUiHierarchyWithHelper( await stopAndroidSnapshotHelperSession(helperDeviceKey); } const capture = await captureAndroidUiHierarchyFromHelper({ - options, adb, adbProvider, artifact, @@ -275,13 +273,12 @@ async function installAndroidSnapshotHelper( } async function captureAndroidUiHierarchyFromHelper(params: { - options: AndroidSnapshotOptions; adb: AndroidAdbExecutor; adbProvider: AndroidAdbProvider; artifact: AndroidSnapshotHelperArtifact; helperDeviceKey: string; }): Promise { - const { options, adb, adbProvider, artifact, helperDeviceKey } = params; + const { adb, adbProvider, artifact, helperDeviceKey } = params; const captureOptions = { adb, adbProvider, @@ -291,8 +288,7 @@ async function captureAndroidUiHierarchyFromHelper(params: { helperSha256: artifact.manifest.sha256, packageName: artifact.manifest.packageName, instrumentationRunner: artifact.manifest.instrumentationRunner, - waitForIdleTimeoutMs: - options.helperWaitForIdleTimeoutMs ?? ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, + waitForIdleTimeoutMs: ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, }; diff --git a/test/ci/trusted-fixture-artifact.test.mjs b/test/ci/trusted-fixture-artifact.test.mjs index a6a537af8..f9071a791 100644 --- a/test/ci/trusted-fixture-artifact.test.mjs +++ b/test/ci/trusted-fixture-artifact.test.mjs @@ -76,19 +76,14 @@ test('Android smoke consumes the restored APK through catalog fixture E2E', (t) (step) => step.name === 'Report fixture cache source', ); const assertion = fs.readFileSync('test/scripts/assert-android-fixture-snapshot.mjs', 'utf8'); - const smokeScript = fs.readFileSync('test/scripts/android-fixture-cache-smoke.sh', 'utf8'); const packageVersion = JSON.parse(fs.readFileSync('package.json', 'utf8')).version; assert.match(smokeStep.with.script, /AGENT_DEVICE_ANDROID_E2E=1/); assert.match(smokeStep.with.script, /steps\.fixture-app\.outputs\.apk-path/); assert.match(smokeStep.with.script, /steps\.fixture-app\.outputs\.app-id/); assert.match(smokeStep.with.script, /smoke-android-emulator\.test\.ts/); - assert.match(smokeStep.with.script, /android-snapshot-helper-release-smoke\.sh/); assert.equal(restoreStep.with['wait-for-artifact-seconds'], '600'); assert.match(sourceStep.run, /steps\.fixture-app\.outputs\.source/); - assert.match(smokeScript, /snapshot -i .*--json > "\$SNAPSHOT_PATH"/); - assert.match(smokeScript, /\[ -z "\$1" \] \|\| \[ -z "\$2" \]/); - assert.match(smokeScript, /assert-android-fixture-snapshot\.mjs/); assert.match(assertion, /metadata\.backend !== 'android-helper'/); assert.match(assertion, /metadata\.helperVersion !== packageVersion/); assert.match(assertion, /Agent Device Tester/); diff --git a/test/integration/android-emulator-e2e/behavior-coverage.ts b/test/integration/android-emulator-e2e/behavior-coverage.ts index 91664c918..d9eec46a9 100644 --- a/test/integration/android-emulator-e2e/behavior-coverage.ts +++ b/test/integration/android-emulator-e2e/behavior-coverage.ts @@ -7,36 +7,30 @@ export type AndroidEmulatorBehaviorId = type BehaviorCoverageEntry = { assertion: string; - level: 'live'; owner: string; }; export const ANDROID_EMULATOR_BEHAVIOR_COVERAGE = { 'android-resource-id-selectors': { assertion: 'fixture test IDs are observed as Android resource IDs before their selectors act', - level: 'live', owner: 'smoke:automation-system', }, 'cold-start-deep-link-navigation': { assertion: 'cold deep link renders payload and normal navigation returns through Back', - level: 'live', owner: 'smoke:automation-system', }, 'home-recents-restoration': { assertion: 'Home and Recents produce distinct Android system evidence before fixture restoration', - level: 'live', owner: 'smoke:automation-system', }, 'orientation-fixture-state': { assertion: 'fixture window state observes landscape and portrait after Android rotation commands', - level: 'live', owner: 'smoke:automation-system', }, 'safe-keyboard-dismissal': { assertion: 'keyboard dismiss hides the IME while Checkout form remains on screen', - level: 'live', - owner: 'smoke:form-input', + owner: 'smoke:keyboard-ime', }, } as const satisfies Record; diff --git a/test/integration/android-emulator-e2e/coverage-manifest.ts b/test/integration/android-emulator-e2e/coverage-manifest.ts index fe6b24906..ed1d2452d 100644 --- a/test/integration/android-emulator-e2e/coverage-manifest.ts +++ b/test/integration/android-emulator-e2e/coverage-manifest.ts @@ -4,34 +4,29 @@ type PublicCommand = (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS]; type RepositoryEvidence = { path: string; - test: string; }; export type AndroidEmulatorCoverageEntry = - | { assertion: string; level: 'live'; owner: string } + | { assertion: string; level: 'live'; scenario: string } | { assertion: string; + evidence: RepositoryEvidence; level: 'command-contract' | 'workflow-live' | 'capability-denial'; - owner: RepositoryEvidence; - } - | { assertion: string; level: 'known-gap'; owner: string; trackingIssue: string }; + }; const C = PUBLIC_COMMANDS; -const live = (owner: string, assertion: string): AndroidEmulatorCoverageEntry => ({ +const live = (scenario: string, assertion: string): AndroidEmulatorCoverageEntry => ({ assertion, level: 'live', - owner, + scenario, }); -const contract = (path: string, test: string, assertion: string): AndroidEmulatorCoverageEntry => ({ +const contract = (path: string, assertion: string): AndroidEmulatorCoverageEntry => ({ assertion, + evidence: { path }, level: 'command-contract', - owner: { path, test }, }); -const ANDROID_LIFECYCLE_CONTRACT = { - path: 'test/integration/provider-scenarios/android-lifecycle.test.ts', - test: 'Provider-backed integration Android Settings flow uses scripted ADB provider', -} as const; +const ANDROID_LIFECYCLE_CONTRACT = 'test/integration/provider-scenarios/android-lifecycle.test.ts'; /** One primary, observable owner for every public command on an Android emulator. */ export const ANDROID_EMULATOR_E2E_COVERAGE = { @@ -47,23 +42,19 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { ), [C.artifacts]: contract( 'src/daemon/__tests__/http-server-artifacts.test.ts', - 'daemon artifact inventory lists artifacts and downloads consume them', 'daemon inventory exposes typed downloadable artifact bytes', ), [C.audio]: contract( 'src/daemon/handlers/__tests__/session-audio.test.ts', - 'audio probe starts host helper for Android emulator audio', 'host audio probing has an Android-emulator session contract', ), [C.back]: live('smoke:automation-system', 'back returns from automation to the Settings tab'), [C.batch]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario asserts typed nested Android batch outcomes', ), [C.boot]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario asserts typed Android boot result', ), [C.capabilities]: live( @@ -72,25 +63,25 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { ), [C.click]: live('smoke:automation-system', 'resource-id selector opens fixture controls'), [C.clipboard]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario round-trips Android clipboard text', ), [C.close]: live('smoke:capture-close', 'session inventory proves fixture lease removal'), [C.devices]: live('smoke:inventory-install', 'selected emulator serial appears in inventory'), - [C.diff]: live('smoke:form-input', 'snapshot diff observes the Form-to-Home transition'), + [C.diff]: live( + 'smoke:automation-system', + 'snapshot diff observes the Automation-to-Settings transition', + ), [C.doctor]: live('smoke:inventory-install', 'doctor discovers the installed fixture package'), [C.events]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario records Android session command events', ), [C.fill]: live('smoke:form-input', 'replacement form text is read back from Android UI'), [C.find]: live('smoke:automation-system', 'find observes the automation landmark'), [C.focus]: live('smoke:form-input', 'snapshot-derived Android field point receives typed text'), [C.gesture]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario verifies Android single- and multi-touch plans', ), [C.get]: live('smoke:automation-system', 'get returns fixture automation canary text'), @@ -101,20 +92,17 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { [C.install]: live('smoke:inventory-install', 'public CLI installs cached/repacked fixture APK'), [C.installFromSource]: contract( 'src/platforms/__tests__/install-source.test.ts', - 'prepareAndroidInstallArtifact resolves package identity for direct APK URL sources', 'Android install-source resolves an installable artifact with typed identity', ), [C.is]: live('smoke:automation-system', 'visible predicate passes for Android fixture node'), - [C.keyboard]: live('smoke:form-input', 'safe dismissal hides keyboard without navigating Back'), + [C.keyboard]: live('smoke:keyboard-ime', 'safe dismissal hides keyboard without navigating Back'), [C.logs]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario starts, inspects, restarts, and stops Android logcat', ), [C.longPress]: live('smoke:automation-system', '800ms hold increments durable fixture counter'), [C.network]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario returns typed Android network entries', ), [C.open]: live( @@ -126,58 +114,49 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { 'fixture window state observes landscape then portrait Android rotation', ), [C.perf]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario validates typed Android process metrics', ), [C.prepare]: { assertion: 'Android emulator capability model rejects Apple runner preparation', - level: 'capability-denial', - owner: { + evidence: { path: 'test/integration/smoke-android-emulator-coverage.test.ts', - test: 'Android emulator capability denial matches the public catalog', }, + level: 'capability-denial', }, [C.press]: live('smoke:automation-system', 'semantic press updates durable fixture input state'), [C.push]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario validates Android intent action and extras delivery', ), [C.reactNative]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario returns Android overlay dismissal state', ), [C.record]: contract( 'test/integration/provider-scenarios/android-recording.test.ts', - 'Provider-backed integration Android recording flow uses scripted ADB provider pull capability', 'Android recording finalizes through its durable manifest and pull contract', ), [C.reinstall]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario validates APK and bundle reinstall identities', ), [C.replay]: { assertion: 'narrow Android Settings replay remains an additive live workflow check', + evidence: { path: '.github/workflows/android.yml' }, level: 'workflow-live', - owner: { path: '.github/workflows/android.yml', test: 'Run Android smoke checks' }, }, [C.screenshot]: live('smoke:capture-close', 'captured fixture file has a valid PNG signature'), [C.scroll]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario validates Android scroll plans and resulting actions', ), [C.settings]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario validates Android device setting mutations', ), [C.shutdown]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario asserts typed Android shutdown result', ), [C.snapshot]: live( @@ -185,50 +164,41 @@ export const ANDROID_EMULATOR_E2E_COVERAGE = { 'interactive tree exposes Android resource-id fixture nodes', ), [C.swipe]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario validates Android swipe execution', ), [C.test]: contract( 'test/integration/provider-scenarios/android-test-suite.test.ts', - 'Provider-backed integration Android replay test suite covers retries and fail-fast flags', 'Android replay suite reports attempt outcomes and JUnit evidence', ), [C.trace]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario verifies Android trace lifecycle output', ), [C.triggerAppEvent]: contract( - ANDROID_LIFECYCLE_CONTRACT.path, - ANDROID_LIFECYCLE_CONTRACT.test, + ANDROID_LIFECYCLE_CONTRACT, 'provider scenario validates Android deep-link event delivery', ), [C.tvRemote]: { assertion: 'Android mobile emulator capability model rejects TV remote input', - level: 'capability-denial', - owner: { + evidence: { path: 'test/integration/smoke-android-emulator-coverage.test.ts', - test: 'Android emulator capability denial matches the public catalog', }, + level: 'capability-denial', }, [C.type]: live('smoke:form-input', 'typed suffix is read back from focused Android field'), [C.viewport]: { assertion: 'Android emulator capability model rejects standalone viewport control', - level: 'capability-denial', - owner: { + evidence: { path: 'test/integration/smoke-android-emulator-coverage.test.ts', - test: 'Android emulator capability denial matches the public catalog', }, + level: 'capability-denial', }, [C.wait]: live('smoke:automation-system', 'wait observes durable fixture landmarks'), } satisfies Record; export function liveCommandsForScenario(scenarioId: string): PublicCommand[] { return Object.entries(ANDROID_EMULATOR_E2E_COVERAGE) - .filter( - ([, entry]) => - (entry.level === 'live' || entry.level === 'known-gap') && entry.owner === scenarioId, - ) + .filter(([, entry]) => entry.level === 'live' && entry.scenario === scenarioId) .map(([command]) => command as PublicCommand); } diff --git a/test/integration/android-emulator-e2e/live-assertions.ts b/test/integration/android-emulator-e2e/live-assertions.ts index 99f8ea2d7..4de6ff414 100644 --- a/test/integration/android-emulator-e2e/live-assertions.ts +++ b/test/integration/android-emulator-e2e/live-assertions.ts @@ -1,46 +1,40 @@ import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import { assertPngFile } from '../provider-scenarios/assertions.ts'; +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import type { RawSnapshotNode } from '../../../src/kernel/snapshot.ts'; +import type { SnapshotDiffLine } from '../../../src/snapshot/snapshot-diff.ts'; +import { + assertFilesDiffer, + assertJsonContains, + createLiveDeviceAssertions, +} from '../live-device-e2e/assertions.ts'; import type { CliJsonResult } from '../cli-json.ts'; +import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; -export function assertJsonContains(result: CliJsonResult, expected: string, message: string): void { - const serialized = JSON.stringify(result.json?.data ?? result.json); - assert.ok(serialized.includes(expected), `${message}\nreceived: ${serialized}`); -} - -export async function assertWaitText(context: LiveContext, expected: string): Promise { - const result = await runStep(context, `wait for ${expected}`, [ - 'wait', - 'text', - expected, - '10000', - ]); - assertJsonContains(result, expected, `wait should observe ${expected}`); - verifyCommand(context, 'wait', `wait observes durable text: ${expected}`); -} - -export async function assertElementText( - context: LiveContext, - selector: string, - expected: string, -): Promise { - const result = await runStep(context, `read ${selector}`, ['get', 'text', selector]); - assert.equal(result.json?.data?.text, expected, `${selector}: ${JSON.stringify(result.json)}`); -} +export { assertFilesDiffer, assertJsonContains }; -export async function capturePng( - context: LiveContext, - step: string, - outputPath: string, -): Promise { - await runStep(context, step, ['screenshot', outputPath, '--max-size', '900']); - assertPngFile(outputPath); -} +export const { assertElementText, assertWaitSelector, assertWaitText, capturePng } = + createLiveDeviceAssertions( + runStep, + verifyCommand, + PUBLIC_COMMANDS.wait, + ); -export function assertFilesDiffer(first: string, second: string, message: string): void { - assert.notDeepEqual(fs.readFileSync(first), fs.readFileSync(second), message); +export function assertDiffLine( + result: CliJsonResult, + kind: SnapshotDiffLine['kind'], + expectedText: string, +): void { + const lines: unknown = result.json?.data?.lines; + assert.ok(Array.isArray(lines), `snapshot diff has no lines: ${JSON.stringify(result.json)}`); + assert.ok( + lines.some( + (line: unknown) => + isSnapshotDiffLine(line) && line.kind === kind && line.text.includes(expectedText), + ), + `expected ${kind} snapshot line containing ${expectedText}: ${JSON.stringify(result.json)}`, + ); } export function requireAndroidResourceId( @@ -50,9 +44,11 @@ export function requireAndroidResourceId( identifier: string; rect: { height: number; width: number; x: number; y: number }; } { - const nodes = Array.isArray(result.json?.data?.nodes) ? result.json.data.nodes : []; + const nodes: unknown = result.json?.data?.nodes; + assert.ok(Array.isArray(nodes), `snapshot has no nodes: ${JSON.stringify(result.json)}`); const node = nodes.find( - (candidate: { identifier?: unknown }) => + (candidate: unknown): candidate is RawSnapshotNode & { identifier: string } => + isSnapshotNode(candidate) && typeof candidate.identifier === 'string' && (candidate.identifier === suffix || candidate.identifier.endsWith(`:id/${suffix}`)), ); @@ -60,11 +56,24 @@ export function requireAndroidResourceId( node, `snapshot missing Android resource-id for ${suffix}: ${JSON.stringify(result.json)}`, ); - const rect = (node as { rect?: unknown }).rect; - assert.ok(rect && typeof rect === 'object', `resource-id ${suffix} has no rect`); - const typedRect = rect as { height: number; width: number; x: number; y: number }; - for (const value of [typedRect.x, typedRect.y, typedRect.width, typedRect.height]) { + assert.ok(node.rect, `resource-id ${suffix} has no rect`); + for (const value of [node.rect.x, node.rect.y, node.rect.width, node.rect.height]) { assert.ok(Number.isFinite(value), `resource-id ${suffix} has invalid rect`); } - return { identifier: (node as { identifier: string }).identifier, rect: typedRect }; + return { identifier: node.identifier, rect: node.rect }; +} + +function isSnapshotNode(value: unknown): value is RawSnapshotNode & { identifier: string } { + if (typeof value !== 'object' || value === null) return false; + const node = value as RawSnapshotNode; + return typeof node.identifier === 'string'; +} + +function isSnapshotDiffLine(value: unknown): value is SnapshotDiffLine { + if (typeof value !== 'object' || value === null) return false; + const line = value as Partial; + return ( + (line.kind === 'added' || line.kind === 'removed' || line.kind === 'unchanged') && + typeof line.text === 'string' + ); } diff --git a/test/integration/android-emulator-e2e/live-automation-scenario.ts b/test/integration/android-emulator-e2e/live-automation-scenario.ts index 0d3a7b0ee..6db4cc386 100644 --- a/test/integration/android-emulator-e2e/live-automation-scenario.ts +++ b/test/integration/android-emulator-e2e/live-automation-scenario.ts @@ -4,8 +4,10 @@ import path from 'node:path'; import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; import { assertElementText, + assertDiffLine, assertFilesDiffer, assertJsonContains, + assertWaitSelector, assertWaitText, capturePng, requireAndroidResourceId, @@ -157,8 +159,17 @@ export async function assertAutomationSystem(context: LiveContext): Promise { + const screenshotPath = path.join(context.artifactDir, 'fixture-smoke.png'); + const screenshot = await runStep(context, 'capture fixture screenshot', [ + 'screenshot', + screenshotPath, + '--max-size', + '900', + ]); + assert.ok( + JSON.stringify(screenshot.json?.data).includes(screenshotPath), + JSON.stringify(screenshot.json), + ); + assertPngFile(screenshotPath); + verifyCommand(context, C.screenshot, 'captured Android fixture file has a valid PNG signature'); + + await runStep(context, 'close fixture session', ['close']); + const sessions = await runStep(context, 'verify fixture session released', ['session', 'list'], { + commonFlags: false, + }); + const inventory = Array.isArray(sessions.json?.data?.sessions) ? sessions.json.data.sessions : []; + assert.equal( + inventory.some((session: { name?: unknown }) => session.name === context.session), + false, + JSON.stringify(sessions.json), + ); + verifyCommand(context, C.close, 'session inventory proves Android fixture lease was removed'); +} diff --git a/test/integration/android-emulator-e2e/live-coverage-report.ts b/test/integration/android-emulator-e2e/live-coverage-report.ts index 65c28ca1e..90e00f1bd 100644 --- a/test/integration/android-emulator-e2e/live-coverage-report.ts +++ b/test/integration/android-emulator-e2e/live-coverage-report.ts @@ -1,64 +1,39 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; - +import { + assertCoverageComplete as assertLiveCoverageComplete, + writeCoverageReport as writeLiveCoverageReport, +} from '../live-device-e2e/coverage.ts'; import { ANDROID_EMULATOR_BEHAVIOR_COVERAGE, type AndroidEmulatorBehaviorId, } from './behavior-coverage.ts'; -import { liveCommandsForScenario } from './coverage-manifest.ts'; +import { ANDROID_EMULATOR_E2E_COVERAGE, liveCommandsForScenario } from './coverage-manifest.ts'; import type { LiveContext } from './live-harness.ts'; -import { ANDROID_EMULATOR_LIVE_SCENARIOS } from './scenarios.ts'; -const MAX_ANDROID_EMULATOR_SCENARIO_BODY_MS = 4 * 60 * 1000; +const REQUIRED_SCENARIOS = [ + ...new Set([ + ...Object.values(ANDROID_EMULATOR_E2E_COVERAGE) + .filter((entry) => entry.level === 'live') + .map((entry) => entry.scenario), + ...Object.values(ANDROID_EMULATOR_BEHAVIOR_COVERAGE).map((entry) => entry.owner), + ]), +].map((id) => ({ id })); export function assertCoverageComplete(context: LiveContext): void { - const missingCommands = ANDROID_EMULATOR_LIVE_SCENARIOS.flatMap((scenario) => - liveCommandsForScenario(scenario.id).filter( - (command) => !context.commandEvidence[command]?.length, - ), - ); - const missingBehaviors = Object.entries(ANDROID_EMULATOR_BEHAVIOR_COVERAGE) - .filter(([, entry]) => entry.level === 'live') - .map(([behavior]) => behavior as AndroidEmulatorBehaviorId) - .filter((behavior) => !context.behaviorEvidence[behavior]?.length); - const missingScenarios = ANDROID_EMULATOR_LIVE_SCENARIOS.map((scenario) => scenario.id).filter( - (id) => !context.completedScenarios.includes(id), - ); - assert.deepEqual( - { missingBehaviors, missingCommands, missingScenarios }, - { missingBehaviors: [], missingCommands: [], missingScenarios: [] }, + assertLiveCoverageComplete( + context, + REQUIRED_SCENARIOS, + liveCommandsForScenario, + liveBehaviorsForScenario, 'Android emulator E2E coverage is incomplete', ); - const totalDurationMs = Date.now() - context.startedAtMs; - assert.ok( - totalDurationMs <= MAX_ANDROID_EMULATOR_SCENARIO_BODY_MS, - `Android emulator scenario body took ${totalDurationMs}ms; limit is ${MAX_ANDROID_EMULATOR_SCENARIO_BODY_MS}ms`, - ); } -export function writeCoverageReport(context: LiveContext): void { - fs.writeFileSync( - path.join(context.artifactDir, 'coverage-report.json'), - JSON.stringify( - { - behaviorEvidence: context.behaviorEvidence, - commandEvidence: context.commandEvidence, - completedScenarios: context.completedScenarios, - stepHistory: context.stepHistory, - timings: { - scenarioBodyDurationMs: Date.now() - context.startedAtMs, - scenarios: context.timings, - }, - }, - null, - 2, - ), - ); +export function writeCoverageReport(context: LiveContext): string { + return writeLiveCoverageReport(context); } export function liveBehaviorsForScenario(scenarioId: string): AndroidEmulatorBehaviorId[] { return Object.entries(ANDROID_EMULATOR_BEHAVIOR_COVERAGE) - .filter(([, entry]) => entry.level === 'live' && entry.owner === scenarioId) + .filter(([, entry]) => entry.owner === scenarioId) .map(([behavior]) => behavior as AndroidEmulatorBehaviorId); } diff --git a/test/integration/android-emulator-e2e/live-form-scenario.ts b/test/integration/android-emulator-e2e/live-form-scenario.ts index f956782cb..ff919a711 100644 --- a/test/integration/android-emulator-e2e/live-form-scenario.ts +++ b/test/integration/android-emulator-e2e/live-form-scenario.ts @@ -15,51 +15,35 @@ import { type LiveContext, runStep, verifyBehavior, verifyCommand } from './live const C = PUBLIC_COMMANDS; export async function assertFormInput(context: LiveContext): Promise { - await runStep(context, 'open form tab', ['click', 'label="Form"']); - await assertWaitText(context, 'Checkout form'); + await openFormTab(context); + await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); + const name = await runStep(context, 'read filled full name', ['get', 'text', 'id="field-name"']); + assertJsonContains(name, 'Ada Lovelace', 'filled name should be observable in Android UI'); + verifyCommand(context, C.fill, 'replacement name text is read back from Android UI'); - const baseline = await runStep(context, 'establish form diff baseline', ['snapshot', '-i']); - requireAndroidResourceId(baseline, 'field-name'); - await runStep(context, 'leave form before snapshot diff', ['back']); - const diff = await runStep(context, 'observe form navigation diff', ['diff', 'snapshot', '-i']); - const additions = Number(diff.json?.data?.summary?.additions ?? 0); - const removals = Number(diff.json?.data?.summary?.removals ?? 0); - assert.ok( - additions + removals > 0, - `expected non-empty form navigation diff: ${JSON.stringify(diff.json)}`, - ); - const lines = Array.isArray(diff.json?.data?.lines) - ? (diff.json.data.lines as { kind?: unknown; text?: unknown }[]) - : []; - assert.ok( - lines.some( - (line) => - line.kind === 'removed' && - typeof line.text === 'string' && - line.text.includes('Checkout form'), - ), - `expected removed Checkout form evidence: ${JSON.stringify(diff.json)}`, - ); - assert.ok( - lines.some( - (line) => - line.kind === 'added' && - typeof line.text === 'string' && - line.text.includes('Agent Device Tester'), - ), - `expected added Home evidence: ${JSON.stringify(diff.json)}`, - ); - verifyCommand(context, C.diff, 'snapshot diff reports the Form-to-Home transition'); + await runStep(context, 'fill email field', ['fill', 'id="field-email"', 'ada@example']); + const snapshot = await runStep(context, 'locate email Android resource-id', ['snapshot', '-i']); + const email = requireAndroidResourceId(snapshot, 'field-email'); + await runStep(context, 'focus email by snapshot-derived point', [ + 'focus', + String(email.rect.x + email.rect.width / 2), + String(email.rect.y + email.rect.height / 2), + ]); + await runStep(context, 'append email suffix', ['type', '.test']); + await assertElementText(context, 'id="field-email"', 'ada@example.test'); + verifyCommand(context, C.focus, `snapshot resource-id ${email.identifier} directs focus`); + verifyCommand(context, C.type, 'typed suffix is read back from focused Android field'); +} - await runStep(context, 'close helper-IME session before keyboard check', ['close']); +export async function assertKeyboardIme(context: LiveContext): Promise { + await runStep(context, 'close test-IME session before keyboard check', ['close']); await runStep(context, 'open fixture with the emulator keyboard', [ 'open', context.appId, '--no-test-ime', ]); await assertWaitText(context, 'Agent Device Tester'); - await runStep(context, 'open form tab for keyboard check', ['click', 'label="Form"']); - await assertWaitText(context, 'Checkout form'); + await openFormTab(context); await runStep(context, 'focus email with the emulator keyboard', [ 'click', 'id="field-email"', @@ -70,6 +54,7 @@ export async function assertFormInput(context: LiveContext): Promise { 'status', ]); assert.equal(keyboardStatus.json?.data?.visible, true, JSON.stringify(keyboardStatus.json)); + const keyboardVisiblePath = path.join(context.artifactDir, 'keyboard-visible.png'); await capturePng(context, 'capture visible Android keyboard', keyboardVisiblePath); const dismiss = await runStep(context, 'dismiss Android keyboard safely', [ @@ -78,6 +63,7 @@ export async function assertFormInput(context: LiveContext): Promise { ]); assert.equal(dismiss.json?.data?.dismissed, true, JSON.stringify(dismiss.json)); assert.equal(dismiss.json?.data?.visible, false, JSON.stringify(dismiss.json)); + const keyboardHiddenPath = path.join(context.artifactDir, 'keyboard-hidden.png'); await capturePng(context, 'capture dismissed Android keyboard', keyboardHiddenPath); assertFilesDiffer( @@ -96,32 +82,9 @@ export async function assertFormInput(context: LiveContext): Promise { 'safe-keyboard-dismissal', 'before/after pixels changed and Checkout form remained visible after non-Back dismissal', ); +} - await runStep(context, 'close emulator-keyboard session before text input', ['close']); - await runStep(context, 'reopen fixture with deterministic test IME', [ - 'open', - context.appId, - '--relaunch', - ]); - await assertWaitText(context, 'Agent Device Tester'); - await runStep(context, 'open form tab for text input', ['click', 'label="Form"']); +async function openFormTab(context: LiveContext): Promise { + await runStep(context, 'open form tab', ['click', 'label="Form"']); await assertWaitText(context, 'Checkout form'); - await runStep(context, 'fill full name', ['fill', 'id="field-name"', 'Ada Lovelace']); - - const name = await runStep(context, 'read filled full name', ['get', 'text', 'id="field-name"']); - assertJsonContains(name, 'Ada Lovelace', 'filled name should be observable in Android UI'); - verifyCommand(context, C.fill, 'replacement name text is read back from Android UI'); - - await runStep(context, 'fill email field', ['fill', 'id="field-email"', 'ada@example']); - const snapshot = await runStep(context, 'locate email Android resource-id', ['snapshot', '-i']); - const email = requireAndroidResourceId(snapshot, 'field-email'); - await runStep(context, 'focus email by snapshot-derived point', [ - 'focus', - String(email.rect.x + email.rect.width / 2), - String(email.rect.y + email.rect.height / 2), - ]); - await runStep(context, 'append email suffix', ['type', '.test']); - await assertElementText(context, 'id="field-email"', 'ada@example.test'); - verifyCommand(context, C.focus, `snapshot resource-id ${email.identifier} directs focus`); - verifyCommand(context, C.type, 'typed suffix is read back from focused Android field'); } diff --git a/test/integration/android-emulator-e2e/live-harness.ts b/test/integration/android-emulator-e2e/live-harness.ts index 64490129e..208e07baf 100644 --- a/test/integration/android-emulator-e2e/live-harness.ts +++ b/test/integration/android-emulator-e2e/live-harness.ts @@ -1,164 +1,66 @@ -import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; -import { type CliJsonResult, formatResultDebug, runBuiltCliJson } from '../cli-json.ts'; +import { + createLiveDeviceContext, + createLiveDeviceHarness, + requiredEnv, + type LiveDeviceContext, +} from '../live-device-e2e/runtime.ts'; import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; import { liveCommandsForScenario } from './coverage-manifest.ts'; import { liveBehaviorsForScenario, writeCoverageReport } from './live-coverage-report.ts'; export { assertCoverageComplete, writeCoverageReport } from './live-coverage-report.ts'; -type StepRecord = { - accepted: boolean; - command: string; - commandName?: string; - durationMs: number; - errorCode?: string; - errorMessage?: string; - scenario: string; - status: number; - step: string; -}; - -export type ScenarioTiming = { - durationMs: number; - id: string; -}; - -export type LiveContext = { +export type LiveContext = LiveDeviceContext & { appId: string; appPath: string; - artifactDir: string; - behaviorEvidence: Partial>; - commandEvidence: Record; - completedScenarios: string[]; - currentScenario: string; - env: NodeJS.ProcessEnv; serial: string; - session: string; - sessionOpen: boolean; - startedAtMs: number; - stepHistory: StepRecord[]; - timings: ScenarioTiming[]; }; export function createContext(): LiveContext { - const runId = `${Date.now()}-${process.pid}`; - const artifactDir = path.resolve('test/artifacts/android-emulator', runId); - fs.mkdirSync(artifactDir, { recursive: true }); return { - appId: requiredEnv('AGENT_DEVICE_FIXTURE_APP_ID'), - appPath: requiredEnv('AGENT_DEVICE_FIXTURE_APP_PATH'), - artifactDir, - behaviorEvidence: {}, - commandEvidence: {}, - completedScenarios: [], - currentScenario: 'bootstrap', - env: process.env, - serial: requiredEnv('AGENT_DEVICE_ANDROID_SERIAL'), - session: `android-e2e-${process.pid.toString(36)}`, - sessionOpen: false, - startedAtMs: Date.now(), - stepHistory: [], - timings: [], + ...createLiveDeviceContext({ + artifactRoot: 'test/artifacts/android-emulator', + session: `android-e2e-${process.pid.toString(36)}`, + }), + appId: requiredEnv('AGENT_DEVICE_FIXTURE_APP_ID', 'AGENT_DEVICE_ANDROID_E2E'), + appPath: requiredEnv('AGENT_DEVICE_FIXTURE_APP_PATH', 'AGENT_DEVICE_ANDROID_E2E'), + serial: requiredEnv('AGENT_DEVICE_ANDROID_SERIAL', 'AGENT_DEVICE_ANDROID_E2E'), }; } -export async function runScenario( - context: LiveContext, - scenario: { id: string; run: (context: LiveContext) => Promise }, -): Promise { - context.currentScenario = scenario.id; - const commandCounts = new Map( - liveCommandsForScenario(scenario.id).map((command) => [ - command, - context.commandEvidence[command]?.length ?? 0, - ]), - ); - const behaviorCounts = new Map( - liveBehaviorsForScenario(scenario.id).map((behavior) => [ - behavior, - context.behaviorEvidence[behavior]?.length ?? 0, - ]), - ); - const startedAt = Date.now(); - try { - await scenario.run(context); - assertNewEvidence(context, scenario.id, commandCounts, behaviorCounts); - context.completedScenarios.push(scenario.id); - } finally { - context.timings.push({ durationMs: Date.now() - startedAt, id: scenario.id }); - writeCoverageReport(context); - } -} - -export async function runStep( - context: LiveContext, - step: string, - args: string[], - options: { commonFlags?: boolean; timeoutMs?: number } = {}, -): Promise { - const fullArgs = options.commonFlags === false ? withJson(args) : withCommonFlags(context, args); - const startedAt = Date.now(); - if (args[0] === 'open') context.sessionOpen = true; - const result = await runBuiltCliJson(fullArgs, context.env, { timeoutMs: options.timeoutMs }); - context.stepHistory.push({ - accepted: result.status === 0, - command: `agent-device ${fullArgs.join(' ')}`, - commandName: args[0], - durationMs: Date.now() - startedAt, - errorCode: stringValue(result.json?.error?.code), - errorMessage: stringValue(result.json?.error?.message), - scenario: context.currentScenario, - status: result.status, - step, - }); - writeStepHistory(context); - if (result.status !== 0) { - const message = [ - formatResultDebug(step, fullArgs, result), - `scenario: ${context.currentScenario}`, - `artifacts: ${context.artifactDir}`, - ].join('\n'); - fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); - assert.fail(message); - } - if (args[0] === 'close') context.sessionOpen = false; - return result; -} - -export function verifyCommand(context: LiveContext, command: string, evidence: string): void { - assert.ok( - context.stepHistory.some( - (step) => - step.scenario === context.currentScenario && step.commandName === command && step.accepted, - ), - `${context.currentScenario} credited ${command} without successful command execution`, - ); - context.commandEvidence[command] = [...(context.commandEvidence[command] ?? []), evidence]; -} +const harness = createLiveDeviceHarness({ + behaviorsForScenario: liveBehaviorsForScenario, + commandsForScenario: liveCommandsForScenario, + commonFlags: (context, args) => [ + ...args, + '--platform', + 'android', + '--serial', + context.serial, + '--session', + context.session, + ...(args.includes('--json') ? [] : ['--json']), + ], + writeCoverageReport, +}); -export function verifyBehavior( - context: LiveContext, - behavior: AndroidEmulatorBehaviorId, - evidence: string, -): void { - context.behaviorEvidence[behavior] = [...(context.behaviorEvidence[behavior] ?? []), evidence]; -} +export const { runScenario, runStep, sessionExists, verifyBehavior, verifyCommand } = harness; export async function cleanupSession(context: LiveContext): Promise { const failures: unknown[] = []; if (context.sessionOpen) { - try { - await runStep(context, 'cleanup: restore portrait orientation', ['orientation', 'portrait']); - } catch (error) { - failures.push(error); - } - try { - await runStep(context, 'cleanup: close fixture session', ['close']); - } catch (error) { - failures.push(error); + for (const [step, args] of [ + ['restore portrait orientation', ['orientation', 'portrait']], + ['close fixture session', ['close']], + ] as const) { + try { + await runStep(context, `cleanup: ${step}`, [...args]); + } catch (error) { + failures.push(error); + } } } if (failures.length === 0) return; @@ -166,57 +68,3 @@ export async function cleanupSession(context: LiveContext): Promise { fs.writeFileSync(errorPath, failures.map(String).join('\n\n')); throw new AggregateError(failures, `Android E2E cleanup failed; details: ${errorPath}`); } - -function assertNewEvidence( - context: LiveContext, - scenarioId: string, - commandCounts: ReadonlyMap, - behaviorCounts: ReadonlyMap, -): void { - for (const command of liveCommandsForScenario(scenarioId)) { - assert.ok( - (context.commandEvidence[command]?.length ?? 0) > (commandCounts.get(command) ?? 0), - `${scenarioId} produced no command-specific evidence for ${command}`, - ); - } - for (const behavior of liveBehaviorsForScenario(scenarioId)) { - assert.ok( - (context.behaviorEvidence[behavior]?.length ?? 0) > (behaviorCounts.get(behavior) ?? 0), - `${scenarioId} produced no behavior evidence for ${behavior}`, - ); - } -} - -function withCommonFlags(context: LiveContext, args: string[]): string[] { - return [ - ...args, - '--platform', - 'android', - '--serial', - context.serial, - '--session', - context.session, - ...(args.includes('--json') ? [] : ['--json']), - ]; -} - -function withJson(args: string[]): string[] { - return args.includes('--json') ? args : [...args, '--json']; -} - -function writeStepHistory(context: LiveContext): void { - fs.writeFileSync( - path.join(context.artifactDir, 'step-history.json'), - JSON.stringify(context.stepHistory, null, 2), - ); -} - -function requiredEnv(name: string): string { - const value = process.env[name]?.trim(); - assert.ok(value, `${name} is required when AGENT_DEVICE_ANDROID_E2E=1`); - return value; -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' ? value : undefined; -} diff --git a/test/integration/android-emulator-e2e/live-runner.ts b/test/integration/android-emulator-e2e/live-runner.ts index 73ab59aac..4d0f0c444 100644 --- a/test/integration/android-emulator-e2e/live-runner.ts +++ b/test/integration/android-emulator-e2e/live-runner.ts @@ -1,37 +1,20 @@ -import assert from 'node:assert/strict'; -import path from 'node:path'; - -import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; -import { assertPngFile } from '../provider-scenarios/assertions.ts'; -import { assertAutomationSystem } from './live-automation-scenario.ts'; -import { assertFormInput } from './live-form-scenario.ts'; -import { assertInventoryAndInstall } from './live-inventory-scenario.ts'; import { assertCoverageComplete, cleanupSession, createContext, runScenario, - runStep, - verifyCommand, + sessionExists, writeCoverageReport, - type LiveContext, } from './live-harness.ts'; -import { bindAndroidEmulatorScenarios } from './scenarios.ts'; - -const C = PUBLIC_COMMANDS; - -const LIVE_SCENARIOS = bindAndroidEmulatorScenarios({ - automationSystem: assertAutomationSystem, - captureClose: assertCaptureAndClose, - formInput: assertFormInput, - inventoryInstall: assertInventoryAndInstall, -}); +import { ANDROID_EMULATOR_LIVE_SCENARIOS } from './scenarios.ts'; export async function runAndroidEmulatorE2E(): Promise { const context = createContext(); let primaryError: unknown; try { - for (const scenario of LIVE_SCENARIOS) await runScenario(context, scenario); + for (const scenario of ANDROID_EMULATOR_LIVE_SCENARIOS) { + await runScenario(context, scenario); + } assertCoverageComplete(context); } catch (error) { primaryError = error; @@ -39,12 +22,18 @@ export async function runAndroidEmulatorE2E(): Promise { let cleanupError: unknown; try { + context.sessionOpen = context.sessionOpen || (await sessionExists(context)); await cleanupSession(context); } catch (error) { cleanupError = error; } try { - writeCoverageReport(context); + const reportPath = writeCoverageReport(context); + console.log(`Android emulator coverage report: ${reportPath}`); + console.log(`Android emulator live run: ${Date.now() - context.startedAtMs}ms`); + for (const timing of context.timings) { + console.log(` ${timing.id}: ${timing.durationMs}ms`); + } } catch (error) { cleanupError = cleanupError ?? error; } @@ -57,31 +46,3 @@ export async function runAndroidEmulatorE2E(): Promise { if (primaryError !== undefined) throw primaryError; if (cleanupError !== undefined) throw cleanupError; } - -async function assertCaptureAndClose(context: LiveContext): Promise { - const screenshotPath = path.join(context.artifactDir, 'fixture-smoke.png'); - const screenshot = await runStep(context, 'capture fixture screenshot', [ - 'screenshot', - screenshotPath, - '--max-size', - '900', - ]); - assert.ok( - JSON.stringify(screenshot.json?.data).includes(screenshotPath), - JSON.stringify(screenshot.json), - ); - assertPngFile(screenshotPath); - verifyCommand(context, C.screenshot, 'captured Android fixture file has a valid PNG signature'); - - await runStep(context, 'close fixture session', ['close']); - const sessions = await runStep(context, 'verify fixture session released', ['session', 'list'], { - commonFlags: false, - }); - const inventory = Array.isArray(sessions.json?.data?.sessions) ? sessions.json.data.sessions : []; - assert.equal( - inventory.some((session: { name?: unknown }) => session.name === context.session), - false, - JSON.stringify(sessions.json), - ); - verifyCommand(context, C.close, 'session inventory proves Android fixture lease was removed'); -} diff --git a/test/integration/android-emulator-e2e/scenarios.ts b/test/integration/android-emulator-e2e/scenarios.ts index 8c09d2272..15e873b1c 100644 --- a/test/integration/android-emulator-e2e/scenarios.ts +++ b/test/integration/android-emulator-e2e/scenarios.ts @@ -1,44 +1,71 @@ +import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; +import type { AndroidEmulatorBehaviorId } from './behavior-coverage.ts'; +import { assertAutomationSystem } from './live-automation-scenario.ts'; +import { assertCaptureAndClose } from './live-capture-scenario.ts'; +import { assertFormInput, assertKeyboardIme } from './live-form-scenario.ts'; +import type { LiveContext } from './live-harness.ts'; +import { assertInventoryAndInstall } from './live-inventory-scenario.ts'; + +const C = PUBLIC_COMMANDS; + export type AndroidEmulatorScenario = { + behaviors: readonly AndroidEmulatorBehaviorId[]; + commands: readonly string[]; id: string; - source: string; + run: (context: LiveContext) => Promise; }; -type ScenarioRunnerKey = 'automationSystem' | 'captureClose' | 'formInput' | 'inventoryInstall'; - -type ScenarioDefinition = AndroidEmulatorScenario & { runner: ScenarioRunnerKey }; - -const SCENARIO_DEFINITIONS: readonly ScenarioDefinition[] = [ +export const ANDROID_EMULATOR_LIVE_SCENARIOS: readonly AndroidEmulatorScenario[] = [ { + behaviors: [], + commands: [C.devices, C.capabilities, C.install, C.apps, C.doctor], id: 'smoke:inventory-install', - runner: 'inventoryInstall', - source: 'test/integration/android-emulator-e2e/live-inventory-scenario.ts', + run: assertInventoryAndInstall, }, { + behaviors: [ + 'android-resource-id-selectors', + 'cold-start-deep-link-navigation', + 'home-recents-restoration', + 'orientation-fixture-state', + ], + commands: [ + C.alert, + C.appSwitcher, + C.appState, + C.back, + C.click, + C.diff, + C.find, + C.get, + C.home, + C.is, + C.longPress, + C.open, + C.orientation, + C.press, + C.snapshot, + C.wait, + ], id: 'smoke:automation-system', - runner: 'automationSystem', - source: 'test/integration/android-emulator-e2e/live-automation-scenario.ts', + run: assertAutomationSystem, }, { + behaviors: [], + commands: [C.fill, C.focus, C.type], id: 'smoke:form-input', - runner: 'formInput', - source: 'test/integration/android-emulator-e2e/live-form-scenario.ts', + run: assertFormInput, }, { + behaviors: ['safe-keyboard-dismissal'], + commands: [C.keyboard], + id: 'smoke:keyboard-ime', + run: assertKeyboardIme, + }, + { + behaviors: [], + commands: [C.close, C.screenshot], id: 'smoke:capture-close', - runner: 'captureClose', - source: 'test/integration/android-emulator-e2e/live-runner.ts', + run: assertCaptureAndClose, }, -] as const; - -export const ANDROID_EMULATOR_LIVE_SCENARIOS: readonly AndroidEmulatorScenario[] = - SCENARIO_DEFINITIONS; - -export function bindAndroidEmulatorScenarios( - runners: Record Promise>, -): readonly (AndroidEmulatorScenario & { run: (context: Context) => Promise })[] { - return SCENARIO_DEFINITIONS.map(({ id, source, runner }) => ({ - id, - run: runners[runner], - source, - })); -} +]; diff --git a/test/integration/ios-simulator-e2e/live-assertions.ts b/test/integration/ios-simulator-e2e/live-assertions.ts index 73bca634b..a259b65f2 100644 --- a/test/integration/ios-simulator-e2e/live-assertions.ts +++ b/test/integration/ios-simulator-e2e/live-assertions.ts @@ -3,38 +3,21 @@ import fs from 'node:fs'; import { PUBLIC_COMMANDS } from '../../../src/command-catalog.ts'; import { isPlayableVideo } from '../../../src/utils/video.ts'; -import { assertPngFile } from '../provider-scenarios/assertions.ts'; +import { + assertFilesDiffer, + assertJsonContains, + createLiveDeviceAssertions, +} from '../live-device-e2e/assertions.ts'; import type { CliJsonResult } from '../cli-json.ts'; +import type { IosSimulatorBehaviorId } from './behavior-coverage.ts'; import { type LiveContext, runStep, verifyCommand } from './live-harness.ts'; -export function assertJsonContains(result: CliJsonResult, expected: string, message: string): void { - const serialized = JSON.stringify(result.json?.data ?? result.json); - assert.ok(serialized.includes(expected), `${message}\nreceived: ${serialized}`); -} - -export async function assertWaitText(context: LiveContext, expected: string): Promise { - const result = await runStep(context, `wait for ${expected}`, [ - 'wait', - 'text', - expected, - '10000', - ]); - assertJsonContains(result, expected, `wait should observe ${expected}`); - verifyCommand(context, PUBLIC_COMMANDS.wait, `wait observes durable text: ${expected}`); -} +export { assertFilesDiffer, assertJsonContains }; -export async function assertElementText( - context: LiveContext, - selector: string, - expected: string, -): Promise { - const result = await runStep(context, `read ${selector}`, ['get', 'text', selector]); - assert.equal( - result.json?.data?.text, - expected, - `${selector} should expose ${expected}: ${JSON.stringify(result.json)}`, - ); -} +export const { assertElementText, assertWaitText, capturePng } = createLiveDeviceAssertions< + IosSimulatorBehaviorId, + LiveContext +>(runStep, verifyCommand, PUBLIC_COMMANDS.wait); export async function assertElementTextAfterScrolling( context: LiveContext, @@ -70,19 +53,6 @@ export async function assertMp4File(filePath: string): Promise { ); } -export async function capturePng( - context: LiveContext, - step: string, - outputPath: string, -): Promise { - await runStep(context, step, ['screenshot', outputPath, '--max-size', '900']); - assertPngFile(outputPath); -} - -export function assertFilesDiffer(first: string, second: string, message: string): void { - assert.notDeepEqual(fs.readFileSync(first), fs.readFileSync(second), message); -} - function requireNode( result: CliJsonResult, identifier: string, diff --git a/test/integration/ios-simulator-e2e/live-coverage-report.ts b/test/integration/ios-simulator-e2e/live-coverage-report.ts index a252d7917..266dd41d1 100644 --- a/test/integration/ios-simulator-e2e/live-coverage-report.ts +++ b/test/integration/ios-simulator-e2e/live-coverage-report.ts @@ -1,7 +1,8 @@ -import assert from 'node:assert/strict'; -import fs from 'node:fs'; -import path from 'node:path'; - +import { + assertCoverageComplete as assertLiveCoverageComplete, + computeMissingCoverage, + writeCoverageReport as writeLiveCoverageReport, +} from '../live-device-e2e/coverage.ts'; import { IOS_SIMULATOR_BEHAVIOR_COVERAGE, type IosSimulatorBehaviorId, @@ -11,45 +12,26 @@ import type { LiveContext, Tier } from './live-harness.ts'; import { IOS_SIMULATOR_LIVE_SCENARIOS, type IosSimulatorScenario } from './scenarios.ts'; export function assertCoverageComplete(context: LiveContext): void { - const missing = computeMissing(context); - assert.deepEqual( - missing, - { missingBehaviors: [], missingCommands: [], missingScenarios: [] }, + assertLiveCoverageComplete( + context, + scenariosForTier(context.tier), + liveCommandsForScenario, + liveBehaviorsForScenario, 'iOS simulator E2E coverage is incomplete', ); } export function writeCoverageReport(context: LiveContext): void { - const missing = computeMissing(context); - fs.writeFileSync( - path.join(context.artifactDir, 'coverage-report.json'), - JSON.stringify( - { - behaviorEvidence: context.behaviorEvidence, - commandEvidence: context.commandEvidence, - completedScenarios: context.completedScenarios, - ...missing, - tier: context.tier, - }, - null, - 2, + const scenarios = scenariosForTier(context.tier); + writeLiveCoverageReport(context, { + ...computeMissingCoverage( + context, + scenarios, + liveCommandsForScenario, + liveBehaviorsForScenario, ), - ); -} - -function computeMissing(context: LiveContext) { - const requiredScenarios = scenariosForTier(context.tier); - return { - missingBehaviors: requiredScenarios - .flatMap((scenario) => liveBehaviorsForScenario(scenario.id)) - .filter((behavior) => (context.behaviorEvidence[behavior]?.length ?? 0) === 0), - missingCommands: requiredScenarios - .flatMap((scenario) => liveCommandsForScenario(scenario.id)) - .filter((command) => (context.commandEvidence[command]?.length ?? 0) === 0), - missingScenarios: requiredScenarios - .map((scenario) => scenario.id) - .filter((id) => !context.completedScenarios.includes(id)), - }; + tier: context.tier, + }); } function scenariosForTier(tier: Tier): readonly IosSimulatorScenario[] { diff --git a/test/integration/ios-simulator-e2e/live-harness.ts b/test/integration/ios-simulator-e2e/live-harness.ts index 4bd5ff6c8..0944e5f97 100644 --- a/test/integration/ios-simulator-e2e/live-harness.ts +++ b/test/integration/ios-simulator-e2e/live-harness.ts @@ -3,8 +3,13 @@ import fs from 'node:fs'; import path from 'node:path'; import { resolveDaemonPaths } from '../../../src/daemon/config.ts'; -import { type CliJsonResult, formatResultDebug, runBuiltCliJson } from '../cli-json.ts'; -import { type IosSimulatorBehaviorId } from './behavior-coverage.ts'; +import { + createLiveDeviceContext, + createLiveDeviceHarness, + requiredEnv, + type LiveDeviceContext, +} from '../live-device-e2e/runtime.ts'; +import type { IosSimulatorBehaviorId } from './behavior-coverage.ts'; import { liveCommandsForScenario } from './coverage-manifest.ts'; import { liveBehaviorsForScenario, writeCoverageReport } from './live-coverage-report.ts'; @@ -12,137 +17,54 @@ export { assertCoverageComplete, writeCoverageReport } from './live-coverage-rep export type Tier = 'smoke' | 'full'; -type StepRecord = { - accepted: boolean; - command: string; - commandName?: string; - durationMs: number; - errorCode?: string; - errorMessage?: string; - scenario: string; - status: number; - step: string; -}; - -export type LiveContext = { +export type LiveContext = LiveDeviceContext & { appId: string; appPath: string; - artifactDir: string; - behaviorEvidence: Partial>; - commandEvidence: Record; - completedScenarios: string[]; - currentScenario: string; - env: NodeJS.ProcessEnv; - session: string; - sessionOpen: boolean; stateDir: string; - stepHistory: StepRecord[]; tier: Tier; udid: string; }; export function createContext(): LiveContext { - const tier = requiredEnv('AGENT_DEVICE_IOS_E2E_TIER'); + const tier = requiredEnv('AGENT_DEVICE_IOS_E2E_TIER', 'AGENT_DEVICE_IOS_E2E'); assert.ok(tier === 'smoke' || tier === 'full', `unsupported iOS E2E tier: ${tier}`); - if (tier === 'full') requiredEnv('AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE'); - const runId = `${Date.now()}-${process.pid}`; - const artifactDir = path.resolve('test/artifacts/ios-simulator', tier, runId); - fs.mkdirSync(artifactDir, { recursive: true }); - + if (tier === 'full') { + requiredEnv('AGENT_DEVICE_IOS_APP_EVENT_URL_TEMPLATE', 'AGENT_DEVICE_IOS_E2E'); + } return { - appId: requiredEnv('AGENT_DEVICE_FIXTURE_APP_ID'), - appPath: requiredEnv('AGENT_DEVICE_FIXTURE_APP_PATH'), - artifactDir, - behaviorEvidence: {}, - commandEvidence: {}, - completedScenarios: [], - currentScenario: 'bootstrap', - env: process.env, - session: `ios-e2e-${tier}-${process.pid.toString(36)}`, - sessionOpen: false, + ...createLiveDeviceContext({ + artifactRoot: `test/artifacts/ios-simulator/${tier}`, + session: `ios-e2e-${tier}-${process.pid.toString(36)}`, + }), + appId: requiredEnv('AGENT_DEVICE_FIXTURE_APP_ID', 'AGENT_DEVICE_IOS_E2E'), + appPath: requiredEnv('AGENT_DEVICE_FIXTURE_APP_PATH', 'AGENT_DEVICE_IOS_E2E'), stateDir: resolveDaemonPaths(process.env.AGENT_DEVICE_STATE_DIR, { env: process.env, }).baseDir, - stepHistory: [], tier, - udid: requiredEnv('AGENT_DEVICE_IOS_UDID'), + udid: requiredEnv('AGENT_DEVICE_IOS_UDID', 'AGENT_DEVICE_IOS_E2E'), }; } -export async function runScenario( - context: LiveContext, - scenario: { - id: string; - run: (context: LiveContext) => Promise; - }, -): Promise { - context.currentScenario = scenario.id; - const commands = liveCommandsForScenario(scenario.id); - const evidenceCounts = new Map( - commands.map((command) => [command, context.commandEvidence[command]?.length ?? 0]), - ); - const behaviorCounts = new Map( - liveBehaviorsForScenario(scenario.id).map((behavior) => [ - behavior, - context.behaviorEvidence[behavior]?.length ?? 0, - ]), - ); - await scenario.run(context); - assertScenarioCommandsVerified(scenario.id, commands, context, evidenceCounts); - assertScenarioBehaviorsVerified(scenario.id, context, behaviorCounts); - context.completedScenarios.push(scenario.id); - writeCoverageReport(context); -} - -export async function runStep( - context: LiveContext, - step: string, - args: string[], - options: { - allowFailure?: boolean; - commonFlags?: boolean; - expectFailure?: boolean; - timeoutMs?: number; - } = {}, -): Promise { - const fullArgs = options.commonFlags === false ? withJson(args) : withCommonFlags(context, args); - const startedAt = Date.now(); - const result = await runBuiltCliJson(fullArgs, context.env, { - timeoutMs: options.timeoutMs, - }); - context.stepHistory.push({ - accepted: result.status === 0 || (options.expectFailure === true && result.status !== 0), - command: `agent-device ${fullArgs.join(' ')}`, - commandName: args[0], - durationMs: Date.now() - startedAt, - errorCode: stringValue(result.json?.error?.code), - errorMessage: stringValue(result.json?.error?.message), - scenario: context.currentScenario, - status: result.status, - step, - }); - writeStepHistory(context); - const failedAsExpected = options.expectFailure === true && result.status !== 0; - if (result.status !== 0 && !failedAsExpected && options.allowFailure !== true) { - const message = [ - formatResultDebug(step, fullArgs, result), - `scenario: ${context.currentScenario}`, - `artifacts: ${context.artifactDir}`, - ].join('\n'); - fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); - assert.fail(message); - } - if (options.expectFailure === true && result.status === 0) { - assert.fail(`${step} unexpectedly succeeded\ncommand: agent-device ${fullArgs.join(' ')}`); - } - if (result.status === 0 && args[0] === 'open') context.sessionOpen = true; - if (result.status === 0 && args[0] === 'close') context.sessionOpen = false; - return result; -} +const harness = createLiveDeviceHarness({ + behaviorsForScenario: liveBehaviorsForScenario, + commandsForScenario: liveCommandsForScenario, + commonFlags: (context, args) => [ + ...args, + '--platform', + 'ios', + '--udid', + context.udid, + '--session', + context.session, + '--state-dir', + context.stateDir, + ...(args.includes('--json') ? [] : ['--json']), + ], + writeCoverageReport, +}); -export function verifyCommand(context: LiveContext, command: string, evidence: string): void { - recordCommandEvidence(context, command, command, evidence); -} +export const { runScenario, runStep, sessionExists, verifyBehavior, verifyCommand } = harness; export function verifyNestedReplayCommand( context: LiveContext, @@ -150,35 +72,7 @@ export function verifyNestedReplayCommand( executedVia: 'replay' | 'test', evidence: string, ): void { - recordCommandEvidence(context, command, executedVia, evidence); -} - -function recordCommandEvidence( - context: LiveContext, - command: string, - executedCommand: string, - evidence: string, -): void { - assert.ok( - context.stepHistory.some( - (step) => - step.scenario === context.currentScenario && - step.commandName === executedCommand && - step.accepted, - ), - `${context.currentScenario} credited ${command} without a successful ${executedCommand} execution`, - ); - const existing = context.commandEvidence[command] ?? []; - context.commandEvidence[command] = [...existing, evidence]; -} - -export function verifyBehavior( - context: LiveContext, - behavior: IosSimulatorBehaviorId, - evidence: string, -): void { - const existing = context.behaviorEvidence[behavior] ?? []; - context.behaviorEvidence[behavior] = [...existing, evidence]; + harness.verifyNestedCommand(context, command, executedVia, evidence); } export async function cleanupSession(context: LiveContext): Promise { @@ -211,74 +105,3 @@ export async function cleanupSession(context: LiveContext): Promise { fs.writeFileSync(errorPath, failures.map(String).join('\n\n')); throw new AggregateError(failures, `iOS E2E cleanup failed; details: ${errorPath}`); } - -export async function sessionExists(context: LiveContext): Promise { - const inventory = await runStep(context, 'inspect final session ownership', ['session', 'list'], { - commonFlags: false, - }); - const sessions = Array.isArray(inventory.json?.data?.sessions) - ? inventory.json.data.sessions - : []; - return sessions.some((session: { name?: unknown }) => session.name === context.session); -} - -function assertScenarioCommandsVerified( - scenarioId: string, - commands: readonly string[], - context: LiveContext, - evidenceCounts: ReadonlyMap, -): void { - for (const command of commands) { - const before = evidenceCounts.get(command) ?? 0; - const after = context.commandEvidence[command]?.length ?? 0; - assert.ok(after > before, `${scenarioId} produced no command-specific evidence for ${command}`); - } -} - -function assertScenarioBehaviorsVerified( - scenarioId: string, - context: LiveContext, - evidenceCounts: ReadonlyMap, -): void { - for (const behavior of liveBehaviorsForScenario(scenarioId)) { - const before = evidenceCounts.get(behavior) ?? 0; - const after = context.behaviorEvidence[behavior]?.length ?? 0; - assert.ok(after > before, `${scenarioId} produced no behavior evidence for ${behavior}`); - } -} - -function withCommonFlags(context: LiveContext, args: string[]): string[] { - return [ - ...args, - '--platform', - 'ios', - '--udid', - context.udid, - '--session', - context.session, - '--state-dir', - context.stateDir, - ...(args.includes('--json') ? [] : ['--json']), - ]; -} - -function withJson(args: string[]): string[] { - return args.includes('--json') ? args : [...args, '--json']; -} - -function writeStepHistory(context: LiveContext): void { - fs.writeFileSync( - path.join(context.artifactDir, 'step-history.json'), - JSON.stringify(context.stepHistory, null, 2), - ); -} - -function requiredEnv(name: string): string { - const value = process.env[name]?.trim(); - assert.ok(value, `${name} is required when AGENT_DEVICE_IOS_E2E=1`); - return value; -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' ? value : undefined; -} diff --git a/test/integration/live-device-e2e/assertions.ts b/test/integration/live-device-e2e/assertions.ts new file mode 100644 index 000000000..18c31625e --- /dev/null +++ b/test/integration/live-device-e2e/assertions.ts @@ -0,0 +1,67 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; + +import { assertPngFile } from '../provider-scenarios/assertions.ts'; +import type { CliJsonResult } from '../cli-json.ts'; +import type { LiveDeviceContext } from './runtime.ts'; + +type RunStep = ( + context: Context, + step: string, + args: string[], + options?: { allowFailure?: boolean }, +) => Promise; + +export function createLiveDeviceAssertions< + BehaviorId extends string, + Context extends LiveDeviceContext, +>( + runStep: RunStep, + verifyCommand: (context: Context, command: string, evidence: string) => void, + waitCommand: string, +) { + async function assertWaitText(context: Context, expected: string): Promise { + const result = await runStep(context, `wait for ${expected}`, [ + 'wait', + 'text', + expected, + '10000', + ]); + assertJsonContains(result, expected, `wait should observe ${expected}`); + verifyCommand(context, waitCommand, `wait observes durable text: ${expected}`); + } + + async function assertWaitSelector(context: Context, selector: string): Promise { + await runStep(context, `wait for ${selector}`, ['wait', selector, '10000']); + verifyCommand(context, waitCommand, `wait observes durable selector: ${selector}`); + } + + async function assertElementText( + context: Context, + selector: string, + expected: string, + ): Promise { + const result = await runStep(context, `read ${selector}`, ['get', 'text', selector]); + assert.equal( + result.json?.data?.text, + expected, + `${selector} should expose ${expected}: ${JSON.stringify(result.json)}`, + ); + } + + async function capturePng(context: Context, step: string, outputPath: string): Promise { + await runStep(context, step, ['screenshot', outputPath, '--max-size', '900']); + assertPngFile(outputPath); + } + + return { assertElementText, assertWaitSelector, assertWaitText, capturePng }; +} + +export function assertJsonContains(result: CliJsonResult, expected: string, message: string): void { + const serialized = JSON.stringify(result.json?.data ?? result.json); + assert.ok(serialized.includes(expected), `${message}\nreceived: ${serialized}`); +} + +export function assertFilesDiffer(first: string, second: string, message: string): void { + assert.notDeepEqual(fs.readFileSync(first), fs.readFileSync(second), message); +} diff --git a/test/integration/live-device-e2e/coverage.ts b/test/integration/live-device-e2e/coverage.ts new file mode 100644 index 000000000..ddf237446 --- /dev/null +++ b/test/integration/live-device-e2e/coverage.ts @@ -0,0 +1,63 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import type { LiveDeviceContext } from './runtime.ts'; + +export function computeMissingCoverage( + context: LiveDeviceContext, + scenarios: readonly { id: string }[], + commandsForScenario: (scenarioId: string) => readonly string[], + behaviorsForScenario: (scenarioId: string) => readonly BehaviorId[], +) { + return { + missingBehaviors: scenarios + .flatMap((scenario) => behaviorsForScenario(scenario.id)) + .filter((behavior) => (context.behaviorEvidence[behavior]?.length ?? 0) === 0), + missingCommands: scenarios + .flatMap((scenario) => commandsForScenario(scenario.id)) + .filter((command) => (context.commandEvidence[command]?.length ?? 0) === 0), + missingScenarios: scenarios + .map((scenario) => scenario.id) + .filter((id) => !context.completedScenarios.includes(id)), + }; +} + +export function assertCoverageComplete( + context: LiveDeviceContext, + scenarios: readonly { id: string }[], + commandsForScenario: (scenarioId: string) => readonly string[], + behaviorsForScenario: (scenarioId: string) => readonly BehaviorId[], + message: string, +): void { + assert.deepEqual( + computeMissingCoverage(context, scenarios, commandsForScenario, behaviorsForScenario), + { missingBehaviors: [], missingCommands: [], missingScenarios: [] }, + message, + ); +} + +export function writeCoverageReport( + context: LiveDeviceContext, + extra: Record = {}, +): string { + const reportPath = path.join(context.artifactDir, 'coverage-report.json'); + fs.writeFileSync( + reportPath, + JSON.stringify( + { + behaviorEvidence: context.behaviorEvidence, + commandEvidence: context.commandEvidence, + completedScenarios: context.completedScenarios, + ...extra, + timings: { + runDurationMs: Date.now() - context.startedAtMs, + scenarios: context.timings, + }, + }, + null, + 2, + ), + ); + return reportPath; +} diff --git a/test/integration/live-device-e2e/runtime.ts b/test/integration/live-device-e2e/runtime.ts new file mode 100644 index 000000000..8b259dbfb --- /dev/null +++ b/test/integration/live-device-e2e/runtime.ts @@ -0,0 +1,286 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; + +import { type CliJsonResult, formatResultDebug, runBuiltCliJson } from '../cli-json.ts'; + +export type StepRecord = { + accepted: boolean; + command: string; + commandName?: string; + durationMs: number; + errorCode?: string; + errorMessage?: string; + scenario: string; + status: number; + step: string; +}; + +export type ScenarioTiming = { + durationMs: number; + id: string; +}; + +export type LiveDeviceContext = { + artifactDir: string; + behaviorEvidence: Partial>; + commandEvidence: Record; + completedScenarios: string[]; + currentScenario: string; + env: NodeJS.ProcessEnv; + session: string; + sessionOpen: boolean; + startedAtMs: number; + stepHistory: StepRecord[]; + timings: ScenarioTiming[]; +}; + +export type LiveScenario = { + id: string; + run: (context: Context) => Promise; +}; + +type HarnessOptions = { + behaviorsForScenario: (scenarioId: string) => readonly BehaviorId[]; + commandsForScenario: (scenarioId: string) => readonly string[]; + commonFlags: (context: Context, args: readonly string[]) => string[]; + writeCoverageReport: (context: Context) => void; +}; + +type RunStepOptions = { + allowFailure?: boolean; + commonFlags?: boolean; + expectFailure?: boolean; + timeoutMs?: number; +}; + +export function createLiveDeviceContext(options: { + artifactRoot: string; + session: string; +}): LiveDeviceContext { + const runId = `${Date.now()}-${process.pid}`; + const artifactDir = path.resolve(options.artifactRoot, runId); + fs.mkdirSync(artifactDir, { recursive: true }); + return { + artifactDir, + behaviorEvidence: {}, + commandEvidence: {}, + completedScenarios: [], + currentScenario: 'bootstrap', + env: process.env, + session: options.session, + sessionOpen: false, + startedAtMs: Date.now(), + stepHistory: [], + timings: [], + }; +} + +export function createLiveDeviceHarness< + Context extends LiveDeviceContext, + BehaviorId extends string, +>(options: HarnessOptions) { + async function runScenario(context: Context, scenario: LiveScenario): Promise { + context.currentScenario = scenario.id; + const commandCounts = evidenceCounts( + options.commandsForScenario(scenario.id), + context.commandEvidence, + ); + const behaviorCounts = evidenceCounts( + options.behaviorsForScenario(scenario.id), + context.behaviorEvidence, + ); + const startedAt = Date.now(); + try { + await scenario.run(context); + assertNewEvidence( + scenario.id, + options.commandsForScenario(scenario.id), + context.commandEvidence, + commandCounts, + ); + assertNewEvidence( + scenario.id, + options.behaviorsForScenario(scenario.id), + context.behaviorEvidence, + behaviorCounts, + ); + context.completedScenarios.push(scenario.id); + } finally { + context.timings.push({ durationMs: Date.now() - startedAt, id: scenario.id }); + options.writeCoverageReport(context); + } + } + + async function runStep( + context: Context, + step: string, + args: string[], + stepOptions: RunStepOptions = {}, + ): Promise { + const fullArgs = buildStepArgs(context, args, stepOptions); + const startedAt = Date.now(); + const result = await runBuiltCliJson(fullArgs, context.env, { + timeoutMs: stepOptions.timeoutMs, + }); + const failedAsExpected = stepOptions.expectFailure === true && result.status !== 0; + recordStep(context, { + accepted: result.status === 0 || failedAsExpected, + command: `agent-device ${fullArgs.join(' ')}`, + commandName: args[0], + durationMs: Date.now() - startedAt, + errorCode: stringValue(result.json?.error?.code), + errorMessage: stringValue(result.json?.error?.message), + scenario: context.currentScenario, + status: result.status, + step, + }); + assertStepOutcome(context, step, fullArgs, result, failedAsExpected, stepOptions); + updateSessionState(context, args[0], result.status); + return result; + } + + function buildStepArgs( + context: Context, + args: readonly string[], + stepOptions: RunStepOptions, + ): string[] { + return stepOptions.commonFlags === false ? withJson(args) : options.commonFlags(context, args); + } + + function recordStep(context: Context, record: StepRecord): void { + context.stepHistory.push(record); + writeStepHistory(context); + } + + function assertStepOutcome( + context: Context, + step: string, + fullArgs: string[], + result: CliJsonResult, + failedAsExpected: boolean, + stepOptions: RunStepOptions, + ): void { + const unexpectedFailure = + result.status !== 0 && !failedAsExpected && stepOptions.allowFailure !== true; + if (unexpectedFailure) { + const message = [ + formatResultDebug(step, fullArgs, result), + `scenario: ${context.currentScenario}`, + `artifacts: ${context.artifactDir}`, + ].join('\n'); + fs.writeFileSync(path.join(context.artifactDir, 'failed-step.txt'), message); + assert.fail(message); + } + if (stepOptions.expectFailure === true && result.status === 0) { + assert.fail(`${step} unexpectedly succeeded\ncommand: agent-device ${fullArgs.join(' ')}`); + } + } + + function updateSessionState(context: Context, command: string | undefined, status: number): void { + if (status !== 0) return; + if (command === 'open') context.sessionOpen = true; + if (command === 'close') context.sessionOpen = false; + } + + function verifyCommand(context: Context, command: string, evidence: string): void { + recordCommandEvidence(context, command, command, evidence); + } + + function verifyNestedCommand( + context: Context, + command: string, + executedCommand: string, + evidence: string, + ): void { + recordCommandEvidence(context, command, executedCommand, evidence); + } + + function recordCommandEvidence( + context: Context, + command: string, + executedCommand: string, + evidence: string, + ): void { + assert.ok( + context.stepHistory.some( + (record) => + record.scenario === context.currentScenario && + record.commandName === executedCommand && + record.accepted, + ), + `${context.currentScenario} credited ${command} without a successful ${executedCommand} execution`, + ); + context.commandEvidence[command] = [...(context.commandEvidence[command] ?? []), evidence]; + } + + function verifyBehavior(context: Context, behavior: BehaviorId, evidence: string): void { + context.behaviorEvidence[behavior] = [...(context.behaviorEvidence[behavior] ?? []), evidence]; + } + + async function sessionExists(context: Context): Promise { + const inventory = await runStep( + context, + 'inspect final session ownership', + ['session', 'list'], + { + commonFlags: false, + }, + ); + const sessions = Array.isArray(inventory.json?.data?.sessions) + ? inventory.json.data.sessions + : []; + return sessions.some((session: { name?: unknown }) => session.name === context.session); + } + + return { + runScenario, + runStep, + sessionExists, + verifyBehavior, + verifyCommand, + verifyNestedCommand, + }; +} + +export function requiredEnv(name: string, enabledFlag: string): string { + const value = process.env[name]?.trim(); + assert.ok(value, `${name} is required when ${enabledFlag}=1`); + return value; +} + +function evidenceCounts( + keys: readonly Key[], + evidence: Partial>, +): ReadonlyMap { + return new Map(keys.map((key) => [key, evidence[key]?.length ?? 0])); +} + +function assertNewEvidence( + scenarioId: string, + keys: readonly Key[], + evidence: Partial>, + counts: ReadonlyMap, +): void { + for (const key of keys) { + assert.ok( + (evidence[key]?.length ?? 0) > (counts.get(key) ?? 0), + `${scenarioId} produced no specific evidence for ${key}`, + ); + } +} + +function withJson(args: readonly string[]): string[] { + return args.includes('--json') ? [...args] : [...args, '--json']; +} + +function writeStepHistory(context: LiveDeviceContext): void { + fs.writeFileSync( + path.join(context.artifactDir, 'step-history.json'), + JSON.stringify(context.stepHistory, null, 2), + ); +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} diff --git a/test/integration/smoke-android-emulator-coverage.test.ts b/test/integration/smoke-android-emulator-coverage.test.ts index e6dc5218a..ba2b70344 100644 --- a/test/integration/smoke-android-emulator-coverage.test.ts +++ b/test/integration/smoke-android-emulator-coverage.test.ts @@ -1,6 +1,5 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; -import path from 'node:path'; import test from 'node:test'; import { PUBLIC_COMMANDS } from '../../src/command-catalog.ts'; @@ -26,84 +25,64 @@ test('Android emulator coverage exhaustively classifies the public catalog', () for (const command of publicCommands) { const entry = ANDROID_EMULATOR_E2E_COVERAGE[command]; assert.ok(entry.assertion.trim().length > 0, `${command} needs an observable assertion`); - if (typeof entry.owner === 'string') { - assert.ok(entry.owner.trim().length > 0, `${command} needs a scenario owner`); + if (entry.level === 'live') { + assert.ok(entry.scenario.trim().length > 0, `${command} needs a scenario owner`); } else { - assert.ok(entry.owner.path.trim().length > 0, `${command} needs an evidence path`); - assert.ok(entry.owner.test.trim().length > 0, `${command} needs named evidence`); - } - if (entry.level === 'known-gap') { - assert.match(entry.trackingIssue, /^#\d+$/, `${command} gap needs a tracking issue`); + assert.ok(entry.evidence.path.trim().length > 0, `${command} needs an evidence path`); } } }); -test('Android live claims execute in exactly one scenario with command-specific evidence', () => { - const scenarios = new Map( - ANDROID_EMULATOR_LIVE_SCENARIOS.map((scenario) => [scenario.id, scenario]), - ); - for (const [command, entry] of Object.entries(ANDROID_EMULATOR_E2E_COVERAGE)) { - if (entry.level !== 'live' && entry.level !== 'known-gap') continue; - assert.ok(scenarios.has(entry.owner), `${command} references missing scenario ${entry.owner}`); - assert.ok( - liveCommandsForScenario(entry.owner).includes( - command as (typeof PUBLIC_COMMANDS)[keyof typeof PUBLIC_COMMANDS], - ), - `${entry.owner} does not execute ${command}`, - ); - const source = [ - fs.readFileSync(scenarios.get(entry.owner)?.source ?? '', 'utf8'), - fs.readFileSync('test/integration/android-emulator-e2e/live-assertions.ts', 'utf8'), - ].join('\n'); - const commandKey = Object.entries(PUBLIC_COMMANDS).find(([, value]) => value === command)?.[0]; - assert.ok(commandKey, `public command key missing for ${command}`); - assert.match( - source, - new RegExp(`verifyCommand\\(\\s*context,\\s*(?:C\\.${commandKey}|['"]${command}['"])`), - `${entry.owner} has no runtime command-specific evidence assertion for ${command}`, +test('Android live command ownership is structural and exhaustive', () => { + for (const scenario of ANDROID_EMULATOR_LIVE_SCENARIOS) { + assert.deepEqual( + [...scenario.commands].sort(), + liveCommandsForScenario(scenario.id).sort(), + `${scenario.id} command declaration must match the coverage manifest`, ); } - const claimed = ANDROID_EMULATOR_LIVE_SCENARIOS.flatMap((scenario) => - liveCommandsForScenario(scenario.id), - ); - assert.equal( - new Set(claimed).size, - claimed.length, - 'live commands need exactly one primary owner', - ); + const claimed = ANDROID_EMULATOR_LIVE_SCENARIOS.flatMap((scenario) => scenario.commands); + const liveCommands = Object.entries(ANDROID_EMULATOR_E2E_COVERAGE) + .filter(([, entry]) => entry.level === 'live') + .map(([command]) => command); + assert.deepEqual([...claimed].sort(), liveCommands.sort()); + assert.equal(new Set(claimed).size, claimed.length, 'live commands need one primary owner'); }); -test('Android emulator non-live owners name executable repository evidence', () => { +test('Android emulator non-live owners name executable repository modules', () => { for (const [command, entry] of Object.entries(ANDROID_EMULATOR_E2E_COVERAGE)) { - if (entry.level === 'live' || entry.level === 'known-gap') continue; - const ownerPath = path.resolve(entry.owner.path); - assert.ok(fs.existsSync(ownerPath), `${command} owner does not exist: ${entry.owner.path}`); + if (entry.level === 'live') continue; assert.ok( - fs.readFileSync(ownerPath, 'utf8').includes(entry.owner.test), - `${command} owner does not contain named evidence: ${entry.owner.test}`, + fs.existsSync(entry.evidence.path), + `${command} evidence does not exist: ${entry.evidence.path}`, ); } }); test('Android behavior patterns are owned by live fixture journeys', () => { - const scenarioIds = new Set(ANDROID_EMULATOR_LIVE_SCENARIOS.map((scenario) => scenario.id)); + const claimedBehaviors = ANDROID_EMULATOR_LIVE_SCENARIOS.flatMap( + (scenario) => scenario.behaviors, + ); for (const [behavior, entry] of Object.entries(ANDROID_EMULATOR_BEHAVIOR_COVERAGE)) { assert.ok(entry.assertion.trim().length > 0, `${behavior} needs observable assertion`); - assert.equal(entry.level, 'live'); - assert.ok( - scenarioIds.has(entry.owner), - `${behavior} references missing scenario ${entry.owner}`, + const scenario = ANDROID_EMULATOR_LIVE_SCENARIOS.find( + (candidate) => candidate.id === entry.owner, ); - const source = fs.readFileSync( - ANDROID_EMULATOR_LIVE_SCENARIOS.find((scenario) => scenario.id === entry.owner)?.source ?? '', - 'utf8', - ); - assert.match( - source, - new RegExp(`verifyBehavior\\(\\s*context,\\s*['"]${behavior}['"]`), - `${entry.owner} has no runtime behavior assertion for ${behavior}`, + assert.ok(scenario, `${behavior} references missing scenario ${entry.owner}`); + assert.ok( + scenario.behaviors.some((claimed) => claimed === behavior), + `${scenario.id} must declare ${behavior}`, ); } + assert.deepEqual( + [...claimedBehaviors].sort(), + Object.keys(ANDROID_EMULATOR_BEHAVIOR_COVERAGE).sort(), + ); + assert.equal( + new Set(claimedBehaviors).size, + claimedBehaviors.length, + 'live behaviors need one primary owner', + ); }); test('Android emulator capability denial matches the public catalog', () => { diff --git a/test/scripts/android-fixture-cache-smoke.sh b/test/scripts/android-fixture-cache-smoke.sh deleted file mode 100644 index b2ee3770b..000000000 --- a/test/scripts/android-fixture-cache-smoke.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -set -eu - -if [ "$#" -ne 2 ] || [ -z "$1" ] || [ -z "$2" ]; then - echo "Usage: android-fixture-cache-smoke.sh " >&2 - exit 2 -fi - -APK_PATH="$1" -APP_ID="$2" -PROJECT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" -SESSION="fixture-cache" -SNAPSHOT_PATH="$PROJECT_DIR/test/artifacts/android-fixture-cache-snapshot.json" - -cleanup() { - node --experimental-strip-types "$PROJECT_DIR/src/bin.ts" close --platform android --session "$SESSION" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -mkdir -p "$(dirname -- "$SNAPSHOT_PATH")" -node --experimental-strip-types "$PROJECT_DIR/src/bin.ts" install "$APK_PATH" --platform android -node --experimental-strip-types "$PROJECT_DIR/src/bin.ts" open "$APP_ID" --platform android --session "$SESSION" --relaunch -node --experimental-strip-types "$PROJECT_DIR/src/bin.ts" snapshot -i --platform android --session "$SESSION" --json > "$SNAPSHOT_PATH" -node "$PROJECT_DIR/test/scripts/assert-android-fixture-snapshot.mjs" "$SNAPSHOT_PATH" "$APP_ID" diff --git a/test/scripts/android-snapshot-helper-release-smoke.sh b/test/scripts/android-snapshot-helper-release-smoke.sh deleted file mode 100755 index 365cc5c8d..000000000 --- a/test/scripts/android-snapshot-helper-release-smoke.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/sh -set -eu - -PROJECT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" -PACKAGE_NAME="com.callstack.agentdevice.snapshotsmoke" -APP_LABEL="Agent Device Snapshot Smoke" -# Keep these in sync with the Android workflow SDK packages and emulator API level. -MIN_SDK=23 -TARGET_SDK=36 - -SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-}}" -if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then - echo "ANDROID_HOME or ANDROID_SDK_ROOT must point to an Android SDK" >&2 - exit 1 -fi - -ANDROID_JAR="$SDK_ROOT/platforms/android-$TARGET_SDK/android.jar" -if [ ! -f "$ANDROID_JAR" ]; then - echo "Missing Android platform jar: $ANDROID_JAR" >&2 - exit 1 -fi - -BUILD_TOOLS_DIR="$( - find "$SDK_ROOT/build-tools" -maxdepth 1 -mindepth 1 -type d 2>/dev/null | sort -V | tail -n 1 -)" -if [ -z "$BUILD_TOOLS_DIR" ] || [ ! -x "$BUILD_TOOLS_DIR/aapt2" ]; then - echo "Missing Android build tools under $SDK_ROOT/build-tools" >&2 - exit 1 -fi - -WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/agent-device-android-helper-smoke.XXXXXX")" -cleanup() { - rm -rf "$WORK_DIR" -} -trap cleanup EXIT - -SRC_DIR="$WORK_DIR/src/com/callstack/agentdevice/snapshotsmoke" -CLASSES_DIR="$WORK_DIR/classes" -DEX_DIR="$WORK_DIR/dex" -UNSIGNED_APK="$WORK_DIR/app-unsigned.apk" -ALIGNED_APK="$WORK_DIR/app-aligned.apk" -APK_PATH="$WORK_DIR/app-release.apk" -KEYSTORE="$PROJECT_DIR/android/snapshot-helper/debug.keystore" - -# This APK is throwaway test code signed with the helper debug keystore only for CI smoke coverage. -mkdir -p "$SRC_DIR" "$CLASSES_DIR" "$DEX_DIR" - -cat > "$WORK_DIR/AndroidManifest.xml" < - - - - - - - - - -EOF_MANIFEST - -mkdir -p "$WORK_DIR/res/values" -cat > "$WORK_DIR/res/values/styles.xml" < -