Skip to content

feat: improve maestro test reporter #811

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
thymikee merged 1 commit into from
Jun 16, 2026
Merged

feat: improve maestro test reporter #811

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 58 additions & 23 deletions src/__tests__/cli-network.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,8 @@ test('test command prints suite summary and exits non-zero on failures', async (
assert.equal(result.calls.length, 1);
assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test');
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
assert.match(
result.stdout,
/FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/,
);
assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/);
assert.doesNotMatch(result.stdout, /⨯ "Checkout failure" in 02-fail\.ad/);
assert.match(result.stdout, /Replay failed at step 1 \(open Demo\): boom/);
assert.match(result.stdout, /artifacts: \/tmp\/test-artifacts\/02-fail/);
assert.doesNotMatch(result.stdout, /SKIP \/tmp\/03-skip\.ad/);
Expand All @@ -125,11 +122,12 @@ test('test command --verbose prints all test statuses', async () => {
assert.equal(result.code, 1);
assert.equal(result.calls[0]?.meta?.debug, false);
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/);
assert.match(result.stdout, /SKIP 03-skip\.ad/);
assert.doesNotMatch(result.stdout, /✓ 01-pass\.ad \(0\.01s\)/);
assert.doesNotMatch(result.stdout, /SKIP 03-skip\.ad/);
assert.match(result.stdout, /Test summary: 1 passed, 1 failed in 0\.025s/);
});

test('test command --verbose prints step telemetry for passing tests without debug mode', async () => {
test('test command --verbose omits step telemetry for passing tests without debug mode', async () => {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-'));
const artifactsDir = path.join(tmpDir, 'auth-flow');
const attemptDir = path.join(artifactsDir, 'attempt-1');
Expand Down Expand Up @@ -203,16 +201,16 @@ test('test command --verbose prints step telemetry for passing tests without deb

assert.equal(result.code, null);
assert.equal(result.calls[0]?.meta?.debug, false);
assert.match(result.stdout, /PASS "Authentication flow" \(0\.5s\)/);
assert.match(result.stdout, /steps:/);
assert.match(result.stdout, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/);
assert.match(result.stdout, /assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/);
assert.doesNotMatch(result.stdout, /✓ "Authentication flow" in auth-flow\.yml \(0\.5s\)/);
assert.doesNotMatch(result.stdout, /steps:/);
assert.doesNotMatch(result.stdout, /tapOn "text=\\"Log in\\""/);
assert.doesNotMatch(result.stdout, /assertVisible "text=\\"Home\\""/);
} finally {
await fs.rm(tmpDir, { recursive: true, force: true });
}
});

test('test command --verbose keeps nested retry and open step telemetry distinct', async () => {
test('test command --verbose omits nested passing step telemetry', async () => {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-verbose-retry-'));
const artifactsDir = path.join(tmpDir, 'material-top-tabs');
const attemptDir = path.join(artifactsDir, 'attempt-1');
Expand Down Expand Up @@ -300,15 +298,15 @@ test('test command --verbose keeps nested retry and open step telemetry distinct
}));

assert.equal(result.code, null);
assert.match(
assert.doesNotMatch(
result.stdout,
/open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 0\.727s\)/,
);
assert.match(
assert.doesNotMatch(
result.stdout,
/assertVisible "label=\\"Chat\\" \|\| text=\\"Chat\\" \|\| id=\\"Chat\\"" "60000" \(line 4, 2\.58s\)/,
);
assert.match(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
assert.doesNotMatch(result.stdout, /retry "3" \(line 4, 3\.31s\)/);
assert.doesNotMatch(
result.stdout,
/open "org\.reactnavigation\.playground" "rne:\/\/material-top-tabs-basic" \(line 4, 3\.31s\)/,
Expand Down Expand Up @@ -354,30 +352,45 @@ test('test command reports flaky passed-on-retry cases in the default summary',
assert.equal(result.code, null);
assert.doesNotMatch(result.stderr, /Running replay suite\.\.\./);
assert.doesNotMatch(result.stdout, /FLAKY/);
assert.match(
assert.doesNotMatch(
result.stdout,
/PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
/^✓ "Authentication flow" in auth-flow\.yml \(passed attempt 17\.5s, total 112\.2s\)$/m,
);
assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/);
assert.match(result.stdout, /Flaky tests:/);
assert.match(
result.stdout,
/PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
/✓ "Authentication flow" in auth-flow\.yml after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/,
);
assert.match(
result.stdout,
/attempt 1 failed \(94\.7s\): Replay failed at step 3 \(tapOn "Log in"\): selector not found/,
);
});

test('test command prints failed attempt step telemetry when timing trace exists', async () => {
test('test command --debug prints failed attempt step window when timing trace exists', async () => {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-device-cli-test-steps-'));
const artifactsDir = path.join(tmpDir, 'checkout-flow');
const attemptDir = path.join(artifactsDir, 'attempt-2');
await fs.mkdir(attemptDir, { recursive: true });
await fs.writeFile(
path.join(attemptDir, 'replay-timing.ndjson'),
[
{
type: 'replay_action_start',
step: 0,
line: 2,
command: 'close',
positionals: ['Demo'],
},
{
type: 'replay_action_stop',
step: 0,
line: 2,
command: 'close',
ok: true,
durationMs: 50,
},
{
type: 'replay_action_start',
step: 1,
Expand Down Expand Up @@ -406,6 +419,21 @@ test('test command prints failed attempt step telemetry when timing trace exists
step: 2,
line: 4,
command: '__maestroTapOn',
ok: true,
durationMs: 200,
},
{
type: 'replay_action_start',
step: 3,
line: 5,
command: '__maestroAssertVisible',
positionals: ['text="Receipt"', '3000'],
},
{
type: 'replay_action_stop',
step: 3,
line: 5,
command: '__maestroAssertVisible',
ok: false,
durationMs: 1500,
errorCode: 'ASSERTION_FAILED',
Expand All @@ -426,10 +454,10 @@ test('test command prints failed attempt step telemetry when timing trace exists
artifactsDir,
error: {
code: 'ASSERTION_FAILED',
message: 'Replay failed at step 2 (click "Pay"): selector not found',
message: 'Replay failed at step 3 (assertVisible "Receipt"): selector not found',
},
};
const result = await runCliCapture(['test', './suite'], async () => ({
const result = await runCliCapture(['test', './suite', '--debug'], async () => ({
ok: true,
data: {
total: 1,
Expand All @@ -445,11 +473,18 @@ test('test command prints failed attempt step telemetry when timing trace exists
}));

assert.equal(result.code, 1);
assert.equal(result.calls[0]?.meta?.debug, true);
assert.match(
result.stdout,
/Replay failed at step 3 \(assertVisible "Receipt"\): selector not found/,
);
assert.match(result.stdout, /steps \(attempt 2\):/);
assert.doesNotMatch(result.stdout, /close "Demo" \(line 2, 0\.050s\)/);
assert.match(result.stdout, /open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/);
assert.match(result.stdout, /tapOn "text=\\"Pay\\"" \(line 4, 0\.2s\)/);
assert.match(
result.stdout,
/\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/,
/\[FAIL\] assertVisible "text=\\"Receipt\\"" "3000" \(line 5, 1\.50s, ASSERTION_FAILED\)/,
);
} finally {
await fs.rm(tmpDir, { recursive: true, force: true });
Expand Down
130 changes: 108 additions & 22 deletions src/__tests__/cli-test-progress.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,19 @@ import assert from 'node:assert/strict';
import { formatReplayTestProgressEvent } from '../cli-test-progress.ts';
import type { RequestProgressEvent } from '../daemon/request-progress.ts';

test('formatReplayTestProgressEvent renders replay suite start context', () => {
function withStreamTty<T>(stream: NodeJS.WriteStream, isTTY: boolean, run: () => T): T {
const descriptor = Object.getOwnPropertyDescriptor(stream, 'isTTY');
const mutableStream = stream as unknown as Record<string, unknown>;
try {
Object.defineProperty(stream, 'isTTY', { configurable: true, value: isTTY });
return run();
} finally {
if (descriptor) Object.defineProperty(stream, 'isTTY', descriptor);
else delete mutableStream.isTTY;
}
}

test('formatReplayTestProgressEvent suppresses replay suite start context', () => {
const line = formatReplayTestProgressEvent({
type: 'replay-test-suite',
status: 'start',
Expand All @@ -15,17 +27,10 @@ test('formatReplayTestProgressEvent renders replay suite start context', () => {
shardCount: 2,
});

assert.equal(
line,
[
'Running replay suite: 4 files',
' sharding: split across 2 devices',
' artifacts: /tmp/replay-suite',
].join('\n'),
);
assert.equal(line, undefined);
});

test('formatReplayTestProgressEvent renders replay test start context with shard metadata', () => {
test('formatReplayTestProgressEvent suppresses replay test start context', () => {
const line = formatReplayTestProgressEvent({
type: 'replay-test',
file: '/tmp/auth-flow.yml',
Expand All @@ -40,14 +45,7 @@ test('formatReplayTestProgressEvent renders replay test start context with shard
deviceId: 'E140A942-965C-4A92-AC63-F3B23756BE02',
});

assert.equal(
line,
[
'[2/5] START "Authentication flow" in auth-flow.yml [shard 2/2 E140A942-965C-4A92-AC63-F3B23756BE02]',
' session: maestro-test:test:suite:2:attempt-1',
' artifacts: /tmp/replay-suite/auth-flow',
].join('\n'),
);
assert.equal(line, undefined);
});

test('formatReplayTestProgressEvent ignores unknown progress event types', () => {
Expand All @@ -72,7 +70,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
maxAttempts: 2,
durationMs: 12_345,
},
expected: /^\[1\/3] PASS 01-login\.ad after 2 attempts \(total 12\.3s\)$/,
expected: /^✓ 01-login\.ad \(12\.3s\)$/,
},
{
event: {
Expand All @@ -87,7 +85,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
retrying: true,
message: 'first attempt failed',
},
expected: /^\[2\/3] RETRY 02-checkout\.ad attempt 1\/2 \(1\.23s\)\n first attempt failed$/,
expected: /^$/,
},
{
event: {
Expand All @@ -104,7 +102,7 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
artifactsDir: '/tmp/replay-suite/payment',
},
expected:
/^\[3\/3] FAIL 03-payment\.ad after 2 attempts \(total 9\.88s\)\n assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/,
/^⨯ 03-payment\.ad \(9\.88s\)\n failed at: assertVisible failed\n session: maestro-test:test:suite:3:attempt-2\n artifacts: \/tmp\/replay-suite\/payment$/,
},
{
event: {
Expand All @@ -115,11 +113,99 @@ test('formatReplayTestProgressEvent renders pass, retry, fail, and skip cases',
total: 5,
message: 'missing platform metadata for --platform ios',
},
expected: /^\[4\/5] SKIP 04-skip\.ad\n missing platform metadata for --platform ios$/,
expected: /^- 04-skip\.ad\n missing platform metadata for --platform ios$/,
},
];

for (const { event, expected } of cases) {
assert.match(formatReplayTestProgressEvent(event) ?? '', expected);
}
});

test('formatReplayTestProgressEvent colors stderr progress rows when stdout is piped', () => {
const originalForceColor = process.env.FORCE_COLOR;
const originalNoColor = process.env.NO_COLOR;
delete process.env.FORCE_COLOR;
delete process.env.NO_COLOR;
try {
const line = withStreamTty(process.stdout, false, () =>
withStreamTty(process.stderr, true, () =>
formatReplayTestProgressEvent({
type: 'replay-test',
file: '/tmp/01-pass.ad',
status: 'pass',
index: 1,
total: 1,
attempt: 1,
durationMs: 10,
}),
),
);

assert.equal(line, '\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)');
} finally {
if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor;
else delete process.env.FORCE_COLOR;
if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor;
else delete process.env.NO_COLOR;
}
});

test('formatReplayTestProgressEvent colors completed result markers when color is enabled', () => {
const originalForceColor = process.env.FORCE_COLOR;
const originalNoColor = process.env.NO_COLOR;
process.env.FORCE_COLOR = '1';
delete process.env.NO_COLOR;
try {
formatReplayTestProgressEvent({
type: 'replay-test-suite',
status: 'start',
total: 3,
runnable: 3,
skipped: 0,
artifactsDir: '/tmp/replay-suite',
});
assert.equal(
formatReplayTestProgressEvent({
type: 'replay-test',
file: '/tmp/01-pass.ad',
status: 'pass',
index: 1,
total: 3,
attempt: 1,
durationMs: 10,
}),
'\u001B[32m✓\u001B[39m 01-pass.ad (0.01s)',
);
assert.equal(
formatReplayTestProgressEvent({
type: 'replay-test',
file: '/tmp/02-flaky.yml',
title: 'Retry flow',
status: 'pass',
index: 2,
total: 3,
attempt: 2,
durationMs: 30,
}),
'\u001B[33m✓\u001B[39m "Retry flow" in 02-flaky.yml (0.03s)',
);
const failedLine = formatReplayTestProgressEvent({
type: 'replay-test',
file: '/tmp/03-fail.ad',
title: 'Checkout failure',
status: 'fail',
index: 3,
total: 3,
attempt: 1,
durationMs: 5,
message: 'boom',
});
assert.ok(failedLine?.startsWith('\u001B[31m⨯\u001B[39m "Checkout failure" in 03-fail.ad'));
} finally {
if (typeof originalForceColor === 'string') process.env.FORCE_COLOR = originalForceColor;
else delete process.env.FORCE_COLOR;
if (typeof originalNoColor === 'string') process.env.NO_COLOR = originalNoColor;
else delete process.env.NO_COLOR;
}
});
Loading
Loading