feat(scenario): attach step output and assertion details to run log
- "Step N passed" log entry carries the step's return value as data
- context.assert accepts an optional details argument; on failure the
assertion:fail entry stores { details, error } as data
- assertion failures throw AssertionFailedError so "Step N failed" and
"Snippet X failed" entries name only the assertion, without repeating
the details; the step run description keeps the full message
- exported e2e runtime accepts and prints assert details
- log output shows elapsed time in whole seconds, matching the stored
timestamp resolution
- run artifact and scenario file names link to the inline file content
in a new tab
This commit is contained in:
9 files changed
+195
-17
No files matched your search
@@ -208,4 +208,83 @@ describe("ScenarioRunStepEntity.output + context.getStepOutput", () => {
|
||||
expect(JSON.parse(rows[2].output as string)).toEqual([1, 2, 3, 4]);
|
||||
});
|
||||
});
|
||||
// ── run log data ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("run log data", () => {
|
||||
async function logRows(
|
||||
runId: string,
|
||||
context: string,
|
||||
): Promise<Array<{ message: string; data: string | null }>> {
|
||||
return dataSource.query(
|
||||
`SELECT message, data FROM scenario_run_logs WHERE runId = '${runId}' AND context = '${context}' ORDER BY seq`,
|
||||
);
|
||||
}
|
||||
|
||||
it("attaches the step output to the 'Step N passed' entry", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-step-output");
|
||||
await createStep(scId, 0, 'return { token: "abc" };');
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const rows = await logRows(runId, "step:1");
|
||||
const passed = rows.find((r) => r.message === "Step 1 passed");
|
||||
|
||||
expect(JSON.parse(passed!.data as string)).toEqual({ token: "abc" });
|
||||
});
|
||||
|
||||
it("stores assertion failure details as log data", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-assert-details");
|
||||
await createStep(
|
||||
scId,
|
||||
0,
|
||||
'await context.assert(() => false, "Validation must pass", { status: 500 });',
|
||||
);
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const [row] = await logRows(runId, "assertion:fail");
|
||||
|
||||
expect(row.message).toBe("Assertion failed: Validation must pass");
|
||||
expect(JSON.parse(row.data as string)).toEqual({
|
||||
details: { status: 500 },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the 'Step N failed' entry of an assertion failure short", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-assert-step-fail");
|
||||
await createStep(
|
||||
scId,
|
||||
0,
|
||||
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
|
||||
);
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const rows = await logRows(runId, "step:1");
|
||||
const failed = rows.find((r) => r.message.startsWith("Step 1 failed:"));
|
||||
|
||||
expect(failed!.message).toBe(
|
||||
"Step 1 failed: Assertion failed: Must not throw",
|
||||
);
|
||||
const stepRow = await dataSource.query(
|
||||
`SELECT description FROM scenario_run_steps WHERE runId = '${runId}'`,
|
||||
);
|
||||
expect(stepRow[0].description).toContain("(boom)");
|
||||
});
|
||||
|
||||
it("stores the exception message of a throwing assertion as log data", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-assert-throw");
|
||||
await createStep(
|
||||
scId,
|
||||
0,
|
||||
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
|
||||
);
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const [row] = await logRows(runId, "assertion:fail");
|
||||
|
||||
expect(JSON.parse(row.data as string)).toEqual({ error: "boom" });
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user