feat(scenario): attach step output and assertion details to run log
- "Step N passed" log entry carries the step's return value as data
- context.assert accepts an optional details argument; on failure the
assertion:fail entry stores { details, error } as data
- assertion failures throw AssertionFailedError so "Step N failed" and
"Snippet X failed" entries name only the assertion, without repeating
the details; the step run description keeps the full message
- exported e2e runtime accepts and prints assert details
- log output shows elapsed time in whole seconds, matching the stored
timestamp resolution
- run artifact and scenario file names link to the inline file content
in a new tab
This commit is contained in:
9 files changed
+195
-17
No files matched your search
@@ -18,6 +18,31 @@ export interface ExecResult {
|
||||
result: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thrown by `context.assert`. Its `message` includes the failure reason;
|
||||
* `summary` doesn't, so failure reports that follow the `assertion:fail` log
|
||||
* entry (which already holds the details) can stay short.
|
||||
*/
|
||||
export class AssertionFailedError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
readonly summary: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "AssertionFailedError";
|
||||
}
|
||||
}
|
||||
|
||||
/** The `AssertionFailedError` behind `err` (directly or via `cause`), if any. */
|
||||
export function findAssertionFailure(
|
||||
err: unknown,
|
||||
): AssertionFailedError | null {
|
||||
for (let e = err; e instanceof Error; e = (e as { cause?: unknown }).cause) {
|
||||
if (e instanceof AssertionFailedError) return e;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export type ScriptLogger = (
|
||||
level: "debug" | "log" | "warn" | "error",
|
||||
message: string,
|
||||
@@ -50,11 +75,13 @@ export interface ScriptContext {
|
||||
/**
|
||||
* Runs `fn` and requires it to return `true`. Logs the outcome under
|
||||
* `description`, then throws if `fn` returned anything other than `true`
|
||||
* or threw an exception.
|
||||
* or threw an exception. On failure, `details` (and the exception message,
|
||||
* if any) are stored as the log entry's data.
|
||||
*/
|
||||
assert: (
|
||||
fn: () => boolean | Promise<boolean>,
|
||||
description: string,
|
||||
details?: unknown,
|
||||
) => Promise<void>;
|
||||
/** Runs a named snippet with the same context, plus any extra positional args. */
|
||||
runSnippet: (alias: string, ...args: unknown[]) => Promise<unknown>;
|
||||
@@ -194,6 +221,7 @@ export class CodeExecutorService {
|
||||
assert: async (
|
||||
fn: () => boolean | Promise<boolean>,
|
||||
description: string,
|
||||
details?: unknown,
|
||||
): Promise<void> => {
|
||||
let passed: boolean;
|
||||
let failureReason: string | undefined;
|
||||
@@ -210,8 +238,18 @@ export class CodeExecutorService {
|
||||
const message = failureReason
|
||||
? `Assertion failed: ${description} (${failureReason})`
|
||||
: `Assertion failed: ${description}`;
|
||||
emit("error", message, "assertion:fail");
|
||||
throw new Error(message);
|
||||
const failureData =
|
||||
failureReason !== undefined || details !== undefined
|
||||
? {
|
||||
...(failureReason !== undefined && { error: failureReason }),
|
||||
...(details !== undefined && { details }),
|
||||
}
|
||||
: undefined;
|
||||
emit("error", message, "assertion:fail", failureData);
|
||||
throw new AssertionFailedError(
|
||||
message,
|
||||
`Assertion failed: ${description}`,
|
||||
);
|
||||
},
|
||||
getStepOutput: getStepOutput ?? (() => Promise.resolve(null)),
|
||||
getCredential: (alias: string): unknown => {
|
||||
@@ -262,7 +300,7 @@ export class CodeExecutorService {
|
||||
} catch (err) {
|
||||
emit(
|
||||
"error",
|
||||
`Snippet "${alias}" failed: ${(err as Error).message}`,
|
||||
`Snippet "${alias}" failed: ${findAssertionFailure(err)?.summary ?? (err as Error).message}`,
|
||||
snippetContext,
|
||||
);
|
||||
throw err;
|
||||
|
||||
@@ -370,7 +370,11 @@ function buildContext(page: Page, browser: BrowserContext, stepOutputs: StepOutp
|
||||
log: (...args: unknown[]) => console.log(...args),
|
||||
warn: (...args: unknown[]) => console.warn(...args),
|
||||
error: (...args: unknown[]) => console.error(...args),
|
||||
assert: async (fn: () => boolean | Promise<boolean>, description: string): Promise<void> => {
|
||||
assert: async (
|
||||
fn: () => boolean | Promise<boolean>,
|
||||
description: string,
|
||||
details?: unknown,
|
||||
): Promise<void> => {
|
||||
let passed: boolean;
|
||||
let failureReason: string | undefined;
|
||||
try {
|
||||
@@ -386,7 +390,11 @@ function buildContext(page: Page, browser: BrowserContext, stepOutputs: StepOutp
|
||||
const message = failureReason
|
||||
? \`Assertion failed: \${description} (\${failureReason})\`
|
||||
: \`Assertion failed: \${description}\`;
|
||||
console.error(message);
|
||||
if (details !== undefined) {
|
||||
console.error(message, details);
|
||||
} else {
|
||||
console.error(message);
|
||||
}
|
||||
throw new Error(message);
|
||||
},
|
||||
getStepOutput: async (order: number): Promise<unknown> => {
|
||||
|
||||
@@ -13,7 +13,10 @@ import { Repository } from "typeorm";
|
||||
import { type AppConfig, TraceMode } from "../config/app.config";
|
||||
import { FileStorageService } from "../file/file-storage.service";
|
||||
import type { ScriptLogger } from "../code-executor/code-executor.service";
|
||||
import { CodeExecutorService } from "../code-executor/code-executor.service";
|
||||
import {
|
||||
CodeExecutorService,
|
||||
findAssertionFailure,
|
||||
} from "../code-executor/code-executor.service";
|
||||
import { ExecContextBuilder } from "../code-executor/exec-context.builder";
|
||||
import { traceStorage } from "../common/trace-context";
|
||||
import { TraceLogger } from "../common/trace-logger";
|
||||
@@ -446,7 +449,11 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
} catch (err) {
|
||||
const msg = (err as Error).message ?? String(err);
|
||||
this.logger.error(`StepRun #${stepRun.id} threw: ${msg}`);
|
||||
await this.failStepRun(stepRun, msg);
|
||||
await this.failStepRun(
|
||||
stepRun,
|
||||
msg,
|
||||
findAssertionFailure(err)?.summary,
|
||||
);
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
@@ -571,6 +578,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
"log",
|
||||
`Step ${stepRun.order} passed`,
|
||||
`step:${stepRun.order}`,
|
||||
output,
|
||||
);
|
||||
|
||||
// Find the next waiting step in this run (next by order)
|
||||
@@ -600,9 +608,14 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* `logSummary` replaces `description` in the "Step N failed" log entry, e.g.
|
||||
* for assertion failures whose details are already logged.
|
||||
*/
|
||||
private async failStepRun(
|
||||
stepRun: ScenarioRunStepEntity,
|
||||
description: string,
|
||||
logSummary?: string,
|
||||
): Promise<void> {
|
||||
stepRun.status = "fail";
|
||||
stepRun.description = description;
|
||||
@@ -612,7 +625,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
stepRun.runId,
|
||||
stepRun.id,
|
||||
"error",
|
||||
`Step ${stepRun.order} failed: ${description}`,
|
||||
`Step ${stepRun.order} failed: ${logSummary ?? description}`,
|
||||
`step:${stepRun.order}`,
|
||||
);
|
||||
|
||||
|
||||
@@ -208,4 +208,83 @@ describe("ScenarioRunStepEntity.output + context.getStepOutput", () => {
|
||||
expect(JSON.parse(rows[2].output as string)).toEqual([1, 2, 3, 4]);
|
||||
});
|
||||
});
|
||||
// ── run log data ───────────────────────────────────────────────────────────
|
||||
|
||||
describe("run log data", () => {
|
||||
async function logRows(
|
||||
runId: string,
|
||||
context: string,
|
||||
): Promise<Array<{ message: string; data: string | null }>> {
|
||||
return dataSource.query(
|
||||
`SELECT message, data FROM scenario_run_logs WHERE runId = '${runId}' AND context = '${context}' ORDER BY seq`,
|
||||
);
|
||||
}
|
||||
|
||||
it("attaches the step output to the 'Step N passed' entry", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-step-output");
|
||||
await createStep(scId, 0, 'return { token: "abc" };');
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const rows = await logRows(runId, "step:1");
|
||||
const passed = rows.find((r) => r.message === "Step 1 passed");
|
||||
|
||||
expect(JSON.parse(passed!.data as string)).toEqual({ token: "abc" });
|
||||
});
|
||||
|
||||
it("stores assertion failure details as log data", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-assert-details");
|
||||
await createStep(
|
||||
scId,
|
||||
0,
|
||||
'await context.assert(() => false, "Validation must pass", { status: 500 });',
|
||||
);
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const [row] = await logRows(runId, "assertion:fail");
|
||||
|
||||
expect(row.message).toBe("Assertion failed: Validation must pass");
|
||||
expect(JSON.parse(row.data as string)).toEqual({
|
||||
details: { status: 500 },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the 'Step N failed' entry of an assertion failure short", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-assert-step-fail");
|
||||
await createStep(
|
||||
scId,
|
||||
0,
|
||||
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
|
||||
);
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const rows = await logRows(runId, "step:1");
|
||||
const failed = rows.find((r) => r.message.startsWith("Step 1 failed:"));
|
||||
|
||||
expect(failed!.message).toBe(
|
||||
"Step 1 failed: Assertion failed: Must not throw",
|
||||
);
|
||||
const stepRow = await dataSource.query(
|
||||
`SELECT description FROM scenario_run_steps WHERE runId = '${runId}'`,
|
||||
);
|
||||
expect(stepRow[0].description).toContain("(boom)");
|
||||
});
|
||||
|
||||
it("stores the exception message of a throwing assertion as log data", async () => {
|
||||
await seedSession();
|
||||
const scId = await createScenario("log-assert-throw");
|
||||
await createStep(
|
||||
scId,
|
||||
0,
|
||||
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
|
||||
);
|
||||
|
||||
const runId = await runScenario(scId);
|
||||
const [row] = await logRows(runId, "assertion:fail");
|
||||
|
||||
expect(JSON.parse(row.data as string)).toEqual({ error: "boom" });
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user