feat(scenario): attach step output and assertion details to run log
CI / changes (pull_request) Successful in 9s
CI / version (pull_request) Skipped
CI / server (pull_request) Successful in 1m44s
CI / client (pull_request) Successful in 1m36s
CI / image (pull_request) Skipped

- "Step N passed" log entry carries the step's return value as data
- context.assert accepts an optional details argument; on failure the
  assertion:fail entry stores { details, error } as data
- assertion failures throw AssertionFailedError so "Step N failed" and
  "Snippet X failed" entries name only the assertion, without repeating
  the details; the step run description keeps the full message
- exported e2e runtime accepts and prints assert details
- log output shows elapsed time in whole seconds, matching the stored
  timestamp resolution
- run artifact and scenario file names link to the inline file content
  in a new tab
This commit is contained in:
ars9 committed 2026-10-02 11:05:29 +03:00
1 parent 2281a8729b
commit 959b3b70c9
9 files changed
+195 -17

No files matched your search

@@ -18,6 +18,31 @@ export interface ExecResult {
result: unknown;
}
/**
* Thrown by `context.assert`. Its `message` includes the failure reason;
* `summary` doesn't, so failure reports that follow the `assertion:fail` log
* entry (which already holds the details) can stay short.
*/
export class AssertionFailedError extends Error {
constructor(
message: string,
readonly summary: string,
) {
super(message);
this.name = "AssertionFailedError";
}
}
/** The `AssertionFailedError` behind `err` (directly or via `cause`), if any. */
export function findAssertionFailure(
err: unknown,
): AssertionFailedError | null {
for (let e = err; e instanceof Error; e = (e as { cause?: unknown }).cause) {
if (e instanceof AssertionFailedError) return e;
}
return null;
}
export type ScriptLogger = (
level: "debug" | "log" | "warn" | "error",
message: string,
@@ -50,11 +75,13 @@ export interface ScriptContext {
/**
* Runs `fn` and requires it to return `true`. Logs the outcome under
* `description`, then throws if `fn` returned anything other than `true`
* or threw an exception.
* or threw an exception. On failure, `details` (and the exception message,
* if any) are stored as the log entry's data.
*/
assert: (
fn: () => boolean | Promise<boolean>,
description: string,
details?: unknown,
) => Promise<void>;
/** Runs a named snippet with the same context, plus any extra positional args. */
runSnippet: (alias: string, ...args: unknown[]) => Promise<unknown>;
@@ -194,6 +221,7 @@ export class CodeExecutorService {
assert: async (
fn: () => boolean | Promise<boolean>,
description: string,
details?: unknown,
): Promise<void> => {
let passed: boolean;
let failureReason: string | undefined;
@@ -210,8 +238,18 @@ export class CodeExecutorService {
const message = failureReason
? `Assertion failed: ${description} (${failureReason})`
: `Assertion failed: ${description}`;
emit("error", message, "assertion:fail");
throw new Error(message);
const failureData =
failureReason !== undefined || details !== undefined
? {
...(failureReason !== undefined && { error: failureReason }),
...(details !== undefined && { details }),
}
: undefined;
emit("error", message, "assertion:fail", failureData);
throw new AssertionFailedError(
message,
`Assertion failed: ${description}`,
);
},
getStepOutput: getStepOutput ?? (() => Promise.resolve(null)),
getCredential: (alias: string): unknown => {
@@ -262,7 +300,7 @@ export class CodeExecutorService {
} catch (err) {
emit(
"error",
`Snippet "${alias}" failed: ${(err as Error).message}`,
`Snippet "${alias}" failed: ${findAssertionFailure(err)?.summary ?? (err as Error).message}`,
snippetContext,
);
throw err;
+10 -2
View File
@@ -370,7 +370,11 @@ function buildContext(page: Page, browser: BrowserContext, stepOutputs: StepOutp
log: (...args: unknown[]) => console.log(...args),
warn: (...args: unknown[]) => console.warn(...args),
error: (...args: unknown[]) => console.error(...args),
assert: async (fn: () => boolean | Promise<boolean>, description: string): Promise<void> => {
assert: async (
fn: () => boolean | Promise<boolean>,
description: string,
details?: unknown,
): Promise<void> => {
let passed: boolean;
let failureReason: string | undefined;
try {
@@ -386,7 +390,11 @@ function buildContext(page: Page, browser: BrowserContext, stepOutputs: StepOutp
const message = failureReason
? \`Assertion failed: \${description} (\${failureReason})\`
: \`Assertion failed: \${description}\`;
console.error(message);
if (details !== undefined) {
console.error(message, details);
} else {
console.error(message);
}
throw new Error(message);
},
getStepOutput: async (order: number): Promise<unknown> => {
@@ -13,7 +13,10 @@ import { Repository } from "typeorm";
import { type AppConfig, TraceMode } from "../config/app.config";
import { FileStorageService } from "../file/file-storage.service";
import type { ScriptLogger } from "../code-executor/code-executor.service";
import { CodeExecutorService } from "../code-executor/code-executor.service";
import {
CodeExecutorService,
findAssertionFailure,
} from "../code-executor/code-executor.service";
import { ExecContextBuilder } from "../code-executor/exec-context.builder";
import { traceStorage } from "../common/trace-context";
import { TraceLogger } from "../common/trace-logger";
@@ -446,7 +449,11 @@ export class ScenarioSchedulerService implements OnModuleInit {
} catch (err) {
const msg = (err as Error).message ?? String(err);
this.logger.error(`StepRun #${stepRun.id} threw: ${msg}`);
await this.failStepRun(stepRun, msg);
await this.failStepRun(
stepRun,
msg,
findAssertionFailure(err)?.summary,
);
} finally {
clearTimeout(timeoutId);
}
@@ -571,6 +578,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
"log",
`Step ${stepRun.order} passed`,
`step:${stepRun.order}`,
output,
);
// Find the next waiting step in this run (next by order)
@@ -600,9 +608,14 @@ export class ScenarioSchedulerService implements OnModuleInit {
}
}
/**
* `logSummary` replaces `description` in the "Step N failed" log entry, e.g.
* for assertion failures whose details are already logged.
*/
private async failStepRun(
stepRun: ScenarioRunStepEntity,
description: string,
logSummary?: string,
): Promise<void> {
stepRun.status = "fail";
stepRun.description = description;
@@ -612,7 +625,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
stepRun.runId,
stepRun.id,
"error",
`Step ${stepRun.order} failed: ${description}`,
`Step ${stepRun.order} failed: ${logSummary ?? description}`,
`step:${stepRun.order}`,
);
+79
View File
@@ -208,4 +208,83 @@ describe("ScenarioRunStepEntity.output + context.getStepOutput", () => {
expect(JSON.parse(rows[2].output as string)).toEqual([1, 2, 3, 4]);
});
});
// ── run log data ───────────────────────────────────────────────────────────
describe("run log data", () => {
async function logRows(
runId: string,
context: string,
): Promise<Array<{ message: string; data: string | null }>> {
return dataSource.query(
`SELECT message, data FROM scenario_run_logs WHERE runId = '${runId}' AND context = '${context}' ORDER BY seq`,
);
}
it("attaches the step output to the 'Step N passed' entry", async () => {
await seedSession();
const scId = await createScenario("log-step-output");
await createStep(scId, 0, 'return { token: "abc" };');
const runId = await runScenario(scId);
const rows = await logRows(runId, "step:1");
const passed = rows.find((r) => r.message === "Step 1 passed");
expect(JSON.parse(passed!.data as string)).toEqual({ token: "abc" });
});
it("stores assertion failure details as log data", async () => {
await seedSession();
const scId = await createScenario("log-assert-details");
await createStep(
scId,
0,
'await context.assert(() => false, "Validation must pass", { status: 500 });',
);
const runId = await runScenario(scId);
const [row] = await logRows(runId, "assertion:fail");
expect(row.message).toBe("Assertion failed: Validation must pass");
expect(JSON.parse(row.data as string)).toEqual({
details: { status: 500 },
});
});
it("keeps the 'Step N failed' entry of an assertion failure short", async () => {
await seedSession();
const scId = await createScenario("log-assert-step-fail");
await createStep(
scId,
0,
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
);
const runId = await runScenario(scId);
const rows = await logRows(runId, "step:1");
const failed = rows.find((r) => r.message.startsWith("Step 1 failed:"));
expect(failed!.message).toBe(
"Step 1 failed: Assertion failed: Must not throw",
);
const stepRow = await dataSource.query(
`SELECT description FROM scenario_run_steps WHERE runId = '${runId}'`,
);
expect(stepRow[0].description).toContain("(boom)");
});
it("stores the exception message of a throwing assertion as log data", async () => {
await seedSession();
const scId = await createScenario("log-assert-throw");
await createStep(
scId,
0,
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
);
const runId = await runScenario(scId);
const [row] = await logRows(runId, "assertion:fail");
expect(JSON.parse(row.data as string)).toEqual({ error: "boom" });
});
});
});