diff --git a/client/src/components/modals/code-help.md b/client/src/components/modals/code-help.md index cd01cd7..1dd3cce 100644 --- a/client/src/components/modals/code-help.md +++ b/client/src/components/modals/code-help.md @@ -112,12 +112,12 @@ Every entry is tagged with where it came from: `step:`, or `snippet:` ## Assertions -### `context.assert(fn, description)` → `Promise` +### `context.assert(fn, description, details?)` → `Promise` Runs `fn` (sync or async) and requires it to return exactly `true`. - On success, it logs `description` (tagged `assertion:pass`). -- If `fn` returns anything else or throws, it logs `Assertion failed: ` as an error (tagged `assertion:fail`) and throws, which fails the step. +- If `fn` returns anything else or throws, it logs `Assertion failed: ` as an error (tagged `assertion:fail`) and throws, which fails the step. The entry's data holds `details` (if given) and the exception message as `error` (if `fn` threw). Await it, or a failure will not stop the step. @@ -128,6 +128,15 @@ await context.assert( ); ``` +Pass large diagnostics (e.g. a response body) as `details` rather than in `description`, so the message stays short: + +```js +await context.assert(() => response.ok(), "Validation must pass", { + status: response.status(), + body: await response.json(), +}); +``` + Use `expect` instead for assertions that wait and retry until the page matches, e.g. `await expect(locator).toBeVisible()`. ## Snippets diff --git a/client/src/pages/run/RunDetailPage.tsx b/client/src/pages/run/RunDetailPage.tsx index 4178e92..aec105c 100644 --- a/client/src/pages/run/RunDetailPage.tsx +++ b/client/src/pages/run/RunDetailPage.tsx @@ -554,7 +554,23 @@ function RunDetail() { rowKey={(f) => f.id} columns={ [ - { key: 'name', header: t('files.name'), render: (f) => f.name }, + { + key: 'name', + header: t('files.name'), + // Served inline, so the browser shows what it can (images, text, PDF). + render: (f) => + isFileExpired(f) ? ( + f.name + ) : ( + + {f.name} + + ), + }, { key: 'mimeType', header: t('files.mimeType'), diff --git a/client/src/pages/scenario/ScenarioDetailPage.tsx b/client/src/pages/scenario/ScenarioDetailPage.tsx index 4cc0d51..2feb26b 100644 --- a/client/src/pages/scenario/ScenarioDetailPage.tsx +++ b/client/src/pages/scenario/ScenarioDetailPage.tsx @@ -761,7 +761,19 @@ export function ScenarioDetailPage() { { key: 'name', header: t('files.name'), - render: (f) => f.name, + // Served inline, so the browser shows what it can (images, text, PDF). + render: (f) => + isFileExpired(f) ? ( + f.name + ) : ( + + {f.name} + + ), }, { key: 'mimeType', diff --git a/client/src/ui/LogOutput/LogOutput.tsx b/client/src/ui/LogOutput/LogOutput.tsx index f53190b..86bc78f 100644 --- a/client/src/ui/LogOutput/LogOutput.tsx +++ b/client/src/ui/LogOutput/LogOutput.tsx @@ -24,10 +24,13 @@ export interface LogOutputProps { className?: string; } -/** "+12.345s" (or "-" if before startTime, e.g. clock skew). */ +/** + * "+12s" (or "-" if before startTime, e.g. clock skew). Log timestamps are + * stored with second resolution, so fractions would be noise. + */ function formatElapsed(deltaMs: number): string { const sign = deltaMs < 0 ? '-' : '+'; - return `${sign}${(Math.abs(deltaMs) / 1000).toFixed(3)}s`; + return `${sign}${Math.round(Math.abs(deltaMs) / 1000)}s`; } /** Pretty-prints `data` if it's valid JSON, otherwise returns it as-is. */ diff --git a/docs/scenario.md b/docs/scenario.md index a159c5e..09eef8a 100644 --- a/docs/scenario.md +++ b/docs/scenario.md @@ -35,7 +35,7 @@ Available on `context`: - `context.getStepOutput(order)`: async (`await` it); output of a previous step by absolute order (`1`, `2`, ...) or relative (`-1` = previous step); `null` if there is none - `context.dumpDom(selector?)`: simplified DOM snapshot of the page (or of one element) - `context.log(...args)`, `context.warn(...args)`, `context.error(...args)`, `context.debug(...args)`: run log entries -- `context.assert(() => boolean, description)`: runs the arrow function and requires it to return `true`. Logs `description` (as a `log` entry on success, as an `error` entry on failure or exception), then throws to stop the step if the function returned anything other than `true` or threw +- `context.assert(() => boolean, description, details?)`: runs the arrow function and requires it to return `true`. Logs `description` (as a `log` entry on success, as an `error` entry on failure or exception), then throws to stop the step if the function returned anything other than `true` or threw. On failure, the log entry's data holds `details` (if given) and the exception message (as `error`, if the function threw) - `context.runSnippet(name, ...args)`: runs stored snippet code with the same context; the snippet receives the extra arguments as `...args`. `name` is the snippet alias for its latest version, or `alias@` (e.g. `login@3`) to pin a specific version. Snippet start/finish/failure is logged - `context.getScenarioFiles(opts?)`: list files uploaded to the scenario (`limit`, `offset`), including their path on disk - `context.downloadFile(url, opts?)`: fetch `url` and save the result as a run file (requires a real scenario run — throws when invoked ad hoc, e.g. via the `exec_code` MCP tool). `opts` may include `method`, `headers`, `body`, `filename`. `url` also accepts a `data:` URI (`data:;base64,` or `data:,`) to save content generated in-script (e.g. a credential JWT) directly, without an actual network fetch. @@ -53,7 +53,7 @@ The step and snippet editors have a (?) button next to the Code label that opens Each run log entry (`level`, `message`, `createdAt`) also carries a `context`: where the message was generated. Contexts: -- `step:`: the step a log entry belongs to, including the automatic "Step N started / passed / failed" entries +- `step:`: the step a log entry belongs to, including the automatic "Step N started / passed / failed" entries ("Step N passed" carries the step's return value as `data`; "Step N failed" for an assertion failure names only the assertion, whose details are in the `assertion:fail` entry) - `snippet:`: entries logged while `context.runSnippet(alias, ...)` is running, including its own automatic "started"/"finished"/"failed" entries. A snippet calling another snippet tags entries with the innermost running snippet's alias - `assertion:pass` / `assertion:fail`: `context.assert()`'s own log entry - `run`: run-level entries not tied to a step (e.g. a scenario timeout, or the Playwright trace upload notice) diff --git a/server/src/code-executor/code-executor.service.ts b/server/src/code-executor/code-executor.service.ts index d908121..fd0d517 100644 --- a/server/src/code-executor/code-executor.service.ts +++ b/server/src/code-executor/code-executor.service.ts @@ -18,6 +18,31 @@ export interface ExecResult { result: unknown; } +/** + * Thrown by `context.assert`. Its `message` includes the failure reason; + * `summary` doesn't, so failure reports that follow the `assertion:fail` log + * entry (which already holds the details) can stay short. + */ +export class AssertionFailedError extends Error { + constructor( + message: string, + readonly summary: string, + ) { + super(message); + this.name = "AssertionFailedError"; + } +} + +/** The `AssertionFailedError` behind `err` (directly or via `cause`), if any. */ +export function findAssertionFailure( + err: unknown, +): AssertionFailedError | null { + for (let e = err; e instanceof Error; e = (e as { cause?: unknown }).cause) { + if (e instanceof AssertionFailedError) return e; + } + return null; +} + export type ScriptLogger = ( level: "debug" | "log" | "warn" | "error", message: string, @@ -50,11 +75,13 @@ export interface ScriptContext { /** * Runs `fn` and requires it to return `true`. Logs the outcome under * `description`, then throws if `fn` returned anything other than `true` - * or threw an exception. + * or threw an exception. On failure, `details` (and the exception message, + * if any) are stored as the log entry's data. */ assert: ( fn: () => boolean | Promise, description: string, + details?: unknown, ) => Promise; /** Runs a named snippet with the same context, plus any extra positional args. */ runSnippet: (alias: string, ...args: unknown[]) => Promise; @@ -194,6 +221,7 @@ export class CodeExecutorService { assert: async ( fn: () => boolean | Promise, description: string, + details?: unknown, ): Promise => { let passed: boolean; let failureReason: string | undefined; @@ -210,8 +238,18 @@ export class CodeExecutorService { const message = failureReason ? `Assertion failed: ${description} (${failureReason})` : `Assertion failed: ${description}`; - emit("error", message, "assertion:fail"); - throw new Error(message); + const failureData = + failureReason !== undefined || details !== undefined + ? { + ...(failureReason !== undefined && { error: failureReason }), + ...(details !== undefined && { details }), + } + : undefined; + emit("error", message, "assertion:fail", failureData); + throw new AssertionFailedError( + message, + `Assertion failed: ${description}`, + ); }, getStepOutput: getStepOutput ?? (() => Promise.resolve(null)), getCredential: (alias: string): unknown => { @@ -262,7 +300,7 @@ export class CodeExecutorService { } catch (err) { emit( "error", - `Snippet "${alias}" failed: ${(err as Error).message}`, + `Snippet "${alias}" failed: ${findAssertionFailure(err)?.summary ?? (err as Error).message}`, snippetContext, ); throw err; diff --git a/server/src/scenario/e2e-export.service.ts b/server/src/scenario/e2e-export.service.ts index f6fa33f..b4cbd7f 100644 --- a/server/src/scenario/e2e-export.service.ts +++ b/server/src/scenario/e2e-export.service.ts @@ -370,7 +370,11 @@ function buildContext(page: Page, browser: BrowserContext, stepOutputs: StepOutp log: (...args: unknown[]) => console.log(...args), warn: (...args: unknown[]) => console.warn(...args), error: (...args: unknown[]) => console.error(...args), - assert: async (fn: () => boolean | Promise, description: string): Promise => { + assert: async ( + fn: () => boolean | Promise, + description: string, + details?: unknown, + ): Promise => { let passed: boolean; let failureReason: string | undefined; try { @@ -386,7 +390,11 @@ function buildContext(page: Page, browser: BrowserContext, stepOutputs: StepOutp const message = failureReason ? \`Assertion failed: \${description} (\${failureReason})\` : \`Assertion failed: \${description}\`; - console.error(message); + if (details !== undefined) { + console.error(message, details); + } else { + console.error(message); + } throw new Error(message); }, getStepOutput: async (order: number): Promise => { diff --git a/server/src/scenario/scenario-scheduler.service.ts b/server/src/scenario/scenario-scheduler.service.ts index b4d5c75..0633158 100644 --- a/server/src/scenario/scenario-scheduler.service.ts +++ b/server/src/scenario/scenario-scheduler.service.ts @@ -13,7 +13,10 @@ import { Repository } from "typeorm"; import { type AppConfig, TraceMode } from "../config/app.config"; import { FileStorageService } from "../file/file-storage.service"; import type { ScriptLogger } from "../code-executor/code-executor.service"; -import { CodeExecutorService } from "../code-executor/code-executor.service"; +import { + CodeExecutorService, + findAssertionFailure, +} from "../code-executor/code-executor.service"; import { ExecContextBuilder } from "../code-executor/exec-context.builder"; import { traceStorage } from "../common/trace-context"; import { TraceLogger } from "../common/trace-logger"; @@ -446,7 +449,11 @@ export class ScenarioSchedulerService implements OnModuleInit { } catch (err) { const msg = (err as Error).message ?? String(err); this.logger.error(`StepRun #${stepRun.id} threw: ${msg}`); - await this.failStepRun(stepRun, msg); + await this.failStepRun( + stepRun, + msg, + findAssertionFailure(err)?.summary, + ); } finally { clearTimeout(timeoutId); } @@ -571,6 +578,7 @@ export class ScenarioSchedulerService implements OnModuleInit { "log", `Step ${stepRun.order} passed`, `step:${stepRun.order}`, + output, ); // Find the next waiting step in this run (next by order) @@ -600,9 +608,14 @@ export class ScenarioSchedulerService implements OnModuleInit { } } + /** + * `logSummary` replaces `description` in the "Step N failed" log entry, e.g. + * for assertion failures whose details are already logged. + */ private async failStepRun( stepRun: ScenarioRunStepEntity, description: string, + logSummary?: string, ): Promise { stepRun.status = "fail"; stepRun.description = description; @@ -612,7 +625,7 @@ export class ScenarioSchedulerService implements OnModuleInit { stepRun.runId, stepRun.id, "error", - `Step ${stepRun.order} failed: ${description}`, + `Step ${stepRun.order} failed: ${logSummary ?? description}`, `step:${stepRun.order}`, ); diff --git a/server/test/scenario-step-output.spec.ts b/server/test/scenario-step-output.spec.ts index 7f2911b..92276c5 100644 --- a/server/test/scenario-step-output.spec.ts +++ b/server/test/scenario-step-output.spec.ts @@ -208,4 +208,83 @@ describe("ScenarioRunStepEntity.output + context.getStepOutput", () => { expect(JSON.parse(rows[2].output as string)).toEqual([1, 2, 3, 4]); }); }); + // ── run log data ─────────────────────────────────────────────────────────── + + describe("run log data", () => { + async function logRows( + runId: string, + context: string, + ): Promise> { + return dataSource.query( + `SELECT message, data FROM scenario_run_logs WHERE runId = '${runId}' AND context = '${context}' ORDER BY seq`, + ); + } + + it("attaches the step output to the 'Step N passed' entry", async () => { + await seedSession(); + const scId = await createScenario("log-step-output"); + await createStep(scId, 0, 'return { token: "abc" };'); + + const runId = await runScenario(scId); + const rows = await logRows(runId, "step:1"); + const passed = rows.find((r) => r.message === "Step 1 passed"); + + expect(JSON.parse(passed!.data as string)).toEqual({ token: "abc" }); + }); + + it("stores assertion failure details as log data", async () => { + await seedSession(); + const scId = await createScenario("log-assert-details"); + await createStep( + scId, + 0, + 'await context.assert(() => false, "Validation must pass", { status: 500 });', + ); + + const runId = await runScenario(scId); + const [row] = await logRows(runId, "assertion:fail"); + + expect(row.message).toBe("Assertion failed: Validation must pass"); + expect(JSON.parse(row.data as string)).toEqual({ + details: { status: 500 }, + }); + }); + + it("keeps the 'Step N failed' entry of an assertion failure short", async () => { + await seedSession(); + const scId = await createScenario("log-assert-step-fail"); + await createStep( + scId, + 0, + 'await context.assert(() => { throw new Error("boom"); }, "Must not throw");', + ); + + const runId = await runScenario(scId); + const rows = await logRows(runId, "step:1"); + const failed = rows.find((r) => r.message.startsWith("Step 1 failed:")); + + expect(failed!.message).toBe( + "Step 1 failed: Assertion failed: Must not throw", + ); + const stepRow = await dataSource.query( + `SELECT description FROM scenario_run_steps WHERE runId = '${runId}'`, + ); + expect(stepRow[0].description).toContain("(boom)"); + }); + + it("stores the exception message of a throwing assertion as log data", async () => { + await seedSession(); + const scId = await createScenario("log-assert-throw"); + await createStep( + scId, + 0, + 'await context.assert(() => { throw new Error("boom"); }, "Must not throw");', + ); + + const runId = await runScenario(scId); + const [row] = await logRows(runId, "assertion:fail"); + + expect(JSON.parse(row.data as string)).toEqual({ error: "boom" }); + }); + }); });