- "Step N passed" log entry carries the step's return value as data
- context.assert accepts an optional details argument; on failure the
assertion:fail entry stores { details, error } as data
- assertion failures throw AssertionFailedError so "Step N failed" and
"Snippet X failed" entries name only the assertion, without repeating
the details; the step run description keeps the full message
- exported e2e runtime accepts and prints assert details
- log output shows elapsed time in whole seconds, matching the stored
timestamp resolution
- run artifact and scenario file names link to the inline file content
in a new tab
291 lines
11 KiB
TypeScript
291 lines
11 KiB
TypeScript
/**
|
|
* Integration tests for ScenarioRunStepEntity.output column and the
|
|
* context.getStepOutput() API available inside exec step scripts.
|
|
*
|
|
* Strategy:
|
|
* - Seed a session row so the scheduler can create a browser context.
|
|
* - Create a scenario + steps with controlled return values.
|
|
* - Call ScenarioSchedulerService.pickUpPendingRuns() directly to process
|
|
* the run synchronously (no real timers needed).
|
|
* - Assert step-run output persisted and getStepOutput() returns it.
|
|
*/
|
|
|
|
import { INestApplication } from "@nestjs/common";
|
|
import { DataSource } from "typeorm";
|
|
import { buildTestApp } from "./app.harness";
|
|
import { ScenarioService } from "../src/scenario/scenario.service";
|
|
import { ScenarioSchedulerService } from "../src/scenario/scenario-scheduler.service";
|
|
import { EnvironmentEntity } from "../src/environment/environment.entity";
|
|
|
|
describe("ScenarioRunStepEntity.output + context.getStepOutput", () => {
|
|
let app: INestApplication;
|
|
let dataSource: DataSource;
|
|
let scenarioService: ScenarioService;
|
|
let scheduler: ScenarioSchedulerService;
|
|
|
|
beforeAll(async () => {
|
|
app = await buildTestApp();
|
|
dataSource = app.get(DataSource);
|
|
scenarioService = app.get(ScenarioService);
|
|
scheduler = app.get(ScenarioSchedulerService);
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await app.close();
|
|
});
|
|
|
|
/** Seed a minimal session so the scheduler can open a browser context. */
|
|
async function seedSession(name = "output-test-session"): Promise<void> {
|
|
await dataSource.query(
|
|
`INSERT OR IGNORE INTO sessions (sessionName, token, cookies, localStorage)
|
|
VALUES ('${name}', 'tok', '[]', '{}')`,
|
|
);
|
|
}
|
|
|
|
/** Create a scenario and return its id. */
|
|
async function createScenario(name = "output-scenario"): Promise<string> {
|
|
const sc = await scenarioService.create({ name });
|
|
return sc.id;
|
|
}
|
|
|
|
/** Create a step and return its id. */
|
|
async function createStep(
|
|
scenarioId: string,
|
|
_order: number,
|
|
execCode: string,
|
|
): Promise<string> {
|
|
const step = await scenarioService.createStep(scenarioId, {
|
|
execCode,
|
|
});
|
|
return step.id;
|
|
}
|
|
|
|
/** Create a test environment and return its id. */
|
|
async function createEnvironment(
|
|
name = `output-env-${Math.random().toString(36).slice(2, 8)}`,
|
|
): Promise<string> {
|
|
const repo = dataSource.getRepository(EnvironmentEntity);
|
|
const env = await repo.save(repo.create({ name, data: {} }));
|
|
return env.id;
|
|
}
|
|
|
|
/** Trigger a run and process it to completion via the scheduler. */
|
|
async function runScenario(scenarioId: string): Promise<string> {
|
|
const environmentId = await createEnvironment();
|
|
const run = await scenarioService.createRun(scenarioId, environmentId);
|
|
// Drive the scheduler directly — keeps tests synchronous and fast.
|
|
await scheduler.pickUpPendingRuns();
|
|
// Wait for the run to reach a terminal state (max 10 s).
|
|
const result = await scenarioService.waitForRun(scenarioId, run.id, 10_000);
|
|
return result.id;
|
|
}
|
|
|
|
// ── output column ──────────────────────────────────────────────────────────
|
|
|
|
describe("output column", () => {
|
|
it("stores the return value of an exec script as JSON", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("output-basic");
|
|
await createStep(scId, 0, "return 42;");
|
|
|
|
const runId = await runScenario(scId);
|
|
|
|
const row = await dataSource.query(
|
|
`SELECT output, status FROM scenario_run_steps WHERE runId = '${runId}'`,
|
|
);
|
|
expect(row[0].status).toBe("pass");
|
|
expect(JSON.parse(row[0].output as string)).toBe(42);
|
|
});
|
|
|
|
it("stores object return values as JSON", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("output-object");
|
|
await createStep(scId, 0, 'return { foo: "bar", n: 7 };');
|
|
|
|
const runId = await runScenario(scId);
|
|
|
|
const row = await dataSource.query(
|
|
`SELECT output FROM scenario_run_steps WHERE runId = '${runId}'`,
|
|
);
|
|
expect(JSON.parse(row[0].output as string)).toEqual({ foo: "bar", n: 7 });
|
|
});
|
|
|
|
it("stores null when the script returns undefined/nothing", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("output-undefined");
|
|
await createStep(scId, 0, "const x = 1;");
|
|
|
|
const runId = await runScenario(scId);
|
|
|
|
const row = await dataSource.query(
|
|
`SELECT output FROM scenario_run_steps WHERE runId = '${runId}'`,
|
|
);
|
|
expect(row[0].output).toBeNull();
|
|
});
|
|
|
|
it("output is exposed on the stepRuns inside GET /scenarios/:id/run/:runId via findRun", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("output-findrun");
|
|
await createStep(scId, 0, "return 'hello';");
|
|
|
|
const runId = await runScenario(scId);
|
|
const run = await scenarioService.findRun(scId, runId);
|
|
|
|
expect(run.stepRuns[0].output).toBe(JSON.stringify("hello"));
|
|
});
|
|
});
|
|
|
|
// ── context.getStepOutput ──────────────────────────────────────────────────
|
|
|
|
describe("context.getStepOutput()", () => {
|
|
it("returns output of a previous step by absolute order", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("getStepOutput-absolute");
|
|
await createStep(scId, 0, "return 99;");
|
|
// Step 1 reads step 1's output via absolute index 1 (1-based)
|
|
await createStep(scId, 1, "return await context.getStepOutput(1);");
|
|
|
|
const runId = await runScenario(scId);
|
|
const rows = await dataSource.query(
|
|
`SELECT "order", output FROM scenario_run_steps WHERE runId = '${runId}' ORDER BY "order"`,
|
|
);
|
|
|
|
expect(JSON.parse(rows[0].output as string)).toBe(99);
|
|
expect(JSON.parse(rows[1].output as string)).toBe(99);
|
|
});
|
|
|
|
it("returns output of the previous step using relative index -1", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("getStepOutput-relative");
|
|
await createStep(scId, 0, 'return "step-zero";');
|
|
// Step 1 uses relative index -1 to reference step 0
|
|
await createStep(scId, 1, "return await context.getStepOutput(-1);");
|
|
|
|
const runId = await runScenario(scId);
|
|
const rows = await dataSource.query(
|
|
`SELECT "order", output FROM scenario_run_steps WHERE runId = '${runId}' ORDER BY "order"`,
|
|
);
|
|
|
|
expect(JSON.parse(rows[1].output as string)).toBe("step-zero");
|
|
});
|
|
|
|
it("returns null for a step that does not exist", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("getStepOutput-missing");
|
|
// Step 0 tries to read step order 99 which does not exist
|
|
await createStep(scId, 0, "return await context.getStepOutput(99);");
|
|
|
|
const runId = await runScenario(scId);
|
|
const row = await dataSource.query(
|
|
`SELECT output FROM scenario_run_steps WHERE runId = '${runId}'`,
|
|
);
|
|
|
|
expect(row[0].output).toBeNull();
|
|
});
|
|
|
|
it("chains output across three steps", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("getStepOutput-chain");
|
|
await createStep(scId, 0, "return [1, 2];");
|
|
await createStep(
|
|
scId,
|
|
1,
|
|
"const prev = await context.getStepOutput(-1); return [...prev, 3];",
|
|
);
|
|
await createStep(
|
|
scId,
|
|
2,
|
|
"const prev = await context.getStepOutput(-1); return [...prev, 4];",
|
|
);
|
|
|
|
const runId = await runScenario(scId);
|
|
const rows = await dataSource.query(
|
|
`SELECT "order", output FROM scenario_run_steps WHERE runId = '${runId}' ORDER BY "order"`,
|
|
);
|
|
|
|
expect(JSON.parse(rows[0].output as string)).toEqual([1, 2]);
|
|
expect(JSON.parse(rows[1].output as string)).toEqual([1, 2, 3]);
|
|
expect(JSON.parse(rows[2].output as string)).toEqual([1, 2, 3, 4]);
|
|
});
|
|
});
|
|
// ── run log data ───────────────────────────────────────────────────────────
|
|
|
|
describe("run log data", () => {
|
|
async function logRows(
|
|
runId: string,
|
|
context: string,
|
|
): Promise<Array<{ message: string; data: string | null }>> {
|
|
return dataSource.query(
|
|
`SELECT message, data FROM scenario_run_logs WHERE runId = '${runId}' AND context = '${context}' ORDER BY seq`,
|
|
);
|
|
}
|
|
|
|
it("attaches the step output to the 'Step N passed' entry", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("log-step-output");
|
|
await createStep(scId, 0, 'return { token: "abc" };');
|
|
|
|
const runId = await runScenario(scId);
|
|
const rows = await logRows(runId, "step:1");
|
|
const passed = rows.find((r) => r.message === "Step 1 passed");
|
|
|
|
expect(JSON.parse(passed!.data as string)).toEqual({ token: "abc" });
|
|
});
|
|
|
|
it("stores assertion failure details as log data", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("log-assert-details");
|
|
await createStep(
|
|
scId,
|
|
0,
|
|
'await context.assert(() => false, "Validation must pass", { status: 500 });',
|
|
);
|
|
|
|
const runId = await runScenario(scId);
|
|
const [row] = await logRows(runId, "assertion:fail");
|
|
|
|
expect(row.message).toBe("Assertion failed: Validation must pass");
|
|
expect(JSON.parse(row.data as string)).toEqual({
|
|
details: { status: 500 },
|
|
});
|
|
});
|
|
|
|
it("keeps the 'Step N failed' entry of an assertion failure short", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("log-assert-step-fail");
|
|
await createStep(
|
|
scId,
|
|
0,
|
|
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
|
|
);
|
|
|
|
const runId = await runScenario(scId);
|
|
const rows = await logRows(runId, "step:1");
|
|
const failed = rows.find((r) => r.message.startsWith("Step 1 failed:"));
|
|
|
|
expect(failed!.message).toBe(
|
|
"Step 1 failed: Assertion failed: Must not throw",
|
|
);
|
|
const stepRow = await dataSource.query(
|
|
`SELECT description FROM scenario_run_steps WHERE runId = '${runId}'`,
|
|
);
|
|
expect(stepRow[0].description).toContain("(boom)");
|
|
});
|
|
|
|
it("stores the exception message of a throwing assertion as log data", async () => {
|
|
await seedSession();
|
|
const scId = await createScenario("log-assert-throw");
|
|
await createStep(
|
|
scId,
|
|
0,
|
|
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
|
|
);
|
|
|
|
const runId = await runScenario(scId);
|
|
const [row] = await logRows(runId, "assertion:fail");
|
|
|
|
expect(JSON.parse(row.data as string)).toEqual({ error: "boom" });
|
|
});
|
|
});
|
|
});
|