Files
liqa/server/test/scenario-step-output.spec.ts
ars9 959b3b70c9
CI / changes (pull_request) Successful in 9s
CI / version (pull_request) Skipped
CI / server (pull_request) Successful in 1m44s
CI / client (pull_request) Successful in 1m36s
CI / image (pull_request) Skipped
feat(scenario): attach step output and assertion details to run log
- "Step N passed" log entry carries the step's return value as data
- context.assert accepts an optional details argument; on failure the
  assertion:fail entry stores { details, error } as data
- assertion failures throw AssertionFailedError so "Step N failed" and
  "Snippet X failed" entries name only the assertion, without repeating
  the details; the step run description keeps the full message
- exported e2e runtime accepts and prints assert details
- log output shows elapsed time in whole seconds, matching the stored
  timestamp resolution
- run artifact and scenario file names link to the inline file content
  in a new tab
2026-10-02 11:05:29 +03:00

291 lines
11 KiB
TypeScript

/**
* Integration tests for ScenarioRunStepEntity.output column and the
* context.getStepOutput() API available inside exec step scripts.
*
* Strategy:
* - Seed a session row so the scheduler can create a browser context.
* - Create a scenario + steps with controlled return values.
* - Call ScenarioSchedulerService.pickUpPendingRuns() directly to process
* the run synchronously (no real timers needed).
* - Assert step-run output persisted and getStepOutput() returns it.
*/
import { INestApplication } from "@nestjs/common";
import { DataSource } from "typeorm";
import { buildTestApp } from "./app.harness";
import { ScenarioService } from "../src/scenario/scenario.service";
import { ScenarioSchedulerService } from "../src/scenario/scenario-scheduler.service";
import { EnvironmentEntity } from "../src/environment/environment.entity";
describe("ScenarioRunStepEntity.output + context.getStepOutput", () => {
let app: INestApplication;
let dataSource: DataSource;
let scenarioService: ScenarioService;
let scheduler: ScenarioSchedulerService;
beforeAll(async () => {
app = await buildTestApp();
dataSource = app.get(DataSource);
scenarioService = app.get(ScenarioService);
scheduler = app.get(ScenarioSchedulerService);
});
afterAll(async () => {
await app.close();
});
/** Seed a minimal session so the scheduler can open a browser context. */
async function seedSession(name = "output-test-session"): Promise<void> {
await dataSource.query(
`INSERT OR IGNORE INTO sessions (sessionName, token, cookies, localStorage)
VALUES ('${name}', 'tok', '[]', '{}')`,
);
}
/** Create a scenario and return its id. */
async function createScenario(name = "output-scenario"): Promise<string> {
const sc = await scenarioService.create({ name });
return sc.id;
}
/** Create a step and return its id. */
async function createStep(
scenarioId: string,
_order: number,
execCode: string,
): Promise<string> {
const step = await scenarioService.createStep(scenarioId, {
execCode,
});
return step.id;
}
/** Create a test environment and return its id. */
async function createEnvironment(
name = `output-env-${Math.random().toString(36).slice(2, 8)}`,
): Promise<string> {
const repo = dataSource.getRepository(EnvironmentEntity);
const env = await repo.save(repo.create({ name, data: {} }));
return env.id;
}
/** Trigger a run and process it to completion via the scheduler. */
async function runScenario(scenarioId: string): Promise<string> {
const environmentId = await createEnvironment();
const run = await scenarioService.createRun(scenarioId, environmentId);
// Drive the scheduler directly — keeps tests synchronous and fast.
await scheduler.pickUpPendingRuns();
// Wait for the run to reach a terminal state (max 10 s).
const result = await scenarioService.waitForRun(scenarioId, run.id, 10_000);
return result.id;
}
// ── output column ──────────────────────────────────────────────────────────
describe("output column", () => {
it("stores the return value of an exec script as JSON", async () => {
await seedSession();
const scId = await createScenario("output-basic");
await createStep(scId, 0, "return 42;");
const runId = await runScenario(scId);
const row = await dataSource.query(
`SELECT output, status FROM scenario_run_steps WHERE runId = '${runId}'`,
);
expect(row[0].status).toBe("pass");
expect(JSON.parse(row[0].output as string)).toBe(42);
});
it("stores object return values as JSON", async () => {
await seedSession();
const scId = await createScenario("output-object");
await createStep(scId, 0, 'return { foo: "bar", n: 7 };');
const runId = await runScenario(scId);
const row = await dataSource.query(
`SELECT output FROM scenario_run_steps WHERE runId = '${runId}'`,
);
expect(JSON.parse(row[0].output as string)).toEqual({ foo: "bar", n: 7 });
});
it("stores null when the script returns undefined/nothing", async () => {
await seedSession();
const scId = await createScenario("output-undefined");
await createStep(scId, 0, "const x = 1;");
const runId = await runScenario(scId);
const row = await dataSource.query(
`SELECT output FROM scenario_run_steps WHERE runId = '${runId}'`,
);
expect(row[0].output).toBeNull();
});
it("output is exposed on the stepRuns inside GET /scenarios/:id/run/:runId via findRun", async () => {
await seedSession();
const scId = await createScenario("output-findrun");
await createStep(scId, 0, "return 'hello';");
const runId = await runScenario(scId);
const run = await scenarioService.findRun(scId, runId);
expect(run.stepRuns[0].output).toBe(JSON.stringify("hello"));
});
});
// ── context.getStepOutput ──────────────────────────────────────────────────
describe("context.getStepOutput()", () => {
it("returns output of a previous step by absolute order", async () => {
await seedSession();
const scId = await createScenario("getStepOutput-absolute");
await createStep(scId, 0, "return 99;");
// Step 1 reads step 1's output via absolute index 1 (1-based)
await createStep(scId, 1, "return await context.getStepOutput(1);");
const runId = await runScenario(scId);
const rows = await dataSource.query(
`SELECT "order", output FROM scenario_run_steps WHERE runId = '${runId}' ORDER BY "order"`,
);
expect(JSON.parse(rows[0].output as string)).toBe(99);
expect(JSON.parse(rows[1].output as string)).toBe(99);
});
it("returns output of the previous step using relative index -1", async () => {
await seedSession();
const scId = await createScenario("getStepOutput-relative");
await createStep(scId, 0, 'return "step-zero";');
// Step 1 uses relative index -1 to reference step 0
await createStep(scId, 1, "return await context.getStepOutput(-1);");
const runId = await runScenario(scId);
const rows = await dataSource.query(
`SELECT "order", output FROM scenario_run_steps WHERE runId = '${runId}' ORDER BY "order"`,
);
expect(JSON.parse(rows[1].output as string)).toBe("step-zero");
});
it("returns null for a step that does not exist", async () => {
await seedSession();
const scId = await createScenario("getStepOutput-missing");
// Step 0 tries to read step order 99 which does not exist
await createStep(scId, 0, "return await context.getStepOutput(99);");
const runId = await runScenario(scId);
const row = await dataSource.query(
`SELECT output FROM scenario_run_steps WHERE runId = '${runId}'`,
);
expect(row[0].output).toBeNull();
});
it("chains output across three steps", async () => {
await seedSession();
const scId = await createScenario("getStepOutput-chain");
await createStep(scId, 0, "return [1, 2];");
await createStep(
scId,
1,
"const prev = await context.getStepOutput(-1); return [...prev, 3];",
);
await createStep(
scId,
2,
"const prev = await context.getStepOutput(-1); return [...prev, 4];",
);
const runId = await runScenario(scId);
const rows = await dataSource.query(
`SELECT "order", output FROM scenario_run_steps WHERE runId = '${runId}' ORDER BY "order"`,
);
expect(JSON.parse(rows[0].output as string)).toEqual([1, 2]);
expect(JSON.parse(rows[1].output as string)).toEqual([1, 2, 3]);
expect(JSON.parse(rows[2].output as string)).toEqual([1, 2, 3, 4]);
});
});
// ── run log data ───────────────────────────────────────────────────────────
describe("run log data", () => {
async function logRows(
runId: string,
context: string,
): Promise<Array<{ message: string; data: string | null }>> {
return dataSource.query(
`SELECT message, data FROM scenario_run_logs WHERE runId = '${runId}' AND context = '${context}' ORDER BY seq`,
);
}
it("attaches the step output to the 'Step N passed' entry", async () => {
await seedSession();
const scId = await createScenario("log-step-output");
await createStep(scId, 0, 'return { token: "abc" };');
const runId = await runScenario(scId);
const rows = await logRows(runId, "step:1");
const passed = rows.find((r) => r.message === "Step 1 passed");
expect(JSON.parse(passed!.data as string)).toEqual({ token: "abc" });
});
it("stores assertion failure details as log data", async () => {
await seedSession();
const scId = await createScenario("log-assert-details");
await createStep(
scId,
0,
'await context.assert(() => false, "Validation must pass", { status: 500 });',
);
const runId = await runScenario(scId);
const [row] = await logRows(runId, "assertion:fail");
expect(row.message).toBe("Assertion failed: Validation must pass");
expect(JSON.parse(row.data as string)).toEqual({
details: { status: 500 },
});
});
it("keeps the 'Step N failed' entry of an assertion failure short", async () => {
await seedSession();
const scId = await createScenario("log-assert-step-fail");
await createStep(
scId,
0,
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
);
const runId = await runScenario(scId);
const rows = await logRows(runId, "step:1");
const failed = rows.find((r) => r.message.startsWith("Step 1 failed:"));
expect(failed!.message).toBe(
"Step 1 failed: Assertion failed: Must not throw",
);
const stepRow = await dataSource.query(
`SELECT description FROM scenario_run_steps WHERE runId = '${runId}'`,
);
expect(stepRow[0].description).toContain("(boom)");
});
it("stores the exception message of a throwing assertion as log data", async () => {
await seedSession();
const scId = await createScenario("log-assert-throw");
await createStep(
scId,
0,
'await context.assert(() => { throw new Error("boom"); }, "Must not throw");',
);
const runId = await runScenario(scId);
const [row] = await logRows(runId, "assertion:fail");
expect(JSON.parse(row.data as string)).toEqual({ error: "boom" });
});
});
});