feat(runs): record playwright traces with per-run trace mode, bump to 1.15.0

- record a trace for each scenario run and upload it as trace.zip
- add PLAYWRIGHT_TRACE_MODE (on, retain-on-failure, off) as the default
- add a trace mode option to the run modal, stored per run and reused
  by repeat run; expose the default via GET /scenarios/run-defaults
- accept traceMode in POST /scenarios/:id/run and run_scenario MCP tool
- share one run modal component between scenario list and detail pages
- git-ignore server/data
This commit is contained in:
ars9 committed 2026-09-30 14:58:07 +03:00
1 parent 7d366e6368
commit a328ef5ea7
24 files changed
+525 -99

No files matched your search

+10
View File
@@ -12,6 +12,16 @@ const makeContext = () => ({
cookies: jest.fn().mockResolvedValue([]),
addCookies: jest.fn().mockResolvedValue(undefined),
addInitScript: jest.fn().mockResolvedValue(undefined),
tracing: {
start: jest.fn().mockResolvedValue(undefined),
// Mimic Playwright writing the trace archive when a path is given
stop: jest.fn().mockImplementation(async (opts?: { path?: string }) => {
if (opts?.path) {
// eslint-disable-next-line @typescript-eslint/no-require-imports
await require("fs/promises").writeFile(opts.path, "fake trace zip");
}
}),
},
});
const makeBrowser = () => ({
+10 -6
View File
@@ -588,11 +588,15 @@ describe("McpController", () => {
total: number;
};
expect(list.total).toBe(1);
expect(list.items).toHaveLength(1);
expect(list.items[0].name).toBe("artifact.bin");
expect(list.items[0].mimeType).toBe("application/octet-stream");
expect(list.items[0].expiresAt).not.toBeNull();
// The script's artifact plus the run's Playwright trace
expect(list.total).toBe(2);
expect(list.items.map((f) => f.name).sort()).toEqual([
"artifact.bin",
"trace.zip",
]);
const artifact = list.items.find((f) => f.name === "artifact.bin")!;
expect(artifact.mimeType).toBe("application/octet-stream");
expect(artifact.expiresAt).not.toBeNull();
});
it("returns MCP error for non-existent scenario", async () => {
@@ -621,7 +625,7 @@ describe("McpController", () => {
) as {
items: Array<{ id: string; name: string; mimeType: string }>;
};
const file = list.items[0];
const file = list.items.find((f) => f.name === "artifact.bin")!;
const { status, rpc } = await mcpCall("get_run_file_content", {
scenarioId: sc.id,
+181
View File
@@ -0,0 +1,181 @@
/**
* Integration tests for Playwright trace recording on scenario runs.
*
* The playwright mock writes a fake archive when tracing.stop() is given a
* path, so we can assert that a `trace.zip` run artifact is (or is not)
* uploaded depending on the run's trace mode and outcome.
*/
import { INestApplication } from "@nestjs/common";
import request from "supertest";
import { DataSource } from "typeorm";
import { buildTestApp } from "./app.harness";
import { TraceMode } from "../src/config/app.config";
import { EnvironmentEntity } from "../src/environment/environment.entity";
import { ScenarioSchedulerService } from "../src/scenario/scenario-scheduler.service";
import { ScenarioService } from "../src/scenario/scenario.service";
describe("ScenarioSchedulerService Playwright traces", () => {
let app: INestApplication;
let dataSource: DataSource;
let scenarioService: ScenarioService;
let scheduler: ScenarioSchedulerService;
beforeAll(async () => {
app = await buildTestApp();
dataSource = app.get(DataSource);
scenarioService = app.get(ScenarioService);
scheduler = app.get(ScenarioSchedulerService);
});
afterAll(async () => {
await app.close();
});
// ── helpers ────────────────────────────────────────────────────────────────
async function runScenario(
name: string,
execCode: string,
traceMode?: TraceMode,
): Promise<string> {
const sc = await scenarioService.create({ name });
await scenarioService.createStep(sc.id, { execCode });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: `${name}-env`, data: {} }),
);
const run = await scenarioService.createRun(
sc.id,
env.id,
false,
traceMode,
);
await scheduler.pickUpPendingRuns();
await scenarioService.waitForRun(sc.id, run.id, 10_000);
// The trace is saved after the run reaches its final status; wait for
// the scheduler to finish its cleanup for this run.
const activeRuns = (scheduler as unknown as { activeRuns: Set<string> })
.activeRuns;
const deadline = Date.now() + 5_000;
while (activeRuns.has(run.id) && Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 20));
}
return run.id;
}
async function traceArtifacts(
runId: string,
): Promise<{ mimeType: string }[]> {
return dataSource.query(
`SELECT f.mimeType FROM scenario_run_files rf
JOIN files f ON f.id = rf.fileId
WHERE rf.runId = ? AND f.originalName = 'trace.zip'`,
[runId],
);
}
// ── tests ──────────────────────────────────────────────────────────────────
it("uploads trace.zip for a passing run in 'on' mode", async () => {
const runId = await runScenario("trace-on-pass", "return 1;", TraceMode.On);
const files = await traceArtifacts(runId);
expect(files).toHaveLength(1);
expect(files[0].mimeType).toBe("application/zip");
});
it("uploads trace.zip for a failing run in 'retain-on-failure' mode", async () => {
const runId = await runScenario(
"trace-rof-fail",
"throw new Error('boom');",
TraceMode.RetainOnFailure,
);
expect(await traceArtifacts(runId)).toHaveLength(1);
});
it("discards the trace of a passing run in 'retain-on-failure' mode", async () => {
const runId = await runScenario(
"trace-rof-pass",
"return 1;",
TraceMode.RetainOnFailure,
);
expect(await traceArtifacts(runId)).toHaveLength(0);
});
it("does not record a trace in 'off' mode", async () => {
const runId = await runScenario(
"trace-off-fail",
"throw new Error('boom');",
TraceMode.Off,
);
expect(await traceArtifacts(runId)).toHaveLength(0);
});
it("stores the configured default trace mode when none is given", async () => {
const runId = await runScenario("trace-default", "return 1;");
const [row] = await dataSource.query(
`SELECT traceMode FROM scenario_runs WHERE id = ?`,
[runId],
);
expect(row.traceMode).toBe("on");
expect(await traceArtifacts(runId)).toHaveLength(1);
});
it("falls back to the configured mode for runs without a stored trace mode", async () => {
const sc = await scenarioService.create({ name: "trace-null-mode" });
await scenarioService.createStep(sc.id, { execCode: "return 1;" });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: "trace-null-mode-env", data: {} }),
);
const run = await scenarioService.createRun(sc.id, env.id);
await dataSource.query(
`UPDATE scenario_runs SET traceMode = NULL WHERE id = ?`,
[run.id],
);
await scheduler.pickUpPendingRuns();
await scenarioService.waitForRun(sc.id, run.id, 10_000);
const activeRuns = (scheduler as unknown as { activeRuns: Set<string> })
.activeRuns;
const deadline = Date.now() + 5_000;
while (activeRuns.has(run.id) && Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 20));
}
expect(await traceArtifacts(run.id)).toHaveLength(1);
});
describe("HTTP", () => {
it("GET /scenarios/run-defaults returns the configured trace mode", async () => {
const res = await request(app.getHttpServer()).get(
"/scenarios/run-defaults",
);
expect(res.status).toBe(200);
expect(res.body).toEqual({ traceMode: "on" });
});
it("POST /scenarios/:id/run stores the requested trace mode", async () => {
const sc = await scenarioService.create({ name: "trace-http" });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: "trace-http-env", data: {} }),
);
const res = await request(app.getHttpServer())
.post(`/scenarios/${sc.id}/run`)
.send({ environmentId: env.id, traceMode: "retain-on-failure" });
expect(res.status).toBe(201);
expect(res.body.traceMode).toBe("retain-on-failure");
});
it("POST /scenarios/:id/run rejects an unknown trace mode", async () => {
const sc = await scenarioService.create({ name: "trace-http-bad" });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: "trace-http-bad-env", data: {} }),
);
const res = await request(app.getHttpServer())
.post(`/scenarios/${sc.id}/run`)
.send({ environmentId: env.id, traceMode: "sometimes" });
expect(res.status).toBe(400);
});
});
});