feat(runs): record playwright traces with per-run trace mode, bump to 1.15.0
- record a trace for each scenario run and upload it as trace.zip - add PLAYWRIGHT_TRACE_MODE (on, retain-on-failure, off) as the default - add a trace mode option to the run modal, stored per run and reused by repeat run; expose the default via GET /scenarios/run-defaults - accept traceMode in POST /scenarios/:id/run and run_scenario MCP tool - share one run modal component between scenario list and detail pages - git-ignore server/data
This commit is contained in:
1 parent
7d366e6368
commit
a328ef5ea7
24 files changed
+525
-99
No files matched your search
@@ -12,6 +12,16 @@ const makeContext = () => ({
|
||||
cookies: jest.fn().mockResolvedValue([]),
|
||||
addCookies: jest.fn().mockResolvedValue(undefined),
|
||||
addInitScript: jest.fn().mockResolvedValue(undefined),
|
||||
tracing: {
|
||||
start: jest.fn().mockResolvedValue(undefined),
|
||||
// Mimic Playwright writing the trace archive when a path is given
|
||||
stop: jest.fn().mockImplementation(async (opts?: { path?: string }) => {
|
||||
if (opts?.path) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
await require("fs/promises").writeFile(opts.path, "fake trace zip");
|
||||
}
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
const makeBrowser = () => ({
|
||||
|
||||
@@ -588,11 +588,15 @@ describe("McpController", () => {
|
||||
total: number;
|
||||
};
|
||||
|
||||
expect(list.total).toBe(1);
|
||||
expect(list.items).toHaveLength(1);
|
||||
expect(list.items[0].name).toBe("artifact.bin");
|
||||
expect(list.items[0].mimeType).toBe("application/octet-stream");
|
||||
expect(list.items[0].expiresAt).not.toBeNull();
|
||||
// The script's artifact plus the run's Playwright trace
|
||||
expect(list.total).toBe(2);
|
||||
expect(list.items.map((f) => f.name).sort()).toEqual([
|
||||
"artifact.bin",
|
||||
"trace.zip",
|
||||
]);
|
||||
const artifact = list.items.find((f) => f.name === "artifact.bin")!;
|
||||
expect(artifact.mimeType).toBe("application/octet-stream");
|
||||
expect(artifact.expiresAt).not.toBeNull();
|
||||
});
|
||||
|
||||
it("returns MCP error for non-existent scenario", async () => {
|
||||
@@ -621,7 +625,7 @@ describe("McpController", () => {
|
||||
) as {
|
||||
items: Array<{ id: string; name: string; mimeType: string }>;
|
||||
};
|
||||
const file = list.items[0];
|
||||
const file = list.items.find((f) => f.name === "artifact.bin")!;
|
||||
|
||||
const { status, rpc } = await mcpCall("get_run_file_content", {
|
||||
scenarioId: sc.id,
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* Integration tests for Playwright trace recording on scenario runs.
|
||||
*
|
||||
* The playwright mock writes a fake archive when tracing.stop() is given a
|
||||
* path, so we can assert that a `trace.zip` run artifact is (or is not)
|
||||
* uploaded depending on the run's trace mode and outcome.
|
||||
*/
|
||||
|
||||
import { INestApplication } from "@nestjs/common";
|
||||
import request from "supertest";
|
||||
import { DataSource } from "typeorm";
|
||||
import { buildTestApp } from "./app.harness";
|
||||
import { TraceMode } from "../src/config/app.config";
|
||||
import { EnvironmentEntity } from "../src/environment/environment.entity";
|
||||
import { ScenarioSchedulerService } from "../src/scenario/scenario-scheduler.service";
|
||||
import { ScenarioService } from "../src/scenario/scenario.service";
|
||||
|
||||
describe("ScenarioSchedulerService Playwright traces", () => {
|
||||
let app: INestApplication;
|
||||
let dataSource: DataSource;
|
||||
let scenarioService: ScenarioService;
|
||||
let scheduler: ScenarioSchedulerService;
|
||||
|
||||
beforeAll(async () => {
|
||||
app = await buildTestApp();
|
||||
dataSource = app.get(DataSource);
|
||||
scenarioService = app.get(ScenarioService);
|
||||
scheduler = app.get(ScenarioSchedulerService);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
// ── helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
async function runScenario(
|
||||
name: string,
|
||||
execCode: string,
|
||||
traceMode?: TraceMode,
|
||||
): Promise<string> {
|
||||
const sc = await scenarioService.create({ name });
|
||||
await scenarioService.createStep(sc.id, { execCode });
|
||||
const envRepo = dataSource.getRepository(EnvironmentEntity);
|
||||
const env = await envRepo.save(
|
||||
envRepo.create({ name: `${name}-env`, data: {} }),
|
||||
);
|
||||
const run = await scenarioService.createRun(
|
||||
sc.id,
|
||||
env.id,
|
||||
false,
|
||||
traceMode,
|
||||
);
|
||||
await scheduler.pickUpPendingRuns();
|
||||
await scenarioService.waitForRun(sc.id, run.id, 10_000);
|
||||
// The trace is saved after the run reaches its final status; wait for
|
||||
// the scheduler to finish its cleanup for this run.
|
||||
const activeRuns = (scheduler as unknown as { activeRuns: Set<string> })
|
||||
.activeRuns;
|
||||
const deadline = Date.now() + 5_000;
|
||||
while (activeRuns.has(run.id) && Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
return run.id;
|
||||
}
|
||||
|
||||
async function traceArtifacts(
|
||||
runId: string,
|
||||
): Promise<{ mimeType: string }[]> {
|
||||
return dataSource.query(
|
||||
`SELECT f.mimeType FROM scenario_run_files rf
|
||||
JOIN files f ON f.id = rf.fileId
|
||||
WHERE rf.runId = ? AND f.originalName = 'trace.zip'`,
|
||||
[runId],
|
||||
);
|
||||
}
|
||||
|
||||
// ── tests ──────────────────────────────────────────────────────────────────
|
||||
|
||||
it("uploads trace.zip for a passing run in 'on' mode", async () => {
|
||||
const runId = await runScenario("trace-on-pass", "return 1;", TraceMode.On);
|
||||
const files = await traceArtifacts(runId);
|
||||
expect(files).toHaveLength(1);
|
||||
expect(files[0].mimeType).toBe("application/zip");
|
||||
});
|
||||
|
||||
it("uploads trace.zip for a failing run in 'retain-on-failure' mode", async () => {
|
||||
const runId = await runScenario(
|
||||
"trace-rof-fail",
|
||||
"throw new Error('boom');",
|
||||
TraceMode.RetainOnFailure,
|
||||
);
|
||||
expect(await traceArtifacts(runId)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("discards the trace of a passing run in 'retain-on-failure' mode", async () => {
|
||||
const runId = await runScenario(
|
||||
"trace-rof-pass",
|
||||
"return 1;",
|
||||
TraceMode.RetainOnFailure,
|
||||
);
|
||||
expect(await traceArtifacts(runId)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("does not record a trace in 'off' mode", async () => {
|
||||
const runId = await runScenario(
|
||||
"trace-off-fail",
|
||||
"throw new Error('boom');",
|
||||
TraceMode.Off,
|
||||
);
|
||||
expect(await traceArtifacts(runId)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("stores the configured default trace mode when none is given", async () => {
|
||||
const runId = await runScenario("trace-default", "return 1;");
|
||||
const [row] = await dataSource.query(
|
||||
`SELECT traceMode FROM scenario_runs WHERE id = ?`,
|
||||
[runId],
|
||||
);
|
||||
expect(row.traceMode).toBe("on");
|
||||
expect(await traceArtifacts(runId)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("falls back to the configured mode for runs without a stored trace mode", async () => {
|
||||
const sc = await scenarioService.create({ name: "trace-null-mode" });
|
||||
await scenarioService.createStep(sc.id, { execCode: "return 1;" });
|
||||
const envRepo = dataSource.getRepository(EnvironmentEntity);
|
||||
const env = await envRepo.save(
|
||||
envRepo.create({ name: "trace-null-mode-env", data: {} }),
|
||||
);
|
||||
const run = await scenarioService.createRun(sc.id, env.id);
|
||||
await dataSource.query(
|
||||
`UPDATE scenario_runs SET traceMode = NULL WHERE id = ?`,
|
||||
[run.id],
|
||||
);
|
||||
await scheduler.pickUpPendingRuns();
|
||||
await scenarioService.waitForRun(sc.id, run.id, 10_000);
|
||||
const activeRuns = (scheduler as unknown as { activeRuns: Set<string> })
|
||||
.activeRuns;
|
||||
const deadline = Date.now() + 5_000;
|
||||
while (activeRuns.has(run.id) && Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
expect(await traceArtifacts(run.id)).toHaveLength(1);
|
||||
});
|
||||
|
||||
describe("HTTP", () => {
|
||||
it("GET /scenarios/run-defaults returns the configured trace mode", async () => {
|
||||
const res = await request(app.getHttpServer()).get(
|
||||
"/scenarios/run-defaults",
|
||||
);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ traceMode: "on" });
|
||||
});
|
||||
|
||||
it("POST /scenarios/:id/run stores the requested trace mode", async () => {
|
||||
const sc = await scenarioService.create({ name: "trace-http" });
|
||||
const envRepo = dataSource.getRepository(EnvironmentEntity);
|
||||
const env = await envRepo.save(
|
||||
envRepo.create({ name: "trace-http-env", data: {} }),
|
||||
);
|
||||
const res = await request(app.getHttpServer())
|
||||
.post(`/scenarios/${sc.id}/run`)
|
||||
.send({ environmentId: env.id, traceMode: "retain-on-failure" });
|
||||
expect(res.status).toBe(201);
|
||||
expect(res.body.traceMode).toBe("retain-on-failure");
|
||||
});
|
||||
|
||||
it("POST /scenarios/:id/run rejects an unknown trace mode", async () => {
|
||||
const sc = await scenarioService.create({ name: "trace-http-bad" });
|
||||
const envRepo = dataSource.getRepository(EnvironmentEntity);
|
||||
const env = await envRepo.save(
|
||||
envRepo.create({ name: "trace-http-bad-env", data: {} }),
|
||||
);
|
||||
const res = await request(app.getHttpServer())
|
||||
.post(`/scenarios/${sc.id}/run`)
|
||||
.send({ environmentId: env.id, traceMode: "sometimes" });
|
||||
expect(res.status).toBe(400);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user