feat(runs): record playwright traces with per-run trace mode, bump to 1.15.0

- record a trace for each scenario run and upload it as trace.zip
- add PLAYWRIGHT_TRACE_MODE (on, retain-on-failure, off) as the default
- add a trace mode option to the run modal, stored per run and reused
  by repeat run; expose the default via GET /scenarios/run-defaults
- accept traceMode in POST /scenarios/:id/run and run_scenario MCP tool
- share one run modal component between scenario list and detail pages
- git-ignore server/data
This commit is contained in:
ars9 committed 2026-09-30 14:58:07 +03:00
1 parent 7d366e6368
commit a328ef5ea7
24 files changed
+525 -99

No files matched your search

+17 -1
View File
@@ -1,8 +1,21 @@
import { plainToInstance } from "class-transformer";
import { IsInt, IsString, Max, Min, validateSync } from "class-validator";
import {
IsEnum,
IsInt,
IsString,
Max,
Min,
validateSync,
} from "class-validator";
import pkg from "../../package.json";
export enum TraceMode {
On = "on",
RetainOnFailure = "retain-on-failure",
Off = "off",
}
export class AppConfig {
@IsString()
NODE_ENV: string = "development";
@@ -41,6 +54,9 @@ export class AppConfig {
@IsString()
FILE_CLEANUP_CRON: string = "0 3 * * *";
@IsEnum(TraceMode)
PLAYWRIGHT_TRACE_MODE: TraceMode = TraceMode.On;
}
export function validateAppConfig(config: Record<string, unknown>): AppConfig {
+9 -1
View File
@@ -7,6 +7,7 @@ import { SessionService } from "../session/session.service";
import { SessionContextService } from "../session/session-context.service";
import { EnvironmentService } from "../environment/environment.service";
import type { EnvironmentData } from "../environment/environment.entity";
import { TraceMode } from "../config/app.config";
import { CredentialService } from "../credential/credential.service";
import { BrowserService } from "../browser/browser.service";
import { CodeExecutorService } from "../code-executor/code-executor.service";
@@ -939,14 +940,21 @@ export class McpService {
.boolean()
.optional()
.describe("Save session after run completes"),
traceMode: z
.enum(TraceMode)
.optional()
.describe(
"Playwright trace recording: on (keep every trace), retain-on-failure, or off. Defaults to the server's PLAYWRIGHT_TRACE_MODE",
),
},
},
async ({ id, environmentId, saveSession }) => {
async ({ id, environmentId, saveSession, traceMode }) => {
try {
const run = await this.scenarioService.createRun(
id,
environmentId,
saveSession,
traceMode,
);
return {
content: [{ type: "text" as const, text: JSON.stringify(run) }],
@@ -1,4 +1,11 @@
import { IsBoolean, IsNotEmpty, IsOptional, IsUUID } from "class-validator";
import {
IsBoolean,
IsEnum,
IsNotEmpty,
IsOptional,
IsUUID,
} from "class-validator";
import { TraceMode } from "../../config/app.config";
export class CreateScenarioRunDto {
@IsUUID()
@@ -8,4 +15,8 @@ export class CreateScenarioRunDto {
@IsBoolean()
@IsOptional()
saveSession?: boolean;
@IsEnum(TraceMode)
@IsOptional()
traceMode?: TraceMode;
}
@@ -8,6 +8,7 @@ import {
PrimaryGeneratedColumn,
UpdateDateColumn,
} from "typeorm";
import type { TraceMode } from "../config/app.config";
import { EnvironmentEntity } from "../environment/environment.entity";
import { SessionEntity } from "../session/session.entity";
import { ScenarioRunStepEntity } from "./scenario-run-step.entity";
@@ -47,6 +48,10 @@ export class ScenarioRunEntity {
@Column({ type: "boolean", default: false })
saveSession: boolean;
// Playwright trace mode for this run; null falls back to PLAYWRIGHT_TRACE_MODE
@Column("text", { nullable: true, default: null })
traceMode: TraceMode | null;
@OneToMany(() => ScenarioRunStepEntity, (rs) => rs.run, {
cascade: true,
eager: false,
@@ -1,11 +1,16 @@
import { Injectable, OnModuleInit } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import { Interval } from "@nestjs/schedule";
import { InjectRepository } from "@nestjs/typeorm";
import * as crypto from "crypto";
import * as fs from "fs/promises";
import * as os from "os";
import * as path from "path";
import { Effect } from "effect";
import type { Browser, BrowserContext, Page } from "playwright";
import { chromium } from "playwright";
import { Repository } from "typeorm";
import { type AppConfig, TraceMode } from "../config/app.config";
import { FileStorageService } from "../file/file-storage.service";
import type { ScriptLogger } from "../code-executor/code-executor.service";
import { CodeExecutorService } from "../code-executor/code-executor.service";
@@ -29,6 +34,7 @@ interface BrowserHandle {
browser: Browser;
context: BrowserContext;
page: Page;
tracing: boolean;
}
@Injectable()
@@ -49,6 +55,9 @@ export class ScenarioSchedulerService implements OnModuleInit {
// Monotonic log sequence per run, since persistLog writes are unawaited and
// can otherwise resolve out of call order
private readonly runLogSeq = new Map<string, number>();
// Cache Playwright trace mode per run
private readonly runTraceModes = new Map<string, TraceMode>();
private readonly defaultTraceMode: TraceMode;
private static readonly DEFAULT_SCENARIO_TIMEOUT_SEC = 600;
private static readonly DEFAULT_STEP_TIMEOUT_SEC = 60;
@@ -69,7 +78,10 @@ export class ScenarioSchedulerService implements OnModuleInit {
private readonly sessionService: SessionService,
private readonly sessionContextService: SessionContextService,
private readonly fileStorageService: FileStorageService,
) {}
configService: ConfigService<AppConfig, true>,
) {
this.defaultTraceMode = configService.get("PLAYWRIGHT_TRACE_MODE");
}
async onModuleInit(): Promise<void> {
const staleRuns = await this.runRepo.find({
@@ -208,6 +220,10 @@ export class ScenarioSchedulerService implements OnModuleInit {
.catch(() => null);
this.runScenarioTimeouts.set(run.id, scenario?.timeoutSeconds ?? null);
this.runScenarioIds.set(run.id, run.scenarioId);
this.runTraceModes.set(
run.id,
run.traceMode ?? this.defaultTraceMode,
);
const traceId = crypto.randomUUID();
void traceStorage.run({ traceId }, () =>
this.processRunToCompletion(run.id),
@@ -221,6 +237,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
this.runEnvironments.delete(run.id);
this.runScenarioTimeouts.delete(run.id);
this.runScenarioIds.delete(run.id);
this.runTraceModes.delete(run.id);
this.runLogSeq.delete(run.id);
this.logger.error(
`Run #${run.id}: setup failed — ${String(err)}`,
@@ -279,12 +296,15 @@ export class ScenarioSchedulerService implements OnModuleInit {
);
await this.failRun(runId, `Unexpected error: ${(err as Error).message}`);
} finally {
// Before runLogSeq is cleared, so the trace log entry keeps its order
await this.saveTrace(runId);
this.activeRuns.delete(runId);
this.runCredentials.delete(runId);
this.runSnippets.delete(runId);
this.runEnvironments.delete(runId);
this.runScenarioTimeouts.delete(runId);
this.runScenarioIds.delete(runId);
this.runTraceModes.delete(runId);
this.runLogSeq.delete(runId);
await this.maybePreserveSession(runId);
}
@@ -431,9 +451,21 @@ export class ScenarioSchedulerService implements OnModuleInit {
executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH,
});
const context = await browser.newContext();
let tracing = false;
const traceMode = this.runTraceModes.get(runId) ?? this.defaultTraceMode;
if (traceMode !== TraceMode.Off) {
try {
await context.tracing.start({ screenshots: true, snapshots: true });
tracing = true;
} catch (err) {
this.logger.warn(
`Run #${runId}: failed to start tracing — ${(err as Error).message}`,
);
}
}
const page = await context.newPage();
const handle: BrowserHandle = { browser, context, page };
const handle: BrowserHandle = { browser, context, page, tracing };
this.runBrowsers.set(runId, handle);
this.logger.log(`Run #${runId}: browser created`);
return handle;
@@ -453,6 +485,53 @@ export class ScenarioSchedulerService implements OnModuleInit {
}
}
/**
* Stops the run's Playwright trace and uploads it as a `trace.zip` run
* artifact. In retain-on-failure mode, traces of runs that did not fail are
* discarded.
*/
private async saveTrace(runId: string): Promise<void> {
const handle = this.runBrowsers.get(runId);
if (!handle?.tracing) return;
handle.tracing = false;
const run = await this.runRepo.findOneBy({ id: runId }).catch(() => null);
const traceMode = this.runTraceModes.get(runId) ?? this.defaultTraceMode;
const keep = traceMode === TraceMode.On || run?.status === "fail";
let tmpDir: string | null = null;
try {
if (!keep) {
await handle.context.tracing.stop();
return;
}
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "liqa-trace-"));
const tracePath = path.join(tmpDir, "trace.zip");
await handle.context.tracing.stop({ path: tracePath });
const trace = await fs.readFile(tracePath);
await this.fileStorageService.createAndSaveRunArtifact(
runId,
trace,
"trace.zip",
"application/zip",
);
this.persistLog(
runId,
null,
"log",
"Playwright trace uploaded to run files (open with `npx playwright show-trace trace.zip` or https://trace.playwright.dev)",
);
} catch (err) {
this.logger.warn(
`Run #${runId}: failed to save trace — ${(err as Error).message}`,
);
} finally {
if (tmpDir) {
await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
}
}
}
// ── Pass / fail helpers ────────────────────────────────────────────────────
private async passStepRun(
@@ -70,6 +70,13 @@ export class ScenarioController {
return this.scenarioService.findAllRuns(query);
}
@Get("run-defaults")
@ApiOperation({ summary: "Get server-configured defaults for new runs" })
@ApiResponse({ status: 200 })
getRunDefaults() {
return this.scenarioService.getRunDefaults();
}
@Get(":id")
@ApiOperation({ summary: "Get a scenario with its steps" })
@ApiResponse({ status: 200 })
@@ -276,6 +283,7 @@ export class ScenarioController {
id,
dto.environmentId,
dto.saveSession,
dto.traceMode,
);
}
+9
View File
@@ -4,6 +4,7 @@ import {
Injectable,
NotFoundException,
} from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import { InjectRepository } from "@nestjs/typeorm";
import * as fs from "fs/promises";
import * as path from "path";
@@ -13,6 +14,7 @@ import {
PaginationQueryDto,
} from "../common/dto/pagination.dto";
import { resolveUsedSnippetAliases } from "../common/snippet-usage";
import type { AppConfig, TraceMode } from "../config/app.config";
import { CredentialEntity } from "../credential/credential.entity";
import { EnvironmentEntity } from "../environment/environment.entity";
import { FileStorageService } from "../file/file-storage.service";
@@ -69,6 +71,7 @@ export class ScenarioService {
private readonly scenarioRunFileRepo: Repository<ScenarioRunFileEntity>,
private readonly fileStorageService: FileStorageService,
private readonly snippetService: SnippetService,
private readonly configService: ConfigService<AppConfig, true>,
) {}
// ── Scenarios ─────────────────────────────────────────────────────────────
@@ -488,10 +491,15 @@ export class ScenarioService {
return this.findRun(scenarioId, runId);
}
getRunDefaults(): { traceMode: TraceMode } {
return { traceMode: this.configService.get("PLAYWRIGHT_TRACE_MODE") };
}
async createRun(
scenarioId: string,
environmentId: string,
saveSession = false,
traceMode?: TraceMode,
): Promise<ScenarioRunEntity> {
const scenario = await this.findOne(scenarioId);
const environment = await this.environmentRepo.findOneBy({
@@ -507,6 +515,7 @@ export class ScenarioService {
environmentId,
status: "pending",
saveSession,
traceMode: traceMode ?? this.getRunDefaults().traceMode,
}),
);
+10
View File
@@ -12,6 +12,16 @@ const makeContext = () => ({
cookies: jest.fn().mockResolvedValue([]),
addCookies: jest.fn().mockResolvedValue(undefined),
addInitScript: jest.fn().mockResolvedValue(undefined),
tracing: {
start: jest.fn().mockResolvedValue(undefined),
// Mimic Playwright writing the trace archive when a path is given
stop: jest.fn().mockImplementation(async (opts?: { path?: string }) => {
if (opts?.path) {
// eslint-disable-next-line @typescript-eslint/no-require-imports
await require("fs/promises").writeFile(opts.path, "fake trace zip");
}
}),
},
});
const makeBrowser = () => ({
+10 -6
View File
@@ -588,11 +588,15 @@ describe("McpController", () => {
total: number;
};
expect(list.total).toBe(1);
expect(list.items).toHaveLength(1);
expect(list.items[0].name).toBe("artifact.bin");
expect(list.items[0].mimeType).toBe("application/octet-stream");
expect(list.items[0].expiresAt).not.toBeNull();
// The script's artifact plus the run's Playwright trace
expect(list.total).toBe(2);
expect(list.items.map((f) => f.name).sort()).toEqual([
"artifact.bin",
"trace.zip",
]);
const artifact = list.items.find((f) => f.name === "artifact.bin")!;
expect(artifact.mimeType).toBe("application/octet-stream");
expect(artifact.expiresAt).not.toBeNull();
});
it("returns MCP error for non-existent scenario", async () => {
@@ -621,7 +625,7 @@ describe("McpController", () => {
) as {
items: Array<{ id: string; name: string; mimeType: string }>;
};
const file = list.items[0];
const file = list.items.find((f) => f.name === "artifact.bin")!;
const { status, rpc } = await mcpCall("get_run_file_content", {
scenarioId: sc.id,
+181
View File
@@ -0,0 +1,181 @@
/**
* Integration tests for Playwright trace recording on scenario runs.
*
* The playwright mock writes a fake archive when tracing.stop() is given a
* path, so we can assert that a `trace.zip` run artifact is (or is not)
* uploaded depending on the run's trace mode and outcome.
*/
import { INestApplication } from "@nestjs/common";
import request from "supertest";
import { DataSource } from "typeorm";
import { buildTestApp } from "./app.harness";
import { TraceMode } from "../src/config/app.config";
import { EnvironmentEntity } from "../src/environment/environment.entity";
import { ScenarioSchedulerService } from "../src/scenario/scenario-scheduler.service";
import { ScenarioService } from "../src/scenario/scenario.service";
describe("ScenarioSchedulerService Playwright traces", () => {
let app: INestApplication;
let dataSource: DataSource;
let scenarioService: ScenarioService;
let scheduler: ScenarioSchedulerService;
beforeAll(async () => {
app = await buildTestApp();
dataSource = app.get(DataSource);
scenarioService = app.get(ScenarioService);
scheduler = app.get(ScenarioSchedulerService);
});
afterAll(async () => {
await app.close();
});
// ── helpers ────────────────────────────────────────────────────────────────
async function runScenario(
name: string,
execCode: string,
traceMode?: TraceMode,
): Promise<string> {
const sc = await scenarioService.create({ name });
await scenarioService.createStep(sc.id, { execCode });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: `${name}-env`, data: {} }),
);
const run = await scenarioService.createRun(
sc.id,
env.id,
false,
traceMode,
);
await scheduler.pickUpPendingRuns();
await scenarioService.waitForRun(sc.id, run.id, 10_000);
// The trace is saved after the run reaches its final status; wait for
// the scheduler to finish its cleanup for this run.
const activeRuns = (scheduler as unknown as { activeRuns: Set<string> })
.activeRuns;
const deadline = Date.now() + 5_000;
while (activeRuns.has(run.id) && Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 20));
}
return run.id;
}
async function traceArtifacts(
runId: string,
): Promise<{ mimeType: string }[]> {
return dataSource.query(
`SELECT f.mimeType FROM scenario_run_files rf
JOIN files f ON f.id = rf.fileId
WHERE rf.runId = ? AND f.originalName = 'trace.zip'`,
[runId],
);
}
// ── tests ──────────────────────────────────────────────────────────────────
it("uploads trace.zip for a passing run in 'on' mode", async () => {
const runId = await runScenario("trace-on-pass", "return 1;", TraceMode.On);
const files = await traceArtifacts(runId);
expect(files).toHaveLength(1);
expect(files[0].mimeType).toBe("application/zip");
});
it("uploads trace.zip for a failing run in 'retain-on-failure' mode", async () => {
const runId = await runScenario(
"trace-rof-fail",
"throw new Error('boom');",
TraceMode.RetainOnFailure,
);
expect(await traceArtifacts(runId)).toHaveLength(1);
});
it("discards the trace of a passing run in 'retain-on-failure' mode", async () => {
const runId = await runScenario(
"trace-rof-pass",
"return 1;",
TraceMode.RetainOnFailure,
);
expect(await traceArtifacts(runId)).toHaveLength(0);
});
it("does not record a trace in 'off' mode", async () => {
const runId = await runScenario(
"trace-off-fail",
"throw new Error('boom');",
TraceMode.Off,
);
expect(await traceArtifacts(runId)).toHaveLength(0);
});
it("stores the configured default trace mode when none is given", async () => {
const runId = await runScenario("trace-default", "return 1;");
const [row] = await dataSource.query(
`SELECT traceMode FROM scenario_runs WHERE id = ?`,
[runId],
);
expect(row.traceMode).toBe("on");
expect(await traceArtifacts(runId)).toHaveLength(1);
});
it("falls back to the configured mode for runs without a stored trace mode", async () => {
const sc = await scenarioService.create({ name: "trace-null-mode" });
await scenarioService.createStep(sc.id, { execCode: "return 1;" });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: "trace-null-mode-env", data: {} }),
);
const run = await scenarioService.createRun(sc.id, env.id);
await dataSource.query(
`UPDATE scenario_runs SET traceMode = NULL WHERE id = ?`,
[run.id],
);
await scheduler.pickUpPendingRuns();
await scenarioService.waitForRun(sc.id, run.id, 10_000);
const activeRuns = (scheduler as unknown as { activeRuns: Set<string> })
.activeRuns;
const deadline = Date.now() + 5_000;
while (activeRuns.has(run.id) && Date.now() < deadline) {
await new Promise((r) => setTimeout(r, 20));
}
expect(await traceArtifacts(run.id)).toHaveLength(1);
});
describe("HTTP", () => {
it("GET /scenarios/run-defaults returns the configured trace mode", async () => {
const res = await request(app.getHttpServer()).get(
"/scenarios/run-defaults",
);
expect(res.status).toBe(200);
expect(res.body).toEqual({ traceMode: "on" });
});
it("POST /scenarios/:id/run stores the requested trace mode", async () => {
const sc = await scenarioService.create({ name: "trace-http" });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: "trace-http-env", data: {} }),
);
const res = await request(app.getHttpServer())
.post(`/scenarios/${sc.id}/run`)
.send({ environmentId: env.id, traceMode: "retain-on-failure" });
expect(res.status).toBe(201);
expect(res.body.traceMode).toBe("retain-on-failure");
});
it("POST /scenarios/:id/run rejects an unknown trace mode", async () => {
const sc = await scenarioService.create({ name: "trace-http-bad" });
const envRepo = dataSource.getRepository(EnvironmentEntity);
const env = await envRepo.save(
envRepo.create({ name: "trace-http-bad-env", data: {} }),
);
const res = await request(app.getHttpServer())
.post(`/scenarios/${sc.id}/run`)
.send({ environmentId: env.id, traceMode: "sometimes" });
expect(res.status).toBe(400);
});
});
});