From a328ef5ea7d08ab5484062f9e4e6290df4d22b0e Mon Sep 17 00:00:00 2001 From: Andrii Arsenin Date: Wed, 30 Sep 2026 14:58:07 +0300 Subject: [PATCH] feat(runs): record playwright traces with per-run trace mode, bump to 1.15.0 - record a trace for each scenario run and upload it as trace.zip - add PLAYWRIGHT_TRACE_MODE (on, retain-on-failure, off) as the default - add a trace mode option to the run modal, stored per run and reused by repeat run; expose the default via GET /scenarios/run-defaults - accept traceMode in POST /scenarios/:id/run and run_scenario MCP tool - share one run modal component between scenario list and detail pages - git-ignore server/data --- .env.example | 5 +- .gitignore | 1 + CHANGELOG.md | 16 ++ client/src/api/client.ts | 14 +- client/src/api/types.ts | 7 + .../components/modals/RunScenarioModal.tsx | 23 ++- client/src/hooks/useRunDefaults.ts | 26 +++ client/src/i18n/locales/en.json | 6 + client/src/pages/run/RunDetailPage.tsx | 15 +- .../src/pages/scenario/ScenarioDetailPage.tsx | 73 ++++--- client/src/pages/scenario/ScenariosPage.tsx | 67 +++---- docs/scenario.md | 12 ++ package-lock.json | 4 +- package.json | 2 +- server/src/config/app.config.ts | 18 +- server/src/mcp/mcp.service.ts | 10 +- .../scenario/dto/create-scenario-run.dto.ts | 13 +- server/src/scenario/scenario-run.entity.ts | 5 + .../scenario/scenario-scheduler.service.ts | 83 +++++++- server/src/scenario/scenario.controller.ts | 8 + server/src/scenario/scenario.service.ts | 9 + server/test/__mocks__/playwright.ts | 10 + server/test/mcp.controller.spec.ts | 16 +- server/test/scenario-trace.spec.ts | 181 ++++++++++++++++++ 24 files changed, 525 insertions(+), 99 deletions(-) create mode 100644 client/src/hooks/useRunDefaults.ts create mode 100644 server/test/scenario-trace.spec.ts diff --git a/.env.example b/.env.example index ee1b48c..ca1cf1b 100644 --- a/.env.example +++ b/.env.example @@ -14,4 +14,7 @@ DB_PATH=data/sessions.db # Close open sessions that have been idle for longer than this many minutes SESSION_IDLE_TIMEOUT_MINUTES=30 # Delete closed sessions whose updatedAt is older than this many days -SESSION_DELETE_CLOSED_DAYS=7 \ No newline at end of file +SESSION_DELETE_CLOSED_DAYS=7 + +# Playwright trace recording for scenario runs: on | retain-on-failure | off +PLAYWRIGHT_TRACE_MODE=on diff --git a/.gitignore b/.gitignore index d6e86c8..3c83f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # runtime /data +server/data/ .env *.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d59904..089da7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file. +## 1.15.0 - 2026-09-30 + +Changes since 1.14.0: + +### Added + +- Scenario runs now record a Playwright trace (screenshots and DOM snapshots) and upload it to the run's files as `trace.zip` when the run finishes. Open it with `npx playwright show-trace trace.zip` or at https://trace.playwright.dev. +- Added the `PLAYWRIGHT_TRACE_MODE` setting (`on` (default), `retain-on-failure`, or `off`) that controls which traces are kept. +- Added a "Playwright trace" option to the run modal to override the trace mode for a single run. It defaults to the server's `PLAYWRIGHT_TRACE_MODE`, served by the new `GET /scenarios/run-defaults` endpoint. +- `POST /scenarios/:id/run` and the `run_scenario` MCP tool accept an optional `traceMode`. The trace mode is stored on the run, shown on the run detail page, and reused by "Repeat run". + +### Changed + +- The run modals on the scenario list and scenario detail pages now share one component. +- `server/data/` is now git-ignored. + ## 1.14.0 - 2026-09-30 Changes since 1.13.0: diff --git a/client/src/api/client.ts b/client/src/api/client.ts index 945247a..72266fd 100644 --- a/client/src/api/client.ts +++ b/client/src/api/client.ts @@ -4,6 +4,7 @@ import type { Environment, FileMetadata, FileListResponse, + RunDefaults, ScenarioCredential, Session, Scenario, @@ -12,6 +13,7 @@ import type { ScenarioRunStep, ScenarioStep, Snippet, + TraceMode, } from './types'; import { emitApiErrorToast } from '../lib/toast-events'; @@ -222,12 +224,20 @@ export const scenarios = { remove(id: string): Promise { return request(`/scenarios/${id}`, { method: 'DELETE' }); }, - run(id: string, environmentId: string, saveSession?: boolean): Promise { + run( + id: string, + environmentId: string, + saveSession?: boolean, + traceMode?: TraceMode, + ): Promise { return request(`/scenarios/${id}/run`, { method: 'POST', - body: JSON.stringify({ environmentId, saveSession }), + body: JSON.stringify({ environmentId, saveSession, traceMode }), }); }, + runDefaults(): Promise { + return request('/scenarios/run-defaults'); + }, getRun(scenarioId: string, runId: string): Promise { return request(`/scenarios/${scenarioId}/run/${runId}`); }, diff --git a/client/src/api/types.ts b/client/src/api/types.ts index f240d82..2ee0241 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -98,6 +98,12 @@ export interface Scenario { export type ScenarioRunStatus = 'pending' | 'in_progress' | 'pass' | 'fail'; +export type TraceMode = 'on' | 'retain-on-failure' | 'off'; + +export interface RunDefaults { + traceMode: TraceMode; +} + export type RunStepStatus = 'waiting' | 'pending' | 'in_progress' | 'pass' | 'fail' | 'cancelled'; export type LogLevel = 'log' | 'warn' | 'error'; @@ -108,6 +114,7 @@ export interface ScenarioRun { environmentId: string | null; sessionId?: string | null; saveSession: boolean; + traceMode: TraceMode | null; scenario?: Pick; environment?: Pick; session?: { id: string; sessionName: string }; diff --git a/client/src/components/modals/RunScenarioModal.tsx b/client/src/components/modals/RunScenarioModal.tsx index 5493db8..9c181f6 100644 --- a/client/src/components/modals/RunScenarioModal.tsx +++ b/client/src/components/modals/RunScenarioModal.tsx @@ -1,7 +1,9 @@ import { useTranslation } from 'react-i18next'; import { X, Play } from 'lucide-react'; import { Modal, Button, Select } from '../../ui'; -import type { Environment } from '../../api'; +import type { Environment, TraceMode } from '../../api'; + +const TRACE_MODES: TraceMode[] = ['on', 'retain-on-failure', 'off']; export interface RunScenarioModalProps { open: boolean; @@ -10,6 +12,9 @@ export interface RunScenarioModalProps { onEnvSelect: (envId: string) => void; saveSessionFlag: boolean; onSaveSessionChange: (flag: boolean) => void; + traceMode: TraceMode; + defaultTraceMode: TraceMode | null; + onTraceModeChange: (mode: TraceMode) => void; onClose: () => void; onRun: () => void; isRunning: boolean; @@ -23,6 +28,9 @@ export function RunScenarioModal({ onEnvSelect, saveSessionFlag, onSaveSessionChange, + traceMode, + defaultTraceMode, + onTraceModeChange, onClose, onRun, isRunning, @@ -73,6 +81,19 @@ export function RunScenarioModal({ /> {t('common.save_session')} +
+ setSelectedEnvId(e.target.value)} - options={envs.map((env) => ({ value: env.id, label: env.name }))} - /> - - - )} - + environments={envs} + selectedEnvId={selectedEnvId} + onEnvSelect={setSelectedEnvId} + saveSessionFlag={saveSessionFlag} + onSaveSessionChange={setSaveSessionFlag} + traceMode={traceMode} + defaultTraceMode={runDefaults?.traceMode ?? null} + onTraceModeChange={setTraceModeOverride} + onClose={() => setRunModalOpen(false)} + onRun={handleRun} + isRunning={running} + /> (null); const [running, setRunning] = useState(false); const [saveSessionFlag, setSaveSessionFlag] = useState(false); + // null = use the server's configured default + const [traceModeOverride, setTraceModeOverride] = useState(null); + const runDefaults = useRunDefaults(); + const traceMode = traceModeOverride ?? runDefaults?.traceMode ?? 'on'; const [envs, setEnvs] = useState([]); const [selectedEnvId, setSelectedEnvId] = useState(''); @@ -104,6 +109,7 @@ export function ScenariosPage() { const openRunModal = (id: string) => { setRunScenarioId(id); + setTraceModeOverride(null); const scenario = items.find((s) => s.id === id); const preferred = scenario?.environmentId ?? null; const exists = preferred ? envs.some((e) => e.id === preferred) : false; @@ -114,7 +120,12 @@ export function ScenariosPage() { if (!runScenarioId || !selectedEnvId) return; setRunning(true); try { - const run = await scenarios.run(runScenarioId, selectedEnvId, saveSessionFlag); + const run = await scenarios.run( + runScenarioId, + selectedEnvId, + saveSessionFlag, + traceModeOverride ?? undefined, + ); toast.success(t('scenarios.run_started')); navigate(`/scenarios/${runScenarioId}/runs/${run.id}`); setRunScenarioId(null); @@ -271,44 +282,20 @@ export function ScenariosPage() { {t('common.confirm_delete_scenario')} - !running && setRunScenarioId(null)} - footer={ - <> - - - - } - > - {envs.length === 0 ? ( -

{t('scenarios.run_modal_no_env')}

- ) : ( - <> - setSaveSessionFlag(e.target.checked)} - /> - {t('common.save_session')} - - - )} -
+ environments={envs} + selectedEnvId={selectedEnvId} + onEnvSelect={setSelectedEnvId} + saveSessionFlag={saveSessionFlag} + onSaveSessionChange={setSaveSessionFlag} + traceMode={traceMode} + defaultTraceMode={runDefaults?.traceMode ?? null} + onTraceModeChange={setTraceModeOverride} + onClose={() => setRunScenarioId(null)} + onRun={handleRun} + isRunning={running} + />
); } diff --git a/docs/scenario.md b/docs/scenario.md index fa6edd1..31e9ad6 100644 --- a/docs/scenario.md +++ b/docs/scenario.md @@ -43,6 +43,18 @@ Available in scope: `validateCode` additionally receives `result` in scope, which is the value returned by `execCode`. +## Playwright traces + +Each run records a [Playwright trace](https://playwright.dev/docs/trace-viewer) of its browser context (screenshots and DOM snapshots). When the run finishes, the trace is uploaded to the run's files as `trace.zip`. Download it from the run detail page, then open it with `npx playwright show-trace trace.zip` or at https://trace.playwright.dev. + +The trace mode is chosen per run: the run modal in the UI, the `traceMode` field of `POST /scenarios/:id/run`, and the `traceMode` argument of the `run_scenario` MCP tool all accept one of the values below. When it is omitted, the server's `PLAYWRIGHT_TRACE_MODE` setting is used (exposed via `GET /scenarios/run-defaults`). "Repeat run" reuses the original run's mode. + +- `on` (default): keep the trace of every run +- `retain-on-failure`: keep traces only for runs that failed +- `off`: do not record traces + +Traces are run artifacts, so they expire after `FILE_RUN_EXPIRATION_DAYS` like other run files. + ## Validation contract `validateCode` may return: diff --git a/package-lock.json b/package-lock.json index 6cca854..202e668 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "liqa", - "version": "1.14.0", + "version": "1.15.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "liqa", - "version": "1.14.0", + "version": "1.15.0", "workspaces": [ "server", "client" diff --git a/package.json b/package.json index 84eb432..c7862f9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "liqa", - "version": "1.14.0", + "version": "1.15.0", "private": true, "workspaces": [ "server", diff --git a/server/src/config/app.config.ts b/server/src/config/app.config.ts index 5a3dc61..ce9081c 100644 --- a/server/src/config/app.config.ts +++ b/server/src/config/app.config.ts @@ -1,8 +1,21 @@ import { plainToInstance } from "class-transformer"; -import { IsInt, IsString, Max, Min, validateSync } from "class-validator"; +import { + IsEnum, + IsInt, + IsString, + Max, + Min, + validateSync, +} from "class-validator"; import pkg from "../../package.json"; +export enum TraceMode { + On = "on", + RetainOnFailure = "retain-on-failure", + Off = "off", +} + export class AppConfig { @IsString() NODE_ENV: string = "development"; @@ -41,6 +54,9 @@ export class AppConfig { @IsString() FILE_CLEANUP_CRON: string = "0 3 * * *"; + + @IsEnum(TraceMode) + PLAYWRIGHT_TRACE_MODE: TraceMode = TraceMode.On; } export function validateAppConfig(config: Record): AppConfig { diff --git a/server/src/mcp/mcp.service.ts b/server/src/mcp/mcp.service.ts index 5469b02..78ff1f7 100644 --- a/server/src/mcp/mcp.service.ts +++ b/server/src/mcp/mcp.service.ts @@ -7,6 +7,7 @@ import { SessionService } from "../session/session.service"; import { SessionContextService } from "../session/session-context.service"; import { EnvironmentService } from "../environment/environment.service"; import type { EnvironmentData } from "../environment/environment.entity"; +import { TraceMode } from "../config/app.config"; import { CredentialService } from "../credential/credential.service"; import { BrowserService } from "../browser/browser.service"; import { CodeExecutorService } from "../code-executor/code-executor.service"; @@ -939,14 +940,21 @@ export class McpService { .boolean() .optional() .describe("Save session after run completes"), + traceMode: z + .enum(TraceMode) + .optional() + .describe( + "Playwright trace recording: on (keep every trace), retain-on-failure, or off. Defaults to the server's PLAYWRIGHT_TRACE_MODE", + ), }, }, - async ({ id, environmentId, saveSession }) => { + async ({ id, environmentId, saveSession, traceMode }) => { try { const run = await this.scenarioService.createRun( id, environmentId, saveSession, + traceMode, ); return { content: [{ type: "text" as const, text: JSON.stringify(run) }], diff --git a/server/src/scenario/dto/create-scenario-run.dto.ts b/server/src/scenario/dto/create-scenario-run.dto.ts index a0402f0..23e07e2 100644 --- a/server/src/scenario/dto/create-scenario-run.dto.ts +++ b/server/src/scenario/dto/create-scenario-run.dto.ts @@ -1,4 +1,11 @@ -import { IsBoolean, IsNotEmpty, IsOptional, IsUUID } from "class-validator"; +import { + IsBoolean, + IsEnum, + IsNotEmpty, + IsOptional, + IsUUID, +} from "class-validator"; +import { TraceMode } from "../../config/app.config"; export class CreateScenarioRunDto { @IsUUID() @@ -8,4 +15,8 @@ export class CreateScenarioRunDto { @IsBoolean() @IsOptional() saveSession?: boolean; + + @IsEnum(TraceMode) + @IsOptional() + traceMode?: TraceMode; } diff --git a/server/src/scenario/scenario-run.entity.ts b/server/src/scenario/scenario-run.entity.ts index 632f55c..4529e12 100644 --- a/server/src/scenario/scenario-run.entity.ts +++ b/server/src/scenario/scenario-run.entity.ts @@ -8,6 +8,7 @@ import { PrimaryGeneratedColumn, UpdateDateColumn, } from "typeorm"; +import type { TraceMode } from "../config/app.config"; import { EnvironmentEntity } from "../environment/environment.entity"; import { SessionEntity } from "../session/session.entity"; import { ScenarioRunStepEntity } from "./scenario-run-step.entity"; @@ -47,6 +48,10 @@ export class ScenarioRunEntity { @Column({ type: "boolean", default: false }) saveSession: boolean; + // Playwright trace mode for this run; null falls back to PLAYWRIGHT_TRACE_MODE + @Column("text", { nullable: true, default: null }) + traceMode: TraceMode | null; + @OneToMany(() => ScenarioRunStepEntity, (rs) => rs.run, { cascade: true, eager: false, diff --git a/server/src/scenario/scenario-scheduler.service.ts b/server/src/scenario/scenario-scheduler.service.ts index afcd559..3ee352c 100644 --- a/server/src/scenario/scenario-scheduler.service.ts +++ b/server/src/scenario/scenario-scheduler.service.ts @@ -1,11 +1,16 @@ import { Injectable, OnModuleInit } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; import { Interval } from "@nestjs/schedule"; import { InjectRepository } from "@nestjs/typeorm"; import * as crypto from "crypto"; +import * as fs from "fs/promises"; +import * as os from "os"; +import * as path from "path"; import { Effect } from "effect"; import type { Browser, BrowserContext, Page } from "playwright"; import { chromium } from "playwright"; import { Repository } from "typeorm"; +import { type AppConfig, TraceMode } from "../config/app.config"; import { FileStorageService } from "../file/file-storage.service"; import type { ScriptLogger } from "../code-executor/code-executor.service"; import { CodeExecutorService } from "../code-executor/code-executor.service"; @@ -29,6 +34,7 @@ interface BrowserHandle { browser: Browser; context: BrowserContext; page: Page; + tracing: boolean; } @Injectable() @@ -49,6 +55,9 @@ export class ScenarioSchedulerService implements OnModuleInit { // Monotonic log sequence per run, since persistLog writes are unawaited and // can otherwise resolve out of call order private readonly runLogSeq = new Map(); + // Cache Playwright trace mode per run + private readonly runTraceModes = new Map(); + private readonly defaultTraceMode: TraceMode; private static readonly DEFAULT_SCENARIO_TIMEOUT_SEC = 600; private static readonly DEFAULT_STEP_TIMEOUT_SEC = 60; @@ -69,7 +78,10 @@ export class ScenarioSchedulerService implements OnModuleInit { private readonly sessionService: SessionService, private readonly sessionContextService: SessionContextService, private readonly fileStorageService: FileStorageService, - ) {} + configService: ConfigService, + ) { + this.defaultTraceMode = configService.get("PLAYWRIGHT_TRACE_MODE"); + } async onModuleInit(): Promise { const staleRuns = await this.runRepo.find({ @@ -208,6 +220,10 @@ export class ScenarioSchedulerService implements OnModuleInit { .catch(() => null); this.runScenarioTimeouts.set(run.id, scenario?.timeoutSeconds ?? null); this.runScenarioIds.set(run.id, run.scenarioId); + this.runTraceModes.set( + run.id, + run.traceMode ?? this.defaultTraceMode, + ); const traceId = crypto.randomUUID(); void traceStorage.run({ traceId }, () => this.processRunToCompletion(run.id), @@ -221,6 +237,7 @@ export class ScenarioSchedulerService implements OnModuleInit { this.runEnvironments.delete(run.id); this.runScenarioTimeouts.delete(run.id); this.runScenarioIds.delete(run.id); + this.runTraceModes.delete(run.id); this.runLogSeq.delete(run.id); this.logger.error( `Run #${run.id}: setup failed — ${String(err)}`, @@ -279,12 +296,15 @@ export class ScenarioSchedulerService implements OnModuleInit { ); await this.failRun(runId, `Unexpected error: ${(err as Error).message}`); } finally { + // Before runLogSeq is cleared, so the trace log entry keeps its order + await this.saveTrace(runId); this.activeRuns.delete(runId); this.runCredentials.delete(runId); this.runSnippets.delete(runId); this.runEnvironments.delete(runId); this.runScenarioTimeouts.delete(runId); this.runScenarioIds.delete(runId); + this.runTraceModes.delete(runId); this.runLogSeq.delete(runId); await this.maybePreserveSession(runId); } @@ -431,9 +451,21 @@ export class ScenarioSchedulerService implements OnModuleInit { executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, }); const context = await browser.newContext(); + let tracing = false; + const traceMode = this.runTraceModes.get(runId) ?? this.defaultTraceMode; + if (traceMode !== TraceMode.Off) { + try { + await context.tracing.start({ screenshots: true, snapshots: true }); + tracing = true; + } catch (err) { + this.logger.warn( + `Run #${runId}: failed to start tracing — ${(err as Error).message}`, + ); + } + } const page = await context.newPage(); - const handle: BrowserHandle = { browser, context, page }; + const handle: BrowserHandle = { browser, context, page, tracing }; this.runBrowsers.set(runId, handle); this.logger.log(`Run #${runId}: browser created`); return handle; @@ -453,6 +485,53 @@ export class ScenarioSchedulerService implements OnModuleInit { } } + /** + * Stops the run's Playwright trace and uploads it as a `trace.zip` run + * artifact. In retain-on-failure mode, traces of runs that did not fail are + * discarded. + */ + private async saveTrace(runId: string): Promise { + const handle = this.runBrowsers.get(runId); + if (!handle?.tracing) return; + handle.tracing = false; + + const run = await this.runRepo.findOneBy({ id: runId }).catch(() => null); + const traceMode = this.runTraceModes.get(runId) ?? this.defaultTraceMode; + const keep = traceMode === TraceMode.On || run?.status === "fail"; + + let tmpDir: string | null = null; + try { + if (!keep) { + await handle.context.tracing.stop(); + return; + } + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "liqa-trace-")); + const tracePath = path.join(tmpDir, "trace.zip"); + await handle.context.tracing.stop({ path: tracePath }); + const trace = await fs.readFile(tracePath); + await this.fileStorageService.createAndSaveRunArtifact( + runId, + trace, + "trace.zip", + "application/zip", + ); + this.persistLog( + runId, + null, + "log", + "Playwright trace uploaded to run files (open with `npx playwright show-trace trace.zip` or https://trace.playwright.dev)", + ); + } catch (err) { + this.logger.warn( + `Run #${runId}: failed to save trace — ${(err as Error).message}`, + ); + } finally { + if (tmpDir) { + await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + } + } + } + // ── Pass / fail helpers ──────────────────────────────────────────────────── private async passStepRun( diff --git a/server/src/scenario/scenario.controller.ts b/server/src/scenario/scenario.controller.ts index 6fe1e81..c64289d 100644 --- a/server/src/scenario/scenario.controller.ts +++ b/server/src/scenario/scenario.controller.ts @@ -70,6 +70,13 @@ export class ScenarioController { return this.scenarioService.findAllRuns(query); } + @Get("run-defaults") + @ApiOperation({ summary: "Get server-configured defaults for new runs" }) + @ApiResponse({ status: 200 }) + getRunDefaults() { + return this.scenarioService.getRunDefaults(); + } + @Get(":id") @ApiOperation({ summary: "Get a scenario with its steps" }) @ApiResponse({ status: 200 }) @@ -276,6 +283,7 @@ export class ScenarioController { id, dto.environmentId, dto.saveSession, + dto.traceMode, ); } diff --git a/server/src/scenario/scenario.service.ts b/server/src/scenario/scenario.service.ts index cc5c34e..5431dc9 100644 --- a/server/src/scenario/scenario.service.ts +++ b/server/src/scenario/scenario.service.ts @@ -4,6 +4,7 @@ import { Injectable, NotFoundException, } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; import { InjectRepository } from "@nestjs/typeorm"; import * as fs from "fs/promises"; import * as path from "path"; @@ -13,6 +14,7 @@ import { PaginationQueryDto, } from "../common/dto/pagination.dto"; import { resolveUsedSnippetAliases } from "../common/snippet-usage"; +import type { AppConfig, TraceMode } from "../config/app.config"; import { CredentialEntity } from "../credential/credential.entity"; import { EnvironmentEntity } from "../environment/environment.entity"; import { FileStorageService } from "../file/file-storage.service"; @@ -69,6 +71,7 @@ export class ScenarioService { private readonly scenarioRunFileRepo: Repository, private readonly fileStorageService: FileStorageService, private readonly snippetService: SnippetService, + private readonly configService: ConfigService, ) {} // ── Scenarios ───────────────────────────────────────────────────────────── @@ -488,10 +491,15 @@ export class ScenarioService { return this.findRun(scenarioId, runId); } + getRunDefaults(): { traceMode: TraceMode } { + return { traceMode: this.configService.get("PLAYWRIGHT_TRACE_MODE") }; + } + async createRun( scenarioId: string, environmentId: string, saveSession = false, + traceMode?: TraceMode, ): Promise { const scenario = await this.findOne(scenarioId); const environment = await this.environmentRepo.findOneBy({ @@ -507,6 +515,7 @@ export class ScenarioService { environmentId, status: "pending", saveSession, + traceMode: traceMode ?? this.getRunDefaults().traceMode, }), ); diff --git a/server/test/__mocks__/playwright.ts b/server/test/__mocks__/playwright.ts index 37ef6b5..06bad94 100644 --- a/server/test/__mocks__/playwright.ts +++ b/server/test/__mocks__/playwright.ts @@ -12,6 +12,16 @@ const makeContext = () => ({ cookies: jest.fn().mockResolvedValue([]), addCookies: jest.fn().mockResolvedValue(undefined), addInitScript: jest.fn().mockResolvedValue(undefined), + tracing: { + start: jest.fn().mockResolvedValue(undefined), + // Mimic Playwright writing the trace archive when a path is given + stop: jest.fn().mockImplementation(async (opts?: { path?: string }) => { + if (opts?.path) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + await require("fs/promises").writeFile(opts.path, "fake trace zip"); + } + }), + }, }); const makeBrowser = () => ({ diff --git a/server/test/mcp.controller.spec.ts b/server/test/mcp.controller.spec.ts index 4c016cb..410a485 100644 --- a/server/test/mcp.controller.spec.ts +++ b/server/test/mcp.controller.spec.ts @@ -588,11 +588,15 @@ describe("McpController", () => { total: number; }; - expect(list.total).toBe(1); - expect(list.items).toHaveLength(1); - expect(list.items[0].name).toBe("artifact.bin"); - expect(list.items[0].mimeType).toBe("application/octet-stream"); - expect(list.items[0].expiresAt).not.toBeNull(); + // The script's artifact plus the run's Playwright trace + expect(list.total).toBe(2); + expect(list.items.map((f) => f.name).sort()).toEqual([ + "artifact.bin", + "trace.zip", + ]); + const artifact = list.items.find((f) => f.name === "artifact.bin")!; + expect(artifact.mimeType).toBe("application/octet-stream"); + expect(artifact.expiresAt).not.toBeNull(); }); it("returns MCP error for non-existent scenario", async () => { @@ -621,7 +625,7 @@ describe("McpController", () => { ) as { items: Array<{ id: string; name: string; mimeType: string }>; }; - const file = list.items[0]; + const file = list.items.find((f) => f.name === "artifact.bin")!; const { status, rpc } = await mcpCall("get_run_file_content", { scenarioId: sc.id, diff --git a/server/test/scenario-trace.spec.ts b/server/test/scenario-trace.spec.ts new file mode 100644 index 0000000..fa44355 --- /dev/null +++ b/server/test/scenario-trace.spec.ts @@ -0,0 +1,181 @@ +/** + * Integration tests for Playwright trace recording on scenario runs. + * + * The playwright mock writes a fake archive when tracing.stop() is given a + * path, so we can assert that a `trace.zip` run artifact is (or is not) + * uploaded depending on the run's trace mode and outcome. + */ + +import { INestApplication } from "@nestjs/common"; +import request from "supertest"; +import { DataSource } from "typeorm"; +import { buildTestApp } from "./app.harness"; +import { TraceMode } from "../src/config/app.config"; +import { EnvironmentEntity } from "../src/environment/environment.entity"; +import { ScenarioSchedulerService } from "../src/scenario/scenario-scheduler.service"; +import { ScenarioService } from "../src/scenario/scenario.service"; + +describe("ScenarioSchedulerService Playwright traces", () => { + let app: INestApplication; + let dataSource: DataSource; + let scenarioService: ScenarioService; + let scheduler: ScenarioSchedulerService; + + beforeAll(async () => { + app = await buildTestApp(); + dataSource = app.get(DataSource); + scenarioService = app.get(ScenarioService); + scheduler = app.get(ScenarioSchedulerService); + }); + + afterAll(async () => { + await app.close(); + }); + + // ── helpers ──────────────────────────────────────────────────────────────── + + async function runScenario( + name: string, + execCode: string, + traceMode?: TraceMode, + ): Promise { + const sc = await scenarioService.create({ name }); + await scenarioService.createStep(sc.id, { execCode }); + const envRepo = dataSource.getRepository(EnvironmentEntity); + const env = await envRepo.save( + envRepo.create({ name: `${name}-env`, data: {} }), + ); + const run = await scenarioService.createRun( + sc.id, + env.id, + false, + traceMode, + ); + await scheduler.pickUpPendingRuns(); + await scenarioService.waitForRun(sc.id, run.id, 10_000); + // The trace is saved after the run reaches its final status; wait for + // the scheduler to finish its cleanup for this run. + const activeRuns = (scheduler as unknown as { activeRuns: Set }) + .activeRuns; + const deadline = Date.now() + 5_000; + while (activeRuns.has(run.id) && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 20)); + } + return run.id; + } + + async function traceArtifacts( + runId: string, + ): Promise<{ mimeType: string }[]> { + return dataSource.query( + `SELECT f.mimeType FROM scenario_run_files rf + JOIN files f ON f.id = rf.fileId + WHERE rf.runId = ? AND f.originalName = 'trace.zip'`, + [runId], + ); + } + + // ── tests ────────────────────────────────────────────────────────────────── + + it("uploads trace.zip for a passing run in 'on' mode", async () => { + const runId = await runScenario("trace-on-pass", "return 1;", TraceMode.On); + const files = await traceArtifacts(runId); + expect(files).toHaveLength(1); + expect(files[0].mimeType).toBe("application/zip"); + }); + + it("uploads trace.zip for a failing run in 'retain-on-failure' mode", async () => { + const runId = await runScenario( + "trace-rof-fail", + "throw new Error('boom');", + TraceMode.RetainOnFailure, + ); + expect(await traceArtifacts(runId)).toHaveLength(1); + }); + + it("discards the trace of a passing run in 'retain-on-failure' mode", async () => { + const runId = await runScenario( + "trace-rof-pass", + "return 1;", + TraceMode.RetainOnFailure, + ); + expect(await traceArtifacts(runId)).toHaveLength(0); + }); + + it("does not record a trace in 'off' mode", async () => { + const runId = await runScenario( + "trace-off-fail", + "throw new Error('boom');", + TraceMode.Off, + ); + expect(await traceArtifacts(runId)).toHaveLength(0); + }); + + it("stores the configured default trace mode when none is given", async () => { + const runId = await runScenario("trace-default", "return 1;"); + const [row] = await dataSource.query( + `SELECT traceMode FROM scenario_runs WHERE id = ?`, + [runId], + ); + expect(row.traceMode).toBe("on"); + expect(await traceArtifacts(runId)).toHaveLength(1); + }); + + it("falls back to the configured mode for runs without a stored trace mode", async () => { + const sc = await scenarioService.create({ name: "trace-null-mode" }); + await scenarioService.createStep(sc.id, { execCode: "return 1;" }); + const envRepo = dataSource.getRepository(EnvironmentEntity); + const env = await envRepo.save( + envRepo.create({ name: "trace-null-mode-env", data: {} }), + ); + const run = await scenarioService.createRun(sc.id, env.id); + await dataSource.query( + `UPDATE scenario_runs SET traceMode = NULL WHERE id = ?`, + [run.id], + ); + await scheduler.pickUpPendingRuns(); + await scenarioService.waitForRun(sc.id, run.id, 10_000); + const activeRuns = (scheduler as unknown as { activeRuns: Set }) + .activeRuns; + const deadline = Date.now() + 5_000; + while (activeRuns.has(run.id) && Date.now() < deadline) { + await new Promise((r) => setTimeout(r, 20)); + } + expect(await traceArtifacts(run.id)).toHaveLength(1); + }); + + describe("HTTP", () => { + it("GET /scenarios/run-defaults returns the configured trace mode", async () => { + const res = await request(app.getHttpServer()).get( + "/scenarios/run-defaults", + ); + expect(res.status).toBe(200); + expect(res.body).toEqual({ traceMode: "on" }); + }); + + it("POST /scenarios/:id/run stores the requested trace mode", async () => { + const sc = await scenarioService.create({ name: "trace-http" }); + const envRepo = dataSource.getRepository(EnvironmentEntity); + const env = await envRepo.save( + envRepo.create({ name: "trace-http-env", data: {} }), + ); + const res = await request(app.getHttpServer()) + .post(`/scenarios/${sc.id}/run`) + .send({ environmentId: env.id, traceMode: "retain-on-failure" }); + expect(res.status).toBe(201); + expect(res.body.traceMode).toBe("retain-on-failure"); + }); + + it("POST /scenarios/:id/run rejects an unknown trace mode", async () => { + const sc = await scenarioService.create({ name: "trace-http-bad" }); + const envRepo = dataSource.getRepository(EnvironmentEntity); + const env = await envRepo.save( + envRepo.create({ name: "trace-http-bad-env", data: {} }), + ); + const res = await request(app.getHttpServer()) + .post(`/scenarios/${sc.id}/run`) + .send({ environmentId: env.id, traceMode: "sometimes" }); + expect(res.status).toBe(400); + }); + }); +});