import { Injectable, OnModuleInit } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { Interval } from "@nestjs/schedule"; import { InjectRepository } from "@nestjs/typeorm"; import * as crypto from "node:crypto"; import * as fs from "node:fs/promises"; import * as os from "node:os"; import * as path from "node:path"; import { Effect } from "effect"; import type { Browser, BrowserContext, Page } from "playwright"; import { chromium } from "playwright"; import { Repository } from "typeorm"; import { type AppConfig, TraceMode } from "../config/app.config"; import { FileStorageService } from "../file/file-storage.service"; import type { ScriptLogger } from "../code-executor/code-executor.service"; import { CodeExecutorService } from "../code-executor/code-executor.service"; import { ExecContextBuilder } from "../code-executor/exec-context.builder"; import { traceStorage } from "../common/trace-context"; import { TraceLogger } from "../common/trace-logger"; import { EnvironmentData, EnvironmentEntity, } from "../environment/environment.entity"; import { SessionContextService } from "../session/session-context.service"; import { SessionService } from "../session/session.service"; import { SnippetService } from "../snippet/snippet.service"; import { ScenarioRunLogEntity } from "./scenario-run-log.entity"; import { ScenarioRunStepEntity } from "./scenario-run-step.entity"; import { ScenarioRunEntity } from "./scenario-run.entity"; import { ScenarioService } from "./scenario.service"; interface BrowserHandle { browser: Browser; context: BrowserContext; page: Page; tracing: boolean; } @Injectable() export class ScenarioSchedulerService implements OnModuleInit { private readonly logger = new TraceLogger(ScenarioSchedulerService.name); private readonly activeRuns = new Set(); private readonly runBrowsers = new Map(); // Cache credential maps per run (built once when a run starts) private readonly runCredentials = new Map>(); // Cache snippet code map per run (built once when a run starts) private readonly runSnippets = new Map>(); // Cache environment values per run (built once when a run starts) private readonly runEnvironments = new Map(); // Cache scenario-level timeout (seconds) per run private readonly runScenarioTimeouts = new Map(); // Cache scenarioId per run private readonly runScenarioIds = new Map(); // Monotonic log sequence per run, since persistLog writes are unawaited and // can otherwise resolve out of call order private readonly runLogSeq = new Map(); // Cache Playwright trace mode per run private readonly runTraceModes = new Map(); private readonly defaultTraceMode: TraceMode; private static readonly DEFAULT_SCENARIO_TIMEOUT_SEC = 600; private static readonly DEFAULT_STEP_TIMEOUT_SEC = 60; private static readonly CONCURRENCY_LIMIT = 10; constructor( @InjectRepository(ScenarioRunEntity) private readonly runRepo: Repository, @InjectRepository(ScenarioRunStepEntity) private readonly runStepRepo: Repository, @InjectRepository(ScenarioRunLogEntity) private readonly runLogRepo: Repository, @InjectRepository(EnvironmentEntity) private readonly environmentRepo: Repository, private readonly codeExecutor: CodeExecutorService, private readonly scenarioService: ScenarioService, private readonly snippetService: SnippetService, private readonly sessionService: SessionService, private readonly sessionContextService: SessionContextService, private readonly fileStorageService: FileStorageService, configService: ConfigService, ) { this.defaultTraceMode = configService.get("PLAYWRIGHT_TRACE_MODE"); } async onModuleInit(): Promise { const staleRuns = await this.runRepo.find({ where: { status: "in_progress" }, }); if (staleRuns.length === 0) return; this.logger.error( `Server restart detected ${staleRuns.length} in-progress run(s) that cannot be recovered — marking as failed`, ); for (const run of staleRuns) { this.logger.error( `Run #${run.id}: marked fail due to server restart (playwright session lost)`, ); } await Effect.runPromise( Effect.forEach( staleRuns, (run) => Effect.tryPromise(() => this.failRun( run.id, "Run aborted: server restarted and playwright session could not be recovered", ), ).pipe( Effect.catchAll((err) => Effect.sync(() => this.logger.error( `Run #${run.id}: failRun threw unexpectedly — ${String(err)}`, ), ), ), ), { concurrency: ScenarioSchedulerService.CONCURRENCY_LIMIT }, ), ); } /** * Marks a run as failed, fails its active step run, cancels any queued * step runs, and persists an error log entry. Use this for all failure paths. */ private async failRun(runId: string, reason: string): Promise { await this.runRepo.update(runId, { status: "fail" }); // Fail the actively-running step; cancel anything still queued await this.runStepRepo .createQueryBuilder() .update() .set({ status: "fail" }) .where("runId = :runId AND status = 'in_progress'", { runId }) .execute(); await this.runStepRepo .createQueryBuilder() .update() .set({ status: "cancelled" }) .where("runId = :runId AND status IN (:...statuses)", { runId, statuses: ["pending", "waiting"], }) .execute(); this.persistLog(runId, null, "error", reason); } private persistLog( runId: string, stepRunId: string | null, level: "log" | "warn" | "error", message: string, ): void { const seq = (this.runLogSeq.get(runId) ?? 0) + 1; this.runLogSeq.set(runId, seq); void this.runLogRepo.save( this.runLogRepo.create({ runId, stepRunId, level, message, seq }), ); } private stepLogger(stepRunId: string, runId: string): ScriptLogger { return (level, msg) => { this.logger[level](`StepRun #${stepRunId} script: ${msg}`); this.persistLog(runId, stepRunId, level, msg); }; } private makeGetStepOutput( stepRun: ScenarioRunStepEntity, ): (order: number) => Promise { return async (order: number) => { const targetOrder = order < 0 ? stepRun.order + order : order; if (targetOrder < 1) return null; const sr = await this.runStepRepo.findOne({ where: { runId: stepRun.runId, order: targetOrder }, }); if (!sr?.output) return null; try { return JSON.parse(sr.output) as unknown; } catch { return sr.output; } }; } // ── Job: pick up pending runs and process each to completion ───────────── @Interval(1000) async pickUpPendingRuns(): Promise { const pending = await this.runRepo.find({ where: { status: "pending" } }); const newRuns = pending.filter((run) => !this.activeRuns.has(run.id)); if (newRuns.length === 0) return; await Effect.runPromise( Effect.forEach( newRuns, (run) => { this.activeRuns.add(run.id); return Effect.tryPromise(async () => { run.status = "in_progress"; await this.runRepo.save(run); this.logger.log(`Run #${run.id} → in_progress`); const credMap = await this.scenarioService .buildCredentialMap(run.scenarioId) .catch(() => ({}) as Record); this.runCredentials.set(run.id, credMap); const snippetMap = await this.snippetService .buildSnippetMap() .catch(() => ({}) as Record); this.runSnippets.set(run.id, snippetMap); const environmentData = await this.environmentRepo .findOneBy({ id: run.environmentId }) .then((env) => env?.data ?? {}) .catch(() => ({}) as EnvironmentData); this.runEnvironments.set(run.id, environmentData); // Legacy runs (no scenarioVersion) fall back to the live scenario const versioned = run.scenarioVersion != null ? await this.scenarioService .findVersion(run.scenarioId, run.scenarioVersion) .catch(() => null) : await this.scenarioService .findOne(run.scenarioId) .catch(() => null); this.runScenarioTimeouts.set( run.id, versioned?.timeoutSeconds ?? null, ); this.runScenarioIds.set(run.id, run.scenarioId); this.runTraceModes.set( run.id, run.traceMode ?? this.defaultTraceMode, ); const traceId = crypto.randomUUID(); void traceStorage.run({ traceId }, () => this.processRunToCompletion(run.id), ); }).pipe( Effect.catchAll((err) => Effect.promise(async () => { this.activeRuns.delete(run.id); this.runCredentials.delete(run.id); this.runSnippets.delete(run.id); this.runEnvironments.delete(run.id); this.runScenarioTimeouts.delete(run.id); this.runScenarioIds.delete(run.id); this.runTraceModes.delete(run.id); this.runLogSeq.delete(run.id); this.logger.error( `Run #${run.id}: setup failed — ${String(err)}`, ); await this.failRun( run.id, `Run setup failed: ${String(err)}`, ).catch(() => {}); }), ), ); }, { concurrency: ScenarioSchedulerService.CONCURRENCY_LIMIT }, ), ); } private async processRunToCompletion(runId: string): Promise { const scenarioTimeoutSec = this.runScenarioTimeouts.get(runId) ?? ScenarioSchedulerService.DEFAULT_SCENARIO_TIMEOUT_SEC; const scenarioDeadline = Date.now() + scenarioTimeoutSec * 1000; try { let stepRun = await this.runStepRepo.findOne({ where: { runId, status: "pending" }, order: { order: "ASC" }, }); while (stepRun) { if (Date.now() >= scenarioDeadline) { this.logger.error( `Run #${runId}: exceeded scenario timeout of ${scenarioTimeoutSec}s`, ); await this.failRun( runId, `Scenario timed out after ${scenarioTimeoutSec}s`, ); return; } await this.executeStepRun(stepRun); stepRun = await this.runStepRepo.findOne({ where: { runId, status: "pending" }, order: { order: "ASC" }, }); } await this.runRepo .createQueryBuilder() .update() .set({ status: "pass" }) .where("id = :id AND status = 'in_progress'", { id: runId }) .execute(); } catch (err) { this.logger.error( `Run #${runId}: unexpected error: ${(err as Error).message}`, ); await this.failRun(runId, `Unexpected error: ${(err as Error).message}`); } finally { // Before runLogSeq is cleared, so the trace log entry keeps its order await this.saveTrace(runId); this.activeRuns.delete(runId); this.runCredentials.delete(runId); this.runSnippets.delete(runId); this.runEnvironments.delete(runId); this.runScenarioTimeouts.delete(runId); this.runScenarioIds.delete(runId); this.runTraceModes.delete(runId); this.runLogSeq.delete(runId); await this.maybePreserveSession(runId); } } /** * If the run was created with saveSession=true, promote the run's browser * context to a named explore session instead of closing it. */ private async maybePreserveSession(runId: string): Promise { const run = await this.runRepo.findOneBy({ id: runId }); if (!run?.saveSession) { await this.closeBrowserHandle(runId); return; } const handle = this.runBrowsers.get(runId); if (!handle) { return; } const sessionName = `run-${runId}`; try { const cookies = await handle.context.cookies(); const localStorage = await handle.page .evaluate(() => ({ ...window.localStorage })) .catch((): Record => ({})); const token = (localStorage["accessToken"] as string | undefined) ?? (localStorage["token"] as string | undefined) ?? cookies.find((c) => c.name === "token")?.value ?? ""; const session = await this.sessionService.upsert( sessionName, token, cookies, localStorage, ); await this.runRepo.update(runId, { sessionId: session.id }); this.sessionContextService.register( sessionName, handle.browser, handle.context, handle.page, ); // Remove from runBrowsers so closeBrowserHandle won't close it this.runBrowsers.delete(runId); this.logger.log( `Run #${runId}: browser preserved as session "${sessionName}" (${session.id})`, ); } catch (err) { this.logger.warn( `Run #${runId}: failed to preserve session — ${(err as Error).message}`, ); await this.closeBrowserHandle(runId); } } private async executeStepRun(stepRun: ScenarioRunStepEntity): Promise { // Run steps carry a copy of the step from the run's scenario version const step = stepRun; stepRun.status = "in_progress"; await this.runStepRepo.save(stepRun); this.logger.log( `StepRun #${stepRun.id} (run #${stepRun.runId}, step #${step.scenarioStepId ?? "deleted"} order=${step.order}) → in_progress`, ); this.persistLog( stepRun.runId, stepRun.id, "log", `Step ${step.order} started${step.title ? `: ${step.title}` : ""}`, ); try { if (!step.execCode) throw new Error("step has no execCode"); this.codeExecutor.validate(step.execCode); const { page, context } = await this.getOrCreateBrowserHandle( stepRun.runId, ); const getStepOutput = this.makeGetStepOutput(stepRun); const creds = this.runCredentials.get(stepRun.runId); const snips = this.runSnippets.get(stepRun.runId); const env = this.runEnvironments.get(stepRun.runId); const scenarioId = this.runScenarioIds.get(stepRun.runId); const execCtx = new ExecContextBuilder() .page(page) .browser(context) .code(step.execCode) .log(this.stepLogger(stepRun.id, stepRun.runId)) .getStepOutput(getStepOutput) .credentials(creds) .environment(env) .snippets(snips) .scenarioId(scenarioId) .runId(stepRun.runId) .fileService(this.fileStorageService) .build(); const scenarioTimeoutSec = this.runScenarioTimeouts.get(stepRun.runId) ?? null; const stepTimeoutSec = step.timeoutSeconds ?? scenarioTimeoutSec ?? ScenarioSchedulerService.DEFAULT_STEP_TIMEOUT_SEC; const stepTimeoutMs = stepTimeoutSec * 1000; let timeoutId: ReturnType | undefined; const timeoutPromise = new Promise((_, reject) => { timeoutId = setTimeout( () => reject(new Error(`Step timed out after ${stepTimeoutSec}s`)), stepTimeoutMs, ); }); try { const { result: execOutput } = await Promise.race([ this.codeExecutor.execute(execCtx), timeoutPromise, ]); await this.passStepRun(stepRun, null, execOutput); } catch (err) { const msg = (err as Error).message ?? String(err); this.logger.error(`StepRun #${stepRun.id} threw: ${msg}`); await this.failStepRun(stepRun, msg); } finally { clearTimeout(timeoutId); } } catch (err) { const msg = (err as Error).message ?? String(err); this.logger.error(`StepRun #${stepRun.id} threw: ${msg}`); await this.failStepRun(stepRun, msg); } } // ── Shared browser per run ───────────────────────────────────────────────── private async getOrCreateBrowserHandle( runId: string, ): Promise { const existing = this.runBrowsers.get(runId); if (existing) return existing; const browser = await chromium.launch({ headless: true, executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, }); const context = await browser.newContext(); let tracing = false; const traceMode = this.runTraceModes.get(runId) ?? this.defaultTraceMode; if (traceMode !== TraceMode.Off) { try { await context.tracing.start({ screenshots: true, snapshots: true }); tracing = true; } catch (err) { this.logger.warn( `Run #${runId}: failed to start tracing — ${(err as Error).message}`, ); } } const page = await context.newPage(); const handle: BrowserHandle = { browser, context, page, tracing }; this.runBrowsers.set(runId, handle); this.logger.log(`Run #${runId}: browser created`); return handle; } private async closeBrowserHandle(runId: string): Promise { const handle = this.runBrowsers.get(runId); if (!handle) return; this.runBrowsers.delete(runId); try { await handle.browser.close(); this.logger.log(`Run #${runId}: browser closed`); } catch (err) { this.logger.warn( `Run #${runId}: error closing browser: ${(err as Error).message}`, ); } } /** * Stops the run's Playwright trace and uploads it as a `trace.zip` run * artifact. In retain-on-failure mode, traces of runs that did not fail are * discarded. */ private async saveTrace(runId: string): Promise { const handle = this.runBrowsers.get(runId); if (!handle?.tracing) return; handle.tracing = false; const run = await this.runRepo.findOneBy({ id: runId }).catch(() => null); const traceMode = this.runTraceModes.get(runId) ?? this.defaultTraceMode; const keep = traceMode === TraceMode.On || run?.status === "fail"; let tmpDir: string | null = null; try { if (!keep) { await handle.context.tracing.stop(); return; } tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "liqa-trace-")); const tracePath = path.join(tmpDir, "trace.zip"); await handle.context.tracing.stop({ path: tracePath }); const trace = await fs.readFile(tracePath); await this.fileStorageService.createAndSaveRunArtifact( runId, trace, "trace.zip", "application/zip", ); this.persistLog( runId, null, "log", "Playwright trace uploaded to run files (open with `npx playwright show-trace trace.zip` or https://trace.playwright.dev)", ); } catch (err) { this.logger.warn( `Run #${runId}: failed to save trace — ${(err as Error).message}`, ); } finally { if (tmpDir) { await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); } } } // ── Pass / fail helpers ──────────────────────────────────────────────────── private async passStepRun( stepRun: ScenarioRunStepEntity, description: string | null, output: unknown = null, ): Promise { stepRun.status = "pass"; stepRun.description = description; stepRun.output = output !== null && output !== undefined ? JSON.stringify(output) : null; await this.runStepRepo.save(stepRun); this.logger.log(`StepRun #${stepRun.id} → pass`); this.persistLog( stepRun.runId, stepRun.id, "log", `Step ${stepRun.order} passed`, ); // Find the next waiting step in this run (next by order) const nextStep = await this.runStepRepo.findOne({ where: { runId: stepRun.runId, status: "waiting" }, order: { order: "ASC" }, }); if (nextStep) { nextStep.status = "pending"; await this.runStepRepo.save(nextStep); } else { // No more waiting steps — check if any are still in_progress/pending (shouldn't be, but guard anyway) const remaining = await this.runStepRepo.count({ where: [ { runId: stepRun.runId, status: "pending" }, { runId: stepRun.runId, status: "in_progress" }, { runId: stepRun.runId, status: "waiting" }, ], }); if (remaining === 0) { // Don't close the browser here — maybePreserveSession will handle it // (either preserving it if saveSession=true, or closing if false) await this.runRepo.update(stepRun.runId, { status: "pass" }); this.logger.log(`Run #${stepRun.runId} → pass (all steps passed)`); } } } private async failStepRun( stepRun: ScenarioRunStepEntity, description: string, ): Promise { stepRun.status = "fail"; stepRun.description = description; await this.runStepRepo.save(stepRun); this.logger.log(`StepRun #${stepRun.id} → fail: ${description}`); this.persistLog( stepRun.runId, stepRun.id, "error", `Step ${stepRun.order} failed: ${description}`, ); await this.dumpPageOnFailure(stepRun); // Cancel all remaining waiting/pending step runs in this run await this.runStepRepo .createQueryBuilder() .update() .set({ status: "cancelled" }) .where("runId = :runId AND status IN (:...statuses)", { runId: stepRun.runId, statuses: ["waiting", "pending"], }) .execute(); this.logger.log(`Run #${stepRun.runId}: remaining steps cancelled`); // Don't close the browser here — maybePreserveSession (called from // processRunToCompletion's finally block) will either preserve it as a // session (saveSession=true) or close it (saveSession=false). await this.runRepo.update(stepRun.runId, { status: "fail" }); this.logger.log(`Run #${stepRun.runId} → fail`); } // ── Page dump on failure ──────────────────────────────────────────────────── private async dumpPageOnFailure( stepRun: ScenarioRunStepEntity, ): Promise { const handle = this.runBrowsers.get(stepRun.runId); if (!handle) return; // no browser was ever opened for this run (e.g. setup error) const namePrefix = `step-${stepRun.order}-failure`; try { const screenshot = await handle.page.screenshot({ fullPage: true }); await this.fileStorageService.createAndSaveRunArtifact( stepRun.runId, screenshot, `${namePrefix}.png`, "image/png", ); } catch (err) { this.logger.warn( `StepRun #${stepRun.id}: failed to capture failure screenshot — ${(err as Error).message}`, ); } try { const html = await handle.page.content(); await this.fileStorageService.createAndSaveRunArtifact( stepRun.runId, Buffer.from(html, "utf-8"), `${namePrefix}.html`, "text/html", ); } catch (err) { this.logger.warn( `StepRun #${stepRun.id}: failed to capture failure page HTML — ${(err as Error).message}`, ); } this.persistLog( stepRun.runId, stepRun.id, "log", `Step ${stepRun.order} failed: page dump uploaded to run files`, ); } }