refactor(scenario): remove validateCode — assertions live in execCode

- drop validateCode column, DTOs, service mappings, and scheduler branch
- remove parseValidateResult helper and ValidateResult interface
- inject playwright expect into code executor for direct use in execCode
- strip validateCode from MCP tool schemas, client types, and UI forms
This commit is contained in:
2026-04-11 00:52:50 +03:00
parent 29e91b0078
commit 238c52610a
16 changed files with 32 additions and 123 deletions
-2
View File
@@ -274,14 +274,12 @@ export const scenarioCredentials = {
export interface CreateStepPayload { export interface CreateStepPayload {
title?: string; title?: string;
execCode?: string; execCode?: string;
validateCode?: string;
} }
export interface UpdateStepPayload { export interface UpdateStepPayload {
title?: string; title?: string;
order?: number; order?: number;
execCode?: string; execCode?: string;
validateCode?: string;
} }
export const steps = { export const steps = {
-1
View File
@@ -66,7 +66,6 @@ export interface ScenarioStep {
order: number; order: number;
title: string | null; title: string | null;
execCode: string | null; execCode: string | null;
validateCode: string | null;
createdAt: string; createdAt: string;
updatedAt: string; updatedAt: string;
} }
+1 -3
View File
@@ -183,9 +183,7 @@
"form_session_placeholder": "e.g. my-session", "form_session_placeholder": "e.g. my-session",
"form_session_required": "Session name is required", "form_session_required": "Session name is required",
"form_exec_code": "Exec code", "form_exec_code": "Exec code",
"form_exec_code_placeholder": "return await page.title();", "form_exec_code_placeholder": "return await page.title();"
"form_validate_code": "Validate code",
"form_validate_code_placeholder": "return { success: result !== null, description: \"…\" };"
}, },
"runs": { "runs": {
"title": "Runs", "title": "Runs",
@@ -16,7 +16,6 @@ export function CreateStepPage() {
const [scenario, setScenario] = useState<Scenario | null>(null); const [scenario, setScenario] = useState<Scenario | null>(null);
const [title, setTitle] = useState(''); const [title, setTitle] = useState('');
const [execCode, setExecCode] = useState(''); const [execCode, setExecCode] = useState('');
const [validateCode, setValidateCode] = useState('');
const [saving, setSaving] = useState(false); const [saving, setSaving] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
@@ -36,7 +35,6 @@ export function CreateStepPage() {
await steps.create(id!, { await steps.create(id!, {
title: title.trim() || undefined, title: title.trim() || undefined,
execCode: execCode.trim() || undefined, execCode: execCode.trim() || undefined,
validateCode: validateCode.trim() || undefined,
}); });
toast.success('Step created'); toast.success('Step created');
navigate(`/scenarios/${id}`); navigate(`/scenarios/${id}`);
@@ -77,10 +75,6 @@ export function CreateStepPage() {
<label className={styles.fieldLabel}>{t('steps.form_exec_code')}</label> <label className={styles.fieldLabel}>{t('steps.form_exec_code')}</label>
<CodeEditor value={execCode} onChange={setExecCode} rows={6} /> <CodeEditor value={execCode} onChange={setExecCode} rows={6} />
</div> </div>
<div className={styles.formField}>
<label className={styles.fieldLabel}>{t('steps.form_validate_code')}</label>
<CodeEditor value={validateCode} onChange={setValidateCode} rows={6} />
</div>
</div> </div>
<div className={styles.formActions}> <div className={styles.formActions}>
@@ -17,7 +17,6 @@ export function EditStepPage() {
const [step, setStep] = useState<ScenarioStep | null>(null); const [step, setStep] = useState<ScenarioStep | null>(null);
const [title, setTitle] = useState(''); const [title, setTitle] = useState('');
const [execCode, setExecCode] = useState(''); const [execCode, setExecCode] = useState('');
const [validateCode, setValidateCode] = useState('');
const [loading, setLoading] = useState(true); const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false); const [saving, setSaving] = useState(false);
const [error, setError] = useState<string | null>(null); const [error, setError] = useState<string | null>(null);
@@ -30,7 +29,6 @@ export function EditStepPage() {
setStep(st); setStep(st);
setTitle(st.title ?? ''); setTitle(st.title ?? '');
setExecCode(st.execCode ?? ''); setExecCode(st.execCode ?? '');
setValidateCode(st.validateCode ?? '');
}) })
.catch((err: Error) => setError(err.message)) .catch((err: Error) => setError(err.message))
.finally(() => setLoading(false)); .finally(() => setLoading(false));
@@ -44,7 +42,6 @@ export function EditStepPage() {
await steps.update(id!, stepId!, { await steps.update(id!, stepId!, {
title: title.trim() || undefined, title: title.trim() || undefined,
execCode: execCode.trim() || undefined, execCode: execCode.trim() || undefined,
validateCode: validateCode.trim() || undefined,
}); });
toast.success('Step updated'); toast.success('Step updated');
navigate(`/scenarios/${id}`); navigate(`/scenarios/${id}`);
@@ -88,10 +85,6 @@ export function EditStepPage() {
<label className={styles.fieldLabel}>{t('steps.form_exec_code')}</label> <label className={styles.fieldLabel}>{t('steps.form_exec_code')}</label>
<CodeEditor value={execCode} onChange={setExecCode} rows={6} /> <CodeEditor value={execCode} onChange={setExecCode} rows={6} />
</div> </div>
<div className={styles.formField}>
<label className={styles.fieldLabel}>{t('steps.form_validate_code')}</label>
<CodeEditor value={validateCode} onChange={setValidateCode} rows={6} />
</div>
</div> </div>
<div className={styles.formActions}> <div className={styles.formActions}>
+16
View File
@@ -3532,6 +3532,21 @@
"url": "https://opencollective.com/pkgr" "url": "https://opencollective.com/pkgr"
} }
}, },
"node_modules/@playwright/test": {
"version": "1.59.1",
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.59.1.tgz",
"integrity": "sha512-PG6q63nQg5c9rIi4/Z5lR5IVF7yU5MqmKaPOe0HSc0O2cX1fPi96sUQu5j7eo4gKCkB2AnNGoWt7y4/Xx3Kcqg==",
"license": "Apache-2.0",
"dependencies": {
"playwright": "1.59.1"
},
"bin": {
"playwright": "cli.js"
},
"engines": {
"node": ">=18"
}
},
"node_modules/@polka/url": { "node_modules/@polka/url": {
"version": "1.0.0-next.29", "version": "1.0.0-next.29",
"resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz", "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
@@ -15942,6 +15957,7 @@
"@nestjs/schedule": "^6.1.1", "@nestjs/schedule": "^6.1.1",
"@nestjs/swagger": "^11.2.6", "@nestjs/swagger": "^11.2.6",
"@nestjs/typeorm": "^11.0.1", "@nestjs/typeorm": "^11.0.1",
"@playwright/test": "^1.59.1",
"acorn": "^8.16.0", "acorn": "^8.16.0",
"better-sqlite3": "^12.8.0", "better-sqlite3": "^12.8.0",
"class-transformer": "^0.5.1", "class-transformer": "^0.5.1",
+1
View File
@@ -27,6 +27,7 @@
"@nestjs/schedule": "^6.1.1", "@nestjs/schedule": "^6.1.1",
"@nestjs/swagger": "^11.2.6", "@nestjs/swagger": "^11.2.6",
"@nestjs/typeorm": "^11.0.1", "@nestjs/typeorm": "^11.0.1",
"@playwright/test": "^1.59.1",
"acorn": "^8.16.0", "acorn": "^8.16.0",
"better-sqlite3": "^12.8.0", "better-sqlite3": "^12.8.0",
"class-transformer": "^0.5.1", "class-transformer": "^0.5.1",
@@ -6,6 +6,7 @@ import {
import { TraceLogger } from "../common/trace-logger"; import { TraceLogger } from "../common/trace-logger";
import { parse } from "acorn"; import { parse } from "acorn";
import type { Page, BrowserContext } from "playwright"; import type { Page, BrowserContext } from "playwright";
import { expect as playwrightExpect } from "@playwright/test";
import { dumpDom } from "./dom-helpers"; import { dumpDom } from "./dom-helpers";
import type { EnvironmentData } from "../environment/environment.entity"; import type { EnvironmentData } from "../environment/environment.entity";
@@ -123,19 +124,28 @@ export class CodeExecutorService {
"helpers", "helpers",
"console", "console",
"snippetArgs", "snippetArgs",
"expect",
`return (async (page, context, helpers, ...args) => { ${snippetCode} })(page, context, helpers, ...snippetArgs)`, `return (async (page, context, helpers, ...args) => { ${snippetCode} })(page, context, helpers, ...snippetArgs)`,
); );
return snippetFn(page, context, pageHelpers, fakeConsole, args); return snippetFn(
page,
context,
pageHelpers,
fakeConsole,
args,
playwrightExpect,
);
}, },
}; };
// Passing `console` as a named parameter shadows the global in the script scope. // Passing `console` and `expect` as named parameters exposes them in script scope.
const fn = new Function( const fn = new Function(
"page", "page",
"context", "context",
"helpers", "helpers",
"console", "console",
"result", "result",
"expect",
`return (async (page, context, helpers) => { ${code} })(page, context, helpers)`, `return (async (page, context, helpers) => { ${code} })(page, context, helpers)`,
); );
this.logger.debug("Executing user code"); this.logger.debug("Executing user code");
@@ -145,6 +155,7 @@ export class CodeExecutorService {
pageHelpers, pageHelpers,
fakeConsole, fakeConsole,
result, result,
playwrightExpect,
); );
return { result: execResult }; return { result: execResult };
} catch (err) { } catch (err) {
-10
View File
@@ -505,10 +505,6 @@ export class McpService {
.string() .string()
.optional() .optional()
.describe("Playwright JS code to execute (exec steps)"), .describe("Playwright JS code to execute (exec steps)"),
validateCode: z
.string()
.optional()
.describe("Validation JS code returning { success, description }"),
}, },
}, },
async ({ scenarioId, ...dto }) => { async ({ scenarioId, ...dto }) => {
@@ -569,7 +565,6 @@ export class McpService {
.describe("New step type"), .describe("New step type"),
title: z.string().optional().describe("New step title"), title: z.string().optional().describe("New step title"),
execCode: z.string().optional().describe("New exec code"), execCode: z.string().optional().describe("New exec code"),
validateCode: z.string().optional().describe("New validation code"),
}, },
}, },
async ({ scenarioId, stepId, ...dto }) => { async ({ scenarioId, stepId, ...dto }) => {
@@ -785,11 +780,6 @@ export class McpService {
.nullable() .nullable()
.optional() .optional()
.describe("Exec/sign code"), .describe("Exec/sign code"),
validateCode: z
.string()
.nullable()
.optional()
.describe("Validation code"),
}), }),
) )
.describe("Ordered list of steps"), .describe("Ordered list of steps"),
@@ -13,12 +13,4 @@ export class CreateScenarioStepDto {
@IsNotEmpty() @IsNotEmpty()
execCode?: string; execCode?: string;
@ApiPropertyOptional({
example:
'return { success: result !== null, description: "title present" };',
})
@IsOptional()
@IsString()
@IsNotEmpty()
validateCode?: string;
} }
@@ -23,11 +23,6 @@ export class ScenarioStepExportDto {
@IsNotEmpty() @IsNotEmpty()
execCode: string | null; execCode: string | null;
@ApiPropertyOptional()
@IsOptional()
@IsString()
@IsNotEmpty()
validateCode: string | null;
} }
export class ScenarioExportDto { export class ScenarioExportDto {
@@ -19,9 +19,4 @@ export class UpdateScenarioStepDto {
@IsNotEmpty() @IsNotEmpty()
execCode?: string; execCode?: string;
@ApiPropertyOptional()
@IsOptional()
@IsString()
@IsNotEmpty()
validateCode?: string;
} }
@@ -17,11 +17,6 @@ import { ScenarioService } from "./scenario.service";
import { SnippetService } from "../snippet/snippet.service"; import { SnippetService } from "../snippet/snippet.service";
import { EnvironmentEntity, EnvironmentData } from "../environment/environment.entity"; import { EnvironmentEntity, EnvironmentData } from "../environment/environment.entity";
interface ValidateResult {
success: boolean;
description?: string;
}
interface BrowserHandle { interface BrowserHandle {
browser: Browser; browser: Browser;
context: BrowserContext; context: BrowserContext;
@@ -188,25 +183,6 @@ export class ScenarioSchedulerService {
snips, snips,
); );
if (step.validateCode) {
this.codeExecutor.validate(step.validateCode);
const { result } = await this.codeExecutor.execute(
page,
context,
step.validateCode,
this.stepLogger(stepRun.id, stepRun.runId),
getStepOutput,
creds,
env,
snips,
execOutput,
);
const vr = this.parseValidateResult(result);
if (!vr.success) throw new Error(vr.description ?? "Validation failed");
await this.passStepRun(stepRun, vr.description ?? null, execOutput);
return;
}
await this.passStepRun(stepRun, null, execOutput); await this.passStepRun(stepRun, null, execOutput);
} catch (err) { } catch (err) {
const msg = (err as Error).message ?? String(err); const msg = (err as Error).message ?? String(err);
@@ -250,21 +226,6 @@ export class ScenarioSchedulerService {
} }
} }
// ── Validation helper ──────────────────────────────────────────────────────
private parseValidateResult(raw: unknown): ValidateResult {
if (typeof raw === "boolean") return { success: raw };
if (raw && typeof raw === "object") {
const r = raw as Record<string, unknown>;
return {
success: Boolean(r["success"]),
description:
r["description"] != null ? String(r["description"]) : undefined,
};
}
return { success: Boolean(raw) };
}
// ── Pass / fail helpers ──────────────────────────────────────────────────── // ── Pass / fail helpers ────────────────────────────────────────────────────
private async passStepRun( private async passStepRun(
@@ -32,9 +32,6 @@ export class ScenarioStepEntity {
@Column({ type: "text", nullable: true }) @Column({ type: "text", nullable: true })
execCode: string | null; execCode: string | null;
@Column({ type: "text", nullable: true })
validateCode: string | null;
@CreateDateColumn() @CreateDateColumn()
createdAt: Date; createdAt: Date;
-4
View File
@@ -124,7 +124,6 @@ export class ScenarioService {
scenarioId, scenarioId,
title: dto.title ?? null, title: dto.title ?? null,
execCode: dto.execCode ?? null, execCode: dto.execCode ?? null,
validateCode: dto.validateCode ?? null,
}), }),
); );
await this.normalizeStepOrder(scenarioId); await this.normalizeStepOrder(scenarioId);
@@ -156,7 +155,6 @@ export class ScenarioService {
order: step.order, order: step.order,
title: dto.title ?? step.title, title: dto.title ?? step.title,
execCode: dto.execCode ?? step.execCode, execCode: dto.execCode ?? step.execCode,
validateCode: dto.validateCode ?? step.validateCode,
}); });
await this.stepRepo.save(step); await this.stepRepo.save(step);
@@ -404,7 +402,6 @@ export class ScenarioService {
steps: scenario.steps.map((s) => ({ steps: scenario.steps.map((s) => ({
title: s.title, title: s.title,
execCode: s.execCode, execCode: s.execCode,
validateCode: s.validateCode,
})), })),
}; };
} }
@@ -436,7 +433,6 @@ export class ScenarioService {
order: index, order: index,
title: s.title ?? null, title: s.title ?? null,
execCode: s.execCode ?? null, execCode: s.execCode ?? null,
validateCode: s.validateCode ?? null,
}), }),
); );
await this.stepRepo.save(steps); await this.stepRepo.save(steps);
+1 -28
View File
@@ -477,7 +477,6 @@ describe("ScenarioController", () => {
order: 1, order: 1,
sessionName: "s", sessionName: "s",
execCode: "return 1;", execCode: "return 1;",
validateCode: "return true;",
}); });
const res = await request(app.getHttpServer()) const res = await request(app.getHttpServer())
@@ -536,25 +535,6 @@ describe("ScenarioController", () => {
expect(step).not.toHaveProperty("updatedAt"); expect(step).not.toHaveProperty("updatedAt");
}); });
it("exports null validateCode as null", async () => {
const sc = await createScenario("export-null-validate");
await createStep(sc.id, {
order: 0,
sessionName: "s",
execCode: "return 1;",
});
const res = await request(app.getHttpServer())
.get(`/scenarios/${sc.id}/export`)
.expect(200);
const exported = yamlParse(res.text) as {
steps: Array<{ validateCode: string | null }>;
};
expect(exported.steps[0].validateCode).toBeNull();
});
it("returns 404 for unknown scenario", async () => { it("returns 404 for unknown scenario", async () => {
await request(app.getHttpServer()) await request(app.getHttpServer())
.get("/scenarios/00000000-0000-0000-0000-000000000001/export") .get("/scenarios/00000000-0000-0000-0000-000000000001/export")
@@ -572,17 +552,14 @@ describe("ScenarioController", () => {
{ {
sessionName: "s", sessionName: "s",
execCode: '{"keyId":"k","environmentName":"e"}', execCode: '{"keyId":"k","environmentName":"e"}',
validateCode: null,
}, },
{ {
sessionName: "s", sessionName: "s",
execCode: "return 1;", execCode: "return 1;",
validateCode: "return true;",
}, },
{ {
sessionName: "s", sessionName: "s",
execCode: '{"keyId":"k"}', execCode: '{"keyId":"k"}',
validateCode: null,
}, },
], ],
}; };
@@ -622,7 +599,6 @@ describe("ScenarioController", () => {
order: 0, order: 0,
sessionName: "rs", sessionName: "rs",
execCode: "return 42;", execCode: "return 42;",
validateCode: "return true;",
}); });
const exportRes = await request(app.getHttpServer()) const exportRes = await request(app.getHttpServer())
@@ -630,7 +606,7 @@ describe("ScenarioController", () => {
.expect(200); .expect(200);
const exported = yamlParse(exportRes.text) as { const exported = yamlParse(exportRes.text) as {
steps: Array<{ execCode: string; validateCode: string | null }>; steps: Array<{ execCode: string }>;
}; };
const importRes = await request(app.getHttpServer()) const importRes = await request(app.getHttpServer())
@@ -643,9 +619,6 @@ describe("ScenarioController", () => {
expect(importRes.body.steps[0].execCode).toBe( expect(importRes.body.steps[0].execCode).toBe(
exported.steps[0].execCode, exported.steps[0].execCode,
); );
expect(importRes.body.steps[0].validateCode).toBe(
exported.steps[0].validateCode,
);
}); });
it("imports with empty steps array", async () => { it("imports with empty steps array", async () => {