feat(import): add central yaml import for all exportable kinds
- add POST /import, an import page and the import_yaml mcp tool; items are validated up front and imported in dependency order - switch list page import buttons to the central import - write scenario exports as separate --- yaml documents per entity - give each test app its own files dir to avoid cross-suite races - use node: prefix for node built-in imports - bump version to 1.18.0
This commit is contained in:
1 parent
cf56840df9
commit
8935783371
39 files changed
+848
-130
No files matched your search
@@ -13,6 +13,7 @@ import { FileModule } from "./file/file.module";
|
||||
import { ScenarioFileEntity } from "./file/scenario-file.entity";
|
||||
import { ScenarioRunFileEntity } from "./file/scenario-run-file.entity";
|
||||
import { HealthModule } from "./health/health.module";
|
||||
import { ImportModule } from "./import/import.module";
|
||||
import { McpModule } from "./mcp/mcp.module";
|
||||
import { ScenarioCredentialEntity } from "./scenario/scenario-credential.entity";
|
||||
import { ScenarioRunLogEntity } from "./scenario/scenario-run-log.entity";
|
||||
@@ -66,6 +67,7 @@ import { SnippetModule } from "./snippet/snippet.module";
|
||||
ScenarioModule,
|
||||
SnippetModule,
|
||||
FileModule,
|
||||
ImportModule,
|
||||
McpModule,
|
||||
HealthModule,
|
||||
],
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
import { expect as playwrightExpect } from "@playwright/test";
|
||||
import { parse } from "acorn";
|
||||
import type { BrowserContext, Page } from "playwright";
|
||||
import { URL } from "url";
|
||||
import { URL } from "node:url";
|
||||
import { downloadFile as downloadFileImpl } from "../common/file-downloader";
|
||||
import { TraceLogger } from "../common/trace-logger";
|
||||
import type { EnvironmentData } from "../environment/environment.entity";
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
import type { NestExpressApplication } from "@nestjs/platform-express";
|
||||
|
||||
/** Content types accepted as raw YAML text (e.g. by `POST /import`). */
|
||||
export const YAML_CONTENT_TYPES = [
|
||||
"application/yaml",
|
||||
"application/x-yaml",
|
||||
"text/yaml",
|
||||
"text/x-yaml",
|
||||
];
|
||||
|
||||
/** Registers body parsers beyond Nest's default JSON/urlencoded ones. */
|
||||
export function configureBodyParsers(app: NestExpressApplication): void {
|
||||
app.useBodyParser("text", { type: YAML_CONTENT_TYPES, limit: "10mb" });
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import * as https from "https";
|
||||
import * as http from "http";
|
||||
import { URL } from "url";
|
||||
import * as https from "node:https";
|
||||
import * as http from "node:http";
|
||||
import { URL } from "node:url";
|
||||
|
||||
const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100 MB
|
||||
const TIMEOUT_MS = 30000; // 30 seconds
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { AsyncLocalStorage } from "async_hooks";
|
||||
import { AsyncLocalStorage } from "node:async_hooks";
|
||||
|
||||
export interface TraceStore {
|
||||
traceId: string;
|
||||
|
||||
@@ -2,9 +2,9 @@ import { Injectable, OnModuleInit } from "@nestjs/common";
|
||||
import { ConfigService } from "@nestjs/config";
|
||||
import { SchedulerRegistry } from "@nestjs/schedule";
|
||||
import { InjectRepository } from "@nestjs/typeorm";
|
||||
import * as crypto from "crypto";
|
||||
import * as fs from "fs/promises";
|
||||
import * as path from "path";
|
||||
import * as crypto from "node:crypto";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as path from "node:path";
|
||||
import { CronJob } from "cron";
|
||||
import { Repository } from "typeorm";
|
||||
import { TraceLogger } from "../common/trace-logger";
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { BadRequestException, Body, Controller, Post } from "@nestjs/common";
|
||||
import {
|
||||
ApiBody,
|
||||
ApiConsumes,
|
||||
ApiOperation,
|
||||
ApiResponse,
|
||||
ApiTags,
|
||||
} from "@nestjs/swagger";
|
||||
import { YAML_CONTENT_TYPES } from "../common/body-parsers";
|
||||
import { ImportService } from "./import.service";
|
||||
|
||||
@ApiTags("import")
|
||||
@Controller("import")
|
||||
export class ImportController {
|
||||
constructor(private readonly importService: ImportService) {}
|
||||
|
||||
@Post()
|
||||
@ApiOperation({
|
||||
summary:
|
||||
"Import any mix of environments, credentials, snippets and scenarios",
|
||||
description:
|
||||
"Send YAML (Content-Type application/yaml): `---`-separated documents, each holding one exported item or a list of them. " +
|
||||
"A JSON array of items (or a single item) is accepted too. Every item needs a `kind`. " +
|
||||
"All items are validated before anything is imported, then imported in dependency order: environments, credentials, snippets, scenarios.",
|
||||
})
|
||||
@ApiConsumes(...YAML_CONTENT_TYPES, "application/json")
|
||||
@ApiBody({ schema: { type: "string" } })
|
||||
@ApiResponse({ status: 201, description: "Imported items" })
|
||||
@ApiResponse({
|
||||
status: 400,
|
||||
description: "Invalid YAML or items; nothing was imported",
|
||||
})
|
||||
import(@Body() body: unknown) {
|
||||
if (typeof body === "string") {
|
||||
return this.importService.importYaml(body);
|
||||
}
|
||||
if (Array.isArray(body)) return this.importService.importItems(body);
|
||||
if (body && typeof body === "object" && Object.keys(body).length > 0) {
|
||||
return this.importService.importItems([body]);
|
||||
}
|
||||
throw new BadRequestException(
|
||||
"Send YAML with Content-Type application/yaml, or a JSON item or array of items",
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { CredentialModule } from "../credential/credential.module";
|
||||
import { EnvironmentModule } from "../environment/environment.module";
|
||||
import { ScenarioModule } from "../scenario/scenario.module";
|
||||
import { SnippetModule } from "../snippet/snippet.module";
|
||||
import { ImportController } from "./import.controller";
|
||||
import { ImportService } from "./import.service";
|
||||
|
||||
@Module({
|
||||
imports: [EnvironmentModule, CredentialModule, SnippetModule, ScenarioModule],
|
||||
controllers: [ImportController],
|
||||
providers: [ImportService],
|
||||
exports: [ImportService],
|
||||
})
|
||||
export class ImportModule {}
|
||||
@@ -0,0 +1,176 @@
|
||||
import { BadRequestException, Injectable } from "@nestjs/common";
|
||||
import { ClassConstructor, plainToInstance } from "class-transformer";
|
||||
import { validateSync, ValidationError } from "class-validator";
|
||||
import { parseAllDocuments } from "yaml";
|
||||
import { CredentialExportDto } from "../credential/dto/credential-export.dto";
|
||||
import { CredentialService } from "../credential/credential.service";
|
||||
import { EnvironmentExportDto } from "../environment/dto/environment-export.dto";
|
||||
import { EnvironmentService } from "../environment/environment.service";
|
||||
import { ScenarioExportDto } from "../scenario/dto/scenario-export.dto";
|
||||
import { ScenarioService } from "../scenario/scenario.service";
|
||||
import { SnippetExportDto } from "../snippet/dto/snippet-export.dto";
|
||||
import { SnippetService } from "../snippet/snippet.service";
|
||||
|
||||
export type ImportKind = "environment" | "credential" | "snippet" | "scenario";
|
||||
|
||||
/** Import order: later kinds may reference earlier ones (scenarios → environments/credentials). */
|
||||
const KIND_ORDER: ImportKind[] = [
|
||||
"environment",
|
||||
"credential",
|
||||
"snippet",
|
||||
"scenario",
|
||||
];
|
||||
|
||||
const DTO_BY_KIND: Record<ImportKind, ClassConstructor<object>> = {
|
||||
environment: EnvironmentExportDto,
|
||||
credential: CredentialExportDto,
|
||||
snippet: SnippetExportDto,
|
||||
scenario: ScenarioExportDto,
|
||||
};
|
||||
|
||||
export interface ImportedItem {
|
||||
kind: ImportKind;
|
||||
id: string;
|
||||
name: string;
|
||||
/** Version stored for versioned kinds (snippets, scenarios). */
|
||||
version?: number;
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
imported: ImportedItem[];
|
||||
}
|
||||
|
||||
function formatErrors(errors: ValidationError[], prefix = ""): string[] {
|
||||
return errors.flatMap((e) => {
|
||||
const field = prefix ? `${prefix}.${e.property}` : e.property;
|
||||
return [
|
||||
...Object.values(e.constraints ?? {}).map((msg) => `${field}: ${msg}`),
|
||||
...formatErrors(e.children ?? [], field),
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Imports any mix of exported environments, credentials, snippets and
|
||||
* scenarios in one go. Each item must carry its `kind`.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ImportService {
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private readonly credentialService: CredentialService,
|
||||
private readonly snippetService: SnippetService,
|
||||
private readonly scenarioService: ScenarioService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Parses a YAML stream: `---`-separated documents, each holding a single
|
||||
* item or a list of items. JSON is valid YAML, so JSON input works too.
|
||||
*/
|
||||
parseYaml(text: string): unknown[] {
|
||||
const docs = parseAllDocuments(text);
|
||||
const items: unknown[] = [];
|
||||
docs.forEach((doc, index) => {
|
||||
if (doc.errors.length > 0) {
|
||||
throw new BadRequestException(
|
||||
`YAML document ${index + 1}: ${doc.errors[0].message}`,
|
||||
);
|
||||
}
|
||||
const value: unknown = doc.toJS();
|
||||
if (value == null) return; // empty document, e.g. a trailing `---`
|
||||
items.push(...(Array.isArray(value) ? value : [value]));
|
||||
});
|
||||
return items;
|
||||
}
|
||||
|
||||
async importYaml(text: string): Promise<ImportResult> {
|
||||
return this.importItems(this.parseYaml(text));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates every item before importing any, then imports them in
|
||||
* dependency order (environments, credentials, snippets, scenarios).
|
||||
*/
|
||||
async importItems(items: unknown[]): Promise<ImportResult> {
|
||||
if (items.length === 0) {
|
||||
throw new BadRequestException("Nothing to import");
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
const validated: { kind: ImportKind; dto: object }[] = [];
|
||||
items.forEach((item, index) => {
|
||||
const label = `Item ${index + 1}`;
|
||||
if (item == null || typeof item !== "object" || Array.isArray(item)) {
|
||||
errors.push(`${label}: must be an object`);
|
||||
return;
|
||||
}
|
||||
const kind = (item as { kind?: unknown }).kind;
|
||||
if (typeof kind !== "string" || !(kind in DTO_BY_KIND)) {
|
||||
errors.push(
|
||||
`${label}: kind must be one of ${KIND_ORDER.join(", ")} (got ${JSON.stringify(kind)})`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const dto = plainToInstance(DTO_BY_KIND[kind as ImportKind], item);
|
||||
const itemErrors = formatErrors(validateSync(dto, { whitelist: true }));
|
||||
if (itemErrors.length > 0) {
|
||||
errors.push(`${label} (${kind}): ${itemErrors.join("; ")}`);
|
||||
return;
|
||||
}
|
||||
validated.push({ kind: kind as ImportKind, dto });
|
||||
});
|
||||
if (errors.length > 0) {
|
||||
throw new BadRequestException(errors);
|
||||
}
|
||||
|
||||
const imported: ImportedItem[] = [];
|
||||
for (const kind of KIND_ORDER) {
|
||||
for (const item of validated.filter((v) => v.kind === kind)) {
|
||||
imported.push(await this.importOne(kind, item.dto));
|
||||
}
|
||||
}
|
||||
return { imported };
|
||||
}
|
||||
|
||||
private async importOne(
|
||||
kind: ImportKind,
|
||||
dto: object,
|
||||
): Promise<ImportedItem> {
|
||||
switch (kind) {
|
||||
case "environment": {
|
||||
const env = await this.environmentService.importEnvironment(
|
||||
dto as EnvironmentExportDto,
|
||||
);
|
||||
return { kind, id: env.id, name: env.name };
|
||||
}
|
||||
case "credential": {
|
||||
const cred = await this.credentialService.importCredential(
|
||||
dto as CredentialExportDto,
|
||||
);
|
||||
return { kind, id: cred.id, name: cred.name };
|
||||
}
|
||||
case "snippet": {
|
||||
const snippet = await this.snippetService.importSnippet(
|
||||
dto as SnippetExportDto,
|
||||
);
|
||||
return {
|
||||
kind,
|
||||
id: snippet.id,
|
||||
name: snippet.alias,
|
||||
version: snippet.version,
|
||||
};
|
||||
}
|
||||
case "scenario": {
|
||||
const scenario = await this.scenarioService.importScenario(
|
||||
dto as ScenarioExportDto,
|
||||
);
|
||||
return {
|
||||
kind,
|
||||
id: scenario.id,
|
||||
name: scenario.name,
|
||||
version: scenario.version,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
Injectable,
|
||||
NestInterceptor,
|
||||
} from "@nestjs/common";
|
||||
import * as crypto from "crypto";
|
||||
import * as crypto from "node:crypto";
|
||||
import type { Request } from "express";
|
||||
import { Observable } from "rxjs";
|
||||
import { traceStorage } from "../common/trace-context";
|
||||
|
||||
+4
-1
@@ -1,9 +1,11 @@
|
||||
import { RequestMethod, ValidationPipe } from "@nestjs/common";
|
||||
import { ConfigService } from "@nestjs/config";
|
||||
import { NestFactory } from "@nestjs/core";
|
||||
import type { NestExpressApplication } from "@nestjs/platform-express";
|
||||
import { DocumentBuilder, SwaggerModule } from "@nestjs/swagger";
|
||||
import { name as pkgName, version as pkgVersion } from "../package.json";
|
||||
import { AppModule } from "./app.module";
|
||||
import { configureBodyParsers } from "./common/body-parsers";
|
||||
import { TraceLogger } from "./common/trace-logger";
|
||||
import { AppConfig } from "./config/app.config";
|
||||
import { HttpExceptionFilter } from "./filters/http-exception.filter";
|
||||
@@ -28,7 +30,7 @@ process.on("unhandledRejection", (reason) => {
|
||||
async function bootstrap() {
|
||||
const API_PREFIX = "api/v1";
|
||||
const logger = new TraceLogger("Bootstrap");
|
||||
const app = await NestFactory.create(AppModule, {
|
||||
const app = await NestFactory.create<NestExpressApplication>(AppModule, {
|
||||
logger: new TraceLogger("Bootstrap", { timestamp: true }),
|
||||
});
|
||||
|
||||
@@ -40,6 +42,7 @@ async function bootstrap() {
|
||||
],
|
||||
});
|
||||
|
||||
configureBodyParsers(app);
|
||||
app.useGlobalPipes(new ValidationPipe({ transform: true }));
|
||||
app.useGlobalFilters(new HttpExceptionFilter());
|
||||
app.useGlobalInterceptors(new TraceInterceptor(), new LoggingInterceptor());
|
||||
|
||||
@@ -6,6 +6,7 @@ import { EnvironmentModule } from "../environment/environment.module";
|
||||
import { ScenarioModule } from "../scenario/scenario.module";
|
||||
import { SessionModule } from "../session/session.module";
|
||||
import { SnippetModule } from "../snippet/snippet.module";
|
||||
import { ImportModule } from "../import/import.module";
|
||||
import { McpController } from "./mcp.controller";
|
||||
import { McpService } from "./mcp.service";
|
||||
|
||||
@@ -18,6 +19,7 @@ import { McpService } from "./mcp.service";
|
||||
CodeExecutorModule,
|
||||
ScenarioModule,
|
||||
SnippetModule,
|
||||
ImportModule,
|
||||
],
|
||||
controllers: [McpController],
|
||||
providers: [McpService],
|
||||
|
||||
@@ -14,6 +14,7 @@ import { CodeExecutorService } from "../code-executor/code-executor.service";
|
||||
import { ScenarioService } from "../scenario/scenario.service";
|
||||
import { E2eExportService } from "../scenario/e2e-export.service";
|
||||
import { SnippetService } from "../snippet/snippet.service";
|
||||
import { ImportService } from "../import/import.service";
|
||||
|
||||
import pkg from "../../package.json";
|
||||
|
||||
@@ -29,6 +30,7 @@ export class McpService {
|
||||
private readonly scenarioService: ScenarioService,
|
||||
private readonly e2eExportService: E2eExportService,
|
||||
private readonly snippetService: SnippetService,
|
||||
private readonly importService: ImportService,
|
||||
) {}
|
||||
|
||||
private createServer(): McpServer {
|
||||
@@ -1307,6 +1309,35 @@ export class McpService {
|
||||
},
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"import_yaml",
|
||||
{
|
||||
description:
|
||||
"Import any mix of exported environments, credentials, snippets and scenarios from YAML: `---`-separated documents, each holding one item or a list of items, each with a `kind`. All items are validated before anything is imported; versioned items (snippets, scenarios) must include `version`",
|
||||
inputSchema: {
|
||||
yaml: z.string().describe("YAML text, e.g. the content of an export file"),
|
||||
},
|
||||
},
|
||||
async ({ yaml }) => {
|
||||
try {
|
||||
const result = await this.importService.importYaml(yaml);
|
||||
return {
|
||||
content: [{ type: "text" as const, text: JSON.stringify(result) }],
|
||||
};
|
||||
} catch (err) {
|
||||
const response = (err as { getResponse?: () => unknown }).getResponse?.();
|
||||
const message =
|
||||
response && typeof response === "object" && "message" in response
|
||||
? JSON.stringify((response as { message: unknown }).message)
|
||||
: (err as Error).message;
|
||||
return {
|
||||
isError: true,
|
||||
content: [{ type: "text" as const, text: message }],
|
||||
};
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
server.registerTool(
|
||||
"export_e2e_test",
|
||||
{
|
||||
|
||||
@@ -148,7 +148,7 @@ export class E2eExportService {
|
||||
const timeoutMs = (scenarioTimeoutSeconds ?? DEFAULT_SCENARIO_TIMEOUT_SEC) * 1000;
|
||||
return `import { defineConfig } from "@playwright/test";
|
||||
import * as dotenv from "dotenv";
|
||||
import * as path from "path";
|
||||
import * as path from "node:path";
|
||||
|
||||
dotenv.config({ path: path.resolve(__dirname, ".env") });
|
||||
|
||||
@@ -235,13 +235,13 @@ the scenario are bundled under \`files/\` and served locally by
|
||||
|
||||
return `import { test, expect } from "@playwright/test";
|
||||
import type { BrowserContext, Page } from "@playwright/test";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
import * as crypto from "crypto";
|
||||
import * as http from "http";
|
||||
import * as https from "https";
|
||||
import { URL } from "url";
|
||||
import { randomUUID } from "crypto";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import * as crypto from "node:crypto";
|
||||
import * as http from "node:http";
|
||||
import * as https from "node:https";
|
||||
import { URL } from "node:url";
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
// Credential data resolved from this scenario's aliases at export time.
|
||||
// Stored in credentials.json (gitignored) - contains sensitive values.
|
||||
|
||||
@@ -2,10 +2,10 @@ import { Injectable, OnModuleInit } from "@nestjs/common";
|
||||
import { ConfigService } from "@nestjs/config";
|
||||
import { Interval } from "@nestjs/schedule";
|
||||
import { InjectRepository } from "@nestjs/typeorm";
|
||||
import * as crypto from "crypto";
|
||||
import * as fs from "fs/promises";
|
||||
import * as os from "os";
|
||||
import * as path from "path";
|
||||
import * as crypto from "node:crypto";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { Effect } from "effect";
|
||||
import type { Browser, BrowserContext, Page } from "playwright";
|
||||
import { chromium } from "playwright";
|
||||
|
||||
@@ -150,20 +150,20 @@ export class ScenarioController {
|
||||
version,
|
||||
});
|
||||
res.setHeader("Content-Type", "application/yaml; charset=utf-8");
|
||||
// Serialize as YAML stream: each entity separated by a comment header
|
||||
// Serialize as a YAML stream: one `---` document per entity, each with a
|
||||
// comment naming it. POST /import reads this format back.
|
||||
const kindLabels: Record<string, string> = {
|
||||
environment: "Environment",
|
||||
credential: "Credential",
|
||||
scenario: "Scenario",
|
||||
};
|
||||
const parts = (entities as ExportEntity[]).map((entity) => {
|
||||
const label = kindLabels[entity.kind ?? ""] ?? entity.kind ?? "Entity";
|
||||
const comment = `# --- ${label}: ${
|
||||
entity.name ?? entity.id ?? ""
|
||||
} ---`;
|
||||
return `${comment}\n${yamlStringify(entity)}`;
|
||||
});
|
||||
return parts.join("\n");
|
||||
return (entities as ExportEntity[])
|
||||
.map((entity) => {
|
||||
const label = kindLabels[entity.kind ?? ""] ?? entity.kind ?? "Entity";
|
||||
const comment = `# ${label}: ${entity.name ?? entity.id ?? ""}`;
|
||||
return `---\n${comment}\n${yamlStringify(entity)}`;
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
@Get(":id/export-e2e")
|
||||
|
||||
@@ -7,8 +7,8 @@ import {
|
||||
} from "@nestjs/common";
|
||||
import { ConfigService } from "@nestjs/config";
|
||||
import { InjectRepository } from "@nestjs/typeorm";
|
||||
import * as fs from "fs/promises";
|
||||
import * as path from "path";
|
||||
import * as fs from "node:fs/promises";
|
||||
import * as path from "node:path";
|
||||
import { Like, Repository } from "typeorm";
|
||||
import {
|
||||
PaginatedResult,
|
||||
|
||||
Reference in new issue
Block a user