feat(files): add file storage subsystem with scenario and run artifact support
- add FileEntity, ScenarioFileEntity, ScenarioRunFileEntity with UUID-sharded disk tree - FileStorageService: write/sha256/expiry, paginated listing, dynamic cleanup cron via SchedulerRegistry - expose getScenarioFiles() and downloadFile() on ScriptContext for use in exec code - wire scenarioId, runId, fileService into ExecContextBuilder and scenario scheduler - add file upload and listing endpoints to ScenarioController - add FILES_DIR, FILE_RUN_EXPIRATION_DAYS, FILE_CLEANUP_CRON to AppConfig
This commit is contained in:
@@ -8,6 +8,10 @@ import { CredentialEntity } from "./credential/credential.entity";
|
||||
import { CredentialModule } from "./credential/credential.module";
|
||||
import { EnvironmentEntity } from "./environment/environment.entity";
|
||||
import { EnvironmentModule } from "./environment/environment.module";
|
||||
import { FileEntity } from "./file/file.entity";
|
||||
import { FileModule } from "./file/file.module";
|
||||
import { ScenarioFileEntity } from "./file/scenario-file.entity";
|
||||
import { ScenarioRunFileEntity } from "./file/scenario-run-file.entity";
|
||||
import { HealthModule } from "./health/health.module";
|
||||
import { McpModule } from "./mcp/mcp.module";
|
||||
import { ScenarioCredentialEntity } from "./scenario/scenario-credential.entity";
|
||||
@@ -45,6 +49,9 @@ import { SnippetModule } from "./snippet/snippet.module";
|
||||
ScenarioRunLogEntity,
|
||||
ScenarioCredentialEntity,
|
||||
SnippetEntity,
|
||||
FileEntity,
|
||||
ScenarioFileEntity,
|
||||
ScenarioRunFileEntity,
|
||||
],
|
||||
synchronize: true,
|
||||
}),
|
||||
@@ -54,6 +61,7 @@ import { SnippetModule } from "./snippet/snippet.module";
|
||||
CredentialModule,
|
||||
ScenarioModule,
|
||||
SnippetModule,
|
||||
FileModule,
|
||||
McpModule,
|
||||
HealthModule,
|
||||
],
|
||||
|
||||
@@ -6,8 +6,11 @@ import {
|
||||
import { expect as playwrightExpect } from "@playwright/test";
|
||||
import { parse } from "acorn";
|
||||
import type { BrowserContext, Page } from "playwright";
|
||||
import { URL } from "url";
|
||||
import { downloadFile as downloadFileImpl } from "../common/file-downloader";
|
||||
import { TraceLogger } from "../common/trace-logger";
|
||||
import type { EnvironmentData } from "../environment/environment.entity";
|
||||
import type { FileStorageService } from "../file/file-storage.service";
|
||||
import type { DomNode } from "./dom-helpers";
|
||||
import { dumpDom } from "./dom-helpers";
|
||||
|
||||
@@ -41,6 +44,16 @@ export interface ScriptContext {
|
||||
error: (...args: unknown[]) => void;
|
||||
/** Runs a named snippet with the same context, plus any extra positional args. */
|
||||
runSnippet: (alias: string, ...args: unknown[]) => Promise<unknown>;
|
||||
/** Returns metadata + absolute disk path for files attached to the current scenario. */
|
||||
getScenarioFiles: (opts?: { limit?: number; offset?: number }) => Promise<{
|
||||
items: Array<{ id: string; name: string; mimeType: string; size: number; sha256: string; path: string }>;
|
||||
total: number;
|
||||
}>;
|
||||
/** Downloads a file and creates a run artifact. */
|
||||
downloadFile: (
|
||||
url: string,
|
||||
opts?: { method?: string; headers?: Record<string, string>; body?: string; filename?: string },
|
||||
) => Promise<{ id: string; name: string; mimeType: string; size: number; sha256: string; path: string }>;
|
||||
}
|
||||
|
||||
export interface ExecContext {
|
||||
@@ -53,6 +66,9 @@ export interface ExecContext {
|
||||
environment?: EnvironmentData | null;
|
||||
snippets?: Record<string, string> | null;
|
||||
result?: unknown;
|
||||
scenarioId?: string;
|
||||
runId?: string;
|
||||
fileService?: FileStorageService;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
@@ -91,6 +107,9 @@ export class CodeExecutorService {
|
||||
environment,
|
||||
snippets,
|
||||
result,
|
||||
scenarioId,
|
||||
runId,
|
||||
fileService,
|
||||
} = ctx;
|
||||
const scriptLog: ScriptLogger =
|
||||
log ?? ((level, msg) => this.logger[level](msg));
|
||||
@@ -158,6 +177,58 @@ export class CodeExecutorService {
|
||||
);
|
||||
return snippetFn(scriptContext, fakeConsole, args, playwrightExpect);
|
||||
},
|
||||
getScenarioFiles: async (opts?: { limit?: number; offset?: number }) => {
|
||||
if (!fileService || !scenarioId) {
|
||||
throw new Error("File service not available in this context");
|
||||
}
|
||||
const result = await fileService.listScenarioFiles(
|
||||
scenarioId,
|
||||
opts?.limit,
|
||||
opts?.offset,
|
||||
);
|
||||
return {
|
||||
items: result.items.map((item) => ({
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
mimeType: item.mimeType,
|
||||
size: item.size,
|
||||
sha256: item.sha256,
|
||||
path: item.path,
|
||||
})),
|
||||
total: result.total,
|
||||
};
|
||||
},
|
||||
downloadFile: async (
|
||||
url: string,
|
||||
opts?: { method?: string; headers?: Record<string, string>; body?: string; filename?: string },
|
||||
) => {
|
||||
if (!fileService || !scenarioId || !runId) {
|
||||
throw new Error("File service or run context not available");
|
||||
}
|
||||
// Download file
|
||||
const buffer = await downloadFileImpl(url, opts);
|
||||
|
||||
// Determine filename and mime type
|
||||
const filename = opts?.filename ?? new URL(url).pathname.split("/").pop() ?? "download";
|
||||
const mimeType = this.getMimeType(filename);
|
||||
|
||||
// Create run artifact with mapping
|
||||
const savedFile = await fileService.createAndSaveRunArtifact(
|
||||
runId,
|
||||
buffer,
|
||||
filename,
|
||||
mimeType,
|
||||
);
|
||||
|
||||
return {
|
||||
id: savedFile.id,
|
||||
name: savedFile.originalName,
|
||||
mimeType: savedFile.mimeType,
|
||||
size: savedFile.size,
|
||||
sha256: savedFile.sha256,
|
||||
path: fileService.getAbsolutePath(savedFile.filePath),
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
// `console`, `result`, and `expect` are injected as named parameters so
|
||||
@@ -184,4 +255,26 @@ export class CodeExecutorService {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Infers MIME type from filename.
|
||||
*/
|
||||
private getMimeType(filename: string): string {
|
||||
const ext = filename.split(".").pop()?.toLowerCase() ?? "";
|
||||
const mimeTypes: Record<string, string> = {
|
||||
pdf: "application/pdf",
|
||||
doc: "application/msword",
|
||||
docx: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
xls: "application/vnd.ms-excel",
|
||||
xlsx: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
jpg: "image/jpeg",
|
||||
jpeg: "image/jpeg",
|
||||
png: "image/png",
|
||||
gif: "image/gif",
|
||||
csv: "text/csv",
|
||||
txt: "text/plain",
|
||||
zip: "application/zip",
|
||||
};
|
||||
return mimeTypes[ext] ?? "application/octet-stream";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { BrowserContext, Page } from "playwright";
|
||||
import type { EnvironmentData } from "../environment/environment.entity";
|
||||
import type { FileStorageService } from "../file/file-storage.service";
|
||||
import type { ExecContext, ScriptLogger } from "./code-executor.service";
|
||||
|
||||
export class ExecContextBuilder {
|
||||
@@ -50,6 +51,21 @@ export class ExecContextBuilder {
|
||||
return this;
|
||||
}
|
||||
|
||||
scenarioId(scenarioId: string): this {
|
||||
this.ctx.scenarioId = scenarioId;
|
||||
return this;
|
||||
}
|
||||
|
||||
runId(runId: string): this {
|
||||
this.ctx.runId = runId;
|
||||
return this;
|
||||
}
|
||||
|
||||
fileService(fileService: FileStorageService): this {
|
||||
this.ctx.fileService = fileService;
|
||||
return this;
|
||||
}
|
||||
|
||||
build(): ExecContext {
|
||||
if (!this.ctx.page) throw new Error("ExecContextBuilder: page is required");
|
||||
if (!this.ctx.browser)
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import * as https from "https";
|
||||
import * as http from "http";
|
||||
import { URL } from "url";
|
||||
|
||||
const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100 MB
|
||||
const TIMEOUT_MS = 30000; // 30 seconds
|
||||
|
||||
export async function downloadFile(
|
||||
urlStr: string,
|
||||
options?: {
|
||||
method?: string;
|
||||
headers?: Record<string, string>;
|
||||
body?: string;
|
||||
},
|
||||
): Promise<Buffer> {
|
||||
const url = new URL(urlStr);
|
||||
const protocol = url.protocol === "https:" ? https : http;
|
||||
const method = options?.method ?? "GET";
|
||||
const headers = options?.headers ?? {};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = protocol.request(
|
||||
{
|
||||
hostname: url.hostname,
|
||||
port: url.port,
|
||||
path: url.pathname + url.search,
|
||||
method,
|
||||
headers,
|
||||
timeout: TIMEOUT_MS,
|
||||
},
|
||||
(res) => {
|
||||
// Follow redirects
|
||||
if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
||||
downloadFile(res.headers.location, options).then(resolve).catch(reject);
|
||||
return;
|
||||
}
|
||||
|
||||
if (res.statusCode && res.statusCode !== 200) {
|
||||
reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks: Buffer[] = [];
|
||||
let totalSize = 0;
|
||||
|
||||
res.on("data", (chunk: Buffer) => {
|
||||
totalSize += chunk.length;
|
||||
if (totalSize > MAX_FILE_SIZE) {
|
||||
req.destroy();
|
||||
reject(new Error(`File exceeds maximum size of ${MAX_FILE_SIZE} bytes`));
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
|
||||
res.on("end", () => {
|
||||
resolve(Buffer.concat(chunks));
|
||||
});
|
||||
|
||||
res.on("error", reject);
|
||||
},
|
||||
);
|
||||
|
||||
req.on("timeout", () => {
|
||||
req.destroy();
|
||||
reject(new Error("Download timeout"));
|
||||
});
|
||||
|
||||
req.on("error", reject);
|
||||
|
||||
if (options?.body && (method === "POST" || method === "PUT")) {
|
||||
req.write(options.body);
|
||||
}
|
||||
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
@@ -31,6 +31,16 @@ export class AppConfig {
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
SESSION_DELETE_CLOSED_DAYS: number = 7;
|
||||
|
||||
@IsString()
|
||||
FILES_DIR: string = "data/files";
|
||||
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
FILE_RUN_EXPIRATION_DAYS: number = 7;
|
||||
|
||||
@IsString()
|
||||
FILE_CLEANUP_CRON: string = "0 3 * * *";
|
||||
}
|
||||
|
||||
export function validateAppConfig(config: Record<string, unknown>): AppConfig {
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
import { Injectable, OnModuleInit } from "@nestjs/common";
|
||||
import { ConfigService } from "@nestjs/config";
|
||||
import { SchedulerRegistry } from "@nestjs/schedule";
|
||||
import { InjectRepository } from "@nestjs/typeorm";
|
||||
import * as crypto from "crypto";
|
||||
import * as fs from "fs/promises";
|
||||
import * as path from "path";
|
||||
import { CronJob } from "cron";
|
||||
import { Repository } from "typeorm";
|
||||
import { TraceLogger } from "../common/trace-logger";
|
||||
import { AppConfig } from "../config/app.config";
|
||||
import { FileEntity } from "./file.entity";
|
||||
import { ScenarioFileEntity } from "./scenario-file.entity";
|
||||
import { ScenarioRunFileEntity } from "./scenario-run-file.entity";
|
||||
|
||||
export interface FileMetadata {
|
||||
id: string;
|
||||
name: string;
|
||||
mimeType: string;
|
||||
size: number;
|
||||
sha256: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class FileStorageService implements OnModuleInit {
|
||||
private static readonly DEFAULT_PAGE_SIZE = 20;
|
||||
private readonly logger = new TraceLogger(FileStorageService.name);
|
||||
private readonly filesDir: string;
|
||||
private readonly fileRunExpirationDays: number;
|
||||
private readonly fileCleanupCron: string;
|
||||
|
||||
constructor(
|
||||
@InjectRepository(FileEntity)
|
||||
readonly fileRepo: Repository<FileEntity>,
|
||||
@InjectRepository(ScenarioFileEntity)
|
||||
private readonly scenarioFileRepo: Repository<ScenarioFileEntity>,
|
||||
@InjectRepository(ScenarioRunFileEntity)
|
||||
private readonly scenarioRunFileRepo: Repository<ScenarioRunFileEntity>,
|
||||
configService: ConfigService<AppConfig, true>,
|
||||
private readonly schedulerRegistry: SchedulerRegistry,
|
||||
) {
|
||||
this.filesDir = configService.get("FILES_DIR");
|
||||
this.fileRunExpirationDays = configService.get("FILE_RUN_EXPIRATION_DAYS");
|
||||
this.fileCleanupCron = configService.get("FILE_CLEANUP_CRON");
|
||||
}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
// Ensure the files directory exists
|
||||
try {
|
||||
await fs.mkdir(this.filesDir, { recursive: true });
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to create FILES_DIR ${this.filesDir}: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
// Register cleanup cron job dynamically so the schedule is runtime-configurable
|
||||
const job = new CronJob(this.fileCleanupCron, () => {
|
||||
void this.cleanupExpiredFiles();
|
||||
});
|
||||
this.schedulerRegistry.addCronJob("file-cleanup", job);
|
||||
job.start();
|
||||
this.logger.debug(`File cleanup cron registered: ${this.fileCleanupCron}`);
|
||||
|
||||
// Run cleanup at startup to catch any backlog
|
||||
await this.cleanupExpiredFiles();
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves a file to disk with sharded directory structure.
|
||||
* Returns the FileEntity (unsaved) with computed filePath.
|
||||
*/
|
||||
async saveFile(
|
||||
buffer: Buffer,
|
||||
originalName: string,
|
||||
mimeType: string,
|
||||
expiresAt?: Date,
|
||||
): Promise<FileEntity> {
|
||||
const fileId = crypto.randomUUID();
|
||||
const sha256 = crypto.createHash("sha256").update(buffer).digest("hex");
|
||||
const shardPath = this.getShardedPath(fileId);
|
||||
const fullPath = path.join(this.filesDir, shardPath);
|
||||
|
||||
// Create sharded directory
|
||||
const dirPath = path.dirname(fullPath);
|
||||
await fs.mkdir(dirPath, { recursive: true });
|
||||
|
||||
// Write file to disk and persist entity
|
||||
await fs.writeFile(fullPath, buffer);
|
||||
|
||||
const file = this.fileRepo.create({
|
||||
id: fileId,
|
||||
originalName,
|
||||
mimeType,
|
||||
size: buffer.length,
|
||||
sha256,
|
||||
filePath: shardPath,
|
||||
expiresAt: expiresAt ?? null,
|
||||
});
|
||||
|
||||
return this.fileRepo.save(file);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the sharded path for a file ID: xx/xxxx/uuid
|
||||
*/
|
||||
private getShardedPath(uuid: string): string {
|
||||
return `${uuid.substring(0, 2)}/${uuid.substring(0, 4)}/${uuid}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns absolute disk path for a file.
|
||||
*/
|
||||
getAbsolutePath(filePath: string): string {
|
||||
return path.join(this.filesDir, filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists scenario files with pagination.
|
||||
*/
|
||||
async listScenarioFiles(
|
||||
scenarioId: string,
|
||||
limit?: number,
|
||||
offset?: number,
|
||||
): Promise<{ items: FileMetadata[]; total: number }> {
|
||||
const query = this.scenarioFileRepo
|
||||
.createQueryBuilder("sf")
|
||||
.innerJoinAndSelect("sf.file", "f")
|
||||
.where("sf.scenarioId = :scenarioId", { scenarioId });
|
||||
|
||||
const total = await query.getCount();
|
||||
|
||||
const items = await query
|
||||
.orderBy("sf.createdAt", "DESC")
|
||||
.skip(offset ?? 0)
|
||||
.take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
|
||||
.getMany();
|
||||
return {
|
||||
items: items.map((sf) => this.mapFileEntityToMetadata(sf.file)),
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists run artifact files with pagination.
|
||||
*/
|
||||
async listRunFiles(
|
||||
runId: string,
|
||||
limit?: number,
|
||||
offset?: number,
|
||||
): Promise<{ items: FileMetadata[]; total: number }> {
|
||||
const query = this.scenarioRunFileRepo
|
||||
.createQueryBuilder("srf")
|
||||
.innerJoinAndSelect("srf.file", "f")
|
||||
.where("srf.runId = :runId", { runId });
|
||||
|
||||
const total = await query.getCount();
|
||||
|
||||
const items = await query
|
||||
.orderBy("srf.createdAt", "DESC")
|
||||
.skip(offset ?? 0)
|
||||
.take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
|
||||
.getMany();
|
||||
return {
|
||||
items: items.map((srf) => this.mapFileEntityToMetadata(srf.file)),
|
||||
total,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets a file entity by ID.
|
||||
*/
|
||||
async getFile(fileId: string): Promise<FileEntity | null> {
|
||||
return this.fileRepo.findOneBy({ id: fileId });
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes a file and checks for orphaned FileEntity rows.
|
||||
* If a FileEntity has no remaining mappings, it is deleted along with its physical file.
|
||||
*/
|
||||
async deleteFile(fileId: string): Promise<void> {
|
||||
const file = await this.fileRepo.findOneBy({ id: fileId });
|
||||
if (!file) return;
|
||||
|
||||
// Check if this file still has any mappings
|
||||
const scenarioCount = await this.scenarioFileRepo.countBy({ fileId });
|
||||
const runCount = await this.scenarioRunFileRepo.countBy({ fileId });
|
||||
|
||||
// If no mappings remain, delete the file and its physical copy
|
||||
if (scenarioCount === 0 && runCount === 0) {
|
||||
const fullPath = this.getAbsolutePath(file.filePath);
|
||||
try {
|
||||
await fs.unlink(fullPath);
|
||||
} catch (err) {
|
||||
this.logger.warn(`Failed to delete physical file ${fullPath}: ${(err as Error).message}`);
|
||||
}
|
||||
|
||||
await this.fileRepo.delete(fileId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a run artifact file entry with auto-expiration.
|
||||
*/
|
||||
async createRunArtifact(buffer: Buffer, originalName: string, mimeType: string): Promise<FileEntity> {
|
||||
const now = new Date();
|
||||
const expiresAt = new Date(now.getTime() + this.fileRunExpirationDays * 24 * 60 * 60 * 1000);
|
||||
return this.saveFile(buffer, originalName, mimeType, expiresAt);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a run artifact with file and mapping in the database.
|
||||
*/
|
||||
async createAndSaveRunArtifact(
|
||||
runId: string,
|
||||
buffer: Buffer,
|
||||
originalName: string,
|
||||
mimeType: string,
|
||||
): Promise<FileEntity> {
|
||||
const savedFile = await this.createRunArtifact(buffer, originalName, mimeType);
|
||||
|
||||
// Create run file mapping
|
||||
const runFile = this.scenarioRunFileRepo.create({
|
||||
runId,
|
||||
fileId: savedFile.id,
|
||||
});
|
||||
await this.scenarioRunFileRepo.save(runFile);
|
||||
|
||||
return savedFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleanup job: deletes expired FileEntity rows and orphaned physical files.
|
||||
* Runs via @Cron scheduler and at module init.
|
||||
*/
|
||||
async cleanupExpiredFiles(): Promise<void> {
|
||||
const now = new Date();
|
||||
const expiredFiles = await this.fileRepo
|
||||
.createQueryBuilder("file")
|
||||
.where("file.expiresAt IS NOT NULL")
|
||||
.andWhere("file.expiresAt <= :now", { now })
|
||||
.setParameters({ now })
|
||||
.getMany();
|
||||
|
||||
if (expiredFiles.length === 0) return;
|
||||
|
||||
this.logger.debug(`Cleaning up ${expiredFiles.length} expired files`);
|
||||
|
||||
for (const file of expiredFiles) {
|
||||
try {
|
||||
// Delete associated mapping rows first (cascade will be handled, but we delete manually to ensure orphan check works)
|
||||
await this.scenarioFileRepo.delete({ fileId: file.id });
|
||||
await this.scenarioRunFileRepo.delete({ fileId: file.id });
|
||||
|
||||
// Delete the file entity and physical file
|
||||
await this.deleteFile(file.id);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to clean up expired file ${file.id}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps FileEntity to FileMetadata with absolute path.
|
||||
*/
|
||||
private mapFileEntityToMetadata(file: FileEntity): FileMetadata {
|
||||
return {
|
||||
id: file.id,
|
||||
name: file.originalName,
|
||||
mimeType: file.mimeType,
|
||||
size: file.size,
|
||||
sha256: file.sha256,
|
||||
path: this.getAbsolutePath(file.filePath),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { Column, CreateDateColumn, Entity, PrimaryGeneratedColumn } from "typeorm";
|
||||
|
||||
@Entity("files")
|
||||
export class FileEntity {
|
||||
@PrimaryGeneratedColumn("uuid")
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
originalName: string;
|
||||
|
||||
@Column()
|
||||
mimeType: string;
|
||||
|
||||
@Column({ type: "integer" })
|
||||
size: number;
|
||||
|
||||
@Column()
|
||||
sha256: string;
|
||||
|
||||
@Column()
|
||||
filePath: string;
|
||||
|
||||
@Column({ type: "datetime", nullable: true })
|
||||
expiresAt: Date | null;
|
||||
|
||||
@CreateDateColumn()
|
||||
createdAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { Module } from "@nestjs/common";
|
||||
import { TypeOrmModule } from "@nestjs/typeorm";
|
||||
import { FileEntity } from "./file.entity";
|
||||
import { FileStorageService } from "./file-storage.service";
|
||||
import { ScenarioFileEntity } from "./scenario-file.entity";
|
||||
import { ScenarioRunFileEntity } from "./scenario-run-file.entity";
|
||||
|
||||
@Module({
|
||||
imports: [TypeOrmModule.forFeature([FileEntity, ScenarioFileEntity, ScenarioRunFileEntity])],
|
||||
providers: [FileStorageService],
|
||||
exports: [FileStorageService],
|
||||
})
|
||||
export class FileModule {}
|
||||
@@ -0,0 +1,33 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
} from "typeorm";
|
||||
import { ScenarioEntity } from "../scenario/scenario.entity";
|
||||
import { FileEntity } from "./file.entity";
|
||||
|
||||
@Entity("scenario_files")
|
||||
export class ScenarioFileEntity {
|
||||
@PrimaryGeneratedColumn("uuid")
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
scenarioId: string;
|
||||
|
||||
@Column()
|
||||
fileId: string;
|
||||
|
||||
@ManyToOne(() => ScenarioEntity, { onDelete: "CASCADE" })
|
||||
@JoinColumn({ name: "scenarioId" })
|
||||
scenario: ScenarioEntity;
|
||||
|
||||
@ManyToOne(() => FileEntity, { onDelete: "CASCADE" })
|
||||
@JoinColumn({ name: "fileId" })
|
||||
file: FileEntity;
|
||||
|
||||
@CreateDateColumn()
|
||||
createdAt: Date;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import {
|
||||
Column,
|
||||
CreateDateColumn,
|
||||
Entity,
|
||||
JoinColumn,
|
||||
ManyToOne,
|
||||
PrimaryGeneratedColumn,
|
||||
} from "typeorm";
|
||||
import { ScenarioRunEntity } from "../scenario/scenario-run.entity";
|
||||
import { FileEntity } from "./file.entity";
|
||||
|
||||
@Entity("scenario_run_files")
|
||||
export class ScenarioRunFileEntity {
|
||||
@PrimaryGeneratedColumn("uuid")
|
||||
id: string;
|
||||
|
||||
@Column()
|
||||
runId: string;
|
||||
|
||||
@Column()
|
||||
fileId: string;
|
||||
|
||||
@ManyToOne(() => ScenarioRunEntity, { onDelete: "CASCADE" })
|
||||
@JoinColumn({ name: "runId" })
|
||||
run: ScenarioRunEntity;
|
||||
|
||||
@ManyToOne(() => FileEntity, { onDelete: "CASCADE" })
|
||||
@JoinColumn({ name: "fileId" })
|
||||
file: FileEntity;
|
||||
|
||||
@CreateDateColumn()
|
||||
createdAt: Date;
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import { Effect } from "effect";
|
||||
import type { Browser, BrowserContext, Page } from "playwright";
|
||||
import { chromium } from "playwright";
|
||||
import { Repository } from "typeorm";
|
||||
import { FileStorageService } from "../file/file-storage.service";
|
||||
import type { ScriptLogger } from "../code-executor/code-executor.service";
|
||||
import { CodeExecutorService } from "../code-executor/code-executor.service";
|
||||
import { ExecContextBuilder } from "../code-executor/exec-context.builder";
|
||||
@@ -43,6 +44,8 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
private readonly runEnvironments = new Map<string, EnvironmentData>();
|
||||
// Cache scenario-level timeout (seconds) per run
|
||||
private readonly runScenarioTimeouts = new Map<string, number | null>();
|
||||
// Cache scenarioId per run
|
||||
private readonly runScenarioIds = new Map<string, string>();
|
||||
|
||||
private static readonly DEFAULT_SCENARIO_TIMEOUT_SEC = 600;
|
||||
private static readonly DEFAULT_STEP_TIMEOUT_SEC = 60;
|
||||
@@ -62,6 +65,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
private readonly snippetService: SnippetService,
|
||||
private readonly sessionService: SessionService,
|
||||
private readonly sessionContextService: SessionContextService,
|
||||
private readonly fileStorageService: FileStorageService,
|
||||
) {}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
@@ -198,6 +202,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
.findOne(run.scenarioId)
|
||||
.catch(() => null);
|
||||
this.runScenarioTimeouts.set(run.id, scenario?.timeoutSeconds ?? null);
|
||||
this.runScenarioIds.set(run.id, run.scenarioId);
|
||||
const traceId = crypto.randomUUID();
|
||||
void traceStorage.run({ traceId }, () =>
|
||||
this.processRunToCompletion(run.id),
|
||||
@@ -210,6 +215,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
this.runSnippets.delete(run.id);
|
||||
this.runEnvironments.delete(run.id);
|
||||
this.runScenarioTimeouts.delete(run.id);
|
||||
this.runScenarioIds.delete(run.id);
|
||||
this.logger.error(
|
||||
`Run #${run.id}: setup failed — ${String(err)}`,
|
||||
);
|
||||
@@ -272,6 +278,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
this.runSnippets.delete(runId);
|
||||
this.runEnvironments.delete(runId);
|
||||
this.runScenarioTimeouts.delete(runId);
|
||||
this.runScenarioIds.delete(runId);
|
||||
await this.maybePreserveSession(runId);
|
||||
}
|
||||
}
|
||||
@@ -349,6 +356,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
const creds = this.runCredentials.get(stepRun.runId);
|
||||
const snips = this.runSnippets.get(stepRun.runId);
|
||||
const env = this.runEnvironments.get(stepRun.runId);
|
||||
const scenarioId = this.runScenarioIds.get(stepRun.runId);
|
||||
const execCtx = new ExecContextBuilder()
|
||||
.page(page)
|
||||
.browser(context)
|
||||
@@ -358,6 +366,9 @@ export class ScenarioSchedulerService implements OnModuleInit {
|
||||
.credentials(creds)
|
||||
.environment(env)
|
||||
.snippets(snips)
|
||||
.scenarioId(scenarioId)
|
||||
.runId(stepRun.runId)
|
||||
.fileService(this.fileStorageService)
|
||||
.build();
|
||||
|
||||
const scenarioTimeoutSec = this.runScenarioTimeouts.get(stepRun.runId) ?? null;
|
||||
|
||||
@@ -10,9 +10,13 @@ import {
|
||||
Post,
|
||||
Query,
|
||||
Res,
|
||||
UseInterceptors,
|
||||
UploadedFile,
|
||||
} from "@nestjs/common";
|
||||
import { FileInterceptor } from "@nestjs/platform-express";
|
||||
import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger";
|
||||
import { Response } from "express";
|
||||
import type { File as MulterFile } from "multer";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { AddScenarioCredentialDto } from "./dto/add-scenario-credential.dto";
|
||||
import { CreateScenarioRunDto } from "./dto/create-scenario-run.dto";
|
||||
@@ -270,4 +274,78 @@ export class ScenarioController {
|
||||
) {
|
||||
return this.scenarioService.waitForRun(id, runId);
|
||||
}
|
||||
|
||||
// ── Files ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@Post(":id/files")
|
||||
@UseInterceptors(FileInterceptor("file"))
|
||||
@ApiOperation({ summary: "Upload a file to a scenario" })
|
||||
@ApiResponse({ status: 201, description: "File uploaded" })
|
||||
@ApiResponse({ status: 404, description: "Scenario not found" })
|
||||
uploadFile(
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
@UploadedFile() file: MulterFile,
|
||||
@Body() body: { expiresAt?: string },
|
||||
) {
|
||||
return this.scenarioService.uploadFile(id, file, body.expiresAt);
|
||||
}
|
||||
|
||||
@Get(":id/files")
|
||||
@ApiOperation({ summary: "List files attached to a scenario" })
|
||||
@ApiResponse({ status: 200 })
|
||||
@ApiResponse({ status: 404, description: "Scenario not found" })
|
||||
listScenarioFiles(
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
@Query("limit") limit?: string,
|
||||
@Query("offset") offset?: string,
|
||||
) {
|
||||
return this.scenarioService.listScenarioFiles(
|
||||
id,
|
||||
limit ? parseInt(limit, 10) : undefined,
|
||||
offset ? parseInt(offset, 10) : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
@Get(":id/files/:fileId/content")
|
||||
@ApiOperation({ summary: "Download file content" })
|
||||
@ApiResponse({ status: 200, description: "File content" })
|
||||
@ApiResponse({ status: 404, description: "Scenario or file not found" })
|
||||
async getScenarioFileContent(
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
@Param("fileId", ParseUUIDPipe) fileId: string,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
return this.scenarioService.getScenarioFileContent(id, fileId, res);
|
||||
}
|
||||
|
||||
@Get(":id/runs/:runId/files")
|
||||
@ApiOperation({ summary: "List files created during a run" })
|
||||
@ApiResponse({ status: 200 })
|
||||
@ApiResponse({ status: 404, description: "Scenario or run not found" })
|
||||
listRunFiles(
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
@Param("runId", ParseUUIDPipe) runId: string,
|
||||
@Query("limit") limit?: string,
|
||||
@Query("offset") offset?: string,
|
||||
) {
|
||||
return this.scenarioService.listRunFiles(
|
||||
id,
|
||||
runId,
|
||||
limit ? parseInt(limit, 10) : undefined,
|
||||
offset ? parseInt(offset, 10) : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
@Get(":id/runs/:runId/files/:fileId/content")
|
||||
@ApiOperation({ summary: "Download run artifact file content" })
|
||||
@ApiResponse({ status: 200, description: "File content" })
|
||||
@ApiResponse({ status: 404, description: "Scenario, run, or file not found" })
|
||||
async getRunFileContent(
|
||||
@Param("id", ParseUUIDPipe) id: string,
|
||||
@Param("runId", ParseUUIDPipe) runId: string,
|
||||
@Param("fileId", ParseUUIDPipe) fileId: string,
|
||||
@Res() res: Response,
|
||||
) {
|
||||
return this.scenarioService.getRunFileContent(id, runId, fileId, res);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,9 @@ import { CodeExecutorModule } from "../code-executor/code-executor.module";
|
||||
import { CredentialEntity } from "../credential/credential.entity";
|
||||
import { EnvironmentEntity } from "../environment/environment.entity";
|
||||
import { EnvironmentModule } from "../environment/environment.module";
|
||||
import { FileModule } from "../file/file.module";
|
||||
import { ScenarioFileEntity } from "../file/scenario-file.entity";
|
||||
import { ScenarioRunFileEntity } from "../file/scenario-run-file.entity";
|
||||
import { SessionModule } from "../session/session.module";
|
||||
import { SnippetModule } from "../snippet/snippet.module";
|
||||
import { ScenarioCredentialEntity } from "./scenario-credential.entity";
|
||||
@@ -27,11 +30,14 @@ import { ScenarioService } from "./scenario.service";
|
||||
ScenarioCredentialEntity,
|
||||
CredentialEntity,
|
||||
EnvironmentEntity,
|
||||
ScenarioFileEntity,
|
||||
ScenarioRunFileEntity,
|
||||
]),
|
||||
CodeExecutorModule,
|
||||
SessionModule,
|
||||
EnvironmentModule,
|
||||
SnippetModule,
|
||||
FileModule,
|
||||
],
|
||||
controllers: [ScenarioController],
|
||||
providers: [ScenarioService, ScenarioSchedulerService],
|
||||
|
||||
@@ -12,6 +12,9 @@ import {
|
||||
} from "../common/dto/pagination.dto";
|
||||
import { CredentialEntity } from "../credential/credential.entity";
|
||||
import { EnvironmentEntity } from "../environment/environment.entity";
|
||||
import { FileStorageService } from "../file/file-storage.service";
|
||||
import { ScenarioFileEntity } from "../file/scenario-file.entity";
|
||||
import { ScenarioRunFileEntity } from "../file/scenario-run-file.entity";
|
||||
import { AddScenarioCredentialDto } from "./dto/add-scenario-credential.dto";
|
||||
import { CreateScenarioStepDto } from "./dto/create-scenario-step.dto";
|
||||
import { CreateScenarioDto } from "./dto/create-scenario.dto";
|
||||
@@ -30,6 +33,8 @@ import { ScenarioRunStepEntity } from "./scenario-run-step.entity";
|
||||
import { ScenarioRunEntity } from "./scenario-run.entity";
|
||||
import { ScenarioStepEntity } from "./scenario-step.entity";
|
||||
import { ScenarioEntity } from "./scenario.entity";
|
||||
import { Response } from "express";
|
||||
import type { File as MulterFile } from "multer";
|
||||
|
||||
export { PaginatedResult } from "../common/dto/pagination.dto";
|
||||
export type ScenarioOrderBy = "id" | "name" | "createdAt" | "updatedAt";
|
||||
@@ -53,6 +58,11 @@ export class ScenarioService {
|
||||
private readonly credentialRepo: Repository<CredentialEntity>,
|
||||
@InjectRepository(EnvironmentEntity)
|
||||
private readonly environmentRepo: Repository<EnvironmentEntity>,
|
||||
@InjectRepository(ScenarioFileEntity)
|
||||
private readonly scenarioFileRepo: Repository<ScenarioFileEntity>,
|
||||
@InjectRepository(ScenarioRunFileEntity)
|
||||
private readonly scenarioRunFileRepo: Repository<ScenarioRunFileEntity>,
|
||||
private readonly fileStorageService: FileStorageService,
|
||||
) {}
|
||||
|
||||
// ── Scenarios ─────────────────────────────────────────────────────────────
|
||||
@@ -651,4 +661,169 @@ export class ScenarioService {
|
||||
}
|
||||
return this.findOne(scenario.id);
|
||||
}
|
||||
|
||||
// ── Files ─────────────────────────────────────────────────────────────────
|
||||
|
||||
async uploadFile(
|
||||
scenarioId: string,
|
||||
file: MulterFile,
|
||||
expiresAtStr?: string,
|
||||
): Promise<any> {
|
||||
// Verify scenario exists
|
||||
await this.findOne(scenarioId);
|
||||
|
||||
if (!file) {
|
||||
throw new BadRequestException("No file provided");
|
||||
}
|
||||
|
||||
// Parse optional expiresAt
|
||||
let expiresAt: Date | undefined = undefined;
|
||||
if (expiresAtStr) {
|
||||
expiresAt = new Date(expiresAtStr);
|
||||
if (isNaN(expiresAt.getTime())) {
|
||||
throw new BadRequestException("Invalid expiresAt date");
|
||||
}
|
||||
}
|
||||
|
||||
// Save file to disk and database
|
||||
const savedFile = await this.fileStorageService.saveFile(
|
||||
file.buffer,
|
||||
file.originalname,
|
||||
file.mimetype,
|
||||
expiresAt,
|
||||
);
|
||||
|
||||
// Create scenario file mapping
|
||||
const scenarioFile = this.scenarioFileRepo.create({
|
||||
scenarioId,
|
||||
fileId: savedFile.id,
|
||||
});
|
||||
await this.scenarioFileRepo.save(scenarioFile);
|
||||
|
||||
// Return file metadata
|
||||
return {
|
||||
id: savedFile.id,
|
||||
name: savedFile.originalName,
|
||||
mimeType: savedFile.mimeType,
|
||||
size: savedFile.size,
|
||||
expiresAt: savedFile.expiresAt,
|
||||
createdAt: savedFile.createdAt,
|
||||
};
|
||||
}
|
||||
|
||||
async listScenarioFiles(
|
||||
scenarioId: string,
|
||||
limit?: number,
|
||||
offset?: number,
|
||||
): Promise<any> {
|
||||
// Verify scenario exists
|
||||
await this.findOne(scenarioId);
|
||||
|
||||
const result = await this.fileStorageService.listScenarioFiles(
|
||||
scenarioId,
|
||||
limit,
|
||||
offset,
|
||||
);
|
||||
|
||||
return {
|
||||
items: result.items.map((item) => ({
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
mimeType: item.mimeType,
|
||||
size: item.size,
|
||||
})),
|
||||
total: result.total,
|
||||
};
|
||||
}
|
||||
|
||||
async getScenarioFileContent(
|
||||
scenarioId: string,
|
||||
fileId: string,
|
||||
res: Response,
|
||||
): Promise<void> {
|
||||
// Verify scenario exists
|
||||
await this.findOne(scenarioId);
|
||||
|
||||
// Check if file is linked to this scenario
|
||||
const link = await this.scenarioFileRepo.findOne({
|
||||
where: { scenarioId, fileId },
|
||||
relations: ["file"],
|
||||
});
|
||||
|
||||
if (!link) {
|
||||
throw new NotFoundException(`File ${fileId} not found in scenario ${scenarioId}`);
|
||||
}
|
||||
|
||||
const file = link.file;
|
||||
const fullPath = this.fileStorageService.getAbsolutePath(file.filePath);
|
||||
|
||||
res.setHeader("Content-Type", file.mimeType);
|
||||
res.setHeader("Content-Disposition", `inline; filename="${file.originalName}"`);
|
||||
res.setHeader("Content-Length", file.size);
|
||||
|
||||
res.sendFile(fullPath);
|
||||
}
|
||||
|
||||
async listRunFiles(
|
||||
scenarioId: string,
|
||||
runId: string,
|
||||
limit?: number,
|
||||
offset?: number,
|
||||
): Promise<any> {
|
||||
// Verify scenario and run exist
|
||||
await this.findOne(scenarioId);
|
||||
const run = await this.runRepo.findOneBy({ id: runId, scenarioId });
|
||||
if (!run) {
|
||||
throw new NotFoundException(`Run ${runId} not found in scenario ${scenarioId}`);
|
||||
}
|
||||
|
||||
const result = await this.fileStorageService.listRunFiles(
|
||||
runId,
|
||||
limit,
|
||||
offset,
|
||||
);
|
||||
|
||||
return {
|
||||
items: result.items.map((item) => ({
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
mimeType: item.mimeType,
|
||||
size: item.size,
|
||||
})),
|
||||
total: result.total,
|
||||
};
|
||||
}
|
||||
|
||||
async getRunFileContent(
|
||||
scenarioId: string,
|
||||
runId: string,
|
||||
fileId: string,
|
||||
res: Response,
|
||||
): Promise<void> {
|
||||
// Verify scenario and run exist
|
||||
await this.findOne(scenarioId);
|
||||
const run = await this.runRepo.findOneBy({ id: runId, scenarioId });
|
||||
if (!run) {
|
||||
throw new NotFoundException(`Run ${runId} not found in scenario ${scenarioId}`);
|
||||
}
|
||||
|
||||
// Check if file is linked to this run
|
||||
const link = await this.scenarioRunFileRepo.findOne({
|
||||
where: { runId, fileId },
|
||||
relations: ["file"],
|
||||
});
|
||||
|
||||
if (!link) {
|
||||
throw new NotFoundException(`File ${fileId} not found in run ${runId}`);
|
||||
}
|
||||
|
||||
const file = link.file;
|
||||
const fullPath = this.fileStorageService.getAbsolutePath(file.filePath);
|
||||
|
||||
res.setHeader("Content-Type", file.mimeType);
|
||||
res.setHeader("Content-Disposition", `inline; filename="${file.originalName}"`);
|
||||
res.setHeader("Content-Length", file.size);
|
||||
|
||||
res.sendFile(fullPath);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +48,9 @@ import { HealthController } from "../src/health/health.controller";
|
||||
import { HttpExceptionFilter } from "../src/filters/http-exception.filter";
|
||||
import { LoggingInterceptor } from "../src/interceptors/logging.interceptor";
|
||||
import { validateAppConfig } from "../src/config/app.config";
|
||||
import { FileEntity } from "../src/file/file.entity";
|
||||
import { ScenarioFileEntity } from "../src/file/scenario-file.entity";
|
||||
import { ScenarioRunFileEntity } from "../src/file/scenario-run-file.entity";
|
||||
|
||||
export async function buildTestApp(): Promise<INestApplication> {
|
||||
const module: TestingModule = await Test.createTestingModule({
|
||||
@@ -71,6 +74,9 @@ export async function buildTestApp(): Promise<INestApplication> {
|
||||
ScenarioCredentialEntity,
|
||||
CredentialEntity,
|
||||
SnippetEntity,
|
||||
FileEntity,
|
||||
ScenarioFileEntity,
|
||||
ScenarioRunFileEntity,
|
||||
],
|
||||
synchronize: true,
|
||||
}),
|
||||
|
||||
Reference in New Issue
Block a user