- add POST /import, an import page and the import_yaml mcp tool; items are validated up front and imported in dependency order - switch list page import buttons to the central import - write scenario exports as separate --- yaml documents per entity - give each test app its own files dir to avoid cross-suite races - use node: prefix for node built-in imports - bump version to 1.18.0
282 lines
8.6 KiB
TypeScript
282 lines
8.6 KiB
TypeScript
import { Injectable, OnModuleInit } from "@nestjs/common";
|
|
import { ConfigService } from "@nestjs/config";
|
|
import { SchedulerRegistry } from "@nestjs/schedule";
|
|
import { InjectRepository } from "@nestjs/typeorm";
|
|
import * as crypto from "node:crypto";
|
|
import * as fs from "node:fs/promises";
|
|
import * as path from "node:path";
|
|
import { CronJob } from "cron";
|
|
import { Repository } from "typeorm";
|
|
import { TraceLogger } from "../common/trace-logger";
|
|
import { AppConfig } from "../config/app.config";
|
|
import { FileEntity } from "./file.entity";
|
|
import { ScenarioFileEntity } from "./scenario-file.entity";
|
|
import { ScenarioRunFileEntity } from "./scenario-run-file.entity";
|
|
|
|
export interface FileMetadata {
|
|
id: string;
|
|
name: string;
|
|
mimeType: string;
|
|
size: number;
|
|
sha256: string;
|
|
expiresAt: Date | null;
|
|
createdAt: Date;
|
|
path: string;
|
|
}
|
|
|
|
@Injectable()
|
|
export class FileStorageService implements OnModuleInit {
|
|
private static readonly DEFAULT_PAGE_SIZE = 20;
|
|
private readonly logger = new TraceLogger(FileStorageService.name);
|
|
private readonly filesDir: string;
|
|
private readonly fileRunExpirationDays: number;
|
|
private readonly fileCleanupCron: string;
|
|
|
|
constructor(
|
|
@InjectRepository(FileEntity)
|
|
readonly fileRepo: Repository<FileEntity>,
|
|
@InjectRepository(ScenarioFileEntity)
|
|
private readonly scenarioFileRepo: Repository<ScenarioFileEntity>,
|
|
@InjectRepository(ScenarioRunFileEntity)
|
|
private readonly scenarioRunFileRepo: Repository<ScenarioRunFileEntity>,
|
|
configService: ConfigService<AppConfig, true>,
|
|
private readonly schedulerRegistry: SchedulerRegistry,
|
|
) {
|
|
this.filesDir = configService.get("FILES_DIR");
|
|
this.fileRunExpirationDays = configService.get("FILE_RUN_EXPIRATION_DAYS");
|
|
this.fileCleanupCron = configService.get("FILE_CLEANUP_CRON");
|
|
}
|
|
|
|
async onModuleInit(): Promise<void> {
|
|
// Ensure the files directory exists
|
|
try {
|
|
await fs.mkdir(this.filesDir, { recursive: true });
|
|
} catch (err) {
|
|
this.logger.error(`Failed to create FILES_DIR ${this.filesDir}: ${(err as Error).message}`);
|
|
}
|
|
|
|
// Register cleanup cron job dynamically so the schedule is runtime-configurable
|
|
const job = new CronJob(this.fileCleanupCron, () => {
|
|
void this.cleanupExpiredFiles();
|
|
});
|
|
this.schedulerRegistry.addCronJob("file-cleanup", job);
|
|
job.start();
|
|
this.logger.debug(`File cleanup cron registered: ${this.fileCleanupCron}`);
|
|
|
|
// Run cleanup at startup to catch any backlog
|
|
await this.cleanupExpiredFiles();
|
|
}
|
|
|
|
/**
|
|
* Saves a file to disk with sharded directory structure.
|
|
* Returns the FileEntity (unsaved) with computed filePath.
|
|
*/
|
|
async saveFile(
|
|
buffer: Buffer,
|
|
originalName: string,
|
|
mimeType: string,
|
|
expiresAt?: Date,
|
|
): Promise<FileEntity> {
|
|
const fileId = crypto.randomUUID();
|
|
const sha256 = crypto.createHash("sha256").update(buffer).digest("hex");
|
|
const shardPath = this.getShardedPath(fileId);
|
|
const fullPath = path.join(this.filesDir, shardPath);
|
|
|
|
// Create sharded directory
|
|
const dirPath = path.dirname(fullPath);
|
|
await fs.mkdir(dirPath, { recursive: true });
|
|
|
|
// Write file to disk and persist entity
|
|
await fs.writeFile(fullPath, buffer);
|
|
|
|
const file = this.fileRepo.create({
|
|
id: fileId,
|
|
originalName,
|
|
mimeType,
|
|
size: buffer.length,
|
|
sha256,
|
|
filePath: shardPath,
|
|
expiresAt: expiresAt ?? null,
|
|
});
|
|
|
|
return this.fileRepo.save(file);
|
|
}
|
|
|
|
/**
|
|
* Returns the sharded path for a file ID: xx/xxxx/uuid
|
|
*/
|
|
private getShardedPath(uuid: string): string {
|
|
return `${uuid.substring(0, 2)}/${uuid.substring(0, 4)}/${uuid}`;
|
|
}
|
|
|
|
/**
|
|
* Returns absolute disk path for a file.
|
|
*/
|
|
getAbsolutePath(filePath: string): string {
|
|
return path.join(this.filesDir, filePath);
|
|
}
|
|
|
|
/**
|
|
* Lists scenario files with pagination.
|
|
*/
|
|
async listScenarioFiles(
|
|
scenarioId: string,
|
|
limit?: number,
|
|
offset?: number,
|
|
): Promise<{ items: FileMetadata[]; total: number }> {
|
|
const query = this.scenarioFileRepo
|
|
.createQueryBuilder("sf")
|
|
.innerJoinAndSelect("sf.file", "f")
|
|
.where("sf.scenarioId = :scenarioId", { scenarioId });
|
|
|
|
const total = await query.getCount();
|
|
|
|
const items = await query
|
|
.orderBy("sf.createdAt", "DESC")
|
|
.skip(offset ?? 0)
|
|
.take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
|
|
.getMany();
|
|
return {
|
|
items: items.map((sf) => this.mapFileEntityToMetadata(sf.file)),
|
|
total,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Lists run artifact files with pagination.
|
|
*/
|
|
async listRunFiles(
|
|
runId: string,
|
|
limit?: number,
|
|
offset?: number,
|
|
): Promise<{ items: FileMetadata[]; total: number }> {
|
|
const query = this.scenarioRunFileRepo
|
|
.createQueryBuilder("srf")
|
|
.innerJoinAndSelect("srf.file", "f")
|
|
.where("srf.runId = :runId", { runId });
|
|
|
|
const total = await query.getCount();
|
|
|
|
const items = await query
|
|
.orderBy("srf.createdAt", "DESC")
|
|
.skip(offset ?? 0)
|
|
.take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
|
|
.getMany();
|
|
return {
|
|
items: items.map((srf) => this.mapFileEntityToMetadata(srf.file)),
|
|
total,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Gets a file entity by ID.
|
|
*/
|
|
async getFile(fileId: string): Promise<FileEntity | null> {
|
|
return this.fileRepo.findOneBy({ id: fileId });
|
|
}
|
|
|
|
/**
|
|
* Deletes a file and checks for orphaned FileEntity rows.
|
|
* If a FileEntity has no remaining mappings, it is deleted along with its physical file.
|
|
*/
|
|
async deleteFile(fileId: string): Promise<void> {
|
|
const file = await this.fileRepo.findOneBy({ id: fileId });
|
|
if (!file) return;
|
|
|
|
// Check if this file still has any mappings
|
|
const scenarioCount = await this.scenarioFileRepo.countBy({ fileId });
|
|
const runCount = await this.scenarioRunFileRepo.countBy({ fileId });
|
|
|
|
// If no mappings remain, delete the file and its physical copy
|
|
if (scenarioCount === 0 && runCount === 0) {
|
|
const fullPath = this.getAbsolutePath(file.filePath);
|
|
try {
|
|
await fs.unlink(fullPath);
|
|
} catch (err) {
|
|
this.logger.warn(`Failed to delete physical file ${fullPath}: ${(err as Error).message}`);
|
|
}
|
|
|
|
await this.fileRepo.delete(fileId);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Creates a run artifact file entry with auto-expiration.
|
|
*/
|
|
async createRunArtifact(buffer: Buffer, originalName: string, mimeType: string): Promise<FileEntity> {
|
|
const now = new Date();
|
|
const expiresAt = new Date(now.getTime() + this.fileRunExpirationDays * 24 * 60 * 60 * 1000);
|
|
return this.saveFile(buffer, originalName, mimeType, expiresAt);
|
|
}
|
|
|
|
/**
|
|
* Creates a run artifact with file and mapping in the database.
|
|
*/
|
|
async createAndSaveRunArtifact(
|
|
runId: string,
|
|
buffer: Buffer,
|
|
originalName: string,
|
|
mimeType: string,
|
|
): Promise<FileEntity> {
|
|
const savedFile = await this.createRunArtifact(buffer, originalName, mimeType);
|
|
|
|
// Create run file mapping
|
|
const runFile = this.scenarioRunFileRepo.create({
|
|
runId,
|
|
fileId: savedFile.id,
|
|
});
|
|
await this.scenarioRunFileRepo.save(runFile);
|
|
|
|
return savedFile;
|
|
}
|
|
|
|
/**
|
|
* Cleanup job: deletes expired FileEntity rows and orphaned physical files.
|
|
* Runs via @Cron scheduler and at module init.
|
|
*/
|
|
async cleanupExpiredFiles(): Promise<void> {
|
|
const now = new Date();
|
|
const expiredFiles = await this.fileRepo
|
|
.createQueryBuilder("file")
|
|
.where("file.expiresAt IS NOT NULL")
|
|
.andWhere("file.expiresAt <= :now", { now })
|
|
.setParameters({ now })
|
|
.getMany();
|
|
|
|
if (expiredFiles.length === 0) return;
|
|
|
|
this.logger.debug(`Cleaning up ${expiredFiles.length} expired files`);
|
|
|
|
for (const file of expiredFiles) {
|
|
try {
|
|
// Delete associated mapping rows first (cascade will be handled, but we delete manually to ensure orphan check works)
|
|
await this.scenarioFileRepo.delete({ fileId: file.id });
|
|
await this.scenarioRunFileRepo.delete({ fileId: file.id });
|
|
|
|
// Delete the file entity and physical file
|
|
await this.deleteFile(file.id);
|
|
} catch (err) {
|
|
this.logger.error(
|
|
`Failed to clean up expired file ${file.id}: ${(err as Error).message}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Maps FileEntity to FileMetadata with absolute path.
|
|
*/
|
|
private mapFileEntityToMetadata(file: FileEntity): FileMetadata {
|
|
return {
|
|
id: file.id,
|
|
name: file.originalName,
|
|
mimeType: file.mimeType,
|
|
size: file.size,
|
|
sha256: file.sha256,
|
|
expiresAt: file.expiresAt,
|
|
createdAt: file.createdAt,
|
|
path: this.getAbsolutePath(file.filePath),
|
|
};
|
|
}
|
|
}
|