feat(files): add file storage subsystem with scenario and run artifact support

- add FileEntity, ScenarioFileEntity, ScenarioRunFileEntity with UUID-sharded disk tree - FileStorageService: write/sha256/expiry, paginated listing, dynamic cleanup cron via SchedulerRegistry - expose getScenarioFiles() and downloadFile() on ScriptContext for use in exec code - wire scenarioId, runId, fileService into ExecContextBuilder and scenario scheduler - add file upload and listing endpoints to ScenarioController - add FILES_DIR, FILE_RUN_EXPIRATION_DAYS, FILE_CLEANUP_CRON to AppConfig
2026-04-21 13:01:09 +03:00
parent aa3bae9020
commit 6a91ce30e3
15 changed files with 864 additions and 0 deletions
@@ -0,0 +1,277 @@
+import { Injectable, OnModuleInit } from "@nestjs/common";
+import { ConfigService } from "@nestjs/config";
+import { SchedulerRegistry } from "@nestjs/schedule";
+import { InjectRepository } from "@nestjs/typeorm";
+import * as crypto from "crypto";
+import * as fs from "fs/promises";
+import * as path from "path";
+import { CronJob } from "cron";
+import { Repository } from "typeorm";
+import { TraceLogger } from "../common/trace-logger";
+import { AppConfig } from "../config/app.config";
+import { FileEntity } from "./file.entity";
+import { ScenarioFileEntity } from "./scenario-file.entity";
+import { ScenarioRunFileEntity } from "./scenario-run-file.entity";
+
+export interface FileMetadata {
+  id: string;
+  name: string;
+  mimeType: string;
+  size: number;
+  sha256: string;
+  path: string;
+}
+
+@Injectable()
+export class FileStorageService implements OnModuleInit {
+  private static readonly DEFAULT_PAGE_SIZE = 20;
+  private readonly logger = new TraceLogger(FileStorageService.name);
+  private readonly filesDir: string;
+  private readonly fileRunExpirationDays: number;
+  private readonly fileCleanupCron: string;
+
+  constructor(
+    @InjectRepository(FileEntity)
+    readonly fileRepo: Repository<FileEntity>,
+    @InjectRepository(ScenarioFileEntity)
+    private readonly scenarioFileRepo: Repository<ScenarioFileEntity>,
+    @InjectRepository(ScenarioRunFileEntity)
+    private readonly scenarioRunFileRepo: Repository<ScenarioRunFileEntity>,
+    configService: ConfigService<AppConfig, true>,
+    private readonly schedulerRegistry: SchedulerRegistry,
+  ) {
+    this.filesDir = configService.get("FILES_DIR");
+    this.fileRunExpirationDays = configService.get("FILE_RUN_EXPIRATION_DAYS");
+    this.fileCleanupCron = configService.get("FILE_CLEANUP_CRON");
+  }
+
+  async onModuleInit(): Promise<void> {
+    // Ensure the files directory exists
+    try {
+      await fs.mkdir(this.filesDir, { recursive: true });
+    } catch (err) {
+      this.logger.error(`Failed to create FILES_DIR ${this.filesDir}: ${(err as Error).message}`);
+    }
+
+    // Register cleanup cron job dynamically so the schedule is runtime-configurable
+    const job = new CronJob(this.fileCleanupCron, () => {
+      void this.cleanupExpiredFiles();
+    });
+    this.schedulerRegistry.addCronJob("file-cleanup", job);
+    job.start();
+    this.logger.debug(`File cleanup cron registered: ${this.fileCleanupCron}`);
+
+    // Run cleanup at startup to catch any backlog
+    await this.cleanupExpiredFiles();
+  }
+
+  /**
+   * Saves a file to disk with sharded directory structure.
+   * Returns the FileEntity (unsaved) with computed filePath.
+   */
+  async saveFile(
+    buffer: Buffer,
+    originalName: string,
+    mimeType: string,
+    expiresAt?: Date,
+  ): Promise<FileEntity> {
+    const fileId = crypto.randomUUID();
+    const sha256 = crypto.createHash("sha256").update(buffer).digest("hex");
+    const shardPath = this.getShardedPath(fileId);
+    const fullPath = path.join(this.filesDir, shardPath);
+
+    // Create sharded directory
+    const dirPath = path.dirname(fullPath);
+    await fs.mkdir(dirPath, { recursive: true });
+
+    // Write file to disk and persist entity
+    await fs.writeFile(fullPath, buffer);
+
+    const file = this.fileRepo.create({
+      id: fileId,
+      originalName,
+      mimeType,
+      size: buffer.length,
+      sha256,
+      filePath: shardPath,
+      expiresAt: expiresAt ?? null,
+    });
+
+    return this.fileRepo.save(file);
+  }
+
+  /**
+   * Returns the sharded path for a file ID: xx/xxxx/uuid
+   */
+  private getShardedPath(uuid: string): string {
+    return `${uuid.substring(0, 2)}/${uuid.substring(0, 4)}/${uuid}`;
+  }
+
+  /**
+   * Returns absolute disk path for a file.
+   */
+  getAbsolutePath(filePath: string): string {
+    return path.join(this.filesDir, filePath);
+  }
+
+  /**
+   * Lists scenario files with pagination.
+   */
+  async listScenarioFiles(
+    scenarioId: string,
+    limit?: number,
+    offset?: number,
+  ): Promise<{ items: FileMetadata[]; total: number }> {
+    const query = this.scenarioFileRepo
+      .createQueryBuilder("sf")
+      .innerJoinAndSelect("sf.file", "f")
+      .where("sf.scenarioId = :scenarioId", { scenarioId });
+
+    const total = await query.getCount();
+
+    const items = await query
+      .orderBy("sf.createdAt", "DESC")
+      .skip(offset ?? 0)
+      .take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
+      .getMany();
+    return {
+      items: items.map((sf) => this.mapFileEntityToMetadata(sf.file)),
+      total,
+    };
+  }
+
+  /**
+   * Lists run artifact files with pagination.
+   */
+  async listRunFiles(
+    runId: string,
+    limit?: number,
+    offset?: number,
+  ): Promise<{ items: FileMetadata[]; total: number }> {
+    const query = this.scenarioRunFileRepo
+      .createQueryBuilder("srf")
+      .innerJoinAndSelect("srf.file", "f")
+      .where("srf.runId = :runId", { runId });
+
+    const total = await query.getCount();
+
+    const items = await query
+      .orderBy("srf.createdAt", "DESC")
+      .skip(offset ?? 0)
+      .take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
+      .getMany();
+    return {
+      items: items.map((srf) => this.mapFileEntityToMetadata(srf.file)),
+      total,
+    };
+  }
+
+  /**
+   * Gets a file entity by ID.
+   */
+  async getFile(fileId: string): Promise<FileEntity | null> {
+    return this.fileRepo.findOneBy({ id: fileId });
+  }
+
+  /**
+   * Deletes a file and checks for orphaned FileEntity rows.
+   * If a FileEntity has no remaining mappings, it is deleted along with its physical file.
+   */
+  async deleteFile(fileId: string): Promise<void> {
+    const file = await this.fileRepo.findOneBy({ id: fileId });
+    if (!file) return;
+
+    // Check if this file still has any mappings
+    const scenarioCount = await this.scenarioFileRepo.countBy({ fileId });
+    const runCount = await this.scenarioRunFileRepo.countBy({ fileId });
+
+    // If no mappings remain, delete the file and its physical copy
+    if (scenarioCount === 0 && runCount === 0) {
+      const fullPath = this.getAbsolutePath(file.filePath);
+      try {
+        await fs.unlink(fullPath);
+      } catch (err) {
+        this.logger.warn(`Failed to delete physical file ${fullPath}: ${(err as Error).message}`);
+      }
+
+      await this.fileRepo.delete(fileId);
+    }
+  }
+
+  /**
+   * Creates a run artifact file entry with auto-expiration.
+   */
+  async createRunArtifact(buffer: Buffer, originalName: string, mimeType: string): Promise<FileEntity> {
+    const now = new Date();
+    const expiresAt = new Date(now.getTime() + this.fileRunExpirationDays * 24 * 60 * 60 * 1000);
+    return this.saveFile(buffer, originalName, mimeType, expiresAt);
+  }
+
+  /**
+   * Creates a run artifact with file and mapping in the database.
+   */
+  async createAndSaveRunArtifact(
+    runId: string,
+    buffer: Buffer,
+    originalName: string,
+    mimeType: string,
+  ): Promise<FileEntity> {
+    const savedFile = await this.createRunArtifact(buffer, originalName, mimeType);
+
+    // Create run file mapping
+    const runFile = this.scenarioRunFileRepo.create({
+      runId,
+      fileId: savedFile.id,
+    });
+    await this.scenarioRunFileRepo.save(runFile);
+
+    return savedFile;
+  }
+
+  /**
+   * Cleanup job: deletes expired FileEntity rows and orphaned physical files.
+   * Runs via @Cron scheduler and at module init.
+   */
+  async cleanupExpiredFiles(): Promise<void> {
+    const now = new Date();
+    const expiredFiles = await this.fileRepo
+      .createQueryBuilder("file")
+      .where("file.expiresAt IS NOT NULL")
+      .andWhere("file.expiresAt <= :now", { now })
+      .setParameters({ now })
+      .getMany();
+
+    if (expiredFiles.length === 0) return;
+
+    this.logger.debug(`Cleaning up ${expiredFiles.length} expired files`);
+
+    for (const file of expiredFiles) {
+      try {
+        // Delete associated mapping rows first (cascade will be handled, but we delete manually to ensure orphan check works)
+        await this.scenarioFileRepo.delete({ fileId: file.id });
+        await this.scenarioRunFileRepo.delete({ fileId: file.id });
+
+        // Delete the file entity and physical file
+        await this.deleteFile(file.id);
+      } catch (err) {
+        this.logger.error(
+          `Failed to clean up expired file ${file.id}: ${(err as Error).message}`,
+        );
+      }
+    }
+  }
+
+  /**
+   * Maps FileEntity to FileMetadata with absolute path.
+   */
+  private mapFileEntityToMetadata(file: FileEntity): FileMetadata {
+    return {
+      id: file.id,
+      name: file.originalName,
+      mimeType: file.mimeType,
+      size: file.size,
+      sha256: file.sha256,
+      path: this.getAbsolutePath(file.filePath),
+    };
+  }
+}
@@ -0,0 +1,28 @@
+import { Column, CreateDateColumn, Entity, PrimaryGeneratedColumn } from "typeorm";
+
+@Entity("files")
+export class FileEntity {
+  @PrimaryGeneratedColumn("uuid")
+  id: string;
+
+  @Column()
+  originalName: string;
+
+  @Column()
+  mimeType: string;
+
+  @Column({ type: "integer" })
+  size: number;
+
+  @Column()
+  sha256: string;
+
+  @Column()
+  filePath: string;
+
+  @Column({ type: "datetime", nullable: true })
+  expiresAt: Date | null;
+
+  @CreateDateColumn()
+  createdAt: Date;
+}
@@ -0,0 +1,13 @@
+import { Module } from "@nestjs/common";
+import { TypeOrmModule } from "@nestjs/typeorm";
+import { FileEntity } from "./file.entity";
+import { FileStorageService } from "./file-storage.service";
+import { ScenarioFileEntity } from "./scenario-file.entity";
+import { ScenarioRunFileEntity } from "./scenario-run-file.entity";
+
+@Module({
+  imports: [TypeOrmModule.forFeature([FileEntity, ScenarioFileEntity, ScenarioRunFileEntity])],
+  providers: [FileStorageService],
+  exports: [FileStorageService],
+})
+export class FileModule {}
@@ -0,0 +1,33 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+} from "typeorm";
+import { ScenarioEntity } from "../scenario/scenario.entity";
+import { FileEntity } from "./file.entity";
+
+@Entity("scenario_files")
+export class ScenarioFileEntity {
+  @PrimaryGeneratedColumn("uuid")
+  id: string;
+
+  @Column()
+  scenarioId: string;
+
+  @Column()
+  fileId: string;
+
+  @ManyToOne(() => ScenarioEntity, { onDelete: "CASCADE" })
+  @JoinColumn({ name: "scenarioId" })
+  scenario: ScenarioEntity;
+
+  @ManyToOne(() => FileEntity, { onDelete: "CASCADE" })
+  @JoinColumn({ name: "fileId" })
+  file: FileEntity;
+
+  @CreateDateColumn()
+  createdAt: Date;
+}
@@ -0,0 +1,33 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+} from "typeorm";
+import { ScenarioRunEntity } from "../scenario/scenario-run.entity";
+import { FileEntity } from "./file.entity";
+
+@Entity("scenario_run_files")
+export class ScenarioRunFileEntity {
+  @PrimaryGeneratedColumn("uuid")
+  id: string;
+
+  @Column()
+  runId: string;
+
+  @Column()
+  fileId: string;
+
+  @ManyToOne(() => ScenarioRunEntity, { onDelete: "CASCADE" })
+  @JoinColumn({ name: "runId" })
+  run: ScenarioRunEntity;
+
+  @ManyToOne(() => FileEntity, { onDelete: "CASCADE" })
+  @JoinColumn({ name: "fileId" })
+  file: FileEntity;
+
+  @CreateDateColumn()
+  createdAt: Date;
+}