Files
liqa/server/src/file/file-storage.service.ts
T
ars9 8935783371 feat(import): add central yaml import for all exportable kinds
- add POST /import, an import page and the import_yaml mcp tool; items
  are validated up front and imported in dependency order
- switch list page import buttons to the central import
- write scenario exports as separate --- yaml documents per entity
- give each test app its own files dir to avoid cross-suite races
- use node: prefix for node built-in imports
- bump version to 1.18.0
2026-09-30 19:25:03 +03:00

282 lines
8.6 KiB
TypeScript

import { Injectable, OnModuleInit } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import { SchedulerRegistry } from "@nestjs/schedule";
import { InjectRepository } from "@nestjs/typeorm";
import * as crypto from "node:crypto";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { CronJob } from "cron";
import { Repository } from "typeorm";
import { TraceLogger } from "../common/trace-logger";
import { AppConfig } from "../config/app.config";
import { FileEntity } from "./file.entity";
import { ScenarioFileEntity } from "./scenario-file.entity";
import { ScenarioRunFileEntity } from "./scenario-run-file.entity";
export interface FileMetadata {
id: string;
name: string;
mimeType: string;
size: number;
sha256: string;
expiresAt: Date | null;
createdAt: Date;
path: string;
}
@Injectable()
export class FileStorageService implements OnModuleInit {
private static readonly DEFAULT_PAGE_SIZE = 20;
private readonly logger = new TraceLogger(FileStorageService.name);
private readonly filesDir: string;
private readonly fileRunExpirationDays: number;
private readonly fileCleanupCron: string;
constructor(
@InjectRepository(FileEntity)
readonly fileRepo: Repository<FileEntity>,
@InjectRepository(ScenarioFileEntity)
private readonly scenarioFileRepo: Repository<ScenarioFileEntity>,
@InjectRepository(ScenarioRunFileEntity)
private readonly scenarioRunFileRepo: Repository<ScenarioRunFileEntity>,
configService: ConfigService<AppConfig, true>,
private readonly schedulerRegistry: SchedulerRegistry,
) {
this.filesDir = configService.get("FILES_DIR");
this.fileRunExpirationDays = configService.get("FILE_RUN_EXPIRATION_DAYS");
this.fileCleanupCron = configService.get("FILE_CLEANUP_CRON");
}
async onModuleInit(): Promise<void> {
// Ensure the files directory exists
try {
await fs.mkdir(this.filesDir, { recursive: true });
} catch (err) {
this.logger.error(`Failed to create FILES_DIR ${this.filesDir}: ${(err as Error).message}`);
}
// Register cleanup cron job dynamically so the schedule is runtime-configurable
const job = new CronJob(this.fileCleanupCron, () => {
void this.cleanupExpiredFiles();
});
this.schedulerRegistry.addCronJob("file-cleanup", job);
job.start();
this.logger.debug(`File cleanup cron registered: ${this.fileCleanupCron}`);
// Run cleanup at startup to catch any backlog
await this.cleanupExpiredFiles();
}
/**
* Saves a file to disk with sharded directory structure.
* Returns the FileEntity (unsaved) with computed filePath.
*/
async saveFile(
buffer: Buffer,
originalName: string,
mimeType: string,
expiresAt?: Date,
): Promise<FileEntity> {
const fileId = crypto.randomUUID();
const sha256 = crypto.createHash("sha256").update(buffer).digest("hex");
const shardPath = this.getShardedPath(fileId);
const fullPath = path.join(this.filesDir, shardPath);
// Create sharded directory
const dirPath = path.dirname(fullPath);
await fs.mkdir(dirPath, { recursive: true });
// Write file to disk and persist entity
await fs.writeFile(fullPath, buffer);
const file = this.fileRepo.create({
id: fileId,
originalName,
mimeType,
size: buffer.length,
sha256,
filePath: shardPath,
expiresAt: expiresAt ?? null,
});
return this.fileRepo.save(file);
}
/**
* Returns the sharded path for a file ID: xx/xxxx/uuid
*/
private getShardedPath(uuid: string): string {
return `${uuid.substring(0, 2)}/${uuid.substring(0, 4)}/${uuid}`;
}
/**
* Returns absolute disk path for a file.
*/
getAbsolutePath(filePath: string): string {
return path.join(this.filesDir, filePath);
}
/**
* Lists scenario files with pagination.
*/
async listScenarioFiles(
scenarioId: string,
limit?: number,
offset?: number,
): Promise<{ items: FileMetadata[]; total: number }> {
const query = this.scenarioFileRepo
.createQueryBuilder("sf")
.innerJoinAndSelect("sf.file", "f")
.where("sf.scenarioId = :scenarioId", { scenarioId });
const total = await query.getCount();
const items = await query
.orderBy("sf.createdAt", "DESC")
.skip(offset ?? 0)
.take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
.getMany();
return {
items: items.map((sf) => this.mapFileEntityToMetadata(sf.file)),
total,
};
}
/**
* Lists run artifact files with pagination.
*/
async listRunFiles(
runId: string,
limit?: number,
offset?: number,
): Promise<{ items: FileMetadata[]; total: number }> {
const query = this.scenarioRunFileRepo
.createQueryBuilder("srf")
.innerJoinAndSelect("srf.file", "f")
.where("srf.runId = :runId", { runId });
const total = await query.getCount();
const items = await query
.orderBy("srf.createdAt", "DESC")
.skip(offset ?? 0)
.take(limit ?? FileStorageService.DEFAULT_PAGE_SIZE)
.getMany();
return {
items: items.map((srf) => this.mapFileEntityToMetadata(srf.file)),
total,
};
}
/**
* Gets a file entity by ID.
*/
async getFile(fileId: string): Promise<FileEntity | null> {
return this.fileRepo.findOneBy({ id: fileId });
}
/**
* Deletes a file and checks for orphaned FileEntity rows.
* If a FileEntity has no remaining mappings, it is deleted along with its physical file.
*/
async deleteFile(fileId: string): Promise<void> {
const file = await this.fileRepo.findOneBy({ id: fileId });
if (!file) return;
// Check if this file still has any mappings
const scenarioCount = await this.scenarioFileRepo.countBy({ fileId });
const runCount = await this.scenarioRunFileRepo.countBy({ fileId });
// If no mappings remain, delete the file and its physical copy
if (scenarioCount === 0 && runCount === 0) {
const fullPath = this.getAbsolutePath(file.filePath);
try {
await fs.unlink(fullPath);
} catch (err) {
this.logger.warn(`Failed to delete physical file ${fullPath}: ${(err as Error).message}`);
}
await this.fileRepo.delete(fileId);
}
}
/**
* Creates a run artifact file entry with auto-expiration.
*/
async createRunArtifact(buffer: Buffer, originalName: string, mimeType: string): Promise<FileEntity> {
const now = new Date();
const expiresAt = new Date(now.getTime() + this.fileRunExpirationDays * 24 * 60 * 60 * 1000);
return this.saveFile(buffer, originalName, mimeType, expiresAt);
}
/**
* Creates a run artifact with file and mapping in the database.
*/
async createAndSaveRunArtifact(
runId: string,
buffer: Buffer,
originalName: string,
mimeType: string,
): Promise<FileEntity> {
const savedFile = await this.createRunArtifact(buffer, originalName, mimeType);
// Create run file mapping
const runFile = this.scenarioRunFileRepo.create({
runId,
fileId: savedFile.id,
});
await this.scenarioRunFileRepo.save(runFile);
return savedFile;
}
/**
* Cleanup job: deletes expired FileEntity rows and orphaned physical files.
* Runs via @Cron scheduler and at module init.
*/
async cleanupExpiredFiles(): Promise<void> {
const now = new Date();
const expiredFiles = await this.fileRepo
.createQueryBuilder("file")
.where("file.expiresAt IS NOT NULL")
.andWhere("file.expiresAt <= :now", { now })
.setParameters({ now })
.getMany();
if (expiredFiles.length === 0) return;
this.logger.debug(`Cleaning up ${expiredFiles.length} expired files`);
for (const file of expiredFiles) {
try {
// Delete associated mapping rows first (cascade will be handled, but we delete manually to ensure orphan check works)
await this.scenarioFileRepo.delete({ fileId: file.id });
await this.scenarioRunFileRepo.delete({ fileId: file.id });
// Delete the file entity and physical file
await this.deleteFile(file.id);
} catch (err) {
this.logger.error(
`Failed to clean up expired file ${file.id}: ${(err as Error).message}`,
);
}
}
}
/**
* Maps FileEntity to FileMetadata with absolute path.
*/
private mapFileEntityToMetadata(file: FileEntity): FileMetadata {
return {
id: file.id,
name: file.originalName,
mimeType: file.mimeType,
size: file.size,
sha256: file.sha256,
expiresAt: file.expiresAt,
createdAt: file.createdAt,
path: this.getAbsolutePath(file.filePath),
};
}
}