diff --git a/server/src/scenario/scenario-scheduler.service.ts b/server/src/scenario/scenario-scheduler.service.ts index e2e0513..49c7a59 100644 --- a/server/src/scenario/scenario-scheduler.service.ts +++ b/server/src/scenario/scenario-scheduler.service.ts @@ -46,6 +46,7 @@ export class ScenarioSchedulerService implements OnModuleInit { private static readonly DEFAULT_SCENARIO_TIMEOUT_SEC = 600; private static readonly DEFAULT_STEP_TIMEOUT_SEC = 60; + private static readonly CONCURRENCY_LIMIT = 10; constructor( @InjectRepository(ScenarioRunEntity) @@ -97,7 +98,7 @@ export class ScenarioSchedulerService implements OnModuleInit { ), ), ), - { concurrency: 10 }, + { concurrency: ScenarioSchedulerService.CONCURRENCY_LIMIT }, ), ); } @@ -168,38 +169,61 @@ export class ScenarioSchedulerService implements OnModuleInit { @Interval(1000) async pickUpPendingRuns(): Promise { const pending = await this.runRepo.find({ where: { status: "pending" } }); - for (const run of pending) { - if (this.activeRuns.has(run.id)) continue; - this.activeRuns.add(run.id); - run.status = "in_progress"; - await this.runRepo.save(run); - this.logger.log(`Run #${run.id} → in_progress`); - // Pre-load credential map for the scenario - const credMap = await this.scenarioService - .buildCredentialMap(run.scenarioId) - .catch(() => ({}) as Record); - this.runCredentials.set(run.id, credMap); - // Pre-load snippet map - const snippetMap = await this.snippetService - .buildSnippetMap() - .catch(() => ({}) as Record); - this.runSnippets.set(run.id, snippetMap); - // Pre-load selected environment data - const environmentData = await this.environmentRepo - .findOneBy({ id: run.environmentId }) - .then((env) => env?.data ?? {}) - .catch(() => ({}) as EnvironmentData); - this.runEnvironments.set(run.id, environmentData); - // Cache scenario-level timeout for the run - const scenario = await this.scenarioService - .findOne(run.scenarioId) - .catch(() => null); - this.runScenarioTimeouts.set(run.id, scenario?.timeoutSeconds ?? null); - const traceId = crypto.randomUUID(); - void traceStorage.run({ traceId }, () => - this.processRunToCompletion(run.id), - ); - } + const newRuns = pending.filter((run) => !this.activeRuns.has(run.id)); + if (newRuns.length === 0) return; + + await Effect.runPromise( + Effect.forEach( + newRuns, + (run) => { + this.activeRuns.add(run.id); + return Effect.tryPromise(async () => { + run.status = "in_progress"; + await this.runRepo.save(run); + this.logger.log(`Run #${run.id} → in_progress`); + const credMap = await this.scenarioService + .buildCredentialMap(run.scenarioId) + .catch(() => ({}) as Record); + this.runCredentials.set(run.id, credMap); + const snippetMap = await this.snippetService + .buildSnippetMap() + .catch(() => ({}) as Record); + this.runSnippets.set(run.id, snippetMap); + const environmentData = await this.environmentRepo + .findOneBy({ id: run.environmentId }) + .then((env) => env?.data ?? {}) + .catch(() => ({}) as EnvironmentData); + this.runEnvironments.set(run.id, environmentData); + const scenario = await this.scenarioService + .findOne(run.scenarioId) + .catch(() => null); + this.runScenarioTimeouts.set(run.id, scenario?.timeoutSeconds ?? null); + const traceId = crypto.randomUUID(); + void traceStorage.run({ traceId }, () => + this.processRunToCompletion(run.id), + ); + }).pipe( + Effect.catchAll((err) => + Effect.promise(async () => { + this.activeRuns.delete(run.id); + this.runCredentials.delete(run.id); + this.runSnippets.delete(run.id); + this.runEnvironments.delete(run.id); + this.runScenarioTimeouts.delete(run.id); + this.logger.error( + `Run #${run.id}: setup failed — ${String(err)}`, + ); + await this.failRun( + run.id, + `Run setup failed: ${String(err)}`, + ).catch(() => {}); + }), + ), + ); + }, + { concurrency: ScenarioSchedulerService.CONCURRENCY_LIMIT }, + ), + ); } private async processRunToCompletion(runId: string): Promise {