perf(scenario-scheduler): parallelize run pickup with effect, add cache cleanup

- replace sequential for-loop in pickUpPendingRuns with Effect.forEach concurrency:10
- extract CONCURRENCY_LIMIT constant shared by both startup and pickup paths
- clear all run caches (creds, snippets, env, timeouts, activeRuns) on setup failure
- call failRun on setup error to leave the db run in a consistent failed state

server/src/scenario/scenario-scheduler.service.ts

@@ -46,6 +46,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
 
   private static readonly DEFAULT_SCENARIO_TIMEOUT_SEC = 600;
   private static readonly DEFAULT_STEP_TIMEOUT_SEC = 60;
+  private static readonly CONCURRENCY_LIMIT = 10;
 
   constructor(
     @InjectRepository(ScenarioRunEntity)
@@ -97,7 +98,7 @@ export class ScenarioSchedulerService implements OnModuleInit {
               ),
             ),
           ),
-        { concurrency: 10 },
+        { concurrency: ScenarioSchedulerService.CONCURRENCY_LIMIT },
       ),
     );
   }
@@ -168,29 +169,31 @@ export class ScenarioSchedulerService implements OnModuleInit {
   @Interval(1000)
   async pickUpPendingRuns(): Promise<void> {
     const pending = await this.runRepo.find({ where: { status: "pending" } });
-    for (const run of pending) {
-      if (this.activeRuns.has(run.id)) continue;
+    const newRuns = pending.filter((run) => !this.activeRuns.has(run.id));
+    if (newRuns.length === 0) return;
+
+    await Effect.runPromise(
+      Effect.forEach(
+        newRuns,
+        (run) => {
           this.activeRuns.add(run.id);
+          return Effect.tryPromise(async () => {
             run.status = "in_progress";
             await this.runRepo.save(run);
             this.logger.log(`Run #${run.id} → in_progress`);
-      // Pre-load credential map for the scenario
             const credMap = await this.scenarioService
               .buildCredentialMap(run.scenarioId)
               .catch(() => ({}) as Record<string, unknown>);
             this.runCredentials.set(run.id, credMap);
-      // Pre-load snippet map
             const snippetMap = await this.snippetService
               .buildSnippetMap()
               .catch(() => ({}) as Record<string, string>);
             this.runSnippets.set(run.id, snippetMap);
-      // Pre-load selected environment data
             const environmentData = await this.environmentRepo
               .findOneBy({ id: run.environmentId })
               .then((env) => env?.data ?? {})
               .catch(() => ({}) as EnvironmentData);
             this.runEnvironments.set(run.id, environmentData);
-      // Cache scenario-level timeout for the run
             const scenario = await this.scenarioService
               .findOne(run.scenarioId)
               .catch(() => null);
@@ -199,7 +202,28 @@ export class ScenarioSchedulerService implements OnModuleInit {
             void traceStorage.run({ traceId }, () =>
               this.processRunToCompletion(run.id),
             );
-    }
+          }).pipe(
+            Effect.catchAll((err) =>
+              Effect.promise(async () => {
+                this.activeRuns.delete(run.id);
+                this.runCredentials.delete(run.id);
+                this.runSnippets.delete(run.id);
+                this.runEnvironments.delete(run.id);
+                this.runScenarioTimeouts.delete(run.id);
+                this.logger.error(
+                  `Run #${run.id}: setup failed — ${String(err)}`,
+                );
+                await this.failRun(
+                  run.id,
+                  `Run setup failed: ${String(err)}`,
+                ).catch(() => {});
+              }),
+            ),
+          );
+        },
+        { concurrency: ScenarioSchedulerService.CONCURRENCY_LIMIT },
+      ),
+    );
   }
 
   private async processRunToCompletion(runId: string): Promise<void> {