Add parallel processing for individual batch builds

a1_swap_mod_packer/batch.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+import os
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from dataclasses import dataclass
+from typing import Sequence
+
+from .builder import build_packed_3mf
+from .models import BuildOptions, BuildResult, PlateJob
+
+INDIVIDUAL_BATCH_MAX_WORKERS = 8
+
+
+@dataclass(frozen=True)
+class IndividualBuildTask:
+    job: PlateJob
+    options: BuildOptions
+
+
+@dataclass(frozen=True)
+class IndividualBuildFailure:
+    index: int
+    job: PlateJob
+    error: str
+
+
+@dataclass(frozen=True)
+class IndividualBatchBuildResult:
+    results: tuple[BuildResult, ...]
+    failures: tuple[IndividualBuildFailure, ...]
+    worker_count: int
+
+
+def individual_batch_worker_count(task_count: int, max_workers: int | None = None) -> int:
+    if task_count <= 0:
+        return 0
+    if max_workers is not None:
+        return min(task_count, max(1, int(max_workers)))
+    detected_cpu_count = os.cpu_count() or 1
+    return min(task_count, max(1, detected_cpu_count), INDIVIDUAL_BATCH_MAX_WORKERS)
+
+
+def _build_individual_task(task: IndividualBuildTask) -> BuildResult:
+    return build_packed_3mf([task.job], task.options)
+
+
+def _run_individual_batch_serial(
+    tasks: Sequence[IndividualBuildTask],
+    worker_count: int,
+) -> IndividualBatchBuildResult:
+    results: list[BuildResult] = []
+    failures: list[IndividualBuildFailure] = []
+    for index, task in enumerate(tasks):
+        try:
+            results.append(_build_individual_task(task))
+        except Exception as exc:
+            failures.append(IndividualBuildFailure(index, task.job, str(exc)))
+    return IndividualBatchBuildResult(tuple(results), tuple(failures), worker_count)
+
+
+def run_individual_batch_builds(
+    tasks: Sequence[IndividualBuildTask],
+    max_workers: int | None = None,
+) -> IndividualBatchBuildResult:
+    task_list = list(tasks)
+    worker_count = individual_batch_worker_count(len(task_list), max_workers)
+    if worker_count <= 1:
+        return _run_individual_batch_serial(task_list, worker_count)
+
+    ordered_results: list[BuildResult | None] = [None] * len(task_list)
+    failures: list[IndividualBuildFailure] = []
+    with ProcessPoolExecutor(max_workers=worker_count) as executor:
+        future_to_index = {
+            executor.submit(_build_individual_task, task): index
+            for index, task in enumerate(task_list)
+        }
+        for future in as_completed(future_to_index):
+            index = future_to_index[future]
+            task = task_list[index]
+            try:
+                ordered_results[index] = future.result()
+            except Exception as exc:
+                failures.append(IndividualBuildFailure(index, task.job, str(exc)))
+
+    results = tuple(result for result in ordered_results if result is not None)
+    return IndividualBatchBuildResult(
+        tuple(results),
+        tuple(sorted(failures, key=lambda item: item.index)),
+        worker_count,
+    )

a1_swap_mod_packer/gui.py

@@ -6,6 +6,7 @@ import re
 import subprocess
 import sys
 import zipfile
+from multiprocessing import freeze_support
 from pathlib import Path
 from typing import Any, Callable
 
@@ -44,6 +45,7 @@ except ImportError as exc:  # pragma: no cover
     raise SystemExit("PySide6 is required to run the GUI. Install it with: pip install PySide6") from exc
 
 from . import APP_NAME, APP_TITLE
+from .batch import IndividualBuildTask, individual_batch_worker_count, run_individual_batch_builds
 from .core import (
     BuildOptions,
     DEFAULT_ZIP_COMPRESS_LEVEL,
@@ -1122,6 +1124,7 @@ class MainWindow(QMainWindow):
         used_paths: set[Path] = set()
         success_count = 0
         errors: list[str] = []
+        tasks: list[IndividualBuildTask] = []
         for job in jobs:
             try:
                 output_path = make_unique_for_run(
@@ -1129,12 +1132,23 @@ class MainWindow(QMainWindow):
                     used_paths,
                 )
                 options = self.build_options_for_output(output_path)
-                result = build_packed_3mf([job], options)
-                self.log_build_result(result)
-                success_count += 1
             except Exception as exc:
                 errors.append(f"{job.source_3mf.name}: {exc}")
                 self.log.append(f"Error building {job.source_3mf}: {exc}")
+                continue
+            tasks.append(IndividualBuildTask(job, options))
+
+        if tasks:
+            worker_count = individual_batch_worker_count(len(tasks))
+            self.log.append(f"Building {len(tasks)} individual output(s) with {worker_count} worker(s).")
+            batch_result = run_individual_batch_builds(tasks, max_workers=worker_count)
+            for result in batch_result.results:
+                self.log_build_result(result)
+            success_count = len(batch_result.results)
+            for failure in batch_result.failures:
+                errors.append(f"{failure.job.source_3mf.name}: {failure.error}")
+                self.log.append(f"Error building {failure.job.source_3mf}: {failure.error}")
+
         if errors:
             QMessageBox.warning(
                 self,
@@ -1166,6 +1180,7 @@ class MainWindow(QMainWindow):
 
 
 def main() -> int:
+    freeze_support()
     app = QApplication(sys.argv)
     app.setFont(QFont("Segoe UI", 10))
     window = MainWindow()