Using Queue for dynamic thread processing

This commit is contained in:
henryruhs 2023-06-22 11:09:43 +02:00
parent fd8a9b6f94
commit 5835cd2b5b

View File

@ -3,6 +3,7 @@ import sys
import importlib import importlib
import psutil import psutil
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
from queue import Queue
from types import ModuleType from types import ModuleType
from typing import Any, List, Callable from typing import Any, List, Callable
from tqdm import tqdm from tqdm import tqdm
@ -43,13 +44,21 @@ def get_frame_processors_modules(frame_processors: List[str]) -> List[ModuleType
def multi_process_frame(source_path: str, temp_frame_paths: List[str], process_frames: Callable[[str, List[str], Any], None], update: Callable[[], None]) -> None: def multi_process_frame(source_path: str, temp_frame_paths: List[str], process_frames: Callable[[str, List[str], Any], None], update: Callable[[], None]) -> None:
with ThreadPoolExecutor(max_workers=roop.globals.execution_threads) as executor: with ThreadPoolExecutor(max_workers=roop.globals.execution_threads) as executor:
futures = [] futures = []
for path in temp_frame_paths: queue = create_queue(temp_frame_paths)
future = executor.submit(process_frames, source_path, [path], update) while not queue.empty():
future = executor.submit(process_frames, source_path, [queue.get()], update)
futures.append(future) futures.append(future)
for future in as_completed(futures): for future in as_completed(futures):
future.result() future.result()
def create_queue(temp_frame_paths: List[str]) -> Queue:
queue = Queue()
for frame_path in temp_frame_paths:
queue.put(frame_path)
return queue
def process_video(source_path: str, frame_paths: list[str], process_frames: Callable[[str, List[str], Any], None]) -> None: def process_video(source_path: str, frame_paths: list[str], process_frames: Callable[[str, List[str], Any], None]) -> None:
progress_bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}{postfix}]' progress_bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, {rate_fmt}{postfix}]'
total = len(frame_paths) total = len(frame_paths)