import os import sys import time import threading from multiprocessing import Process,…
- User
import os import sys import time import threading from multiprocessing import Process, Queue, get_context from multiprocessing.connection import Connection from typing import List, Literal, Dict, Optional import torch import PIL.Image from streamdiffusion.image_utils import pil2tensor import mss import fire import tkinter as tk
sys.path.append(os.path.join(os.path.dirname(file), "..", ".."))
from utils.viewer import receive_images from utils.wrapper import StreamDiffusionWrapper
inputs = [] top = 0 left = 0
def screen( event: threading.Event, height: int = 512, width: int = 512, monitor: Dict[str, int] = {"top": 300, "left": 200, "width": 512, "height": 512}, ): global inputs with mss.mss() as sct: while True: if event.is_set(): print("terminate read thread") break img = sct.grab(monitor) img = PIL.Image.frombytes("RGB", img.size, img.bgra, "raw", "BGRX") img.resize((height, width)) inputs.append(pil2tensor(img)) print('exit : screen') def dummy_screen( width: int, height: int, ): root = tk.Tk() root.title("Press Enter to start") root.geometry(f"{width}x{height}") root.resizable(False, False) root.attributes("-alpha", 0.8) root.configure(bg="black") def destroy(event): root.destroy() root.bind("", destroy) def update_geometry(event): global top, left top = root.winfo_y() left = root.winfo_x() root.bind("", update_geometry) root.mainloop() return {"top": top, "left": left, "width": width, "height": height}
def monitor_setting_process( width: int, height: int, monitor_sender: Connection, ) -> None: monitor = dummy_screen(width, height) monitor_sender.send(monitor)
def image_generation_process( queue: Queue, fps_queue: Queue, close_queue: Queue, model_id_or_path: str, lora_dict: Optional[Dict[str, float]], prompt: str, negative_prompt: str, frame_buffer_size: int, width: int, height: int, acceleration: Literal["none", "xformers", "tensorrt"], use_denoising_batch: bool, seed: int, cfg_type: Literal["none", "full", "self", "initialize"], guidance_scale: float, delta: float, do_add_noise: bool, enable_similar_image_filter: bool, similar_image_filter_threshold: float, similar_image_filter_max_skip_frame: float, monitor_receiver : Connection, ) -> None: """ Process for generating images based on a prompt using a specified model.
Parameters ---------- queue : Queue The queue to put the generated images in. fps_queue : Queue The queue to put the calculated fps. model_id_or_path : str The name of the model to use for image generation. lora_dict : Optional[Dict[str, float]], optional The lora_dict to load, by default None. Keys are the LoRA names and values are the LoRA scales. Example: {'LoRA_1' : 0.5 , 'LoRA_2' : 0.7 ,...} prompt : str The prompt to generate images from. negative_prompt : str, optional The negative prompt to use. frame_buffer_size : int, optional The frame buffer size for denoising batch, by default 1. width : int, optional The width of the image, by default 512. height : int, optional The height of the image, by default 512. acceleration : Literal["none", "xformers", "tensorrt"], optional The acceleration method, by default "tensorrt". use_denoising_batch : bool, optional Whether to use denoising batch or not, by default True. seed : int, optional The seed, by default 2. if -1, use random seed. cfg_type : Literal["none", "full", "self", "initialize"], optional The cfg_type for img2img mode, by default "self". You cannot use anything other than "none" for txt2img mode. guidance_scale : float, optional The CFG scale, by default 1.2. delta : float, optional The delta multiplier of virtual residual noise, by default 1.0. do_add_noise : bool, optional Whether to add noise for following denoising steps or not, by default True. enable_similar_image_filter : bool, optional Whether to enable similar image filter or not, by default False. similar_image_filter_threshold : float, optional The threshold for similar image filter, by default 0.98. similar_image_filter_max_skip_frame : int, optional The max skip frame for similar image filter, by default 10. """ global inputs stream = StreamDiffusionWrapper( model_id_or_path=model_id_or_path, lora_dict=lora_dict, t_index_list=[32, 45], frame_buffer_size=frame_buffer_size, width=width, height=height, warmup=10, acceleration=acceleration, do_add_noise=do_add_noise, enable_similar_image_filter=enable_similar_image_filter, similar_image_filter_threshold=similar_image_filter_threshold, similar_image_filter_max_skip_frame=similar_image_filter_max_skip_frame, mode="img2img", use_denoising_batch=use_denoising_batch, cfg_type=cfg_type, seed=seed, ) stream.prepare( prompt=prompt, negative_prompt=negative_prompt, num_inference_steps=50, guidance_scale=guidance_scale, delta=delta, ) monitor = monitor_receiver.recv() event = threading.Event() input_screen = threading.Thread(target=screen, args=(event, height, width, monitor)) input_screen.start() time.sleep(5) while True: try: if not close_queue.empty(): # closing check break if len(inputs) < frame_buffer_size: time.sleep(0.005) continue start_time = time.time() sampled_inputs = [] for i in range(frame_buffer_size): index = (len(inputs) // frame_buffer_size) * i sampled_inputs.append(inputs[len(inputs) - index - 1]) input_batch = torch.cat(sampled_inputs) inputs.clear() output_images = stream.stream( input_batch.to(device=stream.device, dtype=stream.dtype) ).cpu() if frame_buffer_size == 1: output_images = [output_images] for output_image in output_images: queue.put(output_image, block=False) fps = 1 / (time.time() - start_time) fps_queue.put(fps) except KeyboardInterrupt: break print("closing image_generation_process...") event.set() # stop capture thread input_screen.join() print(f"fps: {fps}")def main( model_id_or_path: str = "KBlueLeaf/kohaku-v2.1", lora_dict: Optional[Dict[str, float]] = None, prompt: str = "1girl with brown dog hair, thick glasses, smiling", negative_prompt: str = "low quality, bad quality, blurry, low resolution", frame_buffer_size: int = 1, width: int = 512, height: int = 512, acceleration: Literal["none", "xformers", "tensorrt"] = "xformers", use_denoising_batch: bool = True, seed: int = 2, cfg_type: Literal["none", "full", "self", "initialize"] = "self", guidance_scale: float = 1.4, delta: float = 0.5, do_add_noise: bool = False, enable_similar_image_filter: bool = True, similar_image_filter_threshold: float = 0.99, similar_image_filter_max_skip_frame: float = 10, ) -> None: """ Main function to start the image generation and viewer processes. """ ctx = get_context('spawn') queue = ctx.Queue() fps_queue = ctx.Queue() close_queue = Queue()
monitor_sender, monitor_receiver = ctx.Pipe() process1 = ctx.Process( target=image_generation_process, args=( queue, fps_queue, close_queue, model_id_or_path, lora_dict, prompt, negative_prompt, frame_buffer_size, width, height, acceleration, use_denoising_batch, seed, cfg_type, guidance_scale, delta, do_add_noise, enable_similar_image_filter, similar_image_filter_threshold, similar_image_filter_max_skip_frame, monitor_receiver, ), ) process1.start() monitor_process = ctx.Process( target=monitor_setting_process, args=( width, height, monitor_sender, ), ) monitor_process.start() monitor_process.join() process2 = ctx.Process(target=receive_images, args=(queue, fps_queue)) process2.start() # terminate process2.join() print("process2 terminated.") close_queue.put(True) print("process1 terminating...") process1.join(5) # with timeout if process1.is_alive(): print("process1 still alive. force killing...") process1.terminate() # force kill... process1.join() print("process1 terminated.")if name == "main": fire.Fire(main) create a stream from multi input
- Assistant
It looks like you're trying to modify your existing code to create a stream from multiple inputs. Specifically, you want your application to handle multiple input sources (e.g., multiple screen regions, cameras, or image sources) and process them concurrently.
To achieve this, you need to modify your code to:
- Capture multiple inputs simultaneously.
- Process each input independently or combine them as needed.
- Stream the outputs in a synchronized manner.
Below, I'll guide you through the steps to modify your code to handle multiple inputs. I'll provide code examples and explanations to help you understand each part.
1. Modify the Screen Capture Function to Handle Multiple Inputs
First, we need to adjust the
screenfunction to handle multiple monitor settings. We'll create separate threads for each input source.a. Define Multiple Monitor Regions
Suppose you want to capture two different regions of the screen. You can define multiple monitor dictionaries:
monitor1 = {"top": 100, "left": 100, "width": 512, "height": 512} monitor2 = {"top": 100, "left": 700, "width": 512, "height": 512}b. Modify the Screen Function
Update the
screenfunction to accept a list of monitors and corresponding input queues.def screen( event: threading.Event, height: int, width: int, monitors: List[Dict[str, int]], input_queues: List[Queue], ): with mss.mss() as sct: while not event.is_set(): for idx, monitor in enumerate(monitors): img = sct.grab(monitor) img = PIL.Image.frombytes("RGB", img.size, img.bgra, "raw", "BGRX") img = img.resize((height, width)) img_tensor = pil2tensor(img) try: input_queues[idx].put_nowait(img_tensor) except queue.Full: pass # Handle full queue if neededc. Create Threads for Each Input
We'll create a separate thread for each input source.
def start_screen_capture( height: int, width: int, monitors: List[Dict[str, int]], ): event = threading.Event() input_queues = [Queue(maxsize=10) for _ in monitors] capture_thread = threading.Thread( target=screen, args=(event, height, width, monitors, input_queues), ) capture_thread.start() return event, input_queues, capture_thread
2. Modify the Image Generation Process to Handle Multiple Inputs
Adjust the
image_generation_processfunction to process inputs from multiple queues.a. Update the Function Signature
Modify the
image_generation_processto accept multiple input queues.def image_generation_process( input_queues: List[Queue], queue: Queue, fps_queue: Queue, close_queue: Queue, # ... other parameters ): # Rest of the codeb. Process Inputs from Multiple Queues
In the processing loop, fetch inputs from each queue.
while True: try: if not close_queue.empty(): break input_batches = [] for input_queue in input_queues: if input_queue.empty(): continue img_tensor = input_queue.get() input_batches.append(img_tensor) if not input_batches: time.sleep(0.005) continue input_batch = torch.cat(input_batches) output_images = stream.stream( input_batch.to(device=stream.device, dtype=stream.dtype) ).cpu() # Handle outputs (depending on your frame buffer size and processing) for output_image in output_images: queue.put(output_image, block=False) fps = 1 / (time.time() - start_time) fps_queue.put(fps) except KeyboardInterrupt: break
3. Update the Main Function to Initialize Multiple Inputs
In your
mainfunction, set up the monitors and initialize the processes to handle multiple inputs.a. Define the Monitors
Define the monitor regions you want to capture.
def main( # ... existing parameters monitors: Optional[List[Dict[str, int]]] = None, ): if monitors is None: monitors = [ {"top": 100, "left": 100, "width": 512, "height": 512}, {"top": 100, "left": 700, "width": 512, "height": 512}, ]b. Start Screen Capture Threads
Initialize the screen capture.
ctx = get_context('spawn') event, input_queues, capture_thread = start_screen_capture( height, width, monitors, )c. Start the Image Generation Process
Pass the
input_queuesto theimage_generation_process.process1 = ctx.Process( target=image_generation_process, args=( input_queues, queue, fps_queue, close_queue, model_id_or_path, lora_dict, prompt, negative_prompt, frame_buffer_size, width, height, acceleration, use_denoising_batch, seed, cfg_type, guidance_scale, delta, do_add_noise, enable_similar_image_filter, similar_image_filter_threshold, similar_image_filter_max_skip_frame, monitor_receiver, ), ) process1.start()
4. Optional: Handle Multiple Prompts or Processing Parameters
If you need each input to be processed with different prompts or parameters, you can adjust your code accordingly.
a. Modify the Image Generation Process
Pass a list of prompts or processing parameters.
def image_generation_process( input_queues: List[Queue], prompts: List[str], # ... other parameters ): # Initialize separate StreamDiffusionWrapper instances if needed streams = [] for prompt in prompts: stream = StreamDiffusionWrapper( # ... initialization parameters ) stream.prepare( prompt=prompt, # ... other parameters ) streams.append(stream) while True: # Process inputs with corresponding streams
5. Combine or Display the Outputs
Finally, you need to handle the output images, whether you want to combine them into a single image or display them separately.
a. Modify the Receive Images Function
Update
receive_imagesto handle multiple outputs.def receive_images(queue: Queue, fps_queue: Queue, num_streams: int): while True: try: output_images = [queue.get(timeout=5) for _ in range(num_streams)] # Combine or display the images except queue.Empty: break
Complete Example
Below is the updated version of your main code sections, incorporating the changes:
# ... [imports and other code unchanged] def screen( event: threading.Event, height: int, width: int, monitors: List[Dict[str, int]], input_queues: List[Queue], ): with mss.mss() as sct: while not event.is_set(): for idx, monitor in enumerate(monitors): img = sct.grab(monitor) img = PIL.Image.frombytes("RGB", img.size, img.bgra, "raw", "BGRX") img = img.resize((height, width)) img_tensor = pil2tensor(img) try: input_queues[idx].put_nowait(img_tensor) except queue.Full: pass def start_screen_capture( height: int, width: int, monitors: List[Dict[str, int]], ): event = threading.Event() input_queues = [Queue(maxsize=10) for _ in monitors] capture_thread = threading.Thread( target=screen, args=(event, height, width, monitors, input_queues), ) capture_thread.start() return event, input_queues, capture_thread def image_generation_process( input_queues: List[Queue], queue: Queue, fps_queue: Queue, close_queue: Queue, model_id_or_path: str, lora_dict: Optional[Dict[str, float]], prompt: str, negative_prompt: str, frame_buffer_size: int, width: int, height: int, acceleration: Literal["none", "xformers", "tensorrt"], use_denoising_batch: bool, seed: int, cfg_type: Literal["none", "full", "self", "initialize"], guidance_scale: float, delta: float, do_add_noise: bool, enable_similar_image_filter: bool, similar_image_filter_threshold: float, similar_image_filter_max_skip_frame: float, monitor_receiver : Connection, ) -> None: streams = [] for _ in input_queues: stream = StreamDiffusionWrapper( model_id_or_path=model_id_or_path, lora_dict=lora_dict, t_index_list=[32, 45], frame_buffer_size=frame_buffer_size, width=width, height=height, warmup=10, acceleration=acceleration, do_add_noise=do_add_noise, enable_similar_image_filter=enable_similar_image_filter, similar_image_filter_threshold=similar_image_filter_threshold, similar_image_filter_max_skip_frame=similar_image_filter_max_skip_frame, mode="img2img", use_denoising_batch=use_denoising_batch, cfg_type=cfg_type, seed=seed, ) stream.prepare( prompt=prompt, negative_prompt=negative_prompt, num_inference_steps=50, guidance_scale=guidance_scale, delta=delta, ) streams.append(stream) monitor = monitor_receiver.recv() start_time = time.time() while True: try: if not close_queue.empty(): break input_batches = [] for input_queue in input_queues: if input_queue.empty(): continue img_tensor = input_queue.get() input_batches.append(img_tensor) if not input_batches: time.sleep(0.005) continue for img_tensor, stream in zip(input_batches, streams): output_image = stream.stream( img_tensor.unsqueeze(0).to(device=stream.device, dtype=stream.dtype) ).cpu() queue.put(output_image, block=False) fps = len(input_batches) / (time.time() - start_time) fps_queue.put(fps) start_time = time.time() except KeyboardInterrupt: break def main( model_id_or_path: str = "KBlueLeaf/kohaku-v2.1", lora_dict: Optional[Dict[str, float]] = None, prompt: str = "1girl with brown dog hair, thick glasses, smiling", negative_prompt: str = "low quality, bad quality, blurry, low resolution", frame_buffer_size: int = 1, width: int = 512, height: int = 512, acceleration: Literal["none", "xformers", "tensorrt"] = "xformers", use_denoising_batch: bool = True, seed: int = 2, cfg_type: Literal["none", "full", "self", "initialize"] = "self", guidance_scale: float = 1.4, delta: float = 0.5, do_add_noise: bool = False, enable_similar_image_filter: bool = True, similar_image_filter_threshold: float = 0.99, similar_image_filter_max_skip_frame: float = 10, monitors: Optional[List[Dict[str, int]]] = None, ) -> None: if monitors is None: monitors = [ {"top": 100, "left": 100, "width": 512, "height": 512}, {"top": 100, "left": 700, "width": 512, "height": 512}, ] ctx = get_context('spawn') queue = ctx.Queue() fps_queue = ctx.Queue() close_queue = Queue() monitor_sender, monitor_receiver = ctx.Pipe() event, input_queues, capture_thread = start_screen_capture( height, width, monitors, ) process1 = ctx.Process( target=image_generation_process, args=( input_queues, queue, fps_queue, close_queue, model_id_or_path, lora_dict, prompt, negative_prompt, frame_buffer_size, width, height, acceleration, use_denoising_batch, seed, cfg_type, guidance_scale, delta, do_add_noise, enable_similar_image_filter, similar_image_filter_threshold, similar_image_filter_max_skip_frame, monitor_receiver, ), ) process1.start() monitor_process = ctx.Process( target=monitor_setting_process, args=( width, height, monitor_sender, ), ) monitor_process.start() monitor_process.join() process2 = ctx.Process( target=receive_images, args=(queue, fps_queue, len(monitors)), ) process2.start() # Terminate processes process2.join() close_queue.put(True) process1.join(5) if process1.is_alive(): process1.terminate() process1.join() event.set() capture_thread.join() if __name__ == "__main__": fire.Fire(main)
Notes and Considerations
- Thread Safety: Be cautious with threading and shared resources. Ensure that data structures accessed by multiple threads are thread-safe.
- Performance: Processing multiple inputs may increase the computational load. Monitor your system's performance and optimize as needed.
- Synchronization: If you need the inputs or outputs to be synchronized (e.g., combined into a single frame), implement synchronization mechanisms.
- Error Handling: Add appropriate error handling to manage exceptions and edge cases.
- Resource Cleanup: Ensure that all threads and processes are properly terminated, and resources are released when the application exits.
By modifying your code as shown, you can create a stream from multiple inputs. Each input is processed independently, and you can handle the outputs according to your application's requirements.
Let me know if you need further assistance or have any questions!