Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

import os import sys import time import threading from multiprocessing import Process,…

  1. User

    import os import sys import time import threading from multiprocessing import Process, Queue, get_context from multiprocessing.connection import Connection from typing import List, Literal, Dict, Optional import torch import PIL.Image from streamdiffusion.image_utils import pil2tensor import mss import fire import tkinter as tk

    sys.path.append(os.path.join(os.path.dirname(file), "..", ".."))

    from utils.viewer import receive_images from utils.wrapper import StreamDiffusionWrapper

    inputs = [] top = 0 left = 0

    def screen( event: threading.Event, height: int = 512, width: int = 512, monitor: Dict[str, int] = {"top": 300, "left": 200, "width": 512, "height": 512}, ): global inputs with mss.mss() as sct: while True: if event.is_set(): print("terminate read thread") break img = sct.grab(monitor) img = PIL.Image.frombytes("RGB", img.size, img.bgra, "raw", "BGRX") img.resize((height, width)) inputs.append(pil2tensor(img)) print('exit : screen') def dummy_screen( width: int, height: int, ): root = tk.Tk() root.title("Press Enter to start") root.geometry(f"{width}x{height}") root.resizable(False, False) root.attributes("-alpha", 0.8) root.configure(bg="black") def destroy(event): root.destroy() root.bind("", destroy) def update_geometry(event): global top, left top = root.winfo_y() left = root.winfo_x() root.bind("", update_geometry) root.mainloop() return {"top": top, "left": left, "width": width, "height": height}

    def monitor_setting_process( width: int, height: int, monitor_sender: Connection, ) -> None: monitor = dummy_screen(width, height) monitor_sender.send(monitor)

    def image_generation_process( queue: Queue, fps_queue: Queue, close_queue: Queue, model_id_or_path: str, lora_dict: Optional[Dict[str, float]], prompt: str, negative_prompt: str, frame_buffer_size: int, width: int, height: int, acceleration: Literal["none", "xformers", "tensorrt"], use_denoising_batch: bool, seed: int, cfg_type: Literal["none", "full", "self", "initialize"], guidance_scale: float, delta: float, do_add_noise: bool, enable_similar_image_filter: bool, similar_image_filter_threshold: float, similar_image_filter_max_skip_frame: float, monitor_receiver : Connection, ) -> None: """ Process for generating images based on a prompt using a specified model.

    Parameters
    ----------
    queue : Queue
        The queue to put the generated images in.
    fps_queue : Queue
        The queue to put the calculated fps.
    model_id_or_path : str
        The name of the model to use for image generation.
    lora_dict : Optional[Dict[str, float]], optional
        The lora_dict to load, by default None.
        Keys are the LoRA names and values are the LoRA scales.
        Example: {'LoRA_1' : 0.5 , 'LoRA_2' : 0.7 ,...}
    prompt : str
        The prompt to generate images from.
    negative_prompt : str, optional
        The negative prompt to use.
    frame_buffer_size : int, optional
        The frame buffer size for denoising batch, by default 1.
    width : int, optional
        The width of the image, by default 512.
    height : int, optional
        The height of the image, by default 512.
    acceleration : Literal["none", "xformers", "tensorrt"], optional
        The acceleration method, by default "tensorrt".
    use_denoising_batch : bool, optional
        Whether to use denoising batch or not, by default True.
    seed : int, optional
        The seed, by default 2. if -1, use random seed.
    cfg_type : Literal["none", "full", "self", "initialize"],
    optional
        The cfg_type for img2img mode, by default "self".
        You cannot use anything other than "none" for txt2img mode.
    guidance_scale : float, optional
        The CFG scale, by default 1.2.
    delta : float, optional
        The delta multiplier of virtual residual noise,
        by default 1.0.
    do_add_noise : bool, optional
        Whether to add noise for following denoising steps or not,
        by default True.
    enable_similar_image_filter : bool, optional
        Whether to enable similar image filter or not,
        by default False.
    similar_image_filter_threshold : float, optional
        The threshold for similar image filter, by default 0.98.
    similar_image_filter_max_skip_frame : int, optional
        The max skip frame for similar image filter, by default 10.
    """
    
    global inputs
    stream = StreamDiffusionWrapper(
        model_id_or_path=model_id_or_path,
        lora_dict=lora_dict,
        t_index_list=[32, 45],
        frame_buffer_size=frame_buffer_size,
        width=width,
        height=height,
        warmup=10,
        acceleration=acceleration,
        do_add_noise=do_add_noise,
        enable_similar_image_filter=enable_similar_image_filter,
        similar_image_filter_threshold=similar_image_filter_threshold,
        similar_image_filter_max_skip_frame=similar_image_filter_max_skip_frame,
        mode="img2img",
        use_denoising_batch=use_denoising_batch,
        cfg_type=cfg_type,
        seed=seed,
    )
    
    stream.prepare(
        prompt=prompt,
        negative_prompt=negative_prompt,
        num_inference_steps=50,
        guidance_scale=guidance_scale,
        delta=delta,
    )
    
    monitor = monitor_receiver.recv()
    
    event = threading.Event()
    input_screen = threading.Thread(target=screen, args=(event, height, width, monitor))
    input_screen.start()
    time.sleep(5)
    
    while True:
        try:
            if not close_queue.empty(): # closing check
                break
            if len(inputs) < frame_buffer_size:
                time.sleep(0.005)
                continue
            start_time = time.time()
            sampled_inputs = []
            for i in range(frame_buffer_size):
                index = (len(inputs) // frame_buffer_size) * i
                sampled_inputs.append(inputs[len(inputs) - index - 1])
            input_batch = torch.cat(sampled_inputs)
            inputs.clear()
            output_images = stream.stream(
                input_batch.to(device=stream.device, dtype=stream.dtype)
            ).cpu()
            if frame_buffer_size == 1:
                output_images = [output_images]
            for output_image in output_images:
                queue.put(output_image, block=False)
    
            fps = 1 / (time.time() - start_time)
            fps_queue.put(fps)
        except KeyboardInterrupt:
            break
    
    print("closing image_generation_process...")
    event.set() # stop capture thread
    input_screen.join()
    print(f"fps: {fps}")
    

    def main( model_id_or_path: str = "KBlueLeaf/kohaku-v2.1", lora_dict: Optional[Dict[str, float]] = None, prompt: str = "1girl with brown dog hair, thick glasses, smiling", negative_prompt: str = "low quality, bad quality, blurry, low resolution", frame_buffer_size: int = 1, width: int = 512, height: int = 512, acceleration: Literal["none", "xformers", "tensorrt"] = "xformers", use_denoising_batch: bool = True, seed: int = 2, cfg_type: Literal["none", "full", "self", "initialize"] = "self", guidance_scale: float = 1.4, delta: float = 0.5, do_add_noise: bool = False, enable_similar_image_filter: bool = True, similar_image_filter_threshold: float = 0.99, similar_image_filter_max_skip_frame: float = 10, ) -> None: """ Main function to start the image generation and viewer processes. """ ctx = get_context('spawn') queue = ctx.Queue() fps_queue = ctx.Queue() close_queue = Queue()

    monitor_sender, monitor_receiver = ctx.Pipe()
    
    process1 = ctx.Process(
        target=image_generation_process,
        args=(
            queue,
            fps_queue,
            close_queue,
            model_id_or_path,
            lora_dict,
            prompt,
            negative_prompt,
            frame_buffer_size,
            width,
            height,
            acceleration,
            use_denoising_batch,
            seed,
            cfg_type,
            guidance_scale,
            delta,
            do_add_noise,
            enable_similar_image_filter,
            similar_image_filter_threshold,
            similar_image_filter_max_skip_frame,
            monitor_receiver,
            ),
    )
    process1.start()
    
    monitor_process = ctx.Process(
        target=monitor_setting_process,
        args=(
            width,
            height,
            monitor_sender,
            ),
    )
    monitor_process.start()
    monitor_process.join()
    
    process2 = ctx.Process(target=receive_images, args=(queue, fps_queue))
    process2.start()
    
    # terminate
    process2.join()
    print("process2 terminated.")
    close_queue.put(True)
    print("process1 terminating...")
    process1.join(5) # with timeout
    if process1.is_alive():
        print("process1 still alive. force killing...")
        process1.terminate() # force kill...
    process1.join()
    print("process1 terminated.")
    

    if name == "main": fire.Fire(main) create a stream from multi input

  2. Assistant

    It looks like you're trying to modify your existing code to create a stream from multiple inputs. Specifically, you want your application to handle multiple input sources (e.g., multiple screen regions, cameras, or image sources) and process them concurrently.

    To achieve this, you need to modify your code to:

    1. Capture multiple inputs simultaneously.
    2. Process each input independently or combine them as needed.
    3. Stream the outputs in a synchronized manner.

    Below, I'll guide you through the steps to modify your code to handle multiple inputs. I'll provide code examples and explanations to help you understand each part.


    1. Modify the Screen Capture Function to Handle Multiple Inputs

    First, we need to adjust the screen function to handle multiple monitor settings. We'll create separate threads for each input source.

    a. Define Multiple Monitor Regions

    Suppose you want to capture two different regions of the screen. You can define multiple monitor dictionaries:

    monitor1 = {"top": 100, "left": 100, "width": 512, "height": 512}
    monitor2 = {"top": 100, "left": 700, "width": 512, "height": 512}
    

    b. Modify the Screen Function

    Update the screen function to accept a list of monitors and corresponding input queues.

    def screen(
        event: threading.Event,
        height: int,
        width: int,
        monitors: List[Dict[str, int]],
        input_queues: List[Queue],
    ):
        with mss.mss() as sct:
            while not event.is_set():
                for idx, monitor in enumerate(monitors):
                    img = sct.grab(monitor)
                    img = PIL.Image.frombytes("RGB", img.size, img.bgra, "raw", "BGRX")
                    img = img.resize((height, width))
                    img_tensor = pil2tensor(img)
                    try:
                        input_queues[idx].put_nowait(img_tensor)
                    except queue.Full:
                        pass  # Handle full queue if needed
    

    c. Create Threads for Each Input

    We'll create a separate thread for each input source.

    def start_screen_capture(
        height: int,
        width: int,
        monitors: List[Dict[str, int]],
    ):
        event = threading.Event()
        input_queues = [Queue(maxsize=10) for _ in monitors]
        capture_thread = threading.Thread(
            target=screen,
            args=(event, height, width, monitors, input_queues),
        )
        capture_thread.start()
        return event, input_queues, capture_thread
    

    2. Modify the Image Generation Process to Handle Multiple Inputs

    Adjust the image_generation_process function to process inputs from multiple queues.

    a. Update the Function Signature

    Modify the image_generation_process to accept multiple input queues.

    def image_generation_process(
        input_queues: List[Queue],
        queue: Queue,
        fps_queue: Queue,
        close_queue: Queue,
        # ... other parameters
    ):
        # Rest of the code
    

    b. Process Inputs from Multiple Queues

    In the processing loop, fetch inputs from each queue.

    while True:
        try:
            if not close_queue.empty():
                break
    
            input_batches = []
            for input_queue in input_queues:
                if input_queue.empty():
                    continue
                img_tensor = input_queue.get()
                input_batches.append(img_tensor)
    
            if not input_batches:
                time.sleep(0.005)
                continue
    
            input_batch = torch.cat(input_batches)
            output_images = stream.stream(
                input_batch.to(device=stream.device, dtype=stream.dtype)
            ).cpu()
    
            # Handle outputs (depending on your frame buffer size and processing)
            for output_image in output_images:
                queue.put(output_image, block=False)
    
            fps = 1 / (time.time() - start_time)
            fps_queue.put(fps)
        except KeyboardInterrupt:
            break
    

    3. Update the Main Function to Initialize Multiple Inputs

    In your main function, set up the monitors and initialize the processes to handle multiple inputs.

    a. Define the Monitors

    Define the monitor regions you want to capture.

    def main(
        # ... existing parameters
        monitors: Optional[List[Dict[str, int]]] = None,
    ):
        if monitors is None:
            monitors = [
                {"top": 100, "left": 100, "width": 512, "height": 512},
                {"top": 100, "left": 700, "width": 512, "height": 512},
            ]
    

    b. Start Screen Capture Threads

    Initialize the screen capture.

    ctx = get_context('spawn')
    event, input_queues, capture_thread = start_screen_capture(
        height,
        width,
        monitors,
    )
    

    c. Start the Image Generation Process

    Pass the input_queues to the image_generation_process.

    process1 = ctx.Process(
        target=image_generation_process,
        args=(
            input_queues,
            queue,
            fps_queue,
            close_queue,
            model_id_or_path,
            lora_dict,
            prompt,
            negative_prompt,
            frame_buffer_size,
            width,
            height,
            acceleration,
            use_denoising_batch,
            seed,
            cfg_type,
            guidance_scale,
            delta,
            do_add_noise,
            enable_similar_image_filter,
            similar_image_filter_threshold,
            similar_image_filter_max_skip_frame,
            monitor_receiver,
        ),
    )
    process1.start()
    

    4. Optional: Handle Multiple Prompts or Processing Parameters

    If you need each input to be processed with different prompts or parameters, you can adjust your code accordingly.

    a. Modify the Image Generation Process

    Pass a list of prompts or processing parameters.

    def image_generation_process(
        input_queues: List[Queue],
        prompts: List[str],
        # ... other parameters
    ):
        # Initialize separate StreamDiffusionWrapper instances if needed
        streams = []
        for prompt in prompts:
            stream = StreamDiffusionWrapper(
                # ... initialization parameters
            )
            stream.prepare(
                prompt=prompt,
                # ... other parameters
            )
            streams.append(stream)
    
        while True:
            # Process inputs with corresponding streams
    

    5. Combine or Display the Outputs

    Finally, you need to handle the output images, whether you want to combine them into a single image or display them separately.

    a. Modify the Receive Images Function

    Update receive_images to handle multiple outputs.

    def receive_images(queue: Queue, fps_queue: Queue, num_streams: int):
        while True:
            try:
                output_images = [queue.get(timeout=5) for _ in range(num_streams)]
                # Combine or display the images
            except queue.Empty:
                break
    

    Complete Example

    Below is the updated version of your main code sections, incorporating the changes:

    # ... [imports and other code unchanged]
    
    def screen(
        event: threading.Event,
        height: int,
        width: int,
        monitors: List[Dict[str, int]],
        input_queues: List[Queue],
    ):
        with mss.mss() as sct:
            while not event.is_set():
                for idx, monitor in enumerate(monitors):
                    img = sct.grab(monitor)
                    img = PIL.Image.frombytes("RGB", img.size, img.bgra, "raw", "BGRX")
                    img = img.resize((height, width))
                    img_tensor = pil2tensor(img)
                    try:
                        input_queues[idx].put_nowait(img_tensor)
                    except queue.Full:
                        pass
    
    def start_screen_capture(
        height: int,
        width: int,
        monitors: List[Dict[str, int]],
    ):
        event = threading.Event()
        input_queues = [Queue(maxsize=10) for _ in monitors]
        capture_thread = threading.Thread(
            target=screen,
            args=(event, height, width, monitors, input_queues),
        )
        capture_thread.start()
        return event, input_queues, capture_thread
    
    def image_generation_process(
        input_queues: List[Queue],
        queue: Queue,
        fps_queue: Queue,
        close_queue: Queue,
        model_id_or_path: str,
        lora_dict: Optional[Dict[str, float]],
        prompt: str,
        negative_prompt: str,
        frame_buffer_size: int,
        width: int,
        height: int,
        acceleration: Literal["none", "xformers", "tensorrt"],
        use_denoising_batch: bool,
        seed: int,
        cfg_type: Literal["none", "full", "self", "initialize"],
        guidance_scale: float,
        delta: float,
        do_add_noise: bool,
        enable_similar_image_filter: bool,
        similar_image_filter_threshold: float,
        similar_image_filter_max_skip_frame: float,
        monitor_receiver : Connection,
    ) -> None:
        streams = []
        for _ in input_queues:
            stream = StreamDiffusionWrapper(
                model_id_or_path=model_id_or_path,
                lora_dict=lora_dict,
                t_index_list=[32, 45],
                frame_buffer_size=frame_buffer_size,
                width=width,
                height=height,
                warmup=10,
                acceleration=acceleration,
                do_add_noise=do_add_noise,
                enable_similar_image_filter=enable_similar_image_filter,
                similar_image_filter_threshold=similar_image_filter_threshold,
                similar_image_filter_max_skip_frame=similar_image_filter_max_skip_frame,
                mode="img2img",
                use_denoising_batch=use_denoising_batch,
                cfg_type=cfg_type,
                seed=seed,
            )
            stream.prepare(
                prompt=prompt,
                negative_prompt=negative_prompt,
                num_inference_steps=50,
                guidance_scale=guidance_scale,
                delta=delta,
            )
            streams.append(stream)
    
        monitor = monitor_receiver.recv()
        start_time = time.time()
        while True:
            try:
                if not close_queue.empty():
                    break
    
                input_batches = []
                for input_queue in input_queues:
                    if input_queue.empty():
                        continue
                    img_tensor = input_queue.get()
                    input_batches.append(img_tensor)
    
                if not input_batches:
                    time.sleep(0.005)
                    continue
    
                for img_tensor, stream in zip(input_batches, streams):
                    output_image = stream.stream(
                        img_tensor.unsqueeze(0).to(device=stream.device, dtype=stream.dtype)
                    ).cpu()
                    queue.put(output_image, block=False)
    
                fps = len(input_batches) / (time.time() - start_time)
                fps_queue.put(fps)
                start_time = time.time()
            except KeyboardInterrupt:
                break
    
    def main(
        model_id_or_path: str = "KBlueLeaf/kohaku-v2.1",
        lora_dict: Optional[Dict[str, float]] = None,
        prompt: str = "1girl with brown dog hair, thick glasses, smiling",
        negative_prompt: str = "low quality, bad quality, blurry, low resolution",
        frame_buffer_size: int = 1,
        width: int = 512,
        height: int = 512,
        acceleration: Literal["none", "xformers", "tensorrt"] = "xformers",
        use_denoising_batch: bool = True,
        seed: int = 2,
        cfg_type: Literal["none", "full", "self", "initialize"] = "self",
        guidance_scale: float = 1.4,
        delta: float = 0.5,
        do_add_noise: bool = False,
        enable_similar_image_filter: bool = True,
        similar_image_filter_threshold: float = 0.99,
        similar_image_filter_max_skip_frame: float = 10,
        monitors: Optional[List[Dict[str, int]]] = None,
    ) -> None:
        if monitors is None:
            monitors = [
                {"top": 100, "left": 100, "width": 512, "height": 512},
                {"top": 100, "left": 700, "width": 512, "height": 512},
            ]
    
        ctx = get_context('spawn')
        queue = ctx.Queue()
        fps_queue = ctx.Queue()
        close_queue = Queue()
    
        monitor_sender, monitor_receiver = ctx.Pipe()
    
        event, input_queues, capture_thread = start_screen_capture(
            height,
            width,
            monitors,
        )
    
        process1 = ctx.Process(
            target=image_generation_process,
            args=(
                input_queues,
                queue,
                fps_queue,
                close_queue,
                model_id_or_path,
                lora_dict,
                prompt,
                negative_prompt,
                frame_buffer_size,
                width,
                height,
                acceleration,
                use_denoising_batch,
                seed,
                cfg_type,
                guidance_scale,
                delta,
                do_add_noise,
                enable_similar_image_filter,
                similar_image_filter_threshold,
                similar_image_filter_max_skip_frame,
                monitor_receiver,
            ),
        )
        process1.start()
    
        monitor_process = ctx.Process(
            target=monitor_setting_process,
            args=(
                width,
                height,
                monitor_sender,
            ),
        )
        monitor_process.start()
        monitor_process.join()
    
        process2 = ctx.Process(
            target=receive_images,
            args=(queue, fps_queue, len(monitors)),
        )
        process2.start()
    
        # Terminate processes
        process2.join()
        close_queue.put(True)
        process1.join(5)
        if process1.is_alive():
            process1.terminate()
        process1.join()
        event.set()
        capture_thread.join()
    
    if __name__ == "__main__":
        fire.Fire(main)
    

    Notes and Considerations

    • Thread Safety: Be cautious with threading and shared resources. Ensure that data structures accessed by multiple threads are thread-safe.
    • Performance: Processing multiple inputs may increase the computational load. Monitor your system's performance and optimize as needed.
    • Synchronization: If you need the inputs or outputs to be synchronized (e.g., combined into a single frame), implement synchronization mechanisms.
    • Error Handling: Add appropriate error handling to manage exceptions and edge cases.
    • Resource Cleanup: Ensure that all threads and processes are properly terminated, and resources are released when the application exits.

    By modifying your code as shown, you can create a stream from multiple inputs. Each input is processed independently, and you can handle the outputs according to your application's requirements.

    Let me know if you need further assistance or have any questions!