Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

why this code generating image slower that another script in trainer program import…

  1. User

    why this code generating image slower that another script in trainer program

    import gradio as gr import numpy as np import random import torch from diffusers import PixArtSigmaPipeline, EulerAncestralDiscreteScheduler,DPMSolverMultistepScheduler

    device = "cuda" if torch.cuda.is_available() else "cpu"

    #torch.set_float32_matmul_precision("high")

    #torch._inductor.config.conv_1x1_as_mm = True #torch._inductor.config.coordinate_descent_tuning = True #torch._inductor.config.epilogue_fusion = False #torch._inductor.config.coordinate_descent_check_all_directions = True

    pipe = PixArtSigmaPipeline.from_pretrained( "H:/OneTrainer/models/test_sigma", torch_dtype=torch.float16, #max_sequence_length = 120 ).to("cuda") pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)

    #pipe.transformer.to(memory_format=torch.channels_last) #pipe.vae.to(memory_format=torch.channels_last)

    #pipe.transformer = torch.compile(pipe.transformer, mode="max-autotune", fullgraph=True) #pipe.vae.decode = torch.compile(pipe.vae.decode, mode="max-autotune", fullgraph=True)

    MAX_SEED = np.iinfo(np.int32).max MAX_IMAGE_SIZE = 2048

    def infer(prompt, negative_prompt="", seed=42, randomize_seed=False, width=1024, height=1024, guidance_scale=5.0, num_inference_steps=28, progress=gr.Progress(track_tqdm=True)):

    if randomize_seed:
        seed = random.randint(0, MAX_SEED)
        
    generator = torch.Generator().manual_seed(seed)
    
    image = pipe(
        prompt = prompt, 
        negative_prompt = negative_prompt,
        width=width,
        height=height,
        guidance_scale = guidance_scale, 
        num_inference_steps = num_inference_steps, 
        generator = generator
    ).images[0] 
    
    return image, seed
    

    examples = [ "A taco food cart in front of a japanese castle", "The spirit of a tamagotchi wandering in the city of Prague", "A flourecent cat on the moon", "A delicious gummy bear cheesecake slice", ]

    css=""" #col-container { margin: 0 auto; max-width: 520px; } """

    with gr.Blocks(css=css) as demo:

    with gr.Column(elem_id="col-container"):
        gr.Markdown(f"""
        # PixArt Sigma 900M
        Demo of the [PixArt Sigma 900M](https://huggingface.co/dataautogpt3/PixArt-Sigma-900M) model, expanded from [PixArt Sigma 600M](https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS)
        """)
        
        with gr.Row():
            
            prompt = gr.Text(
                label="Prompt",
                show_label=False,
                max_lines=1,
                placeholder="Enter your prompt",
                container=False,
            )
            
            run_button = gr.Button("Run", scale=0)
        
        result = gr.Image(label="Result", show_label=False)
    
        with gr.Accordion("Advanced Settings", open=False):
            
            negative_prompt = gr.Text(
                label="Negative prompt",
                max_lines=1,
                value="low quality, bad, watermark",
                placeholder="Enter a negative prompt",
            )
            
            seed = gr.Slider(
                label="Seed",
                minimum=0,
                maximum=MAX_SEED,
                step=1,
                value=0,
            )
            
            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
            
            with gr.Row():
                
                width = gr.Slider(
                    label="Width",
                    minimum=256,
                    maximum=MAX_IMAGE_SIZE,
                    step=32,
                    value=2048,
                )
                
                height = gr.Slider(
                    label="Height",
                    minimum=256,
                    maximum=MAX_IMAGE_SIZE,
                    step=32,
                    value=2048,
                )
            
            with gr.Row():
                
                guidance_scale = gr.Slider(
                    label="Guidance scale",
                    minimum=0.0,
                    maximum=10.0,
                    step=0.1,
                    value=5.0,
                )
                
                num_inference_steps = gr.Slider(
                    label="Number of inference steps",
                    minimum=1,
                    maximum=50,
                    step=1,
                    value=28,
                )
        
        gr.Examples(
            examples = examples,
            fn = infer,
            inputs = [prompt],
            outputs = [result, seed],
            cache_examples="lazy"
        )
    
    gr.on(
        triggers=[run_button.click, prompt.submit, negative_prompt.submit],
        fn = infer,
        inputs = [prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps],
        outputs = [result, seed]
    )
    

    demo.queue().launch()

  2. Assistant

    The slower generation of images compared to another script in a training program can be attributed to multiple factors. Here are several potential reasons and improvement suggestions:

    1. Model Initialization and Device Loading:

      • Model loading and device transfer often take a significant amount of time. Ensure this only happens once and not repeatedly during inferences.
    2. Commented Optimizations:

      • Several commented lines could potentially speed up your script if enabled. For example:
        #torch.set_float32_matmul_precision("high")
        #torch._inductor.config.conv_1x1_as_mm = True
        #torch._inductor.config.coordinate_descent_tuning = True
        #torch._inductor.config.epilogue_fusion = False
        #torch._inductor.config.coordinate_descent_check_all_directions = True
        
        #pipe.transformer.to(memory_format=torch.channels_last)
        #pipe.vae.to(memory_format=torch.channels_last)
        
        #pipe.transformer = torch.compile(pipe.transformer, mode="max-autotune", fullgraph=True)
        #pipe.vae.decode = torch.compile(pipe.vae.decode, mode="max-autotune", fullgraph=True)
        
      • Uncommenting and carefully testing these configurations and optimizations could lead to significant speed improvements.
    3. Scheduler Configuration:

      • The selection of the scheduler can impact performance. Make sure EulerAncestralDiscreteScheduler is optimal for your use case.
    4. Image Size:

      • Generating larger images (2048x2048) is more computationally expensive. Compare the performance impact by testing with smaller sizes.
    5. Batch Processing:

      • Ensure no unnecessary computations are repeated within the infer function. For example, the torch.Generator() can be instantiated outside the inference function if not dependent on a variable seed.
    6. Scheduler Configuration:

      • Make sure the scheduler and model configuration are optimal and up-to-date.
    7. Model Precision:

      • While you set torch_dtype=torch.float16, further optimizations are possible depending on the hardware and framework support.

    Here's a revised version of your script with some optimizations re-enabled:

    import gradio as gr
    import numpy as np
    import random
    import torch
    from diffusers import PixArtSigmaPipeline, EulerAncestralDiscreteScheduler
    
    device = "cuda" if torch.cuda.is_available() else "cpu"
    
    torch.set_float32_matmul_precision("high")
    torch._inductor.config.conv_1x1_as_mm = True
    torch._inductor.config.coordinate_descent_tuning = True
    torch._inductor.config.epilogue_fusion = False
    torch._inductor.config.coordinate_descent_check_all_directions = True
    
    pipe = PixArtSigmaPipeline.from_pretrained(
        "H:/OneTrainer/models/test_sigma",
        torch_dtype=torch.float16,
        # max_sequence_length = 120
    ).to("cuda")
    pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config)
    
    pipe.transformer.to(memory_format=torch.channels_last)
    pipe.vae.to(memory_format=torch.channels_last)
    
    # Using `torch.compile` to optimize transformer and vae.decode
    pipe.transformer = torch.compile(pipe.transformer, mode="max-autotune", fullgraph=True)
    pipe.vae.decode = torch.compile(pipe.vae.decode, mode="max-autotune", fullgraph=True)
    
    MAX_SEED = np.iinfo(np.int32).max
    MAX_IMAGE_SIZE = 2048
    
    def infer(prompt, negative_prompt="", seed=42, randomize_seed=False, width=1024, height=1024, guidance_scale=5.0, num_inference_steps=28, progress=gr.Progress(track_tqdm=True)):
        if randomize_seed:
            seed = random.randint(0, MAX_SEED)
            
        generator = torch.Generator().manual_seed(seed)
        
        image = pipe(
            prompt=prompt,
            negative_prompt=negative_prompt,
            width=width,
            height=height,
            guidance_scale=guidance_scale,
            num_inference_steps=num_inference_steps,
            generator=generator
        ).images[0]
        
        return image, seed
    
    examples = [
        "A taco food cart in front of a japanese castle",
        "The spirit of a tamagotchi wandering in the city of Prague",
        "A flourecent cat on the moon",
        "A delicious gummy bear cheesecake slice",
    ]
    
    css = """
    #col-container {
        margin: 0 auto;
        max-width: 520px;
    }
    """
    
    with gr.Blocks(css=css) as demo:
        with gr.Column(elem_id="col-container"):
            gr.Markdown(f"""
            # PixArt Sigma 900M
            Demo of the [PixArt Sigma 900M](https://huggingface.co/dataautogpt3/PixArt-Sigma-900M) model, expanded from [PixArt Sigma 600M](https://huggingface.co/PixArt-alpha/PixArt-Sigma-XL-2-1024-MS)
            """)
            
            with gr.Row():
                prompt = gr.Text(
                    label="Prompt",
                    show_label=False,
                    max_lines=1,
                    placeholder="Enter your prompt",
                    container=False,
                )
                
                run_button = gr.Button("Run", scale=0)
            
            result = gr.Image(label="Result", show_label=False)
    
            with gr.Accordion("Advanced Settings", open=False):
                negative_prompt = gr.Text(
                    label="Negative prompt",
                    max_lines=1,
                    value="low quality, bad, watermark",
                    placeholder="Enter a negative prompt",
                )
                
                seed = gr.Slider(
                    label="Seed",
                    minimum=0,
                    maximum=MAX_SEED,
                    step=1,
                    value=0,
                )
                
                randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
                
                with gr.Row():
                    width = gr.Slider(
                        label="Width",
                        minimum=256,
                        maximum=MAX_IMAGE_SIZE,
                        step=32,
                        value=2048,
                    )
                    
                    height = gr.Slider(
                        label="Height",
                        minimum=256,
                        maximum=MAX_IMAGE_SIZE,
                        step=32,
                        value=2048,
                    )
                
                with gr.Row():
                    guidance_scale = gr.Slider(
                        label="Guidance scale",
                        minimum=0.0,
                        maximum=10.0,
                        step=0.1,
                        value=5.0,
                    )
                    
                    num_inference_steps = gr.Slider(
                        label="Number of inference steps",
                        minimum=1,
                        maximum=50,
                        step=1,
                        value=28,
                    )
            
            gr.Examples(
                examples=examples,
                fn=infer,
                inputs=[prompt],
                outputs=[result, seed],
                cache_examples="lazy"
            )
    
        gr.on(
            triggers=[run_button.click, prompt.submit, negative_prompt.submit],
            fn=infer,
            inputs=[prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps],
            outputs=[result, seed]
        )
    
    demo.queue().launch()
    

    To summarize:

    • Specific optimizations and configurations related to the PyTorch backend (Inductor, floating point precision).
    • Memory format optimizations in the pipeline.
    • Utilizing torch.compile for critical model components.

    Always profile before and after these changes to ensure they provide the intended performance improvements.