Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

I am trying to run the code that perform llama multi turn chat which posted on…

  1. User

    I am trying to run the code that perform llama multi turn chat which posted on huggingface locally in my pc in vs code, the original huggingface code: " import os import time import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread

    MODEL_LIST = ["meta-llama/Llama-3.2-1B-Instruct"] HF_TOKEN = os.environ.get("HF_TOKEN", None) MODEL = os.environ.get("MODEL_ID")

    TITLE = "

    Llama 3.2 1B " PLACEHOLDER = """

    Hi, I'm Llama. Ask me anything.

    """ CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """

    device = "cuda" # for GPU usage or "cpu" for CPU usage

    quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")

    tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)

    @spaces.GPU() def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')

    conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])

    conversation.append({"role": "user", "content": message})

    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)

    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)

    generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )

    with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()

    buffer = "" for new_text in streamer: buffer += new_text yield buffer

    chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

    with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )

    if name == "main": demo.launch() " My code in vs code: " import os import time #import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread

    MODEL = r'C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model_try'

    TITLE = "

    Llama 3.2 1B " PLACEHOLDER = """

    Hi, I'm Llama. Ask me anything.

    """ CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """

    device = "cuda" # for GPU usage or "cpu" for CPU usage

    quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")

    tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)

    def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')

    conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])

    conversation.append({"role": "user", "content": message})

    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)

    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)

    generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )

    with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()

    buffer = "" for new_text in streamer: buffer += new_text yield buffer

    chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

    with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )

    if name == "main": demo.launch(debug = True) " The huggingface code can be run on huggingface normally, but my code: " ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid. message: Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option. history: [] Traceback (most recent call last): File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\queueing.py", line 624, in process_events response = await route_utils.call_process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\route_utils.py", line 323, in call_process_api output = await app.get_blocks().process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 2015, in process_api result = await self.call_function( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 1574, in call_function prediction = await utils.async_iteration(iterator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 815, in asyncgen_wrapper response = await iterator.anext() ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\chat_interface.py", line 678, in _stream_fn first_response = await async_iteration(generator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 704, in anext return await anyio.to_thread.run_sync( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 2441, in run_sync_in_worker_thread return await future ^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 943, in run result = context.run(func, *args) ^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 687, in run_sync_iterator_async return next(iterator) ^^^^^^^^^^^^^^ File "C:\Users\shermanhonsf\AppData\Local\Temp\ipykernel_23352\3853318046.py", line 91, in stream_chat for new_text in streamer: File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\transformers\generation\streamers.py", line 223, in next value = self.text_queue.get(timeout=self.timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\queue.py", line 179, in get raise Empty _queue.Empty " What happen? Why the code on huggingface can run but mine can't? How to fix it?

  2. Assistant

    The issue you're encountering stems from a mismatch between the parameters your function stream_chat expects and the arguments being passed to it by Gradio's ChatInterface. This mismatch is causing the temperature parameter to receive the value meant for max_new_tokens (1024), leading to the error you're seeing:

    ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid.
    

    Why This Is Happening:

    In the original code from Hugging Face, the gr.ChatInterface uses additional_inputs to collect parameters from the user and pass them to the stream_chat function. The parameters are:

    • System Prompt
    • Temperature
    • Max New Tokens
    • Top P
    • Top K
    • Repetition Penalty

    In the function definition, the parameters are expected in the same order:

    def stream_chat(
        message: str,
        history: list,
        system_prompt: str,
        temperature: float = 0.8,
        max_new_tokens: int = 1024,
        top_p: float = 1.0,
        top_k: int = 20,
        penalty: float = 1.2,
    ):
    

    However, recent versions of Gradio have changed the way gr.ChatInterface handles additional inputs. Specifically, the parameter additional_inputs is deprecated and replaced with parameters. Moreover, the way parameters are passed to the function has changed, potentially disrupting the order in which arguments are supplied.

    How to Fix It:

    1. Update the Parameter Name from additional_inputs to parameters:

      Replace additional_inputs with parameters in your gr.ChatInterface call. This change aligns with the updated Gradio API.

      gr.ChatInterface(
          fn=stream_chat,
          chatbot=chatbot,
          fill_height=True,
          parameters=[
              # Your gradio components go here
          ],
          # Other arguments...
      )
      
    2. Remove render=False from Your Components:

      The render=False flag prevents components from being displayed in the interface, which can cause them not to be included in the inputs passed to your function. By removing this flag, you ensure that the components are rendered and their values are correctly passed.

      gr.Textbox(
          value="You are a helpful assistant",
          label="System Prompt",
          # render=False,  # Remove this line
      ),
      # Do the same for other components
      
    3. Ensure the Order of Parameters in the Function Matches the Components:

      Make sure that the order of parameters in your stream_chat function matches the order of components in the parameters list (after message and history).

      def stream_chat(
          message: str,
          history: list,
          system_prompt: str,
          temperature: float,
          max_new_tokens: int,
          top_p: float,
          top_k: int,
          penalty: float,
      ):
          # Your function code...
      
    4. Update the Function to Use All Parameters:

      Ensure that your function uses all the parameters you're collecting. For example, if you're collecting penalty, make sure to pass it to the model's generate method using the correct argument name (repetition_penalty):

      generate_kwargs = dict(
          input_ids=input_ids, 
          max_new_tokens=max_new_tokens,
          do_sample=False if temperature == 0 else True,
          top_p=top_p,
          top_k=top_k,
          temperature=temperature,
          repetition_penalty=penalty,  # Add this line
          eos_token_id=[128001, 128008, 128009],
          streamer=streamer,
      )
      
    5. Check for Version Compatibility:

      Ensure that the versions of Gradio and Transformers you're using are compatible with your code. The code from Hugging Face might be using an older version, so you might need to adjust your code to work with newer versions, or you can install specific versions matching the original code.

      pip install gradio==3.35 transformers==4.32  # Replace with the appropriate versions
      
    6. Debug by Printing Parameter Values:

      Add print statements in your function to check the values of the parameters being received. This will help you confirm that the parameters are being passed correctly.

      def stream_chat(
          message: str,
          history: list,
          system_prompt: str,
          temperature: float,
          max_new_tokens: int,
          top_p: float,
          top_k: int,
          penalty: float,
      ):
          print(f"message: {message}")
          print(f"history: {history}")
          print(f"system_prompt: {system_prompt}")
          print(f"temperature: {temperature}")
          print(f"max_new_tokens: {max_new_tokens}")
          print(f"top_p: {top_p}")
          print(f"top_k: {top_k}")
          print(f"penalty: {penalty}")
          # Rest of your code...
      

    Updated Code Example:

    import os
    import time
    import torch
    from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
    import gradio as gr
    from threading import Thread
    
    MODEL = r'C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model_try'
    
    TITLE = """
    Llama 3.2 1B
    """
    PLACEHOLDER = """
    Hi, I'm Llama. Ask me anything.
    """
    CSS = """
    .duplicate-button {
        margin: auto !important;
        color: white !important;
        background: black !important;
        border-radius: 100vh !important;
    }
    h3 {
        text-align: center;
    }
    """
    
    device = "cuda"  # for GPU usage or "cpu" for CPU usage
    
    quantization_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_compute_dtype=torch.bfloat16,
        bnb_4bit_use_double_quant=True,
        bnb_4bit_quant_type="nf4"
    )
    
    tokenizer = AutoTokenizer.from_pretrained(MODEL)
    model = AutoModelForCausalLM.from_pretrained(
        MODEL,
        torch_dtype=torch.bfloat16,
        device_map="auto",
        quantization_config=quantization_config
    )
    
    def stream_chat(
        message: str,
        history: list,
        system_prompt: str,
        temperature: float,
        max_new_tokens: int,
        top_p: float,
        top_k: int,
        penalty: float,
    ):
        print(f"message: {message}")
        print(f"history: {history}")
        print(f"system_prompt: {system_prompt}")
        print(f"temperature: {temperature}")
        print(f"max_new_tokens: {max_new_tokens}")
        print(f"top_p: {top_p}")
        print(f"top_k: {top_k}")
        print(f"penalty: {penalty}")
    
        conversation = [{"role": "system", "content": system_prompt}]
        for prompt, answer in history:
            conversation.extend([
                {"role": "user", "content": prompt},
                {"role": "assistant", "content": answer},
            ])
        conversation.append({"role": "user", "content": message})
    
        input_ids = tokenizer.apply_chat_template(
            conversation, add_generation_prompt=True, return_tensors="pt"
        ).to(model.device)
    
        streamer = TextIteratorStreamer(
            tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True
        )
    
        generate_kwargs = dict(
            input_ids=input_ids,
            max_new_tokens=max_new_tokens,
            do_sample=False if temperature == 0 else True,
            top_p=top_p,
            top_k=top_k,
            temperature=temperature,
            repetition_penalty=penalty,  # Make sure to use penalty
            eos_token_id=[128001, 128008, 128009],
            streamer=streamer,
        )
    
        with torch.no_grad():
            thread = Thread(target=model.generate, kwargs=generate_kwargs)
            thread.start()
    
        buffer = ""
        for new_text in streamer:
            buffer += new_text
            yield buffer
    
    chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
    
    with gr.Blocks(css=CSS) as demo:
        gr.HTML(TITLE)
        gr.ChatInterface(
            fn=stream_chat,
            chatbot=chatbot,
            fill_height=True,
            parameters=[  # Use 'parameters' instead of 'additional_inputs'
                gr.Textbox(
                    value="You are a helpful assistant",
                    label="System Prompt",
                ),
                gr.Slider(
                    minimum=0,
                    maximum=1,
                    step=0.1,
                    value=0.8,
                    label="Temperature",
                ),
                gr.Slider(
                    minimum=128,
                    maximum=8192,
                    step=1,
                    value=1024,
                    label="Max new tokens",
                ),
                gr.Slider(
                    minimum=0.0,
                    maximum=1.0,
                    step=0.1,
                    value=1.0,
                    label="top_p",
                ),
                gr.Slider(
                    minimum=1,
                    maximum=20,
                    step=1,
                    value=20,
                    label="top_k",
                ),
                gr.Slider(
                    minimum=0.0,
                    maximum=2.0,
                    step=0.1,
                    value=1.2,
                    label="Repetition penalty",
                ),
            ],
            examples=[
                ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."],
                ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."],
                ["Tell me a random fun fact about the Roman Empire."],
                ["Show me a code snippet of a website's sticky header in CSS and JavaScript."],
            ],
            cache_examples=False,
        )
    
    if __name__ == "__main__":
        demo.launch(debug=True)
    

    Summary:

    The error occurs because of changes in the Gradio library's API. By updating your code to align with the latest Gradio version—specifically, using parameters instead of additional_inputs and ensuring components are rendered—you can fix the parameter mismatch and resolve the error.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.