Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

I am trying to run the code that perform llama multi turn chat which posted on…

  1. User

    I am trying to run the code that perform llama multi turn chat which posted on huggingface locally in my pc in vs code, the original huggingface code: " import os import time import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread

    MODEL_LIST = ["meta-llama/Llama-3.2-1B-Instruct"] HF_TOKEN = os.environ.get("HF_TOKEN", None) MODEL = os.environ.get("MODEL_ID")

    TITLE = "

    Llama 3.2 1B " PLACEHOLDER = """

    Hi, I'm Llama. Ask me anything.

    """ CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """

    device = "cuda" # for GPU usage or "cpu" for CPU usage

    quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")

    tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)

    @spaces.GPU() def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')

    conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])

    conversation.append({"role": "user", "content": message})

    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)

    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)

    generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )

    with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()

    buffer = "" for new_text in streamer: buffer += new_text yield buffer

    chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

    with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )

    if name == "main": demo.launch() " My code in vs code: " import os import time #import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread

    MODEL = r'C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model_try'

    TITLE = "

    Llama 3.2 1B " PLACEHOLDER = """

    Hi, I'm Llama. Ask me anything.

    """ CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """

    device = "cuda" # for GPU usage or "cpu" for CPU usage

    quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")

    tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)

    def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')

    conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])

    conversation.append({"role": "user", "content": message})

    input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)

    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)

    generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )

    with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()

    buffer = "" for new_text in streamer: buffer += new_text yield buffer

    chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

    with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )

    if name == "main": demo.launch(debug = True) " The huggingface code can be run on huggingface normally, but my code: " ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid. message: Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option. history: [] Traceback (most recent call last): File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\queueing.py", line 624, in process_events response = await route_utils.call_process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\route_utils.py", line 323, in call_process_api output = await app.get_blocks().process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 2015, in process_api result = await self.call_function( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 1574, in call_function prediction = await utils.async_iteration(iterator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 815, in asyncgen_wrapper response = await iterator.anext() ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\chat_interface.py", line 678, in _stream_fn first_response = await async_iteration(generator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 704, in anext return await anyio.to_thread.run_sync( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 2441, in run_sync_in_worker_thread return await future ^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 943, in run result = context.run(func, *args) ^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 687, in run_sync_iterator_async return next(iterator) ^^^^^^^^^^^^^^ File "C:\Users\shermanhonsf\AppData\Local\Temp\ipykernel_23352\3853318046.py", line 91, in stream_chat for new_text in streamer: File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\transformers\generation\streamers.py", line 223, in next value = self.text_queue.get(timeout=self.timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\queue.py", line 179, in get raise Empty _queue.Empty " What happen?

  2. Assistant

    The error you are encountering:

    ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid.
    

    indicates that the temperature parameter is receiving a value of 1024, which is not valid. The temperature parameter in language generation models controls the randomness of the responses and should be a positive float (typically between 0 and, say, 5), not an integer like 1024.

    What's happening:

    In your Gradio interface, the additional_inputs are passed to your stream_chat function as positional arguments in the order they are defined. Your additional_inputs are:

    1. System Prompt
    2. Temperature
    3. Max new tokens
    4. top_p
    5. top_k
    6. Repetition penalty

    However, in your stream_chat function, the parameters are defined in this order:

    def stream_chat(
        message: str,
        history: list,
        system_prompt: str,
        max_new_tokens: int = 1024,
        temperature: float = 0.8,
        top_p: float = 1.0,
        top_k: int = 20,
        penalty: float = 1.2,
    ):
    

    This mismatch in the order of parameters means that when Gradio calls your stream_chat function, it assigns the values from the interface components to the function parameters in the wrong order. Specifically, the max_new_tokens value (1024) is being assigned to the temperature parameter, causing the error.

    How to fix it:

    To resolve this issue, you need to ensure that the parameters in your stream_chat function are in the same order as the inputs are provided in your Gradio interface. Here's how you can adjust your function definition:

    def stream_chat(
        message: str,
        history: list,
        system_prompt: str,
        temperature: float = 0.8,
        max_new_tokens: int = 1024,
        top_p: float = 1.0,
        top_k: int = 20,
        penalty: float = 1.2,
    ):
    

    By swapping the positions of temperature and max_new_tokens in the function signature, you align the parameters with the order of the inputs provided by Gradio. This ensures that each value is correctly assigned to the intended parameter.

    Explanation:

    • Gradio Argument Passing: Gradio passes arguments to the function in the order of the components defined in your interface.
    • Positional Mismatch: The mismatch between the order of components in additional_inputs and the parameters in your function signature causes values to be assigned to the wrong parameters.
    • Correct Order: By adjusting your function parameters to mirror the order in your additional_inputs, you ensure that each value from the interface matches the correct parameter in your function.

    Updated Function:

    def stream_chat(
        message: str,
        history: list,
        system_prompt: str,
        temperature: float = 0.8,      # Now correctly receives the temperature value
        max_new_tokens: int = 1024,    # Now correctly receives max_new_tokens value
        top_p: float = 1.0,
        top_k: int = 20,
        penalty: float = 1.2,
    ):
        print(f'message: {message}')
        print(f'history: {history}')
        
        # Rest of your function code remains the same
    

    Additional Tips:

    • Use Keyword Arguments: To prevent such issues in the future, consider using keyword arguments when defining your function. You can use * in your function definition to enforce keyword-only arguments after a certain point:

      def stream_chat(
          message: str,
          history: list,
          system_prompt: str,
          *,
          temperature: float = 0.8,
          max_new_tokens: int = 1024,
          top_p: float = 1.0,
          top_k: int = 20,
          penalty: float = 1.2,
      ):
      

      However, ensure Gradio supports passing keyword arguments in this way.

    • Check Gradio Documentation: Always refer to the Gradio documentation to understand how arguments are passed to the functions, especially when using additional_inputs and custom components.

    In Summary:

    Your error arises from a mismatch in the order of parameters between your Gradio interface and your stream_chat function. By aligning the function parameters with the order of your interface inputs, you ensure that each parameter receives the correct value, resolving the error.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.