I am trying to run the code that perform llama multi turn chat which posted on…
- User
I am trying to run the code that perform llama multi turn chat which posted on huggingface locally in my pc in vs code, the original huggingface code: " import os import time import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread
MODEL_LIST = ["meta-llama/Llama-3.2-1B-Instruct"] HF_TOKEN = os.environ.get("HF_TOKEN", None) MODEL = os.environ.get("MODEL_ID")
TITLE = "
Llama 3.2 1B " PLACEHOLDER = """
Hi, I'm Llama. Ask me anything.
""" CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """
device = "cuda" # for GPU usage or "cpu" for CPU usage
quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")
tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)
@spaces.GPU() def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')
conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )
with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()
buffer = "" for new_text in streamer: buffer += new_text yield buffer
chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )
if name == "main": demo.launch() " My code in vs code: " import os import time #import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread
MODEL = r'C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model_try'
TITLE = "
Llama 3.2 1B " PLACEHOLDER = """
Hi, I'm Llama. Ask me anything.
""" CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """
device = "cuda" # for GPU usage or "cpu" for CPU usage
quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")
tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)
def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')
conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )
with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()
buffer = "" for new_text in streamer: buffer += new_text yield buffer
chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )
if name == "main": demo.launch(debug = True) " The huggingface code can be run on huggingface normally, but my code: " ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid. message: Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option. history: [] Traceback (most recent call last): File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\queueing.py", line 624, in process_events response = await route_utils.call_process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\route_utils.py", line 323, in call_process_api output = await app.get_blocks().process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 2015, in process_api result = await self.call_function( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 1574, in call_function prediction = await utils.async_iteration(iterator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 815, in asyncgen_wrapper response = await iterator.anext() ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\chat_interface.py", line 678, in _stream_fn first_response = await async_iteration(generator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 704, in anext return await anyio.to_thread.run_sync( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 2441, in run_sync_in_worker_thread return await future ^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 943, in run result = context.run(func, *args) ^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 687, in run_sync_iterator_async return next(iterator) ^^^^^^^^^^^^^^ File "C:\Users\shermanhonsf\AppData\Local\Temp\ipykernel_23352\3853318046.py", line 91, in stream_chat for new_text in streamer: File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\transformers\generation\streamers.py", line 223, in next value = self.text_queue.get(timeout=self.timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\queue.py", line 179, in get raise Empty _queue.Empty " What happen? Why the code on huggingface can run but mine can't? How to fix it?
- Assistant
The issue you're encountering stems from a mismatch between the parameters your function
stream_chatexpects and the arguments being passed to it by Gradio'sChatInterface. This mismatch is causing thetemperatureparameter to receive the value meant formax_new_tokens(1024), leading to the error you're seeing:ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid.Why This Is Happening:
In the original code from Hugging Face, the
gr.ChatInterfaceusesadditional_inputsto collect parameters from the user and pass them to thestream_chatfunction. The parameters are:- System Prompt
- Temperature
- Max New Tokens
- Top P
- Top K
- Repetition Penalty
In the function definition, the parameters are expected in the same order:
def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ):However, recent versions of Gradio have changed the way
gr.ChatInterfacehandles additional inputs. Specifically, the parameteradditional_inputsis deprecated and replaced withparameters. Moreover, the way parameters are passed to the function has changed, potentially disrupting the order in which arguments are supplied.How to Fix It:
-
Update the Parameter Name from
additional_inputstoparameters:Replace
additional_inputswithparametersin yourgr.ChatInterfacecall. This change aligns with the updated Gradio API.gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, parameters=[ # Your gradio components go here ], # Other arguments... ) -
Remove
render=Falsefrom Your Components:The
render=Falseflag prevents components from being displayed in the interface, which can cause them not to be included in the inputs passed to your function. By removing this flag, you ensure that the components are rendered and their values are correctly passed.gr.Textbox( value="You are a helpful assistant", label="System Prompt", # render=False, # Remove this line ), # Do the same for other components -
Ensure the Order of Parameters in the Function Matches the Components:
Make sure that the order of parameters in your
stream_chatfunction matches the order of components in theparameterslist (aftermessageandhistory).def stream_chat( message: str, history: list, system_prompt: str, temperature: float, max_new_tokens: int, top_p: float, top_k: int, penalty: float, ): # Your function code... -
Update the Function to Use All Parameters:
Ensure that your function uses all the parameters you're collecting. For example, if you're collecting
penalty, make sure to pass it to the model'sgeneratemethod using the correct argument name (repetition_penalty):generate_kwargs = dict( input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False if temperature == 0 else True, top_p=top_p, top_k=top_k, temperature=temperature, repetition_penalty=penalty, # Add this line eos_token_id=[128001, 128008, 128009], streamer=streamer, ) -
Check for Version Compatibility:
Ensure that the versions of Gradio and Transformers you're using are compatible with your code. The code from Hugging Face might be using an older version, so you might need to adjust your code to work with newer versions, or you can install specific versions matching the original code.
pip install gradio==3.35 transformers==4.32 # Replace with the appropriate versions -
Debug by Printing Parameter Values:
Add print statements in your function to check the values of the parameters being received. This will help you confirm that the parameters are being passed correctly.
def stream_chat( message: str, history: list, system_prompt: str, temperature: float, max_new_tokens: int, top_p: float, top_k: int, penalty: float, ): print(f"message: {message}") print(f"history: {history}") print(f"system_prompt: {system_prompt}") print(f"temperature: {temperature}") print(f"max_new_tokens: {max_new_tokens}") print(f"top_p: {top_p}") print(f"top_k: {top_k}") print(f"penalty: {penalty}") # Rest of your code...
Updated Code Example:
import os import time import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread MODEL = r'C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model_try' TITLE = """ Llama 3.2 1B """ PLACEHOLDER = """ Hi, I'm Llama. Ask me anything. """ CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """ device = "cuda" # for GPU usage or "cpu" for CPU usage quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4" ) tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config ) def stream_chat( message: str, history: list, system_prompt: str, temperature: float, max_new_tokens: int, top_p: float, top_k: int, penalty: float, ): print(f"message: {message}") print(f"history: {history}") print(f"system_prompt: {system_prompt}") print(f"temperature: {temperature}") print(f"max_new_tokens: {max_new_tokens}") print(f"top_p: {top_p}") print(f"top_k: {top_k}") print(f"penalty: {penalty}") conversation = [{"role": "system", "content": system_prompt}] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ]) conversation.append({"role": "user", "content": message}) input_ids = tokenizer.apply_chat_template( conversation, add_generation_prompt=True, return_tensors="pt" ).to(model.device) streamer = TextIteratorStreamer( tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True ) generate_kwargs = dict( input_ids=input_ids, max_new_tokens=max_new_tokens, do_sample=False if temperature == 0 else True, top_p=top_p, top_k=top_k, temperature=temperature, repetition_penalty=penalty, # Make sure to use penalty eos_token_id=[128001, 128008, 128009], streamer=streamer, ) with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start() buffer = "" for new_text in streamer: buffer += new_text yield buffer chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER) with gr.Blocks(css=CSS) as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, parameters=[ # Use 'parameters' instead of 'additional_inputs' gr.Textbox( value="You are a helpful assistant", label="System Prompt", ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, ) if __name__ == "__main__": demo.launch(debug=True)Summary:
The error occurs because of changes in the Gradio library's API. By updating your code to align with the latest Gradio version—specifically, using
parametersinstead ofadditional_inputsand ensuring components are rendered—you can fix the parameter mismatch and resolve the error.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.