I am trying to run the code that perform llama multi turn chat which posted on…
- User
I am trying to run the code that perform llama multi turn chat which posted on huggingface locally in my pc in vs code, the original huggingface code: " import os import time import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread
MODEL_LIST = ["meta-llama/Llama-3.2-1B-Instruct"] HF_TOKEN = os.environ.get("HF_TOKEN", None) MODEL = os.environ.get("MODEL_ID")
TITLE = "
Llama 3.2 1B " PLACEHOLDER = """
Hi, I'm Llama. Ask me anything.
""" CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """
device = "cuda" # for GPU usage or "cpu" for CPU usage
quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")
tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)
@spaces.GPU() def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')
conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )
with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()
buffer = "" for new_text in streamer: buffer += new_text yield buffer
chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )
if name == "main": demo.launch() " My code in vs code: " import os import time #import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig import gradio as gr from threading import Thread
MODEL = r'C:\Users\shermanhonsf\Desktop\Python\llama_develop\model\Llama3.2-1B-Instruct-HF-Model_try'
TITLE = "
Llama 3.2 1B " PLACEHOLDER = """
Hi, I'm Llama. Ask me anything.
""" CSS = """ .duplicate-button { margin: auto !important; color: white !important; background: black !important; border-radius: 100vh !important; } h3 { text-align: center; } """
device = "cuda" # for GPU usage or "cpu" for CPU usage
quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type= "nf4")
tokenizer = AutoTokenizer.from_pretrained(MODEL) model = AutoModelForCausalLM.from_pretrained( MODEL, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config)
def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}')
conversation = [ {"role": "system", "content": system_prompt} ] for prompt, answer in history: conversation.extend([ {"role": "user", "content": prompt}, {"role": "assistant", "content": answer}, ])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
generate_kwargs = dict( input_ids=input_ids, max_new_tokens = max_new_tokens, do_sample = False if temperature == 0 else True, top_p = top_p, top_k = top_k, temperature = temperature, eos_token_id=[128001,128008,128009], streamer=streamer, )
with torch.no_grad(): thread = Thread(target=model.generate, kwargs=generate_kwargs) thread.start()
buffer = "" for new_text in streamer: buffer += new_text yield buffer
chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
with gr.Blocks(css=CSS, theme="Nymbo/Nymbo_Theme") as demo: gr.HTML(TITLE) gr.ChatInterface( fn=stream_chat, chatbot=chatbot, fill_height=True, additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), additional_inputs=[ gr.Textbox( value="You are a helpful assistant", label="System Prompt", render=False, ), gr.Slider( minimum=0, maximum=1, step=0.1, value=0.8, label="Temperature", render=False, ), gr.Slider( minimum=128, maximum=8192, step=1, value=1024, label="Max new tokens", render=False, ), gr.Slider( minimum=0.0, maximum=1.0, step=0.1, value=1.0, label="top_p", render=False, ), gr.Slider( minimum=1, maximum=20, step=1, value=20, label="top_k", render=False, ), gr.Slider( minimum=0.0, maximum=2.0, step=0.1, value=1.2, label="Repetition penalty", render=False, ), ], examples=[ ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], ["Tell me a random fun fact about the Roman Empire."], ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], ], cache_examples=False, )
if name == "main": demo.launch(debug = True) " The huggingface code can be run on huggingface normally, but my code: " ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid. message: Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option. history: [] Traceback (most recent call last): File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\queueing.py", line 624, in process_events response = await route_utils.call_process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\route_utils.py", line 323, in call_process_api output = await app.get_blocks().process_api( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 2015, in process_api result = await self.call_function( ^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\blocks.py", line 1574, in call_function prediction = await utils.async_iteration(iterator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 815, in asyncgen_wrapper response = await iterator.anext() ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\chat_interface.py", line 678, in _stream_fn first_response = await async_iteration(generator) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 710, in async_iteration return await anext(iterator) ^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 704, in anext return await anyio.to_thread.run_sync( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync return await get_async_backend().run_sync_in_worker_thread( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 2441, in run_sync_in_worker_thread return await future ^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\anyio_backends_asyncio.py", line 943, in run result = context.run(func, *args) ^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\gradio\utils.py", line 687, in run_sync_iterator_async return next(iterator) ^^^^^^^^^^^^^^ File "C:\Users\shermanhonsf\AppData\Local\Temp\ipykernel_23352\3853318046.py", line 91, in stream_chat for new_text in streamer: File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\site-packages\transformers\generation\streamers.py", line 223, in next value = self.text_queue.get(timeout=self.timeout) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\shermanhonsf\Desktop\Python.conda\Lib\queue.py", line 179, in get raise Empty _queue.Empty " What happen?
- Assistant
The error you are encountering:
ValueError: temperature (=1024) has to be a strictly positive float, otherwise your next token scores will be invalid.indicates that the
temperatureparameter is receiving a value of1024, which is not valid. Thetemperatureparameter in language generation models controls the randomness of the responses and should be a positive float (typically between 0 and, say, 5), not an integer like1024.What's happening:
In your Gradio interface, the
additional_inputsare passed to yourstream_chatfunction as positional arguments in the order they are defined. Youradditional_inputsare:- System Prompt
- Temperature
- Max new tokens
- top_p
- top_k
- Repetition penalty
However, in your
stream_chatfunction, the parameters are defined in this order:def stream_chat( message: str, history: list, system_prompt: str, max_new_tokens: int = 1024, temperature: float = 0.8, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ):This mismatch in the order of parameters means that when Gradio calls your
stream_chatfunction, it assigns the values from the interface components to the function parameters in the wrong order. Specifically, themax_new_tokensvalue (1024) is being assigned to thetemperatureparameter, causing the error.How to fix it:
To resolve this issue, you need to ensure that the parameters in your
stream_chatfunction are in the same order as the inputs are provided in your Gradio interface. Here's how you can adjust your function definition:def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ):By swapping the positions of
temperatureandmax_new_tokensin the function signature, you align the parameters with the order of the inputs provided by Gradio. This ensures that each value is correctly assigned to the intended parameter.Explanation:
- Gradio Argument Passing: Gradio passes arguments to the function in the order of the components defined in your interface.
- Positional Mismatch: The mismatch between the order of components in
additional_inputsand the parameters in your function signature causes values to be assigned to the wrong parameters. - Correct Order: By adjusting your function parameters to mirror the order in your
additional_inputs, you ensure that each value from the interface matches the correct parameter in your function.
Updated Function:
def stream_chat( message: str, history: list, system_prompt: str, temperature: float = 0.8, # Now correctly receives the temperature value max_new_tokens: int = 1024, # Now correctly receives max_new_tokens value top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ): print(f'message: {message}') print(f'history: {history}') # Rest of your function code remains the sameAdditional Tips:
-
Use Keyword Arguments: To prevent such issues in the future, consider using keyword arguments when defining your function. You can use
*in your function definition to enforce keyword-only arguments after a certain point:def stream_chat( message: str, history: list, system_prompt: str, *, temperature: float = 0.8, max_new_tokens: int = 1024, top_p: float = 1.0, top_k: int = 20, penalty: float = 1.2, ):However, ensure Gradio supports passing keyword arguments in this way.
-
Check Gradio Documentation: Always refer to the Gradio documentation to understand how arguments are passed to the functions, especially when using
additional_inputsand custom components.
In Summary:
Your error arises from a mismatch in the order of parameters between your Gradio interface and your
stream_chatfunction. By aligning the function parameters with the order of your interface inputs, you ensure that each parameter receives the correct value, resolving the error.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.