from fastapi import FastAPI, Query from transformers import AutoTokenizer,…
- User
from fastapi import FastAPI, Query from transformers import AutoTokenizer, AutoModelForCausalLM import torch
app = FastAPI()
MODEL_NAME = "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B" print("Loading model and tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained( MODEL_NAME, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32, device_map="auto", trust_remote_code=True ) model.eval() print("Model loaded successfully!")
@app.get("/inference") async def generate_qwen_response( role: str = Query(...), prompt: str = Query(...), content: str = Query(...), max_tokens: int = Query(256), temperature: float = Query(0.7), top_p: float = Query(0.95), top_k: int = Query(50), ): messages = [{"role": role, "question": prompt, "content": content}] text = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, enable_thinking=False )
model_inputs = tokenizer([text], return_tensors="pt").to(model.device) with torch.no_grad(): generated_ids = model.generate( **model_inputs, max_new_tokens=max_tokens, temperature=temperature, top_p=top_p, top_k=top_k, do_sample=True, eos_token_id=tokenizer.eos_token_id, ) output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist() THINK_END_TOKEN_ID = 151668 try: index = len(output_ids) - output_ids[::-1].index(THINK_END_TOKEN_ID) except ValueError: index = 0 thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n") final_content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n") return { "prompt": prompt, "thinking_content": thinking_content, "response": final_content }in this input way give for google/gemma-3-12b-it-qat-int4-unquantized this llm deployment
Assistant
To adapt your FastAPI code for the
google/gemma-3-12b-it-qat-int4-unquantizedmodel, you'll need to make several adjustments since Gemma models have different tokenization and generation requirements compared to Qwen models. Here's the modified version:from fastapi import FastAPI, Query from transformers import AutoTokenizer, AutoModelForCausalLM import torch app = FastAPI() MODEL_NAME = "google/gemma-3-12b-it-qat-int4-unquantized" print("Loading model and tokenizer...") # Load tokenizer and model with appropriate settings for Gemma tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) model = AutoModelForCausalLM.from_pretrained( MODEL_NAME, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32, device_map="auto", trust_remote_code=True ) model.eval() print("Model loaded successfully!") @app.get("/inference") async def generate_gemma_response( role: str = Query(...), prompt: str = Query(...), content: str = Query(...), max_tokens: int = Query(256), temperature: float = Query(0.7), top_p: float = Query(0.95), top_k: int = Query(50), ): # Gemma uses a different chat template format messages = [ {"role": "user", "content": f"{prompt}\n{content}"} ] # Apply the appropriate chat template for Gemma text = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) model_inputs = tokenizer([text], return_tensors="pt").to(model.device) with torch.no_grad(): generated_ids = model.generate( **model_inputs, max_new_tokens=max_tokens, temperature=temperature, top_p=top_p, top_k=top_k, do_sample=True, eos_token_id=tokenizer.eos_token_id, ) # Decode the full response full_response = tokenizer.decode(generated_ids[0], skip_special_tokens=True) # For Gemma, we might want to split the response differently # This is a simplified approach - you might need to adjust based on actual output format response_parts = full_response.split("<start_of_turn>model") if len(response_parts) > 1: # If the model's response contains the expected markers thinking_content = response_parts[0].strip() final_content = response_parts[1].strip() else: # If no markers found, return the full response as final content thinking_content = "" final_content = full_response.strip() return { "prompt": prompt, "thinking_content": thinking_content, "response": final_content }Key changes made:
- Changed the model name to
google/gemma-3-12b-it-qat-int4-unquantized - Simplified the message format to match Gemma's expected input format
- Removed the
enable_thinkingparameter which is Qwen-specific - Modified the response parsing logic to handle Gemma's output format
- Updated the chat template application to match Gemma's requirements
Note that for Gemma models:
- The chat template format is different from Qwen
- The response structure might not have the same "thinking" markers as Qwen
- You might need to adjust the response parsing logic based on actual output format
You may need to further adjust the code based on:
- The exact format of inputs that work best with this specific Gemma model
- The actual output format you observe from the model
- Any specific requirements for handling the quantized version of the model
Also consider adding error handling for cases where the model generation might fail or produce unexpected outputs.
- Changed the model name to
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.