Skip to content
TheChatArchiveThe Chat Archive
Mistralmistral-medium-2505· Mistral AIHugging Face LMArena Human Preference 140K

from fastapi import FastAPI, Query from transformers import AutoTokenizer,…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Amazon Novaamazon.nova-pro-v1:0’s answer.
  1. User

    from fastapi import FastAPI, Query from transformers import AutoTokenizer, AutoModelForCausalLM import torch

    app = FastAPI()

    MODEL_NAME = "deepseek-ai/DeepSeek-R1-0528-Qwen3-8B" print("Loading model and tokenizer...")

    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained( MODEL_NAME, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32, device_map="auto", trust_remote_code=True ) model.eval() print("Model loaded successfully!")

    @app.get("/inference") async def generate_qwen_response( role: str = Query(...), prompt: str = Query(...), content: str = Query(...), max_tokens: int = Query(256), temperature: float = Query(0.7), top_p: float = Query(0.95), top_k: int = Query(50), ): messages = [{"role": role, "question": prompt, "content": content}] text = tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, enable_thinking=False )

    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
    
    with torch.no_grad():
        generated_ids = model.generate(
            **model_inputs,
            max_new_tokens=max_tokens,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            do_sample=True,
            eos_token_id=tokenizer.eos_token_id,
        )
    
    output_ids = generated_ids[0][len(model_inputs.input_ids[0]):].tolist()
    
    THINK_END_TOKEN_ID = 151668  
    try:
        index = len(output_ids) - output_ids[::-1].index(THINK_END_TOKEN_ID)
    except ValueError:
        index = 0  
    
    thinking_content = tokenizer.decode(output_ids[:index], skip_special_tokens=True).strip("\n")
    final_content = tokenizer.decode(output_ids[index:], skip_special_tokens=True).strip("\n")
    
    return {
        "prompt": prompt,
        "thinking_content": thinking_content,
        "response": final_content
    }
    

    in this input way give for google/gemma-3-12b-it-qat-int4-unquantized this llm deployment

  2. Mistral Assistant

    To adapt your FastAPI code for the google/gemma-3-12b-it-qat-int4-unquantized model, you'll need to make several adjustments since Gemma models have different tokenization and generation requirements compared to Qwen models. Here's the modified version:

    from fastapi import FastAPI, Query
    from transformers import AutoTokenizer, AutoModelForCausalLM
    import torch
    
    app = FastAPI()
    
    MODEL_NAME = "google/gemma-3-12b-it-qat-int4-unquantized"
    print("Loading model and tokenizer...")
    
    # Load tokenizer and model with appropriate settings for Gemma
    tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
    model = AutoModelForCausalLM.from_pretrained(
        MODEL_NAME,
        torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
        device_map="auto",
        trust_remote_code=True
    )
    model.eval()
    print("Model loaded successfully!")
    
    @app.get("/inference")
    async def generate_gemma_response(
        role: str = Query(...),
        prompt: str = Query(...),
        content: str = Query(...),
        max_tokens: int = Query(256),
        temperature: float = Query(0.7),
        top_p: float = Query(0.95),
        top_k: int = Query(50),
    ):
        # Gemma uses a different chat template format
        messages = [
            {"role": "user", "content": f"{prompt}\n{content}"}
        ]
    
        # Apply the appropriate chat template for Gemma
        text = tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True
        )
    
        model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
    
        with torch.no_grad():
            generated_ids = model.generate(
                **model_inputs,
                max_new_tokens=max_tokens,
                temperature=temperature,
                top_p=top_p,
                top_k=top_k,
                do_sample=True,
                eos_token_id=tokenizer.eos_token_id,
            )
    
        # Decode the full response
        full_response = tokenizer.decode(generated_ids[0], skip_special_tokens=True)
    
        # For Gemma, we might want to split the response differently
        # This is a simplified approach - you might need to adjust based on actual output format
        response_parts = full_response.split("<start_of_turn>model")
    
        if len(response_parts) > 1:
            # If the model's response contains the expected markers
            thinking_content = response_parts[0].strip()
            final_content = response_parts[1].strip()
        else:
            # If no markers found, return the full response as final content
            thinking_content = ""
            final_content = full_response.strip()
    
        return {
            "prompt": prompt,
            "thinking_content": thinking_content,
            "response": final_content
        }
    

    Key changes made:

    1. Changed the model name to google/gemma-3-12b-it-qat-int4-unquantized
    2. Simplified the message format to match Gemma's expected input format
    3. Removed the enable_thinking parameter which is Qwen-specific
    4. Modified the response parsing logic to handle Gemma's output format
    5. Updated the chat template application to match Gemma's requirements

    Note that for Gemma models:

    • The chat template format is different from Qwen
    • The response structure might not have the same "thinking" markers as Qwen
    • You might need to adjust the response parsing logic based on actual output format

    You may need to further adjust the code based on:

    1. The exact format of inputs that work best with this specific Gemma model
    2. The actual output format you observe from the model
    3. Any specific requirements for handling the quantized version of the model

    Also consider adding error handling for cases where the model generation might fail or produce unexpected outputs.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.