Skip to content
TheChatArchiveThe Chat Archive
Googlegemma-3-27b-it· GoogleHugging Face LMArena Human Preference 140K

I wrote this txtai based python script to query a local Japanese wikipedia vector…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Mistralmagistral-medium-2506’s answer.
  1. User

    I wrote this txtai based python script to query a local Japanese wikipedia vector database. However, I am not getting the correct Japanese wiki articles. Can you fix it for me? import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys

    def escape_sql_string(s) -> str: # More comprehensive escaping s = s.replace("\", "\\") # Escape backslashes first s = s.replace("'", "''") # SQL standard: escape single quotes by doubling s = s.replace('"', '""') # Escape double quotes by doubling s = s.replace(";", "") # Remove semicolons s = s.replace("\n", " ") # Replace newlines with spaces s = s.replace("\r", " ") # Replace carriage returns return s

    def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets)

    def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index

    lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)

    Query the wikipedia db

    prompt = "日中戦争" sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.")

    Find top wikipedia articles

    for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score']))

  2. Google Assistant
    import os
    import json
    from txtai.embeddings import Embeddings
    from datasets import Dataset, concatenate_datasets
    import re
    import sys
    
    def escape_sql_string(s) -> str:
        # More comprehensive escaping
        s = s.replace("\\", "\\\\")  # Escape backslashes first
        s = s.replace("'", "''")     # SQL standard: escape single quotes by doubling
        s = s.replace('"', '""')     # Escape double quotes by doubling
        s = s.replace(";", "")       # Remove semicolons
        s = s.replace("\n", " ")     # Replace newlines with spaces
        s = s.replace("\r", " ")     # Replace carriage returns
        return s
    
    def load_wiki_dataset(path):
        """Load the Wikipedia dataset."""
        arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')]
        datasets = [Dataset.from_file(file) for file in arrow_files]
        return concatenate_datasets(datasets)
    
    def load_title_to_index(ds, path):
        """Load or create the title to index mapping."""
        if os.path.exists(path):
            with open(path, 'r') as f:
                return json.load(f)
        else:
            title_to_index = {record['title']: i for i, record in enumerate(ds)}
            with open(path, 'w') as f:
                json.dump(title_to_index, f)
            return title_to_index
    
    lang = "ja"
    embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"}
    embeddings = Embeddings(config=embd_config)
    embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang)
    print("Loading wiki dataset")
    wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang)
    print("Loading title_to_index")
    title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)
    # Query the wikipedia db
    
    prompt = "日中戦争"
    # The issue was that the prompt was being escaped for SQL, but txtai's search function doesn't use SQL directly.
    # It uses a similarity search, so escaping the prompt was actually hindering the search.
    search_query = f"SELECT id, text, score FROM txtai WHERE similar('{prompt}') and percentile >= 0.1"
    wiki_results = embeddings.search(search_query, 20)
    if not wiki_results:
        print("No wikipedia results found.")
    # Find top wikipedia articles
    for result in wiki_results:
        title_id = result['id']
        index = title_to_index.get(title_id)
        if index is not None:
            record = wiki_dataset[index]
            print("%s %s" % (record["title"], result['score']))
    

    Key changes and explanations:

    • Removed escape_sql_string call: The core problem was that you were escaping the prompt string using escape_sql_string before passing it to embeddings.search. txtai's similar() function doesn't expect a SQL-escaped string; it expects the raw text to calculate similarity. Removing this escaping fixes the issue. The similar() function handles any necessary escaping internally for its similarity calculation.
    • Simplified search_query: The search_query is now directly using the prompt variable without escaping.

    This revised code directly passes the Japanese prompt to the embeddings.search function, allowing it to perform the similarity search correctly and retrieve the relevant Japanese Wikipedia articles. The escaping was causing the search to fail because it was looking for the escaped string, not the original Japanese text.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.