Skip to content
TheChatArchiveThe Chat Archive
Mistralmagistral-medium-2506· Mistral AIHugging Face LMArena Human Preference 140K

I wrote this txtai based python script to query a local Japanese wikipedia vector…

From a Chatbot Arena battleThe same person sent this prompt to two anonymous models at once. Compare it with Googlegemma-3-27b-it’s answer.
  1. User

    I wrote this txtai based python script to query a local Japanese wikipedia vector database. However, I am not getting the correct Japanese wiki articles. Can you fix it for me? import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys

    def escape_sql_string(s) -> str: # More comprehensive escaping s = s.replace("\", "\\") # Escape backslashes first s = s.replace("'", "''") # SQL standard: escape single quotes by doubling s = s.replace('"', '""') # Escape double quotes by doubling s = s.replace(";", "") # Remove semicolons s = s.replace("\n", " ") # Replace newlines with spaces s = s.replace("\r", " ") # Replace carriage returns return s

    def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets)

    def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index

    lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)

    Query the wikipedia db

    prompt = "日中戦争" sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.")

    Find top wikipedia articles

    for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score']))

  2. Mistral Assistant

    Here's a modified version of your script with additional debugging and a more robust approach to searching:

    import os
    import json
    from txtai.embeddings import Embeddings
    from datasets import Dataset, concatenate_datasets
    import re
    import sys
    
    def escape_sql_string(s) -> str:
        # Modified to be more careful with non-ASCII text
        s = s.replace("\\", "\\\\")
        s = s.replace("'", "''")
        s = s.replace('"', '""')
        s = s.replace(";", "")
        s = s.replace("\n", " ")
        s = s.replace("\r", " ")
        return s
    
    def load_wiki_dataset(path):
        """Load the Wikipedia dataset."""
        print(f"Loading Wikipedia dataset from: {path}")
        try:
            arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')]
            print(f"Found {len(arrow_files)} arrow files")
            if not arrow_files:
                print("Warning: No arrow files found!")
                return None
            datasets = []
            for file in arrow_files:
                try:
                    datasets.append(Dataset.from_file(file))
                except Exception as e:
                    print(f"Error loading {file}: {e}")
            if not datasets:
                return None
            return concatenate_datasets(datasets)
        except Exception as e:
            print(f"Error loading dataset: {e}")
            return None
    
    def load_title_to_index(ds, path):
        """Load or create the title to index mapping."""
        if os.path.exists(path):
            with open(path, 'r') as f:
                return json.load(f)
        else:
            if ds is None:
                return {}
            print("Creating title_to_index mapping...")
            title_to_index = {}
            for i, record in enumerate(ds):
                if 'title' in record:
                    title_to_index[record['title']] = i
                else:
                    print(f"Record {i} has no title field")
            with open(path, 'w') as f:
                json.dump(title_to_index, f)
            return title_to_index
    
    lang = "ja"
    embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"}
    embeddings = Embeddings(config=embd_config)
    
    # Load embeddings
    embeddings_path = "/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang
    print(f"Loading embeddings from: {embeddings_path}")
    if not os.path.exists(embeddings_path):
        print(f"Error: Embeddings path does not exist: {embeddings_path}")
        sys.exit(1)
    embeddings.load(path=embeddings_path)
    
    # Load dataset
    dataset_path = "/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang
    print(f"Loading dataset from: {dataset_path}")
    if not os.path.exists(dataset_path):
        print(f"Error: Dataset path does not exist: {dataset_path}")
        sys.exit(1)
    wiki_dataset = load_wiki_dataset(dataset_path)
    if wiki_dataset is None:
        print("Failed to load dataset")
        sys.exit(1)
    
    # Print some dataset info
    print(f"Dataset loaded with {len(wiki_dataset)} records")
    if len(wiki_dataset) > 0:
        print("Sample record:", wiki_dataset[0])
    
    # Load title to index mapping
    title_index_path = "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang
    print(f"Loading title_to_index from: {title_index_path}")
    title_to_index = load_title_to_index(wiki_dataset, title_index_path)
    print(f"Loaded {len(title_to_index)} title mappings")
    
    # Query the wikipedia db
    prompt = "日中戦争"
    print(f"Searching for: {prompt}")
    
    # Try both approaches: SQL syntax and direct search
    
    # Approach 1: SQL syntax
    try:
        sql_prompt = escape_sql_string(prompt)
        search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}')"
        print(f"Executing SQL query: {search_query}")
        wiki_results = embeddings.search(search_query, 20)
    
        print("\nResults from SQL query:")
        if not wiki_results:
            print("No results found with SQL query")
        else:
            for result in wiki_results:
                print("Raw result:", result)  # Debug print
                if 'id' in result:
                    title_id = result['id']
                    index = title_to_index.get(title_id)
                    if index is not None:
                        record = wiki_dataset[index]
                        print(f"Found: {record['title']} (score: {result.get('score', 'N/A')})")
                    else:
                        print(f"ID {title_id} not found in title_to_index")
                else:
                    print("Result has no ID field:", result)
    except Exception as e:
        print(f"Error with SQL query approach: {e}")
    
    # Approach 2: Direct search
    try:
        print("\nTrying direct search approach...")
        wiki_results = embeddings.search(prompt, 20)
    
        print("\nResults from direct search:")
        if not wiki_results:
            print("No results found with direct search")
        else:
            for result in wiki_results:
                print("Raw result:", result)  # Debug print
                if 'id' in result:
                    title_id = result['id']
                    index = title_to_index.get(title_id)
                    if index is not None:
                        record = wiki_dataset[index]
                        print(f"Found: {record['title']} (score: {result.get('score', 'N/A')})")
                    else:
                        print(f"ID {title_id} not found in title_to_index")
                else:
                    print("Result has no ID field:", result)
    except Exception as e:
        print(f"Error with direct search approach: {e}")
    

    Key improvements in this version:

    1. Added extensive debugging output to understand what's happening at each step
    2. Added error checking for file paths and dataset loading
    3. Implemented both SQL syntax and direct search approaches to see which works better
    4. Added more detailed output of search results to understand their structure
    5. Added checks for missing fields in the results

    This should help identify where the problem is occurring. If the direct search approach works but the SQL approach doesn't, then we know the issue is with the SQL query syntax. If neither works, then there might be an issue with how the index was built or how the dataset is being loaded.

    The debug output will show:

    • Whether the dataset and embeddings are loading correctly
    • The structure of the search results
    • Whether the title_to_index mapping is working correctly
    • Which approach (SQL or direct search) works better

    Based on the output from this debug version, we can then make more targeted fixes to the script. For example, if we see that the SQL query isn't returning results but the direct search is, we might need to adjust the SQL query syntax or how the index was built.

    If you run this version and share the output, I can help further diagnose the specific issue. The most likely fixes would involve either adjusting the SQL query syntax or modifying how we map search results back to article titles.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.