Skip to content
The Chat Archive
magistral-medium-2506 LMArena Human Preference 140K

I wrote this txtai based python script to query a local Japanese wikipedia vector…

  1. User

    I wrote this txtai based python script to query a local Japanese wikipedia vector database. However, I am not getting the correct Japanese wiki articles. Can you fix it for me? import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys

    def escape_sql_string(s) -> str: # More comprehensive escaping s = s.replace("\", "\\") # Escape backslashes first s = s.replace("'", "''") # SQL standard: escape single quotes by doubling s = s.replace('"', '""') # Escape double quotes by doubling s = s.replace(";", "") # Remove semicolons s = s.replace("\n", " ") # Replace newlines with spaces s = s.replace("\r", " ") # Replace carriage returns return s

    def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets)

    def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index

    lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)

    Query the wikipedia db

    prompt = "日中戦争" sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.")

    Find top wikipedia articles

    for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score']))

  2. Assistant

    Here's a modified version of your script that should better handle the mapping between search results and Wikipedia articles:

    import os
    import json
    from txtai.embeddings import Embeddings
    from datasets import Dataset, concatenate_datasets
    import re
    import sys
    
    def escape_sql_string(s) -> str:
        s = s.replace("\\", "\\\\")
        s = s.replace("'", "''")
        s = s.replace('"', '""')
        s = s.replace(";", "")
        s = s.replace("\n", " ")
        s = s.replace("\r", " ")
        return s
    
    def load_wiki_dataset(path):
        """Load the Wikipedia dataset."""
        arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')]
        datasets = [Dataset.from_file(file) for file in arrow_files]
        return concatenate_datasets(datasets)
    
    def load_title_to_index(ds, path):
        """Load or create the title to index mapping."""
        if os.path.exists(path):
            with open(path, 'r') as f:
                return json.load(f)
        else:
            # Check if records have an 'id' field
            if len(ds) > 0 and 'id' in ds[0]:
                # Create mapping from id to index
                title_to_index = {record['id']: i for i, record in enumerate(ds)}
            else:
                # Fall back to using title as key
                title_to_index = {record['title']: i for i, record in enumerate(ds)}
            with open(path, 'w') as f:
                json.dump(title_to_index, f)
            return title_to_index
    
    lang = "ja"
    embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"}
    embeddings = Embeddings(config=embd_config)
    embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang)
    print("Loading wiki dataset")
    wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang)
    print("Loading title_to_index")
    title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)
    
    # Query the wikipedia db
    prompt = "日中戦争"
    sql_prompt = escape_sql_string(prompt)
    search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1"
    wiki_results = embeddings.search(search_query, 20)
    
    if not wiki_results:
        print("No wikipedia results found.")
    else:
        for result in wiki_results:
            title_id = result['id']
            index = title_to_index.get(title_id)
            if index is not None:
                record = wiki_dataset[index]
                print("%s %s" % (record["title"], result['score']))
            else:
                print(f"Could not find record for ID: {title_id}")
    

    Key improvements:

    1. The load_title_to_index function now checks if the dataset has an 'id' field and uses that for mapping if available
    2. Better error handling when mapping search results to articles
    3. Maintains the original SQL query approach but with more robust mapping

    This should better handle the mapping between search results and Wikipedia articles by being more flexible about what field is used as the identifier. If the dataset has an 'id' field, it will use that; otherwise, it falls back to using titles as before.

    If this still doesn't work, you might need to:

    1. Print out a sample record from the dataset to understand its structure
    2. Print out a sample search result to understand what the 'id' field represents
    3. Adjust the mapping strategy based on what you find

    But this modified version should work in most cases where there's a reasonable correspondence between the search results and the dataset records.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.