I wrote this txtai based python script to query a local Japanese wikipedia vector…
- User
I wrote this txtai based python script to query a local Japanese wikipedia vector database. However, I am not getting the correct Japanese wiki articles. Can you fix it for me? import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys
def escape_sql_string(s) -> str: # More comprehensive escaping s = s.replace("\", "\\") # Escape backslashes first s = s.replace("'", "''") # SQL standard: escape single quotes by doubling s = s.replace('"', '""') # Escape double quotes by doubling s = s.replace(";", "") # Remove semicolons s = s.replace("\n", " ") # Replace newlines with spaces s = s.replace("\r", " ") # Replace carriage returns return s
def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets)
def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index
lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)
Query the wikipedia db
prompt = "日中戦争" sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.")
Find top wikipedia articles
for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score']))
- Assistant
Here's a modified version of your script that should better handle the mapping between search results and Wikipedia articles:
import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys def escape_sql_string(s) -> str: s = s.replace("\\", "\\\\") s = s.replace("'", "''") s = s.replace('"', '""') s = s.replace(";", "") s = s.replace("\n", " ") s = s.replace("\r", " ") return s def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets) def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: # Check if records have an 'id' field if len(ds) > 0 and 'id' in ds[0]: # Create mapping from id to index title_to_index = {record['id']: i for i, record in enumerate(ds)} else: # Fall back to using title as key title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang) # Query the wikipedia db prompt = "日中戦争" sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.") else: for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score'])) else: print(f"Could not find record for ID: {title_id}")Key improvements:
- The
load_title_to_indexfunction now checks if the dataset has an 'id' field and uses that for mapping if available - Better error handling when mapping search results to articles
- Maintains the original SQL query approach but with more robust mapping
This should better handle the mapping between search results and Wikipedia articles by being more flexible about what field is used as the identifier. If the dataset has an 'id' field, it will use that; otherwise, it falls back to using titles as before.
If this still doesn't work, you might need to:
- Print out a sample record from the dataset to understand its structure
- Print out a sample search result to understand what the 'id' field represents
- Adjust the mapping strategy based on what you find
But this modified version should work in most cases where there's a reasonable correspondence between the search results and the dataset records.
- The
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.