I wrote this txtai based python script to query a local Japanese wikipedia vector…
- User
I wrote this txtai based python script to query a local Japanese wikipedia vector database. However, I am not getting the correct Japanese wiki articles. Can you fix it for me? import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys
def escape_sql_string(s) -> str: # More comprehensive escaping s = s.replace("\", "\\") # Escape backslashes first s = s.replace("'", "''") # SQL standard: escape single quotes by doubling s = s.replace('"', '""') # Escape double quotes by doubling s = s.replace(";", "") # Remove semicolons s = s.replace("\n", " ") # Replace newlines with spaces s = s.replace("\r", " ") # Replace carriage returns return s
def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets)
def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index
lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)
Query the wikipedia db
prompt = "日中戦争" sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.")
Find top wikipedia articles
for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score']))
Assistant
import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys def escape_sql_string(s) -> str: # More comprehensive escaping s = s.replace("\\", "\\\\") # Escape backslashes first s = s.replace("'", "''") # SQL standard: escape single quotes by doubling s = s.replace('"', '""') # Escape double quotes by doubling s = s.replace(";", "") # Remove semicolons s = s.replace("\n", " ") # Replace newlines with spaces s = s.replace("\r", " ") # Replace carriage returns return s def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets) def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang) # Query the wikipedia db prompt = "日中戦争" # The issue was that the prompt was being escaped for SQL, but txtai's search function doesn't use SQL directly. # It uses a similarity search, so escaping the prompt was actually hindering the search. search_query = f"SELECT id, text, score FROM txtai WHERE similar('{prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.") # Find top wikipedia articles for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score']))Key changes and explanations:
- Removed
escape_sql_stringcall: The core problem was that you were escaping thepromptstring usingescape_sql_stringbefore passing it toembeddings.search.txtai'ssimilar()function doesn't expect a SQL-escaped string; it expects the raw text to calculate similarity. Removing this escaping fixes the issue. Thesimilar()function handles any necessary escaping internally for its similarity calculation. - Simplified
search_query: Thesearch_queryis now directly using thepromptvariable without escaping.
This revised code directly passes the Japanese prompt to the
embeddings.searchfunction, allowing it to perform the similarity search correctly and retrieve the relevant Japanese Wikipedia articles. The escaping was causing the search to fail because it was looking for the escaped string, not the original Japanese text.- Removed
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.