I wrote this txtai based python script to query a local Japanese wikipedia vector…
- User
I wrote this txtai based python script to query a local Japanese wikipedia vector database. However, I am not getting the correct Japanese wiki articles. Can you fix it for me? import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys
def escape_sql_string(s) -> str: # More comprehensive escaping s = s.replace("\", "\\") # Escape backslashes first s = s.replace("'", "''") # SQL standard: escape single quotes by doubling s = s.replace('"', '""') # Escape double quotes by doubling s = s.replace(";", "") # Remove semicolons s = s.replace("\n", " ") # Replace newlines with spaces s = s.replace("\r", " ") # Replace carriage returns return s
def load_wiki_dataset(path): """Load the Wikipedia dataset.""" arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] datasets = [Dataset.from_file(file) for file in arrow_files] return concatenate_datasets(datasets)
def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: title_to_index = {record['title']: i for i, record in enumerate(ds)} with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index
lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) embeddings.load(path="/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang) print("Loading wiki dataset") wiki_dataset = load_wiki_dataset("/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang) print("Loading title_to_index") title_to_index = load_title_to_index(wiki_dataset, "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang)
Query the wikipedia db
prompt = "日中戦争" sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}') and percentile >= 0.1" wiki_results = embeddings.search(search_query, 20) if not wiki_results: print("No wikipedia results found.")
Find top wikipedia articles
for result in wiki_results: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print("%s %s" % (record["title"], result['score']))
Assistant
Here's a modified version of your script with additional debugging and a more robust approach to searching:
import os import json from txtai.embeddings import Embeddings from datasets import Dataset, concatenate_datasets import re import sys def escape_sql_string(s) -> str: # Modified to be more careful with non-ASCII text s = s.replace("\\", "\\\\") s = s.replace("'", "''") s = s.replace('"', '""') s = s.replace(";", "") s = s.replace("\n", " ") s = s.replace("\r", " ") return s def load_wiki_dataset(path): """Load the Wikipedia dataset.""" print(f"Loading Wikipedia dataset from: {path}") try: arrow_files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.arrow')] print(f"Found {len(arrow_files)} arrow files") if not arrow_files: print("Warning: No arrow files found!") return None datasets = [] for file in arrow_files: try: datasets.append(Dataset.from_file(file)) except Exception as e: print(f"Error loading {file}: {e}") if not datasets: return None return concatenate_datasets(datasets) except Exception as e: print(f"Error loading dataset: {e}") return None def load_title_to_index(ds, path): """Load or create the title to index mapping.""" if os.path.exists(path): with open(path, 'r') as f: return json.load(f) else: if ds is None: return {} print("Creating title_to_index mapping...") title_to_index = {} for i, record in enumerate(ds): if 'title' in record: title_to_index[record['title']] = i else: print(f"Record {i} has no title field") with open(path, 'w') as f: json.dump(title_to_index, f) return title_to_index lang = "ja" embd_config = {"path": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"} embeddings = Embeddings(config=embd_config) # Load embeddings embeddings_path = "/home/user/OfflineWikipediaTextApi/txtai-%s-wikipedia" % lang print(f"Loading embeddings from: {embeddings_path}") if not os.path.exists(embeddings_path): print(f"Error: Embeddings path does not exist: {embeddings_path}") sys.exit(1) embeddings.load(path=embeddings_path) # Load dataset dataset_path = "/home/user/OfflineWikipediaTextApi/wikipedia-%s/train" % lang print(f"Loading dataset from: {dataset_path}") if not os.path.exists(dataset_path): print(f"Error: Dataset path does not exist: {dataset_path}") sys.exit(1) wiki_dataset = load_wiki_dataset(dataset_path) if wiki_dataset is None: print("Failed to load dataset") sys.exit(1) # Print some dataset info print(f"Dataset loaded with {len(wiki_dataset)} records") if len(wiki_dataset) > 0: print("Sample record:", wiki_dataset[0]) # Load title to index mapping title_index_path = "/home/user/OfflineWikipediaTextApi/title_to_index_%s.json" % lang print(f"Loading title_to_index from: {title_index_path}") title_to_index = load_title_to_index(wiki_dataset, title_index_path) print(f"Loaded {len(title_to_index)} title mappings") # Query the wikipedia db prompt = "日中戦争" print(f"Searching for: {prompt}") # Try both approaches: SQL syntax and direct search # Approach 1: SQL syntax try: sql_prompt = escape_sql_string(prompt) search_query = f"SELECT id, text, score FROM txtai WHERE similar('{sql_prompt}')" print(f"Executing SQL query: {search_query}") wiki_results = embeddings.search(search_query, 20) print("\nResults from SQL query:") if not wiki_results: print("No results found with SQL query") else: for result in wiki_results: print("Raw result:", result) # Debug print if 'id' in result: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print(f"Found: {record['title']} (score: {result.get('score', 'N/A')})") else: print(f"ID {title_id} not found in title_to_index") else: print("Result has no ID field:", result) except Exception as e: print(f"Error with SQL query approach: {e}") # Approach 2: Direct search try: print("\nTrying direct search approach...") wiki_results = embeddings.search(prompt, 20) print("\nResults from direct search:") if not wiki_results: print("No results found with direct search") else: for result in wiki_results: print("Raw result:", result) # Debug print if 'id' in result: title_id = result['id'] index = title_to_index.get(title_id) if index is not None: record = wiki_dataset[index] print(f"Found: {record['title']} (score: {result.get('score', 'N/A')})") else: print(f"ID {title_id} not found in title_to_index") else: print("Result has no ID field:", result) except Exception as e: print(f"Error with direct search approach: {e}")Key improvements in this version:
- Added extensive debugging output to understand what's happening at each step
- Added error checking for file paths and dataset loading
- Implemented both SQL syntax and direct search approaches to see which works better
- Added more detailed output of search results to understand their structure
- Added checks for missing fields in the results
This should help identify where the problem is occurring. If the direct search approach works but the SQL approach doesn't, then we know the issue is with the SQL query syntax. If neither works, then there might be an issue with how the index was built or how the dataset is being loaded.
The debug output will show:
- Whether the dataset and embeddings are loading correctly
- The structure of the search results
- Whether the title_to_index mapping is working correctly
- Which approach (SQL or direct search) works better
Based on the output from this debug version, we can then make more targeted fixes to the script. For example, if we see that the SQL query isn't returning results but the direct search is, we might need to adjust the SQL query syntax or how the index was built.
If you run this version and share the output, I can help further diagnose the specific issue. The most likely fixes would involve either adjusting the SQL query syntax or modifying how we map search results back to article titles.
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.