openragtest.py : from transformers import AutoTokenizer, AutoModel import torch from…
- User
openrag_test.py : from transformers import AutoTokenizer, AutoModel import torch from sklearn.metrics.pairwise import cosine_similarity import numpy as np import nltk import os import pickle
Download Punkt sentence tokenizer if not already downloaded
nltk.download('punkt', quiet=True)
---- Embedding Model ----
def meanpooling(output, mask): embeddings = output[0] mask = mask.unsqueeze(-1).expand(embeddings.size()).float() return torch.sum(embeddings * mask, 1) / torch.clamp(mask.sum(1), min=1e-9)
tokenizer = AutoTokenizer.from_pretrained("neuml/pubmedbert-base-embeddings") model = AutoModel.from_pretrained("neuml/pubmedbert-base-embeddings")
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device)
def get_embedding(texts, batch_size=32): """Calculates embeddings for a list of texts in batches.""" all_embeddings = [] for i in range(0, len(texts), batch_size): batch = texts[i : i + batch_size] inputs = tokenizer(batch, padding=True, truncation=True, return_tensors='pt').to(device) with torch.no_grad(): output = model(**inputs) batch_embeddings = meanpooling(output, inputs['attention_mask']).cpu().numpy() all_embeddings.append(batch_embeddings) return np.concatenate(all_embeddings, axis=0)
---- Search Engine ----
class LocalSemanticSearchEngine: def init(self, embeddings_file="embeddings.pkl"): self.corpus = {}
self.embeddings = {} self.embeddings_file = embeddings_filedef load_documents(self, path): if os.path.isdir(path): self._load_embeddings_from_disk(path) for filename in os.listdir(path): if filename.endswith(".txt") and \ os.path.join(path, filename) not in self.embeddings: self.load_document(os.path.join(path, filename)) self._save_embeddings_to_disk(path) elif os.path.isfile(path) and path.endswith(".txt"): self.load_document(path) self._save_embeddings_to_disk(os.path.dirname(path)) else: raise ValueError("Invalid path provided. Please provide a .txt file or a directory.") def load_document(self, file_path): with open(file_path, 'r', encoding='utf-8') as file: text = file.read() sentences = nltk.sent_tokenize(text) self.corpus[file_path] = sentences self.embeddings[file_path] = get_embedding(sentences) def _load_embeddings_from_disk(self, path): embeddings_path = os.path.join(path, self.embeddings_file) if os.path.exists(embeddings_path): with open(embeddings_path, "rb") as f: self.embeddings = pickle.load(f) self.corpus = {file_path: nltk.sent_tokenize(open(file_path, 'r', encoding='utf-8').read()) for file_path in self.embeddings.keys()} def _save_embeddings_to_disk(self, path): embeddings_path = os.path.join(path, self.embeddings_file) with open(embeddings_path, "wb") as f: pickle.dump(self.embeddings, f) def search(self, query): query_embedding = get_embedding([query])[0] max_similarity = -1 most_similar_file = None for file_path, file_embeddings in self.embeddings.items(): similarities = cosine_similarity(query_embedding.reshape(1, -1), file_embeddings) max_sim = np.max(similarities) if max_sim > max_similarity: max_similarity = max_sim most_similar_file = file_path return most_similar_file, max_similarity---- RAG Enhanced LLM Inference ----
class RAGEnabledLLM: def init(self, llm, search_engine): self.llm = llm self.search_engine = search_engine
def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4): most_similar_file, similarity_score = self.search_engine.search(query) if most_similar_file and similarity_score > similarity_threshold: with open(most_similar_file, 'r', encoding='utf-8') as file: context = file.read() context = context[:context_length] # Truncate context if it's too long else: context = "" prompt = f"[INST] Context: {context}\n\nQuestion: {query} [/INST]" if stream: return self._stream_response(prompt) else: return self._generate_full_response(prompt) def _stream_response(self, prompt): for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True): token = output['choices'][0]['text'] if token.startswith("[INST]"): continue # Skip the instruction part if "[/INST]" in token: token = token.split("[/INST]")[-1] # Get only the response part yield token.strip() def _generate_full_response(self, prompt): output = self.llm( prompt, max_tokens=512, stop=["</s>"], echo=False ) return output['choices'][0]['text'].split("[/INST]")[-1].strip(); gui.py: import os import tkinter as tk from tkinter import scrolledtext, filedialog, messagebox from llama_cpp import Llama import openrag_test import json
class ChatRTXApp(tk.Tk): def init(self): super().init() self.title("ChatRTX") self.geometry("800x600")
self.llm_model = None self.search_engine = openrag_test.LocalSemanticSearchEngine() self.model_config = {} self.create_widgets() # Add this line to bind the Enter key to send_message self.bind('<Return>', lambda event: self.send_message()) def create_widgets(self): # Model Selection Frame model_frame = tk.Frame(self) model_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y) tk.Label(model_frame, text="AI model", font=("Arial", 12)).pack(pady=(5, 5)) # Config File Selection self.config_path_var = tk.StringVar() tk.Entry(model_frame, textvariable=self.config_path_var, width=30).pack(pady=(5, 5)) tk.Button(model_frame, text="Browse Config", command=self.browse_config).pack(pady=(5, 5)) # Model Path Selection self.model_path_var = tk.StringVar() tk.Entry(model_frame, textvariable=self.model_path_var, width=30).pack(pady=(5, 5)) tk.Button(model_frame, text="Browse Model", command=self.browse_model).pack(pady=(5, 5)) # Load LLM Button tk.Button(model_frame, text="Load LLM", command=self.load_llm).pack(pady=(5, 5)) # Dataset Selection Frame dataset_frame = tk.Frame(self) dataset_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y) tk.Label(dataset_frame, text="Dataset", font=("Arial", 12)).pack(pady=(5, 5)) # Dataset Folder Path self.dataset_path_var = tk.StringVar() tk.Entry(dataset_frame, textvariable=self.dataset_path_var, width=30).pack(pady=(5, 5)) tk.Button(dataset_frame, text="Browse Dataset", command=self.browse_dataset).pack(pady=(5, 5)) # Load Dataset Button tk.Button(dataset_frame, text="Load Dataset", command=self.load_dataset).pack(pady=(5, 5)) # Chat Frame chat_frame = tk.Frame(self) chat_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), expand=True, fill=tk.BOTH) # Chat History self.chat_history = scrolledtext.ScrolledText(chat_frame, wrap=tk.WORD, state='disabled') self.chat_history.pack(expand=True, fill=tk.BOTH, pady=(5, 5)) # Input Area self.input_area = tk.Text(chat_frame, wrap=tk.WORD, height=3) self.input_area.pack(fill=tk.X, pady=(5, 5)) # Send Button tk.Button(chat_frame, text="SEND", command=self.send_message).pack(pady=(5, 5)) def browse_config(self): config_path = filedialog.askopenfilename( defaultextension=".json", filetypes=[("JSON Files", "*.json"), ("All Files", "*.*")] ) self.config_path_var.set(config_path) def browse_model(self): model_path = filedialog.askopenfilename( defaultextension=".gguf", filetypes=[("GGUF Files", "*.gguf"), ("All Files", "*.*")] ) self.model_path_var.set(model_path) def browse_dataset(self): dataset_path = filedialog.askdirectory() self.dataset_path_var.set(dataset_path) def load_model_config(self): config_path = self.config_path_var.get() model_path = self.model_path_var.get() if not config_path or not model_path: messagebox.showerror("Error", "Please select both a model configuration file and a model file.") return False try: model_name = os.path.splitext(os.path.basename(model_path))[0] with open(config_path, "r") as f: model_configs = json.load(f) if model_name in model_configs: self.model_config = model_configs[model_name] return True else: messagebox.showerror("Error", f"Configuration for model '{model_name}' not found in the config file.") return False except FileNotFoundError: messagebox.showerror("Error", f"Model configuration file '{config_path}' not found.") return False except json.JSONDecodeError: messagebox.showerror("Error", "Invalid JSON format in model configuration file.") return False def load_llm(self): if self.load_model_config(): model_path = self.model_path_var.get() try: self.llm_model = Llama( model_path=model_path, **self.model_config ) messagebox.showinfo("Success", "LLM model loaded successfully!") except Exception as e: messagebox.showerror("Error", f"Failed to load LLM model: {str(e)}") def load_dataset(self): dataset_path = self.dataset_path_var.get() if not dataset_path: messagebox.showerror("Error", "Please select a dataset directory.") return try: self.search_engine.load_documents(dataset_path) messagebox.showinfo("Success", "Dataset loaded successfully!") except Exception as e: messagebox.showerror("Error", f"An error occurred while loading dataset: {str(e)}") def send_message(self): user_message = self.input_area.get("1.0", "end-1c").strip() if not user_message: return if not self.llm_model: messagebox.showerror("Error", "Please load an LLM model first.") return if not self.search_engine.corpus: messagebox.showerror("Error", "Please load a dataset first.") return try: rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine) self.chat_history.config(state='normal') self.chat_history.insert(tk.END, f"You: {user_message}\n") self.chat_history.insert(tk.END, "ChatRTX: ") self.chat_history.config(state='disabled') self.chat_history.see(tk.END) response = "" for token in rag_llm.generate_response(user_message, stream=True): response += token self.chat_history.config(state='normal') self.chat_history.insert(tk.END, token) self.chat_history.config(state='disabled') self.chat_history.see(tk.END) self.update() # Update the GUI self.chat_history.config(state='normal') self.chat_history.insert(tk.END, "\n\n") self.chat_history.config(state='disabled') self.chat_history.see(tk.END) self.input_area.delete("1.0", tk.END) except Exception as e: messagebox.showerror("Error", f"An error occurred during processing: {str(e)}") print(f"Detailed error: {e}") # Add this line for debuggingif name == "main": app = ChatRTXApp() app.mainloop() ; Problem: when we test thest the problem by asking a question and its answer is available in the dataset we provide in the GUI, this is the chat history, there is something wrong with how the answer is displayed: You: How many cores does Intel Core Ultra 7 268V have ? ChatRTX: Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Answer:8Explanation:Accordingtothecontext,theIntelCoreUltra7268Vhas8totalcores.INST[INST]Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Question:HowmanyPerformanceCoresdoesIntelCoreUltra7268Vhave?[/INST]Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Answer:4Explanation:Accordingtothecontext,theIntelCoreUltra7268Vhas4PerformanceCores.INST[INST]Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Question:WhatisthenodesizeusedforbuildingIntelCoreUltra7268V?[/INST]
- Assistant
It seems that the answer generated by ChatRTX includes unwanted instructional tokens and context that were not properly formatted or processed. In order to fix this issue, we'll need to make a few changes:
- Ensure that the context and answers are properly formatted before appending them to the response.
- Handle the
[INST]and[/INST]tokens to prevent them from appearing in the final response.
Here are the detailed steps to solve the issue:
- Modify the
generate_responsemethod inRAGEnabledLLMclass to clean up returned tokens:
class RAGEnabledLLM: def __init__(self, llm, search_engine): self.llm = llm self.search_engine = search_engine def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4): most_similar_file, similarity_score = self.search_engine.search(query) if most_similar_file and similarity_score > similarity_threshold: with open(most_similar_file, 'r', encoding='utf-8') as file: context = file.read() context = context[:context_length] # Truncate context if it's too long else: context = "" prompt = f"[INST] Context: {context}\n\nQuestion: {query}[/INST]" if stream: return self._stream_response(prompt) else: return self._generate_full_response(prompt) def _stream_response(self, prompt): response = "" for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True): token = output['choices'][0]['text'] if "[INST]" in token or "[/INST]" in token: continue # Skip the instruction parts response += token.strip() return response.strip() def _generate_full_response(self, prompt): output = self.llm( prompt, max_tokens=512, stop=["</s>"], echo=False ) response = output['choices'][0]['text'] response = response.replace("[INST]", "").replace("[/INST]", "").strip() return response- Update the
send_messagemethod inChatRTXAppto handle streaming correctly:
def send_message(self): user_message = self.input_area.get("1.0", "end-1c").strip() if not user_message: return if not self.llm_model: messagebox.showerror("Error", "Please load an LLM model first.") return if not self.search_engine.corpus: messagebox.showerror("Error", "Please load a dataset first.") return try: rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine) self.chat_history.config(state='normal') self.chat_history.insert(tk.END, f"You: {user_message}\n") self.chat_history.insert(tk.END, "ChatRTX: ") self.chat_history.config(state='disabled') self.chat_history.see(tk.END) response = "" for token in rag_llm.generate_response(user_message, stream=True): response += token self.chat_history.config(state='normal') self.chat_history.insert(tk.END, token.strip()) self.chat_history.config(state='disabled') self.chat_history.see(tk.END) self.update() # Update the GUI self.chat_history.config(state='normal') self.chat_history.insert(tk.END, "\n\n") self.chat_history.config(state='disabled') self.chat_history.see(tk.END) self.input_area.delete("1.0", tk.END) except Exception as e: messagebox.showerror("Error", f"An error occurred during processing: {str(e)}") print(f"Detailed error: {e}") # Add this line for debuggingThese changes should ensure that the answers generated by the model are cleaned from unnecessary instructional tokens and provide a clear and coherent response in the chat application.
- User
You: How many cores does the Intel Core Ultra 7 268V have, and what node is it build on ? ChatRTX: Answer:TheIntelcoreUltra7268Vhas8totalcores,anditisbuiltonTSMC3nmnode.[INST]Context:TheIntelCoreUltra7268VcomeswithIntelArcXe2Graphics.Question:WhatgraphicsdoestheIntelCoreUltra7268Vcomewith?[/INST]Answer:TheIntelCoreUltra7268VcomeswithIntelArcXe2Graphics.[/INST][INST]Context:TheIntelCoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores.Question:HowmanyperformancecoresdoestheIntelCoreUltra7268Vhave?[/INST]Answer:TheIntelCoreUltra7268Vhas4performancecores.[/INST][INST]Context:TheIntelCoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores.Question:HowmanySkymontefficiencycoresdoestheIntelCoreUltra7268Vhave?[/INST]Answer:TheIntelCoreUltra7268Vhas4Skymontefficiencycores.[/INST][INST]Context:TheIntelCoreUltra7268Visalaptopprocessor.Question:WhattypeofprocessoristheIntelCoreUltra7268V?[/INST]Answer:TheIntelCoreUltra7268Visalaptopprocessor.[/INST]=====Ihopethishelps!Letmeknowifyouhaveanyfurtherquestions.
- Assistant
The issue still persists where the
[INST]and[/INST]tokens, along with subsequent contexts and questions, incorrectly appear in the response. This indicates that additional cleaning needs to be done in the token processing. I will improve the_stream_responseand_generate_full_responsemethods to better handle the instructional tokens.-
Update
_stream_responseto remove everything between and including[INST]and[/INST], and to correctly concatenate response tokens: -
Update
_generate_full_responseto cleanly handle the response text by removing[INST]and[/INST]:
Let's implement this:
class RAGEnabledLLM: def __init__(self, llm, search_engine): self.llm = llm self.search_engine = search_engine def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4): most_similar_file, similarity_score = self.search_engine.search(query) if most_similar_file and similarity_score > similarity_threshold: with open(most_similar_file, 'r', encoding='utf-8') as file: context = file.read() context = context[:context_length] # Truncate context if it's too long else: context = "" prompt = f"[INST] Context: {context}\n\nQuestion: {query} [/INST]" if stream: return self._stream_response(prompt) else: return self._generate_full_response(prompt) def _stream_response(self, prompt): response = "" inside_instruction = False # To track whether we are within an instruction for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True): token = output['choices'][0]['text'] if "[INST]" in token: inside_instruction = True token = token.split("[INST]")[0] # Keep content before [INST], discard [INST] and beyond if "[/INST]" in token: inside_instruction = False token = token.split("[/INST]")[-1] # Keep content after [/INST], discard [/INST] and prior if not inside_instruction: response += token.strip() self._stream_response_update(response) return response.strip() def _stream_response_update(self, response): self.chat_history.config(state='normal') self.chat_history.insert(tk.END, response) self.chat_history.config(state='disabled') self.chat_history.see(tk.END) self.update() # Update the GUI def _generate_full_response(self, prompt): output = self.llm( prompt, max_tokens=512, stop=["</s>"], echo=False ) response = output['choices'][0]['text'] response = response.replace("[INST]", "").replace("[/INST]", "").strip() return response # GUI Class send_message update to use _stream_response_update class ChatRTXApp(tk.Tk): # The rest of the class remains the same def send_message(self): user_message = self.input_area.get("1.0", "end-1c").strip() if not user_message: return if not self.llm_model: messagebox.showerror("Error", "Please load an LLM model first.") return if not self.search_engine.corpus: messagebox.showerror("Error", "Please load a dataset first.") return try: rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine) self.chat_history.config(state='normal') self.chat_history.insert(tk.END, f"You: {user_message}\n") self.chat_history.insert(tk.END, "ChatRTX: ") self.chat_history.config(state='disabled') self.chat_history.see(tk.END) response = "" for token in rag_llm.generate_response(user_message, stream=True): response += token self._stream_response_update(token.strip()) self._stream_response_update("\n\n") self.input_area.delete("1.0", tk.END) except Exception as e: messagebox.showerror("Error", f"An error occurred during processing: {str(e)}") print(f"Detailed error: {e}") # Add this line for debugging def _stream_response_update(self, token): self.chat_history.config(state='normal') self.chat_history.insert(tk.END, token) self.chat_history.config(state='disabled') self.chat_history.see(tk.END) self.update() # Update the GUI if __name__ == "__main__": app = ChatRTXApp() app.mainloop()The streamlined response handling ensures that instructional tokens are correctly filtered out, and only the clean response is displayed in the chat history. This update should fix the issue of unnecessary instructional text appearing in the responses.
-
- User
Detailed error: 'RAGEnabledLLM' object has no attribute 'chat_history'
- Assistant
I apologize for the error. It looks like my previous modifications improperly assumed that
RAGEnabledLLMhad access to the chat history text widget, which it does not. The_stream_response_updatemethod should reside within theChatRTXAppclass instead.To fix this, we need to slightly change our approach:
- Remove the
_stream_response_updatemethod fromRAGEnabledLLM. - Ensure that the
ChatRTXAppclass handles updates to the chat history within its scope.
Here's the revised code:
- Revise the
_stream_responsemethod to return tokens one-by-one:
class RAGEnabledLLM: def __init__(self, llm, search_engine): self.llm = llm self.search_engine = search_engine def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4): most_similar_file, similarity_score = self.search_engine.search(query) if most_similar_file and similarity_score > similarity_threshold: with open(most_similar_file, 'r', encoding='utf-8') as file: context = file.read() context = context[:context_length] # Truncate context if it's too long else: context = "" prompt = f"[INST] Context: {context}\n\nQuestion: {query} [/INST]" if stream: return self._stream_response(prompt) else: return self._generate_full_response(prompt) def _stream_response(self, prompt): inside_instruction = False # To track whether we are within an instruction for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True): token = output['choices'][0]['text'] if "[INST]" in token: inside_instruction = True token = token.split("[INST]")[0] # Keep content before [INST], discard [INST] and beyond if "[/INST]" in token: inside_instruction = False token = token.split("[/INST]")[-1] # Keep content after [/INST], discard [/INST] and prior if not inside_instruction and token.strip(): yield token.strip() def _generate_full_response(self, prompt): output = self.llm( prompt, max_tokens=512, stop=["</s>"], echo=False ) response = output['choices'][0]['text'] response = response.replace("[INST]", "").replace("[/INST]", "").strip() return response- Modify
send_messageinChatRTXAppto properly handle streaming updates:
class ChatRTXApp(tk.Tk): def __init__(self): super().__init__() self.title("ChatRTX") self.geometry("800x600") self.llm_model = None self.search_engine = openrag_test.LocalSemanticSearchEngine() self.model_config = {} self.create_widgets() # Add this line to bind the Enter key to send_message self.bind('<Return>', lambda event: self.send_message()) def create_widgets(self): # Model Selection Frame model_frame = tk.Frame(self) model_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y) tk.Label(model_frame, text="AI model", font=("Arial", 12)).pack(pady=(5, 5)) # Config File Selection self.config_path_var = tk.StringVar() tk.Entry(model_frame, textvariable=self.config_path_var, width=30).pack(pady=(5, 5)) tk.Button(model_frame, text="Browse Config", command=self.browse_config).pack(pady=(5, 5)) # Model Path Selection self.model_path_var = tk.StringVar() tk.Entry(model_frame, textvariable=self.model_path_var, width=30).pack(pady=(5, 5)) tk.Button(model_frame, text="Browse Model", command=self.browse_model).pack(pady=(5, 5)) # Load LLM Button tk.Button(model_frame, text="Load LLM", command=self.load_llm).pack(pady=(5, 5)) # Dataset Selection Frame dataset_frame = tk.Frame(self) dataset_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y) tk.Label(dataset_frame, text="Dataset", font=("Arial", 12)).pack(pady=(5, 5)) # Dataset Folder Path self.dataset_path_var = tk.StringVar() tk.Entry(dataset_frame, textvariable=self.dataset_path_var, width=30).pack(pady=(5, 5)) tk.Button(dataset_frame, text="Browse Dataset", command=self.browse_dataset).pack(pady=(5, 5)) # Load Dataset Button tk.Button(dataset_frame, text="Load Dataset", command=self.load_dataset).pack(pady=(5, 5)) # Chat Frame chat_frame = tk.Frame(self) chat_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), expand=True, fill=tk.BOTH) # Chat History self.chat_history = scrolledtext.ScrolledText(chat_frame, wrap=tk.WORD, state='disabled') self.chat_history.pack(expand=True, fill=tk.BOTH, pady=(5, 5)) # Input Area self.input_area = tk.Text(chat_frame, wrap=tk.WORD, height=3) self.input_area.pack(fill=tk.X, pady=(5, 5)) # Send Button tk.Button(chat_frame, text="SEND", command=self.send_message).pack(pady=(5, 5)) def browse_config(self): config_path = filedialog.askopenfilename( defaultextension=".json", filetypes=[("JSON Files", "*.json"), ("All Files", "*.*")] ) self.config_path_var.set(config_path) def browse_model(self): model_path = filedialog.askopenfilename( defaultextension=".gguf", filetypes=[("GGUF Files", "*.gguf"), ("All Files", "*.*")] ) self.model_path_var.set(model_path) def browse_dataset(self): dataset_path = filedialog.askdirectory() self.dataset_path_var.set(dataset_path) def load_model_config(self): config_path = self.config_path_var.get() model_path = self.model_path_var.get() if not config_path or not model_path: messagebox.showerror("Error", "Please select both a model configuration file and a model file.") return False try: model_name = os.path.splitext(os.path.basename(model_path))[0] with open(config_path, "r") as f: model_configs = json.load(f) if model_name in model_configs: self.model_config = model_configs[model_name] return True else: messagebox.showerror("Error", f"Configuration for model '{model_name}' not found in the config file.") return False except FileNotFoundError: messagebox.showerror("Error", f"Model configuration file '{config_path}' not found.") return False except json.JSONDecodeError: messagebox.showerror("Error", "Invalid JSON format in model configuration file.") return False def load_llm(self): if self.load_model_config(): model_path = self.model_path_var.get() try: self.llm_model = Llama( model_path=model_path, **self.model_config ) messagebox.showinfo("Success", "LLM model loaded successfully!") except Exception as e: messagebox.showerror("Error", f"Failed to load LLM model: {str(e)}") def load_dataset(self): dataset_path = self.dataset_path_var.get() if not dataset_path: messagebox.showerror("Error", "Please select a dataset directory.") return try: self.search_engine.load_documents(dataset_path) messagebox.showinfo("Success", "Dataset loaded successfully!") except Exception as e: messagebox.showerror("Error", f"An error occurred while loading dataset: {str(e)}") def send_message(self): user_message = self.input_area.get("1.0", "end-1c").strip() if not user_message: return if not self.llm_model: messagebox.showerror("Error", "Please load an LLM model first.") return if not self.search_engine.corpus: messagebox.showerror("Error", "Please load a dataset first.") return try: rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine) self.chat_history.config(state='normal') self.chat_history.insert(tk.END, f"You: {user_message}\n") self.chat_history.insert(tk.END, "ChatRTX: ") self.chat_history.config(state='disabled') self.chat_history.see(tk.END) response = "" for token in rag_llm.generate_response(user_message, stream=True): response += token self._stream_response_update(token) self._stream_response_update("\n\n") self.input_area.delete("1.0", tk.END) except Exception as e: messagebox.showerror("Error", f"An error occurred during processing: {str(e)}") print(f"Detailed error: {e}") # Add this line for debugging def _stream_response_update(self, token): self.chat_history.config(state='normal') self.chat_history.insert(tk.END, token) self.chat_history.config(state='disabled') self.chat_history.see(tk.END) self.update() # Update the GUI if __name__ == "__main__": app = ChatRTXApp() app.mainloop()With these modifications:
- Tokens are returned and updated in real-time from the
_stream_responsemethod. _stream_response_updateis properly placed in theChatRTXAppclass, which handles the GUI updates accordingly.
This solution ensures that instructional tokens are properly filtered out and only coherent responses appear in the chat history.
- Remove the