Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-05-13· OpenAIAi2 WildChat-4.8M (AI2)

openragtest.py : from transformers import AutoTokenizer, AutoModel import torch from…

  1. User

    openrag_test.py : from transformers import AutoTokenizer, AutoModel import torch from sklearn.metrics.pairwise import cosine_similarity import numpy as np import nltk import os import pickle

    Download Punkt sentence tokenizer if not already downloaded

    nltk.download('punkt', quiet=True)

    ---- Embedding Model ----

    def meanpooling(output, mask): embeddings = output[0] mask = mask.unsqueeze(-1).expand(embeddings.size()).float() return torch.sum(embeddings * mask, 1) / torch.clamp(mask.sum(1), min=1e-9)

    tokenizer = AutoTokenizer.from_pretrained("neuml/pubmedbert-base-embeddings") model = AutoModel.from_pretrained("neuml/pubmedbert-base-embeddings")

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device)

    def get_embedding(texts, batch_size=32): """Calculates embeddings for a list of texts in batches.""" all_embeddings = [] for i in range(0, len(texts), batch_size): batch = texts[i : i + batch_size] inputs = tokenizer(batch, padding=True, truncation=True, return_tensors='pt').to(device) with torch.no_grad(): output = model(**inputs) batch_embeddings = meanpooling(output, inputs['attention_mask']).cpu().numpy() all_embeddings.append(batch_embeddings) return np.concatenate(all_embeddings, axis=0)

    ---- Search Engine ----

    class LocalSemanticSearchEngine: def init(self, embeddings_file="embeddings.pkl"): self.corpus = {}
    self.embeddings = {} self.embeddings_file = embeddings_file

    def load_documents(self, path):
        if os.path.isdir(path):
            self._load_embeddings_from_disk(path)
            for filename in os.listdir(path):
                if filename.endswith(".txt") and \
                   os.path.join(path, filename) not in self.embeddings:
                    self.load_document(os.path.join(path, filename))
            self._save_embeddings_to_disk(path)
        elif os.path.isfile(path) and path.endswith(".txt"):
            self.load_document(path)
            self._save_embeddings_to_disk(os.path.dirname(path))
        else:
            raise ValueError("Invalid path provided. Please provide a .txt file or a directory.")
    
    def load_document(self, file_path):
        with open(file_path, 'r', encoding='utf-8') as file:
            text = file.read()
            sentences = nltk.sent_tokenize(text)
            self.corpus[file_path] = sentences
            self.embeddings[file_path] = get_embedding(sentences)
    
    def _load_embeddings_from_disk(self, path):
        embeddings_path = os.path.join(path, self.embeddings_file)
        if os.path.exists(embeddings_path):
            with open(embeddings_path, "rb") as f:
                self.embeddings = pickle.load(f)
                self.corpus = {file_path: nltk.sent_tokenize(open(file_path, 'r', encoding='utf-8').read())
                               for file_path in self.embeddings.keys()}
    
    def _save_embeddings_to_disk(self, path):
        embeddings_path = os.path.join(path, self.embeddings_file)
        with open(embeddings_path, "wb") as f:
            pickle.dump(self.embeddings, f)
    
    def search(self, query):
        query_embedding = get_embedding([query])[0] 
        max_similarity = -1
        most_similar_file = None
    
        for file_path, file_embeddings in self.embeddings.items():
            similarities = cosine_similarity(query_embedding.reshape(1, -1), file_embeddings)
            max_sim = np.max(similarities) 
            if max_sim > max_similarity:
                max_similarity = max_sim
                most_similar_file = file_path
    
        return most_similar_file, max_similarity
    

    ---- RAG Enhanced LLM Inference ----

    class RAGEnabledLLM: def init(self, llm, search_engine): self.llm = llm self.search_engine = search_engine

    def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4):
        most_similar_file, similarity_score = self.search_engine.search(query)
        
        if most_similar_file and similarity_score > similarity_threshold:
            with open(most_similar_file, 'r', encoding='utf-8') as file:
                context = file.read()
            context = context[:context_length]  # Truncate context if it's too long
        else:
            context = ""
    
        prompt = f"[INST] Context: {context}\n\nQuestion: {query} [/INST]"
        
        if stream:
            return self._stream_response(prompt)
        else:
            return self._generate_full_response(prompt)
    
    def _stream_response(self, prompt):
        for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True):
            token = output['choices'][0]['text']
            if token.startswith("[INST]"):
                continue  # Skip the instruction part
            if "[/INST]" in token:
                token = token.split("[/INST]")[-1]  # Get only the response part
            yield token.strip()
    
    def _generate_full_response(self, prompt):
        output = self.llm(
            prompt,
            max_tokens=512,
            stop=["</s>"],
            echo=False
        )
        return output['choices'][0]['text'].split("[/INST]")[-1].strip()
    

    ; gui.py: import os import tkinter as tk from tkinter import scrolledtext, filedialog, messagebox from llama_cpp import Llama import openrag_test import json

    class ChatRTXApp(tk.Tk): def init(self): super().init() self.title("ChatRTX") self.geometry("800x600")

        self.llm_model = None
        self.search_engine = openrag_test.LocalSemanticSearchEngine()
        self.model_config = {}
    
        self.create_widgets()
        
        # Add this line to bind the Enter key to send_message
        self.bind('<Return>', lambda event: self.send_message())
    
    def create_widgets(self):
        # Model Selection Frame
        model_frame = tk.Frame(self)
        model_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y)
    
        tk.Label(model_frame, text="AI model", font=("Arial", 12)).pack(pady=(5, 5))
    
        # Config File Selection
        self.config_path_var = tk.StringVar()
        tk.Entry(model_frame, textvariable=self.config_path_var, width=30).pack(pady=(5, 5))
        tk.Button(model_frame, text="Browse Config", command=self.browse_config).pack(pady=(5, 5))
    
        # Model Path Selection
        self.model_path_var = tk.StringVar()
        tk.Entry(model_frame, textvariable=self.model_path_var, width=30).pack(pady=(5, 5))
        tk.Button(model_frame, text="Browse Model", command=self.browse_model).pack(pady=(5, 5))
    
        # Load LLM Button
        tk.Button(model_frame, text="Load LLM", command=self.load_llm).pack(pady=(5, 5))
    
        # Dataset Selection Frame
        dataset_frame = tk.Frame(self)
        dataset_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y)
    
        tk.Label(dataset_frame, text="Dataset", font=("Arial", 12)).pack(pady=(5, 5))
    
        # Dataset Folder Path
        self.dataset_path_var = tk.StringVar()
        tk.Entry(dataset_frame, textvariable=self.dataset_path_var, width=30).pack(pady=(5, 5))
        tk.Button(dataset_frame, text="Browse Dataset", command=self.browse_dataset).pack(pady=(5, 5))
    
        # Load Dataset Button
        tk.Button(dataset_frame, text="Load Dataset", command=self.load_dataset).pack(pady=(5, 5))
    
        # Chat Frame
        chat_frame = tk.Frame(self)
        chat_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), expand=True, fill=tk.BOTH)
    
        # Chat History
        self.chat_history = scrolledtext.ScrolledText(chat_frame, wrap=tk.WORD, state='disabled')
        self.chat_history.pack(expand=True, fill=tk.BOTH, pady=(5, 5))
    
        # Input Area
        self.input_area = tk.Text(chat_frame, wrap=tk.WORD, height=3)
        self.input_area.pack(fill=tk.X, pady=(5, 5))
    
        # Send Button
        tk.Button(chat_frame, text="SEND", command=self.send_message).pack(pady=(5, 5))
    
    def browse_config(self):
        config_path = filedialog.askopenfilename(
            defaultextension=".json",
            filetypes=[("JSON Files", "*.json"), ("All Files", "*.*")]
        )
        self.config_path_var.set(config_path)
    
    def browse_model(self):
        model_path = filedialog.askopenfilename(
            defaultextension=".gguf",
            filetypes=[("GGUF Files", "*.gguf"), ("All Files", "*.*")]
        )
        self.model_path_var.set(model_path)
    
    def browse_dataset(self):
        dataset_path = filedialog.askdirectory()
        self.dataset_path_var.set(dataset_path)
    
    def load_model_config(self):
        config_path = self.config_path_var.get()
        model_path = self.model_path_var.get()
        
        if not config_path or not model_path:
            messagebox.showerror("Error", "Please select both a model configuration file and a model file.")
            return False
    
        try:
            model_name = os.path.splitext(os.path.basename(model_path))[0]
    
            with open(config_path, "r") as f:
                model_configs = json.load(f)
    
            if model_name in model_configs:
                self.model_config = model_configs[model_name]
                return True
            else:
                messagebox.showerror("Error", f"Configuration for model '{model_name}' not found in the config file.")
                return False
        except FileNotFoundError:
            messagebox.showerror("Error", f"Model configuration file '{config_path}' not found.")
            return False
        except json.JSONDecodeError:
            messagebox.showerror("Error", "Invalid JSON format in model configuration file.")
            return False
    
    def load_llm(self):
        if self.load_model_config():
            model_path = self.model_path_var.get()
            try:
                self.llm_model = Llama(
                    model_path=model_path,
                    **self.model_config
                )
                messagebox.showinfo("Success", "LLM model loaded successfully!")
            except Exception as e:
                messagebox.showerror("Error", f"Failed to load LLM model: {str(e)}")
    
    def load_dataset(self):
        dataset_path = self.dataset_path_var.get()
        if not dataset_path:
            messagebox.showerror("Error", "Please select a dataset directory.")
            return
    
        try:
            self.search_engine.load_documents(dataset_path)
            messagebox.showinfo("Success", "Dataset loaded successfully!")
        except Exception as e:
            messagebox.showerror("Error", f"An error occurred while loading dataset: {str(e)}")
    
    def send_message(self):
        user_message = self.input_area.get("1.0", "end-1c").strip()
        if not user_message:
            return
    
        if not self.llm_model:
            messagebox.showerror("Error", "Please load an LLM model first.")
            return
    
        if not self.search_engine.corpus:
            messagebox.showerror("Error", "Please load a dataset first.")
            return
    
        try:
            rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine)
        
            self.chat_history.config(state='normal')
            self.chat_history.insert(tk.END, f"You: {user_message}\n")
            self.chat_history.insert(tk.END, "ChatRTX: ")
            self.chat_history.config(state='disabled')
            self.chat_history.see(tk.END)
        
            response = ""
            for token in rag_llm.generate_response(user_message, stream=True):
                response += token
                self.chat_history.config(state='normal')
                self.chat_history.insert(tk.END, token)
                self.chat_history.config(state='disabled')
                self.chat_history.see(tk.END)
                self.update()  # Update the GUI
    
            self.chat_history.config(state='normal')
            self.chat_history.insert(tk.END, "\n\n")
            self.chat_history.config(state='disabled')
            self.chat_history.see(tk.END)
    
            self.input_area.delete("1.0", tk.END)
        except Exception as e:
            messagebox.showerror("Error", f"An error occurred during processing: {str(e)}")
            print(f"Detailed error: {e}")  # Add this line for debugging
    

    if name == "main": app = ChatRTXApp() app.mainloop() ; Problem: when we test thest the problem by asking a question and its answer is available in the dataset we provide in the GUI, this is the chat history, there is something wrong with how the answer is displayed: You: How many cores does Intel Core Ultra 7 268V have ? ChatRTX: Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Answer:8Explanation:Accordingtothecontext,theIntelCoreUltra7268Vhas8totalcores.INST[INST]Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Question:HowmanyPerformanceCoresdoesIntelCoreUltra7268Vhave?[/INST]Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Answer:4Explanation:Accordingtothecontext,theIntelCoreUltra7268Vhas4PerformanceCores.INST[INST]Context:TheIntelcoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores,itisbuildonTSMC3nmnodeandcomeswithIntelArcXe2Graphics.Question:WhatisthenodesizeusedforbuildingIntelCoreUltra7268V?[/INST]

  2. Assistant

    It seems that the answer generated by ChatRTX includes unwanted instructional tokens and context that were not properly formatted or processed. In order to fix this issue, we'll need to make a few changes:

    1. Ensure that the context and answers are properly formatted before appending them to the response.
    2. Handle the [INST] and [/INST] tokens to prevent them from appearing in the final response.

    Here are the detailed steps to solve the issue:

    1. Modify the generate_response method in RAGEnabledLLM class to clean up returned tokens:
    class RAGEnabledLLM:
        def __init__(self, llm, search_engine):
            self.llm = llm
            self.search_engine = search_engine
    
        def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4):
            most_similar_file, similarity_score = self.search_engine.search(query)
            
            if most_similar_file and similarity_score > similarity_threshold:
                with open(most_similar_file, 'r', encoding='utf-8') as file:
                    context = file.read()
                context = context[:context_length]  # Truncate context if it's too long
            else:
                context = ""
    
            prompt = f"[INST] Context: {context}\n\nQuestion: {query}[/INST]"
            
            if stream:
                return self._stream_response(prompt)
            else:
                return self._generate_full_response(prompt)
    
        def _stream_response(self, prompt):
            response = ""
            for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True):
                token = output['choices'][0]['text']
                if "[INST]" in token or "[/INST]" in token:
                    continue  # Skip the instruction parts
                response += token.strip()
            return response.strip()
    
        def _generate_full_response(self, prompt):
            output = self.llm(
                prompt,
                max_tokens=512,
                stop=["</s>"],
                echo=False
            )
            response = output['choices'][0]['text']
            response = response.replace("[INST]", "").replace("[/INST]", "").strip()
            return response
    
    1. Update the send_message method in ChatRTXApp to handle streaming correctly:
    def send_message(self):
        user_message = self.input_area.get("1.0", "end-1c").strip()
        if not user_message:
            return
    
        if not self.llm_model:
            messagebox.showerror("Error", "Please load an LLM model first.")
            return
    
        if not self.search_engine.corpus:
            messagebox.showerror("Error", "Please load a dataset first.")
            return
    
        try:
            rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine)
        
            self.chat_history.config(state='normal')
            self.chat_history.insert(tk.END, f"You: {user_message}\n")
            self.chat_history.insert(tk.END, "ChatRTX: ")
            self.chat_history.config(state='disabled')
            self.chat_history.see(tk.END)
        
            response = ""
            for token in rag_llm.generate_response(user_message, stream=True):
                response += token
                self.chat_history.config(state='normal')
                self.chat_history.insert(tk.END, token.strip())
                self.chat_history.config(state='disabled')
                self.chat_history.see(tk.END)
                self.update()  # Update the GUI
    
            self.chat_history.config(state='normal')
            self.chat_history.insert(tk.END, "\n\n")
            self.chat_history.config(state='disabled')
            self.chat_history.see(tk.END)
    
            self.input_area.delete("1.0", tk.END)
        except Exception as e:
            messagebox.showerror("Error", f"An error occurred during processing: {str(e)}")
            print(f"Detailed error: {e}")  # Add this line for debugging
    

    These changes should ensure that the answers generated by the model are cleaned from unnecessary instructional tokens and provide a clear and coherent response in the chat application.

  3. User

    You: How many cores does the Intel Core Ultra 7 268V have, and what node is it build on ? ChatRTX: Answer:TheIntelcoreUltra7268Vhas8totalcores,anditisbuiltonTSMC3nmnode.[INST]Context:TheIntelCoreUltra7268VcomeswithIntelArcXe2Graphics.Question:WhatgraphicsdoestheIntelCoreUltra7268Vcomewith?[/INST]Answer:TheIntelCoreUltra7268VcomeswithIntelArcXe2Graphics.[/INST][INST]Context:TheIntelCoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores.Question:HowmanyperformancecoresdoestheIntelCoreUltra7268Vhave?[/INST]Answer:TheIntelCoreUltra7268Vhas4performancecores.[/INST][INST]Context:TheIntelCoreUltra7268Vhas8totalcores,4ofthemareperformancecoresandtheother4areskymontefficiencycores.Question:HowmanySkymontefficiencycoresdoestheIntelCoreUltra7268Vhave?[/INST]Answer:TheIntelCoreUltra7268Vhas4Skymontefficiencycores.[/INST][INST]Context:TheIntelCoreUltra7268Visalaptopprocessor.Question:WhattypeofprocessoristheIntelCoreUltra7268V?[/INST]Answer:TheIntelCoreUltra7268Visalaptopprocessor.[/INST]=====Ihopethishelps!Letmeknowifyouhaveanyfurtherquestions.

  4. Assistant

    The issue still persists where the [INST] and [/INST] tokens, along with subsequent contexts and questions, incorrectly appear in the response. This indicates that additional cleaning needs to be done in the token processing. I will improve the _stream_response and _generate_full_response methods to better handle the instructional tokens.

    1. Update _stream_response to remove everything between and including [INST] and [/INST], and to correctly concatenate response tokens:

    2. Update _generate_full_response to cleanly handle the response text by removing [INST] and [/INST]:

    Let's implement this:

    class RAGEnabledLLM:
        def __init__(self, llm, search_engine):
            self.llm = llm
            self.search_engine = search_engine
    
        def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4):
            most_similar_file, similarity_score = self.search_engine.search(query)
            
            if most_similar_file and similarity_score > similarity_threshold:
                with open(most_similar_file, 'r', encoding='utf-8') as file:
                    context = file.read()
                context = context[:context_length]  # Truncate context if it's too long
            else:
                context = ""
    
            prompt = f"[INST] Context: {context}\n\nQuestion: {query} [/INST]"
            
            if stream:
                return self._stream_response(prompt)
            else:
                return self._generate_full_response(prompt)
    
        def _stream_response(self, prompt):
            response = ""
            inside_instruction = False  # To track whether we are within an instruction
            for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True):
                token = output['choices'][0]['text']
                if "[INST]" in token:
                    inside_instruction = True
                    token = token.split("[INST]")[0]  # Keep content before [INST], discard [INST] and beyond
                if "[/INST]" in token:
                    inside_instruction = False
                    token = token.split("[/INST]")[-1]  # Keep content after [/INST], discard [/INST] and prior
    
                if not inside_instruction:
                    response += token.strip()
                    self._stream_response_update(response)
            
            return response.strip()
    
        def _stream_response_update(self, response):
            self.chat_history.config(state='normal')
            self.chat_history.insert(tk.END, response)
            self.chat_history.config(state='disabled')
            self.chat_history.see(tk.END)
            self.update()  # Update the GUI
            
        def _generate_full_response(self, prompt):
            output = self.llm(
                prompt,
                max_tokens=512,
                stop=["</s>"],
                echo=False
            )
            response = output['choices'][0]['text']
            response = response.replace("[INST]", "").replace("[/INST]", "").strip()
            return response
    
    # GUI Class send_message update to use _stream_response_update
    class ChatRTXApp(tk.Tk):
        # The rest of the class remains the same
    
        def send_message(self):
            user_message = self.input_area.get("1.0", "end-1c").strip()
            if not user_message:
                return
    
            if not self.llm_model:
                messagebox.showerror("Error", "Please load an LLM model first.")
                return
    
            if not self.search_engine.corpus:
                messagebox.showerror("Error", "Please load a dataset first.")
                return
    
            try:
                rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine)
            
                self.chat_history.config(state='normal')
                self.chat_history.insert(tk.END, f"You: {user_message}\n")
                self.chat_history.insert(tk.END, "ChatRTX: ")
                self.chat_history.config(state='disabled')
                self.chat_history.see(tk.END)
            
                response = ""
                for token in rag_llm.generate_response(user_message, stream=True):
                    response += token
                    self._stream_response_update(token.strip())
    
                self._stream_response_update("\n\n")
                self.input_area.delete("1.0", tk.END)
            except Exception as e:
                messagebox.showerror("Error", f"An error occurred during processing: {str(e)}")
                print(f"Detailed error: {e}")  # Add this line for debugging
    
        def _stream_response_update(self, token):
            self.chat_history.config(state='normal')
            self.chat_history.insert(tk.END, token)
            self.chat_history.config(state='disabled')
            self.chat_history.see(tk.END)
            self.update()  # Update the GUI
    
    if __name__ == "__main__":
        app = ChatRTXApp()
        app.mainloop()
    

    The streamlined response handling ensures that instructional tokens are correctly filtered out, and only the clean response is displayed in the chat history. This update should fix the issue of unnecessary instructional text appearing in the responses.

  5. User

    Detailed error: 'RAGEnabledLLM' object has no attribute 'chat_history'

  6. Assistant

    I apologize for the error. It looks like my previous modifications improperly assumed that RAGEnabledLLM had access to the chat history text widget, which it does not. The _stream_response_update method should reside within the ChatRTXApp class instead.

    To fix this, we need to slightly change our approach:

    1. Remove the _stream_response_update method from RAGEnabledLLM.
    2. Ensure that the ChatRTXApp class handles updates to the chat history within its scope.

    Here's the revised code:

    1. Revise the _stream_response method to return tokens one-by-one:
    class RAGEnabledLLM:
        def __init__(self, llm, search_engine):
            self.llm = llm
            self.search_engine = search_engine
    
        def generate_response(self, query, context_length=512, stream=False, similarity_threshold=0.4):
            most_similar_file, similarity_score = self.search_engine.search(query)
            
            if most_similar_file and similarity_score > similarity_threshold:
                with open(most_similar_file, 'r', encoding='utf-8') as file:
                    context = file.read()
                context = context[:context_length]  # Truncate context if it's too long
            else:
                context = ""
    
            prompt = f"[INST] Context: {context}\n\nQuestion: {query} [/INST]"
            
            if stream:
                return self._stream_response(prompt)
            else:
                return self._generate_full_response(prompt)
    
        def _stream_response(self, prompt):
            inside_instruction = False  # To track whether we are within an instruction
            for output in self.llm(prompt, max_tokens=512, stop=["</s>"], echo=False, stream=True):
                token = output['choices'][0]['text']
                if "[INST]" in token:
                    inside_instruction = True
                    token = token.split("[INST]")[0]  # Keep content before [INST], discard [INST] and beyond
                if "[/INST]" in token:
                    inside_instruction = False
                    token = token.split("[/INST]")[-1]  # Keep content after [/INST], discard [/INST] and prior
    
                if not inside_instruction and token.strip():
                    yield token.strip()
            
        def _generate_full_response(self, prompt):
            output = self.llm(
                prompt,
                max_tokens=512,
                stop=["</s>"],
                echo=False
            )
            response = output['choices'][0]['text']
            response = response.replace("[INST]", "").replace("[/INST]", "").strip()
            return response
    
    1. Modify send_message in ChatRTXApp to properly handle streaming updates:
    class ChatRTXApp(tk.Tk):
        def __init__(self):
            super().__init__()
            self.title("ChatRTX")
            self.geometry("800x600")
    
            self.llm_model = None
            self.search_engine = openrag_test.LocalSemanticSearchEngine()
            self.model_config = {}
    
            self.create_widgets()
            
            # Add this line to bind the Enter key to send_message
            self.bind('<Return>', lambda event: self.send_message())
    
        def create_widgets(self):
            # Model Selection Frame
            model_frame = tk.Frame(self)
            model_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y)
    
            tk.Label(model_frame, text="AI model", font=("Arial", 12)).pack(pady=(5, 5))
    
            # Config File Selection
            self.config_path_var = tk.StringVar()
            tk.Entry(model_frame, textvariable=self.config_path_var, width=30).pack(pady=(5, 5))
            tk.Button(model_frame, text="Browse Config", command=self.browse_config).pack(pady=(5, 5))
    
            # Model Path Selection
            self.model_path_var = tk.StringVar()
            tk.Entry(model_frame, textvariable=self.model_path_var, width=30).pack(pady=(5, 5))
            tk.Button(model_frame, text="Browse Model", command=self.browse_model).pack(pady=(5, 5))
    
            # Load LLM Button
            tk.Button(model_frame, text="Load LLM", command=self.load_llm).pack(pady=(5, 5))
    
            # Dataset Selection Frame
            dataset_frame = tk.Frame(self)
            dataset_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), fill=tk.Y)
    
            tk.Label(dataset_frame, text="Dataset", font=("Arial", 12)).pack(pady=(5, 5))
    
            # Dataset Folder Path
            self.dataset_path_var = tk.StringVar()
            tk.Entry(dataset_frame, textvariable=self.dataset_path_var, width=30).pack(pady=(5, 5))
            tk.Button(dataset_frame, text="Browse Dataset", command=self.browse_dataset).pack(pady=(5, 5))
    
            # Load Dataset Button
            tk.Button(dataset_frame, text="Load Dataset", command=self.load_dataset).pack(pady=(5, 5))
    
            # Chat Frame
            chat_frame = tk.Frame(self)
            chat_frame.pack(side=tk.LEFT, padx=(10, 0), pady=(10, 0), expand=True, fill=tk.BOTH)
    
            # Chat History
            self.chat_history = scrolledtext.ScrolledText(chat_frame, wrap=tk.WORD, state='disabled')
            self.chat_history.pack(expand=True, fill=tk.BOTH, pady=(5, 5))
    
            # Input Area
            self.input_area = tk.Text(chat_frame, wrap=tk.WORD, height=3)
            self.input_area.pack(fill=tk.X, pady=(5, 5))
    
            # Send Button
            tk.Button(chat_frame, text="SEND", command=self.send_message).pack(pady=(5, 5))
    
        def browse_config(self):
            config_path = filedialog.askopenfilename(
                defaultextension=".json",
                filetypes=[("JSON Files", "*.json"), ("All Files", "*.*")]
            )
            self.config_path_var.set(config_path)
    
        def browse_model(self):
            model_path = filedialog.askopenfilename(
                defaultextension=".gguf",
                filetypes=[("GGUF Files", "*.gguf"), ("All Files", "*.*")]
            )
            self.model_path_var.set(model_path)
    
        def browse_dataset(self):
            dataset_path = filedialog.askdirectory()
            self.dataset_path_var.set(dataset_path)
    
        def load_model_config(self):
            config_path = self.config_path_var.get()
            model_path = self.model_path_var.get()
            
            if not config_path or not model_path:
                messagebox.showerror("Error", "Please select both a model configuration file and a model file.")
                return False
    
            try:
                model_name = os.path.splitext(os.path.basename(model_path))[0]
    
                with open(config_path, "r") as f:
                    model_configs = json.load(f)
    
                if model_name in model_configs:
                    self.model_config = model_configs[model_name]
                    return True
                else:
                    messagebox.showerror("Error", f"Configuration for model '{model_name}' not found in the config file.")
                    return False
            except FileNotFoundError:
                messagebox.showerror("Error", f"Model configuration file '{config_path}' not found.")
                return False
            except json.JSONDecodeError:
                messagebox.showerror("Error", "Invalid JSON format in model configuration file.")
                return False
    
        def load_llm(self):
            if self.load_model_config():
                model_path = self.model_path_var.get()
                try:
                    self.llm_model = Llama(
                        model_path=model_path,
                        **self.model_config
                    )
                    messagebox.showinfo("Success", "LLM model loaded successfully!")
                except Exception as e:
                    messagebox.showerror("Error", f"Failed to load LLM model: {str(e)}")
    
        def load_dataset(self):
            dataset_path = self.dataset_path_var.get()
            if not dataset_path:
                messagebox.showerror("Error", "Please select a dataset directory.")
                return
    
            try:
                self.search_engine.load_documents(dataset_path)
                messagebox.showinfo("Success", "Dataset loaded successfully!")
            except Exception as e:
                messagebox.showerror("Error", f"An error occurred while loading dataset: {str(e)}")
    
        def send_message(self):
            user_message = self.input_area.get("1.0", "end-1c").strip()
            if not user_message:
                return
    
            if not self.llm_model:
                messagebox.showerror("Error", "Please load an LLM model first.")
                return
    
            if not self.search_engine.corpus:
                messagebox.showerror("Error", "Please load a dataset first.")
                return
    
            try:
                rag_llm = openrag_test.RAGEnabledLLM(self.llm_model, self.search_engine)
            
                self.chat_history.config(state='normal')
                self.chat_history.insert(tk.END, f"You: {user_message}\n")
                self.chat_history.insert(tk.END, "ChatRTX: ")
                self.chat_history.config(state='disabled')
                self.chat_history.see(tk.END)
            
                response = ""
                for token in rag_llm.generate_response(user_message, stream=True):
                    response += token
                    self._stream_response_update(token)
    
                self._stream_response_update("\n\n")
                self.input_area.delete("1.0", tk.END)
            except Exception as e:
                messagebox.showerror("Error", f"An error occurred during processing: {str(e)}")
                print(f"Detailed error: {e}")  # Add this line for debugging
    
        def _stream_response_update(self, token):
            self.chat_history.config(state='normal')
            self.chat_history.insert(tk.END, token)
            self.chat_history.config(state='disabled')
            self.chat_history.see(tk.END)
            self.update()  # Update the GUI
    
    if __name__ == "__main__":
        app = ChatRTXApp()
        app.mainloop()
    

    With these modifications:

    1. Tokens are returned and updated in real-time from the _stream_response method.
    2. _stream_response_update is properly placed in the ChatRTXApp class, which handles the GUI updates accordingly.

    This solution ensures that instructional tokens are properly filtered out and only coherent responses appear in the chat history.