I have developed a code - """import numpy as np from langchain.embeddings import…
- User
I have developed a code - """import numpy as np from langchain.embeddings import HuggingFaceEmbeddings from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import Chroma from langchain.document_loaders import PyPDFLoader from langchain import HuggingFacePipeline from langchain import PromptTemplate from langchain.chains import ConversationalRetrievalChain from langchain.memory import ConversationBufferWindowMemory from sentence_transformers import SentenceTransformer import gradio as gr from langchain_community.vectorstores import FAISS from flashrank.Ranker import Ranker, RerankRequest
#nltk.download() import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline, TextStreamer
def create_multipage_pdf(text, filename='output.pdf', font_size=12, margin=50): # Split text into chunks that fit on one page chunks = [] chunk_size = 50 # Number of characters per chunk for i in range(0, len(text), chunk_size): chunks.append(text[i:i+chunk_size])
# Create PDF c = canvas.Canvas(filename, pagesize=letter) width, height = letter y = height - margin for chunk in chunks: # Draw text text_object = c.beginText(margin, y) text_object.setFont("Helvetica", font_size) text_object.textLines(chunk) c.drawText(text_object) # Update y position y -= (font_size + 4) # Adjust spacing between lines # Check if we need to start a new page if y <= margin: c.showPage() y = height - margin c.save()Example usage
#create_multipage_pdf(text, filename='document.pdf')
bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=False)
model_id = "meta-llama/Llama-2-7b-chat-hf"
tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config = bnb_config,device_map={"":0})
import json import textwrap
B_INST, E_INST = "[INST]", "[/INST]" B_SYS, E_SYS = "<>\n", "\n<>\n\n" DEFAULT_SYSTEM_PROMPT = """
You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."""
def get_prompt(instruction, new_system_prompt=DEFAULT_SYSTEM_PROMPT ): SYSTEM_PROMPT = B_SYS + new_system_prompt + E_SYS prompt_template = B_INST + SYSTEM_PROMPT + instruction + E_INST return prompt_template
loader = PyPDFLoader("document.pdf")
text_splitter = RecursiveCharacterTextSplitter( # Set a really small chunk size, just to show. chunk_size = 500, chunk_overlap = 20, length_function = len, )
pages = loader.load_and_split(text_splitter)
testing with embeddings
embeddings = HuggingFaceEmbeddings(model_name="hkunlp/instructor-xl", model_kwargs={'device': 'cuda'}) #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={'device': 'cuda'}) print("OK1") db = FAISS.from_documents(pages, embedding=embeddings) #db = Chroma.from_documents(pages, embedding=embeddings) print("OK2") instruction = "Given the context that has been provided. \n {context}, Answer the following question - \n{question}"
This part was remove from first line of the system_prompt for testing purposes. Be precise in your answers wherever possible.
system_prompt = """You are an honest virtual assistant. You will be given a context to answer from. In case you are sure you don't know the answer then you say that based on the context you don't know the answer. In all other instances you provide an answer to the best of your capability. Cite context when you can access them maintaining formatting. Don't say 'based on the context provided more than once."""
get_prompt(instruction, system_prompt)
"""## Setting up with LangChain"""
template = get_prompt(instruction, system_prompt) print(template)
prompt = PromptTemplate(template=template, input_variables=["context", "question"])
memory = ConversationBufferWindowMemory( memory_key="chat_history", k=5, return_messages=True )
retriever = db.as_retriever(search_kwargs={'k': 10})
def create_pipeline(max_new_tokens=1024): pipe = pipeline("text-generation", model=model, tokenizer = tokenizer, max_new_tokens = max_new_tokens, temperature = 0.6) return pipe
class ChessBot:
send re_ranked docs instead of retriever for advanced RAG
def init(self, memory, prompt, task:str = "text-generation", retriever = retriever): self.memory = memory self.prompt = prompt self.retriever = retriever
def create_chat_bot(self, max_new_tokens = 1024): hf_pipe = create_pipeline(max_new_tokens) llm = HuggingFacePipeline(pipeline =hf_pipe) qa = ConversationalRetrievalChain.from_llm( llm=llm, retriever=self.retriever, memory=self.memory, combine_docs_chain_kwargs={"prompt": self.prompt} ) return qa
chess_bot = ChessBot(memory = memory, prompt = prompt)
bot = chess_bot.create_chat_bot()
def clear_llm_memory(): bot.memory.clear()
def update_prompt(sys_prompt): if sys_prompt == "": sys_prompt = system_prompt template = get_prompt(instruction, sys_prompt)
prompt = PromptTemplate(template=template, input_variables=["context", "question"])
bot.combine_docs_chain.llm_chain.prompt = prompt
"""1. Not using API 2. Use cases are not defined 3. Just a POC emphasis """
with gr.Blocks() as demo: gr.Markdown( """ # Please ask your questions! """) # Commenting update prompt option #update_sys_prompt = gr.Textbox(label = "Update System Prompt") chatbot = gr.Chatbot(label="SAMSUNG SDS", height = 300) msg = gr.Textbox(label = "Enter your query!") with gr.Column(scale=1): clear = gr.ClearButton([msg, chatbot]) with gr.Column(scale=1): clear_memory = gr.Button(value = "Clear LLM Memory")
def respond(message, chat_history): print("Query:::", message) bot_message = bot({"question": message})['answer'] chat_history.append((message, bot_message)) return "", chat_history msg.submit(respond, inputs=[msg, chatbot], outputs=[msg, chatbot]) clear_memory.click(clear_llm_memory) # Commenting update prompt option #update_sys_prompt.submit(update_prompt, inputs=update_sys_prompt)demo.launch(share=False, server_name='216.48.177.144', server_port=8502)
""" Now I want to modify this code to include this concept - """Long-Context Reorder
No matter the architecture of your model, there is a substantial performance degradation when you include 10+ retrieved documents. In brief: When models must access relevant information in the middle of long contexts, they tend to ignore the provided documents. See: https://arxiv.org/abs/2307.03172
To avoid this issue you can re-order documents after retrieval to avoid performance degradation.
%pip install --upgrade --quiet sentence-transformers > /dev/null
from langchain.chains import LLMChain, StuffDocumentsChain from langchain.prompts import PromptTemplate from langchain_community.document_transformers import ( LongContextReorder, ) from langchain_community.embeddings import HuggingFaceEmbeddings from langchain_community.vectorstores import Chroma from langchain_openai import OpenAI
Get embeddings.
embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
texts = [ "Basquetball is a great sport.", "Fly me to the moon is one of my favourite songs.", "The Celtics are my favourite team.", "This is a document about the Boston Celtics", "I simply love going to the movies", "The Boston Celtics won the game by 20 points", "This is just a random text.", "Elden Ring is one of the best games in the last 15 years.", "L. Kornet is one of the best Celtics players.", "Larry Bird was an iconic NBA player.", ]
Create a retriever
retriever = Chroma.from_texts(texts, embedding=embeddings).as_retriever( search_kwargs={"k": 10} ) query = "What can you tell me about the Celtics?"
Get relevant documents ordered by relevance score
docs = retriever.get_relevant_documents(query) docs
[Document(page_content='This is a document about the Boston Celtics'), Document(page_content='The Celtics are my favourite team.'), Document(page_content='L. Kornet is one of the best Celtics players.'), Document(page_content='The Boston Celtics won the game by 20 points'), Document(page_content='Larry Bird was an iconic NBA player.'), Document(page_content='Elden Ring is one of the best games in the last 15 years.'), Document(page_content='Basquetball is a great sport.'), Document(page_content='I simply love going to the movies'), Document(page_content='Fly me to the moon is one of my favourite songs.'), Document(page_content='This is just a random text.')]
Reorder the documents:
Less relevant document will be at the middle of the list and more
relevant elements at beginning / end.
reordering = LongContextReorder() reordered_docs = reordering.transform_documents(docs)
Confirm that the 4 relevant documents are at beginning and end.
reordered_docs
[Document(page_content='The Celtics are my favourite team.'), Document(page_content='The Boston Celtics won the game by 20 points'), Document(page_content='Elden Ring is one of the best games in the last 15 years.'), Document(page_content='I simply love going to the movies'), Document(page_content='This is just a random text.'), Document(page_content='Fly me to the moon is one of my favourite songs.'), Document(page_content='Basquetball is a great sport.'), Document(page_content='Larry Bird was an iconic NBA player.'), Document(page_content='L. Kornet is one of the best Celtics players.'), Document(page_content='This is a document about the Boston Celtics')]
We prepare and run a custom Stuff chain with reordered docs as context.
Override prompts
document_prompt = PromptTemplate( input_variables=["page_content"], template="{page_content}" ) document_variable_name = "context" llm = OpenAI() stuff_prompt_override = """Given this text extracts:
{context}
Please answer the following question: {query}""" prompt = PromptTemplate( template=stuff_prompt_override, input_variables=["context", "query"] )
Instantiate the chain
llm_chain = LLMChain(llm=llm, prompt=prompt) chain = StuffDocumentsChain( llm_chain=llm_chain, document_prompt=document_prompt, document_variable_name=document_variable_name, ) chain.run(input_documents=reordered_docs, query=query)
'\n\nThe Celtics are referenced in four of the nine text extracts. They are mentioned as the favorite team of the author, the winner of a basketball game, a team with one of the best players, and a team with a specific player. Additionally, the last extract states that the document is about the Boston Celtics. This suggests that the Celtics are a basketball team, possibly from Boston, that is well-known and has had successful players and games in the past. '
Help us out by providing feedback on this documentation page: """ to improve my code's output. How can I merge it?
- Assistant
To integrate the concept of "Long-Context Reorder" into your existing code which uses LangChain for conversation, document retrieval, and responses, you need to adapt and insert the new concepts at the appropriate steps. Assuming you want to integrate this into the flow where documents are retrieved and then reordered as per the concept from the referenced arXiv paper before sending them to the LLM (Large Language Model) for generating an answer, the steps would look like something like this:
- After retrieving documents with your retriever (
db.as_retriever(search_kwargs={'k': 10})in your current setup), you'll apply the reordering logic to these documents. - Utilize the
LongContextReorderclass fromlangchain_community.document_transformersto reorder the retrieved documents as described in the concept. - Pass these reordered documents into the LLM (such as LLaMA in your case) for generating the response to the query.
Let's walk through the modifications needed in your existing code to incorporate these steps:
Step 1 & 2: Retrieve and Reorder Documents
from langchain_community.document_transformers import LongContextReorder from langchain_community.embeddings import HuggingFaceEmbeddings from langchain_community.vectorstores import Chroma # Assuming 'pages' are the documents extracted from your PDF or other sources # and you have already instantiated your `embeddings` # Use your retriever as in your existing setup retriever = db.as_retriever(search_kwargs={'k': 10}) # Example query query = "Your Query Here" docs = retriever.get_relevant_documents(query) # Reorder the documents reordering = LongContextReorder() reordered_docs = reordering.transform_documents(docs)Step 3: Pass Reordered Documents to LLM for Generating Response
Modify your existing
ChessBotor similar class to accept and use these reordered documents. You need to integrate it into the place where your retrieval and LLM interaction takes place. This might require you to adjust your existing flow slightly.However, a direct replacement isn't straightforward without altering your
ConversationalRetrievalChainusage or creating a custom interaction model within LangChain that can handle the process of:- Retrieving documents based on a query.
- Reordering these documents for optimal context positioning.
- Passing the reordered context to your LLM for response generation.
Integrating the Custom Flow
Given the structure of LangChain, integrating such a flow might involve more in-depth customization of the library's classes and methods, potentially extending or modifying how
ConversationalRetrievalChainmanages document retrieval, processing, and LLM querying.Here's a high-level outline of what needs to be done:
- After retrieval, reorder the documents as shown above.
- Modify the part of your code where the LLM generates answers to include these reordered docs in its context. This might involve changing how you set up and use
ConversationalRetrievalChainor extendingHuggingFacePipelineto consider the reordered documents' context explicitly.
Since your code integrates various libraries and functionalities (like Gradio for UI, LangChain for NLP processing, transformers for models), a seamless integration of the "Long-Context Reorder" concept requires careful consideration of how data flows between these components, especially regarding document retrieval, reordering, and LLM querying for generating responses.
Implementing this fully requires a deeper dive into your application's logic and possibly some structural changes to accommodate the document reordering process efficiently within your interactive chatbot setup.
- After retrieving documents with your retriever (
- User
Please modify my code like a professional to include this logic
- Assistant
Integrating "Long-Context Reorder" logic into your existing code requires careful restructuring to fit the new step seamlessly. Below, I've modified your existing code to incorporate the reordering of retrieved documents before passing them to the language model (LLaMA, in your case) for generating responses. This adaptation assumes you've already set up your environment with the necessary libraries (
langchain,langchain_community, etc.) and their dependencies.Note: The core logic focuses on document retrieval, reordering, and then using these reordered documents for generating responses. Some specifics about your environment or additional functionality you've implemented might need further adaptation.
from reportlab.lib.pagesizes import letter from reportlab.pdfgen import canvas import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline import gradio as gr from langchain.embeddings import HuggingFaceEmbeddings from langchain.document_loaders import PyPDFLoader from langchain.chains import ConversationalRetrievalChain from langchain.memory import ConversationBufferWindowMemory from langchain.prompts import PromptTemplate from langchain.llms import HuggingFacePipeline from langchain_community.vectorstores import FAISS from langchain_community.document_transformers import LongContextReorder # Assuming other necessary imports and nltk.download() as needed model_id = "meta-llama/Llama-2-7b-chat-hf" tokenizer = AutoTokenizer.from_pretrained(model_id) bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=False) model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config,device_map={"":0}) # Document reordering and retrieval def reorder_documents(documents): """ Reorder documents using the LongContextReorder strategy. """ reordering = LongContextReorder() reordered_docs = reordering.transform_documents(documents) return reordered_docs def retrieve_and_reorder_documents(query, retriever): """ Retrieve documents based on a query and reorder them for better context arrangement. """ docs = retriever.get_relevant_documents(query) reordered_docs = reorder_documents(docs) return reordered_docs # Adjusting ChessBot or similar to incorporate reordered_docs class EnhancedChessBot: def __init__(self, memory, prompt, retriever, llm_pipeline): self.memory = memory self.prompt = prompt self.retriever = retriever self.llm_pipeline = llm_pipeline def answer_query(self, query): """ Retrieve documents, reorder them, and generate an answer to the query. """ reordered_docs = retrieve_and_reorder_documents(query, self.retriever) # Convert reordered docs into text context = "\n".join([doc.page_content for doc in reordered_docs]) # Generate answer with provided context and query response = self.llm_pipeline({"context": context, "question": query})['answer'] return response # Function to create the HuggingFace pipeline def create_llm_pipeline(model, tokenizer): hf_pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens=1024, temperature=0.6) return HuggingFacePipeline(pipeline=hf_pipe) # Initialization and usage # Your retrieval setup; assuming 'db' is correctly set up for document retrieval # Setup the retriever from db retriever = db.as_retriever(search_kwargs={'k': 10}) # Initialize HuggingFace pipeline llm_pipeline = create_llm_pipeline(model, tokenizer) # Initialize your bot with memory, prompt template, retriever, and LLM pipeline memory = ConversationBufferWindowMemory(memory_key="chat_history", k=5, return_messages=True) prompt = PromptTemplate(template="{context}\n\n{question}", input_variables=["context", "question"]) chess_bot = EnhancedChessBot(memory=memory, prompt=prompt, retriever=retriever, llm_pipeline=llm_pipeline) # You can now use chess_bot.answer_query("Your query here") to get answers. # Note: Integration with Gradio UI and additional functionalities would follow the initialization and setup pattern demonstrated above. # Due to space constraints, we're focusing on the core logic of document retrieval, reordering, and LLM-based response generation.Notes:
- This revised version focuses on the core logic: document retrieval, reordering, and passing reordered contexts to generate responses. It simplifies some aspects like Gradio UI interaction for focus and clarity.
- The
EnhancedChessBotclass introduces an encapsulated approach, where retrieval, reordering, and answering are integrated. This allows for more straightforward adaptation and testing. - The code snippet above integrates document reordering with the language model interaction while abstracting over specific details of document and embedding management. You might need to adapt it further based on your specific setup, especially concerning document retrieval (
db) and LLM creation (modelandtokenizersetup). - Ensure all the necessary imports are correctly included at the top of your script, along with any required setup for libraries and models.
Incorporating this logic into your codebase aligns the retrieval and response generation process more closely with the concept of leveraging reordered long contexts for improved language model performance, as discussed in the research you referenced.
- User
Please write the complete code
- Assistant
For the purpose of providing a complete example, I'll streamline the process and focus on integrating the Long-Context Reorder logic with a chatbot interface that generates responses based on retrieved and reordered documents. This example assumes you're familiar with and have access to the libraries mentioned, have set up your models, and are working within an environment where these dependencies are available.
Setting Up the Environment:
First, ensure you've installed all necessary packages. You would typically do this outside of the script, in your environment shell, terminal, or notebook:
pip install transformers reportlab gradio langchain langchain-community torch sentence-transformers # Note: Some libraries or modules may not exist as named, ensure correct packages.Complete Code Example:
This Python script demonstrates how to set up a conversational AI service using Gradio for the frontend interface, Transformers for the model, and includes the Long-Context Reorder logic for document retrieval and reordering.
# First, the necessary imports: import gradio as gr from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline from langchain.document_loaders import PyPDFLoader from langchain.embeddings import HuggingFaceEmbeddings from langchain.chains import ConversationalRetrievalChain from langchain.memory import ConversationBufferWindowMemory from langchain.prompts import PromptTemplate from langchain.llms import HuggingFacePipeline from langchain_community.vectorstores import FAISS from langchain_community.document_transformers import LongContextReorder # Replace 'your-model-name' with actual model names for tokenizer and sequence-to-sequence models tokenizer = AutoTokenizer.from_pretrained('gpt2') # Example with GPT-2 model = AutoModelForSeq2SeqLM.from_pretrained('gpt2') # Creating the Long-Context Reorder Transformer reordering = LongContextReorder() # Document retrieval and embeddings setup # Note: Replace 'sentence-transformers/all-MiniLM-L6-v2' with the actual model for HuggingFaceEmbeddings embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2") # Load and embed documents (for demonstration, manually defined texts) texts = ["Document text 1", "Document text 2", "..."] retriever = FAISS.from_texts(texts, embedding=embeddings).as_retriever(search_kwargs={"k": 10}) # Setup the memory for Conversational Chain memory = ConversationBufferWindowMemory(memory_key="chat_history", k=5, return_messages=True) # Define the prompt template prompt_template = PromptTemplate(template="Given the context: {context}. Answer the following question: {question}", input_variables=["context", "question"]) # Create Gradio Interface def ask_bot(question): # Perform document retrieval docs = retriever.get_relevant_documents(question) # Apply Long-Context Reorder reordered_docs = reordering.transform_documents(docs) # Aggregate reordered document texts aggregated_context = " ".join([doc.page_content for doc in reordered_docs]) # Setup the pipeline for question answering hf_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, framework='pt') llm = HuggingFacePipeline(pipeline=hf_pipeline) # Create LLM conversational chain qa_chain = ConversationalRetrievalChain.from_llm(llm=llm, retriever=retriever, memory=memory, combine_docs_chain_kwargs={"prompt": prompt_template}) # Generate response based on reordered context response = qa_chain.run({"question": question, "context": aggregated_context}) return response # Gradio app iface = gr.Interface(fn=ask_bot, inputs=["text"], outputs=["text"]) iface.launch()Explanation and Modifications:
- This example uses GPT-2 as a placeholder; substitute with the desired model like Meta LLaMA or another suitable for your application. Adjust the model and tokenizer imports accordingly.
- The document retrieval and embedding part (with FAISS and
HuggingFaceEmbeddings) is simplified here. You'll need real texts or documents instead of the placeholder"Document text 1", "Document text 2", "..."to embed and retrieve effectively. - The Long-Context Reorder logic (
reordering) is applied to the documents retrieved based on the query, but you should ensure that your document objects (docs) are compatible with whatLongContextReorderexpects. - The conversational component (
ask_botfunction) integrates the Gradio interface, providing a simple text input for queries and displaying the bot's responses. This function retrieves and reorders documents relevant to the user's query, then processes them using a transformer-based model to generate a response. - The script initializes a Gradio web interface for interactive querying.
Before Running:
- Ensure all necessary models (
HuggingFaceEmbeddings, your choice of language model, etc.) and their respective embeddings are correctly identified and accessible. - This script assumes the presence of "langchain", "langchain-community", and other libraries which might have to be adapted based on actual package names or custom implementations if they are not directly available as imports.
- This script is a conceptual demonstration. Specific functionalities (like PDF document loading, actual conversation memory integration, etc.) may need to be tested and adjusted according to your specific use case and data formats.
- User
modify this code """import numpy as np from langchain.embeddings import HuggingFaceEmbeddings from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import Chroma from langchain.document_loaders import PyPDFLoader from langchain import HuggingFacePipeline from langchain import PromptTemplate from langchain.chains import ConversationalRetrievalChain from langchain.memory import ConversationBufferWindowMemory from sentence_transformers import SentenceTransformer import gradio as gr from langchain_community.vectorstores import FAISS from flashrank.Ranker import Ranker, RerankRequest
#nltk.download() import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline, TextStreamer
def create_multipage_pdf(text, filename='output.pdf', font_size=12, margin=50): # Split text into chunks that fit on one page chunks = [] chunk_size = 50 # Number of characters per chunk for i in range(0, len(text), chunk_size): chunks.append(text[i:i+chunk_size])
# Create PDF c = canvas.Canvas(filename, pagesize=letter) width, height = letter y = height - margin for chunk in chunks: # Draw text text_object = c.beginText(margin, y) text_object.setFont("Helvetica", font_size) text_object.textLines(chunk) c.drawText(text_object) # Update y position y -= (font_size + 4) # Adjust spacing between lines # Check if we need to start a new page if y <= margin: c.showPage() y = height - margin c.save()Example usage
#create_multipage_pdf(text, filename='document.pdf')
bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=False)
model_id = "meta-llama/Llama-2-7b-chat-hf"
tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config = bnb_config,device_map={"":0})
import json import textwrap
B_INST, E_INST = "[INST]", "[/INST]" B_SYS, E_SYS = "<>\n", "\n<>\n\n" DEFAULT_SYSTEM_PROMPT = """
You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."""
def get_prompt(instruction, new_system_prompt=DEFAULT_SYSTEM_PROMPT ): SYSTEM_PROMPT = B_SYS + new_system_prompt + E_SYS prompt_template = B_INST + SYSTEM_PROMPT + instruction + E_INST return prompt_template
loader = PyPDFLoader("document.pdf")
text_splitter = RecursiveCharacterTextSplitter( # Set a really small chunk size, just to show. chunk_size = 500, chunk_overlap = 20, length_function = len, )
pages = loader.load_and_split(text_splitter)
testing with embeddings
embeddings = HuggingFaceEmbeddings(model_name="hkunlp/instructor-xl", model_kwargs={'device': 'cuda'}) #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={'device': 'cuda'}) print("OK1") db = FAISS.from_documents(pages, embedding=embeddings) #db = Chroma.from_documents(pages, embedding=embeddings) print("OK2") instruction = "Given the context that has been provided. \n {context}, Answer the following question - \n{question}"
This part was remove from first line of the system_prompt for testing purposes. Be precise in your answers wherever possible.
system_prompt = """You are an honest virtual assistant. You will be given a context to answer from. In case you are sure you don't know the answer then you say that based on the context you don't know the answer. In all other instances you provide an answer to the best of your capability. Cite context when you can access them maintaining formatting. Don't say 'based on the context provided more than once."""
get_prompt(instruction, system_prompt)
"""## Setting up with LangChain"""
template = get_prompt(instruction, system_prompt) print(template)
prompt = PromptTemplate(template=template, input_variables=["context", "question"])
memory = ConversationBufferWindowMemory( memory_key="chat_history", k=5, return_messages=True )
retriever = db.as_retriever(search_kwargs={'k': 10})
def create_pipeline(max_new_tokens=1024): pipe = pipeline("text-generation", model=model, tokenizer = tokenizer, max_new_tokens = max_new_tokens, temperature = 0.6) return pipe
class ChessBot:
send re_ranked docs instead of retriever for advanced RAG
def init(self, memory, prompt, task:str = "text-generation", retriever = retriever): self.memory = memory self.prompt = prompt self.retriever = retriever
def create_chat_bot(self, max_new_tokens = 1024): hf_pipe = create_pipeline(max_new_tokens) llm = HuggingFacePipeline(pipeline =hf_pipe) qa = ConversationalRetrievalChain.from_llm( llm=llm, retriever=self.retriever, memory=self.memory, combine_docs_chain_kwargs={"prompt": self.prompt} ) return qa
chess_bot = ChessBot(memory = memory, prompt = prompt)
bot = chess_bot.create_chat_bot()
def clear_llm_memory(): bot.memory.clear()
def update_prompt(sys_prompt): if sys_prompt == "": sys_prompt = system_prompt template = get_prompt(instruction, sys_prompt)
prompt = PromptTemplate(template=template, input_variables=["context", "question"])
bot.combine_docs_chain.llm_chain.prompt = prompt
"""1. Not using API 2. Use cases are not defined 3. Just a POC emphasis """
with gr.Blocks() as demo: gr.Markdown( """ # Please ask your questions! """) # Commenting update prompt option #update_sys_prompt = gr.Textbox(label = "Update System Prompt") chatbot = gr.Chatbot(label="SAMSUNG SDS", height = 300) msg = gr.Textbox(label = "Enter your query!") with gr.Column(scale=1): clear = gr.ClearButton([msg, chatbot]) with gr.Column(scale=1): clear_memory = gr.Button(value = "Clear LLM Memory")
def respond(message, chat_history): print("Query:::", message) bot_message = bot({"question": message})['answer'] chat_history.append((message, bot_message)) return "", chat_history msg.submit(respond, inputs=[msg, chatbot], outputs=[msg, chatbot]) clear_memory.click(clear_llm_memory) # Commenting update prompt option #update_sys_prompt.submit(update_prompt, inputs=update_sys_prompt)demo.launch(share=False, server_name='216.48.177.144', server_port=8502)
""" to include the complete longtext reorder technique
- Assistant
To include the Long-Context Reorder technique in the code you've provided requires adding a few necessary steps and modifications. We will incorporate the mechanism of reordering documents after retrieval but before passing them to the conversational chain for response generation. This technique, as proposed, aims to reposition documents so that the most relevant ones are positioned in a manner that helps the model process them more effectively.
Here’s how you can modify the given script to include Long-Context Reorder:
-
Setup for Reordering: First, you need an implementation of the Long-Context Reorder technique. This involves using embeddings for the query and the retrieved documents, computing similarity, and then reordering based on the similarity scores.
-
Modifications in the Bot: The
ChessBotclass logic will include steps to reorder the documents based on relevance to the query before generating the response.
Given those points, here’s how the modified sections of the script would look:
from langchain_community.document_transformers import LongContextReorder class ChessBot: def __init__(self, memory, prompt, task: str = "text-generation", retriever, embeddings): self.memory = memory self.prompt = prompt self.retriever = retriever self.embeddings = embeddings self.reordering = LongContextReorder(embeddings) # Assuming a compatible constructor def reorder_documents(self, documents, query): # Convert query and documents to embeddings query_embedding = self.embeddings.encode(query) doc_embeddings = [self.embeddings.encode(doc) for doc in documents] # Calculate similarity and sort documents (This part is highly simplified) similarities = [np.dot(query_embedding, doc_emb.T) for doc_emb in doc_embeddings] sorted_docs = [doc for _, doc in sorted(zip(similarities, documents), reverse=True)] return sorted_docs def create_chat_bot(self, max_new_tokens=1024): hf_pipe = create_pipeline(max_new_tokens) llm = HuggingFacePipeline(pipeline=hf_pipe) def custom_retrieval_function(query): # Retrieve documents based on the query documents = self.retriever.get_relevant_documents(query) # Reorder the documents to improve context positioning reordered_documents = self.reorder_documents(documents, query) return reordered_documents qa = ConversationalRetrievalChain.from_llm( llm=llm, retriever=custom_retrieval_function, # Pass the custom function memory=self.memory, combine_docs_chain_kwargs={"prompt": self.prompt} ) return qaKey Changes Explained:
- Embeddings Dependency: The
ChessBotinitialization now requiresembeddingsto compute similarities between the query and retrieved documents for reordering. - Reorder Documents: A simplified method
reorder_documentsis added to reorder documents based on their similarity to the query. The actual implementation would need to be more sophisticated, depending on how your documents are structured and how exactly you want to compute similarity (e.g., cosine similarity). - Custom Retrieval Function: In
create_chat_bot, we definecustom_retrieval_functionto integrate our retrieval and reordering logic. This function retrieves documents using the original retriever, reorders them, and returns the reordered list for further processing.
Caveats and Simplifications:
- The reordering logic here is highly simplified. For practical purposes, you might want to use cosine similarity for comparing embeddings, handle edge cases, and optimize the reordering process based on your application's specific needs.
- Error handling, edge cases (like empty document lists), and embedding cache mechanisms are omitted for brevity but are essential for production code.
- Make sure to adapt the reordering logic based on how your documents are represented. The example assumes you can directly compute embeddings for each document and the query.
Remember, implementing Long-Context Reorder effectively depends significantly on the nature of your documents, the embeddings used, and the specifics of how your language model processes context. This example should serve as a starting point that you will likely need to adapt and refine.
-