Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0125-preview· OpenAIAi2 WildChat-4.8M (AI2)

I have developed a code - """import numpy as np from langchain.embeddings import…

  1. User

    I have developed a code - """import numpy as np from langchain.embeddings import HuggingFaceEmbeddings from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import Chroma from langchain.document_loaders import PyPDFLoader from langchain import HuggingFacePipeline from langchain import PromptTemplate from langchain.chains import ConversationalRetrievalChain from langchain.memory import ConversationBufferWindowMemory from sentence_transformers import SentenceTransformer import gradio as gr from langchain_community.vectorstores import FAISS from flashrank.Ranker import Ranker, RerankRequest

    #nltk.download() import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline, TextStreamer

    def create_multipage_pdf(text, filename='output.pdf', font_size=12, margin=50): # Split text into chunks that fit on one page chunks = [] chunk_size = 50 # Number of characters per chunk for i in range(0, len(text), chunk_size): chunks.append(text[i:i+chunk_size])

    # Create PDF
    c = canvas.Canvas(filename, pagesize=letter)
    width, height = letter
    y = height - margin
    for chunk in chunks:
        # Draw text
        text_object = c.beginText(margin, y)
        text_object.setFont("Helvetica", font_size)
        text_object.textLines(chunk)
        c.drawText(text_object)
    
        # Update y position
        y -= (font_size + 4)  # Adjust spacing between lines
    
        # Check if we need to start a new page
        if y <= margin:
            c.showPage()
            y = height - margin
    
    c.save()
    

    Example usage

    #create_multipage_pdf(text, filename='document.pdf')

    bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=False)

    model_id = "meta-llama/Llama-2-7b-chat-hf"

    tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config = bnb_config,device_map={"":0})

    import json import textwrap

    B_INST, E_INST = "[INST]", "[/INST]" B_SYS, E_SYS = "<>\n", "\n<>\n\n" DEFAULT_SYSTEM_PROMPT = """
    You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.

    If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."""

    def get_prompt(instruction, new_system_prompt=DEFAULT_SYSTEM_PROMPT ): SYSTEM_PROMPT = B_SYS + new_system_prompt + E_SYS prompt_template = B_INST + SYSTEM_PROMPT + instruction + E_INST return prompt_template

    loader = PyPDFLoader("document.pdf")

    text_splitter = RecursiveCharacterTextSplitter( # Set a really small chunk size, just to show. chunk_size = 500, chunk_overlap = 20, length_function = len, )

    pages = loader.load_and_split(text_splitter)

    testing with embeddings

    embeddings = HuggingFaceEmbeddings(model_name="hkunlp/instructor-xl", model_kwargs={'device': 'cuda'}) #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={'device': 'cuda'}) print("OK1") db = FAISS.from_documents(pages, embedding=embeddings) #db = Chroma.from_documents(pages, embedding=embeddings) print("OK2") instruction = "Given the context that has been provided. \n {context}, Answer the following question - \n{question}"

    This part was remove from first line of the system_prompt for testing purposes. Be precise in your answers wherever possible.

    system_prompt = """You are an honest virtual assistant. You will be given a context to answer from. In case you are sure you don't know the answer then you say that based on the context you don't know the answer. In all other instances you provide an answer to the best of your capability. Cite context when you can access them maintaining formatting. Don't say 'based on the context provided more than once."""

    get_prompt(instruction, system_prompt)

    """## Setting up with LangChain"""

    template = get_prompt(instruction, system_prompt) print(template)

    prompt = PromptTemplate(template=template, input_variables=["context", "question"])

    memory = ConversationBufferWindowMemory( memory_key="chat_history", k=5, return_messages=True )

    retriever = db.as_retriever(search_kwargs={'k': 10})

    def create_pipeline(max_new_tokens=1024): pipe = pipeline("text-generation", model=model, tokenizer = tokenizer, max_new_tokens = max_new_tokens, temperature = 0.6) return pipe

    class ChessBot:

    send re_ranked docs instead of retriever for advanced RAG

    def init(self, memory, prompt, task:str = "text-generation", retriever = retriever): self.memory = memory self.prompt = prompt self.retriever = retriever

    def create_chat_bot(self, max_new_tokens = 1024): hf_pipe = create_pipeline(max_new_tokens) llm = HuggingFacePipeline(pipeline =hf_pipe) qa = ConversationalRetrievalChain.from_llm( llm=llm, retriever=self.retriever, memory=self.memory, combine_docs_chain_kwargs={"prompt": self.prompt} ) return qa

    chess_bot = ChessBot(memory = memory, prompt = prompt)

    bot = chess_bot.create_chat_bot()

    def clear_llm_memory(): bot.memory.clear()

    def update_prompt(sys_prompt): if sys_prompt == "": sys_prompt = system_prompt template = get_prompt(instruction, sys_prompt)

    prompt = PromptTemplate(template=template, input_variables=["context", "question"])

    bot.combine_docs_chain.llm_chain.prompt = prompt

    """1. Not using API 2. Use cases are not defined 3. Just a POC emphasis """

    with gr.Blocks() as demo: gr.Markdown( """ # Please ask your questions! """) # Commenting update prompt option #update_sys_prompt = gr.Textbox(label = "Update System Prompt") chatbot = gr.Chatbot(label="SAMSUNG SDS", height = 300) msg = gr.Textbox(label = "Enter your query!") with gr.Column(scale=1): clear = gr.ClearButton([msg, chatbot]) with gr.Column(scale=1): clear_memory = gr.Button(value = "Clear LLM Memory")

    def respond(message, chat_history):
        print("Query:::", message)
        bot_message = bot({"question": message})['answer']
        chat_history.append((message, bot_message))
        return "", chat_history
    
    msg.submit(respond, inputs=[msg, chatbot], outputs=[msg, chatbot])
    clear_memory.click(clear_llm_memory)
    # Commenting update prompt option
    #update_sys_prompt.submit(update_prompt, inputs=update_sys_prompt)
    

    demo.launch(share=False, server_name='216.48.177.144', server_port=8502)

    """ Now I want to modify this code to include this concept - """Long-Context Reorder

    No matter the architecture of your model, there is a substantial performance degradation when you include 10+ retrieved documents. In brief: When models must access relevant information in the middle of long contexts, they tend to ignore the provided documents. See: https://arxiv.org/abs/2307.03172

    To avoid this issue you can re-order documents after retrieval to avoid performance degradation.

    %pip install --upgrade --quiet sentence-transformers > /dev/null

    from langchain.chains import LLMChain, StuffDocumentsChain from langchain.prompts import PromptTemplate from langchain_community.document_transformers import ( LongContextReorder, ) from langchain_community.embeddings import HuggingFaceEmbeddings from langchain_community.vectorstores import Chroma from langchain_openai import OpenAI

    Get embeddings.

    embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")

    texts = [ "Basquetball is a great sport.", "Fly me to the moon is one of my favourite songs.", "The Celtics are my favourite team.", "This is a document about the Boston Celtics", "I simply love going to the movies", "The Boston Celtics won the game by 20 points", "This is just a random text.", "Elden Ring is one of the best games in the last 15 years.", "L. Kornet is one of the best Celtics players.", "Larry Bird was an iconic NBA player.", ]

    Create a retriever

    retriever = Chroma.from_texts(texts, embedding=embeddings).as_retriever( search_kwargs={"k": 10} ) query = "What can you tell me about the Celtics?"

    Get relevant documents ordered by relevance score

    docs = retriever.get_relevant_documents(query) docs

    [Document(page_content='This is a document about the Boston Celtics'), Document(page_content='The Celtics are my favourite team.'), Document(page_content='L. Kornet is one of the best Celtics players.'), Document(page_content='The Boston Celtics won the game by 20 points'), Document(page_content='Larry Bird was an iconic NBA player.'), Document(page_content='Elden Ring is one of the best games in the last 15 years.'), Document(page_content='Basquetball is a great sport.'), Document(page_content='I simply love going to the movies'), Document(page_content='Fly me to the moon is one of my favourite songs.'), Document(page_content='This is just a random text.')]

    Reorder the documents:

    Less relevant document will be at the middle of the list and more

    relevant elements at beginning / end.

    reordering = LongContextReorder() reordered_docs = reordering.transform_documents(docs)

    Confirm that the 4 relevant documents are at beginning and end.

    reordered_docs

    [Document(page_content='The Celtics are my favourite team.'), Document(page_content='The Boston Celtics won the game by 20 points'), Document(page_content='Elden Ring is one of the best games in the last 15 years.'), Document(page_content='I simply love going to the movies'), Document(page_content='This is just a random text.'), Document(page_content='Fly me to the moon is one of my favourite songs.'), Document(page_content='Basquetball is a great sport.'), Document(page_content='Larry Bird was an iconic NBA player.'), Document(page_content='L. Kornet is one of the best Celtics players.'), Document(page_content='This is a document about the Boston Celtics')]

    We prepare and run a custom Stuff chain with reordered docs as context.

    Override prompts

    document_prompt = PromptTemplate( input_variables=["page_content"], template="{page_content}" ) document_variable_name = "context" llm = OpenAI() stuff_prompt_override = """Given this text extracts:

    {context}

    Please answer the following question: {query}""" prompt = PromptTemplate( template=stuff_prompt_override, input_variables=["context", "query"] )

    Instantiate the chain

    llm_chain = LLMChain(llm=llm, prompt=prompt) chain = StuffDocumentsChain( llm_chain=llm_chain, document_prompt=document_prompt, document_variable_name=document_variable_name, ) chain.run(input_documents=reordered_docs, query=query)

    '\n\nThe Celtics are referenced in four of the nine text extracts. They are mentioned as the favorite team of the author, the winner of a basketball game, a team with one of the best players, and a team with a specific player. Additionally, the last extract states that the document is about the Boston Celtics. This suggests that the Celtics are a basketball team, possibly from Boston, that is well-known and has had successful players and games in the past. '

    Help us out by providing feedback on this documentation page: """ to improve my code's output. How can I merge it?

  2. Assistant

    To integrate the concept of "Long-Context Reorder" into your existing code which uses LangChain for conversation, document retrieval, and responses, you need to adapt and insert the new concepts at the appropriate steps. Assuming you want to integrate this into the flow where documents are retrieved and then reordered as per the concept from the referenced arXiv paper before sending them to the LLM (Large Language Model) for generating an answer, the steps would look like something like this:

    1. After retrieving documents with your retriever (db.as_retriever(search_kwargs={'k': 10}) in your current setup), you'll apply the reordering logic to these documents.
    2. Utilize the LongContextReorder class from langchain_community.document_transformers to reorder the retrieved documents as described in the concept.
    3. Pass these reordered documents into the LLM (such as LLaMA in your case) for generating the response to the query.

    Let's walk through the modifications needed in your existing code to incorporate these steps:

    Step 1 & 2: Retrieve and Reorder Documents

    from langchain_community.document_transformers import LongContextReorder
    from langchain_community.embeddings import HuggingFaceEmbeddings
    from langchain_community.vectorstores import Chroma
    
    # Assuming 'pages' are the documents extracted from your PDF or other sources
    # and you have already instantiated your `embeddings`
    
    # Use your retriever as in your existing setup
    retriever = db.as_retriever(search_kwargs={'k': 10})
    
    # Example query
    query = "Your Query Here"
    docs = retriever.get_relevant_documents(query)
    
    # Reorder the documents
    reordering = LongContextReorder()
    reordered_docs = reordering.transform_documents(docs)
    

    Step 3: Pass Reordered Documents to LLM for Generating Response

    Modify your existing ChessBot or similar class to accept and use these reordered documents. You need to integrate it into the place where your retrieval and LLM interaction takes place. This might require you to adjust your existing flow slightly.

    However, a direct replacement isn't straightforward without altering your ConversationalRetrievalChain usage or creating a custom interaction model within LangChain that can handle the process of:

    • Retrieving documents based on a query.
    • Reordering these documents for optimal context positioning.
    • Passing the reordered context to your LLM for response generation.

    Integrating the Custom Flow

    Given the structure of LangChain, integrating such a flow might involve more in-depth customization of the library's classes and methods, potentially extending or modifying how ConversationalRetrievalChain manages document retrieval, processing, and LLM querying.

    Here's a high-level outline of what needs to be done:

    • After retrieval, reorder the documents as shown above.
    • Modify the part of your code where the LLM generates answers to include these reordered docs in its context. This might involve changing how you set up and use ConversationalRetrievalChain or extending HuggingFacePipeline to consider the reordered documents' context explicitly.

    Since your code integrates various libraries and functionalities (like Gradio for UI, LangChain for NLP processing, transformers for models), a seamless integration of the "Long-Context Reorder" concept requires careful consideration of how data flows between these components, especially regarding document retrieval, reordering, and LLM querying for generating responses.

    Implementing this fully requires a deeper dive into your application's logic and possibly some structural changes to accommodate the document reordering process efficiently within your interactive chatbot setup.

  3. User

    Please modify my code like a professional to include this logic

  4. Assistant

    Integrating "Long-Context Reorder" logic into your existing code requires careful restructuring to fit the new step seamlessly. Below, I've modified your existing code to incorporate the reordering of retrieved documents before passing them to the language model (LLaMA, in your case) for generating responses. This adaptation assumes you've already set up your environment with the necessary libraries (langchain, langchain_community, etc.) and their dependencies.

    Note: The core logic focuses on document retrieval, reordering, and then using these reordered documents for generating responses. Some specifics about your environment or additional functionality you've implemented might need further adaptation.

    from reportlab.lib.pagesizes import letter
    from reportlab.pdfgen import canvas
    import torch
    from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline
    import gradio as gr
    
    from langchain.embeddings import HuggingFaceEmbeddings
    from langchain.document_loaders import PyPDFLoader
    from langchain.chains import ConversationalRetrievalChain
    from langchain.memory import ConversationBufferWindowMemory
    from langchain.prompts import PromptTemplate
    from langchain.llms import HuggingFacePipeline
    
    from langchain_community.vectorstores import FAISS
    from langchain_community.document_transformers import LongContextReorder
    
    # Assuming other necessary imports and nltk.download() as needed
    
    model_id = "meta-llama/Llama-2-7b-chat-hf"
    tokenizer = AutoTokenizer.from_pretrained(model_id)
    bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=False)
    model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config,device_map={"":0})
    
    # Document reordering and retrieval
    def reorder_documents(documents):
        """
        Reorder documents using the LongContextReorder strategy.
        """
        reordering = LongContextReorder()
        reordered_docs = reordering.transform_documents(documents)
        return reordered_docs
    
    def retrieve_and_reorder_documents(query, retriever):
        """
        Retrieve documents based on a query and reorder them for better context arrangement.
        """
        docs = retriever.get_relevant_documents(query)
        reordered_docs = reorder_documents(docs)
        return reordered_docs
    
    # Adjusting ChessBot or similar to incorporate reordered_docs
    class EnhancedChessBot:
        def __init__(self, memory, prompt, retriever, llm_pipeline):
            self.memory = memory
            self.prompt = prompt
            self.retriever = retriever
            self.llm_pipeline = llm_pipeline
        
        def answer_query(self, query):
            """
            Retrieve documents, reorder them, and generate an answer to the query.
            """
            reordered_docs = retrieve_and_reorder_documents(query, self.retriever)
            # Convert reordered docs into text
            context = "\n".join([doc.page_content for doc in reordered_docs])
            # Generate answer with provided context and query
            response = self.llm_pipeline({"context": context, "question": query})['answer']
            return response
    
    # Function to create the HuggingFace pipeline
    def create_llm_pipeline(model, tokenizer):
        hf_pipe = pipeline("text-generation",
                           model=model,
                           tokenizer=tokenizer,
                           max_new_tokens=1024,
                           temperature=0.6)
        return HuggingFacePipeline(pipeline=hf_pipe)
    
    # Initialization and usage
    # Your retrieval setup; assuming 'db' is correctly set up for document retrieval
    # Setup the retriever from db
    retriever = db.as_retriever(search_kwargs={'k': 10})
    
    # Initialize HuggingFace pipeline
    llm_pipeline = create_llm_pipeline(model, tokenizer)
    
    # Initialize your bot with memory, prompt template, retriever, and LLM pipeline
    memory = ConversationBufferWindowMemory(memory_key="chat_history", k=5, return_messages=True)
    prompt = PromptTemplate(template="{context}\n\n{question}", input_variables=["context", "question"])
    chess_bot = EnhancedChessBot(memory=memory, prompt=prompt, retriever=retriever, llm_pipeline=llm_pipeline)
    
    # You can now use chess_bot.answer_query("Your query here") to get answers.
    
    # Note: Integration with Gradio UI and additional functionalities would follow the initialization and setup pattern demonstrated above.
    # Due to space constraints, we're focusing on the core logic of document retrieval, reordering, and LLM-based response generation.
    

    Notes:

    • This revised version focuses on the core logic: document retrieval, reordering, and passing reordered contexts to generate responses. It simplifies some aspects like Gradio UI interaction for focus and clarity.
    • The EnhancedChessBot class introduces an encapsulated approach, where retrieval, reordering, and answering are integrated. This allows for more straightforward adaptation and testing.
    • The code snippet above integrates document reordering with the language model interaction while abstracting over specific details of document and embedding management. You might need to adapt it further based on your specific setup, especially concerning document retrieval (db) and LLM creation (model and tokenizer setup).
    • Ensure all the necessary imports are correctly included at the top of your script, along with any required setup for libraries and models.

    Incorporating this logic into your codebase aligns the retrieval and response generation process more closely with the concept of leveraging reordered long contexts for improved language model performance, as discussed in the research you referenced.

  5. User

    Please write the complete code

  6. Assistant

    For the purpose of providing a complete example, I'll streamline the process and focus on integrating the Long-Context Reorder logic with a chatbot interface that generates responses based on retrieved and reordered documents. This example assumes you're familiar with and have access to the libraries mentioned, have set up your models, and are working within an environment where these dependencies are available.

    Setting Up the Environment:

    First, ensure you've installed all necessary packages. You would typically do this outside of the script, in your environment shell, terminal, or notebook:

    pip install transformers reportlab gradio langchain langchain-community torch sentence-transformers
    # Note: Some libraries or modules may not exist as named, ensure correct packages.
    

    Complete Code Example:

    This Python script demonstrates how to set up a conversational AI service using Gradio for the frontend interface, Transformers for the model, and includes the Long-Context Reorder logic for document retrieval and reordering.

    # First, the necessary imports:
    import gradio as gr
    from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline
    from langchain.document_loaders import PyPDFLoader
    from langchain.embeddings import HuggingFaceEmbeddings
    from langchain.chains import ConversationalRetrievalChain
    from langchain.memory import ConversationBufferWindowMemory
    from langchain.prompts import PromptTemplate
    from langchain.llms import HuggingFacePipeline
    from langchain_community.vectorstores import FAISS
    from langchain_community.document_transformers import LongContextReorder
    
    # Replace 'your-model-name' with actual model names for tokenizer and sequence-to-sequence models
    tokenizer = AutoTokenizer.from_pretrained('gpt2')  # Example with GPT-2
    model = AutoModelForSeq2SeqLM.from_pretrained('gpt2')
    
    # Creating the Long-Context Reorder Transformer
    reordering = LongContextReorder()
    
    # Document retrieval and embeddings setup
    # Note: Replace 'sentence-transformers/all-MiniLM-L6-v2' with the actual model for HuggingFaceEmbeddings
    embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
    
    # Load and embed documents (for demonstration, manually defined texts)
    texts = ["Document text 1", "Document text 2", "..."]
    retriever = FAISS.from_texts(texts, embedding=embeddings).as_retriever(search_kwargs={"k": 10})
    
    # Setup the memory for Conversational Chain
    memory = ConversationBufferWindowMemory(memory_key="chat_history", k=5, return_messages=True)
    
    # Define the prompt template
    prompt_template = PromptTemplate(template="Given the context: {context}. Answer the following question: {question}", input_variables=["context", "question"])
    
    # Create Gradio Interface
    def ask_bot(question):
        # Perform document retrieval
        docs = retriever.get_relevant_documents(question)
        # Apply Long-Context Reorder
        reordered_docs = reordering.transform_documents(docs)
        # Aggregate reordered document texts
        aggregated_context = " ".join([doc.page_content for doc in reordered_docs])
        # Setup the pipeline for question answering
        hf_pipeline = pipeline("text-generation", model=model, tokenizer=tokenizer, framework='pt')
        llm = HuggingFacePipeline(pipeline=hf_pipeline)
        # Create LLM conversational chain
        qa_chain = ConversationalRetrievalChain.from_llm(llm=llm, retriever=retriever, memory=memory, combine_docs_chain_kwargs={"prompt": prompt_template})
        # Generate response based on reordered context
        response = qa_chain.run({"question": question, "context": aggregated_context})
        return response
    
    # Gradio app
    iface = gr.Interface(fn=ask_bot, inputs=["text"], outputs=["text"])
    iface.launch()
    

    Explanation and Modifications:

    • This example uses GPT-2 as a placeholder; substitute with the desired model like Meta LLaMA or another suitable for your application. Adjust the model and tokenizer imports accordingly.
    • The document retrieval and embedding part (with FAISS and HuggingFaceEmbeddings) is simplified here. You'll need real texts or documents instead of the placeholder "Document text 1", "Document text 2", "..." to embed and retrieve effectively.
    • The Long-Context Reorder logic (reordering) is applied to the documents retrieved based on the query, but you should ensure that your document objects (docs) are compatible with what LongContextReorder expects.
    • The conversational component (ask_bot function) integrates the Gradio interface, providing a simple text input for queries and displaying the bot's responses. This function retrieves and reorders documents relevant to the user's query, then processes them using a transformer-based model to generate a response.
    • The script initializes a Gradio web interface for interactive querying.

    Before Running:

    • Ensure all necessary models (HuggingFaceEmbeddings, your choice of language model, etc.) and their respective embeddings are correctly identified and accessible.
    • This script assumes the presence of "langchain", "langchain-community", and other libraries which might have to be adapted based on actual package names or custom implementations if they are not directly available as imports.
    • This script is a conceptual demonstration. Specific functionalities (like PDF document loading, actual conversation memory integration, etc.) may need to be tested and adjusted according to your specific use case and data formats.
  7. User

    modify this code """import numpy as np from langchain.embeddings import HuggingFaceEmbeddings from langchain.text_splitter import RecursiveCharacterTextSplitter from langchain.vectorstores import Chroma from langchain.document_loaders import PyPDFLoader from langchain import HuggingFacePipeline from langchain import PromptTemplate from langchain.chains import ConversationalRetrievalChain from langchain.memory import ConversationBufferWindowMemory from sentence_transformers import SentenceTransformer import gradio as gr from langchain_community.vectorstores import FAISS from flashrank.Ranker import Ranker, RerankRequest

    #nltk.download() import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline, TextStreamer

    def create_multipage_pdf(text, filename='output.pdf', font_size=12, margin=50): # Split text into chunks that fit on one page chunks = [] chunk_size = 50 # Number of characters per chunk for i in range(0, len(text), chunk_size): chunks.append(text[i:i+chunk_size])

    # Create PDF
    c = canvas.Canvas(filename, pagesize=letter)
    width, height = letter
    y = height - margin
    for chunk in chunks:
        # Draw text
        text_object = c.beginText(margin, y)
        text_object.setFont("Helvetica", font_size)
        text_object.textLines(chunk)
        c.drawText(text_object)
    
        # Update y position
        y -= (font_size + 4)  # Adjust spacing between lines
    
        # Check if we need to start a new page
        if y <= margin:
            c.showPage()
            y = height - margin
    
    c.save()
    

    Example usage

    #create_multipage_pdf(text, filename='document.pdf')

    bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=False)

    model_id = "meta-llama/Llama-2-7b-chat-hf"

    tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config = bnb_config,device_map={"":0})

    import json import textwrap

    B_INST, E_INST = "[INST]", "[/INST]" B_SYS, E_SYS = "<>\n", "\n<>\n\n" DEFAULT_SYSTEM_PROMPT = """
    You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.

    If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."""

    def get_prompt(instruction, new_system_prompt=DEFAULT_SYSTEM_PROMPT ): SYSTEM_PROMPT = B_SYS + new_system_prompt + E_SYS prompt_template = B_INST + SYSTEM_PROMPT + instruction + E_INST return prompt_template

    loader = PyPDFLoader("document.pdf")

    text_splitter = RecursiveCharacterTextSplitter( # Set a really small chunk size, just to show. chunk_size = 500, chunk_overlap = 20, length_function = len, )

    pages = loader.load_and_split(text_splitter)

    testing with embeddings

    embeddings = HuggingFaceEmbeddings(model_name="hkunlp/instructor-xl", model_kwargs={'device': 'cuda'}) #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={'device': 'cuda'}) print("OK1") db = FAISS.from_documents(pages, embedding=embeddings) #db = Chroma.from_documents(pages, embedding=embeddings) print("OK2") instruction = "Given the context that has been provided. \n {context}, Answer the following question - \n{question}"

    This part was remove from first line of the system_prompt for testing purposes. Be precise in your answers wherever possible.

    system_prompt = """You are an honest virtual assistant. You will be given a context to answer from. In case you are sure you don't know the answer then you say that based on the context you don't know the answer. In all other instances you provide an answer to the best of your capability. Cite context when you can access them maintaining formatting. Don't say 'based on the context provided more than once."""

    get_prompt(instruction, system_prompt)

    """## Setting up with LangChain"""

    template = get_prompt(instruction, system_prompt) print(template)

    prompt = PromptTemplate(template=template, input_variables=["context", "question"])

    memory = ConversationBufferWindowMemory( memory_key="chat_history", k=5, return_messages=True )

    retriever = db.as_retriever(search_kwargs={'k': 10})

    def create_pipeline(max_new_tokens=1024): pipe = pipeline("text-generation", model=model, tokenizer = tokenizer, max_new_tokens = max_new_tokens, temperature = 0.6) return pipe

    class ChessBot:

    send re_ranked docs instead of retriever for advanced RAG

    def init(self, memory, prompt, task:str = "text-generation", retriever = retriever): self.memory = memory self.prompt = prompt self.retriever = retriever

    def create_chat_bot(self, max_new_tokens = 1024): hf_pipe = create_pipeline(max_new_tokens) llm = HuggingFacePipeline(pipeline =hf_pipe) qa = ConversationalRetrievalChain.from_llm( llm=llm, retriever=self.retriever, memory=self.memory, combine_docs_chain_kwargs={"prompt": self.prompt} ) return qa

    chess_bot = ChessBot(memory = memory, prompt = prompt)

    bot = chess_bot.create_chat_bot()

    def clear_llm_memory(): bot.memory.clear()

    def update_prompt(sys_prompt): if sys_prompt == "": sys_prompt = system_prompt template = get_prompt(instruction, sys_prompt)

    prompt = PromptTemplate(template=template, input_variables=["context", "question"])

    bot.combine_docs_chain.llm_chain.prompt = prompt

    """1. Not using API 2. Use cases are not defined 3. Just a POC emphasis """

    with gr.Blocks() as demo: gr.Markdown( """ # Please ask your questions! """) # Commenting update prompt option #update_sys_prompt = gr.Textbox(label = "Update System Prompt") chatbot = gr.Chatbot(label="SAMSUNG SDS", height = 300) msg = gr.Textbox(label = "Enter your query!") with gr.Column(scale=1): clear = gr.ClearButton([msg, chatbot]) with gr.Column(scale=1): clear_memory = gr.Button(value = "Clear LLM Memory")

    def respond(message, chat_history):
        print("Query:::", message)
        bot_message = bot({"question": message})['answer']
        chat_history.append((message, bot_message))
        return "", chat_history
    
    msg.submit(respond, inputs=[msg, chatbot], outputs=[msg, chatbot])
    clear_memory.click(clear_llm_memory)
    # Commenting update prompt option
    #update_sys_prompt.submit(update_prompt, inputs=update_sys_prompt)
    

    demo.launch(share=False, server_name='216.48.177.144', server_port=8502)

    """ to include the complete longtext reorder technique

  8. Assistant

    To include the Long-Context Reorder technique in the code you've provided requires adding a few necessary steps and modifications. We will incorporate the mechanism of reordering documents after retrieval but before passing them to the conversational chain for response generation. This technique, as proposed, aims to reposition documents so that the most relevant ones are positioned in a manner that helps the model process them more effectively.

    Here’s how you can modify the given script to include Long-Context Reorder:

    1. Setup for Reordering: First, you need an implementation of the Long-Context Reorder technique. This involves using embeddings for the query and the retrieved documents, computing similarity, and then reordering based on the similarity scores.

    2. Modifications in the Bot: The ChessBot class logic will include steps to reorder the documents based on relevance to the query before generating the response.

    Given those points, here’s how the modified sections of the script would look:

    from langchain_community.document_transformers import LongContextReorder
    
    class ChessBot:
        def __init__(self, memory, prompt, task: str = "text-generation", retriever, embeddings):
            self.memory = memory
            self.prompt = prompt
            self.retriever = retriever
            self.embeddings = embeddings
            self.reordering = LongContextReorder(embeddings)  # Assuming a compatible constructor
    
        def reorder_documents(self, documents, query):
            # Convert query and documents to embeddings
            query_embedding = self.embeddings.encode(query)
            doc_embeddings = [self.embeddings.encode(doc) for doc in documents]
            
            # Calculate similarity and sort documents (This part is highly simplified)
            similarities = [np.dot(query_embedding, doc_emb.T) for doc_emb in doc_embeddings]
            sorted_docs = [doc for _, doc in sorted(zip(similarities, documents), reverse=True)]
            
            return sorted_docs
    
        def create_chat_bot(self, max_new_tokens=1024):
            hf_pipe = create_pipeline(max_new_tokens)
            llm = HuggingFacePipeline(pipeline=hf_pipe)
            
            def custom_retrieval_function(query):
                # Retrieve documents based on the query
                documents = self.retriever.get_relevant_documents(query)
                # Reorder the documents to improve context positioning
                reordered_documents = self.reorder_documents(documents, query)
                return reordered_documents
    
            qa = ConversationalRetrievalChain.from_llm(
                llm=llm,
                retriever=custom_retrieval_function,  # Pass the custom function
                memory=self.memory,
                combine_docs_chain_kwargs={"prompt": self.prompt}
            )
            return qa
    

    Key Changes Explained:

    • Embeddings Dependency: The ChessBot initialization now requires embeddings to compute similarities between the query and retrieved documents for reordering.
    • Reorder Documents: A simplified method reorder_documents is added to reorder documents based on their similarity to the query. The actual implementation would need to be more sophisticated, depending on how your documents are structured and how exactly you want to compute similarity (e.g., cosine similarity).
    • Custom Retrieval Function: In create_chat_bot, we define custom_retrieval_function to integrate our retrieval and reordering logic. This function retrieves documents using the original retriever, reorders them, and returns the reordered list for further processing.

    Caveats and Simplifications:

    • The reordering logic here is highly simplified. For practical purposes, you might want to use cosine similarity for comparing embeddings, handle edge cases, and optimize the reordering process based on your application's specific needs.
    • Error handling, edge cases (like empty document lists), and embedding cache mechanisms are omitted for brevity but are essential for production code.
    • Make sure to adapt the reordering logic based on how your documents are represented. The example assumes you can directly compute embeddings for each document and the query.

    Remember, implementing Long-Context Reorder effectively depends significantly on the nature of your documents, the embeddings used, and the specifics of how your language model processes context. This example should serve as a starting point that you will likely need to adapt and refine.