import logging import numpy as np from pathlib import Path from typing import Optional…
- User
import logging import numpy as np from pathlib import Path from typing import Optional from sentence_transformers import SentenceTransformer from transformers import AutoTokenizer, pipeline import tabula import pytesseract from pdf2image import convert_from_path import re import faiss import pickle from tqdm import tqdm import torch
Correct imports for your version of llama_index
from llama_index.core import StorageContext, Document from llama_index.core.storage.docstore import SimpleDocumentStore from llama_index.core.vector_stores import SimpleVectorStore
Configure logging
logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s - %(levelname)s - %(message)s', handlers=[ logging.StreamHandler() ] )
Initialize Sentence-BERT model and tokenizer
model_name = 'all-MiniLM-L6-v2' logging.info(f"Loading SentenceTransformer model: {model_name}") sentence_model = SentenceTransformer(model_name) tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/all-MiniLM-L6-v2")
Define directories
PDFS_DIRECTORY = Path('D:/Contracts') # Replace with your actual path STORAGE_PERSIST_DIR = Path("D:/Storage_test4") # Replace with your actual path FAISS_INDEX_PATH = STORAGE_PERSIST_DIR / "faiss_index.bin" EMBEDDINGS_FILE = STORAGE_PERSIST_DIR / "document_embeddings.pkl" DOC_METADATA_FILE = STORAGE_PERSIST_DIR / "document_metadata.pkl"
Ensure the storage directory exists
STORAGE_PERSIST_DIR.mkdir(parents=True, exist_ok=True)
Initialize variables to store embeddings and document metadata
all_embeddings = [] all_metadata = []
Initialize summarizer (BART model)
logging.info("Initializing BART summarizer") summarizer = pipeline( "summarization", model="facebook/bart-large-cnn", tokenizer="facebook/bart-large-cnn", device=0 if torch.cuda.is_available() else -1 )
Define the embedding dimension
dimension = sentence_model.get_sentence_embedding_dimension()
Summarize text using BART summarizer with detailed debugging
def summarize_text(text, max_length=200, min_length=30): if len(text) < min_length: logging.warning("Text too short for summarization.") return "" try: if not text.strip(): logging.warning("Text is empty or whitespace-only, skipping summarization.") return ""
# Set explicit maximum input length max_input_length = 1024 # Maximum tokens for BART # Tokenize the text tokens = summarizer.tokenizer.encode(text, truncation=False) total_tokens = len(tokens) if total_tokens > max_input_length: logging.debug(f"Text too long ({total_tokens} tokens), splitting into chunks.") # Split the tokens into manageable chunks chunks = [] start_idx = 0 while start_idx < total_tokens: end_idx = min(start_idx + max_input_length, total_tokens) chunk_tokens = tokens[start_idx:end_idx] chunk_text = summarizer.tokenizer.decode(chunk_tokens, skip_special_tokens=True) chunks.append(chunk_text) start_idx = end_idx # Summarize each chunk and combine the summaries summaries = [] for chunk in chunks: summary = summarizer( chunk, max_length=max_length, min_length=min_length, do_sample=False )[0]['summary_text'] summaries.append(summary) combined_summary = ' '.join(summaries) logging.info("Generated combined summary for text.") return combined_summary else: # Text is within the acceptable length summary = summarizer( text, max_length=max_length, min_length=min_length, do_sample=False )[0]['summary_text'] logging.info("Generated summary for text.") return summary except Exception as e: logging.error(f"Error generating summary: {e}") return ""... Rest of your code remains the same ...
Remember to update any location where summarization is called, which should already be compatible.
if name == "main": main()
2024-09-27 15:17:06,913 - INFO - Loading SentenceTransformer model: all-MiniLM-L6-v2 2024-09-27 15:17:06,924 - INFO - Use pytorch device_name: cpu 2024-09-27 15:17:06,925 - INFO - Load pretrained SentenceTransformer: all-MiniLM-L6-v2 C:\Users\FourieH\AppData\Local\anaconda3\Lib\site-packages\transformers\tokenization_utils_base.py:1601: FutureWarning:
clean_up_tokenization_spaceswas not set. It will be set toTrueby default. This behavior will be depracted in transformers v4.45, and will be then set toFalseby default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884 warnings.warn( 2024-09-27 15:17:10,214 - INFO - Initializing BART summarizer 2024-09-27 15:17:16,982 - INFO - Found 2 client directories to process. Processing Clients: 0%| | 0/2 [00:00<?, ?it/s]2024-09-27 15:17:16,988 - INFO - Processing client: Absa 2024-09-27 15:17:16,992 - INFO - Processing PDF: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf 2024-09-27 15:17:16,992 - INFO - Converting PDF to images: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf 2024-09-27 15:17:23,201 - INFO - Extracting text from page 1 2024-09-27 15:17:24,714 - INFO - Extracted text from image. 2024-09-27 15:17:24,714 - INFO - Extracting text from page 2 2024-09-27 15:17:29,763 - INFO - Extracted text from image. 2024-09-27 15:17:29,778 - INFO - Extracting text from page 3 2024-09-27 15:17:35,199 - INFO - Extracted text from image. 2024-09-27 15:17:35,199 - INFO - Extracting text from page 4 2024-09-27 15:17:40,886 - INFO - Extracted text from image. 2024-09-27 15:17:40,886 - INFO - Extracting text from page 5 2024-09-27 15:17:46,280 - INFO - Extracted text from image. 2024-09-27 15:17:46,280 - INFO - Extracting text from page 6 2024-09-27 15:17:48,403 - INFO - Extracted text from image. 2024-09-27 15:17:48,419 - INFO - Extracted text from PDF: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf 2024-09-27 15:17:48,435 - INFO - Extracting tables from PDF: D:\Contracts\Absa\ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0.pdf 2024-09-27 15:17:49,793 - WARNING - No tables found. Token indices sequence length is longer than the specified maximum sequence length for this model (5644 > 512). Running this sequence through the model will result in indexing errors 2024-09-27 15:17:51,074 - INFO - Created 13 text chunks with overlapping. Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget Error displaying widget 2024-09-27 15:17:52,618 - INFO - Generating document-level summary for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0 2024-09-27 15:17:52,761 - ERROR - Error generating summary: index out of range in self 2024-09-27 15:17:52,764 - INFO - Document-level summary generated successfully for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0 2024-09-27 15:17:52,764 - INFO - Generating full document summary for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0 2024-09-27 15:17:52,957 - ERROR - Error generating summary: index out of range in self 2024-09-27 15:17:52,957 - INFO - Full document summary generated successfully for: ABSA 37 OCCUPATIONAL HEALTH AND SAFETY AGREEMENT MOBILISATION_v3.0 2024-09-27 15:17:52,957 - WARNING - Received empty text for embedding. 2024-09-27 15:17:52,957 - WARNING - Received empty text for embedding. 2024-09-27 15:17:53,168 - INFO - Created 13 text chunks with overlapping. 2024-09-27 15:17:53,175 - INFO - Number of text chunks: 13 2024-09-27 15:17:53,191 - INFO - Number of tables: 0 2024-09-27 15:17:53,191 - INFO - Document ABSA 37 OCCUPAT - Assistant
From the logs you provided, it appears you're encountering an error during the summarization step of your script:
2024-09-27 15:17:52,761 - ERROR - Error generating summary: index out of range in self ... Token indices sequence length is longer than the specified maximum sequence length for this model (5644 > 512). Running this sequence through the model will result in indexing errorsThis error suggests that the input text you're providing to the BART summarizer is exceeding the maximum allowed sequence length for the model, which is causing an "index out of range" error internally within the model.
Let's break down the problem and provide a solution.
Problem Analysis
-
Tokenization Warning:
Token indices sequence length is longer than the specified maximum sequence length for this model (5644 > 512). Running this sequence through the model will result in indexing errorsThis warning indicates that the input text, when tokenized, exceeds the model's maximum sequence length.
-
Error During Summarization:
2024-09-27 15:17:52,761 - ERROR - Error generating summary: index out of range in selfThis error arises because the model cannot process input sequences longer than its maximum allowed length, leading to an internal indexing error.
-
Text Splitting Logic:
In your
summarize_textfunction, you attempt to handle long texts by splitting tokens into chunks:if total_tokens > max_input_length: # Split the tokens into manageable chunks chunks = [] start_idx = 0 while start_idx < total_tokens: end_idx = min(start_idx + max_input_length, total_tokens) chunk_tokens = tokens[start_idx:end_idx] chunk_text = summarizer.tokenizer.decode(chunk_tokens, skip_special_tokens=True) chunks.append(chunk_text) start_idx = end_idxHowever, the way you're splitting the tokens might not align with how the model expects input data, especially regarding special tokens and attention masks.
Proposed Solution
To resolve this issue, we need to ensure that:
-
Input Sequences Do Not Exceed Maximum Length:
We have to make sure that the input sequences fed into the model are within the maximum sequence length supported.
-
Proper Handling of Tokenization and Decoding:
When splitting the text, we need to respect the model's tokenization process, including special tokens.
-
Adjusting the Summarization Function:
Modify the
summarize_textfunction to correctly handle long texts and avoid the indexing error.
Step-by-Step Fix
-
Update the
summarize_textFunction:We'll modify the
summarize_textfunction to:- Use sentence splitting to break down the text into smaller chunks based on sentences.
- Ensure each chunk is within the maximum allowed length for the model.
- Aggregate summaries appropriately.
-
Install Necessary Libraries:
To split text into sentences, we'll use
nltklibrary.pip install nltk -
Initialize NLTK:
We'll need to download the required NLTK data:
import nltk nltk.download('punkt') -
Rewriting
summarize_textFunction:import nltk def summarize_text(text, max_length=200, min_length=30): if not text.strip(): logging.warning("Text is empty or whitespace-only, skipping summarization.") return "" # Tokenization parameters max_input_length = 1024 # Maximum tokens for BART try: # Split text into sentences sentences = nltk.sent_tokenize(text) chunks = [] chunk = "" for sentence in sentences: # Check if adding the next sentence exceeds the max length chunk_tokens = summarizer.tokenizer.encode(chunk + ' ' + sentence, add_special_tokens=False) if len(chunk_tokens) > max_input_length: if chunk: chunks.append(chunk.strip()) chunk = sentence else: # Sentence itself is longer than max_input_length, so we split the sentence words = sentence.split() sub_chunk = "" for word in words: sub_chunk_tokens = summarizer.tokenizer.encode(sub_chunk + ' ' + word, add_special_tokens=False) if len(sub_chunk_tokens) > max_input_length: chunks.append(sub_chunk.strip()) sub_chunk = word else: sub_chunk += ' ' + word if sub_chunk: chunks.append(sub_chunk.strip()) chunk = "" else: chunk += ' ' + sentence if chunk: chunks.append(chunk.strip()) # Summarize each chunk summaries = [] for chunk in chunks: summary = summarizer( chunk, max_length=max_length, min_length=min_length, do_sample=False )[0]['summary_text'] summaries.append(summary) combined_summary = ' '.join(summaries) logging.info("Generated combined summary for text.") return combined_summary except Exception as e: logging.error(f"Error generating summary: {e}") return "" -
Explanation of Changes:
-
Sentence Splitting:
We use
nltk.sent_tokenizeto split the text into sentences. This helps in preserving the meaning when we split the text. -
Chunking Based on Token Length:
We accumulate sentences into a
chunkuntil adding another sentence would exceed the maximum token length (max_input_length). If it does, we start a new chunk. -
Handling Long Sentences:
If a single sentence exceeds
max_input_length, we split it into words and create sub-chunks that fit within the limit. -
Summarizing Chunks:
Each
chunkis passed to the summarizer individually, ensuring that none exceed the model's maximum sequence length.
-
-
Test the Updated Function:
Run your script again and monitor the logs for errors. The warnings and errors related to sequence length should no longer appear.
Additional Recommendations
-
Adjust Max Input Length Based on Model:
The BART model (
facebook/bart-large-cnn) has a maximum sequence length of 1024 tokens. Ensure thatmax_input_lengthmatches the model you're using. -
Reduce Max Length for Summaries:
If you're still encountering issues, consider reducing the
max_lengthparameter when calling the summarizer.summary = summarizer( chunk, max_length=min(max_length, int(0.2 * len(chunk.split()))), # Ensure summary is not longer than 20% of chunk min_length=min_length, do_sample=False )[0]['summary_text'] -
Monitor GPU/CPU Memory Usage:
Long texts can consume significant memory. Ensure that your system has enough resources, or process smaller batches.
-
Upgrade Transformers Library:
Ensure you're using an updated version of the
transformerslibrary, as some issues might have been fixed in newer releases.pip install --upgrade transformers -
Handle Special Tokens Properly:
When encoding and decoding, use
add_special_tokens=Trueif necessary. This ensures that the model's special tokens (e.g.,[CLS],[SEP], etc.) are used correctly.
Conclusion
The "index out of range" error was caused by input sequences exceeding the maximum length that the BART model can handle. By modifying the
summarize_textfunction to split the text into appropriately sized chunks, we ensure that each piece of text fed into the model is within its capacity, thus preventing the error.Remember to test the updated script thoroughly to confirm that the issue is resolved and that summaries are generated as expected.
Let me know if you have any questions or need further assistance!
-