only take out these functinality and make a separate script, drawing recatngle using…
- User
only take out these functinality and make a separate script, drawing recatngle using mouse, screenshot being taken, sent to ocr, extraced text and than that extracted text should be sent to clipboard : import re import mss from PIL import Image, ImageEnhance import pytesseract import speech_recognition as sr import pyttsx3 import keyboard import threading import os import logging import tkinter as tk import ctypes import sys import ollama import asyncio from datetime import datetime from word2number import w2n from langchain.vectorstores import FAISS from langchain_huggingface import HuggingFaceEmbeddings from sentence_transformers import SentenceTransformer
-------------------- Configuration --------------------
Define a custom logging filter to exclude specific messages
class ExcludeFilter(logging.Filter): def init(self, exclude_keywords): super().init() self.exclude_keywords = exclude_keywords
def filter(self, record): return not any(keyword in record.getMessage() for keyword in self.exclude_keywords)Configure logging
logger = logging.getLogger() logger.setLevel(logging.INFO) # Set to INFO to include INFO and higher level logs
Create a stream handler
stream_handler = logging.StreamHandler() stream_handler.setLevel(logging.INFO)
Define the log format
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s') stream_handler.setFormatter(formatter)
Define keywords to exclude from logging
excluded_keywords = [ "Mouse moved to", "Width:", "Height:", "Monitor for capture:" ]
Add the custom filter to the handler
stream_handler.addFilter(ExcludeFilter(excluded_keywords))
Add the handler to the logger
if not logger.handlers: logger.addHandler(stream_handler) else: # Remove existing handlers to prevent duplicate logs logger.handlers = [] logger.addHandler(stream_handler)
Initialize TTS engine
tts_engine = pyttsx3.init() is_listening = False ocr_text = ""
lock = threading.Lock() tts_lock = threading.Lock()
OUTPUT_DIR = r"D:\AI\EXP2\TempS"
Initialize RAG components
embedding_model_name = "all-mpnet-base-v2" # Use a more robust model embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) vector_store = FAISS.load_local(r"D:\AI\EXP2\faiss_index", embeddings,allow_dangerous_deserialization=True)
def set_dpi_awareness(): try: if sys.platform.startswith('win'): ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
def get_scaling_factor(): try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0
def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text
def strip_markdown(text): """Function to remove Markdown syntax from text.""" # Remove asterisks, underscores, and backticks text = re.sub(r'[*_`]', '', text) # Remove links text text = re.sub(r'[([^]]+)]([^)]+)', r'\1', text) # Remove headings ## Heading text = re.sub(r'#+\s?', '', text) # Add more patterns as necessary return text
def speak_text(text): global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: tts_engine.say(text)
# Add hotkeys for pausing (F4) and stopping (F5) keyboard.add_hotkey('f4', pause_speaking) keyboard.add_hotkey('f5', stop_speaking) tts_engine.runAndWait() logging.info("Text-to-Speech completed.") except RuntimeError as e: logging.error(f"TTS Error: {e}") finally: keyboard.remove_hotkey('f4') keyboard.remove_hotkey('f5')def pause_speaking(): global tts_engine logging.info("Paused speaking...") # pyttsx3 does not support pausing natively. Placeholder.
def stop_speaking(): global tts_engine logging.info("Stopped speaking.") tts_engine.stop()
def save_captured_image(image, output_dir, prefix="captured_region"): try: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png") image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")
def save_preprocessed_image(image, output_dir, prefix="preprocessed_image"): try: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png") image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")
def perform_ocr(image): global ocr_text try: logging.info("Starting OCR process with image preprocessing...")
save_captured_image(image, OUTPUT_DIR) gray = image.convert('L') # Removed DEBUG logs related to image processing enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(1.0) # Removed DEBUG logs related to image processing threshold = 190 gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') # Removed DEBUG logs related to image processing final_image = gray_threshold save_preprocessed_image(final_image, OUTPUT_DIR) logging.info("Image preprocessing complete. Performing OCR...") custom_config = r'--oem 3 --psm 6' raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config) if not raw_text.strip(): logging.warning("OCR returned empty text.") speak_text("I couldn't detect any text in the selected area. Please try again with a different region.") else: ocr_text = raw_text.strip() logging.info(f"OCR Text: {ocr_text}") # Log the extracted OCR text logging.info("OCR process completed successfully.") except pytesseract.TesseractNotFoundError: logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.") speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.") ocr_text = "" except Exception as e: logging.error(f"OCR Error: {e}") speak_text("An error occurred during OCR processing. Please try again.") ocr_text = ""def get_selected_region(scaling_factor): root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) root.attributes("-topmost", True) root.overrideredirect(True)
screen_width = root.winfo_screenwidth() screen_height = root.winfo_screenheight() root.geometry(f"{screen_width}x{screen_height}") canvas = tk.Canvas(root, cursor="cross", bg="grey") canvas.pack(fill=tk.BOTH, expand=True) rect = None start_x = start_y = end_x = end_y = 0 def on_button_press(event): nonlocal start_x, start_y, rect start_x = int(event.x * scaling_factor) start_y = int(event.y * scaling_factor) rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) logging.info("Please select the region by clicking and dragging the mouse.") def on_move_press(event): nonlocal end_x, end_y end_x = int(event.x * scaling_factor) end_y = int(event.y * scaling_factor) canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y) def on_button_release(event): root.quit() root.destroy() def on_escape(event): logging.info("Capture cancelled by user.") root.quit() root.destroy() sys.exit() canvas.bind("<ButtonPress-1>", on_button_press) canvas.bind("<B1-Motion>", on_move_press) canvas.bind("<ButtonRelease-1>", on_button_release) root.bind("<Escape>", on_escape) root.mainloop() left = min(start_x, end_x) top = min(start_y, end_y) right = max(start_x, end_x) bottom = max(start_y, end_y) logging.info(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}") return (left, top, right, bottom)def generate_final_response(ocr_text): logging.info("Generating final response...")
# Retrieve relevant documents from the knowledge base query = ocr_text retrieved_docs = vector_store.similarity_search(query, k=5) # Adjust 'k' as needed retrieved_text = "\n".join([doc.page_content for doc in retrieved_docs]) # Combine OCR text with retrieved context context = f"""Assume the role of a strategist who excels at logical problem-solving and planning based on the following information.Retrieved Information: {retrieved_text}
OCR Text: {ocr_text}
Instructions:
- Understand the OCR text and treat it as a comprehensive multiple-choice question.
- Identify the correct answer from options (if provided within text).
- Respond in plain text without any Markdown formatting.
- Eliminate all Markdown syntax such as asterisks (*), underscores (_), backticks (`), and brackets ([]).
- Keep your answer strictly to 100 words.
- If there is calculation involved, follow this structure:
- We have been given (calculation_data),
- The formula to be used is (formula),
- We need to find (answer_variable).
- Perform calculations attentively.
Provide the answer below: """
try: # Generate response from the LLM with the enriched context response = ollama.generate(model='llama3.1', prompt=context) raw_response = response.get('response', '').strip() # Strip Markdown formatting plain_text_response = strip_markdown(raw_response) # Provide the response if "because" in plain_text_response: formatted_answer = plain_text_response logging.info("Generated formatted response.") else: formatted_answer = "The response could not be structured correctly." logging.warning(formatted_answer) if formatted_answer: speak_text(formatted_answer) else: logging.warning("No valid response could be generated.") return formatted_answer except Exception as e: logging.error(f"LLM Error during final response generation: {e}") return ""def reset_state(): global ocr_text ocr_text = ""
-------------------- Helper Functions --------------------
def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.
Args: short_timeout (int): Maximum number of seconds to wait for a phrase to start. phrase_time_limit (int): Maximum number of seconds a phrase can last. """ global is_listening, speech_text recognizer = sr.Recognizer() with sr.Microphone() as source: logging.info("Listening for voice input... Please say your query or question.") recognizer.adjust_for_ambient_noise(source, duration=1) # Adjust for noise for 1 second try: # Listen for speech with shorter timeout and phrase time limit audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit) logging.info("Processing the audio with Google Speech Recognition...") text = recognizer.recognize_google(audio) # Convert number words to digits if necessary speech_text = convert_number_words(text) logging.info(f"Speech Transcribed: {speech_text}") # Provide confirmation speak_text(f"I heard: {speech_text}.") except sr.WaitTimeoutError: logging.warning("Listening timed out, no speech detected. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.UnknownValueError: logging.warning("Speech recognition could not understand audio. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.RequestError as e: logging.error(f"Could not request results from speech recognition service; {e}") speech_text = "" # Ensure speech_text is empty finally: with lock: is_listening = False # Ensure listening flag is resetdef configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')
# List available voices (optional) # Commented out DEBUG logs as they are now suppressed # for index, voice in enumerate(voices): # logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}") # Change voice (select male/female, adjust the index as per your preference) if len(voices) > 1: tts_engine.setProperty('voice', voices[1].id) # Change index to select a different voice logging.info(f"TTS voice set to: {voices[1].name}") else: logging.info("Only one voice available. Using default voice.") # Adjust speaking rate (lower is slower and more human-like) tts_engine.setProperty('rate', 150) # Default is 200, lowering makes it more human-like # Adjust volume (0.0 to 1.0) tts_engine.setProperty('volume', 1.0)def capture_and_save_image(): global ocr_text, is_listening # Added is_listening to globals
region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top # Removed detailed dimension logging logging.info("Starting OCR process with image preprocessing...") if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() with lock: is_listening = False # Reset the listening flag return with mss.mss() as sct: monitor = { "top": int(top / scaling_factor), "left": int(left / scaling_factor), "width": int(width / scaling_factor), "height": int(height / scaling_factor) } try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") except Exception as e: logging.error(f"Screen capture failed: {e}") speak_text("Screen capture failed. Please ensure the selected area is correct and try again.") reset_state() with lock: is_listening = False # Reset the listening flag return perform_ocr(img) if not ocr_text.strip(): logging.error("OCR returned empty text. Please ensure the selected region contains readable text.") speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.") reset_state() with lock: is_listening = False # Reset the listening flag return generate_final_response(ocr_text) reset_state() with lock: is_listening = False # Reset the listening flag after processingdef start_listening(scaling): global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()
def stop_listening(): global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")
async def main(): global scaling_factor
configure_tts_voice() set_dpi_awareness() scaling_factor = get_scaling_factor() keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,)) keyboard.add_hotkey('f3', stop_listening) logging.info("QA Assistant is running.") logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.") while True: await asyncio.sleep(0.1) if keyboard.is_pressed('esc'): logging.info("Exiting...") speak_text("Exiting the QA Assistant. Goodbye!") break keyboard.unhook_all()if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: keyboard.unhook_all()
- Assistant
Certainly! Below is a separate Python script that allows you to:
- Draw a Rectangle Using the Mouse: Select a specific region on your screen by clicking and dragging the mouse.
- Take a Screenshot of the Selected Region: Capture the selected area of the screen.
- Perform OCR on the Screenshot: Extract text from the captured image using Optical Character Recognition (OCR).
- Copy Extracted Text to Clipboard: Automatically copy the extracted text to your clipboard for easy pasting.
Prerequisites
Before running the script, ensure you have the following installed on your system:
-
Python 3.x: If you don't have Python installed, download it from here.
-
Tesseract OCR Engine:
-
Windows:
- Download the installer from Tesseract at UB Mannheim.
- Run the installer and follow the installation instructions.
- Add the Tesseract installation directory (e.g.,
C:\Program Files\Tesseract-OCR) to your system's PATH environment variable.
-
macOS:
brew install tesseract -
Linux:
sudo apt-get update sudo apt-get install tesseract-ocr
-
-
Python Libraries: Install the required Python libraries using
pip:pip install mss pillow pytesseract pyperclip- mss: For taking screenshots.
- Pillow: For image processing.
- pytesseract: Python wrapper for Tesseract OCR.
- pyperclip: For copying text to the clipboard.
The Script
import sys import tkinter as tk from tkinter import messagebox from PIL import Image import mss import pytesseract import pyperclip # Optional: If Tesseract is not in your PATH, uncomment and set the path manually # pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe' def select_region(): """ Allows the user to select a region on the screen by dragging the mouse. Returns the coordinates of the selected region. """ root = tk.Tk() root.attributes('-alpha', 0.3) # Make the window semi-transparent root.attributes("-topmost", True) # Keep the window on top root.overrideredirect(True) # Remove window decorations screen_width = root.winfo_screenwidth() screen_height = root.winfo_screenheight() root.geometry(f"{screen_width}x{screen_height}") canvas = tk.Canvas(root, cursor="cross", bg="grey") canvas.pack(fill=tk.BOTH, expand=True) rect = None start_x = start_y = end_x = end_y = 0 def on_button_press(event): nonlocal start_x, start_y, rect start_x = root.winfo_pointerx() start_y = root.winfo_pointery() rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) def on_move_press(event): nonlocal end_x, end_y end_x = root.winfo_pointerx() end_y = root.winfo_pointery() canvas.coords(rect, start_x - root.winfo_rootx(), start_y - root.winfo_rooty(), event.x, event.y) def on_button_release(event): root.quit() canvas.bind("<ButtonPress-1>", on_button_press) canvas.bind("<B1-Motion>", on_move_press) canvas.bind("<ButtonRelease-1>", on_button_release) try: root.mainloop() except KeyboardInterrupt: sys.exit() # Calculate the selected region left = min(start_x, end_x) top = min(start_y, end_y) right = max(start_x, end_x) bottom = max(start_y, end_y) root.destroy() return (left, top, right, bottom) def capture_screenshot(region): """ Captures a screenshot of the specified region. :param region: Tuple of (left, top, right, bottom) :return: PIL Image of the captured region """ left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: messagebox.showerror("Error", "Invalid region selected. Please try again.") sys.exit() with mss.mss() as sct: monitor = {"top": top, "left": left, "width": width, "height": height} try: sct_img = sct.grab(monitor) img = Image.frombytes("RGB", sct_img.size, sct_img.rgb) return img except Exception as e: messagebox.showerror("Error", f"Failed to capture the screen: {e}") sys.exit() def perform_ocr(image): """ Performs OCR on the given image and returns the extracted text. :param image: PIL Image :return: Extracted text as string """ try: text = pytesseract.image_to_string(image, lang='eng') if not text.strip(): messagebox.showwarning("No Text Detected", "No text was detected in the selected region.") return text except pytesseract.TesseractNotFoundError: messagebox.showerror("Tesseract Not Found", "Tesseract OCR is not installed or not found in PATH.") sys.exit() except Exception as e: messagebox.showerror("OCR Error", f"An error occurred during OCR: {e}") sys.exit() def copy_to_clipboard(text): """ Copies the given text to the clipboard. :param text: Text to copy """ try: pyperclip.copy(text) messagebox.showinfo("Success", "Extracted text has been copied to the clipboard.") except Exception as e: messagebox.showerror("Clipboard Error", f"Failed to copy text to clipboard: {e}") sys.exit() def main(): messagebox.showinfo("Start", "Press OK and then click and drag to select a region for OCR.") region = select_region() if region == (0, 0, 0, 0): messagebox.showwarning("No Selection", "No region was selected.") sys.exit() image = capture_screenshot(region) text = perform_ocr(image) if text.strip(): copy_to_clipboard(text) if __name__ == "__main__": main()How It Works
-
Select Region: When you run the script, a semi-transparent fullscreen window appears. Click and drag your mouse to select the desired region. Release the mouse button to finalize the selection.
-
Capture Screenshot: The selected region is captured as an image.
-
Perform OCR: The captured image is processed using Tesseract OCR to extract any text present.
-
Copy to Clipboard: The extracted text is automatically copied to your clipboard. A confirmation message will appear to notify you of the successful operation.
Running the Script
-
Save the Script: Copy the above code into a file named, for example,
ocr_to_clipboard.py. -
Run the Script:
python ocr_to_clipboard.py -
Follow the Prompts:
- A message box will appear instructing you to press OK and select the region.
- After selecting the region, the script will process the image, perform OCR, and copy the text to your clipboard.
- Notifications will inform you of success or any errors encountered.
Troubleshooting
-
Tesseract Not Found: If you encounter an error stating that Tesseract is not found, ensure that:
- Tesseract OCR is installed on your system.
- The Tesseract executable is added to your system's PATH.
- If Tesseract is installed in a non-standard directory, uncomment and set the
pytesseract.pytesseract.tesseract_cmdpath in the script accordingly.
-
No Text Detected: If no text is detected:
- Ensure that the selected region contains clear and readable text.
- High-contrast images yield better OCR results.
- Consider preprocessing the image (e.g., converting to grayscale, increasing contrast) for improved accuracy.
-
Clipboard Issues: If the text is not copied to the clipboard:
- Ensure that the
pypercliplibrary is properly installed. - Check for any operating system restrictions or permissions that may prevent clipboard access.
- Ensure that the
Customization
-
Change OCR Language: To perform OCR in a language other than English, ensure that the corresponding language data is installed for Tesseract and modify the
langparameter in thepytesseract.image_to_stringfunction.text = pytesseract.image_to_string(image, lang='your_language_code') -
Image Preprocessing: For better OCR results, consider adding image preprocessing steps such as resizing, thresholding, or denoising before performing OCR.
Conclusion
This script provides a straightforward way to extract text from any region of your screen and have it readily available in your clipboard. It leverages powerful libraries and tools to deliver accurate OCR results with minimal user intervention.
Feel free to enhance and customize the script further to fit your specific needs!