Skip to content
TheChatArchiveThe Chat Archive
o1-mini-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

only take out these functinality and make a separate script, drawing recatngle using…

  1. User

    only take out these functinality and make a separate script, drawing recatngle using mouse, screenshot being taken, sent to ocr, extraced text and than that extracted text should be sent to clipboard : import re import mss from PIL import Image, ImageEnhance import pytesseract import speech_recognition as sr import pyttsx3 import keyboard import threading import os import logging import tkinter as tk import ctypes import sys import ollama import asyncio from datetime import datetime from word2number import w2n from langchain.vectorstores import FAISS from langchain_huggingface import HuggingFaceEmbeddings from sentence_transformers import SentenceTransformer

    -------------------- Configuration --------------------

    Define a custom logging filter to exclude specific messages

    class ExcludeFilter(logging.Filter): def init(self, exclude_keywords): super().init() self.exclude_keywords = exclude_keywords

    def filter(self, record):
        return not any(keyword in record.getMessage() for keyword in self.exclude_keywords)
    

    Configure logging

    logger = logging.getLogger() logger.setLevel(logging.INFO) # Set to INFO to include INFO and higher level logs

    Create a stream handler

    stream_handler = logging.StreamHandler() stream_handler.setLevel(logging.INFO)

    Define the log format

    formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s') stream_handler.setFormatter(formatter)

    Define keywords to exclude from logging

    excluded_keywords = [ "Mouse moved to", "Width:", "Height:", "Monitor for capture:" ]

    Add the custom filter to the handler

    stream_handler.addFilter(ExcludeFilter(excluded_keywords))

    Add the handler to the logger

    if not logger.handlers: logger.addHandler(stream_handler) else: # Remove existing handlers to prevent duplicate logs logger.handlers = [] logger.addHandler(stream_handler)

    Initialize TTS engine

    tts_engine = pyttsx3.init() is_listening = False ocr_text = ""

    lock = threading.Lock() tts_lock = threading.Lock()

    OUTPUT_DIR = r"D:\AI\EXP2\TempS"

    Initialize RAG components

    embedding_model_name = "all-mpnet-base-v2" # Use a more robust model embeddings = HuggingFaceEmbeddings(model_name=embedding_model_name) vector_store = FAISS.load_local(r"D:\AI\EXP2\faiss_index", embeddings,allow_dangerous_deserialization=True)

    def set_dpi_awareness(): try: if sys.platform.startswith('win'): ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")

    def get_scaling_factor(): try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0

    def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text

    def strip_markdown(text): """Function to remove Markdown syntax from text.""" # Remove asterisks, underscores, and backticks text = re.sub(r'[*_`]', '', text) # Remove links text text = re.sub(r'[([^]]+)]([^)]+)', r'\1', text) # Remove headings ## Heading text = re.sub(r'#+\s?', '', text) # Add more patterns as necessary return text

    def speak_text(text): global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: tts_engine.say(text)

            # Add hotkeys for pausing (F4) and stopping (F5)
            keyboard.add_hotkey('f4', pause_speaking)
            keyboard.add_hotkey('f5', stop_speaking)
    
            tts_engine.runAndWait()
            logging.info("Text-to-Speech completed.")
        except RuntimeError as e:
            logging.error(f"TTS Error: {e}")
        finally:
            keyboard.remove_hotkey('f4')
            keyboard.remove_hotkey('f5')
    

    def pause_speaking(): global tts_engine logging.info("Paused speaking...") # pyttsx3 does not support pausing natively. Placeholder.

    def stop_speaking(): global tts_engine logging.info("Stopped speaking.") tts_engine.stop()

    def save_captured_image(image, output_dir, prefix="captured_region"): try: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png") image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")

    def save_preprocessed_image(image, output_dir, prefix="preprocessed_image"): try: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png") image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")

    def perform_ocr(image): global ocr_text try: logging.info("Starting OCR process with image preprocessing...")

        save_captured_image(image, OUTPUT_DIR)
    
        gray = image.convert('L')
        # Removed DEBUG logs related to image processing
    
        enhancer = ImageEnhance.Contrast(gray)
        gray_enhanced = enhancer.enhance(1.0)
        # Removed DEBUG logs related to image processing
    
        threshold = 190
        gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
        # Removed DEBUG logs related to image processing
    
        final_image = gray_threshold
    
        save_preprocessed_image(final_image, OUTPUT_DIR)
    
        logging.info("Image preprocessing complete. Performing OCR...")
    
        custom_config = r'--oem 3 --psm 6'
        raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config)
    
        if not raw_text.strip():
            logging.warning("OCR returned empty text.")
            speak_text("I couldn't detect any text in the selected area. Please try again with a different region.")
        else:
            ocr_text = raw_text.strip()
            logging.info(f"OCR Text: {ocr_text}")  # Log the extracted OCR text
            logging.info("OCR process completed successfully.")
    except pytesseract.TesseractNotFoundError:
        logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.")
        speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.")
        ocr_text = ""
    except Exception as e:
        logging.error(f"OCR Error: {e}")
        speak_text("An error occurred during OCR processing. Please try again.")
        ocr_text = ""
    

    def get_selected_region(scaling_factor): root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) root.attributes("-topmost", True) root.overrideredirect(True)

    screen_width = root.winfo_screenwidth()
    screen_height = root.winfo_screenheight()
    root.geometry(f"{screen_width}x{screen_height}")
    
    canvas = tk.Canvas(root, cursor="cross", bg="grey")
    canvas.pack(fill=tk.BOTH, expand=True)
    
    rect = None
    start_x = start_y = end_x = end_y = 0
    
    def on_button_press(event):
        nonlocal start_x, start_y, rect
        start_x = int(event.x * scaling_factor)
        start_y = int(event.y * scaling_factor)
        rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
        logging.info("Please select the region by clicking and dragging the mouse.")
    
    def on_move_press(event):
        nonlocal end_x, end_y
        end_x = int(event.x * scaling_factor)
        end_y = int(event.y * scaling_factor)
        canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y)
    
    def on_button_release(event):
        root.quit()
        root.destroy()
    
    def on_escape(event):
        logging.info("Capture cancelled by user.")
        root.quit()
        root.destroy()
        sys.exit()
    
    canvas.bind("<ButtonPress-1>", on_button_press)
    canvas.bind("<B1-Motion>", on_move_press)
    canvas.bind("<ButtonRelease-1>", on_button_release)
    root.bind("<Escape>", on_escape)
    
    root.mainloop()
    
    left = min(start_x, end_x)
    top = min(start_y, end_y)
    right = max(start_x, end_x)
    bottom = max(start_y, end_y)
    
    logging.info(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}")
    return (left, top, right, bottom)
    

    def generate_final_response(ocr_text): logging.info("Generating final response...")

    # Retrieve relevant documents from the knowledge base
    query = ocr_text
    retrieved_docs = vector_store.similarity_search(query, k=5)  # Adjust 'k' as needed
    retrieved_text = "\n".join([doc.page_content for doc in retrieved_docs])
    
    # Combine OCR text with retrieved context
    context = f"""Assume the role of a strategist who excels at logical problem-solving and planning based on the following information.
    

    Retrieved Information: {retrieved_text}

    OCR Text: {ocr_text}

    Instructions:

    1. Understand the OCR text and treat it as a comprehensive multiple-choice question.
    2. Identify the correct answer from options (if provided within text).
    3. Respond in plain text without any Markdown formatting.
    4. Eliminate all Markdown syntax such as asterisks (*), underscores (_), backticks (`), and brackets ([]).
    5. Keep your answer strictly to 100 words.
    6. If there is calculation involved, follow this structure:
      • We have been given (calculation_data),
      • The formula to be used is (formula),
      • We need to find (answer_variable).
    7. Perform calculations attentively.

    Provide the answer below: """

    try:
        # Generate response from the LLM with the enriched context
        response = ollama.generate(model='llama3.1', prompt=context)
        raw_response = response.get('response', '').strip()
    
        # Strip Markdown formatting
        plain_text_response = strip_markdown(raw_response)
    
        # Provide the response
        if "because" in plain_text_response:
            formatted_answer = plain_text_response
            logging.info("Generated formatted response.")
        else:
            formatted_answer = "The response could not be structured correctly."
            logging.warning(formatted_answer)
    
        if formatted_answer:
            speak_text(formatted_answer)
        else:
            logging.warning("No valid response could be generated.")
        
        return formatted_answer
    except Exception as e:
        logging.error(f"LLM Error during final response generation: {e}")
        return ""
    

    def reset_state(): global ocr_text ocr_text = ""

    -------------------- Helper Functions --------------------

    def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.

    Args:
        short_timeout (int): Maximum number of seconds to wait for a phrase to start.
        phrase_time_limit (int): Maximum number of seconds a phrase can last.
    """
    global is_listening, speech_text
    recognizer = sr.Recognizer()
    with sr.Microphone() as source:
        logging.info("Listening for voice input... Please say your query or question.")
        recognizer.adjust_for_ambient_noise(source, duration=1)  # Adjust for noise for 1 second
        try:
            # Listen for speech with shorter timeout and phrase time limit
            audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit)
            logging.info("Processing the audio with Google Speech Recognition...")
            text = recognizer.recognize_google(audio)
            # Convert number words to digits if necessary
            speech_text = convert_number_words(text)
            logging.info(f"Speech Transcribed: {speech_text}")
    
            # Provide confirmation
            speak_text(f"I heard: {speech_text}.")
        except sr.WaitTimeoutError:
            logging.warning("Listening timed out, no speech detected. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.UnknownValueError:
            logging.warning("Speech recognition could not understand audio. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.RequestError as e:
            logging.error(f"Could not request results from speech recognition service; {e}")
            speech_text = ""  # Ensure speech_text is empty
        finally:
            with lock:
                is_listening = False  # Ensure listening flag is reset
    

    def configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')

    # List available voices (optional)
    # Commented out DEBUG logs as they are now suppressed
    # for index, voice in enumerate(voices):
    #     logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}")
    
    # Change voice (select male/female, adjust the index as per your preference)
    if len(voices) > 1:
        tts_engine.setProperty('voice', voices[1].id)  # Change index to select a different voice
        logging.info(f"TTS voice set to: {voices[1].name}")
    else:
        logging.info("Only one voice available. Using default voice.")
    
    # Adjust speaking rate (lower is slower and more human-like)
    tts_engine.setProperty('rate', 150)  # Default is 200, lowering makes it more human-like
    
    # Adjust volume (0.0 to 1.0)
    tts_engine.setProperty('volume', 1.0)
    

    def capture_and_save_image(): global ocr_text, is_listening # Added is_listening to globals

    region = get_selected_region(scaling_factor)
    left, top, right, bottom = region
    width = right - left
    height = bottom - top
    
    # Removed detailed dimension logging
    logging.info("Starting OCR process with image preprocessing...")
    
    if width <= 0 or height <= 0:
        logging.error("Invalid region selected. Please try again.")
        speak_text("Invalid region selected. Please try again by pressing F2.")
        reset_state()
        with lock:
            is_listening = False  # Reset the listening flag
        return
    
    with mss.mss() as sct:
        monitor = {
            "top": int(top / scaling_factor),
            "left": int(left / scaling_factor),
            "width": int(width / scaling_factor),
            "height": int(height / scaling_factor)
        }
    
        try:
            screenshot = sct.grab(monitor)
            img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
        except Exception as e:
            logging.error(f"Screen capture failed: {e}")
            speak_text("Screen capture failed. Please ensure the selected area is correct and try again.")
            reset_state()
            with lock:
                is_listening = False  # Reset the listening flag
            return
    
    perform_ocr(img)
    if not ocr_text.strip():
        logging.error("OCR returned empty text. Please ensure the selected region contains readable text.")
        speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.")
        reset_state()
        with lock:
            is_listening = False  # Reset the listening flag
        return
    
    generate_final_response(ocr_text)
    reset_state()
    with lock:
        is_listening = False  # Reset the listening flag after processing
    

    def start_listening(scaling): global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()

    def stop_listening(): global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")

    async def main(): global scaling_factor

    configure_tts_voice()
    
    set_dpi_awareness()
    scaling_factor = get_scaling_factor()
    
    keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,))
    keyboard.add_hotkey('f3', stop_listening)
    
    logging.info("QA Assistant is running.")
    logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.")
    
    while True:
        await asyncio.sleep(0.1)
        if keyboard.is_pressed('esc'):
            logging.info("Exiting...")
            speak_text("Exiting the QA Assistant. Goodbye!")
            break
    
    keyboard.unhook_all()
    

    if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: keyboard.unhook_all()

  2. Assistant

    Certainly! Below is a separate Python script that allows you to:

    1. Draw a Rectangle Using the Mouse: Select a specific region on your screen by clicking and dragging the mouse.
    2. Take a Screenshot of the Selected Region: Capture the selected area of the screen.
    3. Perform OCR on the Screenshot: Extract text from the captured image using Optical Character Recognition (OCR).
    4. Copy Extracted Text to Clipboard: Automatically copy the extracted text to your clipboard for easy pasting.

    Prerequisites

    Before running the script, ensure you have the following installed on your system:

    1. Python 3.x: If you don't have Python installed, download it from here.

    2. Tesseract OCR Engine:

      • Windows:

        • Download the installer from Tesseract at UB Mannheim.
        • Run the installer and follow the installation instructions.
        • Add the Tesseract installation directory (e.g., C:\Program Files\Tesseract-OCR) to your system's PATH environment variable.
      • macOS:

        brew install tesseract
        
      • Linux:

        sudo apt-get update
        sudo apt-get install tesseract-ocr
        
    3. Python Libraries: Install the required Python libraries using pip:

      pip install mss pillow pytesseract pyperclip
      
      • mss: For taking screenshots.
      • Pillow: For image processing.
      • pytesseract: Python wrapper for Tesseract OCR.
      • pyperclip: For copying text to the clipboard.

    The Script

    import sys
    import tkinter as tk
    from tkinter import messagebox
    from PIL import Image
    import mss
    import pytesseract
    import pyperclip
    
    # Optional: If Tesseract is not in your PATH, uncomment and set the path manually
    # pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
    
    def select_region():
        """
        Allows the user to select a region on the screen by dragging the mouse.
        Returns the coordinates of the selected region.
        """
        root = tk.Tk()
        root.attributes('-alpha', 0.3)  # Make the window semi-transparent
        root.attributes("-topmost", True)  # Keep the window on top
        root.overrideredirect(True)  # Remove window decorations
    
        screen_width = root.winfo_screenwidth()
        screen_height = root.winfo_screenheight()
        root.geometry(f"{screen_width}x{screen_height}")
    
        canvas = tk.Canvas(root, cursor="cross", bg="grey")
        canvas.pack(fill=tk.BOTH, expand=True)
    
        rect = None
        start_x = start_y = end_x = end_y = 0
    
        def on_button_press(event):
            nonlocal start_x, start_y, rect
            start_x = root.winfo_pointerx()
            start_y = root.winfo_pointery()
            rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
    
        def on_move_press(event):
            nonlocal end_x, end_y
            end_x = root.winfo_pointerx()
            end_y = root.winfo_pointery()
            canvas.coords(rect, start_x - root.winfo_rootx(), start_y - root.winfo_rooty(),
                          event.x, event.y)
    
        def on_button_release(event):
            root.quit()
    
        canvas.bind("<ButtonPress-1>", on_button_press)
        canvas.bind("<B1-Motion>", on_move_press)
        canvas.bind("<ButtonRelease-1>", on_button_release)
    
        try:
            root.mainloop()
        except KeyboardInterrupt:
            sys.exit()
    
        # Calculate the selected region
        left = min(start_x, end_x)
        top = min(start_y, end_y)
        right = max(start_x, end_x)
        bottom = max(start_y, end_y)
    
        root.destroy()
        return (left, top, right, bottom)
    
    def capture_screenshot(region):
        """
        Captures a screenshot of the specified region.
        :param region: Tuple of (left, top, right, bottom)
        :return: PIL Image of the captured region
        """
        left, top, right, bottom = region
        width = right - left
        height = bottom - top
    
        if width <= 0 or height <= 0:
            messagebox.showerror("Error", "Invalid region selected. Please try again.")
            sys.exit()
    
        with mss.mss() as sct:
            monitor = {"top": top, "left": left, "width": width, "height": height}
            try:
                sct_img = sct.grab(monitor)
                img = Image.frombytes("RGB", sct_img.size, sct_img.rgb)
                return img
            except Exception as e:
                messagebox.showerror("Error", f"Failed to capture the screen: {e}")
                sys.exit()
    
    def perform_ocr(image):
        """
        Performs OCR on the given image and returns the extracted text.
        :param image: PIL Image
        :return: Extracted text as string
        """
        try:
            text = pytesseract.image_to_string(image, lang='eng')
            if not text.strip():
                messagebox.showwarning("No Text Detected", "No text was detected in the selected region.")
            return text
        except pytesseract.TesseractNotFoundError:
            messagebox.showerror("Tesseract Not Found", "Tesseract OCR is not installed or not found in PATH.")
            sys.exit()
        except Exception as e:
            messagebox.showerror("OCR Error", f"An error occurred during OCR: {e}")
            sys.exit()
    
    def copy_to_clipboard(text):
        """
        Copies the given text to the clipboard.
        :param text: Text to copy
        """
        try:
            pyperclip.copy(text)
            messagebox.showinfo("Success", "Extracted text has been copied to the clipboard.")
        except Exception as e:
            messagebox.showerror("Clipboard Error", f"Failed to copy text to clipboard: {e}")
            sys.exit()
    
    def main():
        messagebox.showinfo("Start", "Press OK and then click and drag to select a region for OCR.")
        region = select_region()
        if region == (0, 0, 0, 0):
            messagebox.showwarning("No Selection", "No region was selected.")
            sys.exit()
        image = capture_screenshot(region)
        text = perform_ocr(image)
        if text.strip():
            copy_to_clipboard(text)
    
    if __name__ == "__main__":
        main()
    

    How It Works

    1. Select Region: When you run the script, a semi-transparent fullscreen window appears. Click and drag your mouse to select the desired region. Release the mouse button to finalize the selection.

    2. Capture Screenshot: The selected region is captured as an image.

    3. Perform OCR: The captured image is processed using Tesseract OCR to extract any text present.

    4. Copy to Clipboard: The extracted text is automatically copied to your clipboard. A confirmation message will appear to notify you of the successful operation.

    Running the Script

    1. Save the Script: Copy the above code into a file named, for example, ocr_to_clipboard.py.

    2. Run the Script:

      python ocr_to_clipboard.py
      
    3. Follow the Prompts:

      • A message box will appear instructing you to press OK and select the region.
      • After selecting the region, the script will process the image, perform OCR, and copy the text to your clipboard.
      • Notifications will inform you of success or any errors encountered.

    Troubleshooting

    • Tesseract Not Found: If you encounter an error stating that Tesseract is not found, ensure that:

      • Tesseract OCR is installed on your system.
      • The Tesseract executable is added to your system's PATH.
      • If Tesseract is installed in a non-standard directory, uncomment and set the pytesseract.pytesseract.tesseract_cmd path in the script accordingly.
    • No Text Detected: If no text is detected:

      • Ensure that the selected region contains clear and readable text.
      • High-contrast images yield better OCR results.
      • Consider preprocessing the image (e.g., converting to grayscale, increasing contrast) for improved accuracy.
    • Clipboard Issues: If the text is not copied to the clipboard:

      • Ensure that the pyperclip library is properly installed.
      • Check for any operating system restrictions or permissions that may prevent clipboard access.

    Customization

    • Change OCR Language: To perform OCR in a language other than English, ensure that the corresponding language data is installed for Tesseract and modify the lang parameter in the pytesseract.image_to_string function.

      text = pytesseract.image_to_string(image, lang='your_language_code')
      
    • Image Preprocessing: For better OCR results, consider adding image preprocessing steps such as resizing, thresholding, or denoising before performing OCR.

    Conclusion

    This script provides a straightforward way to extract text from any region of your screen and have it readily available in your clipboard. It leverages powerful libraries and tools to deliver accurate OCR results with minimal user intervention.

    Feel free to enhance and customize the script further to fit your specific needs!