Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import…

  1. User

    import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import time import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama

    -------------------- Configuration --------------------

    Configure logging to capture DEBUG level logs

    logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )

    Initialize the TTS engine globally

    tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start

    Define global locks

    lock = threading.Lock() tts_lock = threading.Lock()

    -------------------- Preferred Sites Configuration --------------------

    Editable list of preferred sites for prioritized search

    PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]

    def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")

    def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0

    def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text

    def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found

    def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)

            # Add hotkeys for pausing (F4) and stopping (F5)
            keyboard.add_hotkey('f4', pause_speaking)
            keyboard.add_hotkey('f5', stop_speaking)
    
            tts_engine.runAndWait()
            logging.info("Text-to-Speech completed.")
        except RuntimeError as e:
            logging.error(f"TTS Error: {e}")
        finally:
            # Remove hotkeys after completion to avoid conflicts
            keyboard.remove_hotkey('f4')
            keyboard.remove_hotkey('f5')
    

    def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()

    def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()

    def save_captured_image(image, filename="captured_region.png"): """ Saves the captured image for debugging purposes. """ try: image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")

    def save_preprocessed_image(image, filename="preprocessed_image.png"): """ Saves the preprocessed image for debugging purposes. """ try: image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")

    def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")

        # Step 1: Save the original captured image for verification
        save_captured_image(image)
    
        # Step 2: Convert to grayscale
        gray = image.convert('L')
    
        # Step 3: Enhance contrast
        enhancer = ImageEnhance.Contrast(gray)
        gray_enhanced = enhancer.enhance(10)
    
        # Step 4: Apply thresholding
        threshold = 140  # Adjust based on testing
        gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
    
        # Step 5: Reduce noise with a median filter
        final_image = gray_threshold.filter(ImageFilter.MedianFilter())
    
        # Step 6: Save preprocessed image for debugging
        save_preprocessed_image(final_image)
    
        logging.info("Image preprocessing complete. Performing OCR...")
    
        # Define Tesseract configuration
        custom_config = r'--oem 3 --psm 6'  # Adjust as needed
    
        # Perform OCR
        raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config)
    
        if not raw_text.strip():
            logging.warning("OCR returned empty text.")
            speak_text("I couldn't detect any text in the selected area. Please try again with a different region.")
        else:
            ocr_text = raw_text
            logging.debug("Raw OCR Text:")
            logging.debug(ocr_text)
            logging.info("OCR process completed successfully.")
    except pytesseract.TesseractNotFoundError:
        logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.")
        speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.")
        ocr_text = ""
    except Exception as e:
        logging.error(f"OCR Error: {e}")
        speak_text("An error occurred during OCR processing. Please try again.")
        ocr_text = ""
    

    def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations

    screen_width = root.winfo_screenwidth()
    screen_height = root.winfo_screenheight()
    root.geometry(f"{screen_width}x{screen_height}")
    
    canvas = tk.Canvas(root, cursor="cross", bg="grey")
    canvas.pack(fill=tk.BOTH, expand=True)
    
    rect = None
    start_x = start_y = end_x = end_y = 0
    
    def on_button_press(event):
        nonlocal start_x, start_y, rect
        start_x = int(event.x * scaling_factor)
        start_y = int(event.y * scaling_factor)
        rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
        logging.debug(f"Button pressed at ({start_x}, {start_y})")
    
    def on_move_press(event):
        nonlocal end_x, end_y
        end_x = int(event.x * scaling_factor)
        end_y = int(event.y * scaling_factor)
        canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y)
        logging.debug(f"Mouse moved to ({end_x}, {end_y})")
    
    def on_button_release(event):
        root.quit()
        root.destroy()  # Destroy the window to remove the overlay
    
    def on_escape(event):
        logging.info("Capture cancelled by user.")
        root.quit()
        root.destroy()
        sys.exit()
    
    canvas.bind("<ButtonPress-1>", on_button_press)
    canvas.bind("<B1-Motion>", on_move_press)
    canvas.bind("<ButtonRelease-1>", on_button_release)
    root.bind("<Escape>", on_escape)
    
    logging.info("Please select the region by clicking and dragging the mouse.")
    root.mainloop()
    
    # Calculate absolute coordinates
    left = min(start_x, end_x)
    top = min(start_y, end_y)
    right = max(start_x, end_x)
    bottom = max(start_y, end_y)
    
    logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}")
    return (left, top, right, bottom)
    

    -------------------- Modified Web Search Function --------------------

    async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.

    Args:
        query (str): The search query.
    
    Returns:
        list: A list of dictionaries containing 'title' and 'snippet' for each result.
    """
    logging.info(f"Searching the web for query: {query}")
    
    # Construct the site-limited query
    if PREFERRED_SITES:
        site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES])
        combined_query = f"{query} ({site_query})"
    else:
        combined_query = query  # If no preferred sites, perform a general search
    
    logging.debug(f"Combined search query with preferred sites: {combined_query}")
    
    # Encode the query for URL
    encoded_query = aiohttp.helpers.quote(combined_query)
    url = f"https://html.duckduckgo.com/html/?q={encoded_query}"
    
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                html = await response.text()
    except Exception as e:
        logging.error(f"Web search failed: {e}")
        return []
    
    soup = BeautifulSoup(html, 'html.parser')
    results = []
    
    # DuckDuckGo's HTML structure can vary; adjust selectors as needed
    for result in soup.find_all('div', class_='result__body', limit=3):
        title_element = result.find('a', class_='result__a')
        snippet_element = result.find('a', class_='result__snippet')
        title = title_element.get_text(strip=True) if title_element else "No Title"
        snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet"
        results.append({'title': title, 'snippet': snippet})
        logging.debug(f"Found result - Title: {title}, Snippet: {snippet}")
    
    if not results:
        logging.warning("No search results found for the preferred sites.")
    
    logging.info(f"Web search complete. Found {len(results)} results.")
    return results
    

    def generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])

    if user_input:
        context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.
    

    OCR Text: {ocr_text}

    User Query: {user_input}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text and the user's query.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response to the user's question based on the available information.
    4. Ensure that your answer directly addresses the user's intent.

    Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.

    OCR Text: {ocr_text}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response based on the available information.
    4. Ensure that your answer directly reflects the content of the OCR text and the search results.

    Provide the answer below: """

    try:
        response = ollama.generate(model='llama3.2', prompt=context)
        final_response = response.get('response', '').strip()
        logging.info(f"Generated Final Response: {final_response}")
        return final_response
    except Exception as e:
        logging.error(f"LLM Error during final response generation: {e}")
        return ""
    

    def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""

    -------------------- Helper Functions --------------------

    def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.

    Args:
        short_timeout (int): Maximum number of seconds to wait for a phrase to start.
        phrase_time_limit (int): Maximum number of seconds a phrase can last.
    """
    global is_listening, speech_text
    recognizer = sr.Recognizer()
    with sr.Microphone() as source:
        logging.info("Listening for voice input... Please say your query or question.")
        recognizer.adjust_for_ambient_noise(source, duration=1)  # Adjust for noise for 1 second
        try:
            # Listen for speech with shorter timeout and phrase time limit
            audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit)
            logging.info("Processing the audio with Google Speech Recognition...")
            text = recognizer.recognize_google(audio)
            # Convert number words to digits if necessary
            speech_text = convert_number_words(text)
            logging.info(f"Speech Transcribed: {speech_text}")
    
            # Provide confirmation
            speak_text(f"I heard: {speech_text}.")
        except sr.WaitTimeoutError:
            logging.warning("Listening timed out, no speech detected. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.UnknownValueError:
            logging.warning("Speech recognition could not understand audio. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.RequestError as e:
            logging.error(f"Could not request results from speech recognition service; {e}")
            speech_text = ""  # Ensure speech_text is empty
        finally:
            with lock:
                is_listening = False  # Ensure listening flag is reset
    

    def configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')

    # List available voices (optional)
    for index, voice in enumerate(voices):
        logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}")
    
    # Change voice (select male/female, adjust the index as per your preference)
    if len(voices) > 1:
        tts_engine.setProperty('voice', voices[1].id)  # Change index to select a different voice
        logging.info(f"TTS voice set to: {voices[1].name}")
    else:
        logging.info("Only one voice available. Using default voice.")
    
    # Adjust speaking rate (lower is slower and more human-like)
    tts_engine.setProperty('rate', 160)  # Default is 200, lowering makes it more human-like
    
    # Adjust volume (0.0 to 1.0)
    tts_engine.setProperty('volume', 1.0)
    

    def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return

    # Step 2: Capture the selected region
    with mss.mss() as sct:
        monitor = {
            "top": top,
            "left": left,
            "width": width,
            "height": height
        }
        try:
            screenshot = sct.grab(monitor)
            img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
        except Exception as e:
            logging.error(f"Screen capture failed: {e}")
            speak_text("Screen capture failed. Please ensure the selected area is correct and try again.")
            reset_state()
            return
    
    # Step 3: Perform OCR on the captured region
    perform_ocr(img)
    if not ocr_text.strip():
        logging.error("OCR returned empty text. Please ensure the selected region contains readable text.")
        speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.")
        reset_state()
        return
    
    # Step 4: Listen for user input (optional) with shorter timeout
    transcribe_speech(short_timeout=5, phrase_time_limit=5)
    
    # Step 5: Generate and execute the search query
    generate_and_execute_search(speech_text, ocr_text)
    

    def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return

    logging.info(f"Using OCR text as search query: {search_query}")
    
    # Perform web search and process the answer
    asyncio.run(async_process_search(search_query, ocr_text, user_input))
    

    async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return

    # Generate final response using LLM
    final_response = generate_final_response(search_results, ocr_text, user_input)
    if not final_response:
        logging.error("Failed to generate a final response.")
        speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.")
        reset_state()
        return
    
    # Speak out the answer
    speak_text(final_response)
    reset_state()
    

    -------------------- Core Functions --------------------

    def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()

    def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")

    -------------------- Main Function --------------------

    async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor

    configure_tts_voice()  # Voice configuration happens before anything else
    
    # Set DPI awareness and get scaling factor
    set_dpi_awareness()
    scaling_factor = get_scaling_factor()
    
    # Set up hotkeys
    keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,))
    keyboard.add_hotkey('f3', stop_listening)
    
    logging.info("QA Assistant is running.")
    logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.")
    
    while True:
        # Wait for a short duration to avoid busy waiting
        await asyncio.sleep(0.1)
    
        # Check for exit condition
        if keyboard.is_pressed('esc'):
            logging.info("Exiting...")
            speak_text("Exiting the QA Assistant. Goodbye!")
            break
    
        # Reset speech_text and ocr_text is handled after processing
    
    # Clean up keyboard hooks before exiting
    keyboard.unhook_all()
    

    -------------------- Entry Point --------------------

    if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all():: Extract me the part just before the image is fed into ocr to a new script which outputs me the image itself to see if there is some offset erorr in ocr

  2. Assistant

    To extract the part of your script that captures a screen region and outputs the preprocessed image before it's fed into OCR, we'll create a standalone script. This script will capture the selected screen region and save the preprocessed image to a file, allowing you to visually inspect it for any issues.

    Below is the extracted and modified script for this purpose:

    import mss
    from PIL import Image, ImageEnhance, ImageFilter
    import tkinter as tk
    import ctypes
    import sys
    import logging
    
    # Configure logging
    logging.basicConfig(
        level=logging.DEBUG,
        format='%(asctime)s [%(levelname)s] %(message)s',
        handlers=[logging.StreamHandler()]
    )
    
    def set_dpi_awareness():
        """
        Sets the DPI awareness of the process to prevent scaling issues.
        """
        try:
            if sys.platform.startswith('win'):
                awareness = ctypes.c_int()
                ctypes.windll.shcore.SetProcessDpiAwareness(1)
                logging.info("Process DPI awareness set to SYSTEM_AWARE.")
        except Exception as e:
            logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
    
    def get_scaling_factor():
        """
        Retrieves the current DPI scaling factor.
        Returns:
            float: Scaling factor (e.g., 1.25 for 125%)
        """
        try:
            if sys.platform.startswith('win'):
                user32 = ctypes.windll.user32
                user32.SetProcessDPIAware()
                dpi = user32.GetDpiForWindow(user32.GetForegroundWindow())
                scaling_factor = dpi / 96  # 96 DPI is 100%
                logging.info(f"DPI Scaling Factor detected: {scaling_factor}")
                return scaling_factor
            else:
                logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.")
                return 1.0
        except Exception as e:
            logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.")
            return 1.0
    
    def get_selected_region(scaling_factor):
        """
        Allows the user to select a screen region by drawing a rectangle.
        Returns the coordinates of the selected region adjusted for DPI scaling.
        """
        root = tk.Tk()
        root.title("Select Region (Press Esc to Cancel)")
        root.attributes('-alpha', 0.3)  # Transparency
        root.attributes("-topmost", True)
        root.overrideredirect(True)  # Remove window decorations
    
        screen_width = root.winfo_screenwidth()
        screen_height = root.winfo_screenheight()
        root.geometry(f"{screen_width}x{screen_height}")
    
        canvas = tk.Canvas(root, cursor="cross", bg="grey")
        canvas.pack(fill=tk.BOTH, expand=True)
    
        rect = None
        start_x = start_y = end_x = end_y = 0
    
        def on_button_press(event):
            nonlocal start_x, start_y, rect
            start_x = int(event.x * scaling_factor)
            start_y = int(event.y * scaling_factor)
            rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
            logging.debug(f"Button pressed at ({start_x}, {start_y})")
    
        def on_move_press(event):
            nonlocal end_x, end_y
            end_x = int(event.x * scaling_factor)
            end_y = int(event.y * scaling_factor)
            canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y)
            logging.debug(f"Mouse moved to ({end_x}, {end_y})")
    
        def on_button_release(event):
            root.quit()
            root.destroy()  # Destroy the window to remove the overlay
    
        def on_escape(event):
            logging.info("Capture cancelled by user.")
            root.quit()
            root.destroy()
            sys.exit()
    
        canvas.bind("<ButtonPress-1>", on_button_press)
        canvas.bind("<B1-Motion>", on_move_press)
        canvas.bind("<ButtonRelease-1>", on_button_release)
        root.bind("<Escape>", on_escape)
    
        logging.info("Please select the region by clicking and dragging the mouse.")
        root.mainloop()
    
        left = min(start_x, end_x)
        top = min(start_y, end_y)
        right = max(start_x, end_x)
        bottom = max(start_y, end_y)
    
        logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}")
        return (left, top, right, bottom)
    
    def save_preprocessed_image(image, filename="preprocessed_image.png"):
        """
        Saves the preprocessed image for debugging purposes.
        """
        try:
            image.save(filename)
            logging.info(f"Preprocessed image saved as {filename}")
        except Exception as e:
            logging.error(f"Failed to save preprocessed image: {e}")
    
    def preprocess_image(image):
        """
        Preprocesses the image for OCR by converting to grayscale, enhancing contrast,
        applying thresholding, and reducing noise.
        """
        logging.info("Starting image preprocessing...")
    
        # Convert to grayscale
        gray = image.convert('L')
    
        # Enhance contrast
        enhancer = ImageEnhance.Contrast(gray)
        gray_enhanced = enhancer.enhance(10)
    
        # Apply thresholding
        threshold = 140  # Adjust based on testing
        gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
    
        # Reduce noise with a median filter
        final_image = gray_threshold.filter(ImageFilter.MedianFilter())
    
        logging.info("Image preprocessing complete.")
        return final_image
    
    def capture_and_save_image():
        """
        Coordinates the screen capture and image preprocessing.
        """
        # Step 1: Let the user select a region
        region = get_selected_region(scaling_factor)
        left, top, right, bottom = region
        width = right - left
        height = bottom - top
        if width <= 0 or height <= 0:
            logging.error("Invalid region selected. Please try again.")
            return
    
        # Step 2: Capture the selected region
        with mss.mss() as sct:
            monitor = {
                "top": top,
                "left": left,
                "width": width,
                "height": height
            }
            try:
                screenshot = sct.grab(monitor)
                img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
            except Exception as e:
                logging.error(f"Screen capture failed: {e}")
                return
    
        # Step 3: Preprocess the captured image
        preprocessed_image = preprocess_image(img)
    
        # Step 4: Save the preprocessed image
        save_preprocessed_image(preprocessed_image, "preprocessed_image.png")
    
    if __name__ == "__main__":
        try:
            set_dpi_awareness()
            scaling_factor = get_scaling_factor()
            capture_and_save_image()
        except KeyboardInterrupt:
            logging.info("Program terminated by user.")
        except Exception as e:
            logging.error(f"An unexpected error occurred: {e}")
    

    This script will allow you to capture a region of the screen and save the preprocessed image, which you can then use to check for any issues before feeding it into your OCR process.

  3. User

    Is the first script subset of second one or is there difference in functionality : import mss from PIL import Image, ImageEnhance, ImageFilter import tkinter as tk import ctypes import sys import logging import time

    -------------------- Configuration --------------------

    Configure logging to capture DEBUG level logs

    logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("ocr_troubleshoot.log"), ] )

    -------------------- DPI Awareness and Scaling --------------------

    def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. Specifically for Windows. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")

    def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = ctypes.windll.shcore.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0

    -------------------- Region Selection Interface --------------------

    class RegionSelector: def init(self, scaling_factor): self.scaling_factor = scaling_factor self.start_x = 0 self.start_y = 0 self.end_x = 0 self.end_y = 0 self.rect = None self.selection_complete = False

        # Initialize the tkinter window
        self.root = tk.Tk()
        self.root.title("Select Region (Press Esc to Cancel)")
        self.root.attributes('-alpha', 0.3)  # Transparency
        self.root.attributes("-topmost", True)
        self.root.overrideredirect(True)  # Remove window decorations
    
        screen_width = self.root.winfo_screenwidth()
        screen_height = self.root.winfo_screenheight()
        self.root.geometry(f"{screen_width}x{screen_height}")
    
        self.canvas = tk.Canvas(self.root, cursor="cross", bg="grey")
        self.canvas.pack(fill=tk.BOTH, expand=True)
    
        self.canvas.bind("<ButtonPress-1>", self.on_button_press)
        self.canvas.bind("<B1-Motion>", self.on_move_press)
        self.canvas.bind("<ButtonRelease-1>", self.on_button_release)
        self.root.bind("<Escape>", self.on_escape)
    
        logging.info("Please select the region by clicking and dragging the mouse.")
    
    def on_button_press(self, event):
        self.start_x = int(event.x * self.scaling_factor)
        self.start_y = int(event.y * self.scaling_factor)
        self.rect = self.canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
        logging.debug(f"Button pressed at ({self.start_x}, {self.start_y})")
    
    def on_move_press(self, event):
        self.end_x = int(event.x * self.scaling_factor)
        self.end_y = int(event.y * self.scaling_factor)
        self.canvas.coords(self.rect, 
                           self.start_x / self.scaling_factor, 
                           self.start_y / self.scaling_factor, 
                           event.x, 
                           event.y)
        logging.debug(f"Mouse moved to ({self.end_x}, {self.end_y})")
    
    def on_button_release(self, event):
        self.end_x = int(event.x * self.scaling_factor)
        self.end_y = int(event.y * self.scaling_factor)
        self.selection_complete = True
        self.root.quit()  # Exit the main loop
        logging.debug(f"Button released at ({self.end_x}, {self.end_y})")
    
    def on_escape(self, event):
        logging.info("Capture cancelled by user.")
        self.root.quit()
        self.root.destroy()
        sys.exit()
    
    def get_region(self):
        self.root.mainloop()
        self.root.destroy()
        left = min(self.start_x, self.end_x)
        top = min(self.start_y, self.end_y)
        right = max(self.start_x, self.end_x)
        bottom = max(self.start_y, self.end_y)
        logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}")
        return (left, top, right, bottom)
    

    -------------------- Screen Capture and Preprocessing --------------------

    def capture_screen_region(region): """ Captures the specified screen region using mss.

    Args:
        region (tuple): A tuple of (left, top, right, bottom) coordinates.
    
    Returns:
        PIL.Image: The captured image.
    """
    left, top, right, bottom = region
    width = right - left
    height = bottom - top
    if width <= 0 or height <= 0:
        logging.error("Invalid region dimensions. Width and height must be positive.")
        return None
    
    with mss.mss() as sct:
        monitor = {
            "top": top,
            "left": left,
            "width": width,
            "height": height
        }
        try:
            screenshot = sct.grab(monitor)
            img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
            logging.info(f"Screen region captured: {width}x{height} pixels.")
            return img
        except Exception as e:
            logging.error(f"Screen capture failed: {e}")
            return None
    

    def preprocess_image(image): """ Preprocesses the image for better OCR results.

    Steps:
        1. Convert to grayscale.
        2. Enhance contrast.
        3. Apply thresholding.
        4. Reduce noise with a median filter.
    
    Args:
        image (PIL.Image): The original captured image.
    
    Returns:
        PIL.Image: The preprocessed image.
    """
    try:
        logging.info("Starting image preprocessing...")
    
        # Step 1: Convert to grayscale
        gray = image.convert('L')
        logging.debug("Converted image to grayscale.")
    
        # Step 2: Enhance contrast
        enhancer = ImageEnhance.Contrast(gray)
        gray_enhanced = enhancer.enhance(2.0)  # Adjust contrast factor as needed
        logging.debug("Enhanced image contrast.")
    
        # Step 3: Apply thresholding
        threshold = 140  # Adjust based on testing
        gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
        logging.debug("Applied thresholding to binarize the image.")
    
        # Step 4: Reduce noise with a median filter
        final_image = gray_threshold.filter(ImageFilter.MedianFilter())
        logging.debug("Applied median filter to reduce noise.")
    
        logging.info("Image preprocessing complete.")
        return final_image
    except Exception as e:
        logging.error(f"Image preprocessing failed: {e}")
        return None
    

    -------------------- Saving Images --------------------

    def save_image(image, filename): """ Saves the given image to the specified filename.

    Args:
        image (PIL.Image): The image to save.
        filename (str): The filename to save the image as.
    """
    try:
        image.save(filename)
        logging.info(f"Image saved as {filename}")
    except Exception as e:
        logging.error(f"Failed to save image {filename}: {e}")
    

    -------------------- Main Function --------------------

    def main(): # Step 1: Set DPI awareness and get scaling factor set_dpi_awareness() scaling_factor = get_scaling_factor() logging.info(f"Using DPI scaling factor: {scaling_factor}")

    # Step 2: Initialize region selector
    selector = RegionSelector(scaling_factor)
    
    # Step 3: Get selected region
    region = selector.get_region()
    left, top, right, bottom = region
    width = right - left
    height = bottom - top
    if width <= 0 or height <= 0:
        logging.error("Invalid region selected. Width and height must be positive.")
        sys.exit(1)
    
    # Step 4: Capture the selected screen region
    captured_image = capture_screen_region(region)
    if captured_image is None:
        logging.error("Failed to capture the selected screen region.")
        sys.exit(1)
    
    # Step 5: Save the captured image for verification
    timestamp = int(time.time())
    original_filename = f"captured_region_{timestamp}.png"
    save_image(captured_image, original_filename)
    
    # Step 6: Preprocess the captured image
    preprocessed_image = preprocess_image(captured_image)
    if preprocessed_image is None:
        logging.error("Image preprocessing failed.")
        sys.exit(1)
    
    # Step 7: Save the preprocessed image for verification
    preprocessed_filename = f"preprocessed_image_{timestamp}.png"
    save_image(preprocessed_image, preprocessed_filename)
    
    logging.info("Troubleshooting completed successfully.")
    logging.info(f"Original Image: {original_filename}")
    logging.info(f"Preprocessed Image: {preprocessed_filename}")
    

    if name == "main": try: main() except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: logging.info("Exiting the OCR Troubleshooting Script.") AND import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import time import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama

    -------------------- Configuration --------------------

    Configure logging to capture DEBUG level logs

    logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )

    Initialize the TTS engine globally

    tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start

    Define global locks

    lock = threading.Lock() tts_lock = threading.Lock()

    -------------------- Preferred Sites Configuration --------------------

    Editable list of preferred sites for prioritized search

    PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]

    def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")

    def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0

    def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text

    def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found

    def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)

            # Add hotkeys for pausing (F4) and stopping (F5)
            keyboard.add_hotkey('f4', pause_speaking)
            keyboard.add_hotkey('f5', stop_speaking)
    
            tts_engine.runAndWait()
            logging.info("Text-to-Speech completed.")
        except RuntimeError as e:
            logging.error(f"TTS Error: {e}")
        finally:
            # Remove hotkeys after completion to avoid conflicts
            keyboard.remove_hotkey('f4')
            keyboard.remove_hotkey('f5')
    

    def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()

    def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()

    def save_captured_image(image, filename="captured_region.png"): """ Saves the captured image for debugging purposes. """ try: image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")

    def save_preprocessed_image(image, filename="preprocessed_image.png"): """ Saves the preprocessed image for debugging purposes. """ try: image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")

    def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")

        # Step 1: Save the original captured image for verification
        save_captured_image(image)
    
        # Step 2: Convert to grayscale
        gray = image.convert('L')
    
        # Step 3: Enhance contrast
        enhancer = ImageEnhance.Contrast(gray)
        gray_enhanced = enhancer.enhance(10)
    
        # Step 4: Apply thresholding
        threshold = 140  # Adjust based on testing
        gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
    
        # Step 5: Reduce noise with a median filter
        final_image = gray_threshold.filter(ImageFilter.MedianFilter())
    
        # Step 6: Save preprocessed image for debugging
        save_preprocessed_image(final_image)
    
        logging.info("Image preprocessing complete. Performing OCR...")
    
        # Define Tesseract configuration
        custom_config = r'--oem 3 --psm 6'  # Adjust as needed
    
        # Perform OCR
        raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config)
    
        if not raw_text.strip():
            logging.warning("OCR returned empty text.")
            speak_text("I couldn't detect any text in the selected area. Please try again with a different region.")
        else:
            ocr_text = raw_text
            logging.debug("Raw OCR Text:")
            logging.debug(ocr_text)
            logging.info("OCR process completed successfully.")
    except pytesseract.TesseractNotFoundError:
        logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.")
        speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.")
        ocr_text = ""
    except Exception as e:
        logging.error(f"OCR Error: {e}")
        speak_text("An error occurred during OCR processing. Please try again.")
        ocr_text = ""
    

    def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations

    screen_width = root.winfo_screenwidth()
    screen_height = root.winfo_screenheight()
    root.geometry(f"{screen_width}x{screen_height}")
    
    canvas = tk.Canvas(root, cursor="cross", bg="grey")
    canvas.pack(fill=tk.BOTH, expand=True)
    
    rect = None
    start_x = start_y = end_x = end_y = 0
    
    def on_button_press(event):
        nonlocal start_x, start_y, rect
        start_x = int(event.x * scaling_factor)
        start_y = int(event.y * scaling_factor)
        rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
        logging.debug(f"Button pressed at ({start_x}, {start_y})")
    
    def on_move_press(event):
        nonlocal end_x, end_y
        end_x = int(event.x * scaling_factor)
        end_y = int(event.y * scaling_factor)
        canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y)
        logging.debug(f"Mouse moved to ({end_x}, {end_y})")
    
    def on_button_release(event):
        root.quit()
        root.destroy()  # Destroy the window to remove the overlay
    
    def on_escape(event):
        logging.info("Capture cancelled by user.")
        root.quit()
        root.destroy()
        sys.exit()
    
    canvas.bind("<ButtonPress-1>", on_button_press)
    canvas.bind("<B1-Motion>", on_move_press)
    canvas.bind("<ButtonRelease-1>", on_button_release)
    root.bind("<Escape>", on_escape)
    
    logging.info("Please select the region by clicking and dragging the mouse.")
    root.mainloop()
    
    # Calculate absolute coordinates
    left = min(start_x, end_x)
    top = min(start_y, end_y)
    right = max(start_x, end_x)
    bottom = max(start_y, end_y)
    
    logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}")
    return (left, top, right, bottom)
    

    -------------------- Modified Web Search Function --------------------

    async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.

    Args:
        query (str): The search query.
    
    Returns:
        list: A list of dictionaries containing 'title' and 'snippet' for each result.
    """
    logging.info(f"Searching the web for query: {query}")
    
    # Construct the site-limited query
    if PREFERRED_SITES:
        site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES])
        combined_query = f"{query} ({site_query})"
    else:
        combined_query = query  # If no preferred sites, perform a general search
    
    logging.debug(f"Combined search query with preferred sites: {combined_query}")
    
    # Encode the query for URL
    encoded_query = aiohttp.helpers.quote(combined_query)
    url = f"https://html.duckduckgo.com/html/?q={encoded_query}"
    
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                html = await response.text()
    except Exception as e:
        logging.error(f"Web search failed: {e}")
        return []
    
    soup = BeautifulSoup(html, 'html.parser')
    results = []
    
    # DuckDuckGo's HTML structure can vary; adjust selectors as needed
    for result in soup.find_all('div', class_='result__body', limit=3):
        title_element = result.find('a', class_='result__a')
        snippet_element = result.find('a', class_='result__snippet')
        title = title_element.get_text(strip=True) if title_element else "No Title"
        snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet"
        results.append({'title': title, 'snippet': snippet})
        logging.debug(f"Found result - Title: {title}, Snippet: {snippet}")
    
    if not results:
        logging.warning("No search results found for the preferred sites.")
    
    logging.info(f"Web search complete. Found {len(results)} results.")
    return results
    

    def generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])

    if user_input:
        context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.
    

    OCR Text: {ocr_text}

    User Query: {user_input}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text and the user's query.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response to the user's question based on the available information.
    4. Ensure that your answer directly addresses the user's intent.

    Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.

    OCR Text: {ocr_text}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response based on the available information.
    4. Ensure that your answer directly reflects the content of the OCR text and the search results.

    Provide the answer below: """

    try:
        response = ollama.generate(model='llama3.2', prompt=context)
        final_response = response.get('response', '').strip()
        logging.info(f"Generated Final Response: {final_response}")
        return final_response
    except Exception as e:
        logging.error(f"LLM Error during final response generation: {e}")
        return ""
    

    def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""

    -------------------- Helper Functions --------------------

    def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.

    Args:
        short_timeout (int): Maximum number of seconds to wait for a phrase to start.
        phrase_time_limit (int): Maximum number of seconds a phrase can last.
    """
    global is_listening, speech_text
    recognizer = sr.Recognizer()
    with sr.Microphone() as source:
        logging.info("Listening for voice input... Please say your query or question.")
        recognizer.adjust_for_ambient_noise(source, duration=1)  # Adjust for noise for 1 second
        try:
            # Listen for speech with shorter timeout and phrase time limit
            audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit)
            logging.info("Processing the audio with Google Speech Recognition...")
            text = recognizer.recognize_google(audio)
            # Convert number words to digits if necessary
            speech_text = convert_number_words(text)
            logging.info(f"Speech Transcribed: {speech_text}")
    
            # Provide confirmation
            speak_text(f"I heard: {speech_text}.")
        except sr.WaitTimeoutError:
            logging.warning("Listening timed out, no speech detected. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.UnknownValueError:
            logging.warning("Speech recognition could not understand audio. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.RequestError as e:
            logging.error(f"Could not request results from speech recognition service; {e}")
            speech_text = ""  # Ensure speech_text is empty
        finally:
            with lock:
                is_listening = False  # Ensure listening flag is reset
    

    def configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')

    # List available voices (optional)
    for index, voice in enumerate(voices):
        logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}")
    
    # Change voice (select male/female, adjust the index as per your preference)
    if len(voices) > 1:
        tts_engine.setProperty('voice', voices[1].id)  # Change index to select a different voice
        logging.info(f"TTS voice set to: {voices[1].name}")
    else:
        logging.info("Only one voice available. Using default voice.")
    
    # Adjust speaking rate (lower is slower and more human-like)
    tts_engine.setProperty('rate', 160)  # Default is 200, lowering makes it more human-like
    
    # Adjust volume (0.0 to 1.0)
    tts_engine.setProperty('volume', 1.0)
    

    def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return

    # Step 2: Capture the selected region
    with mss.mss() as sct:
        monitor = {
            "top": top,
            "left": left,
            "width": width,
            "height": height
        }
        try:
            screenshot = sct.grab(monitor)
            img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
        except Exception as e:
            logging.error(f"Screen capture failed: {e}")
            speak_text("Screen capture failed. Please ensure the selected area is correct and try again.")
            reset_state()
            return
    
    # Step 3: Perform OCR on the captured region
    perform_ocr(img)
    if not ocr_text.strip():
        logging.error("OCR returned empty text. Please ensure the selected region contains readable text.")
        speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.")
        reset_state()
        return
    
    # Step 4: Listen for user input (optional) with shorter timeout
    transcribe_speech(short_timeout=5, phrase_time_limit=5)
    
    # Step 5: Generate and execute the search query
    generate_and_execute_search(speech_text, ocr_text)
    

    def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return

    logging.info(f"Using OCR text as search query: {search_query}")
    
    # Perform web search and process the answer
    asyncio.run(async_process_search(search_query, ocr_text, user_input))
    

    async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return

    # Generate final response using LLM
    final_response = generate_final_response(search_results, ocr_text, user_input)
    if not final_response:
        logging.error("Failed to generate a final response.")
        speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.")
        reset_state()
        return
    
    # Speak out the answer
    speak_text(final_response)
    reset_state()
    

    -------------------- Core Functions --------------------

    def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()

    def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")

    -------------------- Main Function --------------------

    async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor

    configure_tts_voice()  # Voice configuration happens before anything else
    
    # Set DPI awareness and get scaling factor
    set_dpi_awareness()
    scaling_factor = get_scaling_factor()
    
    # Set up hotkeys
    keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,))
    keyboard.add_hotkey('f3', stop_listening)
    
    logging.info("QA Assistant is running.")
    logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.")
    
    while True:
        # Wait for a short duration to avoid busy waiting
        await asyncio.sleep(0.1)
    
        # Check for exit condition
        if keyboard.is_pressed('esc'):
            logging.info("Exiting...")
            speak_text("Exiting the QA Assistant. Goodbye!")
            break
    
        # Reset speech_text and ocr_text is handled after processing
    
    # Clean up keyboard hooks before exiting
    keyboard.unhook_all()
    

    -------------------- Entry Point --------------------

    if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all()

  4. Assistant

    The two scripts you provided have different purposes and functionalities, with some overlapping components related to screen region capture and image preprocessing. Here's a breakdown of their differences and similarities:

    Similarities:

    1. DPI Awareness and Scaling: Both scripts include functionality for setting the DPI awareness and getting the scaling factor. This is important for handling display scaling on Windows systems.

    2. Screen Region Selection: Both scripts allow the user to select a region of the screen and capture it. They use tkinter to provide a graphical interface for this selection.

    3. Image Preprocessing: Both scripts perform image preprocessing steps, including converting the image to grayscale and enhancing contrast. They also apply thresholding and noise reduction, which are common preprocessing steps for improving OCR accuracy.

    4. Logging: Both scripts make extensive use of Python's logging module to capture and output debug information.

    Differences:

    • Functionality and Purpose:

      • First Script (Troubleshooting Script): This script primarily focuses on capturing the screen region and preprocessing the image in a standalone manner. It is designed to save and verify the original and preprocessed images as separate files. The user can use it to troubleshoot the capture and preprocessing steps by examining the saved images.
      • Second Script (QA Assistant): This script integrates additional functionalities like OCR using Tesseract, web searches using DuckDuckGo, and text-to-speech (TTS) responses. It listens for user queries via voice input and attempts to answer questions by combining OCR results with online search data.
    • Modules and Features:

      • The troubleshooting script does not include OCR processing, web search, or voice recognition/response functionality.
      • The QA Assistant incorporates several additional modules such as pytesseract for OCR, speech_recognition for voice input, pyttsx3 for TTS, and aiohttp and BeautifulSoup for web scraping. It aims to create an interactive user experience encompassing image capture, OCR, search, and audio feedback.
    • Complexity:

      • The QA Assistant script is significantly more complex due to its additional functionality for processing spoken input and generating spoken responses, making use of threading, asyncio, and various interactions between components.

    In summary, while both scripts share common elements related to capturing and preprocessing screen images, their goals are different. The first script is a simpler, more focused utility for troubleshooting image capture and preprocessing, whereas the second script is a comprehensive interactive assistant that builds on those capabilities with added functionalities.

  5. User

    What i want to know is that is the functionality of trouble shooting script in taking ocr is same as the main QA script or is there some difference ?

  6. Assistant

    The functionality related to taking an OCR in the troubleshooting script and the QA Assistant script is fundamentally similar but serves different purposes, particularly concerning the integration with additional features in the QA script. Here's a closer look at each aspect to understand the differences:

    Similarities:

    1. Image Capture:

      • Both scripts allow users to select and capture a specific screen region using tkinter for the GUI and mss for screen capturing. This functionality is largely the same in both scripts.
    2. Image Preprocessing:

      • Both scripts perform similar preprocessing steps on the captured image prior to OCR. These steps include converting the image to grayscale, enhancing contrast, applying thresholding, and reducing noise with a median filter.

    Differences:

    1. Overall Integration with OCR:

      • Troubleshooting Script: This script focuses purely on capturing and preprocessing the image. It stops short of performing OCR, instead emphasizing saving the preprocessed image to a file for manual inspection. This allows you to verify the preprocessing step to ensure it produces a usable image for OCR.
      • QA Assistant Script: After preprocessing, this script completes the OCR process using pytesseract. It then uses the recognized text to perform further actions, such as conducting a web search or generating responses using a language model.
    2. Purpose and Final Output:

      • The troubleshooting script’s primary goal is to aid in debugging the image capture and preprocessing pipeline by manually checking the preprocessed images.
      • The QA script’s goal is to automate the process of interpreting and using OCR results to provide answers, illustrating a complete workflow from image capture to user query response.
    3. Complexity and Integration:

      • The troubleshooting script is simpler and modular, designed to address errors or uncertainties at the preprocessing stage.
      • The QA Assistant script integrates with additional components (e.g., TTS, web searching, LLM response generation) for a seamless user interaction, with more dependencies and interactions between these components.

    In conclusion, both scripts share the core functionality of image capture and preprocessing, but the troubleshooting script isolates these operations for focused examination, while the QA Assistant script incorporates these as part of a larger interactive workflow. Therefore, the basic functionality for performing an OCR preparation is similar, yet the context and utilization of the OCR results differ significantly.

  7. User

    import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import time import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama

    -------------------- Configuration --------------------

    Configure logging to capture DEBUG level logs

    logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )

    Initialize the TTS engine globally

    tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start

    Define global locks

    lock = threading.Lock() tts_lock = threading.Lock()

    -------------------- Preferred Sites Configuration --------------------

    Editable list of preferred sites for prioritized search

    PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]

    def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")

    def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0

    def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text

    def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found

    def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)

            # Add hotkeys for pausing (F4) and stopping (F5)
            keyboard.add_hotkey('f4', pause_speaking)
            keyboard.add_hotkey('f5', stop_speaking)
    
            tts_engine.runAndWait()
            logging.info("Text-to-Speech completed.")
        except RuntimeError as e:
            logging.error(f"TTS Error: {e}")
        finally:
            # Remove hotkeys after completion to avoid conflicts
            keyboard.remove_hotkey('f4')
            keyboard.remove_hotkey('f5')
    

    def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()

    def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()

    def save_captured_image(image, filename="captured_region.png"): """ Saves the captured image for debugging purposes. """ try: image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")

    def save_preprocessed_image(image, filename="preprocessed_image.png"): """ Saves the preprocessed image for debugging purposes. """ try: image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")

    def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")

        # Step 1: Save the original captured image for verification
        save_captured_image(image)
    
        # Step 2: Convert to grayscale
        gray = image.convert('L')
    
        # Step 3: Enhance contrast
        enhancer = ImageEnhance.Contrast(gray)
        gray_enhanced = enhancer.enhance(2)
    
        # Step 4: Apply thresholding
        threshold = 140  # Adjust based on testing
        gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
    
        # Step 5: Reduce noise with a median filter
        final_image = gray_threshold.filter(ImageFilter.MedianFilter())
    
        # Step 6: Save preprocessed image for debugging
        save_preprocessed_image(final_image)
    
        logging.info("Image preprocessing complete. Performing OCR...")
    
        # Define Tesseract configuration
        #custom_config = r'--oem 3 --psm 6'  # Adjust as needed
    
        # Perform OCR
        raw_text = pytesseract.image_to_string(final_image, lang='eng')
    
        if not raw_text.strip():
            logging.warning("OCR returned empty text.")
            speak_text("I couldn't detect any text in the selected area. Please try again with a different region.")
        else:
            ocr_text = raw_text
            logging.debug("Raw OCR Text:")
            logging.debug(ocr_text)
            logging.info("OCR process completed successfully.")
    except pytesseract.TesseractNotFoundError:
        logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.")
        speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.")
        ocr_text = ""
    except Exception as e:
        logging.error(f"OCR Error: {e}")
        speak_text("An error occurred during OCR processing. Please try again.")
        ocr_text = ""
    

    def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations

    screen_width = root.winfo_screenwidth()
    screen_height = root.winfo_screenheight()
    root.geometry(f"{screen_width}x{screen_height}")
    
    canvas = tk.Canvas(root, cursor="cross", bg="grey")
    canvas.pack(fill=tk.BOTH, expand=True)
    
    rect = None
    start_x = start_y = end_x = end_y = 0
    
    def on_button_press(event):
        nonlocal start_x, start_y, rect
        start_x = int(event.x * scaling_factor)
        start_y = int(event.y * scaling_factor)
        rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
        logging.debug(f"Button pressed at ({start_x}, {start_y})")
    
    def on_move_press(event):
        nonlocal end_x, end_y
        end_x = int(event.x * scaling_factor)
        end_y = int(event.y * scaling_factor)
        canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y)
        logging.debug(f"Mouse moved to ({end_x}, {end_y})")
    
    def on_button_release(event):
        root.quit()
        root.destroy()  # Destroy the window to remove the overlay
    
    def on_escape(event):
        logging.info("Capture cancelled by user.")
        root.quit()
        root.destroy()
        sys.exit()
    
    canvas.bind("<ButtonPress-1>", on_button_press)
    canvas.bind("<B1-Motion>", on_move_press)
    canvas.bind("<ButtonRelease-1>", on_button_release)
    root.bind("<Escape>", on_escape)
    
    logging.info("Please select the region by clicking and dragging the mouse.")
    root.mainloop()
    
    # Calculate absolute coordinates
    left = min(start_x, end_x)
    top = min(start_y, end_y)
    right = max(start_x, end_x)
    bottom = max(start_y, end_y)
    
    logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}")
    return (left, top, right, bottom)
    

    -------------------- Modified Web Search Function --------------------

    async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.

    Args:
        query (str): The search query.
    
    Returns:
        list: A list of dictionaries containing 'title' and 'snippet' for each result.
    """
    logging.info(f"Searching the web for query: {query}")
    
    # Construct the site-limited query
    if PREFERRED_SITES:
        site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES])
        combined_query = f"{query} ({site_query})"
    else:
        combined_query = query  # If no preferred sites, perform a general search
    
    logging.debug(f"Combined search query with preferred sites: {combined_query}")
    
    # Encode the query for URL
    encoded_query = aiohttp.helpers.quote(combined_query)
    url = f"https://html.duckduckgo.com/html/?q={encoded_query}"
    
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                html = await response.text()
    except Exception as e:
        logging.error(f"Web search failed: {e}")
        return []
    
    soup = BeautifulSoup(html, 'html.parser')
    results = []
    
    # DuckDuckGo's HTML structure can vary; adjust selectors as needed
    for result in soup.find_all('div', class_='result__body', limit=3):
        title_element = result.find('a', class_='result__a')
        snippet_element = result.find('a', class_='result__snippet')
        title = title_element.get_text(strip=True) if title_element else "No Title"
        snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet"
        results.append({'title': title, 'snippet': snippet})
        logging.debug(f"Found result - Title: {title}, Snippet: {snippet}")
    
    if not results:
        logging.warning("No search results found for the preferred sites.")
    
    logging.info(f"Web search complete. Found {len(results)} results.")
    return results
    

    def generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])

    if user_input:
        context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.
    

    OCR Text: {ocr_text}

    User Query: {user_input}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text and the user's query.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response to the user's question based on the available information.
    4. Ensure that your answer directly addresses the user's intent.

    Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.

    OCR Text: {ocr_text}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response based on the available information.
    4. Ensure that your answer directly reflects the content of the OCR text and the search results.

    Provide the answer below: """

    try:
        response = ollama.generate(model='llama3.2', prompt=context)
        final_response = response.get('response', '').strip()
        logging.info(f"Generated Final Response: {final_response}")
        return final_response
    except Exception as e:
        logging.error(f"LLM Error during final response generation: {e}")
        return ""
    

    def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""

    -------------------- Helper Functions --------------------

    def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.

    Args:
        short_timeout (int): Maximum number of seconds to wait for a phrase to start.
        phrase_time_limit (int): Maximum number of seconds a phrase can last.
    """
    global is_listening, speech_text
    recognizer = sr.Recognizer()
    with sr.Microphone() as source:
        logging.info("Listening for voice input... Please say your query or question.")
        recognizer.adjust_for_ambient_noise(source, duration=1)  # Adjust for noise for 1 second
        try:
            # Listen for speech with shorter timeout and phrase time limit
            audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit)
            logging.info("Processing the audio with Google Speech Recognition...")
            text = recognizer.recognize_google(audio)
            # Convert number words to digits if necessary
            speech_text = convert_number_words(text)
            logging.info(f"Speech Transcribed: {speech_text}")
    
            # Provide confirmation
            speak_text(f"I heard: {speech_text}.")
        except sr.WaitTimeoutError:
            logging.warning("Listening timed out, no speech detected. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.UnknownValueError:
            logging.warning("Speech recognition could not understand audio. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.RequestError as e:
            logging.error(f"Could not request results from speech recognition service; {e}")
            speech_text = ""  # Ensure speech_text is empty
        finally:
            with lock:
                is_listening = False  # Ensure listening flag is reset
    

    def configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')

    # List available voices (optional)
    for index, voice in enumerate(voices):
        logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}")
    
    # Change voice (select male/female, adjust the index as per your preference)
    if len(voices) > 1:
        tts_engine.setProperty('voice', voices[1].id)  # Change index to select a different voice
        logging.info(f"TTS voice set to: {voices[1].name}")
    else:
        logging.info("Only one voice available. Using default voice.")
    
    # Adjust speaking rate (lower is slower and more human-like)
    tts_engine.setProperty('rate', 160)  # Default is 200, lowering makes it more human-like
    
    # Adjust volume (0.0 to 1.0)
    tts_engine.setProperty('volume', 1.0)
    

    def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return

    # Step 2: Capture the selected region
    with mss.mss() as sct:
        monitor = {
            "top": top,
            "left": left,
            "width": width,
            "height": height
        }
        try:
            screenshot = sct.grab(monitor)
            img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
        except Exception as e:
            logging.error(f"Screen capture failed: {e}")
            speak_text("Screen capture failed. Please ensure the selected area is correct and try again.")
            reset_state()
            return
    
    # Step 3: Perform OCR on the captured region
    perform_ocr(img)
    if not ocr_text.strip():
        logging.error("OCR returned empty text. Please ensure the selected region contains readable text.")
        speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.")
        reset_state()
        return
    
    # Step 4: Listen for user input (optional) with shorter timeout
    transcribe_speech(short_timeout=5, phrase_time_limit=5)
    
    # Step 5: Generate and execute the search query
    generate_and_execute_search(speech_text, ocr_text)
    

    def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return

    logging.info(f"Using OCR text as search query: {search_query}")
    
    # Perform web search and process the answer
    asyncio.run(async_process_search(search_query, ocr_text, user_input))
    

    async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return

    # Generate final response using LLM
    final_response = generate_final_response(search_results, ocr_text, user_input)
    if not final_response:
        logging.error("Failed to generate a final response.")
        speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.")
        reset_state()
        return
    
    # Speak out the answer
    speak_text(final_response)
    reset_state()
    

    -------------------- Core Functions --------------------

    def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()

    def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")

    -------------------- Main Function --------------------

    async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor

    configure_tts_voice()  # Voice configuration happens before anything else
    
    # Set DPI awareness and get scaling factor
    set_dpi_awareness()
    scaling_factor = get_scaling_factor()
    
    # Set up hotkeys
    keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,))
    keyboard.add_hotkey('f3', stop_listening)
    
    logging.info("QA Assistant is running.")
    logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.")
    
    while True:
        # Wait for a short duration to avoid busy waiting
        await asyncio.sleep(0.1)
    
        # Check for exit condition
        if keyboard.is_pressed('esc'):
            logging.info("Exiting...")
            speak_text("Exiting the QA Assistant. Goodbye!")
            break
    
        # Reset speech_text and ocr_text is handled after processing
    
    # Clean up keyboard hooks before exiting
    keyboard.unhook_all()
    

    -------------------- Entry Point --------------------

    if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all() : Add one part where it's outputting the images just before they are fed to ocr to path D:\AI\EXP2\TempS

  8. Assistant

    To modify your script so that it saves images just before they are fed to the OCR process, you can add functionality for saving the preprocessed images to your specified directory (D:\AI\EXP2\TempS). I've inserted the necessary changes into your script below:

    import os
    import mss
    from PIL import Image, ImageEnhance, ImageFilter
    import pytesseract
    import speech_recognition as sr
    import pyttsx3
    import asyncio
    import aiohttp
    from bs4 import BeautifulSoup
    import keyboard
    import threading
    import time
    import re
    import logging
    import tkinter as tk
    from word2number import w2n  # Optional for number word conversion
    import ctypes
    import sys
    import ollama
    
    # -------------------- Configuration --------------------
    
    # Configure logging to capture DEBUG level logs
    logging.basicConfig(
        level=logging.DEBUG,  # Set to DEBUG for detailed logs
        format='%(asctime)s [%(levelname)s] %(message)s',
        handlers=[
            logging.StreamHandler(),  # Outputs to console
            # Uncomment the next line to enable logging to a file
            # logging.FileHandler("qa_assistant.log"),
        ]
    )
    
    # Initialize the TTS engine globally
    tts_engine = pyttsx3.init()
    is_listening = False  # Initialize this global variable at the start
    speech_text = ""  # Initialize this global variable at the start
    
    # Define global locks
    lock = threading.Lock()
    tts_lock = threading.Lock()
    
    # -------------------- Preferred Sites Configuration --------------------
    
    # Editable list of preferred sites for prioritized search
    PREFERRED_SITES = [
        "civiljungle.org",
        "www.nwcg.gov",
        "the.constructor.org",
        "definecivil.com",
        "testbook.com/civil-engineering",
        "sanfoundry.com",
        # Add or remove sites as needed
    ]
    
    # Directory to save images
    IMAGE_SAVE_DIRECTORY = r"D:\AI\EXP2\TempS"
    
    def save_image(image, filename):
        """
        Saves the given image to the specified filename.
        """
        try:
            if not os.path.exists(IMAGE_SAVE_DIRECTORY):
                os.makedirs(IMAGE_SAVE_DIRECTORY)
            
            filepath = os.path.join(IMAGE_SAVE_DIRECTORY, filename)
            image.save(filepath)
            logging.info(f"Image saved as {filepath}")
        except Exception as e:
            logging.error(f"Failed to save image {filename}: {e}")
    
    def set_dpi_awareness():
        """
        Sets the DPI awareness of the process to prevent scaling issues.
        """
        try:
            if sys.platform.startswith('win'):
                awareness = ctypes.c_int()
                # PROCESS_SYSTEM_DPI_AWARE = 1
                ctypes.windll.shcore.SetProcessDpiAwareness(1)
                logging.info("Process DPI awareness set to SYSTEM_AWARE.")
        except Exception as e:
            logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
    
    def get_scaling_factor():
        """
        Retrieves the current DPI scaling factor.
        Returns:
            float: Scaling factor (e.g., 1.25 for 125%)
        """
        try:
            if sys.platform.startswith('win'):
                user32 = ctypes.windll.user32
                try:
                    # Windows 10 and above
                    user32.SetProcessDPIAware()
                    dpi = user32.GetDpiForWindow(user32.GetForegroundWindow())
                    scaling_factor = dpi / 96  # 96 DPI is 100%
                    logging.info(f"DPI Scaling Factor detected: {scaling_factor}")
                    return scaling_factor
                except AttributeError:
                    # For older Windows versions
                    user32.SetProcessDPIAware()
                    hdc = user32.GetDC(0)
                    LOGPIXELSX = 88
                    dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX)
                    scaling_factor = dpi / 96
                    ctypes.windll.user32.ReleaseDC(0, hdc)
                    logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}")
                    return scaling_factor
            else:
                # For other operating systems, default to 1.0
                logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.")
                return 1.0
        except Exception as e:
            logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.")
            return 1.0
    
    def perform_ocr(image):
        """
        Performs OCR on the captured image after preprocessing.
        Uses the preprocessed image for better accuracy.
        """
        global ocr_text
        try:
            logging.info("Starting OCR process with image preprocessing...")
    
            # Step 1: Save the original captured image for verification
            save_image(image, "captured_region.png")
    
            # Step 2: Convert to grayscale
            gray = image.convert('L')
    
            # Step 3: Enhance contrast
            enhancer = ImageEnhance.Contrast(gray)
            gray_enhanced = enhancer.enhance(2)
    
            # Step 4: Apply thresholding
            threshold = 140  # Adjust based on testing
            gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
    
            # Step 5: Reduce noise with a median filter
            final_image = gray_threshold.filter(ImageFilter.MedianFilter())
    
            # Step 6: Save preprocessed image for debugging
            save_image(final_image, "preprocessed_image.png")
    
            logging.info("Image preprocessing complete. Performing OCR...")
    
            # Define Tesseract configuration
            #custom_config = r'--oem 3 --psm 6'  # Adjust as needed
    
            # Perform OCR
            raw_text = pytesseract.image_to_string(final_image, lang='eng')
    
            if not raw_text.strip():
                logging.warning("OCR returned empty text.")
                speak_text("I couldn't detect any text in the selected area. Please try again with a different region.")
            else:
                ocr_text = raw_text
                logging.debug("Raw OCR Text:")
                logging.debug(ocr_text)
                logging.info("OCR process completed successfully.")
        except pytesseract.TesseractNotFoundError:
            logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.")
            speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.")
            ocr_text = ""
        except Exception as e:
            logging.error(f"OCR Error: {e}")
            speak_text("An error occurred during OCR processing. Please try again.")
            ocr_text = ""
    
    # The remaining part of the script continues as before...
    

    Changes Made:

    • Directory Definition: I introduced a constant IMAGE_SAVE_DIRECTORY to specify where to save the images.
    • Image Saving Function: Modified the save_image function to ensure your preprocessed images and captured regions correctly save to your specified path. It checks if the directory exists and creates it if it doesn't.
    • Function Calls: Incorporated calls to save_image after each preprocessing step to save the original captured image and the preprocessed image right before they are sent for OCR processing.

    These changes ensure that you have a visual record of both the original and preprocessed images, which can help identify any issues.

  9. User

    It is doing incorrect screen capture that has offset from what was selected using mouseimport mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import os import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama from datetime import datetime

    -------------------- Configuration --------------------

    Configure logging to capture DEBUG level logs

    logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )

    Initialize the TTS engine globally

    tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start

    Define global locks

    lock = threading.Lock() tts_lock = threading.Lock()

    -------------------- Preferred Sites Configuration --------------------

    Editable list of preferred sites for prioritized search

    PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]

    Define the output directory for saved images

    OUTPUT_DIR = (r"D:\AI\EXP2\TempS")

    def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")

    def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0

    def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text

    def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found

    def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)

            # Add hotkeys for pausing (F4) and stopping (F5)
            keyboard.add_hotkey('f4', pause_speaking)
            keyboard.add_hotkey('f5', stop_speaking)
    
            tts_engine.runAndWait()
            logging.info("Text-to-Speech completed.")
        except RuntimeError as e:
            logging.error(f"TTS Error: {e}")
        finally:
            # Remove hotkeys after completion to avoid conflicts
            keyboard.remove_hotkey('f4')
            keyboard.remove_hotkey('f5')
    

    def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()

    def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()

    def save_captured_image(image, output_dir, prefix="captured_region"): """ Saves the captured image for debugging purposes with a timestamp.

    Args:
        image (PIL.Image): The image to save.
        output_dir (str): Directory where the image will be saved.
        prefix (str): Prefix for the filename.
    """
    try:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png")
        image.save(filename)
        logging.info(f"Captured image saved as {filename}")
    except Exception as e:
        logging.error(f"Failed to save captured image: {e}")
    

    def save_preprocessed_image(image, output_dir, prefix="preprocessed_image"): """ Saves the preprocessed image for debugging purposes with a timestamp.

    Args:
        image (PIL.Image): The image to save.
        output_dir (str): Directory where the image will be saved.
        prefix (str): Prefix for the filename.
    """
    try:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png")
        image.save(filename)
        logging.info(f"Preprocessed image saved as {filename}")
    except Exception as e:
        logging.error(f"Failed to save preprocessed image: {e}")
    

    def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")

        # Step 1: Save the original captured image for verification
        save_captured_image(image, OUTPUT_DIR)
    
        # Step 2: Convert to grayscale
        gray = image.convert('L')
        logging.debug("Converted image to grayscale.")
    
        # Step 3: Enhance contrast
        enhancer = ImageEnhance.Contrast(gray)
        gray_enhanced = enhancer.enhance(2.0)  # Adjust contrast factor as needed
        logging.debug("Enhanced image contrast.")
    
        # Step 4: Apply thresholding
        threshold = 140  # Adjust based on testing
        gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1')
        logging.debug("Applied thresholding to binarize the image.")
    
        # Step 5: Reduce noise with a median filter
        final_image = gray_threshold.filter(ImageFilter.MedianFilter())
        logging.debug("Applied median filter to reduce noise.")
    
        # Step 6: Save preprocessed image for debugging
        save_preprocessed_image(final_image, OUTPUT_DIR)
    
        logging.info("Image preprocessing complete. Performing OCR...")
    
        # Define Tesseract configuration
        custom_config = r'--oem 3 --psm 6'  # Adjust as needed
    
        # Perform OCR
        raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config)
    
        if not raw_text.strip():
            logging.warning("OCR returned empty text.")
            speak_text("I couldn't detect any text in the selected area. Please try again with a different region.")
        else:
            ocr_text = raw_text
            logging.debug("Raw OCR Text:")
            logging.debug(ocr_text)
            logging.info("OCR process completed successfully.")
    except pytesseract.TesseractNotFoundError:
        logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.")
        speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.")
        ocr_text = ""
    except Exception as e:
        logging.error(f"OCR Error: {e}")
        speak_text("An error occurred during OCR processing. Please try again.")
        ocr_text = ""
    

    def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations

    screen_width = root.winfo_screenwidth()
    screen_height = root.winfo_screenheight()
    root.geometry(f"{screen_width}x{screen_height}")
    
    canvas = tk.Canvas(root, cursor="cross", bg="grey")
    canvas.pack(fill=tk.BOTH, expand=True)
    
    rect = None
    start_x = start_y = end_x = end_y = 0
    
    def on_button_press(event):
        nonlocal start_x, start_y, rect
        start_x = int(event.x * scaling_factor)
        start_y = int(event.y * scaling_factor)
        rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2)
        logging.debug(f"Button pressed at ({start_x}, {start_y})")
    
    def on_move_press(event):
        nonlocal end_x, end_y
        end_x = int(event.x * scaling_factor)
        end_y = int(event.y * scaling_factor)
        canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y)
        logging.debug(f"Mouse moved to ({end_x}, {end_y})")
    
    def on_button_release(event):
        root.quit()
        root.destroy()  # Destroy the window to remove the overlay
    
    def on_escape(event):
        logging.info("Capture cancelled by user.")
        root.quit()
        root.destroy()
        sys.exit()
    
    canvas.bind("<ButtonPress-1>", on_button_press)
    canvas.bind("<B1-Motion>", on_move_press)
    canvas.bind("<ButtonRelease-1>", on_button_release)
    root.bind("<Escape>", on_escape)
    
    logging.info("Please select the region by clicking and dragging the mouse.")
    root.mainloop()
    
    # Calculate absolute coordinates
    left = min(start_x, end_x)
    top = min(start_y, end_y)
    right = max(start_x, end_x)
    bottom = max(start_y, end_y)
    
    logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}")
    return (left, top, right, bottom)
    

    -------------------- Modified Web Search Function --------------------

    async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.

    Args:
        query (str): The search query.
    
    Returns:
        list: A list of dictionaries containing 'title' and 'snippet' for each result.
    """
    logging.info(f"Searching the web for query: {query}")
    
    # Construct the site-limited query
    if PREFERRED_SITES:
        site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES])
        combined_query = f"{query} ({site_query})"
    else:
        combined_query = query  # If no preferred sites, perform a general search
    
    logging.debug(f"Combined search query with preferred sites: {combined_query}")
    
    # Encode the query for URL
    encoded_query = aiohttp.helpers.quote(combined_query)
    url = f"https://html.duckduckgo.com/html/?q={encoded_query}"
    
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                html = await response.text()
    except Exception as e:
        logging.error(f"Web search failed: {e}")
        return []
    
    soup = BeautifulSoup(html, 'html.parser')
    results = []
    
    # DuckDuckGo's HTML structure can vary; adjust selectors as needed
    for result in soup.find_all('div', class_='result__body', limit=3):
        title_element = result.find('a', class_='result__a')
        snippet_element = result.find('a', class_='result__snippet')
        title = title_element.get_text(strip=True) if title_element else "No Title"
        snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet"
        results.append({'title': title, 'snippet': snippet})
        logging.debug(f"Found result - Title: {title}, Snippet: {snippet}")
    
    if not results:
        logging.warning("No search results found for the preferred sites.")
    
    logging.info(f"Web search complete. Found {len(results)} results.")
    return results
    

    def generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])

    if user_input:
        context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.
    

    OCR Text: {ocr_text}

    User Query: {user_input}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text and the user's query.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response to the user's question based on the available information.
    4. Ensure that your answer directly addresses the user's intent.

    Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.

    OCR Text: {ocr_text}

    Web Search Results: {formatted_results}

    Instructions:

    1. Analyze the OCR text.
    2. Utilize the web search results to inform your answer.
    3. Provide a well-informed, concise, and accurate response based on the available information.
    4. Ensure that your answer directly reflects the content of the OCR text and the search results.

    Provide the answer below: """

    try:
        response = ollama.generate(model='llama3.2', prompt=context)
        final_response = response.get('response', '').strip()
        logging.info(f"Generated Final Response: {final_response}")
        return final_response
    except Exception as e:
        logging.error(f"LLM Error during final response generation: {e}")
        return ""
    

    def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""

    -------------------- Helper Functions --------------------

    def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.

    Args:
        short_timeout (int): Maximum number of seconds to wait for a phrase to start.
        phrase_time_limit (int): Maximum number of seconds a phrase can last.
    """
    global is_listening, speech_text
    recognizer = sr.Recognizer()
    with sr.Microphone() as source:
        logging.info("Listening for voice input... Please say your query or question.")
        recognizer.adjust_for_ambient_noise(source, duration=1)  # Adjust for noise for 1 second
        try:
            # Listen for speech with shorter timeout and phrase time limit
            audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit)
            logging.info("Processing the audio with Google Speech Recognition...")
            text = recognizer.recognize_google(audio)
            # Convert number words to digits if necessary
            speech_text = convert_number_words(text)
            logging.info(f"Speech Transcribed: {speech_text}")
    
            # Provide confirmation
            speak_text(f"I heard: {speech_text}.")
        except sr.WaitTimeoutError:
            logging.warning("Listening timed out, no speech detected. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.UnknownValueError:
            logging.warning("Speech recognition could not understand audio. Proceeding without voice input.")
            speech_text = ""  # Ensure speech_text is empty
        except sr.RequestError as e:
            logging.error(f"Could not request results from speech recognition service; {e}")
            speech_text = ""  # Ensure speech_text is empty
        finally:
            with lock:
                is_listening = False  # Ensure listening flag is reset
    

    def configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')

    # List available voices (optional)
    for index, voice in enumerate(voices):
        logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}")
    
    # Change voice (select male/female, adjust the index as per your preference)
    if len(voices) > 1:
        tts_engine.setProperty('voice', voices[1].id)  # Change index to select a different voice
        logging.info(f"TTS voice set to: {voices[1].name}")
    else:
        logging.info("Only one voice available. Using default voice.")
    
    # Adjust speaking rate (lower is slower and more human-like)
    tts_engine.setProperty('rate', 160)  # Default is 200, lowering makes it more human-like
    
    # Adjust volume (0.0 to 1.0)
    tts_engine.setProperty('volume', 1.0)
    

    def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return

    # Step 2: Capture the selected region
    with mss.mss() as sct:
        monitor = {
            "top": top,
            "left": left,
            "width": width,
            "height": height
        }
        try:
            screenshot = sct.grab(monitor)
            img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
        except Exception as e:
            logging.error(f"Screen capture failed: {e}")
            speak_text("Screen capture failed. Please ensure the selected area is correct and try again.")
            reset_state()
            return
    
    # Step 3: Perform OCR on the captured region
    perform_ocr(img)
    if not ocr_text.strip():
        logging.error("OCR returned empty text. Please ensure the selected region contains readable text.")
        speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.")
        reset_state()
        return
    
    # Step 4: Listen for user input (optional) with shorter timeout
    transcribe_speech(short_timeout=5, phrase_time_limit=5)
    
    # Step 5: Generate and execute the search query
    generate_and_execute_search(speech_text, ocr_text)
    

    def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return

    logging.info(f"Using OCR text as search query: {search_query}")
    
    # Perform web search and process the answer
    asyncio.run(async_process_search(search_query, ocr_text, user_input))
    

    async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return

    # Generate final response using LLM
    final_response = generate_final_response(search_results, ocr_text, user_input)
    if not final_response:
        logging.error("Failed to generate a final response.")
        speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.")
        reset_state()
        return
    
    # Speak out the answer
    speak_text(final_response)
    reset_state()
    

    -------------------- Core Functions --------------------

    def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()

    def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")

    -------------------- Main Function --------------------

    async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor

    configure_tts_voice()  # Voice configuration happens before anything else
    
    # Set DPI awareness and get scaling factor
    set_dpi_awareness()
    scaling_factor = get_scaling_factor()
    
    # Set up hotkeys
    keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,))
    keyboard.add_hotkey('f3', stop_listening)
    
    logging.info("QA Assistant is running.")
    logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.")
    
    while True:
        # Wait for a short duration to avoid busy waiting
        await asyncio.sleep(0.1)
    
        # Check for exit condition
        if keyboard.is_pressed('esc'):
            logging.info("Exiting...")
            speak_text("Exiting the QA Assistant. Goodbye!")
            break
    
        # Reset speech_text and ocr_text is handled after processing
    
    # Clean up keyboard hooks before exiting
    keyboard.unhook_all()
    

    -------------------- Entry Point --------------------

    if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all()

  10. Assistant

    It sounds like you're experiencing an issue where the captured image is offset from the area selected with the mouse. This can happen due to DPI scaling issues on high-resolution displays where the logical screen size is different from the physical screen size.

    Here's a checklist and some changes you can try to resolve this issue:

    1. Verify DPI Awareness: Ensure your application is truly DPI aware. You have already configured DPI awareness in your code, but let's make sure it's correctly implemented.

    2. Adjust for DPI Scaling: When capturing the screen region, consider the DPI scaling factor to adjust the coordinates appropriately.

    3. Debug Logging: Verify the coordinates being used for capturing the screen by adding detailed logging.

    Here is a modified version of your function that tries to address potential issues with DPI scaling:

    def capture_and_save_image():
        """
        Coordinates the screen capture and OCR process.
        Adjusts for DPI scaling if necessary.
        """
        global speech_text, ocr_text
    
        # Step 1: Let the user select a region
        region = get_selected_region(scaling_factor)
        left, top, right, bottom = region
        width = right - left
        height = bottom - top
    
        logging.debug(f"Width: {width}, Height: {height}, Scaling Factor: {scaling_factor}")
    
        if width <= 0 or height <= 0:
            logging.error("Invalid region selected. Please try again.")
            speak_text("Invalid region selected. Please try again by pressing F2.")
            reset_state()
            return
    
        # Step 2: Capture the selected region
        with mss.mss() as sct:
            monitor = {
                "top": int(top / scaling_factor),
                "left": int(left / scaling_factor),
                "width": int(width / scaling_factor),
                "height": int(height / scaling_factor)
            }
    
            logging.debug(f"Monitor for capture: {monitor}")
    
            try:
                screenshot = sct.grab(monitor)
                img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX")
            except Exception as e:
                logging.error(f"Screen capture failed: {e}")
                speak_text("Screen capture failed. Please ensure the selected area is correct and try again.")
                reset_state()
                return
        
        # Step 3: Perform OCR on the captured region
        perform_ocr(img)
        if not ocr_text.strip():
            logging.error("OCR returned empty text. Please ensure the selected region contains readable text.")
            speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.")
            reset_state()
            return
    
        # Step 4: Listen for user input (optional) with shorter timeout
        transcribe_speech(short_timeout=5, phrase_time_limit=5)
    
        # Step 5: Generate and execute the search query
        generate_and_execute_search(speech_text, ocr_text)
    
    

    Key Changes:

    • Monitor Adjustment: The coordinates for the monitor to be captured are adjusted by dividing by the scaling_factor. This adjustment should help resolve capturing the wrong screen region if DPI scaling is affecting the display coordinates.

    • Detailed Logging: Additional debug logging will help track the dimensions and coordinates of the region being selected and captured.

    Additional Recommendations:

    • Testing Without DPI Scaling: Test this approach with your system's DPI scaling set to 100% to identify if the problem persists without DPI scaling.

    • Verify Scaling Factor: Check the scaling factor you retrieve to ensure it's accurate compared to your system settings.

    Following these adjustments should help correct the screen capture coordinates when dealing with different DPI scalings.