import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import…
- User
import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import time import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama
-------------------- Configuration --------------------
Configure logging to capture DEBUG level logs
logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )
Initialize the TTS engine globally
tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start
Define global locks
lock = threading.Lock() tts_lock = threading.Lock()
-------------------- Preferred Sites Configuration --------------------
Editable list of preferred sites for prioritized search
PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]
def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0
def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text
def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found
def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)
# Add hotkeys for pausing (F4) and stopping (F5) keyboard.add_hotkey('f4', pause_speaking) keyboard.add_hotkey('f5', stop_speaking) tts_engine.runAndWait() logging.info("Text-to-Speech completed.") except RuntimeError as e: logging.error(f"TTS Error: {e}") finally: # Remove hotkeys after completion to avoid conflicts keyboard.remove_hotkey('f4') keyboard.remove_hotkey('f5')def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()
def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()
def save_captured_image(image, filename="captured_region.png"): """ Saves the captured image for debugging purposes. """ try: image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")
def save_preprocessed_image(image, filename="preprocessed_image.png"): """ Saves the preprocessed image for debugging purposes. """ try: image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")
def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")
# Step 1: Save the original captured image for verification save_captured_image(image) # Step 2: Convert to grayscale gray = image.convert('L') # Step 3: Enhance contrast enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(10) # Step 4: Apply thresholding threshold = 140 # Adjust based on testing gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') # Step 5: Reduce noise with a median filter final_image = gray_threshold.filter(ImageFilter.MedianFilter()) # Step 6: Save preprocessed image for debugging save_preprocessed_image(final_image) logging.info("Image preprocessing complete. Performing OCR...") # Define Tesseract configuration custom_config = r'--oem 3 --psm 6' # Adjust as needed # Perform OCR raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config) if not raw_text.strip(): logging.warning("OCR returned empty text.") speak_text("I couldn't detect any text in the selected area. Please try again with a different region.") else: ocr_text = raw_text logging.debug("Raw OCR Text:") logging.debug(ocr_text) logging.info("OCR process completed successfully.") except pytesseract.TesseractNotFoundError: logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.") speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.") ocr_text = "" except Exception as e: logging.error(f"OCR Error: {e}") speak_text("An error occurred during OCR processing. Please try again.") ocr_text = ""def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations
screen_width = root.winfo_screenwidth() screen_height = root.winfo_screenheight() root.geometry(f"{screen_width}x{screen_height}") canvas = tk.Canvas(root, cursor="cross", bg="grey") canvas.pack(fill=tk.BOTH, expand=True) rect = None start_x = start_y = end_x = end_y = 0 def on_button_press(event): nonlocal start_x, start_y, rect start_x = int(event.x * scaling_factor) start_y = int(event.y * scaling_factor) rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) logging.debug(f"Button pressed at ({start_x}, {start_y})") def on_move_press(event): nonlocal end_x, end_y end_x = int(event.x * scaling_factor) end_y = int(event.y * scaling_factor) canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y) logging.debug(f"Mouse moved to ({end_x}, {end_y})") def on_button_release(event): root.quit() root.destroy() # Destroy the window to remove the overlay def on_escape(event): logging.info("Capture cancelled by user.") root.quit() root.destroy() sys.exit() canvas.bind("<ButtonPress-1>", on_button_press) canvas.bind("<B1-Motion>", on_move_press) canvas.bind("<ButtonRelease-1>", on_button_release) root.bind("<Escape>", on_escape) logging.info("Please select the region by clicking and dragging the mouse.") root.mainloop() # Calculate absolute coordinates left = min(start_x, end_x) top = min(start_y, end_y) right = max(start_x, end_x) bottom = max(start_y, end_y) logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}") return (left, top, right, bottom)-------------------- Modified Web Search Function --------------------
async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.
Args: query (str): The search query. Returns: list: A list of dictionaries containing 'title' and 'snippet' for each result. """ logging.info(f"Searching the web for query: {query}") # Construct the site-limited query if PREFERRED_SITES: site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES]) combined_query = f"{query} ({site_query})" else: combined_query = query # If no preferred sites, perform a general search logging.debug(f"Combined search query with preferred sites: {combined_query}") # Encode the query for URL encoded_query = aiohttp.helpers.quote(combined_query) url = f"https://html.duckduckgo.com/html/?q={encoded_query}" try: async with aiohttp.ClientSession() as session: async with session.get(url) as response: html = await response.text() except Exception as e: logging.error(f"Web search failed: {e}") return [] soup = BeautifulSoup(html, 'html.parser') results = [] # DuckDuckGo's HTML structure can vary; adjust selectors as needed for result in soup.find_all('div', class_='result__body', limit=3): title_element = result.find('a', class_='result__a') snippet_element = result.find('a', class_='result__snippet') title = title_element.get_text(strip=True) if title_element else "No Title" snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet" results.append({'title': title, 'snippet': snippet}) logging.debug(f"Found result - Title: {title}, Snippet: {snippet}") if not results: logging.warning("No search results found for the preferred sites.") logging.info(f"Web search complete. Found {len(results)} results.") return resultsdef generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])
if user_input: context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.OCR Text: {ocr_text}
User Query: {user_input}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text and the user's query.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response to the user's question based on the available information.
- Ensure that your answer directly addresses the user's intent.
Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.
OCR Text: {ocr_text}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response based on the available information.
- Ensure that your answer directly reflects the content of the OCR text and the search results.
Provide the answer below: """
try: response = ollama.generate(model='llama3.2', prompt=context) final_response = response.get('response', '').strip() logging.info(f"Generated Final Response: {final_response}") return final_response except Exception as e: logging.error(f"LLM Error during final response generation: {e}") return ""def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""
-------------------- Helper Functions --------------------
def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.
Args: short_timeout (int): Maximum number of seconds to wait for a phrase to start. phrase_time_limit (int): Maximum number of seconds a phrase can last. """ global is_listening, speech_text recognizer = sr.Recognizer() with sr.Microphone() as source: logging.info("Listening for voice input... Please say your query or question.") recognizer.adjust_for_ambient_noise(source, duration=1) # Adjust for noise for 1 second try: # Listen for speech with shorter timeout and phrase time limit audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit) logging.info("Processing the audio with Google Speech Recognition...") text = recognizer.recognize_google(audio) # Convert number words to digits if necessary speech_text = convert_number_words(text) logging.info(f"Speech Transcribed: {speech_text}") # Provide confirmation speak_text(f"I heard: {speech_text}.") except sr.WaitTimeoutError: logging.warning("Listening timed out, no speech detected. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.UnknownValueError: logging.warning("Speech recognition could not understand audio. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.RequestError as e: logging.error(f"Could not request results from speech recognition service; {e}") speech_text = "" # Ensure speech_text is empty finally: with lock: is_listening = False # Ensure listening flag is resetdef configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')
# List available voices (optional) for index, voice in enumerate(voices): logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}") # Change voice (select male/female, adjust the index as per your preference) if len(voices) > 1: tts_engine.setProperty('voice', voices[1].id) # Change index to select a different voice logging.info(f"TTS voice set to: {voices[1].name}") else: logging.info("Only one voice available. Using default voice.") # Adjust speaking rate (lower is slower and more human-like) tts_engine.setProperty('rate', 160) # Default is 200, lowering makes it more human-like # Adjust volume (0.0 to 1.0) tts_engine.setProperty('volume', 1.0)def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return
# Step 2: Capture the selected region with mss.mss() as sct: monitor = { "top": top, "left": left, "width": width, "height": height } try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") except Exception as e: logging.error(f"Screen capture failed: {e}") speak_text("Screen capture failed. Please ensure the selected area is correct and try again.") reset_state() return # Step 3: Perform OCR on the captured region perform_ocr(img) if not ocr_text.strip(): logging.error("OCR returned empty text. Please ensure the selected region contains readable text.") speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.") reset_state() return # Step 4: Listen for user input (optional) with shorter timeout transcribe_speech(short_timeout=5, phrase_time_limit=5) # Step 5: Generate and execute the search query generate_and_execute_search(speech_text, ocr_text)def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return
logging.info(f"Using OCR text as search query: {search_query}") # Perform web search and process the answer asyncio.run(async_process_search(search_query, ocr_text, user_input))async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return
# Generate final response using LLM final_response = generate_final_response(search_results, ocr_text, user_input) if not final_response: logging.error("Failed to generate a final response.") speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.") reset_state() return # Speak out the answer speak_text(final_response) reset_state()-------------------- Core Functions --------------------
def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()
def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")
-------------------- Main Function --------------------
async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor
configure_tts_voice() # Voice configuration happens before anything else # Set DPI awareness and get scaling factor set_dpi_awareness() scaling_factor = get_scaling_factor() # Set up hotkeys keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,)) keyboard.add_hotkey('f3', stop_listening) logging.info("QA Assistant is running.") logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.") while True: # Wait for a short duration to avoid busy waiting await asyncio.sleep(0.1) # Check for exit condition if keyboard.is_pressed('esc'): logging.info("Exiting...") speak_text("Exiting the QA Assistant. Goodbye!") break # Reset speech_text and ocr_text is handled after processing # Clean up keyboard hooks before exiting keyboard.unhook_all()-------------------- Entry Point --------------------
if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all():: Extract me the part just before the image is fed into ocr to a new script which outputs me the image itself to see if there is some offset erorr in ocr
- Assistant
To extract the part of your script that captures a screen region and outputs the preprocessed image before it's fed into OCR, we'll create a standalone script. This script will capture the selected screen region and save the preprocessed image to a file, allowing you to visually inspect it for any issues.
Below is the extracted and modified script for this purpose:
import mss from PIL import Image, ImageEnhance, ImageFilter import tkinter as tk import ctypes import sys import logging # Configure logging logging.basicConfig( level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s', handlers=[logging.StreamHandler()] ) def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.") def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor else: logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0 def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations screen_width = root.winfo_screenwidth() screen_height = root.winfo_screenheight() root.geometry(f"{screen_width}x{screen_height}") canvas = tk.Canvas(root, cursor="cross", bg="grey") canvas.pack(fill=tk.BOTH, expand=True) rect = None start_x = start_y = end_x = end_y = 0 def on_button_press(event): nonlocal start_x, start_y, rect start_x = int(event.x * scaling_factor) start_y = int(event.y * scaling_factor) rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) logging.debug(f"Button pressed at ({start_x}, {start_y})") def on_move_press(event): nonlocal end_x, end_y end_x = int(event.x * scaling_factor) end_y = int(event.y * scaling_factor) canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y) logging.debug(f"Mouse moved to ({end_x}, {end_y})") def on_button_release(event): root.quit() root.destroy() # Destroy the window to remove the overlay def on_escape(event): logging.info("Capture cancelled by user.") root.quit() root.destroy() sys.exit() canvas.bind("<ButtonPress-1>", on_button_press) canvas.bind("<B1-Motion>", on_move_press) canvas.bind("<ButtonRelease-1>", on_button_release) root.bind("<Escape>", on_escape) logging.info("Please select the region by clicking and dragging the mouse.") root.mainloop() left = min(start_x, end_x) top = min(start_y, end_y) right = max(start_x, end_x) bottom = max(start_y, end_y) logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}") return (left, top, right, bottom) def save_preprocessed_image(image, filename="preprocessed_image.png"): """ Saves the preprocessed image for debugging purposes. """ try: image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}") def preprocess_image(image): """ Preprocesses the image for OCR by converting to grayscale, enhancing contrast, applying thresholding, and reducing noise. """ logging.info("Starting image preprocessing...") # Convert to grayscale gray = image.convert('L') # Enhance contrast enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(10) # Apply thresholding threshold = 140 # Adjust based on testing gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') # Reduce noise with a median filter final_image = gray_threshold.filter(ImageFilter.MedianFilter()) logging.info("Image preprocessing complete.") return final_image def capture_and_save_image(): """ Coordinates the screen capture and image preprocessing. """ # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") return # Step 2: Capture the selected region with mss.mss() as sct: monitor = { "top": top, "left": left, "width": width, "height": height } try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") except Exception as e: logging.error(f"Screen capture failed: {e}") return # Step 3: Preprocess the captured image preprocessed_image = preprocess_image(img) # Step 4: Save the preprocessed image save_preprocessed_image(preprocessed_image, "preprocessed_image.png") if __name__ == "__main__": try: set_dpi_awareness() scaling_factor = get_scaling_factor() capture_and_save_image() except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}")This script will allow you to capture a region of the screen and save the preprocessed image, which you can then use to check for any issues before feeding it into your OCR process.
- User
Is the first script subset of second one or is there difference in functionality : import mss from PIL import Image, ImageEnhance, ImageFilter import tkinter as tk import ctypes import sys import logging import time
-------------------- Configuration --------------------
Configure logging to capture DEBUG level logs
logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("ocr_troubleshoot.log"), ] )
-------------------- DPI Awareness and Scaling --------------------
def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. Specifically for Windows. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = ctypes.windll.shcore.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0
-------------------- Region Selection Interface --------------------
class RegionSelector: def init(self, scaling_factor): self.scaling_factor = scaling_factor self.start_x = 0 self.start_y = 0 self.end_x = 0 self.end_y = 0 self.rect = None self.selection_complete = False
# Initialize the tkinter window self.root = tk.Tk() self.root.title("Select Region (Press Esc to Cancel)") self.root.attributes('-alpha', 0.3) # Transparency self.root.attributes("-topmost", True) self.root.overrideredirect(True) # Remove window decorations screen_width = self.root.winfo_screenwidth() screen_height = self.root.winfo_screenheight() self.root.geometry(f"{screen_width}x{screen_height}") self.canvas = tk.Canvas(self.root, cursor="cross", bg="grey") self.canvas.pack(fill=tk.BOTH, expand=True) self.canvas.bind("<ButtonPress-1>", self.on_button_press) self.canvas.bind("<B1-Motion>", self.on_move_press) self.canvas.bind("<ButtonRelease-1>", self.on_button_release) self.root.bind("<Escape>", self.on_escape) logging.info("Please select the region by clicking and dragging the mouse.") def on_button_press(self, event): self.start_x = int(event.x * self.scaling_factor) self.start_y = int(event.y * self.scaling_factor) self.rect = self.canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) logging.debug(f"Button pressed at ({self.start_x}, {self.start_y})") def on_move_press(self, event): self.end_x = int(event.x * self.scaling_factor) self.end_y = int(event.y * self.scaling_factor) self.canvas.coords(self.rect, self.start_x / self.scaling_factor, self.start_y / self.scaling_factor, event.x, event.y) logging.debug(f"Mouse moved to ({self.end_x}, {self.end_y})") def on_button_release(self, event): self.end_x = int(event.x * self.scaling_factor) self.end_y = int(event.y * self.scaling_factor) self.selection_complete = True self.root.quit() # Exit the main loop logging.debug(f"Button released at ({self.end_x}, {self.end_y})") def on_escape(self, event): logging.info("Capture cancelled by user.") self.root.quit() self.root.destroy() sys.exit() def get_region(self): self.root.mainloop() self.root.destroy() left = min(self.start_x, self.end_x) top = min(self.start_y, self.end_y) right = max(self.start_x, self.end_x) bottom = max(self.start_y, self.end_y) logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}") return (left, top, right, bottom)-------------------- Screen Capture and Preprocessing --------------------
def capture_screen_region(region): """ Captures the specified screen region using mss.
Args: region (tuple): A tuple of (left, top, right, bottom) coordinates. Returns: PIL.Image: The captured image. """ left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region dimensions. Width and height must be positive.") return None with mss.mss() as sct: monitor = { "top": top, "left": left, "width": width, "height": height } try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") logging.info(f"Screen region captured: {width}x{height} pixels.") return img except Exception as e: logging.error(f"Screen capture failed: {e}") return Nonedef preprocess_image(image): """ Preprocesses the image for better OCR results.
Steps: 1. Convert to grayscale. 2. Enhance contrast. 3. Apply thresholding. 4. Reduce noise with a median filter. Args: image (PIL.Image): The original captured image. Returns: PIL.Image: The preprocessed image. """ try: logging.info("Starting image preprocessing...") # Step 1: Convert to grayscale gray = image.convert('L') logging.debug("Converted image to grayscale.") # Step 2: Enhance contrast enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(2.0) # Adjust contrast factor as needed logging.debug("Enhanced image contrast.") # Step 3: Apply thresholding threshold = 140 # Adjust based on testing gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') logging.debug("Applied thresholding to binarize the image.") # Step 4: Reduce noise with a median filter final_image = gray_threshold.filter(ImageFilter.MedianFilter()) logging.debug("Applied median filter to reduce noise.") logging.info("Image preprocessing complete.") return final_image except Exception as e: logging.error(f"Image preprocessing failed: {e}") return None-------------------- Saving Images --------------------
def save_image(image, filename): """ Saves the given image to the specified filename.
Args: image (PIL.Image): The image to save. filename (str): The filename to save the image as. """ try: image.save(filename) logging.info(f"Image saved as {filename}") except Exception as e: logging.error(f"Failed to save image {filename}: {e}")-------------------- Main Function --------------------
def main(): # Step 1: Set DPI awareness and get scaling factor set_dpi_awareness() scaling_factor = get_scaling_factor() logging.info(f"Using DPI scaling factor: {scaling_factor}")
# Step 2: Initialize region selector selector = RegionSelector(scaling_factor) # Step 3: Get selected region region = selector.get_region() left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Width and height must be positive.") sys.exit(1) # Step 4: Capture the selected screen region captured_image = capture_screen_region(region) if captured_image is None: logging.error("Failed to capture the selected screen region.") sys.exit(1) # Step 5: Save the captured image for verification timestamp = int(time.time()) original_filename = f"captured_region_{timestamp}.png" save_image(captured_image, original_filename) # Step 6: Preprocess the captured image preprocessed_image = preprocess_image(captured_image) if preprocessed_image is None: logging.error("Image preprocessing failed.") sys.exit(1) # Step 7: Save the preprocessed image for verification preprocessed_filename = f"preprocessed_image_{timestamp}.png" save_image(preprocessed_image, preprocessed_filename) logging.info("Troubleshooting completed successfully.") logging.info(f"Original Image: {original_filename}") logging.info(f"Preprocessed Image: {preprocessed_filename}")if name == "main": try: main() except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: logging.info("Exiting the OCR Troubleshooting Script.") AND import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import time import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama
-------------------- Configuration --------------------
Configure logging to capture DEBUG level logs
logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )
Initialize the TTS engine globally
tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start
Define global locks
lock = threading.Lock() tts_lock = threading.Lock()
-------------------- Preferred Sites Configuration --------------------
Editable list of preferred sites for prioritized search
PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]
def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0
def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text
def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found
def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)
# Add hotkeys for pausing (F4) and stopping (F5) keyboard.add_hotkey('f4', pause_speaking) keyboard.add_hotkey('f5', stop_speaking) tts_engine.runAndWait() logging.info("Text-to-Speech completed.") except RuntimeError as e: logging.error(f"TTS Error: {e}") finally: # Remove hotkeys after completion to avoid conflicts keyboard.remove_hotkey('f4') keyboard.remove_hotkey('f5')def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()
def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()
def save_captured_image(image, filename="captured_region.png"): """ Saves the captured image for debugging purposes. """ try: image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")
def save_preprocessed_image(image, filename="preprocessed_image.png"): """ Saves the preprocessed image for debugging purposes. """ try: image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")
def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")
# Step 1: Save the original captured image for verification save_captured_image(image) # Step 2: Convert to grayscale gray = image.convert('L') # Step 3: Enhance contrast enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(10) # Step 4: Apply thresholding threshold = 140 # Adjust based on testing gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') # Step 5: Reduce noise with a median filter final_image = gray_threshold.filter(ImageFilter.MedianFilter()) # Step 6: Save preprocessed image for debugging save_preprocessed_image(final_image) logging.info("Image preprocessing complete. Performing OCR...") # Define Tesseract configuration custom_config = r'--oem 3 --psm 6' # Adjust as needed # Perform OCR raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config) if not raw_text.strip(): logging.warning("OCR returned empty text.") speak_text("I couldn't detect any text in the selected area. Please try again with a different region.") else: ocr_text = raw_text logging.debug("Raw OCR Text:") logging.debug(ocr_text) logging.info("OCR process completed successfully.") except pytesseract.TesseractNotFoundError: logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.") speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.") ocr_text = "" except Exception as e: logging.error(f"OCR Error: {e}") speak_text("An error occurred during OCR processing. Please try again.") ocr_text = ""def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations
screen_width = root.winfo_screenwidth() screen_height = root.winfo_screenheight() root.geometry(f"{screen_width}x{screen_height}") canvas = tk.Canvas(root, cursor="cross", bg="grey") canvas.pack(fill=tk.BOTH, expand=True) rect = None start_x = start_y = end_x = end_y = 0 def on_button_press(event): nonlocal start_x, start_y, rect start_x = int(event.x * scaling_factor) start_y = int(event.y * scaling_factor) rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) logging.debug(f"Button pressed at ({start_x}, {start_y})") def on_move_press(event): nonlocal end_x, end_y end_x = int(event.x * scaling_factor) end_y = int(event.y * scaling_factor) canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y) logging.debug(f"Mouse moved to ({end_x}, {end_y})") def on_button_release(event): root.quit() root.destroy() # Destroy the window to remove the overlay def on_escape(event): logging.info("Capture cancelled by user.") root.quit() root.destroy() sys.exit() canvas.bind("<ButtonPress-1>", on_button_press) canvas.bind("<B1-Motion>", on_move_press) canvas.bind("<ButtonRelease-1>", on_button_release) root.bind("<Escape>", on_escape) logging.info("Please select the region by clicking and dragging the mouse.") root.mainloop() # Calculate absolute coordinates left = min(start_x, end_x) top = min(start_y, end_y) right = max(start_x, end_x) bottom = max(start_y, end_y) logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}") return (left, top, right, bottom)-------------------- Modified Web Search Function --------------------
async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.
Args: query (str): The search query. Returns: list: A list of dictionaries containing 'title' and 'snippet' for each result. """ logging.info(f"Searching the web for query: {query}") # Construct the site-limited query if PREFERRED_SITES: site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES]) combined_query = f"{query} ({site_query})" else: combined_query = query # If no preferred sites, perform a general search logging.debug(f"Combined search query with preferred sites: {combined_query}") # Encode the query for URL encoded_query = aiohttp.helpers.quote(combined_query) url = f"https://html.duckduckgo.com/html/?q={encoded_query}" try: async with aiohttp.ClientSession() as session: async with session.get(url) as response: html = await response.text() except Exception as e: logging.error(f"Web search failed: {e}") return [] soup = BeautifulSoup(html, 'html.parser') results = [] # DuckDuckGo's HTML structure can vary; adjust selectors as needed for result in soup.find_all('div', class_='result__body', limit=3): title_element = result.find('a', class_='result__a') snippet_element = result.find('a', class_='result__snippet') title = title_element.get_text(strip=True) if title_element else "No Title" snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet" results.append({'title': title, 'snippet': snippet}) logging.debug(f"Found result - Title: {title}, Snippet: {snippet}") if not results: logging.warning("No search results found for the preferred sites.") logging.info(f"Web search complete. Found {len(results)} results.") return resultsdef generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])
if user_input: context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.OCR Text: {ocr_text}
User Query: {user_input}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text and the user's query.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response to the user's question based on the available information.
- Ensure that your answer directly addresses the user's intent.
Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.
OCR Text: {ocr_text}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response based on the available information.
- Ensure that your answer directly reflects the content of the OCR text and the search results.
Provide the answer below: """
try: response = ollama.generate(model='llama3.2', prompt=context) final_response = response.get('response', '').strip() logging.info(f"Generated Final Response: {final_response}") return final_response except Exception as e: logging.error(f"LLM Error during final response generation: {e}") return ""def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""
-------------------- Helper Functions --------------------
def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.
Args: short_timeout (int): Maximum number of seconds to wait for a phrase to start. phrase_time_limit (int): Maximum number of seconds a phrase can last. """ global is_listening, speech_text recognizer = sr.Recognizer() with sr.Microphone() as source: logging.info("Listening for voice input... Please say your query or question.") recognizer.adjust_for_ambient_noise(source, duration=1) # Adjust for noise for 1 second try: # Listen for speech with shorter timeout and phrase time limit audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit) logging.info("Processing the audio with Google Speech Recognition...") text = recognizer.recognize_google(audio) # Convert number words to digits if necessary speech_text = convert_number_words(text) logging.info(f"Speech Transcribed: {speech_text}") # Provide confirmation speak_text(f"I heard: {speech_text}.") except sr.WaitTimeoutError: logging.warning("Listening timed out, no speech detected. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.UnknownValueError: logging.warning("Speech recognition could not understand audio. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.RequestError as e: logging.error(f"Could not request results from speech recognition service; {e}") speech_text = "" # Ensure speech_text is empty finally: with lock: is_listening = False # Ensure listening flag is resetdef configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')
# List available voices (optional) for index, voice in enumerate(voices): logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}") # Change voice (select male/female, adjust the index as per your preference) if len(voices) > 1: tts_engine.setProperty('voice', voices[1].id) # Change index to select a different voice logging.info(f"TTS voice set to: {voices[1].name}") else: logging.info("Only one voice available. Using default voice.") # Adjust speaking rate (lower is slower and more human-like) tts_engine.setProperty('rate', 160) # Default is 200, lowering makes it more human-like # Adjust volume (0.0 to 1.0) tts_engine.setProperty('volume', 1.0)def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return
# Step 2: Capture the selected region with mss.mss() as sct: monitor = { "top": top, "left": left, "width": width, "height": height } try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") except Exception as e: logging.error(f"Screen capture failed: {e}") speak_text("Screen capture failed. Please ensure the selected area is correct and try again.") reset_state() return # Step 3: Perform OCR on the captured region perform_ocr(img) if not ocr_text.strip(): logging.error("OCR returned empty text. Please ensure the selected region contains readable text.") speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.") reset_state() return # Step 4: Listen for user input (optional) with shorter timeout transcribe_speech(short_timeout=5, phrase_time_limit=5) # Step 5: Generate and execute the search query generate_and_execute_search(speech_text, ocr_text)def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return
logging.info(f"Using OCR text as search query: {search_query}") # Perform web search and process the answer asyncio.run(async_process_search(search_query, ocr_text, user_input))async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return
# Generate final response using LLM final_response = generate_final_response(search_results, ocr_text, user_input) if not final_response: logging.error("Failed to generate a final response.") speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.") reset_state() return # Speak out the answer speak_text(final_response) reset_state()-------------------- Core Functions --------------------
def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()
def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")
-------------------- Main Function --------------------
async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor
configure_tts_voice() # Voice configuration happens before anything else # Set DPI awareness and get scaling factor set_dpi_awareness() scaling_factor = get_scaling_factor() # Set up hotkeys keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,)) keyboard.add_hotkey('f3', stop_listening) logging.info("QA Assistant is running.") logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.") while True: # Wait for a short duration to avoid busy waiting await asyncio.sleep(0.1) # Check for exit condition if keyboard.is_pressed('esc'): logging.info("Exiting...") speak_text("Exiting the QA Assistant. Goodbye!") break # Reset speech_text and ocr_text is handled after processing # Clean up keyboard hooks before exiting keyboard.unhook_all()-------------------- Entry Point --------------------
if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all()
- Assistant
The two scripts you provided have different purposes and functionalities, with some overlapping components related to screen region capture and image preprocessing. Here's a breakdown of their differences and similarities:
Similarities:
-
DPI Awareness and Scaling: Both scripts include functionality for setting the DPI awareness and getting the scaling factor. This is important for handling display scaling on Windows systems.
-
Screen Region Selection: Both scripts allow the user to select a region of the screen and capture it. They use
tkinterto provide a graphical interface for this selection. -
Image Preprocessing: Both scripts perform image preprocessing steps, including converting the image to grayscale and enhancing contrast. They also apply thresholding and noise reduction, which are common preprocessing steps for improving OCR accuracy.
-
Logging: Both scripts make extensive use of Python's logging module to capture and output debug information.
Differences:
-
Functionality and Purpose:
- First Script (Troubleshooting Script): This script primarily focuses on capturing the screen region and preprocessing the image in a standalone manner. It is designed to save and verify the original and preprocessed images as separate files. The user can use it to troubleshoot the capture and preprocessing steps by examining the saved images.
- Second Script (QA Assistant): This script integrates additional functionalities like OCR using Tesseract, web searches using DuckDuckGo, and text-to-speech (TTS) responses. It listens for user queries via voice input and attempts to answer questions by combining OCR results with online search data.
-
Modules and Features:
- The troubleshooting script does not include OCR processing, web search, or voice recognition/response functionality.
- The QA Assistant incorporates several additional modules such as
pytesseractfor OCR,speech_recognitionfor voice input,pyttsx3for TTS, andaiohttpandBeautifulSoupfor web scraping. It aims to create an interactive user experience encompassing image capture, OCR, search, and audio feedback.
-
Complexity:
- The QA Assistant script is significantly more complex due to its additional functionality for processing spoken input and generating spoken responses, making use of threading, asyncio, and various interactions between components.
In summary, while both scripts share common elements related to capturing and preprocessing screen images, their goals are different. The first script is a simpler, more focused utility for troubleshooting image capture and preprocessing, whereas the second script is a comprehensive interactive assistant that builds on those capabilities with added functionalities.
-
- User
What i want to know is that is the functionality of trouble shooting script in taking ocr is same as the main QA script or is there some difference ?
- Assistant
The functionality related to taking an OCR in the troubleshooting script and the QA Assistant script is fundamentally similar but serves different purposes, particularly concerning the integration with additional features in the QA script. Here's a closer look at each aspect to understand the differences:
Similarities:
-
Image Capture:
- Both scripts allow users to select and capture a specific screen region using
tkinterfor the GUI andmssfor screen capturing. This functionality is largely the same in both scripts.
- Both scripts allow users to select and capture a specific screen region using
-
Image Preprocessing:
- Both scripts perform similar preprocessing steps on the captured image prior to OCR. These steps include converting the image to grayscale, enhancing contrast, applying thresholding, and reducing noise with a median filter.
Differences:
-
Overall Integration with OCR:
- Troubleshooting Script: This script focuses purely on capturing and preprocessing the image. It stops short of performing OCR, instead emphasizing saving the preprocessed image to a file for manual inspection. This allows you to verify the preprocessing step to ensure it produces a usable image for OCR.
- QA Assistant Script: After preprocessing, this script completes the OCR process using
pytesseract. It then uses the recognized text to perform further actions, such as conducting a web search or generating responses using a language model.
-
Purpose and Final Output:
- The troubleshooting script’s primary goal is to aid in debugging the image capture and preprocessing pipeline by manually checking the preprocessed images.
- The QA script’s goal is to automate the process of interpreting and using OCR results to provide answers, illustrating a complete workflow from image capture to user query response.
-
Complexity and Integration:
- The troubleshooting script is simpler and modular, designed to address errors or uncertainties at the preprocessing stage.
- The QA Assistant script integrates with additional components (e.g., TTS, web searching, LLM response generation) for a seamless user interaction, with more dependencies and interactions between these components.
In conclusion, both scripts share the core functionality of image capture and preprocessing, but the troubleshooting script isolates these operations for focused examination, while the QA Assistant script incorporates these as part of a larger interactive workflow. Therefore, the basic functionality for performing an OCR preparation is similar, yet the context and utilization of the OCR results differ significantly.
-
- User
import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import time import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama
-------------------- Configuration --------------------
Configure logging to capture DEBUG level logs
logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )
Initialize the TTS engine globally
tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start
Define global locks
lock = threading.Lock() tts_lock = threading.Lock()
-------------------- Preferred Sites Configuration --------------------
Editable list of preferred sites for prioritized search
PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]
def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0
def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text
def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found
def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)
# Add hotkeys for pausing (F4) and stopping (F5) keyboard.add_hotkey('f4', pause_speaking) keyboard.add_hotkey('f5', stop_speaking) tts_engine.runAndWait() logging.info("Text-to-Speech completed.") except RuntimeError as e: logging.error(f"TTS Error: {e}") finally: # Remove hotkeys after completion to avoid conflicts keyboard.remove_hotkey('f4') keyboard.remove_hotkey('f5')def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()
def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()
def save_captured_image(image, filename="captured_region.png"): """ Saves the captured image for debugging purposes. """ try: image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")
def save_preprocessed_image(image, filename="preprocessed_image.png"): """ Saves the preprocessed image for debugging purposes. """ try: image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")
def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")
# Step 1: Save the original captured image for verification save_captured_image(image) # Step 2: Convert to grayscale gray = image.convert('L') # Step 3: Enhance contrast enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(2) # Step 4: Apply thresholding threshold = 140 # Adjust based on testing gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') # Step 5: Reduce noise with a median filter final_image = gray_threshold.filter(ImageFilter.MedianFilter()) # Step 6: Save preprocessed image for debugging save_preprocessed_image(final_image) logging.info("Image preprocessing complete. Performing OCR...") # Define Tesseract configuration #custom_config = r'--oem 3 --psm 6' # Adjust as needed # Perform OCR raw_text = pytesseract.image_to_string(final_image, lang='eng') if not raw_text.strip(): logging.warning("OCR returned empty text.") speak_text("I couldn't detect any text in the selected area. Please try again with a different region.") else: ocr_text = raw_text logging.debug("Raw OCR Text:") logging.debug(ocr_text) logging.info("OCR process completed successfully.") except pytesseract.TesseractNotFoundError: logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.") speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.") ocr_text = "" except Exception as e: logging.error(f"OCR Error: {e}") speak_text("An error occurred during OCR processing. Please try again.") ocr_text = ""def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations
screen_width = root.winfo_screenwidth() screen_height = root.winfo_screenheight() root.geometry(f"{screen_width}x{screen_height}") canvas = tk.Canvas(root, cursor="cross", bg="grey") canvas.pack(fill=tk.BOTH, expand=True) rect = None start_x = start_y = end_x = end_y = 0 def on_button_press(event): nonlocal start_x, start_y, rect start_x = int(event.x * scaling_factor) start_y = int(event.y * scaling_factor) rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) logging.debug(f"Button pressed at ({start_x}, {start_y})") def on_move_press(event): nonlocal end_x, end_y end_x = int(event.x * scaling_factor) end_y = int(event.y * scaling_factor) canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y) logging.debug(f"Mouse moved to ({end_x}, {end_y})") def on_button_release(event): root.quit() root.destroy() # Destroy the window to remove the overlay def on_escape(event): logging.info("Capture cancelled by user.") root.quit() root.destroy() sys.exit() canvas.bind("<ButtonPress-1>", on_button_press) canvas.bind("<B1-Motion>", on_move_press) canvas.bind("<ButtonRelease-1>", on_button_release) root.bind("<Escape>", on_escape) logging.info("Please select the region by clicking and dragging the mouse.") root.mainloop() # Calculate absolute coordinates left = min(start_x, end_x) top = min(start_y, end_y) right = max(start_x, end_x) bottom = max(start_y, end_y) logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}") return (left, top, right, bottom)-------------------- Modified Web Search Function --------------------
async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.
Args: query (str): The search query. Returns: list: A list of dictionaries containing 'title' and 'snippet' for each result. """ logging.info(f"Searching the web for query: {query}") # Construct the site-limited query if PREFERRED_SITES: site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES]) combined_query = f"{query} ({site_query})" else: combined_query = query # If no preferred sites, perform a general search logging.debug(f"Combined search query with preferred sites: {combined_query}") # Encode the query for URL encoded_query = aiohttp.helpers.quote(combined_query) url = f"https://html.duckduckgo.com/html/?q={encoded_query}" try: async with aiohttp.ClientSession() as session: async with session.get(url) as response: html = await response.text() except Exception as e: logging.error(f"Web search failed: {e}") return [] soup = BeautifulSoup(html, 'html.parser') results = [] # DuckDuckGo's HTML structure can vary; adjust selectors as needed for result in soup.find_all('div', class_='result__body', limit=3): title_element = result.find('a', class_='result__a') snippet_element = result.find('a', class_='result__snippet') title = title_element.get_text(strip=True) if title_element else "No Title" snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet" results.append({'title': title, 'snippet': snippet}) logging.debug(f"Found result - Title: {title}, Snippet: {snippet}") if not results: logging.warning("No search results found for the preferred sites.") logging.info(f"Web search complete. Found {len(results)} results.") return resultsdef generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])
if user_input: context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.OCR Text: {ocr_text}
User Query: {user_input}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text and the user's query.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response to the user's question based on the available information.
- Ensure that your answer directly addresses the user's intent.
Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.
OCR Text: {ocr_text}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response based on the available information.
- Ensure that your answer directly reflects the content of the OCR text and the search results.
Provide the answer below: """
try: response = ollama.generate(model='llama3.2', prompt=context) final_response = response.get('response', '').strip() logging.info(f"Generated Final Response: {final_response}") return final_response except Exception as e: logging.error(f"LLM Error during final response generation: {e}") return ""def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""
-------------------- Helper Functions --------------------
def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.
Args: short_timeout (int): Maximum number of seconds to wait for a phrase to start. phrase_time_limit (int): Maximum number of seconds a phrase can last. """ global is_listening, speech_text recognizer = sr.Recognizer() with sr.Microphone() as source: logging.info("Listening for voice input... Please say your query or question.") recognizer.adjust_for_ambient_noise(source, duration=1) # Adjust for noise for 1 second try: # Listen for speech with shorter timeout and phrase time limit audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit) logging.info("Processing the audio with Google Speech Recognition...") text = recognizer.recognize_google(audio) # Convert number words to digits if necessary speech_text = convert_number_words(text) logging.info(f"Speech Transcribed: {speech_text}") # Provide confirmation speak_text(f"I heard: {speech_text}.") except sr.WaitTimeoutError: logging.warning("Listening timed out, no speech detected. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.UnknownValueError: logging.warning("Speech recognition could not understand audio. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.RequestError as e: logging.error(f"Could not request results from speech recognition service; {e}") speech_text = "" # Ensure speech_text is empty finally: with lock: is_listening = False # Ensure listening flag is resetdef configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')
# List available voices (optional) for index, voice in enumerate(voices): logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}") # Change voice (select male/female, adjust the index as per your preference) if len(voices) > 1: tts_engine.setProperty('voice', voices[1].id) # Change index to select a different voice logging.info(f"TTS voice set to: {voices[1].name}") else: logging.info("Only one voice available. Using default voice.") # Adjust speaking rate (lower is slower and more human-like) tts_engine.setProperty('rate', 160) # Default is 200, lowering makes it more human-like # Adjust volume (0.0 to 1.0) tts_engine.setProperty('volume', 1.0)def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return
# Step 2: Capture the selected region with mss.mss() as sct: monitor = { "top": top, "left": left, "width": width, "height": height } try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") except Exception as e: logging.error(f"Screen capture failed: {e}") speak_text("Screen capture failed. Please ensure the selected area is correct and try again.") reset_state() return # Step 3: Perform OCR on the captured region perform_ocr(img) if not ocr_text.strip(): logging.error("OCR returned empty text. Please ensure the selected region contains readable text.") speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.") reset_state() return # Step 4: Listen for user input (optional) with shorter timeout transcribe_speech(short_timeout=5, phrase_time_limit=5) # Step 5: Generate and execute the search query generate_and_execute_search(speech_text, ocr_text)def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return
logging.info(f"Using OCR text as search query: {search_query}") # Perform web search and process the answer asyncio.run(async_process_search(search_query, ocr_text, user_input))async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return
# Generate final response using LLM final_response = generate_final_response(search_results, ocr_text, user_input) if not final_response: logging.error("Failed to generate a final response.") speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.") reset_state() return # Speak out the answer speak_text(final_response) reset_state()-------------------- Core Functions --------------------
def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()
def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")
-------------------- Main Function --------------------
async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor
configure_tts_voice() # Voice configuration happens before anything else # Set DPI awareness and get scaling factor set_dpi_awareness() scaling_factor = get_scaling_factor() # Set up hotkeys keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,)) keyboard.add_hotkey('f3', stop_listening) logging.info("QA Assistant is running.") logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.") while True: # Wait for a short duration to avoid busy waiting await asyncio.sleep(0.1) # Check for exit condition if keyboard.is_pressed('esc'): logging.info("Exiting...") speak_text("Exiting the QA Assistant. Goodbye!") break # Reset speech_text and ocr_text is handled after processing # Clean up keyboard hooks before exiting keyboard.unhook_all()-------------------- Entry Point --------------------
if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all() : Add one part where it's outputting the images just before they are fed to ocr to path D:\AI\EXP2\TempS
- Assistant
To modify your script so that it saves images just before they are fed to the OCR process, you can add functionality for saving the preprocessed images to your specified directory (
D:\AI\EXP2\TempS). I've inserted the necessary changes into your script below:import os import mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import time import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama # -------------------- Configuration -------------------- # Configure logging to capture DEBUG level logs logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] ) # Initialize the TTS engine globally tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start # Define global locks lock = threading.Lock() tts_lock = threading.Lock() # -------------------- Preferred Sites Configuration -------------------- # Editable list of preferred sites for prioritized search PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ] # Directory to save images IMAGE_SAVE_DIRECTORY = r"D:\AI\EXP2\TempS" def save_image(image, filename): """ Saves the given image to the specified filename. """ try: if not os.path.exists(IMAGE_SAVE_DIRECTORY): os.makedirs(IMAGE_SAVE_DIRECTORY) filepath = os.path.join(IMAGE_SAVE_DIRECTORY, filename) image.save(filepath) logging.info(f"Image saved as {filepath}") except Exception as e: logging.error(f"Failed to save image {filename}: {e}") def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.") def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0 def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...") # Step 1: Save the original captured image for verification save_image(image, "captured_region.png") # Step 2: Convert to grayscale gray = image.convert('L') # Step 3: Enhance contrast enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(2) # Step 4: Apply thresholding threshold = 140 # Adjust based on testing gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') # Step 5: Reduce noise with a median filter final_image = gray_threshold.filter(ImageFilter.MedianFilter()) # Step 6: Save preprocessed image for debugging save_image(final_image, "preprocessed_image.png") logging.info("Image preprocessing complete. Performing OCR...") # Define Tesseract configuration #custom_config = r'--oem 3 --psm 6' # Adjust as needed # Perform OCR raw_text = pytesseract.image_to_string(final_image, lang='eng') if not raw_text.strip(): logging.warning("OCR returned empty text.") speak_text("I couldn't detect any text in the selected area. Please try again with a different region.") else: ocr_text = raw_text logging.debug("Raw OCR Text:") logging.debug(ocr_text) logging.info("OCR process completed successfully.") except pytesseract.TesseractNotFoundError: logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.") speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.") ocr_text = "" except Exception as e: logging.error(f"OCR Error: {e}") speak_text("An error occurred during OCR processing. Please try again.") ocr_text = "" # The remaining part of the script continues as before...Changes Made:
- Directory Definition: I introduced a constant
IMAGE_SAVE_DIRECTORYto specify where to save the images. - Image Saving Function: Modified the
save_imagefunction to ensure your preprocessed images and captured regions correctly save to your specified path. It checks if the directory exists and creates it if it doesn't. - Function Calls: Incorporated calls to
save_imageafter each preprocessing step to save the original captured image and the preprocessed image right before they are sent for OCR processing.
These changes ensure that you have a visual record of both the original and preprocessed images, which can help identify any issues.
- Directory Definition: I introduced a constant
- User
It is doing incorrect screen capture that has offset from what was selected using mouseimport mss from PIL import Image, ImageEnhance, ImageFilter import pytesseract import speech_recognition as sr import pyttsx3 import asyncio import aiohttp from bs4 import BeautifulSoup import keyboard import threading import os import re import logging import tkinter as tk from word2number import w2n # Optional for number word conversion import ctypes import sys import ollama from datetime import datetime
-------------------- Configuration --------------------
Configure logging to capture DEBUG level logs
logging.basicConfig( level=logging.DEBUG, # Set to DEBUG for detailed logs format='%(asctime)s [%(levelname)s] %(message)s', handlers=[ logging.StreamHandler(), # Outputs to console # Uncomment the next line to enable logging to a file # logging.FileHandler("qa_assistant.log"), ] )
Initialize the TTS engine globally
tts_engine = pyttsx3.init() is_listening = False # Initialize this global variable at the start speech_text = "" # Initialize this global variable at the start
Define global locks
lock = threading.Lock() tts_lock = threading.Lock()
-------------------- Preferred Sites Configuration --------------------
Editable list of preferred sites for prioritized search
PREFERRED_SITES = [ "civiljungle.org", "www.nwcg.gov", "the.constructor.org", "definecivil.com", "testbook.com/civil-engineering", "sanfoundry.com", # Add or remove sites as needed ]
Define the output directory for saved images
OUTPUT_DIR = (r"D:\AI\EXP2\TempS")
def set_dpi_awareness(): """ Sets the DPI awareness of the process to prevent scaling issues. """ try: if sys.platform.startswith('win'): awareness = ctypes.c_int() # PROCESS_SYSTEM_DPI_AWARE = 1 ctypes.windll.shcore.SetProcessDpiAwareness(1) logging.info("Process DPI awareness set to SYSTEM_AWARE.") except Exception as e: logging.warning(f"Could not set DPI awareness: {e}. Continuing without setting DPI awareness.")
def get_scaling_factor(): """ Retrieves the current DPI scaling factor. Returns: float: Scaling factor (e.g., 1.25 for 125%) """ try: if sys.platform.startswith('win'): user32 = ctypes.windll.user32 try: # Windows 10 and above user32.SetProcessDPIAware() dpi = user32.GetDpiForWindow(user32.GetForegroundWindow()) scaling_factor = dpi / 96 # 96 DPI is 100% logging.info(f"DPI Scaling Factor detected: {scaling_factor}") return scaling_factor except AttributeError: # For older Windows versions user32.SetProcessDPIAware() hdc = user32.GetDC(0) LOGPIXELSX = 88 dpi = ctypes.windll.gdi32.GetDeviceCaps(hdc, LOGPIXELSX) scaling_factor = dpi / 96 ctypes.windll.user32.ReleaseDC(0, hdc) logging.info(f"DPI Scaling Factor detected via fallback: {scaling_factor}") return scaling_factor else: # For other operating systems, default to 1.0 logging.info("Non-Windows OS detected. Defaulting DPI scaling factor to 1.0.") return 1.0 except Exception as e: logging.warning(f"Could not retrieve DPI scaling factor: {e}. Defaulting to 1.0.") return 1.0
def convert_number_words(text): """ Converts number words in the speech text to digits. Example: "five hundred ninety-seven" -> "597" """ try: # Attempt to convert the entire text number = w2n.word_to_num(text) return str(number) except: # Attempt to extract digits directly if conversion fails digits = re.findall(r'\d+', text) return ''.join(digits) if digits else text # Return the number if found, else original text
def normalize_option_label(label): """ Normalizes OCR-misread option labels to standard A-E. """ label = label.upper() if label in ['A', 'B', 'C', 'D', 'E']: return label # Handle common OCR misinterpretations ocr_misreads = { '©': 'C', 'S': '5', 'À': 'A', 'ß': 'B', '(': '', # Remove any stray characters ')': '', '-': '', # Add more mappings if needed } return ocr_misreads.get(label, label) # Default to original if no mapping found
def speak_text(text): """ Speaks out the provided text using TTS. Adds hotkeys for pausing (F4) and stopping (F5) speech. """ global tts_engine logging.info(f"Speaking out the response: {text}") with tts_lock: try: # Speak the text tts_engine.say(text)
# Add hotkeys for pausing (F4) and stopping (F5) keyboard.add_hotkey('f4', pause_speaking) keyboard.add_hotkey('f5', stop_speaking) tts_engine.runAndWait() logging.info("Text-to-Speech completed.") except RuntimeError as e: logging.error(f"TTS Error: {e}") finally: # Remove hotkeys after completion to avoid conflicts keyboard.remove_hotkey('f4') keyboard.remove_hotkey('f5')def pause_speaking(): """ Pauses the TTS engine. """ global tts_engine logging.info("Paused speaking...") tts_engine.pause()
def stop_speaking(): """ Stops the TTS engine. """ global tts_engine logging.info("Stopped speaking.") tts_engine.stop()
def save_captured_image(image, output_dir, prefix="captured_region"): """ Saves the captured image for debugging purposes with a timestamp.
Args: image (PIL.Image): The image to save. output_dir (str): Directory where the image will be saved. prefix (str): Prefix for the filename. """ try: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png") image.save(filename) logging.info(f"Captured image saved as {filename}") except Exception as e: logging.error(f"Failed to save captured image: {e}")def save_preprocessed_image(image, output_dir, prefix="preprocessed_image"): """ Saves the preprocessed image for debugging purposes with a timestamp.
Args: image (PIL.Image): The image to save. output_dir (str): Directory where the image will be saved. prefix (str): Prefix for the filename. """ try: timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") filename = os.path.join(output_dir, f"{prefix}_{timestamp}.png") image.save(filename) logging.info(f"Preprocessed image saved as {filename}") except Exception as e: logging.error(f"Failed to save preprocessed image: {e}")def perform_ocr(image): """ Performs OCR on the captured image after preprocessing. Uses the preprocessed image for better accuracy. """ global ocr_text try: logging.info("Starting OCR process with image preprocessing...")
# Step 1: Save the original captured image for verification save_captured_image(image, OUTPUT_DIR) # Step 2: Convert to grayscale gray = image.convert('L') logging.debug("Converted image to grayscale.") # Step 3: Enhance contrast enhancer = ImageEnhance.Contrast(gray) gray_enhanced = enhancer.enhance(2.0) # Adjust contrast factor as needed logging.debug("Enhanced image contrast.") # Step 4: Apply thresholding threshold = 140 # Adjust based on testing gray_threshold = gray_enhanced.point(lambda x: 0 if x < threshold else 255, '1') logging.debug("Applied thresholding to binarize the image.") # Step 5: Reduce noise with a median filter final_image = gray_threshold.filter(ImageFilter.MedianFilter()) logging.debug("Applied median filter to reduce noise.") # Step 6: Save preprocessed image for debugging save_preprocessed_image(final_image, OUTPUT_DIR) logging.info("Image preprocessing complete. Performing OCR...") # Define Tesseract configuration custom_config = r'--oem 3 --psm 6' # Adjust as needed # Perform OCR raw_text = pytesseract.image_to_string(final_image, lang='eng', config=custom_config) if not raw_text.strip(): logging.warning("OCR returned empty text.") speak_text("I couldn't detect any text in the selected area. Please try again with a different region.") else: ocr_text = raw_text logging.debug("Raw OCR Text:") logging.debug(ocr_text) logging.info("OCR process completed successfully.") except pytesseract.TesseractNotFoundError: logging.error("Tesseract executable not found. Please ensure Tesseract is installed and PATH is set correctly.") speak_text("Tesseract is not installed or not found. Please install Tesseract and try again.") ocr_text = "" except Exception as e: logging.error(f"OCR Error: {e}") speak_text("An error occurred during OCR processing. Please try again.") ocr_text = ""def get_selected_region(scaling_factor): """ Allows the user to select a screen region by drawing a rectangle. Returns the coordinates of the selected region adjusted for DPI scaling. """ # Initialize the tkinter window root = tk.Tk() root.title("Select Region (Press Esc to Cancel)") root.attributes('-alpha', 0.3) # Transparency root.attributes("-topmost", True) root.overrideredirect(True) # Remove window decorations
screen_width = root.winfo_screenwidth() screen_height = root.winfo_screenheight() root.geometry(f"{screen_width}x{screen_height}") canvas = tk.Canvas(root, cursor="cross", bg="grey") canvas.pack(fill=tk.BOTH, expand=True) rect = None start_x = start_y = end_x = end_y = 0 def on_button_press(event): nonlocal start_x, start_y, rect start_x = int(event.x * scaling_factor) start_y = int(event.y * scaling_factor) rect = canvas.create_rectangle(event.x, event.y, event.x, event.y, outline='red', width=2) logging.debug(f"Button pressed at ({start_x}, {start_y})") def on_move_press(event): nonlocal end_x, end_y end_x = int(event.x * scaling_factor) end_y = int(event.y * scaling_factor) canvas.coords(rect, start_x / scaling_factor, start_y / scaling_factor, event.x, event.y) logging.debug(f"Mouse moved to ({end_x}, {end_y})") def on_button_release(event): root.quit() root.destroy() # Destroy the window to remove the overlay def on_escape(event): logging.info("Capture cancelled by user.") root.quit() root.destroy() sys.exit() canvas.bind("<ButtonPress-1>", on_button_press) canvas.bind("<B1-Motion>", on_move_press) canvas.bind("<ButtonRelease-1>", on_button_release) root.bind("<Escape>", on_escape) logging.info("Please select the region by clicking and dragging the mouse.") root.mainloop() # Calculate absolute coordinates left = min(start_x, end_x) top = min(start_y, end_y) right = max(start_x, end_x) bottom = max(start_y, end_y) logging.debug(f"Selected region coordinates (scaled): Left={left}, Top={top}, Right={right}, Bottom={bottom}") return (left, top, right, bottom)-------------------- Modified Web Search Function --------------------
async def async_web_search(query): """ Performs a web search using DuckDuckGo and retrieves the top 3 results from the preferred sites.
Args: query (str): The search query. Returns: list: A list of dictionaries containing 'title' and 'snippet' for each result. """ logging.info(f"Searching the web for query: {query}") # Construct the site-limited query if PREFERRED_SITES: site_query = " OR ".join([f"site:{site}" for site in PREFERRED_SITES]) combined_query = f"{query} ({site_query})" else: combined_query = query # If no preferred sites, perform a general search logging.debug(f"Combined search query with preferred sites: {combined_query}") # Encode the query for URL encoded_query = aiohttp.helpers.quote(combined_query) url = f"https://html.duckduckgo.com/html/?q={encoded_query}" try: async with aiohttp.ClientSession() as session: async with session.get(url) as response: html = await response.text() except Exception as e: logging.error(f"Web search failed: {e}") return [] soup = BeautifulSoup(html, 'html.parser') results = [] # DuckDuckGo's HTML structure can vary; adjust selectors as needed for result in soup.find_all('div', class_='result__body', limit=3): title_element = result.find('a', class_='result__a') snippet_element = result.find('a', class_='result__snippet') title = title_element.get_text(strip=True) if title_element else "No Title" snippet = snippet_element.get_text(strip=True) if snippet_element else "No Snippet" results.append({'title': title, 'snippet': snippet}) logging.debug(f"Found result - Title: {title}, Snippet: {snippet}") if not results: logging.warning("No search results found for the preferred sites.") logging.info(f"Web search complete. Found {len(results)} results.") return resultsdef generate_final_response(search_results, ocr_text, user_input): """ Generates the final response based on web search results and OCR text. """ logging.info("Generating final response...") # Format the search results for better readability formatted_results = "\n".join([f"Title: {res['title']}\nSnippet: {res['snippet']}\n" for res in search_results])
if user_input: context = f"""You are an intelligent assistant specialized in answering questions based on OCR text and web search results.OCR Text: {ocr_text}
User Query: {user_input}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text and the user's query.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response to the user's question based on the available information.
- Ensure that your answer directly addresses the user's intent.
Provide the answer below: """ else: context = f"""You are an intelligent assistant specialized in answering questions based solely on OCR text and web search results.
OCR Text: {ocr_text}
Web Search Results: {formatted_results}
Instructions:
- Analyze the OCR text.
- Utilize the web search results to inform your answer.
- Provide a well-informed, concise, and accurate response based on the available information.
- Ensure that your answer directly reflects the content of the OCR text and the search results.
Provide the answer below: """
try: response = ollama.generate(model='llama3.2', prompt=context) final_response = response.get('response', '').strip() logging.info(f"Generated Final Response: {final_response}") return final_response except Exception as e: logging.error(f"LLM Error during final response generation: {e}") return ""def reset_state(): """ Resets the global state variables. """ global speech_text, ocr_text speech_text = "" ocr_text = ""
-------------------- Helper Functions --------------------
def transcribe_speech(short_timeout=5, phrase_time_limit=5): """ Transcribes speech from the microphone with a short timeout. If no speech is detected within the timeout, proceeds without user input.
Args: short_timeout (int): Maximum number of seconds to wait for a phrase to start. phrase_time_limit (int): Maximum number of seconds a phrase can last. """ global is_listening, speech_text recognizer = sr.Recognizer() with sr.Microphone() as source: logging.info("Listening for voice input... Please say your query or question.") recognizer.adjust_for_ambient_noise(source, duration=1) # Adjust for noise for 1 second try: # Listen for speech with shorter timeout and phrase time limit audio = recognizer.listen(source, timeout=short_timeout, phrase_time_limit=phrase_time_limit) logging.info("Processing the audio with Google Speech Recognition...") text = recognizer.recognize_google(audio) # Convert number words to digits if necessary speech_text = convert_number_words(text) logging.info(f"Speech Transcribed: {speech_text}") # Provide confirmation speak_text(f"I heard: {speech_text}.") except sr.WaitTimeoutError: logging.warning("Listening timed out, no speech detected. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.UnknownValueError: logging.warning("Speech recognition could not understand audio. Proceeding without voice input.") speech_text = "" # Ensure speech_text is empty except sr.RequestError as e: logging.error(f"Could not request results from speech recognition service; {e}") speech_text = "" # Ensure speech_text is empty finally: with lock: is_listening = False # Ensure listening flag is resetdef configure_tts_voice(): """ Configures the TTS engine's voice, rate, and volume. """ global tts_engine voices = tts_engine.getProperty('voices')
# List available voices (optional) for index, voice in enumerate(voices): logging.debug(f"Voice {index}: {voice.name}, {voice.gender}, {voice.languages}") # Change voice (select male/female, adjust the index as per your preference) if len(voices) > 1: tts_engine.setProperty('voice', voices[1].id) # Change index to select a different voice logging.info(f"TTS voice set to: {voices[1].name}") else: logging.info("Only one voice available. Using default voice.") # Adjust speaking rate (lower is slower and more human-like) tts_engine.setProperty('rate', 160) # Default is 200, lowering makes it more human-like # Adjust volume (0.0 to 1.0) tts_engine.setProperty('volume', 1.0)def capture_and_save_image(): """ Coordinates the screen capture and OCR process. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return
# Step 2: Capture the selected region with mss.mss() as sct: monitor = { "top": top, "left": left, "width": width, "height": height } try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") except Exception as e: logging.error(f"Screen capture failed: {e}") speak_text("Screen capture failed. Please ensure the selected area is correct and try again.") reset_state() return # Step 3: Perform OCR on the captured region perform_ocr(img) if not ocr_text.strip(): logging.error("OCR returned empty text. Please ensure the selected region contains readable text.") speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.") reset_state() return # Step 4: Listen for user input (optional) with shorter timeout transcribe_speech(short_timeout=5, phrase_time_limit=5) # Step 5: Generate and execute the search query generate_and_execute_search(speech_text, ocr_text)def generate_and_execute_search(user_input, ocr_text): """ Generates and executes the search query using OCR text directly. """ # Use OCR text directly as search query search_query = ocr_text.strip() if not search_query: logging.error("No OCR text available for search query.") speak_text("No text detected from the screen. Please try again by pressing F2.") reset_state() return
logging.info(f"Using OCR text as search query: {search_query}") # Perform web search and process the answer asyncio.run(async_process_search(search_query, ocr_text, user_input))async def async_process_search(search_query, ocr_text, user_input): """ Asynchronously processes the web search and generates the final response. """ # Perform web search search_results = await async_web_search(search_query) if not search_results: logging.warning("No search results found. Unable to answer the question.") speak_text("I'm sorry, I couldn't find an answer to your question.") reset_state() return
# Generate final response using LLM final_response = generate_final_response(search_results, ocr_text, user_input) if not final_response: logging.error("Failed to generate a final response.") speak_text("I couldn't determine the answer to your question. Please try again by pressing F2.") reset_state() return # Speak out the answer speak_text(final_response) reset_state()-------------------- Core Functions --------------------
def start_listening(scaling): """ Starts the listening process by initiating region selection and OCR. Runs in a separate thread to prevent blocking. """ global is_listening with lock: if not is_listening: is_listening = True logging.info("Started listening...") # Perform region selection and OCR in a separate thread threading.Thread(target=capture_and_save_image, args=(), daemon=True).start()
def stop_listening(): """ Stops the listening process if it's currently active. """ global is_listening with lock: if is_listening: is_listening = False logging.info("Stopped listening.") speak_text("Stopped listening.")
-------------------- Main Function --------------------
async def main(): """ Main asynchronous function that sets up the hotkeys and runs the event loop. """ global scaling_factor
configure_tts_voice() # Voice configuration happens before anything else # Set DPI awareness and get scaling factor set_dpi_awareness() scaling_factor = get_scaling_factor() # Set up hotkeys keyboard.add_hotkey('f2', start_listening, args=(scaling_factor,)) keyboard.add_hotkey('f3', stop_listening) logging.info("QA Assistant is running.") logging.info("Press 'F2' to select a region and answer a question, 'F3' to stop listening, 'Esc' to exit.") while True: # Wait for a short duration to avoid busy waiting await asyncio.sleep(0.1) # Check for exit condition if keyboard.is_pressed('esc'): logging.info("Exiting...") speak_text("Exiting the QA Assistant. Goodbye!") break # Reset speech_text and ocr_text is handled after processing # Clean up keyboard hooks before exiting keyboard.unhook_all()-------------------- Entry Point --------------------
if name == "main": try: asyncio.run(main()) except KeyboardInterrupt: logging.info("Program terminated by user.") except Exception as e: logging.error(f"An unexpected error occurred: {e}") finally: # Ensure all keyboard hooks are removed before exiting keyboard.unhook_all()
- Assistant
It sounds like you're experiencing an issue where the captured image is offset from the area selected with the mouse. This can happen due to DPI scaling issues on high-resolution displays where the logical screen size is different from the physical screen size.
Here's a checklist and some changes you can try to resolve this issue:
-
Verify DPI Awareness: Ensure your application is truly DPI aware. You have already configured DPI awareness in your code, but let's make sure it's correctly implemented.
-
Adjust for DPI Scaling: When capturing the screen region, consider the DPI scaling factor to adjust the coordinates appropriately.
-
Debug Logging: Verify the coordinates being used for capturing the screen by adding detailed logging.
Here is a modified version of your function that tries to address potential issues with DPI scaling:
def capture_and_save_image(): """ Coordinates the screen capture and OCR process. Adjusts for DPI scaling if necessary. """ global speech_text, ocr_text # Step 1: Let the user select a region region = get_selected_region(scaling_factor) left, top, right, bottom = region width = right - left height = bottom - top logging.debug(f"Width: {width}, Height: {height}, Scaling Factor: {scaling_factor}") if width <= 0 or height <= 0: logging.error("Invalid region selected. Please try again.") speak_text("Invalid region selected. Please try again by pressing F2.") reset_state() return # Step 2: Capture the selected region with mss.mss() as sct: monitor = { "top": int(top / scaling_factor), "left": int(left / scaling_factor), "width": int(width / scaling_factor), "height": int(height / scaling_factor) } logging.debug(f"Monitor for capture: {monitor}") try: screenshot = sct.grab(monitor) img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") except Exception as e: logging.error(f"Screen capture failed: {e}") speak_text("Screen capture failed. Please ensure the selected area is correct and try again.") reset_state() return # Step 3: Perform OCR on the captured region perform_ocr(img) if not ocr_text.strip(): logging.error("OCR returned empty text. Please ensure the selected region contains readable text.") speak_text("OCR did not detect any text. Please ensure the selected area contains readable text and try again.") reset_state() return # Step 4: Listen for user input (optional) with shorter timeout transcribe_speech(short_timeout=5, phrase_time_limit=5) # Step 5: Generate and execute the search query generate_and_execute_search(speech_text, ocr_text)Key Changes:
-
Monitor Adjustment: The coordinates for the monitor to be captured are adjusted by dividing by the
scaling_factor. This adjustment should help resolve capturing the wrong screen region if DPI scaling is affecting the display coordinates. -
Detailed Logging: Additional debug logging will help track the dimensions and coordinates of the region being selected and captured.
Additional Recommendations:
-
Testing Without DPI Scaling: Test this approach with your system's DPI scaling set to 100% to identify if the problem persists without DPI scaling.
-
Verify Scaling Factor: Check the scaling factor you retrieve to ensure it's accurate compared to your system settings.
Following these adjustments should help correct the screen capture coordinates when dealing with different DPI scalings.
-