help me figure out why this isnt working as intented, it's suppose to look on the left…
- User
help me figure out why this isnt working as intented, it's suppose to look on the left side of the cropped image and react when there's a black dot showing, but it just says "The two images have different sizes The two images have different sizes" "The cropped frames are empty" "Not found":
def check_for_dot(frame, previous_frame, x, y, w, h, threshold): # Convert the image to grayscale current_frame = frame.copy()
# Check if the two images have the same size if current_frame.shape != previous_frame.shape: print("The two images have different sizes") return "Not found" # Check if the two images have the same number of channels if current_frame.ndim != previous_frame.ndim: print("The two images have a different number of channels") return "Not found" # Crop the images to the specified area current_frame = current_frame[y : y + h, x : x + w] previous_frame = previous_frame[y : y + h, x : x + w] # Check if the cropped frames are not empty if current_frame.size != 0 and previous_frame.size != 0: # Convert the image to grayscale current_frame = cv2.cvtColor(current_frame, cv2.COLOR_BGR2GRAY) previous_frame = cv2.cvtColor(previous_frame, cv2.COLOR_BGR2GRAY) else: print("The cropped frames are empty") # Compute the absolute difference between the current frame and the previous frame frame_diff = cv2.absdiff(current_frame, previous_frame) # Convert the difference image to binary _, binary_diff = cv2.threshold(frame_diff, 10, 255, cv2.THRESH_BINARY) # Count the number of black pixels num_black_pixels = cv2.countNonZero(binary_diff) # Check if the number of black pixels is above the threshold if num_black_pixels > threshold: print("Found it") return "Found it" else: print("Not found") return "Not found"x, y, w, h = 100, 100, 200, 200 threshold = 400
@app.post("/capture_image") async def capture_image(): global SERVER_RESPONSE previous_frame = None
# Adjust the green color range green_lower = np.array([1, 177, 115], np.uint8) green_upper = np.array([85, 255, 255], np.uint8) # Load image from file instead of camera url = "http://192.168.229.55:8080/shot.jpg" while True: img_resp = requests.get(url) img_arr = np.frombuffer(img_resp.content, np.uint8) frame = cv2.imdecode(img_arr, -1) # Convert the image from BGR to HSV color space hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) # Create a binary mask where white represents the green color green_mask = cv2.inRange(hsv, green_lower, green_upper) # Apply morphological operations to reduce noise kernel = np.ones((5, 5), np.uint8) green_mask = cv2.dilate(green_mask, kernel, iterations=1) green_mask = cv2.erode(green_mask, kernel, iterations=1) # Find contours in the green mask contours, _ = cv2.findContours( green_mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE ) if len(contours) != 0: max_contour = max(contours, key=cv2.contourArea) x, y, w, h = cv2.boundingRect(max_contour) # Ensure the coordinates are in the correct order cropped = frame[y : y + h, x : x + w] # Calculate the area of the largest contour green_area = cv2.contourArea(max_contour) # Check if the detected green area is of a certain size AREA_THRESHOLD = 5000 # adjust this value based on your requirement if green_area > AREA_THRESHOLD: # Check if the directory exists, if not create it if not os.path.isdir("images"): os.mkdir("images") # Save the image height, width = cropped.shape[:2] if width < height: cropped = cv2.rotate(cropped, cv2.ROTATE_90_CLOCKWISE) # Ensure the coordinates are in the correct order cropped = frame[y : y + h, x : x + w] # Check if the cropped frame is not empty if cropped.size != 0: # Check if this is the first frame if previous_frame is not None: # Check if the sizes of the current frame and the previous frame are the same if cropped.shape == previous_frame.shape: # Check for dot result = check_for_dot( cropped, previous_frame, x, y, w, h, threshold ) if result == "Found it": print("Changed") cv2.imwrite("images/captured_image2.jpg", cropped) result = check_picture("images/captured_image2.jpg") SERVER_RESPONSE = result[0] await generate_audio() return result else: print("The two images have different sizes") # Store current frame as previous frame for next iteration previous_frame = cropped.copy() # Display the resulting frame cv2.imshow("Frame", cropped) # Press Q on keyboard to exit if cv2.waitKey(1) & 0xFF == ord("q"): break else: print("The cropped frame is empty") return "Not found" - Assistant
There are a few potential issues with the code that can cause it not to work as intended. Here are some points of interest that might help you troubleshoot the problem:
-
Image Size and Channel Mismatch Checks: Ensure that previous_frame is set at least once before being used. If it's never set, it will always be
None, and the size and channel checks will always fail. You might want to move the check outside thewhile Trueloop and after you have captured the first frame. -
Previous Frame Initialization: The
previous_framevariable is initialized asNoneand is intended to store the previous frame. However, it only gets assigned after all processing. If your intent is to check the difference between subsequent frames, you need a proper initialization after the first frame has been processed. -
Cropped Frames and Valid Regions: The
check_for_dotfunction expects to work withx, y, w, hparameters relative to the originalframe. However, when you passcroppedasframetocheck_for_dot, you should not crop it again with the same coordinates within the function as it will result in incorrect or out-of-bound indices. Instead, use0, 0, w, hif you intend to scan the entire cropped image. -
Contour Finding and Area Filtering: You are finding contours inside a loop and filtering by area. However, not all frames might contain the green marker, and not all found contours may exceed the threshold. Ensure that you handle cases when no suitable contours are found.
-
Green Mask Application: While you're creating a mask for green areas and performing morphological operations on it, there is no further use of this
green_maskto filter out the green areas before looking for the black dot. -
Server Response and Async Behavior: The server's endpoint is defined as async, yet the loop doesn't seem to be designed to work with async behavior properly. If the intention is to return from an endpoint, remember that a loop like
while Truemay block the async loop unless there are awaits inside it that can yield control. -
Saving and Reading from File: You are saving an image to "images/captured_image2.jpg" and then immediately calling
check_pictureon it, but the implementation ofcheck_pictureis not provided. A potential problem could be related to how the saved image is evaluated incheck_picture. -
Image Display and Exit: The condition that checks for a key press and exits
if cv2.waitKey(1) & 0xFF == ord('q')is obsolete in an async endpoint function. It works only if the display window is properly handled within a desktop application.
Overall, the debugging should proceed with ensuring that:
- The server can handle incoming requests in an asynchronous manner.
- The algorithm for processing frames properly initializes and keeps track of the previous frame.
- The
check_for_dotfunction receives correctly cropped images with the correct dimensions and does not attempt to re-crop them further. - The conditions under which a black dot is searched for are clearly defined and met (e.g., a suitable area is detected).
- All potential exceptions or edge cases are handled, such as empty cropped frames or different image sizes.
Simplify and test each part of the function separately, then make sure they integrate correctly, respecting the asynchronous behavior and image processing requirements.
-
- User
suggest ways to fix all of them for me with code
- Assistant
Sure, I'll provide you with some guidance to address the highlighted issues.
- Initialize
previous_frameafter the first frame is processed:
# After processing the first frame, set the previous_frame if previous_frame is None: previous_frame = current_frame.copy()- Modify the
check_for_dotfunction to use the entire cropped image:
def check_for_dot(current_frame, previous_frame, threshold): # Assume that current_frame and previous_frame are already cropped # and no further cropping is needed here. Remove x, y, w, h parameters. ...- Use the green mask to identify areas of interest:
# After finding contours and selecting the largest one if green_area > AREA_THRESHOLD: # Fill out the area outside the largest contour with black (ignore smaller contours) mask_inverted = cv2.bitwise_not(green_mask) frame_masked = cv2.bitwise_and(frame, frame, mask=mask_inverted) # Rest of your code ...- Simplify the loop to handle frames in order, and only process when green areas are found:
# Replace the while True loop in capture_image with a single iteration # This function will process one image at a time and must be called repeatedly async def capture_image(): global SERVER_RESPONSE, previous_frame # Rest of your image processing logic to capture frame and find green area ... # Check if a suitable area was found if green_area > AREA_THRESHOLD: # Rest of the logic to check for black dot and process the image ... else: print("No suitable area found or the area is too small") # Return a suitable response or status if needed ...- Modify your server response logic to integrate with your async endpoint:
async def capture_image(): global SERVER_RESPONSE, previous_frame # Code to capture and process frame ... if found_black_dot: SERVER_RESPONSE = "Found Black Dot" else: SERVER_RESPONSE = "Black Dot Not Found" # Return SERVER_RESPONSE as a part of an HTTP response return SERVER_RESPONSEThe above changes aim to correct the issues identified in the original code. Remember to correctly initialize
previous_frameafter the first-frame processing, adjust your function to handle cropped images and restructure your loop within the asynchronous endpoint.- Since
cv2.imshowandcv2.waitKeyfunctions are blocking and don't work well with async, you might want to remove them or replace them with a suitable web-based viewing mechanism if you’re working in a web server context.
Please note that without the complete context and functions like
check_picture, the exact correction may still require additional modifications. The suggestions above assume thatprevious_frameis already the correctly cropped previous image to compare with the current cropped frame. Also, thecapture_imagefunction now expects to be called repeatedly for each frame and does not have a loop in it; this means you would need to change the logic that calls this function to repeatedly invoke it for each frame to be processed. - Initialize
- User
what is the easiest way to extract a window from a video? the color way I'm doing is too dependet on the lightning
- Assistant
If the window has a consistent appearance across video frames and is distinguishable from its surroundings, there are more robust methods you can employ other than color extraction that may improve detection under varying lighting conditions:
-
Feature Matching: You can use feature matching techniques, such as SIFT, SURF, or ORB, to find the window within each frame. These techniques are less sensitive to changes in lighting or slight variations in perspective.
- First, extract key features from an image of the window.
- Then, in each video frame, detect features and compare them with the features of the window to find the best match.
-
Template Matching: When the window size and orientation are constant and the video’s scale doesn’t change much, template matching can be an effective tool. It uses a template image to scan the video frames and find areas that match.
- Cut out a portion of the window that you want to track as a template image.
- Use
cv2.matchTemplate()to search for matching areas in each frame.
-
Edge Detection and Contour-based Methods: If the window can be identified by its edges, use edge detection (e.g., Canny) followed by finding contours with
cv2.findContours(). -
Background Subtraction: If the window area remains static and everything else moves, background subtraction allows you to detect static foreground objects.
-
Deep Learning: Train a custom object detection or instance segmentation model to detect the window directly. This requires a dataset of images labeled with the window's location.
Here’s a simple template matching example using OpenCV:
import cv2 video_path = 'path_to_your_video.mp4' template_image = cv2.imread('path_to_window_template.jpg') h, w = template_image.shape[:2] # Convert template to grayscale template_gray = cv2.cvtColor(template_image, cv2.COLOR_BGR2GRAY) # Open the video cap = cv2.VideoCapture(video_path) while True: ret, frame = cap.read() if not ret: break # End of video # Convert frame to grayscale gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Perform template matching res = cv2.matchTemplate(gray_frame, template_gray, cv2.TM_CCOEFF_NORMED) # Set a threshold and find where the matching result exceeds it threshold = 0.8 loc = np.where(res >= threshold) # Draw rectangles around detected windows for pt in zip(*loc[::-1]): cv2.rectangle(frame, pt, (pt[0] + w, pt[1] + h), (0, 255, 0), 2) # Show the frame cv2.imshow('Detected Window', frame) # Exit on pressing 'q' if cv2.waitKey(1) & 0xFF == ord('q'): break # Release the video and destroy windows cap.release() cv2.destroyAllWindows()Each method has its advantages and limitations, and the effectiveness will depend on the specifics of your video. Multiple methods might be combined for improved detection accuracy, such as using feature matching to refine the regions detected by edges or colors.
-
- User
rewrite this to work with the template:
@app.post("/capture_image") async def capture_image(): global SERVER_RESPONSE previous_frame = None
# Load image from file instead of camera url = "http://10.30.225.127:8080/shot.jpg" while True: img_resp = requests.get(url) img_arr = np.frombuffer(img_resp.content, np.uint8) frame = cv2.imdecode(img_arr, -1) gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Convert the image from BGR to HSV color space hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV) # Create a binary mask where white represents the green color # Apply morphological operations to reduce noise res = cv2.matchTemplate(gray_frame, template_gray, cv2.TM_CCOEFF_NORMED) if len(res) != 0: x, y, w, h = cv2.boundingRect(res) # Ensure the coordinates are in the correct order cropped = frame[y : y + h, x : x + w] # Calculate the area of the largest contour green_area = cv2.contourArea(res) # Check if the detected green area is of a certain size AREA_THRESHOLD = 2000 # adjust this value based on your requirement if green_area > AREA_THRESHOLD: # Fill out the area outside the largest contour with black (ignore smaller contours) # Check if the directory exists, if not create it if not os.path.isdir("images"): os.mkdir("images") # Save the image height, width = cropped.shape[:2] if width < height: cropped = cv2.rotate(cropped, cv2.ROTATE_90_CLOCKWISE) # Ensure the coordinates are in the correct order cropped = frame[y : y + h, x : x + w] # Convert the image to grayscale # Apply adaptive thresholding # Check if the cropped frame is not empty if cropped.size != 0: # Check if this is the first frame if previous_frame is not None: # Check if the sizes of the current frame and the previous frame are the same if cropped.shape == previous_frame.shape: # Check for dot result = check_for_dot(cropped, previous_frame, threshold) if result == "Found its": print("Changed") cv2.imwrite("images/captured_image2.jpg", cropped) result = check_picture("images/captured_image2.jpg") SERVER_RESPONSE = result[0] await generate_audio() return result else: print("The two images have different sizes") # Store current frame as previous frame for next iteration previous_frame = cropped.copy() # Display the resulting frame cv2.imshow("Frame", cropped) # Press Q on keyboard to exit if cv2.waitKey(1) & 0xFF == ord("q"): break else: print("The cropped frame is empty") return "Not found" - Assistant
Before rewriting the function, I will address some logical inconsistencies and suggest improvements:
-
This code is returning from within a
while Trueloop. The first time you get a match and process it, the function will return a result, ending the loop and not processing any more frames. This might not be what you want in a live capture scenario. For a REST API, you might want to process just one frame per request. -
You are trying to threshold a grayscale image to find contours directly from template matching results—which is incorrect, as
matchTemplatereturns a score map, not a binary mask. Instead, find the location with the highest score. -
The code seems to be a mix of approaches (color thresholding and now template matching). Ensure you're only using the required process.
With these points in mind, here's a reworked version of your script:
from fastapi import FastAPI import numpy as np import cv2 import os import requests from aiohttp import ClientSession app = FastAPI() # Load template image template_image_path = 'path_to_template_image.jpg' template_image = cv2.imread(template_image_path, 0) template_height, template_width = template_image.shape[:2] # Define the threshold for detection MATCH_THRESHOLD = 0.7 async def fetch_frame(url): async with ClientSession() as session: async with session.get(url) as response: content = await response.read() return np.frombuffer(content, np.uint8) @app.post("/capture_image") async def capture_image(): global SERVER_RESPONSE previous_frame = None url = "http://10.30.225.127:8080/shot.jpg" # Fetch the frame img_arr = await fetch_frame(url) frame = cv2.imdecode(img_arr, -1) gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Perform template matching res = cv2.matchTemplate(gray_frame, template_image, cv2.TM_CCOEFF_NORMED) # Find locations where matches are above the threshold match_locations = np.where(res >= MATCH_THRESHOLD) if len(match_locations[0]) > 0: # Take the first found location for simplicity min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) top_left = max_loc bottom_right = (top_left[0] + template_width, top_left[1] + template_height) # Extract window cropped = frame[top_left[1]:bottom_right[1], top_left[0]:bottom_right[0]] if previous_frame is not None: # Here you would check for changes between cropped and previous_frame # Assuming check_for_dot and check_picture are defined elsewhere result = check_for_dot(cropped, previous_frame, threshold) if result == "Found it": print("Changed") cv2.imwrite("images/captured_image2.jpg", cropped) result = check_picture("images/captured_image2.jpg") SERVER_RESPONSE = result[0] # assuming generate_audio is an async function await generate_audio() return result # Store current frame as previous frame for next iteration previous_frame = cropped.copy() else: print("No template match found") return "Not found"This function will only process a single frame per request now. Adjust according to your needs. Keep in mind that continuous processing in a loop is not typical for a web server endpoint. If you need constant processing, set up a background task or a separate processing service that constantly fetches frames and processes them, updating some state the web server can read when requested.
Remember that depending on your actual requirements, you might need further modifications to the code. If you have details like
check_for_dotandcheck_picturefunctions and the complete functionality you're aiming for, I could provide a more accurate modification. -
- User
So I want it to continue looping and I will have another template check to see if a dot is visable or not and if it is then it will take the photo and send it to the function that will look at the numbers in the picture
- Assistant
Given your requirement for continuous looping and performing template matching for both a window and a dot within that window, we will need to adjust the server endpoint to operate in a non-blocking manner and trigger a separate background task that constantly processes the video stream.
Below is a combination of FastAPI with Starlette background tasks to achieve the looping and image processing. The API will have one endpoint to start the processing and another to check the most recent "SERVER_RESPONSE".
from fastapi import FastAPI, BackgroundTasks import numpy as np import cv2 import os import requests app = FastAPI() # Paths to template images window_template_path = 'path_to_window_template.jpg' dot_template_path = 'path_to_dot_template.jpg' # Load template images window_template = cv2.imread(window_template_path, 0) dot_template = cv2.imread(dot_template_path, 0) # Global variable to store the server's response SERVER_RESPONSE = None # Image processing function def process_images(): global SERVER_RESPONSE url = "http://10.30.225.127:8080/shot.jpg" previous_frame = None window_match_threshold = 0.8 # Adjust according to your needs dot_match_threshold = 0.8 # Adjust according to your needs while True: # Fetch the frame img_resp = requests.get(url) img_arr = np.frombuffer(img_resp.content, np.uint8) frame = cv2.imdecode(img_arr, -1) gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Perform template matching for the window res = cv2.matchTemplate(gray_frame, window_template, cv2.TM_CCOEFF_NORMED) loc = np.where(res >= window_match_threshold) if len(loc[0]) > 0: min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) top_left = max_loc bottom_right = (top_left[0] + window_template.shape[1], top_left[1] + window_template.shape[0]) window_roi = frame[top_left[1]:bottom_right[1], top_left[0]:bottom_right[0]] # Convert the ROI to grayscale for dot detection gray_window_roi = cv2.cvtColor(window_roi, cv2.COLOR_BGR2GRAY) # Perform template matching for the dot within the window ROI dot_res = cv2.matchTemplate(gray_window_roi, dot_template, cv2.TM_CCOEFF_NORMED) dot_loc = np.where(dot_res >= dot_match_threshold) if len(dot_loc[0]) > 0: # A dot is found, perform the necessary tasks # Save the image if not os.path.isdir("images"): os.mkdir("images") image_path = "images/captured_image.jpg" cv2.imwrite(image_path, window_roi) # Check the saved image result = check_picture(image_path) # Assuming this function is defined SERVER_RESPONSE = result # You can break or continue depending on your process # break # If you decide to continue, you can throttle the loop # time.sleep(1) # Pause the loop for a given time (in seconds) # Throttle the loop to avoid excessive CPU usage time.sleep(0.5) @app.post("/start_processing") async def start_processing(background_tasks: BackgroundTasks): background_tasks.add_task(process_images) return {"message": "Image processing started"} @app.get("/get_response") async def get_response(): return {"response": SERVER_RESPONSE}In the code above, the
process_imagesfunction continuously loops to fetch frames and process them, looking for the window and dot using template matching. When a dot match is found, the image is saved, processed bycheck_picture, and theSERVER_RESPONSEis updated.The
/start_processingendpoint starts the image processing loop as a background task, allowing the FastAPI app to continue servicing other requests. It doesn't stop the processing, which runs indefinitely in the background.The
/get_responseendpoint provides the most recentSERVER_RESPONSE.This code snippet assumes you have the
check_picturefunction properly defined to accept an image path and return the result of the number recognition.Note on Production Use: Running an infinite loop in a background task within a web server is not ideal for production environments. Generally, you should have a dedicated image processing service separated from your web service, such as a worker process managed by a task queue (e.g., using Celery). This separation allows better scalability and reliability.
- User
so how do I start it now?
- Assistant
To start the image processing using the FastAPI application, you'll need to follow these steps:
-
Make sure all placeholder functions and paths (
check_picture,path_to_window_template.jpg,path_to_dot_template.jpg, etc.) in your script are replaced with actual, working code and file paths. -
Run the FastAPI application. This typically involves running a script (such as
app.py) with a command like:
uvicorn app:app --reloadHere,
apprefers to the variable name of theFastAPI()instance, and the--reloadflag is useful in development because it restarts the server on code changes.-
Once the FastAPI app is running, it will listen for HTTP requests at the specified host and port (by default,
http://127.0.0.1:8000). -
To start the image processing, make an HTTP
POSTrequest to the/start_processingendpoint. You can do this using a tool likecurl, a graphical client like Postman, or directly from within your code using an HTTP client library such asrequestsfor Python.
Using
curlfrom the command line:curl -X POST http://127.0.0.1:8000/start_processingUsing Python
requestslibrary:import requests response = requests.post('http://127.0.0.1:8000/start_processing') print(response.json())- After starting the image processing, the background task will continuously process frames. You can check the SERVER_RESPONSE by making a
GETrequest to/get_response.
Using
curl:curl http://127.0.0.1:8000/get_responseUsing Python
requestslibrary:response = requests.get('http://127.0.0.1:8000/get_response') print(response.json())Remember to ensure that the video streaming source (in the
urlvariable) is active and accessible from where the FastAPI app is running. Also, thecheck_picturefunction that gets called after a dot is detected should be a working function that returns the expected response.If you want the service to run persistently and be restarted automatically in case of failure, consider running FastAPI through a production server such as Gunicorn and using a process manager like
systemd,supervisord, or Docker for better control and stability. -
- User
Traceback (most recent call last): File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\uvicorn\protocols\http\h11_impl.py", line 408, in run_asgi result = await app( # type: ignore[func-returns-value] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 84, in call return await self.app(scope, receive, send) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\fastapi\applications.py", line 292, in call await super().call(scope, receive, send) File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\applications.py", line 122, in call await self.middleware_stack(scope, receive, send) File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\middleware\errors.py", line 184, in call raise exc File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\middleware\errors.py", line 162, in call await self.app(scope, receive, _send) File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\middleware\exceptions.py", line 79, in call raise exc File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\middleware\exceptions.py", line 68, in call await self.app(scope, receive, sender) File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 20, in call raise e File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 17, in call await self.app(scope, receive, send) File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\routing.py", line 718, in call await route.handle(scope, receive, send) File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\routing.py", line 276, in handle await self.app(scope, receive, send) File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\starlette\routing.py", line 66, in app response = await func(request) ^^^^^^^^^^^^^^^^^^^ File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\fastapi\routing.py", line 273, in app raw_response = await run_endpoint_function( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "C:\Users\AME\AppData\Local\Programs\Python\Python311\Lib\site-packages\fastapi\routing.py", line 190, in run_endpoint_function return await dependant.call(**values) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "c:\Users\AME\Documents\rs232\new.py", line 264, in main update_csv(SERVER_RESPONSE) File "c:\Users\AME\Documents\rs232\new.py", line 308, in update_csv while float(result) > 30.0: ^^^^^^^^^^^^^ TypeError: float() argument must be a string or a real number, not 'NoneType'
html:
server:
from fastapi import FastAPI, Request, BackgroundTasks
from fastapi.responses import StreamingResponse import uvicorn from fastapi.templating import Jinja2Templates
import numpy as np import cv2
from paddleocr import PaddleOCR import requests import datetime import os import csv import time
templates = Jinja2Templates(directory="templates")
class ImageProcessor: def init(self, original): self.original = original self.img = original.copy()
def process_image(self, blur, threshold, adjustment, erode, iterations): debug_images = [] alpha = float(2.5) debug_images.append(("Original", self.original)) # Adjust the exposure exposure_img = cv2.multiply(self.img, np.array([alpha])) debug_images.append(("Exposure Adjust", exposure_img)) # Convert to grayscale img2gray = cv2.cvtColor(exposure_img, cv2.COLOR_BGR2GRAY) debug_images.append(("Grayscale", img2gray)) # Blur to reduce noise img_blurred = cv2.GaussianBlur(img2gray, (blur, blur), 0) debug_images.append(("Blurred", img_blurred)) cropped = img_blurred # Threshold the image cropped_threshold = cv2.adaptiveThreshold( cropped, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, threshold, adjustment, ) debug_images.append(("Cropped Threshold", cropped_threshold)) # Erode the lcd digits to make them continuous for easier contouring kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (erode, erode)) eroded = cv2.erode(cropped_threshold, kernel, iterations=iterations) debug_images.append(("Eroded", eroded)) # Reverse the image to so the white text is found when looking for the contours inverse = cv2.bitwise_not(eroded) debug_images.append(("Inversed", inverse)) # Find the lcd digit contours # Assuming we find some, we'll sort them in order left -> right return debug_imagesapp = FastAPI()
Paddleocr supports Chinese, English, French, German, Korean and Japanese.
You can set the parameter
langasch,en,fr,german,korean,japanto switch the language model in order.
ocr = PaddleOCR( use_angle_cls=False, lang="en", ) # need to run only once to download and load model into memory
def check_for_dot(frame, previous_frame, threshold): # Convert the image to grayscale current_frame = frame.copy()
# Check if the two images have the same size if current_frame.shape != previous_frame.shape: print("The two images have different sizes") return "Not found" # Check if the two images have the same number of channels if current_frame.ndim != previous_frame.ndim: print("The two images have a different number of channels") return "Not found" # Crop the images to the specified area # Check if the cropped frames are not empty if current_frame.size != 0 and previous_frame.size != 0: # Convert the image to grayscale current_frame = cv2.cvtColor(current_frame, cv2.COLOR_BGR2GRAY) previous_frame = cv2.cvtColor(previous_frame, cv2.COLOR_BGR2GRAY) else: print("The cropped frames are empty") # Compute the absolute difference between the current frame and the previous frame frame_diff = cv2.absdiff(current_frame, previous_frame) # Convert the difference image to binary _, binary_diff = cv2.threshold(frame_diff, 10, 255, cv2.THRESH_BINARY) # Count the number of black pixels num_black_pixels = cv2.countNonZero(binary_diff) # Check if the number of black pixels is above the threshold if num_black_pixels > threshold: print("Found it") return "Found it" else: print("Not found") return "Not found"x, y, w, h = 100, 100, 200, 200 threshold = 400
Paths to template images
window_template_path = "shot2Target.png" dot_template_path = "dottemplate.png"
Load template images
window_template = cv2.imread(window_template_path, 0) dot_template = cv2.imread(dot_template_path, 0)
Global variable to store the server’s response
SERVER_RESPONSE = None
Image processing function
def process_images(): global SERVER_RESPONSE url = "http://10.30.225.127:8080/shot.jpg"
window_match_threshold = 0.8 # Adjust according to your needs dot_match_threshold = 0.8 # Adjust according to your needs while True: # Fetch the frame img_resp = requests.get(url) img_arr = np.frombuffer(img_resp.content, np.uint8) frame = cv2.imdecode(img_arr, -1) gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Perform template matching for the window res = cv2.matchTemplate(gray_frame, window_template, cv2.TM_CCOEFF_NORMED) loc = np.where(res >= window_match_threshold) if len(loc[0]) > 0: min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) top_left = max_loc bottom_right = ( top_left[0] + window_template.shape[1], top_left[1] + window_template.shape[0], ) window_roi = frame[ top_left[1] : bottom_right[1], top_left[0] : bottom_right[0] ] # Convert the ROI to grayscale for dot detection gray_window_roi = cv2.cvtColor(window_roi, cv2.COLOR_BGR2GRAY) # Perform template matching for the dot within the window ROI dot_res = cv2.matchTemplate( gray_window_roi, dot_template, cv2.TM_CCOEFF_NORMED ) dot_loc = np.where(dot_res >= dot_match_threshold) if len(dot_loc[0]) > 0: # A dot is found, perform the necessary tasks # Save the image if not os.path.isdir("images"): os.mkdir("images") image_path = "images/captured_image.jpg" cv2.imwrite(image_path, window_roi) # Check the saved image result = check_picture(image_path) # Assuming this function is defined SERVER_RESPONSE = result # You can break or continue depending on your process # break # If you decide to continue, you can throttle the loop # time.sleep(1) # Pause the loop for a given time (in seconds) # Throttle the loop to avoid excessive CPU usage time.sleep(0.5)@app.post("/start_processing") async def start_processing(background_tasks: BackgroundTasks): background_tasks.add_task(process_images) return {"message": "Image processing started"}
@app.get("/get_response") async def get_response(): return {"response": SERVER_RESPONSE}
async def generate_audio(): global SERVER_RESPONSE clenedUpResponse = SERVER_RESPONSE already_exists = False
while float(clenedUpResponse) > 30.0: clenedUpResponse = float(clenedUpResponse) / 10 if "." in SERVER_RESPONSE: clenedUpResponse = SERVER_RESPONSE.replace(".", " dot ") with open("last_number", "r+") as f: lars = ( f.readline().strip() ) # Read the first line and remove any leading/trailing whitespaces if lars == clenedUpResponse: already_exists = True else: f.seek(0) # Move the cursor to the beginning of the file f.write(str(clenedUpResponse)) f.truncate() # Remove any content after the new value url = "https://api.mandrillai.tech/v1/audio/tts" headers = { "Authorization": "Bearer md-tqjJTsIZujWmBUzZZxBXyLsBllHgEFFBydEGhhrRRiScqIKM" } data = { "voice_id": "piTKgcLEGmPE4e6mEKli", "text": f"Added {clenedUpResponse} Kilo", } try: if already_exists: return else: response = requests.post(url, json=data, headers=headers) response.raise_for_status() # If the request was successful, no Exception will be raised with open("audio.mp3", "wb") as f: f.write(response.content) return response except requests.exceptions.HTTPError as errh: print("HTTP Error:", errh) except requests.exceptions.ConnectionError as errc: print("Error Connecting:", errc) except requests.exceptions.Timeout as errt: print("Timeout Error:", errt) except requests.exceptions.RequestException as err: print("Something went wrong", err)@app.get("/audio") async def main(): global SERVER_RESPONSE # Generate the audio here with open("audio.mp3", "rb") as f: audio_data = f.read() update_csv(SERVER_RESPONSE)
return StreamingResponse(iter([audio_data]), media_type="audio/mp3")@app.get("/video") async def video_endpoint(request: Request): return templates.TemplateResponse("video.html", {"request": request})
def check_picture(img_path): result = ocr.ocr(img_path, cls=False) new_result = [] for idx in range(len(result)): res = result[idx] for line in res: print(line[1][0]) if line[1][0].replace(".", "", 1).isdigit() or line[0][1] == ".": new_result.append(line[1][0])
return new_resultdef update_csv(result): today = datetime.datetime.now().strftime("%Y-%m-%d") # Check if the file exists file_name = f"Elektronik_{today}.csv" file_exists = os.path.isfile(file_name) # If the file doesn't exist, create a new one with the header row if not file_exists: with open(file_name, "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=",", quotechar='"') writer.writerow(["Vikt", "Datum", "Total Vikt"]) csvfile.close() # Read the existing CSV file with open(file_name, "r") as csvfile: reader = csv.reader(csvfile, delimiter=",", quotechar='"') rows = list(reader) # Check if the date already exists # Add the weights for the current date total_weight = 0 for row in rows: if row[1] == today: total_weight += float(row[0]) while float(result) > 30.0: result = float(result) / 10 # Round the result to 2 decimal places result = round(float(result), 2) total_weight += float(result) rows.append([str(result), today, total_weight]) # Write the updated rows to the CSV file with open(file_name, "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=",", quotechar='"') writer.writerows(rows)
if name == "main": uvicorn.run(app, host="localhost", port=3333)
- Assistant
The error message you received indicates that a
TypeErroroccurred because aNonewas passed as an argument to thefloat()conversion, which meansSERVER_RESPONSEwasNoneat that moment. This likely happened in theupdate_csv()function or in thegenerate_audio()function when you attempted to convert the response to a float:while float(result) > 30.0: # result is NoneTo address this issue, you need to ensure that
SERVER_RESPONSEis a valid number as a string before attempting to convert it to a float. You should check ifSERVER_RESPONSEis notNonebefore proceeding with the conversion. You should also ensure thatcheck_picture()returns a proper result that can be converted to float.Here's how you can modify the
generate_audio()and theupdate_csv()functions:In the
generate_audio()function:async def generate_audio(): global SERVER_RESPONSE # Ensure SERVER_RESPONSE is not None and can be converted to float if SERVER_RESPONSE is None: print("No server response to generate audio from.") return # rest of generate_audio() code...And in the
update_csv()function:def update_csv(result): # Ensure result is not None and can be converted to float if result is None: print("No result to update CSV.") return # rest of update_csv() code...Furthermore, the
check_picture()function should return a recognizable and consistent number format, ensuring that when text is converted to float, it will not raise an error. Make sure thatcheck_picture()always returns a valid string representation of a number.Additionally, it's a good practice to add exception handling to your code to catch any unexpected errors. For example:
try: while float(result) > 30.0: result = float(result) / 10 except ValueError as ve: print(f"Error converting result to float: {ve}") # Handle the error or return from the functionMake these modifications in your code to ensure that type conversion errors are handled properly and that
SERVER_RESPONSEis checked before being used in calculations. - User
how can I see the frames the webcam sees in the background?:
x, y, w, h = 100, 100, 200, 200 threshold = 400
Paths to template images
window_template_path = "shot2Target.png" dot_template_path = "dottemplate.png"
Load template images
window_template = cv2.imread(window_template_path, 0) dot_template = cv2.imread(dot_template_path, 0)
Global variable to store the server’s response
SERVER_RESPONSE = None
Image processing function
def process_images(): global SERVER_RESPONSE # url = "http://10.30.225.127:8080/shot.jpg" url = "http://192.168.229.55:8080/shot.jpg" window_match_threshold = 0.8 # Adjust according to your needs dot_match_threshold = 0.8 # Adjust according to your needs
while True: # Fetch the frame img_resp = requests.get(url) img_arr = np.frombuffer(img_resp.content, np.uint8) frame = cv2.imdecode(img_arr, -1) gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Perform template matching for the window res = cv2.matchTemplate(gray_frame, window_template, cv2.TM_CCOEFF_NORMED) loc = np.where(res >= window_match_threshold) cv2.imshow("Frame", gray_frame) if len(loc[0]) > 0: min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) top_left = max_loc bottom_right = ( top_left[0] + window_template.shape[1], top_left[1] + window_template.shape[0], ) window_roi = frame[ top_left[1] : bottom_right[1], top_left[0] : bottom_right[0] ] # Convert the ROI to grayscale for dot detection gray_window_roi = cv2.cvtColor(window_roi, cv2.COLOR_BGR2GRAY) # Perform template matching for the dot within the window ROI dot_res = cv2.matchTemplate( gray_window_roi, dot_template, cv2.TM_CCOEFF_NORMED ) dot_loc = np.where(dot_res >= dot_match_threshold) if len(dot_loc[0]) > 0: # A dot is found, perform the necessary tasks # Save the image if not os.path.isdir("images"): os.mkdir("images") image_path = "images/captured_image.jpg" cv2.imwrite(image_path, window_roi) # Check the saved image result = check_picture(image_path) # Assuming this function is defined SERVER_RESPONSE = result # You can break or continue depending on your process # break # If you decide to continue, you can throttle the loop # time.sleep(1) # Pause the loop for a given time (in seconds) # Throttle the loop to avoid excessive CPU usage time.sleep(0.5)@app.post("/start_processing") async def start_processing(background_tasks: BackgroundTasks): background_tasks.add_task(process_images) return {"message": "Image processing started"}
@app.get("/get_response") async def get_response(): return {"response": SERVER_RESPONSE}
- Assistant
To display the frames that the webcam is capturing, you can use OpenCV's
cv2.imshow()function to create a window and display the frame. You've already included thecv2.imshow()line in yourprocess_imagesfunction, but in order to update the display and allow OpenCV to process window events (like closing the window), you also need to callcv2.waitKey()periodically. Here's a modified snippet of yourprocess_imagesfunction with these updates:# Image processing function def process_images(): global SERVER_RESPONSE url = "http://192.168.229.55:8080/shot.jpg" window_match_threshold = 0.8 # Adjust according to your needs dot_match_threshold = 0.8 # Adjust according to your needs cv2.namedWindow("Live Stream", cv2.WINDOW_NORMAL) # Create a named window while True: # Fetch the frame img_resp = requests.get(url) img_arr = np.frombuffer(img_resp.content, np.uint8) frame = cv2.imdecode(img_arr, -1) gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Perform template matching for the window res = cv2.matchTemplate(gray_frame, window_template, cv2.TM_CCOEFF_NORMED) loc = np.where(res >= window_match_threshold) # Display the frame cv2.imshow("Live Stream", frame) # Change to `gray_frame` if you want to show grayscale # Periodically process events such as keypresses key = cv2.waitKey(1) if key == ord('q'): # Exit loop if 'q' is pressed break # (rest of your existing code)Make sure to include this check at some point in the processing loop, so the window can update itself and respond to user actions. The code now includes a call to
cv2.namedWindow()to create a resizable window and waits for 1 millisecond oncv2.waitKey()to allow OpenCV to process the operating system's windowing events.If you want to stop displaying the window and break out of the loop, pressing 'q' will allow you to do so.
Please note: Since you're intending to display the window on a server or background process, be aware that you might not have a graphical user interface (GUI) environment set up in certain server environments (like virtual private servers, docker containers without GUI, etc.). The code above might not work if your environment doesn't support GUI window creation (in that case you could look into GUI-less alternatives like streaming the video over a web interface).
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.