robotutils/camera.py from ultralytics import YOLO import cv2 import pybullet as p import…
- User
robot_utils/camera.py
from ultralytics import YOLO import cv2 import pybullet as p import numpy as np
class Camera: def init(self, robot_id, physics_client_id, yolo_model_path, classes_path, img_width=640, img_height=480): self.robot_id = robot_id self.physics_client_id = physics_client_id self.active = False self.model = YOLO(yolo_model_path) # Load YOLOv8 model self.img_width = img_width self.img_height = img_height
with open(classes_path, 'r') as f: self.classes = [line.strip() for line in f.readlines()] def activate(self): self.active = True print("Camera activated.") def deactivate(self): self.active = False print("Camera deactivated.") def capture_image(self, robot_id, physics_client_id): if not self.active: print("Camera is off.") return None # Get robot's position and orientation robot_pos, robot_orn = p.getBasePositionAndOrientation(robot_id, physicsClientId=physics_client_id) yaw = p.getEulerFromQuaternion(robot_orn)[-1] # Set camera parameters distance = 2.5 camera_eye_position = [robot_pos[0], robot_pos[1], robot_pos[2] + 1] camera_target_position = [ robot_pos[0] + distance * np.cos(yaw), robot_pos[1] + distance * np.sin(yaw), robot_pos[2] ] # Compute view and projection matrices view_matrix = p.computeViewMatrix( cameraEyePosition=camera_eye_position, cameraTargetPosition=camera_target_position, cameraUpVector=[0, 1, 0], physicsClientId=physics_client_id ) projection_matrix = p.computeProjectionMatrixFOV( fov=90, aspect=self.img_width / self.img_height, nearVal=0.1, farVal=100, physicsClientId=physics_client_id ) # Capture the image _, _, img_arr, _, _ = p.getCameraImage( width=self.img_width, height=self.img_height, viewMatrix=view_matrix, projectionMatrix=projection_matrix, physicsClientId=physics_client_id ) # Convert the image array to a proper format img = np.array(img_arr, dtype=np.uint8).reshape((self.img_height, self.img_width, 4)) img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR) return img def detect_objects(self): img = self.capture_image() if img is None: return [] results = self.model(img) # Perform object detection # Extract detected classes detected_classes = [self.model.names[int(cls)] for result in results for cls in result.boxes.cls] unique_classes = list(set(detected_classes)) print(f"Detected objects: {unique_classes}") return unique_classes def get_annotated_frame(self): img = self.capture_image(self.robot_id, self.physics_client_id) if img is None: return None results = self.model(img) # Perform object detection annotated_frame = img.copy() for result in results: for box in result.boxes: cls_id = int(box.cls) label = self.model.names[cls_id] conf = box.conf x1, y1, x2, y2 = map(int, box.xyxy) cv2.rectangle(annotated_frame, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.putText(annotated_frame, f"{label} {conf:.2f}", (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) return annotated_frameTraceback (most recent call last): File "/Users/apple/Music/major project/multimodal_robot_simulation/main.py", line 137, in main() File "/Users/apple/Music/major project/multimodal_robot_simulation/main.py", line 120, in main execute_instructions(robot.functions, instructions) File "/Users/apple/Music/major project/multimodal_robot_simulation/main.py", line 20, in execute_instructions robot_functions.move_forward(params['distance']) File "/Users/apple/Music/major project/multimodal_robot_simulation/functions.py", line 49, in move_forward self._move(distance, forward=True) File "/Users/apple/Music/major project/multimodal_robot_simulation/functions.py", line 63, in _move self.display_annotated_frame() File "/Users/apple/Music/major project/multimodal_robot_simulation/functions.py", line 39, in display_annotated_frame annotated_frame = self.camera.get_annotated_frame() File "/Users/apple/Music/major project/multimodal_robot_simulation/robot_utils/camera.py", line 102, in get_annotated_frame x1, y1, x2, y2 = map(int, box.xyxy) ValueError: only one element tensors can be converted to Python scalars
- Assistant
The error you're encountering:
ValueError: only one element tensors can be converted to Python scalarsis occurring because you're attempting to directly convert tensor elements to integers using
map(int, box.xyxy). Thebox.xyxyattribute from the Ultralytics YOLO model returns a tensor, and directly mappingintover tensor elements can lead to the mentioned ValueError.Understanding the Issue
-
box.xyxyStructure: In Ultralytics YOLOv8,box.xyxyreturns a tensor containing the bounding box coordinates in the format[x1, y1, x2, y2]. -
Direct Conversion Issue: Directly applying
intto tensor elements without first converting them to a list or NumPy array leads to the error becauseintexpects a single scalar value, not a tensor.
Solution
To fix this issue, you should convert the tensor to a list (or NumPy array) before mapping
intover its elements. This ensures that each coordinate is a Python scalar that can be converted to an integer.Step-by-Step Fix
-
Convert Tensor to List: Use the
.tolist()method to convert the tensor to a list of floats. -
Map
intOver the List: Apply theintfunction to each element in the list to obtain integer coordinates. -
Unpack the Coordinates: Assign the converted integers to
x1,y1,x2, andy2.
Updated Code
Here's how you can modify the
get_annotated_framemethod in yourcamera.pyfile:def get_annotated_frame(self): img = self.capture_image(self.robot_id, self.physics_client_id) if img is None: return None results = self.model(img) # Perform object detection annotated_frame = img.copy() for result in results: for box in result.boxes: cls_id = int(box.cls) label = self.model.names[cls_id] conf = box.conf # Convert tensor to list before mapping to int x1, y1, x2, y2 = map(int, box.xyxy.tolist()) cv2.rectangle(annotated_frame, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.putText( annotated_frame, f"{label} {conf:.2f}", (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2 ) return annotated_frameAlternative Approach
Alternatively, you can use a list comprehension for clarity:
x1, y1, x2, y2 = [int(coord) for coord in box.xyxy.tolist()]Final Code with Fix Applied
Here's the complete
camera.pywith the necessary fix applied:# robot_utils/camera.py from ultralytics import YOLO import cv2 import pybullet as p import numpy as np class Camera: def __init__(self, robot_id, physics_client_id, yolo_model_path, classes_path, img_width=640, img_height=480): self.robot_id = robot_id self.physics_client_id = physics_client_id self.active = False self.model = YOLO(yolo_model_path) # Load YOLOv8 model self.img_width = img_width self.img_height = img_height with open(classes_path, 'r') as f: self.classes = [line.strip() for line in f.readlines()] def activate(self): self.active = True print("Camera activated.") def deactivate(self): self.active = False print("Camera deactivated.") def capture_image(self, robot_id, physics_client_id): if not self.active: print("Camera is off.") return None # Get robot's position and orientation robot_pos, robot_orn = p.getBasePositionAndOrientation(robot_id, physicsClientId=physics_client_id) yaw = p.getEulerFromQuaternion(robot_orn)[-1] # Set camera parameters distance = 2.5 camera_eye_position = [robot_pos[0], robot_pos[1], robot_pos[2] + 1] camera_target_position = [ robot_pos[0] + distance * np.cos(yaw), robot_pos[1] + distance * np.sin(yaw), robot_pos[2] ] # Compute view and projection matrices view_matrix = p.computeViewMatrix( cameraEyePosition=camera_eye_position, cameraTargetPosition=camera_target_position, cameraUpVector=[0, 1, 0], physicsClientId=physics_client_id ) projection_matrix = p.computeProjectionMatrixFOV( fov=90, aspect=self.img_width / self.img_height, nearVal=0.1, farVal=100, physicsClientId=physics_client_id ) # Capture the image _, _, img_arr, _, _ = p.getCameraImage( width=self.img_width, height=self.img_height, viewMatrix=view_matrix, projectionMatrix=projection_matrix, physicsClientId=physics_client_id ) # Convert the image array to a proper format img = np.array(img_arr, dtype=np.uint8).reshape((self.img_height, self.img_width, 4)) img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR) return img def detect_objects(self): img = self.capture_image() if img is None: return [] results = self.model(img) # Perform object detection # Extract detected classes detected_classes = [self.model.names[int(cls)] for result in results for cls in result.boxes.cls] unique_classes = list(set(detected_classes)) print(f"Detected objects: {unique_classes}") return unique_classes def get_annotated_frame(self): img = self.capture_image(self.robot_id, self.physics_client_id) if img is None: return None results = self.model(img) # Perform object detection annotated_frame = img.copy() for result in results: for box in result.boxes: cls_id = int(box.cls) label = self.model.names[cls_id] conf = box.conf # Fix applied here: Convert tensor to list before mapping to int x1, y1, x2, y2 = map(int, box.xyxy.tolist()) cv2.rectangle(annotated_frame, (x1, y1), (x2, y2), (0, 255, 0), 2) cv2.putText(annotated_frame, f"{label} {conf:.2f}", (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) return annotated_frameAdditional Recommendations
-
Error Handling: Consider adding more robust error handling to manage unexpected inputs or states, especially when dealing with hardware interactions or external libraries.
-
Performance Optimization: If performance becomes an issue, you might want to process frames asynchronously or utilize GPU acceleration for the YOLO model.
-
Logging: Instead of using
printstatements, using Python'sloggingmodule can provide more flexibility in managing debug and error messages. -
Unit Testing: Implement unit tests for the
Cameraclass to ensure that each method behaves as expected, especially after making changes.
Conclusion
By converting the tensor to a list before mapping
intover its elements, you resolve the ValueError and ensure that the bounding box coordinates are correctly processed for annotation. This fix should allow yourCameraclass to function as intended without encountering the mentioned error.
If you encounter any further issues or have additional questions, feel free to ask!
-