Please use the following script and update my script so that the individually calculated…
- User
Please use the following script and update my script so that the individually calculated timestamp_offset is used throughout all calculations (e.g., in filtering_data, match_images_dataframes), we need to modify the way timestamps are handled in the data matching process. The main challenge here is that timestamp_offset depends on the line_number (which we obtain from tip_2d_y), but tip_2d_y comes from the coordinate data that hasn't been matched yet. This creates a circular dependency. Here is the script: import os import re import imageio import pandas as pd import numpy as np import cv2 import matplotlib.pyplot as plt import argparse import glob
from src.camera_param import CameraParameters from src.transformation.projective_geometry import project_points
from src.transformation.coordinates_loader import ( add_fiducial_coordinates, filter_blurry, filter_quality_recorded_trackers, match_images_dataframes, pre_filter_unrecorded_nir, )
from src.selector_configs import SelectedData, SelectorParameters from src.raw_dataset_configs import MetadataFile, RawDataFolder from src.transformation.camera_loader import load_camera_parameters from src.transformation.projective_geometry import add_tip_coordinates from src.data_loader import _join_2d_positions from src.raw_dataset_configs import CoordinateDataKeys
TOOL_TIP_ESTIMATION = np.array([300, 110, 0])
def create_csv(filename, image_id, x1, y1, x2, y2): columns = ['image_id', 'tip_x', 'tip_y', 'center_x', 'center_y'] df = pd.DataFrame(columns=columns) df.loc[0] = [image_id, x1, y1, x2, y2]
csvFilePath = filename if not os.path.isfile(csvFilePath): df.to_csv(csvFilePath, sep=',', index=False) else: df.to_csv(csvFilePath, mode='a', sep=',', header=False, index=False) import glob
def load_raw_data( calibration_data, data_folder: str, timestamp_offset: float = 0.0, verbose: bool = True, image_list: list[str] | None = None, ) -> SelectedData: image_folder = os.path.join(data_folder, RawDataFolder.IMAGE_FOLDER) camera_configuration_file = os.path.join( data_folder, RawDataFolder.CAMERA_CONFIGURATION_FILE )
Define the path to the logs folder where the coordinate files are located
logs_folder = os.path.join(data_folder, "logs")
Find all coordinate files within the 'logs' subfolder
coordinate_files = glob.glob(os.path.join(logs_folder, "coord_log_led_based_*.txt"))
Check if any files were found
if not coordinate_files: raise FileNotFoundError(f"No coordinate files found in {logs_folder} with pattern 'coord_log_led_based_*.txt'.")
Load each file, skip if empty
raw_coordinates_dataframe_list = [] for coord_file in coordinate_files: df = pd.read_csv(coord_file, delimiter=r"\s+") if not df.empty: raw_coordinates_dataframe_list.append(df) else: print(f"Warning: {coord_file} is empty and was skipped.")
Check if any data was loaded
if not raw_coordinates_dataframe_list: raise ValueError("No data loaded from any coordinate files; all files are empty.")
Concatenate all dataframes into one
raw_coordinates_dataframe = pd.concat(raw_coordinates_dataframe_list, ignore_index=True)
if image_list is None: image_list = os.listdir(image_folder) image_list = pre_filter_unrecorded_nir(image_list, raw_coordinates_dataframe)
metadata = MetadataFile.parse_obj({ 'date': '2023-10-19:14-45-32', 'bur_id': '107-030', 'bur_name': 'match head 3.0', 'bur_level': '1', 'attachment_id': '284', 'tracker_id': '0', 'bur_offset_mm': [0, 0, -1.5] })
tip_coordinates = np.array([ [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [calibration_data[2], calibration_data[0], calibration_data[1], 1] ]) camera_parameters = load_camera_parameters(camera_configuration_file)
selected_data = SelectedData( data_folder=data_folder, image_folder=image_folder, coordinate_dataframe=match_images_dataframes( image_list, raw_coordinates_dataframe, timestamp_offset=timestamp_offset ), camera_parameters=camera_parameters, metadata=metadata, tip_coordinates=tip_coordinates, )
if verbose: print( f"Found {selected_data.coordinate_dataframe.shape[0]} matches in {data_folder} across {len(coordinate_files)} files." )
return selected_data def filtering_data( calibration_data, data_folder: str, selector_parameters: SelectorParameters, verbose: bool = True, image_list: list[str] | None = None, ) -> SelectedData: selected_data = load_raw_data( calibration_data, data_folder, timestamp_offset=selector_parameters.timestamp_offset, verbose=verbose, image_list=image_list, )
if selector_parameters.quality_threshold is not None: selected_data.coordinate_dataframe = filter_quality_recorded_trackers( selected_data.coordinate_dataframe, quality_threshold=selector_parameters.quality_threshold, verbose=verbose, )
if selector_parameters.movement_threshold_mm is not None: selected_data.coordinate_dataframe = filter_blurry( selected_data.coordinate_dataframe, movement_threshold_mm=selector_parameters.movement_threshold_mm, verbose=verbose, )
if selector_parameters.join_fiducials: add_fiducial_coordinates(selected_data, verbose=verbose)
if selector_parameters.join_tip_coordinates: if verbose: print("Joining tip coordinates...")
add_tip_coordinates(selected_data, corrected=False) if selector_parameters.join_tip_correction: add_tip_coordinates(selected_data, corrected=True)if selector_parameters.join_2d_positions: _join_2d_positions( selected_data, selector_parameters, verbose=verbose, )
return selected_data def plot_metrics(timestamps, metrics, metric_name, output_filename):
Convert timestamps to seconds from the first timestamp
reference_time = min(timestamps) x_values = [(t - reference_time) / 1000.0 for t in timestamps] # Convert ms to seconds
plt.figure(figsize=(10, 6)) plt.plot(x_values, metrics, marker='o', linestyle='-', color='b') plt.title(f'{metric_name} over Time') plt.xlabel('Time (seconds from start)') plt.ylabel(metric_name) plt.grid(True)
Save the plot as a PNG image file
plt.savefig(output_filename) plt.close() print(f"{metric_name} plot saved to {output_filename}") def extract_timestamp(filename: str) -> int: """ Extracts the timestamp from the filename. Assumes the filename contains a large numeric timestamp in milliseconds. Example: '106840846.png' -> 106840846 (milliseconds) """ pattern = re.compile(r'(\d+)') # Adjust this regex pattern to match the timestamp format match = pattern.search(filename) if match: return int(match.group(1)) # Return timestamp as integer return None
def get_blur_score(grey_image):
return cv2.Laplacian(grey_image, cv2.CV_64F).var() def detect_blur_fft(image,size: int = 60): height, width = image.shape center_x, center_y = int(width / 2.0), int(height / 2.0) fft = np.fft.fft2(image) fft_shift = np.fft.fftshift(fft) fft_shift[center_y - size : center_y + size, center_x - size : center_x + size] = 0 fft_shift = np.fft.ifftshift(fft_shift) recon = np.fft.ifft2(fft_shift) magnitude = 20 * np.log(np.abs(recon))
return np.mean(magnitude)
def get_2d_position_estimation( tool_center_3d: np.ndarray, tool_rotation_matrix: np.ndarray, translation_vector: np.ndarray, camera_parameters: CameraParameters, ) -> np.ndarray: translation_vector = translation_vector[[1, 2, 0]]tip_coordinates = tool_rotation_matrix @ translation_vector + tool_center_3d
return project_points(tip_coordinates, camera_parameters) The 10 pixel margin for cropping the ROI. Should be adapted based on the resolution of images. margin = 10
def get_bounding_box_estimation_center( tool_center_2d: np.ndarray, tool_center_3d: np.ndarray, tool_rotation_matrix: np.ndarray, width: int, height: int, camera_parameters: CameraParameters, TOOL_TIP_ESTIMATION, ) -> tuple[int, int, int, int]: tip_2d_position_estimation = get_2d_position_estimation( tool_center_3d, tool_rotation_matrix, TOOL_TIP_ESTIMATION, camera_parameters )
points = np.stack( ( np.array([tip_2d_position_estimation[0], tip_2d_position_estimation[1]]), np.array([tool_center_2d[0], tool_center_2d[1]]), ) )
x1 = int(np.min(points[:, 0])) y1 = int(np.min(points[:, 1])) x2 = int(np.max(points[:, 0])) y2 = int(np.max(points[:, 1]))
h = y2 - y1 w = x2 - x1
if w > h: x1 = x1 - margin x2 = x2 + margin ym = int((y2 + y1)*0.5) y1 = ym - int(w/2) - margin y2 = ym + int(w / 2) + margin else: y1 = y1 - margin y2 = y2 + margin xm = int((x2 + x1) * 0.5) x1 = xm - int(h / 2) - margin x2 = xm + int(h / 2) + margin
return ( max(int(x1), 0), max(int(y1), 0), min(int(x2), width), min(int(y2), height)), (tip_2d_position_estimation[0], tip_2d_position_estimation[1]) def crop_frame(frame_array, bounding_box): return frame_array[ bounding_box[1] : bounding_box[3], bounding_box[0] : bounding_box[2] ], (bounding_box[0], bounding_box[1])
def resize_frame( frame: np.ndarray, size: tuple[int, int] ) -> tuple[np.ndarray, tuple[float, float]]: return cv2.resize(frame, size), (size[1] / frame.shape[1], size[0] / frame.shape[0])
def _reduce_point_scale( point: np.ndarray | None, x_min: int, y_min: int, scale: tuple[float, float] ) -> np.ndarray | None: return (point - np.array([x_min, y_min])) * np.array(scale) if point is not None else None
def reduce_labels_scale(point: np.ndarray, bouding_box: tuple[int, int, int, int], scale: tuple[float, float]): x_min, y_min, _, _ = bouding_box return _reduce_point_scale(point, x_min, y_min, scale)
import cv2
def save_cropped_frame(size, width, height, tool_center_2d, tool_center_3d, tool_center_rotation, bur_tip_x, bur_tip_y, camera_parameters, frame_array, image_name, quality, movement, output_subfolder): bounding_box, tip_2d_position_estimation = get_bounding_box_estimation_center( tool_center_2d, tool_center_3d, tool_center_rotation, width, height, camera_parameters, TOOL_TIP_ESTIMATION ) input_size = (size, size) cropped_frame, offset = crop_frame(frame_array, bounding_box) resized_frame, scale = resize_frame(cropped_frame, input_size)
Paint tip_2d_position_estimation and tool_center_2d on the resized frame
tip_color = (0, 255, 0) # Green for tip center_color = (255, 0, 0) # Blue for center radius = 3 thickness = 2
Adjust coordinates for cropping and resizing
tip_point = reduce_labels_scale(np.array(tip_2d_position_estimation), bounding_box, scale) center_point = reduce_labels_scale(tool_center_2d, bounding_box, scale)
Draw circles for tip and center
cv2.circle(resized_frame, tuple(map(int, tip_point)), radius, tip_color, thickness) cv2.circle(resized_frame, tuple(map(int, center_point)), radius, center_color, thickness)
grey_image = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY) laplacian_blur = cv2.Laplacian(grey_image, cv2.CV_64F).var() fft_blur = detect_blur_fft(grey_image) print(f"Laplacian Blur: {laplacian_blur:.2f}, FFT Blur: {fft_blur:.2f}")
Prepare output path with subfolder
os.makedirs(output_subfolder, exist_ok=True) new_image_name = f"{os.path.splitext(image_name)[0]}_q{quality:.2f}_m{movement:.2f}_lap{laplacian_blur:.2f}_fft{fft_blur:.2f}{os.path.splitext(image_name)[1]}" imageio.v2.imwrite(os.path.join(output_subfolder, new_image_name), resized_frame)
return laplacian_blur, fft_blur selector_parameters = SelectorParameters( timestamp_offset= 0.059, quality_threshold=0.5, movement_threshold_mm=10000, join_fiducials=False, join_tip_coordinates=True, join_tip_correction=True, join_2d_positions=True, interest_region_radius=400, tip_inclusion_radius=200, final_tool_radius=100, small_component_threshold=0, cropping_margin=5, pixel_margin=50, )
def main(file_path, UHD, output_size, output_folder, framerate, exposure_time): file_list = os.listdir(file_path) file_list.sort(key=lambda f: extract_timestamp(f)) timestamps = [] laplacian_blurs = [] fft_blurs = [] movements = []
total_lines = 2160 if UHD else 1080 # Set total lines based on UHD flag
for file in file_list: data_folder = os.path.join(file_path, file) raw_data_folder = data_folder camera_parameters = load_camera_parameters( os.path.join(data_folder, RawDataFolder.CAMERA_CONFIGURATION_FILE) ) solution = [149.421361, 0, 200.342416] width, height = (3840, 2160) if UHD else (1920, 1080) selected_data = filtering_data(solution, raw_data_folder, selector_parameters, verbose=True)
max_row = selected_data.coordinate_dataframe.shape[0] for row in range(max_row): image_name = selected_data.coordinate_dataframe.iloc[row]["image_name"] image_path = os.path.join(selected_data.image_folder, image_name) try: image = imageio.v2.imread(image_path) except Exception as e: print(f"Failed to read {image_name}: {e}") continue timestamp = extract_timestamp(image_name) if timestamp is not None: timestamps.append(timestamp) quality_value = selected_data.coordinate_dataframe.iloc[row]["quality"] movement_value = selected_data.coordinate_dataframe.iloc[row]["estimated_movement"] movements.append(movement_value) # Get line_number from y-coordinate of tool tip position line_number = int(selected_data.coordinate_dataframe.iloc[row]["tip_2d_y"]) # Calculate timestamp_offset for this image timestamp_offset = (1000.0 / framerate - exposure_time / 1000.0) * (line_number / total_lines) + (exposure_time / 1000.0 / 2.0) tool_center_x = selected_data.coordinate_dataframe.iloc[row]["center_x"] tool_center_y = selected_data.coordinate_dataframe.iloc[row]["center_y"] bur_tip_x = selected_data.coordinate_dataframe.iloc[row]["tip_2d_x"] bur_tip_y = selected_data.coordinate_dataframe.iloc[row]["tip_2d_y"] tool_center_3d = selected_data.coordinate_dataframe.iloc[row][CoordinateDataKeys.TOOL_CENTER_3D].to_numpy() rotation_matrix = ( selected_data.coordinate_dataframe.iloc[row][CoordinateDataKeys.TOOL_ROTATION_MATRIX].to_numpy().reshape(-1, 3, 3) ).transpose(0, 2, 1)[0] tool_center_2d = selected_data.coordinate_dataframe.iloc[row][CoordinateDataKeys.TOOL_CENTER_2D].to_numpy() # Get relative path to maintain folder structure relative_path = os.path.relpath(image_path, file_path) output_subfolder = os.path.join(output_folder, os.path.dirname(relative_path)) try: print(f"Processed row {row+1}/{max_row}: {image_path}") laplacian_blur, fft_blur = save_cropped_frame( output_size, width, height, tool_center_2d, tool_center_3d, rotation_matrix, bur_tip_x, bur_tip_y, camera_parameters, image, image_name, quality_value, movement_value, output_subfolder ) laplacian_blurs.append(laplacian_blur) fft_blurs.append(fft_blur) # Store the calculated timestamp_offset if needed for further processing print(f"Timestamp Offset for {image_name}: {timestamp_offset:.4f}") except Exception as e: print(f"Failed to write {image_name}: {e}")results_df = pd.DataFrame({ 'timestamp': timestamps, 'laplacian_blur': laplacian_blurs, 'fft_blur': fft_blurs, 'movement': movements }) output_csv_path = os.path.join(output_folder, 'blur_analysis_results.csv') results_df.to_csv(output_csv_path, index=False) print(f"Results saved to {output_csv_path}")
if timestamps: plot_metrics(timestamps, laplacian_blurs, "Laplacian Blur", os.path.join(output_folder, "laplacian_blur_plot.png")) plot_metrics(timestamps, fft_blurs, "FFT Blur", os.path.join(output_folder, "fft_blur_plot.png")) plot_metrics(timestamps, movements, "Movement", os.path.join(output_folder, "movement_plot.png")) if name == 'main': parser = argparse.ArgumentParser(description="Process parameters") parser.add_argument('--file_path', type=str, required=True, help='Path to the data files.') parser.add_argument('--output_size', type=int, required=True, help='Input size for the MV model.') parser.add_argument('--UHD', action='store_true', help='Flag to set UHD (4K resolution).') parser.add_argument('--output_folder', type=str, required=True, help='Path to save output images and CSV with the same subfolder structure as file_path.') parser.add_argument('--framerate', type=float, required=True, help='Framerate input from user.') parser.add_argument('--exposure_time', type=float, required=True, help='Exposure time input from user.')
args = parser.parse_args() main(args.file_path, args.UHD, args.output_size, args.output_folder, args.framerate, args.exposure_time)
- Assistant
To address the circular dependency in your script, we'll adjust how
timestamp_offsetis handled. Instead of using a singletimestamp_offsetfor all images, we'll calculate an individualtimestamp_offsetfor each coordinate data row based ontip_2d_y. We'll adjust the timestamps inraw_coordinates_dataframebefore matching, so thatmatch_images_dataframescan proceed without requiringtip_2d_yfrom matched data.Here's how we'll update your script:
- Modify
load_raw_datato acceptframerate,exposure_time, andtotal_linesas parameters. - Compute the
timestamp_offsetfor each row inraw_coordinates_dataframeusing thetip_2d_yvalue. - Adjust the
ts(timestamp) column inraw_coordinates_dataframeby adding the computedtimestamp_offsetper row. - Call
match_images_dataframeswithtimestamp_offset=0.0since timestamps are already adjusted. - Update the functions
filtering_dataandmainto pass the new parameters (framerate,exposure_time,total_lines) through.
Below is the updated script with these changes:
import os import re import imageio import pandas as pd import numpy as np import cv2 import matplotlib.pyplot as plt import argparse import glob from src.camera_param import CameraParameters from src.transformation.projective_geometry import project_points from src.transformation.coordinates_loader import ( add_fiducial_coordinates, filter_blurry, filter_quality_recorded_trackers, match_images_dataframes, pre_filter_unrecorded_nir, ) from src.selector_configs import SelectedData, SelectorParameters from src.raw_dataset_configs import MetadataFile, RawDataFolder from src.transformation.camera_loader import load_camera_parameters from src.transformation.projective_geometry import add_tip_coordinates from src.data_loader import _join_2d_positions from src.raw_dataset_configs import CoordinateDataKeys TOOL_TIP_ESTIMATION = np.array([300, 110, 0]) def create_csv(filename, image_id, x1, y1, x2, y2): columns = ['image_id', 'tip_x', 'tip_y', 'center_x', 'center_y'] df = pd.DataFrame(columns=columns) df.loc[0] = [image_id, x1, y1, x2, y2] csvFilePath = filename if not os.path.isfile(csvFilePath): df.to_csv(csvFilePath, sep=',', index=False) else: df.to_csv(csvFilePath, mode='a', sep=',', header=False, index=False) import glob def load_raw_data( calibration_data, data_folder: str, framerate: float, exposure_time: float, total_lines: int, verbose: bool = True, image_list: list[str] | None = None, ) -> SelectedData: image_folder = os.path.join(data_folder, RawDataFolder.IMAGE_FOLDER) camera_configuration_file = os.path.join( data_folder, RawDataFolder.CAMERA_CONFIGURATION_FILE ) # Define the path to the logs folder where the coordinate files are located logs_folder = os.path.join(data_folder, "logs") # Find all coordinate files within the 'logs' subfolder coordinate_files = glob.glob(os.path.join(logs_folder, "coord_log_led_based_*.txt")) # Check if any files were found if not coordinate_files: raise FileNotFoundError(f"No coordinate files found in {logs_folder} with pattern 'coord_log_led_based_*.txt'.") # Load each file, skip if empty raw_coordinates_dataframe_list = [] for coord_file in coordinate_files: df = pd.read_csv(coord_file, delimiter=r"\s+") if not df.empty: raw_coordinates_dataframe_list.append(df) else: print(f"Warning: {coord_file} is empty and was skipped.") # Check if any data was loaded if not raw_coordinates_dataframe_list: raise ValueError("No data loaded from any coordinate files; all files are empty.") # Concatenate all dataframes into one raw_coordinates_dataframe = pd.concat(raw_coordinates_dataframe_list, ignore_index=True) # Ensure that 'tip_2d_y' is present in 'raw_coordinates_dataframe' if 'tip_2d_y' not in raw_coordinates_dataframe.columns: raise ValueError("'tip_2d_y' column not found in coordinate data.") # Compute 'timestamp_offset' per row based on 'tip_2d_y' line_number = raw_coordinates_dataframe['tip_2d_y'] timestamp_offset = ( (1000.0 / framerate - exposure_time / 1000.0) * (line_number / total_lines) + (exposure_time / 1000.0 / 2.0) ) # Adjust 'ts' per row raw_coordinates_dataframe['ts'] += timestamp_offset if image_list is None: image_list = os.listdir(image_folder) image_list = pre_filter_unrecorded_nir(image_list, raw_coordinates_dataframe) metadata = MetadataFile.parse_obj({ 'date': '2023-10-19:14-45-32', 'bur_id': '107-030', 'bur_name': 'match head 3.0', 'bur_level': '1', 'attachment_id': '284', 'tracker_id': '0', 'bur_offset_mm': [0, 0, -1.5] }) tip_coordinates = np.array([ [1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [calibration_data[2], calibration_data[0], calibration_data[1], 1] ]) camera_parameters = load_camera_parameters(camera_configuration_file) selected_data = SelectedData( data_folder=data_folder, image_folder=image_folder, coordinate_dataframe=match_images_dataframes( image_list, raw_coordinates_dataframe, timestamp_offset=0.0 # Set to zero as we adjusted 'ts' already ), camera_parameters=camera_parameters, metadata=metadata, tip_coordinates=tip_coordinates, ) if verbose: print( f"Found {selected_data.coordinate_dataframe.shape[0]} matches in {data_folder} across {len(coordinate_files)} files." ) return selected_data def filtering_data( calibration_data, data_folder: str, selector_parameters: SelectorParameters, framerate: float, exposure_time: float, total_lines: int, verbose: bool = True, image_list: list[str] | None = None, ) -> SelectedData: selected_data = load_raw_data( calibration_data, data_folder, framerate=framerate, exposure_time=exposure_time, total_lines=total_lines, verbose=verbose, image_list=image_list, ) if selector_parameters.quality_threshold is not None: selected_data.coordinate_dataframe = filter_quality_recorded_trackers( selected_data.coordinate_dataframe, quality_threshold=selector_parameters.quality_threshold, verbose=verbose, ) if selector_parameters.movement_threshold_mm is not None: selected_data.coordinate_dataframe = filter_blurry( selected_data.coordinate_dataframe, movement_threshold_mm=selector_parameters.movement_threshold_mm, verbose=verbose, ) if selector_parameters.join_fiducials: add_fiducial_coordinates(selected_data, verbose=verbose) if selector_parameters.join_tip_coordinates: if verbose: print("Joining tip coordinates...") add_tip_coordinates(selected_data, corrected=False) if selector_parameters.join_tip_correction: add_tip_coordinates(selected_data, corrected=True) if selector_parameters.join_2d_positions: _join_2d_positions( selected_data, selector_parameters, verbose=verbose, ) return selected_data def plot_metrics(timestamps, metrics, metric_name, output_filename): # Convert timestamps to seconds from the first timestamp reference_time = min(timestamps) x_values = [(t - reference_time) / 1000.0 for t in timestamps] # Convert ms to seconds plt.figure(figsize=(10, 6)) plt.plot(x_values, metrics, marker='o', linestyle='-', color='b') plt.title(f'{metric_name} over Time') plt.xlabel('Time (seconds from start)') plt.ylabel(metric_name) plt.grid(True) # Save the plot as a PNG image file plt.savefig(output_filename) plt.close() print(f"{metric_name} plot saved to {output_filename}") def extract_timestamp(filename: str) -> int: """ Extracts the timestamp from the filename. Assumes the filename contains a large numeric timestamp in milliseconds. Example: '106840846.png' -> 106840846 (milliseconds) """ pattern = re.compile(r'(\d+)') # Adjust this regex pattern to match the timestamp format match = pattern.search(filename) if match: return int(match.group(1)) # Return timestamp as integer return None def get_blur_score(grey_image): return cv2.Laplacian(grey_image, cv2.CV_64F).var() def detect_blur_fft(image,size: int = 60): height, width = image.shape center_x, center_y = int(width / 2.0), int(height / 2.0) fft = np.fft.fft2(image) fft_shift = np.fft.fftshift(fft) fft_shift[center_y - size : center_y + size, center_x - size : center_x + size] = 0 fft_shift = np.fft.ifftshift(fft_shift) recon = np.fft.ifft2(fft_shift) magnitude = 20 * np.log(np.abs(recon)) return np.mean(magnitude) def get_2d_position_estimation( tool_center_3d: np.ndarray, tool_rotation_matrix: np.ndarray, translation_vector: np.ndarray, camera_parameters: CameraParameters, ) -> np.ndarray: translation_vector = translation_vector[[1, 2, 0]] tip_coordinates = tool_rotation_matrix @ translation_vector + tool_center_3d return project_points(tip_coordinates, camera_parameters) margin = 10 # The 10 pixel margin for cropping the ROI. Should be adapted based on the resolution of images. def get_bounding_box_estimation_center( tool_center_2d: np.ndarray, tool_center_3d: np.ndarray, tool_rotation_matrix: np.ndarray, width: int, height: int, camera_parameters: CameraParameters, TOOL_TIP_ESTIMATION, ) -> tuple[int, int, int, int]: tip_2d_position_estimation = get_2d_position_estimation( tool_center_3d, tool_rotation_matrix, TOOL_TIP_ESTIMATION, camera_parameters ) points = np.stack( ( np.array([tip_2d_position_estimation[0], tip_2d_position_estimation[1]]), np.array([tool_center_2d[0], tool_center_2d[1]]), ) ) x1 = int(np.min(points[:, 0])) y1 = int(np.min(points[:, 1])) x2 = int(np.max(points[:, 0])) y2 = int(np.max(points[:, 1])) h = y2 - y1 w = x2 - x1 if w > h: x1 = x1 - margin x2 = x2 + margin ym = int((y2 + y1)*0.5) y1 = ym - int(w/2) - margin y2 = ym + int(w / 2) + margin else: y1 = y1 - margin y2 = y2 + margin xm = int((x2 + x1) * 0.5) x1 = xm - int(h / 2) - margin x2 = xm + int(h / 2) + margin return ( max(int(x1), 0), max(int(y1), 0), min(int(x2), width), min(int(y2), height) ), (tip_2d_position_estimation[0], tip_2d_position_estimation[1]) def crop_frame(frame_array, bounding_box): return frame_array[ bounding_box[1] : bounding_box[3], bounding_box[0] : bounding_box[2] ], (bounding_box[0], bounding_box[1]) def resize_frame( frame: np.ndarray, size: tuple[int, int] ) -> tuple[np.ndarray, tuple[float, float]]: return cv2.resize(frame, size), (size[1] / frame.shape[1], size[0] / frame.shape[0]) def _reduce_point_scale( point: np.ndarray | None, x_min: int, y_min: int, scale: tuple[float, float] ) -> np.ndarray | None: return (point - np.array([x_min, y_min])) * np.array(scale) if point is not None else None def reduce_labels_scale(point: np.ndarray, bouding_box: tuple[int, int, int, int], scale: tuple[float, float]): x_min, y_min, _, _ = bouding_box return _reduce_point_scale(point, x_min, y_min, scale) import cv2 def save_cropped_frame(size, width, height, tool_center_2d, tool_center_3d, tool_center_rotation, bur_tip_x, bur_tip_y, camera_parameters, frame_array, image_name, quality, movement, output_subfolder): bounding_box, tip_2d_position_estimation = get_bounding_box_estimation_center( tool_center_2d, tool_center_3d, tool_center_rotation, width, height, camera_parameters, TOOL_TIP_ESTIMATION ) input_size = (size, size) cropped_frame, offset = crop_frame(frame_array, bounding_box) resized_frame, scale = resize_frame(cropped_frame, input_size) # Paint tip_2d_position_estimation and tool_center_2d on the resized frame tip_color = (0, 255, 0) # Green for tip center_color = (255, 0, 0) # Blue for center radius = 3 thickness = 2 # Adjust coordinates for cropping and resizing tip_point = reduce_labels_scale(np.array(tip_2d_position_estimation), bounding_box, scale) center_point = reduce_labels_scale(tool_center_2d, bounding_box, scale) # Draw circles for tip and center cv2.circle(resized_frame, tuple(map(int, tip_point)), radius, tip_color, thickness) cv2.circle(resized_frame, tuple(map(int, center_point)), radius, center_color, thickness) grey_image = cv2.cvtColor(resized_frame, cv2.COLOR_BGR2GRAY) laplacian_blur = cv2.Laplacian(grey_image, cv2.CV_64F).var() fft_blur = detect_blur_fft(grey_image) print(f"Laplacian Blur: {laplacian_blur:.2f}, FFT Blur: {fft_blur:.2f}") # Prepare output path with subfolder os.makedirs(output_subfolder, exist_ok=True) new_image_name = f"{os.path.splitext(image_name)[0]}_q{quality:.2f}_m{movement:.2f}_lap{laplacian_blur:.2f}_fft{fft_blur:.2f}{os.path.splitext(image_name)[1]}" imageio.v2.imwrite(os.path.join(output_subfolder, new_image_name), resized_frame) return laplacian_blur, fft_blur # Remove 'timestamp_offset' from 'selector_parameters', or set it to zero selector_parameters = SelectorParameters( timestamp_offset=0.0, # Set to zero or remove if not needed quality_threshold=0.5, movement_threshold_mm=10000, join_fiducials=False, join_tip_coordinates=True, join_tip_correction=True, join_2d_positions=True, interest_region_radius=400, tip_inclusion_radius=200, final_tool_radius=100, small_component_threshold=0, cropping_margin=5, pixel_margin=50, ) def main(file_path, UHD, output_size, output_folder, framerate, exposure_time): file_list = os.listdir(file_path) file_list.sort(key=lambda f: extract_timestamp(f)) timestamps = [] laplacian_blurs = [] fft_blurs = [] movements = [] total_lines = 2160 if UHD else 1080 # Set total lines based on UHD flag for file in file_list: data_folder = os.path.join(file_path, file) raw_data_folder = data_folder camera_parameters = load_camera_parameters( os.path.join(data_folder, RawDataFolder.CAMERA_CONFIGURATION_FILE) ) solution = [149.421361, 0, 200.342416] width, height = (3840, 2160) if UHD else (1920, 1080) selected_data = filtering_data( solution, raw_data_folder, selector_parameters, framerate=framerate, exposure_time=exposure_time, total_lines=total_lines, verbose=True ) max_row = selected_data.coordinate_dataframe.shape[0] for row in range(max_row): image_name = selected_data.coordinate_dataframe.iloc[row]["image_name"] image_path = os.path.join(selected_data.image_folder, image_name) try: image = imageio.v2.imread(image_path) except Exception as e: print(f"Failed to read {image_name}: {e}") continue timestamp = extract_timestamp(image_name) if timestamp is not None: timestamps.append(timestamp) quality_value = selected_data.coordinate_dataframe.iloc[row]["quality"] movement_value = selected_data.coordinate_dataframe.iloc[row]["estimated_movement"] movements.append(movement_value) # Get line_number from y-coordinate of tool tip position line_number = int(selected_data.coordinate_dataframe.iloc[row]["tip_2d_y"]) # Calculate timestamp_offset for this image (for logging or other purposes) timestamp_offset = ( (1000.0 / framerate - exposure_time / 1000.0) * (line_number / total_lines) + (exposure_time / 1000.0 / 2.0) ) tool_center_x = selected_data.coordinate_dataframe.iloc[row]["center_x"] tool_center_y = selected_data.coordinate_dataframe.iloc[row]["center_y"] bur_tip_x = selected_data.coordinate_dataframe.iloc[row]["tip_2d_x"] bur_tip_y = selected_data.coordinate_dataframe.iloc[row]["tip_2d_y"] tool_center_3d = selected_data.coordinate_dataframe.iloc[row][CoordinateDataKeys.TOOL_CENTER_3D].to_numpy() rotation_matrix = ( selected_data.coordinate_dataframe.iloc[row][CoordinateDataKeys.TOOL_ROTATION_MATRIX].to_numpy().reshape(-1, 3, 3) ).transpose(0, 2, 1)[0] tool_center_2d = selected_data.coordinate_dataframe.iloc[row][CoordinateDataKeys.TOOL_CENTER_2D].to_numpy() # Get relative path to maintain folder structure relative_path = os.path.relpath(image_path, file_path) output_subfolder = os.path.join(output_folder, os.path.dirname(relative_path)) try: print(f"Processed row {row+1}/{max_row}: {image_path}") laplacian_blur, fft_blur = save_cropped_frame( output_size, width, height, tool_center_2d, tool_center_3d, rotation_matrix, bur_tip_x, bur_tip_y, camera_parameters, image, image_name, quality_value, movement_value, output_subfolder ) laplacian_blurs.append(laplacian_blur) fft_blurs.append(fft_blur) # Store the calculated timestamp_offset if needed for further processing print(f"Timestamp Offset for {image_name}: {timestamp_offset:.4f}") except Exception as e: print(f"Failed to write {image_name}: {e}") results_df = pd.DataFrame({ 'timestamp': timestamps, 'laplacian_blur': laplacian_blurs, 'fft_blur': fft_blurs, 'movement': movements }) output_csv_path = os.path.join(output_folder, 'blur_analysis_results.csv') results_df.to_csv(output_csv_path, index=False) print(f"Results saved to {output_csv_path}") if timestamps: plot_metrics(timestamps, laplacian_blurs, "Laplacian Blur", os.path.join(output_folder, "laplacian_blur_plot.png")) plot_metrics(timestamps, fft_blurs, "FFT Blur", os.path.join(output_folder, "fft_blur_plot.png")) plot_metrics(timestamps, movements, "Movement", os.path.join(output_folder, "movement_plot.png")) if __name__ == '__main__': parser = argparse.ArgumentParser(description="Process parameters") parser.add_argument('--file_path', type=str, required=True, help='Path to the data files.') parser.add_argument('--output_size', type=int, required=True, help='Input size for the MV model.') parser.add_argument('--UHD', action='store_true', help='Flag to set UHD (4K resolution).') parser.add_argument('--output_folder', type=str, required=True, help='Path to save output images and CSV with the same subfolder structure as file_path.') parser.add_argument('--framerate', type=float, required=True, help='Framerate input from user.') parser.add_argument('--exposure_time', type=float, required=True, help='Exposure time input from user.') args = parser.parse_args() main(args.file_path, args.UHD, args.output_size, args.output_folder, args.framerate, args.exposure_time)Explanation of the Changes:
-
Adjust
load_raw_data:- Added
framerate,exposure_time, andtotal_linesas parameters. - Computed
timestamp_offsetper row inraw_coordinates_dataframeusingtip_2d_y. - Adjusted the
tscolumn inraw_coordinates_dataframeby adding the computedtimestamp_offset. - Updated the call to
match_images_dataframeswithtimestamp_offset=0.0since the timestamps are already adjusted.
- Added
-
Update
filtering_data:- Passed
framerate,exposure_time, andtotal_linestoload_raw_data.
- Passed
-
Modify
main:- Determined
total_linesbased on theUHDflag. - Passed
framerate,exposure_time, andtotal_linesthrough tofiltering_data. - Adjusted logging to compute
timestamp_offsetper image if needed for logging but not for matching.
- Determined
-
Adjust
selector_parameters:- Set
timestamp_offsetto0.0or removed it if not necessary.
- Set
By adjusting the timestamps in
raw_coordinates_dataframebefore matching, we avoid the circular dependency betweentimestamp_offsetandtip_2d_y. Thematch_images_dataframesfunction will now match images and coordinate data based on the adjusted timestamps without needing prior knowledge oftip_2d_y. - Modify