Skip to content
The Chat Archive
o1-preview-2024-09-12· OpenAI WildChat-4.8M (AI2)

import cv2 import mediapipe as mp import csv import numpy as np Define an enhanced…

  1. User

    import cv2 import mediapipe as mp import csv import numpy as np

    Define an enhanced Kalman Filter class with velocity

    class KalmanFilter1D: def init(self, initial_position=0.0, initial_velocity=0.0): # State vector [position, velocity] self.state = np.array([[initial_position], # Position [initial_velocity]]) # Velocity # State covariance matrix self.covariance = np.eye(2) * 1.0 # State transition matrix (assuming constant velocity) self.F = np.array([[1, 1], [0, 1]], dtype=np.float32) # Measurement matrix (we only measure position) self.H = np.array([[1, 0]], dtype=np.float32) # Process noise covariance self.Q = np.array([[1e-4, 0], [0, 1e-4]], dtype=np.float32) # Measurement noise covariance self.R = np.array([[1e-1]], dtype=np.float32) # Kalman gain self.K = np.zeros((2, 1), dtype=np.float32) # Flag to check if initialized self.initialized = False # Residual (innovation) self.residual = None

    def initialize(self, measurement):
        self.state = np.array([[measurement], [0.0]], dtype=np.float32)
        self.initialized = True
    
    def predict(self):
        # Predict the next state
        self.state = self.F @ self.state
        self.covariance = self.F @ self.covariance @ self.F.T + self.Q
    
    def update(self, measurement):
        if not self.initialized:
            self.initialize(measurement)
            return self.state[0][0], self.state[1][0]  # Position, Velocity
    
        # Measurement prediction
        S = self.H @ self.covariance @ self.H.T + self.R
        # Kalman Gain
        self.K = self.covariance @ self.H.T @ np.linalg.inv(S)
        # Update the state with measurement
        y = np.array([[measurement]]) - (self.H @ self.state)
        self.state = self.state + self.K @ y
        # Update the covariance
        I = np.eye(self.F.shape[0])
        self.covariance = (I - self.K @ self.H) @ self.covariance
        # Store residual
        self.residual = y[0][0]
        return self.state[0][0], self.state[1][0]  # Position, Velocity
    

    Initialize MediaPipe Hand Tracking

    mp_hands = mp.solutions.hands hands = mp_hands.Hands( static_image_mode=False, # For video input max_num_hands=2, # Maximum number of hands to detect min_detection_confidence=0.5, # Minimum confidence for detection min_tracking_confidence=0.5 # Minimum confidence for tracking ) mp_drawing = mp.solutions.drawing_utils # For drawing hand landmarks

    Open the input video

    cap = cv2.VideoCapture('input_video.mp4')

    Get video properties

    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # Video frame width height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # Video frame height fps = cap.get(cv2.CAP_PROP_FPS) # Original frames per second

    Desired processing frame rate (2 frames per second)

    desired_fps = 10
    frame_skip = max(int(fps / desired_fps), 1) # Ensure at least 1 frame_count = 0 # To keep track of the frame number

    Time difference between frames

    delta_t = 1 / desired_fps

    Initialize Video Writer for output video

    output_video = cv2.VideoWriter( 'output_video_kf.mp4', cv2.VideoWriter_fourcc(*'mp4v'), desired_fps, # Set output video FPS to desired FPS (width, height) )

    Prepare to write to CSV file

    csv_file = open('landmarks_with_velocity_and_kalman.csv', mode='w', newline='') csv_writer = csv.writer(csv_file)

    Create CSV header

    header = ['Frame', 'Hand'] for i in range(21): # 21 landmarks in MediaPipe Hands header.extend([ f'Landmark_{i}x', f'Landmark{i}y', f'Landmark{i}z', f'Landmark{i}velocity_x', f'Landmark{i}velocity_y', f'Landmark{i}velocity_z', f'Landmark{i}kalman_x', f'Landmark{i}kalman_y', f'Landmark{i}_kalman_z' ])

    Write header to CSV

    csv_writer.writerow(header)

    Initialize Kalman filters for all 21 landmarks (for each x, y, and z separately)

    Structure: kalman_filters[hand_label][landmark_index]['x'/'y'/'z']

    kalman_filters = { 'Left': [ {'x': KalmanFilter1D(), 'y': KalmanFilter1D(), 'z': KalmanFilter1D()} for _ in range(21) ], 'Right': [ {'x': KalmanFilter1D(), 'y': KalmanFilter1D(), 'z': KalmanFilter1D()} for _ in range(21) ] }

    To store previous frame's landmarks and velocities for velocity calculation

    previous_landmarks = {'Left': None, 'Right': None} previous_velocities = {'Left': None, 'Right': None}

    Data structures for plotting

    plot_data = { 'Left': [ {'frames': [], 'positions': [], 'velocities': [], 'kalman_positions': [], 'residuals': [], 'error_covariances': []} for _ in range(21) ], 'Right': [ {'frames': [], 'positions': [], 'velocities': [], 'kalman_positions': [], 'residuals': [], 'error_covariances': []} for _ in range(21) ] }

    while cap.isOpened(): ret, frame = cap.read() if not ret: break

    # Process every 'frame_skip' frames to achieve desired FPS
    if frame_count % frame_skip == 0:
        # Convert frame to RGB for MediaPipe processing
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        result = hands.process(frame_rgb)
    
        # Create a copy of the frame for drawing
        annotated_frame = frame.copy()
    
        # Check if hand landmarks are detected
        if result.multi_hand_landmarks and result.multi_handedness:
            for hand_landmarks, handedness in zip(result.multi_hand_landmarks, result.multi_handedness):
                # Get hand label ('Left' or 'Right')
                hand_label = handedness.classification[0].label  # 'Left' or 'Right'
    
                # Draw the hand landmarks on the frame
                mp_drawing.draw_landmarks(
                    annotated_frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
    
                # Get the wrist landmark as reference (landmark index 0)
                wrist = hand_landmarks.landmark[mp_hands.HandLandmark.WRIST]
                wrist_x = wrist.x
                wrist_y = wrist.y
                wrist_z = wrist.z
    
                # Initialize a list to store adjusted landmarks
                landmarks_row = [frame_count, hand_label]
                current_landmarks = []  # To store current frame's landmarks
                current_velocities = []  # To store current frame's velocities
    
                for i, landmark in enumerate(hand_landmarks.landmark):
                    # Original normalized coordinates
                    x_raw = landmark.x
                    y_raw = landmark.y
                    z_raw = landmark.z
    
                    # Adjust landmarks relative to wrist
                    x = x_raw - wrist_x
                    y = y_raw - wrist_y
                    z = z_raw - wrist_z
    
                    # Calculate velocity using previous landmarks
                    if previous_landmarks[hand_label] is not None:
                        previous_x, previous_y, previous_z = previous_landmarks[hand_label][i]
                        velocity_x = (x - previous_x) / delta_t
                        velocity_y = (y - previous_y) / delta_t
                        velocity_z = (z - previous_z) / delta_t
    
                        # Optionally, apply a moving average filter for smoothing
                        if previous_velocities[hand_label] is not None:
                            prev_velocity_x, prev_velocity_y, prev_velocity_z = previous_velocities[hand_label][i]
                            smoothed_velocity_x = (velocity_x + prev_velocity_x) / 2
                            smoothed_velocity_y = (velocity_y + prev_velocity_y) / 2
                            smoothed_velocity_z = (velocity_z + prev_velocity_z) / 2
                        else:
                            smoothed_velocity_x = velocity_x
                            smoothed_velocity_y = velocity_y
                            smoothed_velocity_z = velocity_z
    
                    else:
                        velocity_x = 0.0
                        velocity_y = 0.0
                        velocity_z = 0.0
                        smoothed_velocity_x = 0.0
                        smoothed_velocity_y = 0.0
                        smoothed_velocity_z = 0.0
    
                    current_landmarks.append((x, y, z))
                    current_velocities.append((velocity_x, velocity_y, velocity_z))
    
                    # Apply Kalman filter to the x, y, and z coordinates
                    kalman_x = kalman_filters[hand_label][i]['x']
                    kalman_y = kalman_filters[hand_label][i]['y']
                    kalman_z = kalman_filters[hand_label][i]['z']
    
                    # Prediction step
                    kalman_x.predict()
                    kalman_y.predict()
                    kalman_z.predict()
    
                    # Update step with measurement
                    kalman_x_estimated, kalman_velocity_x = kalman_x.update(x)
                    kalman_y_estimated, kalman_velocity_y = kalman_y.update(y)
                    kalman_z_estimated, kalman_velocity_z = kalman_z.update(z)
    
                    # Visualization of Raw and Kalman-Filtered Landmarks
                    # Convert normalized coordinates to pixel coordinates for visualization
                    raw_x_pixel = int(x_raw * width)
                    raw_y_pixel = int(y_raw * height)
                    kalman_x_abs = kalman_x_estimated + wrist_x
                    kalman_y_abs = kalman_y_estimated + wrist_y
                    kalman_x_pixel = int(kalman_x_abs * width)
                    kalman_y_pixel = int(kalman_y_abs * height)
    
                    # Ensure pixel values are within image bounds
                    raw_x_pixel = np.clip(raw_x_pixel, 0, width - 1)
                    raw_y_pixel = np.clip(raw_y_pixel, 0, height - 1)
                    kalman_x_pixel = np.clip(kalman_x_pixel, 0, width - 1)
                    kalman_y_pixel = np.clip(kalman_y_pixel, 0, height - 1)
    
                    # Draw raw landmark (green)
                    cv2.circle(annotated_frame, (raw_x_pixel, raw_y_pixel), 3, (0, 255, 0), -1)  # Green for raw landmark
    
                    # Draw Kalman-filtered landmark (red)
                    cv2.circle(annotated_frame, (kalman_x_pixel, kalman_y_pixel), 3, (0, 0, 255), -1)  # Red for Kalman filtered landmark
    
                    # Append landmarks, velocity, and filtered coordinates to the row
                    landmarks_row.extend([
                        x, y, z,
                        velocity_x, velocity_y, velocity_z,
                        kalman_x_estimated, kalman_y_estimated, kalman_z_estimated
                    ])
    
                    # Collect data for plotting
                    plot_data[hand_label][i]['frames'].append(frame_count)
                    plot_data[hand_label][i]['positions'].append((x, y, z))
                    plot_data[hand_label][i]['velocities'].append((velocity_x, velocity_y, velocity_z))
                    plot_data[hand_label][i]['kalman_positions'].append((kalman_x_estimated, kalman_y_estimated, kalman_z_estimated))
                    plot_data[hand_label][i]['residuals'].append(kalman_x.residual)
                    plot_data[hand_label][i]['error_covariances'].append(kalman_x.covariance[0, 0])
    
                # Write the landmarks to CSV
                csv_writer.writerow(landmarks_row)
    
                # Update previous landmarks and velocities
                previous_landmarks[hand_label] = current_landmarks
                previous_velocities[hand_label] = current_velocities
    
        # Write the annotated frame to output video
        output_video.write(annotated_frame)
    
    frame_count += 1  # Increment frame count regardless of processing
    
    # Optional: Display the annotated frame (uncomment to use)
    cv2.imshow('Annotated Frame', annotated_frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    

    Release resources

    cap.release() output_video.release() hands.close() csv_file.close() cv2.destroyAllWindows() rewrite the code to only save the position and velocity of landmarks, delete the kalman filter portion from it? also delete any irrelevant code and also add important checks for enhanced velocity capturing as well? also explain the modified code in detial??

  2. Assistant

    Here's the modified code that captures and saves the positions and velocities of hand landmarks using MediaPipe, without the Kalman Filter portion. I've also added some important checks to enhance velocity capturing.

    import cv2
    import mediapipe as mp
    import csv
    import numpy as np
    
    # Initialize MediaPipe Hand Tracking
    mp_hands = mp.solutions.hands
    hands = mp_hands.Hands(
        static_image_mode=False,       # For video input
        max_num_hands=2,               # Maximum number of hands to detect
        min_detection_confidence=0.5,  # Minimum confidence for detection
        min_tracking_confidence=0.5    # Minimum confidence for tracking
    )
    mp_drawing = mp.solutions.drawing_utils  # For drawing hand landmarks
    
    # Open the input video
    cap = cv2.VideoCapture('input_video.mp4')
    
    # Get video properties
    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))   # Video frame width
    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # Video frame height
    fps = cap.get(cv2.CAP_PROP_FPS)                  # Original frames per second
    
    # Desired processing frame rate
    desired_fps = 10  # You can adjust this value as needed
    frame_skip = max(int(fps / desired_fps), 1)  # Ensure at least 1
    frame_count = 0  # To keep track of the frame number
    
    # Time difference between frames
    delta_t = 1 / desired_fps
    
    # Initialize Video Writer for output video
    output_video = cv2.VideoWriter(
        'output_video_velocity.mp4',
        cv2.VideoWriter_fourcc(*'mp4v'),
        desired_fps,  # Set output video FPS to desired FPS
        (width, height)
    )
    
    # Prepare to write to CSV file
    csv_file = open('landmarks_with_velocity.csv', mode='w', newline='')
    csv_writer = csv.writer(csv_file)
    
    # Create CSV header
    header = ['Frame', 'Hand']
    for i in range(21):  # 21 landmarks in MediaPipe Hands
        header.extend([
            f'Landmark_{i}_x', f'Landmark_{i}_y', f'Landmark_{i}_z',
            f'Landmark_{i}_velocity_x', f'Landmark_{i}_velocity_y', f'Landmark_{i}_velocity_z'
        ])
    # Write header to CSV
    csv_writer.writerow(header)
    
    # To store previous frame's landmarks and velocities for velocity calculation
    previous_landmarks = {'Left': None, 'Right': None}
    previous_frame_count = {'Left': None, 'Right': None}
    
    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
    
        # Process every 'frame_skip' frames to achieve desired FPS
        if frame_count % frame_skip == 0:
            # Convert frame to RGB for MediaPipe processing
            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            result = hands.process(frame_rgb)
    
            # Create a copy of the frame for drawing
            annotated_frame = frame.copy()
    
            # Check if hand landmarks are detected
            if result.multi_hand_landmarks and result.multi_handedness:
                for hand_landmarks, handedness in zip(result.multi_hand_landmarks, result.multi_handedness):
                    # Get hand label ('Left' or 'Right')
                    hand_label = handedness.classification[0].label  # 'Left' or 'Right'
    
                    # Draw the hand landmarks on the frame
                    mp_drawing.draw_landmarks(
                        annotated_frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
    
                    # Get the wrist landmark as reference (landmark index 0)
                    wrist = hand_landmarks.landmark[mp_hands.HandLandmark.WRIST]
                    wrist_x = wrist.x
                    wrist_y = wrist.y
                    wrist_z = wrist.z
    
                    # Initialize a list to store adjusted landmarks
                    landmarks_row = [frame_count, hand_label]
                    current_landmarks = []  # To store current frame's landmarks
    
                    for i, landmark in enumerate(hand_landmarks.landmark):
                        # Original normalized coordinates
                        x_raw = landmark.x
                        y_raw = landmark.y
                        z_raw = landmark.z
    
                        # Adjust landmarks relative to wrist
                        x = x_raw - wrist_x
                        y = y_raw - wrist_y
                        z = z_raw - wrist_z
    
                        current_landmarks.append((x, y, z))
    
                        # Initialize velocity
                        velocity_x = 0.0
                        velocity_y = 0.0
                        velocity_z = 0.0
    
                        # Calculate velocity using previous landmarks
                        if previous_landmarks[hand_label] is not None:
                            # Compute actual time difference
                            frames_skipped = frame_count - previous_frame_count[hand_label]
                            actual_delta_t = frames_skipped * delta_t
    
                            previous_x, previous_y, previous_z = previous_landmarks[hand_label][i]
    
                            # Compute velocities
                            velocity_x = (x - previous_x) / actual_delta_t
                            velocity_y = (y - previous_y) / actual_delta_t
                            velocity_z = (z - previous_z) / actual_delta_t
    
                            # Apply a simple threshold to filter out unrealistic velocities
                            max_velocity = 1.0  # Adjust this threshold as needed
                            if abs(velocity_x) > max_velocity or abs(velocity_y) > max_velocity or abs(velocity_z) > max_velocity:
                                velocity_x = 0.0
                                velocity_y = 0.0
                                velocity_z = 0.0
    
                        # Append landmarks and velocities to the row
                        landmarks_row.extend([
                            x, y, z,
                            velocity_x, velocity_y, velocity_z
                        ])
    
                        # Draw the landmark on the annotated frame
                        x_pixel = int((x_raw) * width)
                        y_pixel = int((y_raw) * height)
                        cv2.circle(annotated_frame, (x_pixel, y_pixel), 3, (0, 255, 0), -1)  # Green dot
    
                    # Write the landmarks to CSV
                    csv_writer.writerow(landmarks_row)
    
                    # Update previous landmarks and frame count
                    previous_landmarks[hand_label] = current_landmarks
                    previous_frame_count[hand_label] = frame_count
    
            else:
                # If no hand is detected, reset previous landmarks and frame count
                previous_landmarks = {'Left': None, 'Right': None}
                previous_frame_count = {'Left': None, 'Right': None}
    
            # Write the annotated frame to output video
            output_video.write(annotated_frame)
    
        frame_count += 1  # Increment frame count regardless of processing
    
        # Optional: Display the annotated frame (uncomment to use)
        # cv2.imshow('Annotated Frame', annotated_frame)
        # if cv2.waitKey(1) & 0xFF == ord('q'):
        #     break
    
    # Release resources
    cap.release()
    output_video.release()
    hands.close()
    csv_file.close()
    cv2.destroyAllWindows()
    

    Explanation of the Modified Code

    Imports and Initialization

    import cv2
    import mediapipe as mp
    import csv
    import numpy as np
    
    • cv2: OpenCV library for video processing.
    • mediapipe: Library for hand landmark detection.
    • csv: For writing the output to a CSV file.
    • numpy: For numerical computations.

    MediaPipe Hand Tracking Initialization

    mp_hands = mp.solutions.hands
    hands = mp_hands.Hands(
        static_image_mode=False,       # Video input (not static images)
        max_num_hands=2,               # Detect up to 2 hands
        min_detection_confidence=0.5,  # Minimum confidence for detection
        min_tracking_confidence=0.5    # Minimum confidence for tracking
    )
    mp_drawing = mp.solutions.drawing_utils  # Utilities for drawing landmarks
    
    • mp_hands: Accesses MediaPipe's hand solutions.
    • hands: Initializes the hand tracking model with specified parameters.
    • mp_drawing: Provides functions to draw landmarks on images.

    Video Input and Output Setup

    # Open the input video
    cap = cv2.VideoCapture('input_video.mp4')
    
    # Get video properties
    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))   # Width of the frames
    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # Height of the frames
    fps = cap.get(cv2.CAP_PROP_FPS)                  # Frames per second of the input video
    
    # Desired processing frame rate
    desired_fps = 10  # Adjust this value as needed
    frame_skip = max(int(fps / desired_fps), 1)  # Calculate how many frames to skip to achieve desired FPS
    frame_count = 0  # To keep track of the frame number
    
    # Time difference between frames
    delta_t = 1 / desired_fps
    
    # Initialize Video Writer for output video
    output_video = cv2.VideoWriter(
        'output_video_velocity.mp4',
        cv2.VideoWriter_fourcc(*'mp4v'),
        desired_fps,  # Set output video FPS to desired FPS
        (width, height)
    )
    
    • cap: Captures the input video.
    • width, height, fps: Extract properties from the input video.
    • desired_fps: Sets the rate at which we process frames (to reduce computational load).
    • frame_skip: Calculates the number of frames to skip to achieve the desired FPS.
    • delta_t: Time difference between the processed frames.
    • output_video: Video writer to save the annotated frames.

    CSV File Setup

    # Prepare to write to CSV file
    csv_file = open('landmarks_with_velocity.csv', mode='w', newline='')
    csv_writer = csv.writer(csv_file)
    
    # Create CSV header
    header = ['Frame', 'Hand']
    for i in range(21):  # There are 21 landmarks in MediaPipe Hands model
        header.extend([
            f'Landmark_{i}_x', f'Landmark_{i}_y', f'Landmark_{i}_z',
            f'Landmark_{i}_velocity_x', f'Landmark_{i}_velocity_y', f'Landmark_{i}_velocity_z'
        ])
    # Write header to CSV
    csv_writer.writerow(header)
    
    • csv_file: Opens the CSV file for writing.
    • csv_writer: CSV writer object.
    • header: Constructs the header row for the CSV file, including positions and velocities for each landmark.

    Variables to Store Previous Data

    # To store previous frame's landmarks and frame counts for velocity calculation
    previous_landmarks = {'Left': None, 'Right': None}
    previous_frame_count = {'Left': None, 'Right': None}
    
    • previous_landmarks: Stores the landmarks from the previous frame for each hand.
    • previous_frame_count: Stores the frame number of the previous landmarks for each hand.

    Main Processing Loop

    while cap.isOpened():
        ret, frame = cap.read()
        if not ret:
            break
    
        # Process every 'frame_skip' frames to achieve desired FPS
        if frame_count % frame_skip == 0:
            # Convert frame to RGB for MediaPipe processing
            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            result = hands.process(frame_rgb)
    
            # Create a copy of the frame for drawing
            annotated_frame = frame.copy()
            ...
    
    • cap.isOpened(): Checks if the video capture is open.
    • ret, frame: Reads a frame from the video.
    • frame_count % frame_skip == 0: Processes every nth frame to achieve the desired FPS.
    • frame_rgb: Converts the frame to RGB as MediaPipe expects RGB input.
    • result: Processes the frame to detect hands and landmarks.
    • annotated_frame: A copy of the frame where annotations will be drawn.

    Hand Detection and Landmark Processing

            # Check if hand landmarks are detected
            if result.multi_hand_landmarks and result.multi_handedness:
                for hand_landmarks, handedness in zip(result.multi_hand_landmarks, result.multi_handedness):
                    # Get hand label ('Left' or 'Right')
                    hand_label = handedness.classification[0].label  # 'Left' or 'Right'
    
                    # Draw the hand landmarks on the frame
                    mp_drawing.draw_landmarks(
                        annotated_frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
    
                    # Get the wrist landmark as reference (landmark index 0)
                    wrist = hand_landmarks.landmark[mp_hands.HandLandmark.WRIST]
                    wrist_x = wrist.x
                    wrist_y = wrist.y
                    wrist_z = wrist.z
    
                    # Initialize a list to store adjusted landmarks
                    landmarks_row = [frame_count, hand_label]
                    current_landmarks = []  # To store current frame's landmarks
                    ...
    
    • result.multi_hand_landmarks: List of detected hand landmarks.
    • hand_label: Determines whether the hand is left or right.
    • mp_drawing.draw_landmarks: Draws the hand landmarks on the frame for visualization.
    • wrist: Uses the wrist landmark as a reference for relative positioning.
    • landmarks_row: List to store the data that will be written to the CSV.

    Processing Each Landmark

                    for i, landmark in enumerate(hand_landmarks.landmark):
                        # Original normalized coordinates
                        x_raw = landmark.x
                        y_raw = landmark.y
                        z_raw = landmark.z
    
                        # Adjust landmarks relative to wrist
                        x = x_raw - wrist_x
                        y = y_raw - wrist_y
                        z = z_raw - wrist_z
    
                        current_landmarks.append((x, y, z))
    
                        # Initialize velocity
                        velocity_x = 0.0
                        velocity_y = 0.0
                        velocity_z = 0.0
    
                        # Calculate velocity using previous landmarks
                        if previous_landmarks[hand_label] is not None:
                            # Compute actual time difference
                            frames_skipped = frame_count - previous_frame_count[hand_label]
                            actual_delta_t = frames_skipped * delta_t
    
                            previous_x, previous_y, previous_z = previous_landmarks[hand_label][i]
    
                            # Compute velocities
                            velocity_x = (x - previous_x) / actual_delta_t
                            velocity_y = (y - previous_y) / actual_delta_t
                            velocity_z = (z - previous_z) / actual_delta_t
    
                            # Apply a simple threshold to filter out unrealistic velocities
                            max_velocity = 1.0  # Adjust this threshold as needed
                            if abs(velocity_x) > max_velocity or abs(velocity_y) > max_velocity or abs(velocity_z) > max_velocity:
                                velocity_x = 0.0
                                velocity_y = 0.0
                                velocity_z = 0.0
    
                        # Append landmarks and velocities to the row
                        landmarks_row.extend([
                            x, y, z,
                            velocity_x, velocity_y, velocity_z
                        ])
    
                        # Draw the landmark on the annotated frame
                        x_pixel = int((x_raw) * width)
                        y_pixel = int((y_raw) * height)
                        cv2.circle(annotated_frame, (x_pixel, y_pixel), 3, (0, 255, 0), -1)  # Green dot
    
    • for i, landmark in enumerate(...): Iterates over each landmark point.
    • x_raw, y_raw, z_raw: Original coordinates normalized between 0 and 1.
    • x, y, z: Adjusted coordinates relative to the wrist.
    • current_landmarks: Stores the adjusted landmarks for use in the next frame.
    • velocity_x, velocity_y, velocity_z: Initialized to zero.
    • if previous_landmarks[hand_label] is not None: Checks if there is a previous frame to compare for velocity calculation.
    • frames_skipped: Calculates how many frames have been skipped since the last processed frame.
    • actual_delta_t: Computes the actual time difference based on frames skipped.
    • Compute velocities: Calculates the change in position over time.
    • max_velocity: A threshold to filter out unrealistic velocities (could be due to noise or sudden spikes).
    • if abs(velocity_x) > max_velocity ...: If the calculated velocity exceeds the threshold, reset it to zero.
    • landmarks_row.extend(...): Adds the position and velocity data to the row.
    • cv2.circle(...): Draws the landmark on the annotated frame.

    Writing Data and Updating Previous State

                    # Write the landmarks to CSV
                    csv_writer.writerow(landmarks_row)
    
                    # Update previous landmarks and frame count
                    previous_landmarks[hand_label] = current_landmarks
                    previous_frame_count[hand_label] = frame_count
    
    • csv_writer.writerow(landmarks_row): Writes the current data row to the CSV file.
    • previous_landmarks[hand_label] = current_landmarks: Updates the previous landmarks for the next frame's velocity calculation.
    • previous_frame_count[hand_label] = frame_count: Updates the frame count when the landmarks were last updated.

    Handling Cases When No Hands Are Detected

            else:
                # If no hand is detected, reset previous landmarks and frame count
                previous_landmarks = {'Left': None, 'Right': None}
                previous_frame_count = {'Left': None, 'Right': None}
    
    • If no hands are detected in the current frame, reset the previous landmarks and frame counts to prevent incorrect velocity calculations when the hand reappears.

    Writing the Annotated Frame and Incrementing Frame Count

            # Write the annotated frame to output video
            output_video.write(annotated_frame)
    
        frame_count += 1  # Increment frame count regardless of processing
    
    • output_video.write(annotated_frame): Writes the current annotated frame to the output video.
    • frame_count += 1: Increments the frame count.

    Clean Up and Resource Release

    # Release resources
    cap.release()
    output_video.release()
    hands.close()
    csv_file.close()
    cv2.destroyAllWindows()
    
    • cap.release(): Releases the video capture object.
    • output_video.release(): Releases the video writer object.
    • hands.close(): Closes the MediaPipe hands process.
    • csv_file.close(): Closes the CSV file.
    • cv2.destroyAllWindows(): Closes any OpenCV windows.

    Enhancements for Velocity Capturing

    • Actual Time Difference Calculation: Instead of assuming a fixed delta_t, the code calculates the actual time difference based on the number of frames skipped (frames_skipped). This improves the accuracy of the velocity calculation, especially if the processing frame rate is not constant.

      frames_skipped = frame_count - previous_frame_count[hand_label]
      actual_delta_t = frames_skipped * delta_t
      
    • Velocity Thresholding: A maximum velocity threshold (max_velocity) is applied to filter out unrealistic velocities that may result from sudden noise or tracking errors.

      max_velocity = 1.0  # Adjust this threshold as needed
      if abs(velocity_x) > max_velocity or abs(velocity_y) > max_velocity or abs(velocity_z) > max_velocity:
          velocity_x = 0.0
          velocity_y = 0.0
          velocity_z = 0.0
      
    • Resetting State When No Hands Are Detected: When no hands are detected, the previous landmarks and frame counts are reset. This prevents incorrect velocity calculations when the hand appears again after being missing.

      else:
          # If no hand is detected, reset previous landmarks and frame count
          previous_landmarks = {'Left': None, 'Right': None}
          previous_frame_count = {'Left': None, 'Right': None}
      

    Summary

    This modified code processes an input video to detect hand landmarks using MediaPipe, calculates the velocities of each landmark, and saves the positions and velocities to a CSV file. The code removes the Kalman filter portions and irrelevant code, adds important checks for enhanced velocity capturing, and provides a clear, step-by-step explanation of the modifications.