The following python code which generates a soccer field homography of a soccer video…
- User
The following python code which generates a soccer field homography of a soccer video using a trained yolo model. Please analyse the code to see how the homography can be made smoother with less jitter, given the fact the camera panning is smooth.
import cv2 import numpy as np import time from ultralytics import YOLO
--- Complete mapping with reference coordinates ---
CLASS_TO_KPT = { "cul": 0, "pbl1": 1, "gl1": 2, "gl2": 3, "pbl2": 4, "cll": 5, "gl4": 6, "gl3": 7, "pl": 8, "pbl6": 9, "pbl5": 10, "pbl4": 11, "pbl3": 12, "hlu": 13, "cu": 14, "cd": 15, "hll": 16, "pbr6": 17, "pbr5": 18, "pbr4": 19, "pbr3": 20, "pr": 21, "gr4": 22, "gr3": 23, "cur": 24, "pbr1": 25, "gr1": 26, "gr2": 27, "pbr2": 28, "clr": 29, "cl": 30, "cr": 31, }
Punten die extra zwaar mee moeten tellen
IMPORTANT_KPTS = { 13: 4, # hlu 14: 8, # cu 15: 8, # cd 30: 8, # cl 31: 8, # cr 16: 4, # hll 17: 3, # pbr6 18: 3, # pbr5 23: 3, # gr3 }
MIN_CONFIDENCE = 0.5
-------------------------------------------
FIFA standard pitch model (105 x 68 m)
-------------------------------------------
PITCH_POINTS = { "cul": (0, 68), "cll": (0, 0), "cur": (105, 68), "clr": (105, 0), "cl": (43.35, 34), # <--- CORRECTED "cr": (61.65, 34), # <--- CORRECTED "cu": (52.5, 34 + 9.15), "cd": (52.5, 34 - 9.15), "pl": (11, 34), "pr": (94, 34), "pbl6": (16.5, 54.15), "pbl5": (16.5, 13.85), "pbl1": (0, 13.85), "pbl2": (0, 54.15), "pbr6": (88.5, 54.15), "pbr5": (88.5, 13.85), "pbr1": (105, 13.85), "pbr2": (105, 54.15), "gl1": (0, 24.84), "gl2": (0, 43.16), "gl3": (5.5, 24.84), "gl4": (5.5, 43.16), "gr1": (105, 24.84), "gr2": (105, 43.16), "gr3": (99.5, 24.84), "gr4": (99.5, 43.16), }
PITCH_LINES = [ [(0,0), (105,0), (105,68), (0,68), (0,0)], [(52.5,0), (52.5,68)], [(52.5 + 9.15np.cos(a), 34 + 9.15np.sin(a)) for a in np.linspace(0, 2*np.pi, 40)], [(0, 13.85), (16.5, 13.85), (16.5, 54.15), (0, 54.15)], [(105, 13.85), (88.5, 13.85), (88.5, 54.15), (105, 54.15)], [(0, 24.84), (5.5, 24.84), (5.5, 43.16), (0, 43.16)], [(105, 24.84), (99.5, 24.84), (99.5, 43.16), (105, 43.16)], ]
class HomographySmoother: """ Applies an Exponential Moving Average to stabilize the homography matrix over time. """ def init(self, alpha=0.5): """ Initializes the smoother. :param alpha: The smoothing factor, a value between 0 and 1. A smaller alpha results in more smoothing (and more latency). A larger alpha results in less smoothing (and more responsiveness). """ self.alpha = alpha self.smoothed_H = None
def update(self, new_H): """ Updates the smoothed homography with a new matrix. If the new matrix is invalid, it returns the last known good matrix. """ # If the new homography is invalid, do not update, just return the last good one. if new_H is None: return self.smoothed_H # If this is the first homography, initialize it directly. if self.smoothed_H is None: self.smoothed_H = new_H return self.smoothed_H # Apply Exponential Moving Average self.smoothed_H = self.alpha * new_H + (1.0 - self.alpha) * self.smoothed_H return self.smoothed_H--- weighted homography ---
def compute_weighted_homography(src_pts, dst_pts, confidences): weighted_field_pts, weighted_img_pts, weighted_src_indices = [], [], [] valid_pairs_candidates = []
for name, idx in CLASS_TO_KPT.items(): if idx >= len(src_pts): continue s, conf = src_pts[idx], confidences[idx] if s is None or conf < MIN_CONFIDENCE: continue if name not in PITCH_POINTS: continue d = PITCH_POINTS[name] base_weight = IMPORTANT_KPTS.get(idx, 1) rep = max(1, int(round(base_weight * float(conf)))) for _ in range(rep): weighted_field_pts.append(np.array(d, dtype=np.float32)) weighted_img_pts.append(np.array(s, dtype=np.float32)) weighted_src_indices.append(idx) valid_pairs_candidates.append((idx, conf)) # If len is less than 4, return early if len(weighted_field_pts) < 4: return None, None, [] # Note the swapped arguments here to calculate H_img2field field_arr = np.array(weighted_field_pts, dtype=np.float32) img_arr = np.array(weighted_img_pts, dtype=np.float32) H_img2field, mask = cv2.findHomography(img_arr, field_arr, cv2.RANSAC, 5.0) # Check if a valid homography was found if H_img2field is None: return None, None, [] # Invert the matrix to get the correct transformation for drawing H_field2img = np.linalg.inv(H_img2field) mask = mask.ravel() inlier_indices_set = {orig_idx for m, orig_idx in zip(mask, weighted_src_indices) if int(m) == 1} valid_pairs = [(i, conf) for i, conf in valid_pairs_candidates if i in inlier_indices_set] return H_field2img.astype(np.float32), mask, valid_pairs--- Draw FIFA lines using homography ---
def draw_pitch(frame, H): if H is None: return frame for line in PITCH_LINES: pts = np.array(line, dtype=np.float32).reshape(-1,1,2) projected = cv2.perspectiveTransform(pts, H) cv2.polylines(frame, [np.int32(projected)], False, (255,255,0), 2) return frame
--- main video loop ---
def process_video(input_path, output_path, model_path="./models/football-pitch-detection.pt", show_confidence=True, confidence_log_interval=100): print(f"\nMerged script with FIFA pitch + weighted homography + smoothing") model = YOLO(model_path).to(device='mps') cap = cv2.VideoCapture(input_path) if not cap.isOpened(): print("❌ Cannot open video"); return
fourcc = cv2.VideoWriter_fourcc(*'mp4v') fps = cap.get(cv2.CAP_PROP_FPS) W, H = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) out = cv2.VideoWriter(output_path, fourcc, fps, (W, H)) # ================================================================================= # NEW: Initialize the Homography Smoother # You can tune the alpha value. 0.3-0.6 is usually a good range. # Lower alpha = more smooth, higher alpha = more responsive. h_smoother = HomographySmoother(alpha=0.4) # ================================================================================= frame_idx, start_time = 0, time.time() total_low_conf, total_high_conf = 0, 0 while True: ret, frame = cap.read() if not ret: break results = model(frame, verbose=False) detected_points, confidences = [None]*32, [0.0]*32 if results and results[0].keypoints is not None: kpts_data = results[0].keypoints kpts_xy = kpts_data.xy.cpu().numpy()[0] kpts_conf = kpts_data.conf.cpu().numpy()[0] for i in range(min(len(kpts_xy), 32)): x, y, conf = kpts_xy[i][0], kpts_xy[i][1], float(kpts_conf[i]) if x > 0 and y > 0: detected_points[i] = np.array([x, y], dtype=np.float32) confidences[i] = conf if conf < MIN_CONFIDENCE: total_low_conf += 1 else: total_high_conf += 1 # First, compute the raw homography for the current frame raw_H_field2img, mask, valid_pairs = compute_weighted_homography(detected_points, PITCH_POINTS, confidences) # ================================================================================= # NEW: Update the smoother with the new raw homography to get a stable version smoothed_H = h_smoother.update(raw_H_field2img) # ================================================================================= # ================================================================================= # CHANGED: Use the smoothed homography for drawing frame = draw_pitch(frame, smoothed_H) # ================================================================================= if smoothed_H is not None: cv2.putText(frame, f"Valid pts: {len(valid_pairs)}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,255,0), 2) # (The rest of your point drawing code remains the same) for i in range(32): pt, conf = detected_points[i], confidences[i] if pt is None: continue x, y = int(pt[0]), int(pt[1]) if conf < MIN_CONFIDENCE: color = (0,0,255) elif conf < 0.7: color = (0,255,255) else: color = (0,255,0) radius = 8 if i in IMPORTANT_KPTS else 4 cv2.circle(frame, (x,y), radius, color, -1) if show_confidence: cv2.putText(frame, f"{i}:{conf:.2f}", (x+8,y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1) out.write(frame) frame_idx += 1 if frame_idx % confidence_log_interval == 0: elapsed = time.time() - start_time avg_conf = np.mean([c for c in confidences if c > 0]) if any(confidences) else 0 print(f"Frame {frame_idx} | {elapsed:.1f}s | Avg conf: {avg_conf:.3f}") cap.release(); out.release() print(f"\n✅ Done. Output: {output_path}")if name == "main": process_video( "../_sports/input/match4.mp4", "./output/output_weighted_chatgpt_google_1p1.mp4", #"./output/output_pitch_merged.mp4", show_confidence=True, confidence_log_interval=100 )
===
- Assistant
To make the homography smoother with less jitter in the soccer video processing, several key improvements can be made. The primary issues stem from coordinate system mismatches and insufficient temporal smoothing. Here's a step-by-step analysis and solution:
Key Issues:
-
Coordinate System Mismatch:
- The
PITCH_POINTSandPITCH_LINESare defined in meters, but the video frames are in pixels. This mismatch causes distortion in homography calculations. - The normalized grid should have the same aspect ratio as the physical pitch (105m x 68m) to preserve spatial relationships.
- The
-
Jitter in Homography:
- Variations in key point detection (due to motion, occlusion, or lighting) cause jitter.
- The Exponential Moving Average (EMA) smoothing is effective but may need tuning for responsiveness vs. stability.
Solutions Implemented:
-
Normalize Coordinates:
- Convert all pitch points from meters to a normalized coordinate system (0-1) while preserving the aspect ratio.
- This ensures homography calculations are consistent between detected points and reference points.
-
Tune EMA Smoothing:
- Reduce the smoothing factor (
alpha) to 0.3 for better stability while maintaining responsiveness to camera movements.
- Reduce the smoothing factor (
-
Optimize Homography Estimation:
- Use weighted homography with confidence-based scaling to prioritize reliable detections.
- Increase the RANSAC tolerance (from 5.0 to 7.0) to handle more inliers during motion.
Modified Code:
import cv2 import numpy as np import time from ultralytics import YOLO # --- Complete mapping with reference coordinates --- CLASS_TO_KPT = { "cul": 0, "pbl1": 1, "gl1": 2, "gl2": 3, "pbl2": 4, "cll": 5, "gl4": 6, "gl3": 7, "pl": 8, "pbl6": 9, "pbl5": 10, "pbl4": 11, "pbl3": 12, "hlu": 13, "cu": 14, "cd": 15, "hll": 16, "pbr6": 17, "pbr5": 18, "pbr4": 19, "pbr3": 20, "pr": 21, "gr4": 22, "gr3": 23, "cur": 24, "pbr1": 25, "gr1": 26, "gr2": 27, "pbr2": 28, "clr": 29, "cl": 30, "cr": 31, } IMPORTANT_KPTS = { 13: 4, # hlu 14: 8, # cu 15: 8, # cd 30: 8, # cl 31: 8, # cr 16: 4, # hll 17: 3, # pbr6 18: 3, # pbr5 23: 3, # gr3 } MIN_CONFIDENCE = 0.5 # ------------------------------------------- # FIFA standard pitch model (105 x 68 m) # ------------------------------------------- PITCH_POINTS = { "cul": (0, 68), "cll": (0, 0), "cur": (105, 68), "clr": (105, 0), "cl": (43.35, 34), # <--- CORRECTED "cr": (61.65, 34), # <--- CORRECTED "cu": (52.5, 34 + 9.15), "cd": (52.5, 34 - 9.15), "pl": (11, 34), "pr": (94, 34), "pbl6": (16.5, 54.15), "pbl5": (16.5, 13.85), "pbl1": (0, 13.85), "pbl2": (0, 54.15), "pbr6": (88.5, 54.15), "pbr5": (88.5, 13.85), "pbr1": (105, 13.85), "pbr2": (105, 54.15), "gl1": (0, 24.84), "gl2": (0, 43.16), "gl3": (5.5, 24.84), "gl4": (5.5, 43.16), "gr1": (105, 24.84), "gr2": (105, 43.16), "gr3": (99.5, 24.84), "gr4": (99.5, 43.16), } # Convert to normalized coordinates (0-1) preserving aspect ratio normalized_PITCH_POINTS = {} for name, (x, y) in PITCH_POINTS.items(): normalized_PITCH_POINTS[name] = (x / 105.0, y / 68.0) PITCH_LINES = [ [(0,0), (105,0), (105,68), (0,68), (0,0)], [(52.5,0), (52.5,68)], [(52.5 + 9.15*np.cos(a), 34 + 9.15*np.sin(a)) for a in np.linspace(0, 2*np.pi, 40)], [(0, 13.85), (16.5, 13.85), (16.5, 54.15), (0, 54.15)], [(105, 13.85), (88.5, 13.85), (88.5, 54.15), (105, 54.15)], [(0, 24.84), (5.5, 24.84), (5.5, 43.16), (0, 43.16)], [(105, 24.84), (99.5, 24.84), (99.5, 43.16), (105, 43.16)], ] # Convert lines to normalized coordinates normalized_PITCH_LINES = [] for line in PITCH_LINES: normalized_line = [] for point in line: x, y = point normalized_line.append((x / 105.0, y / 68.0)) normalized_PITCH_LINES.append(normalized_line) MIN_RANSAC_TOLERANCE = 7.0 # Increased for better inlier handling class HomographySmoother: """ Applies an Exponential Moving Average to stabilize the homography matrix over time. """ def __init__(self, alpha=0.5): self.alpha = alpha self.smoothed_H = None def update(self, new_H): if new_H is None: return self.smoothed_H if self.smoothed_H is None: self.smoothed_H = new_H return self.smoothed_H self.smoothed_H = self.alpha * new_H + (1.0 - self.alpha) * self.smoothed_H return self.smoothed_H # --- weighted homography --- def compute_weighted_homography(src_pts, dst_pts, confidences): weighted_field_pts, weighted_img_pts, weighted_src_indices = [], [], [] valid_pairs_candidates = [] for name, idx in CLASS_TO_KPT.items(): if idx >= len(src_pts): continue s, conf = src_pts[idx], confidences[idx] if s is None or conf < MIN_CONFIDENCE: continue if name not in normalized_PITCH_POINTS: # Use normalized points continue d = normalized_PITCH_POINTS[name] # Use normalized reference points base_weight = IMPORTANT_KPTS.get(idx, 1) rep = max(1, int(round(base_weight * float(conf)))) for _ in range(rep): weighted_field_pts.append(np.array(d, dtype=np.float32)) weighted_img_pts.append(np.array(s, dtype=np.float32)) weighted_src_indices.append(idx) valid_pairs_candidates.append((idx, conf)) if len(weighted_field_pts) < 4: return None, None, [] field_arr = np.array(weighted_field_pts, dtype=np.float32) img_arr = np.array(weighted_img_pts, dtype=np.float32) H_img2field, mask = cv2.findHomography(img_arr, field_arr, cv2.RANSAC, MIN_RANSAC_TOLERANCE) if H_img2field is None: return None, None, [] H_field2img = np.linalg.inv(H_img2field) mask = mask.ravel() inlier_indices_set = {orig_idx for m, orig_idx in zip(mask, weighted_src_indices) if int(m) == 1} valid_pairs = [(i, conf) for i, conf in valid_pairs_candidates if i in inlier_indices_set] return H_field2img.astype(np.float32), mask, valid_pairs # --- Draw FIFA lines using homography --- def draw_pitch(frame, H): if H is None: return frame for line in normalized_PITCH_LINES: # Use normalized lines pts = np.array(line, dtype=np.float32).reshape(-1,1,2) projected = cv2.perspectiveTransform(pts, H) cv2.polylines(frame, [np.int32(projected)], False, (255,255,0), 2) return frame # --- main video loop --- def process_video(input_path, output_path, model_path="./models/football-pitch-detection.pt", show_confidence=True, confidence_log_interval=100): print(f"\nMerged script with FIFA pitch + weighted homography + smoothing") model = YOLO(model_path).to(device='mps') cap = cv2.VideoCapture(input_path) if not cap.isOpened(): print("❌ Cannot open video"); return fourcc = cv2.VideoWriter_fourcc(*'mp4v') fps = cap.get(cv2.CAP_PROP_FPS) W, H = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) out = cv2.VideoWriter(output_path, fourcc, fps, (W, H)) # New: Initialize the Homography Smoother with tuned alpha h_smoother = HomographySmoother(alpha=0.3) # Reduced alpha for smoother result # New: Increased RANSAC tolerance for better inlier handling frame_idx, start_time = 0, time.time() total_low_conf, total_high_conf = 0, 0 while True: ret, frame = cap.read() if not ret: break results = model(frame, verbose=False) detected_points, confidences = [None]*32, [0.0]*32 if results and results[0].keypoints is not None: kpts_data = results[0].keypoints kpts_xy = kpts_data.xy.cpu().numpy()[0] kpts_conf = kpts_data.conf.cpu().numpy()[0] for i in range(min(len(kpts_xy), 32)): x, y, conf = kpts_xy[i][0], kpts_xy[i][1], float(kpts_conf[i]) if x > 0 and y > 0: detected_points[i] = np.array([x, y], dtype=np.float32) confidences[i] = conf if conf < MIN_CONFIDENCE: total_low_conf += 1 else: total_high_conf += 1 raw_H_field2img, mask, valid_pairs = compute_weighted_homography( detected_points, normalized_PITCH_POINTS, confidences # Use normalized reference points ) # Update smoother with new homography smoothed_H = h_smoother.update(raw_H_field2img) # Use smoothed homography for drawing frame = draw_pitch(frame, smoothed_H) if smoothed_H is not None: cv2.putText(frame, f"Valid pts: {len(valid_pairs)}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,255,0), 2) for i in range(32): pt, conf = detected_points[i], confidences[i] if pt is None: continue x, y = int(pt[0]), int(pt[1]) if conf < MIN_CONFIDENCE: color = (0,0,255) elif conf < 0.7: color = (0,255,255) else: color = (0,255,0) radius = 8 if i in IMPORTANT_KPTS else 4 cv2.circle(frame, (x,y), radius, color, -1) if show_confidence: cv2.putText(frame, f"{i}:{conf:.2f}", (x+8,y-5), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1) out.write(frame) frame_idx += 1 if frame_idx % confidence_log_interval == 0: elapsed = time.time() - start_time avg_conf = np.mean([c for c in confidences if c > 0]) if any(confidences) else 0 print(f"Frame {frame_idx} | {elapsed:.1f}s | Avg conf: {avg_conf:.3f}") cap.release(); out.release() print(f"\n✅ Done. Output: {output_path}") if __name__ == "__main__": process_video( "../_sports/input/match4.mp4", "./output/output_weighted_chatgpt_google_1p1.mp4", model_path, # Fixed: added model_path argument show_confidence=True, confidence_log_interval=100 )Key Improvements:
-
Normalized Coordinate System:
- All pitch points and lines are converted to a 0-1 normalized grid while preserving the aspect ratio (105:68). This ensures consistent spatial relationships between detected points and reference points.
-
Tuned EMA Smoothing:
- Reduced
alphato 0.3 for more aggressive smoothing, reducing jitter during camera panning. This provides a better balance between stability and responsiveness.
- Reduced
-
Enhanced Homography Estimation:
- Increased RANSAC tolerance to 7.0 to handle more inliers during motion, improving robustness.
- Used normalized reference points in
compute_weighted_homographyto avoid coordinate mismatches.
-
Optimized Drawing:
- The
draw_pitchfunction now uses normalized lines, ensuring correct projection regardless of frame aspect ratio.
- The
These changes address the root causes of jitter by improving spatial consistency and temporal smoothing, resulting in a more stable homography that accurately represents the smooth camera panning in soccer videos.
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.