I am getting error: in mergeimages(subimages, keypoints, filteredmatches, homographies)…
- User
I am getting error: in merge_images(sub_images, keypoints, filtered_matches, homographies) 84 for i in range(1, len(sub_images)): 85 img_i = sub_images[i] ---> 86 H_i = homographies[i-1] 87 88 h1, w1 = ref_img.shape[:2]
IndexError: list index out of range In this code: import numpy as np import cv2 import os import glob import matplotlib.pyplot as plt import time
def feature_extraction(sub_images, method="SIFT"): if method == "SIFT": keypoint_extractor = cv2.xfeatures2d.SIFT_create() elif method == "SURF": keypoint_extractor = cv2.xfeatures2d.SURF_create() elif method == "ORB": keypoint_extractor = cv2.ORB_create()
keypoints = [] descriptors = [] for sub_image in sub_images: keypoint, descriptor = keypoint_extractor.detectAndCompute(sub_image, None) keypoints.append(keypoint) descriptors.append(descriptor) return keypoints, descriptorsdef feature_matching(descriptors, matcher_type="BF"): if matcher_type == "BF": matcher = cv2.BFMatcher()
matches = [] for i in range(1, len(descriptors)): match = matcher.knnMatch(descriptors[0], descriptors[i], k=2) matches.append(match) return matchesdef filter_matches(matches, ratio_thres=0.7): filtered_matches = [] for match in matches: good_match = [] for m, n in match: if m.distance < ratio_thres * n.distance: good_match.append(m) filtered_matches.append(good_match)
return filtered_matchesdef find_homography(keypoints, filtered_matches): homographies = []
for i, matches in enumerate(filtered_matches): src_pts = np.float32([keypoints[0][m.queryIdx].pt for m in matches]).reshape(-1, 1, 2) dst_pts = np.float32([keypoints[i + 1][m.trainIdx].pt for m in matches]).reshape(-1, 1, 2) H, _ = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC) if H is not None: H = H.astype(np.float32) homographies.append(H) else: print(f"Warning: Homography computation failed for image pair (0, {i + 1}). Skipping.") continue return homographiesdef read_ground_truth_homographies(dataset_path): H_files = sorted(glob.glob(os.path.join(dataset_path, "H_.txt"))) ground_truth_homographies = []
for filename in H_files: H = np.loadtxt(filename) ground_truth_homographies.append(H) return ground_truth_homographiesdef merge_images(sub_images, keypoints, filtered_matches, homographies): ref_img = sub_images[0]
for i in range(1, len(sub_images)): img_i = sub_images[i] H_i = homographies[i-1] h1, w1 = ref_img.shape[:2] h2, w2 = img_i.shape[:2] pts1 = np.float32([[0, 0], [0, h1], [w1, h1], [w1, 0]]).reshape(-1, 1, 2) pts2 = np.float32([[0, 0], [0, h2], [w2, h2], [w2, 0]]).reshape(-1, 1, 2) pts2_trans = cv2.perspectiveTransform(pts2, H_i) new_pts = np.concatenate((pts1, pts2_trans), axis=0) min_x, min_y = np.int32(new_pts.min(axis=0).ravel()) max_x, max_y = np.int32(new_pts.max(axis=0).ravel()) trans_dst = [-min_x, -min_y] H_trans = np.array([[1.0, 0, trans_dst[0]], [0, 1.0, trans_dst[1]], [0, 0, 1.0]]) result = cv2.warpPerspective(ref_img, H_trans, (max_x - min_x, max_y - min_y)) img_warped = cv2.warpPerspective(img_i, H_i, (result.shape[1], result.shape[0])) mask = (result == 0) result[mask] = img_warped[mask] ref_img = result return ref_imgdef main(dataset_path): filenames = sorted(glob.glob(os.path.join(dataset_path, "*.png"))) sub_images = []
for filename in filenames: img = cv2.imread(filename, cv2.IMREAD_GRAYSCALE) sub_images.append(img) ground_truth_homographies = read_ground_truth_homographies(dataset_path) methods = ["SIFT", "SURF", "ORB"] for method in methods: start_time = time.time() keypoints, descriptors = feature_extraction(sub_images, method=method) matches = feature_matching(descriptors) filtered_matches = filter_matches(matches) homographies = find_homography(keypoints, filtered_matches) panorama = merge_images(sub_images, keypoints, filtered_matches, homographies) end_time = time.time() runtime = end_time - start_time print(f"Method: {method} - Runtime: {runtime:.2f} seconds") for idx, (image, kp) in enumerate(zip(sub_images, keypoints)): feature_plot = cv2.drawKeypoints(image, kp, None) plt.figure() plt.imshow(feature_plot, cmap="gray") plt.title(f"Feature Points - {method} - Image {idx}") for i, match in enumerate(filtered_matches): matching_plot = cv2.drawMatches(sub_images[0], keypoints[0], sub_images[i + 1], keypoints[i + 1], match, None, flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS) plt.figure() plt.imshow(matching_plot, cmap="gray") plt.title(f"Feature Point Matching - {method} - Image 0 - Image {i + 1}") plt.figure() plt.imshow(panorama, cmap="gray") plt.title(f"Panorama - {method}") print("\nGround truth homographies") for i, H_gt in enumerate(ground_truth_homographies): print(f"Image 0 to {i+1}:") print(H_gt) print("\nComputed homographies") for i, H_est in enumerate(homographies): print(f"Image 0 to {i+1}:") print(H_est) plt.show()if name == "main": dataset_path = "dataset/v_bird" main(dataset_path)
- Assistant
It seems that the code is trying to access elements in the "homographies" list that are not available, causing an IndexError ("list index out of range"). I suspect this is happening when the find_homography function fails for an image pair and skips the homography computation, creating a mismatch in the length of the "homographies" list when compared with the length of the "sub_images" list.
You can fix this issue by keeping track of the skipped images and their indices, and then making sure you avoid processing them in the merge_images function. I have modified the relevant functions below:
find_homographyfunction:
def find_homography(keypoints, filtered_matches): homographies = [] skipped_indices = [] # Keep track of skipped images and their indices for i, matches in enumerate(filtered_matches): src_pts = np.float32([keypoints[0][m.queryIdx].pt for m in matches]).reshape(-1, 1, 2) dst_pts = np.float32([keypoints[i + 1][m.trainIdx].pt for m in matches]).reshape(-1, 1, 2) H, _ = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC) if H is not None: H = H.astype(np.float32) homographies.append(H) else: print(f"Warning: Homography computation failed for image pair (0, {i + 1}). Skipping.") skipped_indices.append(i + 1) # Add indices of skipped images to the list continue return homographies, skipped_indices- Modify the
mainfunction to get the "skipped_indices" variable:
... homographies, skipped_indices = find_homography(keypoints, filtered_matches) ...- Modify the
merge_imagesfunction to check if an image index is in the "skipped_indices" list before processing it:
def merge_images(sub_images, keypoints, filtered_matches, homographies, skipped_indices): ref_img = sub_images[0] for i in range(1, len(sub_images)): if i in skipped_indices: # Check if the image index is in the skipped_indices list print(f"Image {i} was skipped due to homography computation failure.") continue img_i = sub_images[i] H_i = homographies[i - 1 - len([idx for idx in skipped_indices if idx < i])] # Adjust the index based on skipped images ...- Finally, pass the "skipped_indices" variable to the
merge_imagesfunction in themainfunction:
... panorama = merge_images(sub_images, keypoints, filtered_matches, homographies, skipped_indices) ...With these changes, the code should handle cases when the homography computation fails for an image pair, avoiding the IndexError issue. Make sure you thoroughly test these changes with various datasets to make sure they work correctly for your case.
- User
ok code produces result but the result does not look right. First of all the image seems to be vertically grown but the merged image supposed to be horizontal as the images provided are horizontal angles. Can you also check how the ground truth homography which is read from the files provided used. They must be used as ground truth homography and used to compute final homography.
- Assistant
It appears that the ground truth homographies are only displayed in the output and not actually used in the merging process. If you want to use the ground truth homographies for merging the images instead of the computed homographies, you can change the following lines in the
mainfunction:From:
panorama = merge_images(sub_images, keypoints, filtered_matches, homographies)To:
panorama = merge_images(sub_images, keypoints, filtered_matches, ground_truth_homographies, skipped_indices)These changes should ensure the ground truth homographies are used in the merge_images function.
Regarding the stretched image issue:
It seems the problem is related to how the image is being displayed by matplotlib, resulting in a vertically stretched final panorama. You can change the aspect ratio for the imshow function in the
mainfunction to have it display the image correctly:Change this line:
plt.imshow(panorama, cmap="gray")To:
plt.imshow(panorama, cmap="gray", aspect='auto')By setting the aspect='auto', matplotlib will automatically adapt the aspect ratio to the size of the axes, which should display the final panorama correctly.
- User
no, may be I am wrong but shouldn't ground truth homographies be used to computed homographies. Like initial reference ? Here is my assignment description if that helps: "Here is the description of the dataset: "Each image sequence contains a reference image and 5 target images taken under a different illumination and/or, for a planar scenes, a different viewpoint. For all images we have the estimated ground truth homography with respect to the reference"
In the images, key points should be found to determine how to merge the multiple images into single one (Such as from which locations and as how much rotated). To detect keypoints in the image in a robust way (rotation and scale invariant) SIFT,SURF and ORB methods have been developed. Dataset includes various scenes consisting of sub-images groups and the estimated ground truth homography with respect to the reference. Here is The Implementation Details that you must follow:
-
Feature Extraction: Firstly you are expected to the extract keypoints in the sub-images by a keypoint extraction method (SIFT/SURF and ORB).
-
Feature Matching : Then you are expected to code a matching function (for example this can be based k-nearest neighbor method) to match extracted keypoints between pairs of sub-images. You can use libraries for this part.
-
Finding Homography: Then you should calculate a Homography Matrix for each pair of sub-images (by using RANSAC method) and you must implement this part by your own. That means you can not use opencv or any library other than numpy.
-
Merging by Transformation: Merge sub-images into single panorama by applying transformation operations on sub-images by using the Homography Matrix. You must also implement this part by your own. That means you can not use opencv or any library other than numpy."
-
- Assistant
Now that I understand your assignment better, I see that your main goal is to implement the homography calculation and merging parts without using the OpenCV library except for feature extraction and matching. In the current code, both
find_homographyandmerge_imagesfunctions use OpenCV, which doesn't follow the assignment requirements.To address this issue, I suggest you rewrite the
find_homographyandmerge_imagesfunctions to avoid using OpenCV for computation, except for feature extraction and matching.Here's an example of how you can implement a RANSAC based homography computation function:
def ransac_homography(src_pts, dst_pts, iterations=2000, threshold=3): best_inlier_count = 0 best_homography = None for _ in range(iterations): indices = np.random.choice(len(src_pts), 4, replace=False) src_subset = src_pts[indices] dst_subset = dst_pts[indices] homography = compute_homography_matrix(src_subset, dst_subset) if homography is None: continue inliers = 0 for i in range(len(src_pts)): projected_point = np.dot(homography, np.append(src_pts[i], 1)) projected_point = projected_point / projected_point[-1] distance = np.linalg.norm(projected_point - np.append(dst_pts[i], 1)) if distance < threshold: inliers += 1 if inliers > best_inlier_count: best_inlier_count = inliers best_homography = homography return best_homographyYou can use this RANSAC function to replace the OpenCV
findHomographycall in thefind_homographyfunction. Note that you'll need to implement thecompute_homography_matrixfunction, which calculates the homography matrix from four corresponding point pairs.For the merge_images function, replace the OpenCV
warpPerspectivecalls with a custom implementation using NumPy. Here's a suggestion on how to warp an image using a homography matrix:def warp_perspective(img, H, target_shape): h, w = target_shape warped_image = np.zeros((h, w), dtype=np.uint8) for i in range(h): for j in range(w): destination_point = np.array([j, i, 1]) source_point = np.dot(np.linalg.inv(H), destination_point) source_x, source_y, w = source_point source_x = int(source_x / w) source_y = int(source_y / w) if source_y < img.shape[0] and source_x < img.shape[1]: warped_image[i][j] = img[source_y][source_x] return warped_imageYou'll need to find the target_shape for each merged image to adjust the final target panorama size accordingly.
Using these custom functions, you can continue to use the computed homographies for merging the sub-images, as the goal is to implement the homography calculation and merging parts without relying on OpenCV. Keep in mind that this code serves as a starting point and may require optimization and tweaking to get the best results.
- User
can you show me example of what you mean by a compute_homography_matrix function ?
- Assistant
Sure, here's an example of the
compute_homography_matrixfunction that takes two sets of four corresponding points and returns the homography matrix between them:def compute_homography_matrix(src_pts, dst_pts): A = [] for i in range(4): u1, v1 = src_pts[i] u2, v2 = dst_pts[i] A.append([-u1, -v1, -1, 0, 0, 0, u1 * u2, v1 * u2, u2]) A.append([0, 0, 0, -u1, -v1, -1, u1 * v2, v1 * v2, v2]) A = np.array(A) _, _, VT = np.linalg.svd(A) h = VT[-1] H = h.reshape(3, 3) return HYou can use this function in the
ransac_homographyfunction I provided in the previous response.Here's how the function works:
compute_homography_matrixtakes two sets of corresponding points,src_ptsanddst_pts, as input.- The function initializes an empty list
Ato construct the coefficient matrix for the linear system Ah = 0. - It iterates through each pair of corresponding points in
src_ptsanddst_pts. - For each pair of points, the function appends two rows to the list
A. These rows correspond to the equationsrc_points * H = dst_pointsin homogenized coordinates. - The function converts
Ainto a NumPy array. - It computes the SVD of the matrix
A. - It retrieves the last column of V from the SVD decomposition, which stores the normalized eigenvector associated with the smallest singular value, and reshapes it into a 3x3 matrix
H. - The function returns the computed homography matrix
H.
Keep in mind that this implementation assumes the provided points are valid for computing a homography (i.e., the points are not collinear). You may want to add a check to ensure the given points are valid before proceeding with the computation.
- User
I think the line u1, v1 = src_pts[i] is wrong because we already reshape our matrix at src_pts = np.float32([keypoints[0][m.queryIdx].pt for m in matches]).reshape(-1, 1, 2) dst_pts = np.float32([keypoints[i + 1][m.trainIdx].pt for m in matches]).reshape(-1, 1, 2)
- Assistant
You're right. The shape of
src_ptsanddst_ptsis indeed (-1, 1, 2). In that case, you need to modify accessing the points in theransac_homographyfunction. Change the lines:src_subset = src_pts[indices] dst_subset = dst_pts[indices]To:
src_subset = src_pts[indices].reshape(-1, 2) dst_subset = dst_pts[indices].reshape(-1, 2)Now the
src_subsetanddst_subsetvariables will have the correct shape, and the points will be accessible as expected in thecompute_homography_matrixfunction.With these changes, the functions should work correctly for your input arrays.
- User
there is an error in warp_perspective(img, H, target_shape) 131 132 for i in range(h): --> 133 for j in range(w): 134 destination_point = np.array([j, i, 1]) 135 source_point = np.dot(np.linalg.inv(H), destination_point)
TypeError: 'numpy.float64' object cannot be interpreted as an integer
- Assistant
It seems that the target_shape values (h, w) are of type float. To fix this, you can convert them to integers. Update
warp_perspectivefunction as follows:def warp_perspective(img, H, target_shape): h, w = target_shape h, w = int(h), int(w) # Convert h and w to integers before iterating warped_image = np.zeros((h, w), dtype=np.uint8) for i in range(h): for j in range(w): destination_point = np.array([j, i, 1]) source_point = np.dot(np.linalg.inv(H), destination_point) source_x, source_y, w = source_point source_x = int(source_x / w) source_y = int(source_y / w) if source_y < img.shape[0] and source_x < img.shape[1]: warped_image[i][j] = img[source_y][source_x] return warped_imageAdding
h, w = int(h), int(w)afterh, w = target_shapewill convert the height and width to integers, which should resolve the TypeError. - User
okay well there is a thing with the image sizes "Size of image 0: (1068, 973) Size of image 1: (986, 653) Size of image 2: (986, 653) Size of image 3: (986, 653) Size of image 4: (1232, 816) Size of image 5: (986, 653)" I am sharing this because it might be relevant to the error that just came up in merge_images(sub_images, keypoints, filtered_matches, homographies, skipped_indices) 173 174 mask = (result == 0) --> 175 result[mask] = img_warped[mask] 176 177 ref_img = result
IndexError: boolean index did not match indexed array along dimension 0; dimension is 991 but corresponding boolean dimension is 1068
- Assistant
This error occurs because the dimensions of
resultandimg_warpeddon't match along dimension 0 when using the boolean mask. To fix this, we can create a mask specifically forimg_warpedand use it to modify only the overlapping regions.Here's the modified
merge_imagesfunction:def merge_images(sub_images, keypoints, filtered_matches, homographies, skipped_indices): ref_img = sub_images[0] for i in range(1, len(sub_images)): if i in skipped_indices: # Check if the image index is in the skipped_indices list print(f"Image {i} was skipped due to homography computation failure.") continue img_i = sub_images[i] H_i = homographies[i - 1 - len([idx for idx in skipped_indices if idx < i])] # Adjust the index based on skipped images h1, w1 = ref_img.shape[:2] h2, w2 = img_i.shape[:2] pts1 = np.float32([[0, 0], [0, h1], [w1, h1], [w1, 0]]).reshape(-1, 1, 2) pts2 = np.float32([[0, 0], [0, h2], [w2, h2], [w2, 0]]).reshape(-1, 1, 2) pts2_trans = cv2.perspectiveTransform(pts2, H_i) new_pts = np.concatenate((pts1, pts2_trans), axis=0) min_x, min_y = np.int32(new_pts.min(axis=0).ravel()) max_x, max_y = np.int32(new_pts.max(axis=0).ravel()) trans_dst = [-min_x, -min_y] H_trans = np.array([[1.0, 0, trans_dst[0]], [0, 1.0, trans_dst[1]], [0, 0, 1.0]]) result = warp_perspective(ref_img, H_trans, (max_y - min_y, max_x - min_x)) img_warped = warp_perspective(img_i, H_i, (result.shape[1], result.shape[0])) mask_result = (result > 0) mask_img_warped = (img_warped > 0) result[mask_img_warped] = img_warped[mask_img_warped] result[mask_result] = ref_img[mask_result] ref_img = result return ref_imgI modified these lines in the function:
- Replace
mask = (result == 0)withmask_result = (result > 0)andmask_img_warped = (img_warped > 0). - Combine the masks when setting the pixel values:
result[mask_img_warped] = img_warped[mask_img_warped]andresult[mask_result] = ref_img[mask_result].
With these changes, the function should work correctly for images with different dimensions.
- Replace
- User
got the same error in merge_images(sub_images, keypoints, filtered_matches, homographies, skipped_indices) 174 mask_result = (result > 0) 175 mask_img_warped = (img_warped > 0) --> 176 result[mask_img_warped] = img_warped[mask_img_warped] 177 result[mask_result] = ref_img[mask_result] 178
IndexError: boolean index did not match indexed array along dimension 0; dimension is 992 but corresponding boolean dimension is 1068
- Assistant
Apologies for the confusion, it looks like the masks are still not matching the corresponding dimensions. Let's update the
merge_imagesfunction again by creating a combined mask from bothresultandimg_warped:def merge_images(sub_images, keypoints, filtered_matches, homographies, skipped_indices): ref_img = sub_images[0] for i in range(1, len(sub_images)): if i in skipped_indices: # Check if the image index is in the skipped_indices list print(f"Image {i} was skipped due to homography computation failure.") continue img_i = sub_images[i] H_i = homographies[i - 1 - len([idx for idx in skipped_indices if idx < i])] # Adjust the index based on skipped images h1, w1 = ref_img.shape[:2] h2, w2 = img_i.shape[:2] pts1 = np.float32([[0, 0], [0, h1], [w1, h1], [w1, 0]]).reshape(-1, 1, 2) pts2 = np.float32([[0, 0], [0, h2], [w2, h2], [w2, 0]]).reshape(-1, 1, 2) pts2_trans = cv2.perspectiveTransform(pts2, H_i) new_pts = np.concatenate((pts1, pts2_trans), axis=0) min_x, min_y = np.int32(new_pts.min(axis=0).ravel()) max_x, max_y = np.int32(new_pts.max(axis=0).ravel()) trans_dst = [-min_x, -min_y] H_trans = np.array([[1.0, 0, trans_dst[0]], [0, 1.0, trans_dst[1]], [0, 0, 1.0]]) result = warp_perspective(ref_img, H_trans, (max_y - min_y, max_x - min_x)) img_warped = warp_perspective(img_i, H_i, (result.shape[1], result.shape[0])) mask_result = (result > 0) mask_img_warped = (img_warped > 0) combined_mask = mask_result & mask_img_warped result[combined_mask] = img_warped[combined_mask] ref_img = result return ref_imgNow, in this updated function:
- Create a
combined_maskby performing a bitwise AND operation onmask_resultandmask_img_warped. - Replace the lines previously using both masks with a single line:
result[combined_mask] = img_warped[combined_mask].
This should fix the dimension issue in the merging process.
- Create a
- User
ah look at this error, I think one of the shapes is reversed: in merge_images(sub_images, keypoints, filtered_matches, homographies, skipped_indices) 174 mask_result = (result > 0) 175 mask_img_warped = (img_warped > 0) --> 176 combined_mask = mask_result & mask_img_warped 177 178 result[combined_mask] = img_warped[combined_mask]
ValueError: operands could not be broadcast together with shapes (993,1068) (1068,993)
- Assistant
My apologies for overlooking this. Let's modify the
warp_perspectivefunction to generate the output image with the correct shape. In the current implementation, the width and height are interchanged when setting the output image. To fix it, update thewarp_perspectivefunction as follows:def warp_perspective(img, H, target_shape): h, w = target_shape h, w = int(h), int(w) # Convert h and w to integers before iterating warped_image = np.zeros((h, w), dtype=np.uint8) for i in range(h): for j in range(w): destination_point = np.array([j, i, 1]) source_point = np.dot(np.linalg.inv(H), destination_point) source_x, source_y, w = source_point source_x = int(source_x / w) source_y = int(source_y / w) if source_y < img.shape[0] and source_x < img.shape[1]: warped_image[i][j] = img[source_y][source_x] return warped_imageNow, update the lines calling the
warp_perspectivefunction in themerge_imagesfunction:From:
result = warp_perspective(ref_img, H_trans, (max_y - min_y, max_x - min_x)) img_warped = warp_perspective(img_i, H_i, (result.shape[1], result.shape[0]))To:
result = warp_perspective(ref_img, H_trans, (max_x - min_x, max_y - min_y)) img_warped = warp_perspective(img_i, H_i, (result.shape[0], result.shape[1]))With these changes, the shapes should now be set correctly, and the error should be resolved.
- User
there is error in warp_perspective(img, H, target_shape) 141 142 if source_y < img.shape[0] and source_x < img.shape[1]: --> 143 warped_image[i][j] = img[source_y][source_x] 144 145 return warped_image
IndexError: index -1344 is out of bounds for axis 0 with size 1232
- Assistant
My apologies for the mistake. It seems that the coordinates
source_xandsource_ygo out of bounds due to the inversion of the homography matrix. To fix this, add a condition in thewarp_perspectivefunction to ensure that the source coordinates are within the valid range before setting the value in thewarped_imagearray:def warp_perspective(img, H, target_shape): h, w = target_shape h, w = int(h), int(w) # Convert h and w to integers before iterating warped_image = np.zeros((h, w), dtype=np.uint8) for i in range(h): for j in range(w): destination_point = np.array([j, i, 1]) source_point = np.dot(np.linalg.inv(H), destination_point) source_x, source_y, w = source_point source_x = int(source_x / w) source_y = int(source_y / w) # Add an additional condition here to make sure indices are within the valid range if 0 <= source_x < img.shape[1] and 0 <= source_y < img.shape[0]: warped_image[i][j] = img[source_y][source_x] return warped_imageAdding an additional condition
0 <= source_x < img.shape[1] and 0 <= source_y < img.shape[0]should avoid the IndexError by only updating thewarped_imagearray when the source coordinates are within valid bounds. With this change, the function should work correctly. - User
there is an error in warp_perspective(img, H, target_shape) 141 # Add an additional condition here to make sure indices are within the valid range 142 if 0 <= source_x < img.shape[1] and 0 <= source_y < img.shape[0]: --> 143 warped_image[i][j] = img[source_y][source_x] 144 145 return warped_image
IndexError: index 1351 is out of bounds for axis 0 with size 1351
- Assistant
It appears there is still an issue with the indices going out of bounds. I apologize for the confusion. Let's make some adjustments to both the
merge_imagesand thewarp_perspectivefunctions to ensure that the indices are within the valid bounds.First, change the part where you compute the min/max values of
xandyin themerge_imagesfunction to take the image dimensions into account:new_pts = np.concatenate((pts1, pts2_trans), axis=0) min_x, min_y = 0, 0 # Set the minimum values to 0 instead max_x, max_y = max(img_i.shape[1], ref_img.shape[1]), max(img_i.shape[0], ref_img.shape[0]) # Use the max dimensions of img_i and ref_img trans_dst = [-min_x, -min_y] H_trans = np.array([[1.0, 0, trans_dst[0]], [0, 1.0, trans_dst[1]], [0, 0, 1.0]])Then, in the
warp_perspectivefunction, modify the condition for assigning the pixel value:Replace:
if 0 <= source_x < img.shape[1] and 0 <= source_y < img.shape[0]:With:
if 0 <= source_x < img.shape[1] - 1 and 0 <= source_y < img.shape[0] - 1:Now, the functions should not produce an IndexError, and the resulting merged image should be generated without issues.
- User
it has been 10 hours an d the code is still running and there are no outputs yet. Why do you think that is because I am about to cancel this 10 hour run since it seems to be stuck.
- Assistant
The most likely reason for the long runtime is that the image warping and merging process is inefficient when implemented using nested for loops. This becomes especially apparent when working with larger images.
To speed up the process, we will use NumPy's vectorized operations to perform the image warping and merging more efficiently, reducing the runtime significantly.
Here's an updated implementation of the
warp_perspectivefunction using NumPy's vectorized operations:def warp_perspective(img, H, target_shape): h, w = target_shape h, w = int(h), int(w) warped_image = np.zeros((h, w), dtype=img.dtype) y_coords, x_coords = np.indices((h, w)) homogenous_coords = np.vstack([x_coords.flatten(), y_coords.flatten(), np.ones((h * w), dtype=np.float64)]) source_coords = np.dot(np.linalg.inv(H), homogenous_coords).T source_coords /= source_coords[:, 2, None] mask_x = np.logical_and(0 <= source_coords[:, 0], source_coords[:, 0] < img.shape[1] - 1) mask_y = np.logical_and(0 <= source_coords[:, 1], source_coords[:, 1] < img.shape[0] - 1) valid_mask = np.logical_and(mask_x, mask_y) int_src_x = np.clip(np.floor(source_coords[:, 0]), 0, img.shape[1] - 2).astype(np.int64) int_src_y = np.clip(np.floor(source_coords[:, 1]), 0, img.shape[0] - 2).astype(np.int64) x_alpha = source_coords[:, 0] - int_src_x y_alpha = source_coords[:, 1] - int_src_y warped_image[y_coords[valid_mask], x_coords[valid_mask]] = ( img[int_src_y[valid_mask], int_src_x[valid_mask]] * (1 - x_alpha[valid_mask]) * (1 - y_alpha[valid_mask]) + img[int_src_y[valid_mask], int_src_x[valid_mask] + 1] * x_alpha[valid_mask] * (1 - y_alpha[valid_mask]) + img[int_src_y[valid_mask] + 1, int_src_x[valid_mask]] * (1 - x_alpha[valid_mask]) * y_alpha[valid_mask] + img[int_src_y[valid_mask] + 1, int_src_x[valid_mask] + 1] * x_alpha[valid_mask]