import os import glob import cv2 import scipy.misc as misc from skimage.transform import…
- User
import os import glob import cv2 import scipy.misc as misc from skimage.transform import resize import numpy as np from functools import reduce from operator import mul import torch from torch import nn import matplotlib.pyplot as plt import re try: import cynetworkx as netx except ImportError: import networkx as netx from scipy.ndimage import gaussian_filter from skimage.feature import canny import collections import shutil import imageio import copy from matplotlib import pyplot as plt from mpl_toolkits.mplot3d import Axes3D import time from scipy.interpolate import interp1d from collections import namedtuple
def path_planning(num_frames, x, y, z, path_type=''): if path_type == 'straight-line': corner_points = np.array([[0, 0, 0], [(0 + x) * 0.5, (0 + y) * 0.5, (0 + z) * 0.5], [x, y, z]]) corner_t = np.linspace(0, 1, len(corner_points)) t = np.linspace(0, 1, num_frames) cs = interp1d(corner_t, corner_points, axis=0, kind='quadratic') spline = cs(t) xs, ys, zs = [xx.squeeze() for xx in np.split(spline, 3, 1)] elif path_type == 'double-straight-line': corner_points = np.array([[-x, -y, -z], [0, 0, 0], [x, y, z]]) corner_t = np.linspace(0, 1, len(corner_points)) t = np.linspace(0, 1, num_frames) cs = interp1d(corner_t, corner_points, axis=0, kind='quadratic') spline = cs(t) xs, ys, zs = [xx.squeeze() for xx in np.split(spline, 3, 1)]
elif path_type == 'circle': xs, ys, zs = [], [], [] for frame_id, bs_shift_val in enumerate(np.arange(-2.0, 2.0, (4./num_frames))): xs += [np.cos(bs_shift_val * np.pi) * 1 * x] ys += [np.sin(bs_shift_val * np.pi) * 1 * y] zs += [np.cos(bs_shift_val * np.pi/2.) * 1 * z] xs, ys, zs = np.array(xs), np.array(ys), np.array(zs)return xs, ys, zsdef open_small_mask(mask, context, open_iteration, kernel): np_mask = mask.cpu().data.numpy().squeeze().astype(np.uint8) raw_mask = np_mask.copy() np_context = context.cpu().data.numpy().squeeze().astype(np.uint8) np_input = np_mask + np_context for _ in range(open_iteration): np_input = cv2.erode(cv2.dilate(np_input, np.ones((kernel, kernel)), iterations=1), np.ones((kernel,kernel)), iterations=1) np_mask[(np_input - np_context) > 0] = 1 out_mask = torch.FloatTensor(np_mask).to(mask)[None, None, ...]
return out_maskdef filter_irrelevant_edge_new(self_edge, comp_edge, other_edges, other_edges_with_id, current_edge_id, context, depth, mesh, context_cc, spdb=False): other_edges = other_edges.squeeze().astype(np.uint8) other_edges_with_id = other_edges_with_id.squeeze() self_edge = self_edge.squeeze() dilate_bevel_self_edge = cv2.dilate((self_edge + comp_edge).astype(np.uint8), np.array([[1,1,1],[1,1,1],[1,1,1]]), iterations=1) dilate_cross_self_edge = cv2.dilate((self_edge + comp_edge).astype(np.uint8), np.array([[0,1,0],[1,1,1],[0,1,0]]).astype(np.uint8), iterations=1) edge_ids = np.unique(other_edges_with_id * context + (-1) * (1 - context)).astype(np.int) end_depth_maps = np.zeros_like(self_edge) self_edge_ids = np.sort(np.unique(other_edges_with_id[self_edge > 0]).astype(np.int)) self_edge_ids = self_edge_ids[1:] if self_edge_ids.shape[0] > 0 and self_edge_ids[0] == -1 else self_edge_ids self_comp_ids = np.sort(np.unique(other_edges_with_id[comp_edge > 0]).astype(np.int)) self_comp_ids = self_comp_ids[1:] if self_comp_ids.shape[0] > 0 and self_comp_ids[0] == -1 else self_comp_ids edge_ids = edge_ids[1:] if edge_ids[0] == -1 else edge_ids other_edges_info = [] extend_other_edges = np.zeros_like(other_edges) if spdb is True: f, ((ax1, ax2, ax3)) = plt.subplots(1, 3, sharex=True, sharey=True); ax1.imshow(self_edge); ax2.imshow(context); ax3.imshow(other_edges_with_id * context + (-1) * (1 - context)); plt.show() import pdb; pdb.set_trace() filter_self_edge = np.zeros_like(self_edge) for self_edge_id in self_edge_ids: filter_self_edge[other_edges_with_id == self_edge_id] = 1 dilate_self_comp_edge = cv2.dilate(comp_edge, kernel=np.ones((3, 3)), iterations=2) valid_self_comp_edge = np.zeros_like(comp_edge) for self_comp_id in self_comp_ids: valid_self_comp_edge[self_comp_id == other_edges_with_id] = 1 self_comp_edge = dilate_self_comp_edge * valid_self_comp_edge filter_self_edge = (filter_self_edge + self_comp_edge).clip(0, 1) for edge_id in edge_ids: other_edge_locs = (other_edges_with_id == edge_id).astype(np.uint8) condition = (other_edge_locs * other_edges * context.astype(np.uint8)) end_cross_point = dilate_cross_self_edge * condition * (1 - filter_self_edge) end_bevel_point = dilate_bevel_self_edge * condition * (1 - filter_self_edge) if end_bevel_point.max() != 0: end_depth_maps[end_bevel_point != 0] = depth[end_bevel_point != 0] if end_cross_point.max() == 0: nxs, nys = np.where(end_bevel_point != 0) for nx, ny in zip(nxs, nys): bevel_node = [xx for xx in context_cc if xx[0] == nx and xx[1] == ny][0] for ne in mesh.neighbors(bevel_node): if other_edges_with_id[ne[0], ne[1]] > -1 and dilate_cross_self_edge[ne[0], ne[1]] > 0: extend_other_edges[ne[0], ne[1]] = 1 break else: other_edges[other_edges_with_id == edge_id] = 0 other_edges = (other_edges + extend_other_edges).clip(0, 1) * context
return other_edges, end_depth_maps, other_edges_infodef clean_far_edge_new(input_edge, end_depth_maps, mask, context, global_mesh, info_on_pix, self_edge, inpaint_id, config): mesh = netx.Graph() hxs, hys = np.where(input_edge * mask > 0) valid_near_edge = (input_edge != 0).astype(np.uint8) * context valid_map = mask + context invalid_edge_ids = [] for hx, hy in zip(hxs, hys): node = (hx ,hy) mesh.add_node((hx, hy)) eight_nes = [ne for ne in [(hx + 1, hy), (hx - 1, hy), (hx, hy + 1), (hx, hy - 1),
(hx + 1, hy + 1), (hx - 1, hy - 1), (hx - 1, hy + 1), (hx + 1, hy - 1)]
if 0 <= ne[0] < input_edge.shape[0] and 0 <= ne[1] < input_edge.shape[1] and 0 < input_edge[ne[0], ne[1]]] # or end_depth_maps[ne[0], ne[1]] != 0] for ne in eight_nes: mesh.add_edge(node, ne, length=np.hypot(ne[0] - hx, ne[1] - hy)) if end_depth_maps[ne[0], ne[1]] != 0: mesh.nodes[ne[0], ne[1]]['cnt'] = True if end_depth_maps[ne[0], ne[1]] == 0: import pdb; pdb.set_trace() mesh.nodes[ne[0], ne[1]]['depth'] = end_depth_maps[ne[0], ne[1]] elif mask[ne[0], ne[1]] != 1: four_nes = [nne for nne in [(ne[0] + 1, ne[1]), (ne[0] - 1, ne[1]), (ne[0], ne[1] + 1), (ne[0], ne[1] - 1)]
if nne[0] < end_depth_maps.shape[0] and nne[0] >= 0 and nne[1] < end_depth_maps.shape[1] and nne[1] >= 0] for nne in four_nes: if end_depth_maps[nne[0], nne[1]] != 0: mesh.add_edge(nne, ne, length=np.hypot(nne[0] - ne[0], nne[1] - ne[1])) mesh.nodes[nne[0], nne[1]]['cnt'] = True mesh.nodes[nne[0], nne[1]]['depth'] = end_depth_maps[nne[0], nne[1]] ccs = [*netx.connected_components(mesh)] end_pts = [] for cc in ccs: end_pts.append(set()) for node in cc: if mesh.nodes[node].get('cnt') is not None: end_pts[-1].add((node[0], node[1], mesh.nodes[node]['depth']))
predef_npaths = [None for _ in range(len(ccs))] fpath_map = np.zeros_like(input_edge) - 1 npath_map = np.zeros_like(input_edge) - 1 npaths, fpaths = dict(), dict() break_flag = False end_idx = 0 while end_idx < len(end_pts): end_pt, cc = [*zip(end_pts, ccs)][end_idx] end_idx += 1 sorted_end_pt = [] fpath = [] iter_fpath = [] if len(end_pt) > 2 or len(end_pt) == 0: if len(end_pt) > 2: continue continue if len(end_pt) == 2: ravel_end = [*end_pt] tmp_sub_mesh = mesh.subgraph(list(cc)).copy() tmp_npath = [*netx.shortest_path(tmp_sub_mesh, (ravel_end[0][0], ravel_end[0][1]), (ravel_end[1][0], ravel_end[1][1]), weight='length')] fpath_map1, npath_map1, disp_diff1 = plan_path(mesh, info_on_pix, cc, ravel_end[0:1], global_mesh, input_edge, mask, valid_map, inpaint_id, npath_map=None, fpath_map=None, npath=tmp_npath) fpath_map2, npath_map2, disp_diff2 = plan_path(mesh, info_on_pix, cc, ravel_end[1:2], global_mesh, input_edge, mask, valid_map, inpaint_id, npath_map=None, fpath_map=None, npath=tmp_npath) tmp_disp_diff = [disp_diff1, disp_diff2] self_end = [] edge_len = [] ds_edge = cv2.dilate(self_edge.astype(np.uint8), np.ones((3, 3)), iterations=1) if ds_edge[ravel_end[0][0], ravel_end[0][1]] > 0: self_end.append(1) else: self_end.append(0) if ds_edge[ravel_end[1][0], ravel_end[1][1]] > 0: self_end.append(1) else: self_end.append(0) edge_len = [np.count_nonzero(npath_map1), np.count_nonzero(npath_map2)] sorted_end_pts = [xx[0] for xx in sorted(zip(ravel_end, self_end, edge_len, [disp_diff1, disp_diff2]), key=lambda x: (x[1], x[2]), reverse=True)] re_npath_map1, re_fpath_map1 = (npath_map1 != -1).astype(np.uint8), (fpath_map1 != -1).astype(np.uint8) re_npath_map2, re_fpath_map2 = (npath_map2 != -1).astype(np.uint8), (fpath_map2 != -1).astype(np.uint8) if np.count_nonzero(re_npath_map1 * re_npath_map2 * mask) /
(np.count_nonzero((re_npath_map1 + re_npath_map2) * mask) + 1e-6) > 0.5
and np.count_nonzero(re_fpath_map1 * re_fpath_map2 * mask) /
(np.count_nonzero((re_fpath_map1 + re_fpath_map2) * mask) + 1e-6) > 0.5
and tmp_disp_diff[0] != -1 and tmp_disp_diff[1] != -1: my_fpath_map, my_npath_map, npath, fpath =
plan_path_e2e(mesh, cc, sorted_end_pts, global_mesh, input_edge, mask, valid_map, inpaint_id, npath_map=None, fpath_map=None) npath_map[my_npath_map != -1] = my_npath_map[my_npath_map != -1] fpath_map[my_fpath_map != -1] = my_fpath_map[my_fpath_map != -1] if len(fpath) > 0: edge_id = global_mesh.nodes[[*sorted_end_pts][0]]['edge_id'] fpaths[edge_id] = fpath npaths[edge_id] = npath invalid_edge_ids.append(edge_id) else: if tmp_disp_diff[0] != -1: ratio_a = tmp_disp_diff[0] / (np.sum(tmp_disp_diff) + 1e-8) else: ratio_a = 0 if tmp_disp_diff[1] != -1: ratio_b = tmp_disp_diff[1] / (np.sum(tmp_disp_diff) + 1e-8) else: ratio_b = 0 npath_len = len(tmp_npath) if npath_len > config['depth_edge_dilate_2'] * 2: npath_len = npath_len - (config['depth_edge_dilate_2'] * 1) tmp_npath_a = tmp_npath[:int(np.floor(npath_len * ratio_a))] tmp_npath_b = tmp_npath[::-1][:int(np.floor(npath_len * ratio_b))] tmp_merge = [] if len(tmp_npath_a) > 0 and sorted_end_pts[0][0] == tmp_npath_a[0][0] and sorted_end_pts[0][1] == tmp_npath_a[0][1]: if len(tmp_npath_a) > 0 and mask[tmp_npath_a[-1][0], tmp_npath_a[-1][1]] > 0: tmp_merge.append([sorted_end_pts[:1], tmp_npath_a]) if len(tmp_npath_b) > 0 and mask[tmp_npath_b[-1][0], tmp_npath_b[-1][1]] > 0: tmp_merge.append([sorted_end_pts[1:2], tmp_npath_b]) elif len(tmp_npath_b) > 0 and sorted_end_pts[0][0] == tmp_npath_b[0][0] and sorted_end_pts[0][1] == tmp_npath_b[0][1]: if len(tmp_npath_b) > 0 and mask[tmp_npath_b[-1][0], tmp_npath_b[-1][1]] > 0: tmp_merge.append([sorted_end_pts[:1], tmp_npath_b]) if len(tmp_npath_a) > 0 and mask[tmp_npath_a[-1][0], tmp_npath_a[-1][1]] > 0: tmp_merge.append([sorted_end_pts[1:2], tmp_npath_a]) for tmp_idx in range(len(tmp_merge)): if len(tmp_merge[tmp_idx][1]) == 0: continue end_pts.append(tmp_merge[tmp_idx][0]) ccs.append(set(tmp_merge[tmp_idx][1])) if len(end_pt) == 1: sub_mesh = mesh.subgraph(list(cc)).copy() pnodes = netx.periphery(sub_mesh) if len(end_pt) == 1: ends = [*end_pt] elif len(sorted_end_pt) == 1: ends = [*sorted_end_pt] else: import pdb; pdb.set_trace() try: edge_id = global_mesh.nodes[ends[0]]['edge_id'] except: import pdb; pdb.set_trace() pnodes = sorted(pnodes, key=lambda x: np.hypot((x[0] - ends[0][0]), (x[1] - ends[0][1])), reverse=True)[0] npath = [*netx.shortest_path(sub_mesh, (ends[0][0], ends[0][1]), pnodes, weight='length')] for np_node in npath: npath_map[np_node[0], np_node[1]] = edge_id fpath = [] if global_mesh.nodes[ends[0]].get('far') is None: print("None far") else: fnodes = global_mesh.nodes[ends[0]].get('far') dmask = mask + 0 did = 0 while True: did += 1 dmask = cv2.dilate(dmask, np.ones((3, 3)), iterations=1) if did > 3: break ffnode = [fnode for fnode in fnodes if (dmask[fnode[0], fnode[1]] > 0 and mask[fnode[0], fnode[1]] == 0 and
global_mesh.nodes[fnode].get('inpaint_id') != inpaint_id + 1)] if len(ffnode) > 0: fnode = ffnode[0] break if len(ffnode) == 0: continue fpath.append((fnode[0], fnode[1])) barrel_dir = np.array([[1, 0], [1, 1], [0, 1], [-1, 1], [-1, 0], [-1, -1], [0, -1], [1, -1]]) n2f_dir = (int(fnode[0] - npath[0][0]), int(fnode[1] - npath[0][1])) while True: if barrel_dir[0, 0] == n2f_dir[0] and barrel_dir[0, 1] == n2f_dir[1]: n2f_barrel = barrel_dir.copy() break barrel_dir = np.roll(barrel_dir, 1, axis=0) for step in range(0, len(npath)): if step == 0: continue elif step == 1: next_dir = (npath[step][0] - npath[step - 1][0], npath[step][1] - npath[step - 1][1]) while True: if barrel_dir[0, 0] == next_dir[0] and barrel_dir[0, 1] == next_dir[1]: next_barrel = barrel_dir.copy() break barrel_dir = np.roll(barrel_dir, 1, axis=0) barrel_pair = np.stack((n2f_barrel, next_barrel), axis=0) n2f_dir = (barrel_pair[0, 0, 0], barrel_pair[0, 0, 1]) elif step > 1: next_dir = (npath[step][0] - npath[step - 1][0], npath[step][1] - npath[step - 1][1]) while True: if barrel_pair[1, 0, 0] == next_dir[0] and barrel_pair[1, 0, 1] == next_dir[1]: next_barrel = barrel_pair.copy() break barrel_pair = np.roll(barrel_pair, 1, axis=1) n2f_dir = (barrel_pair[0, 0, 0], barrel_pair[0, 0, 1]) new_locs = [] if abs(n2f_dir[0]) == 1: new_locs.append((npath[step][0] + n2f_dir[0], npath[step][1])) if abs(n2f_dir[1]) == 1: new_locs.append((npath[step][0], npath[step][1] + n2f_dir[1])) if len(new_locs) > 1: new_locs = sorted(new_locs, key=lambda xx: np.hypot((xx[0] - fpath[-1][0]), (xx[1] - fpath[-1][1]))) break_flag = False for new_loc in new_locs: new_loc_nes = [xx for xx in [(new_loc[0] + 1, new_loc[1]), (new_loc[0] - 1, new_loc[1]), (new_loc[0], new_loc[1] + 1), (new_loc[0], new_loc[1] - 1)]
if xx[0] >= 0 and xx[0] < fpath_map.shape[0] and xx[1] >= 0 and xx[1] < fpath_map.shape[1]] if np.all([(fpath_map[nlne[0], nlne[1]] == -1) for nlne in new_loc_nes]) != True: break if npath_map[new_loc[0], new_loc[1]] != -1: if npath_map[new_loc[0], new_loc[1]] != edge_id: break_flag = True break else: continue if valid_map[new_loc[0], new_loc[1]] == 0: break_flag = True break fpath.append(new_loc) if break_flag is True: break if step != len(npath) - 1: for xx in npath[step:]: if npath_map[xx[0], xx[1]] == edge_id: npath_map[xx[0], xx[1]] = -1 npath = npath[:step] if len(fpath) > 0: for fp_node in fpath: fpath_map[fp_node[0], fp_node[1]] = edge_id fpaths[edge_id] = fpath npaths[edge_id] = npath fpath_map[valid_near_edge != 0] = -1 if len(fpath) > 0: iter_fpath = copy.deepcopy(fpaths[edge_id]) for node in iter_fpath: if valid_near_edge[node[0], node[1]] != 0: fpaths[edge_id].remove(node)return fpath_map, npath_map, False, npaths, fpaths, invalid_edge_idsdef plan_path_e2e(mesh, cc, end_pts, global_mesh, input_edge, mask, valid_map, inpaint_id, npath_map=None, fpath_map=None): my_npath_map = np.zeros_like(input_edge) - 1 my_fpath_map = np.zeros_like(input_edge) - 1 sub_mesh = mesh.subgraph(list(cc)).copy() ends_1, ends_2 = end_pts[0], end_pts[1] edge_id = global_mesh.nodes[ends_1]['edge_id'] npath = [*netx.shortest_path(sub_mesh, (ends_1[0], ends_1[1]), (ends_2[0], ends_2[1]), weight='length')] for np_node in npath: my_npath_map[np_node[0], np_node[1]] = edge_id fpath = [] if global_mesh.nodes[ends_1].get('far') is None: print("None far") else: fnodes = global_mesh.nodes[ends_1].get('far') dmask = mask + 0 while True: dmask = cv2.dilate(dmask, np.ones((3, 3)), iterations=1) ffnode = [fnode for fnode in fnodes if (dmask[fnode[0], fnode[1]] > 0 and mask[fnode[0], fnode[1]] == 0 and
global_mesh.nodes[fnode].get('inpaint_id') != inpaint_id + 1)] if len(ffnode) > 0: fnode = ffnode[0] break e_fnodes = global_mesh.nodes[ends_2].get('far') dmask = mask + 0 while True: dmask = cv2.dilate(dmask, np.ones((3, 3)), iterations=1) e_ffnode = [e_fnode for e_fnode in e_fnodes if (dmask[e_fnode[0], e_fnode[1]] > 0 and mask[e_fnode[0], e_fnode[1]] == 0 and
global_mesh.nodes[e_fnode].get('inpaint_id') != inpaint_id + 1)] if len(e_ffnode) > 0: e_fnode = e_ffnode[0] break
fpath.append((fnode[0], fnode[1])) if len(e_ffnode) == 0 or len(ffnode) == 0: return my_npath_map, my_fpath_map, [], [] barrel_dir = np.array([[1, 0], [1, 1], [0, 1], [-1, 1], [-1, 0], [-1, -1], [0, -1], [1, -1]]) n2f_dir = (int(fnode[0] - npath[0][0]), int(fnode[1] - npath[0][1])) while True: if barrel_dir[0, 0] == n2f_dir[0] and barrel_dir[0, 1] == n2f_dir[1]: n2f_barrel = barrel_dir.copy() break barrel_dir = np.roll(barrel_dir, 1, axis=0) for step in range(0, len(npath)): if step == 0: continue elif step == 1: next_dir = (npath[step][0] - npath[step - 1][0], npath[step][1] - npath[step - 1][1]) while True: if barrel_dir[0, 0] == next_dir[0] and barrel_dir[0, 1] == next_dir[1]: next_barrel = barrel_dir.copy() break barrel_dir = np.roll(barrel_dir, 1, axis=0) barrel_pair = np.stack((n2f_barrel, next_barrel), axis=0) n2f_dir = (barrel_pair[0, 0, 0], barrel_pair[0, 0, 1]) elif step > 1: next_dir = (npath[step][0] - npath[step - 1][0], npath[step][1] - npath[step - 1][1]) while True: if barrel_pair[1, 0, 0] == next_dir[0] and barrel_pair[1, 0, 1] == next_dir[1]: next_barrel = barrel_pair.copy() break barrel_pair = np.roll(barrel_pair, 1, axis=1) n2f_dir = (barrel_pair[0, 0, 0], barrel_pair[0, 0, 1]) new_locs = [] if abs(n2f_dir[0]) == 1: new_locs.append((npath[step][0] + n2f_dir[0], npath[step][1])) if abs(n2f_dir[1]) == 1: new_locs.append((npath[step][0], npath[step][1] + n2f_dir[1])) if len(new_locs) > 1: new_locs = sorted(new_locs, key=lambda xx: np.hypot((xx[0] - fpath[-1][0]), (xx[1] - fpath[-1][1]))) break_flag = False for new_loc in new_locs: new_loc_nes = [xx for xx in [(new_loc[0] + 1, new_loc[1]), (new_loc[0] - 1, new_loc[1]), (new_loc[0], new_loc[1] + 1), (new_loc[0], new_loc[1] - 1)]
if xx[0] >= 0 and xx[0] < my_fpath_map.shape[0] and xx[1] >= 0 and xx[1] < my_fpath_map.shape[1]] if fpath_map is not None and np.sum([fpath_map[nlne[0], nlne[1]] for nlne in new_loc_nes]) != 0: break_flag = True break if my_npath_map[new_loc[0], new_loc[1]] != -1: continue if npath_map is not None and npath_map[new_loc[0], new_loc[1]] != edge_id: break_flag = True break fpath.append(new_loc) if break_flag is True: break if (e_fnode[0], e_fnode[1]) not in fpath: fpath.append((e_fnode[0], e_fnode[1])) if step != len(npath) - 1: for xx in npath[step:]: if my_npath_map[xx[0], xx[1]] == edge_id: my_npath_map[xx[0], xx[1]] = -1 npath = npath[:step] if len(fpath) > 0: for fp_node in fpath: my_fpath_map[fp_node[0], fp_node[1]] = edge_idreturn my_fpath_map, my_npath_map, npath, fpathdef plan_path(mesh, info_on_pix, cc, end_pt, global_mesh, input_edge, mask, valid_map, inpaint_id, npath_map=None, fpath_map=None, npath=None): my_npath_map = np.zeros_like(input_edge) - 1 my_fpath_map = np.zeros_like(input_edge) - 1 sub_mesh = mesh.subgraph(list(cc)).copy() pnodes = netx.periphery(sub_mesh) ends = [*end_pt] edge_id = global_mesh.nodes[ends[0]]['edge_id'] pnodes = sorted(pnodes, key=lambda x: np.hypot((x[0] - ends[0][0]), (x[1] - ends[0][1])), reverse=True)[0] if npath is None: npath = [*netx.shortest_path(sub_mesh, (ends[0][0], ends[0][1]), pnodes, weight='length')] else: if (ends[0][0], ends[0][1]) == npath[0]: npath = npath elif (ends[0][0], ends[0][1]) == npath[-1]: npath = npath[::-1] else: import pdb; pdb.set_trace() for np_node in npath: my_npath_map[np_node[0], np_node[1]] = edge_id fpath = [] if global_mesh.nodes[ends[0]].get('far') is None: print("None far") else: fnodes = global_mesh.nodes[ends[0]].get('far') dmask = mask + 0 did = 0 while True: did += 1 if did > 3: return my_fpath_map, my_npath_map, -1 dmask = cv2.dilate(dmask, np.ones((3, 3)), iterations=1) ffnode = [fnode for fnode in fnodes if (dmask[fnode[0], fnode[1]] > 0 and mask[fnode[0], fnode[1]] == 0 and
global_mesh.nodes[fnode].get('inpaint_id') != inpaint_id + 1)] if len(ffnode) > 0: fnode = ffnode[0] breakfpath.append((fnode[0], fnode[1])) disp_diff = 0. for n_loc in npath: if mask[n_loc[0], n_loc[1]] != 0: disp_diff = abs(abs(1. / info_on_pix[(n_loc[0], n_loc[1])][0]['depth']) - abs(1. / ends[0][2])) break barrel_dir = np.array([[1, 0], [1, 1], [0, 1], [-1, 1], [-1, 0], [-1, -1], [0, -1], [1, -1]]) n2f_dir = (int(fnode[0] - npath[0][0]), int(fnode[1] - npath[0][1])) while True: if barrel_dir[0, 0] == n2f_dir[0] and barrel_dir[0, 1] == n2f_dir[1]: n2f_barrel = barrel_dir.copy() break barrel_dir = np.roll(barrel_dir, 1, axis=0) for step in range(0, len(npath)): if step == 0: continue elif step == 1: next_dir = (npath[step][0] - npath[step - 1][0], npath[step][1] - npath[step - 1][1]) while True: if barrel_dir[0, 0] == next_dir[0] and barrel_dir[0, 1] == next_dir[1]: next_barrel = barrel_dir.copy() break barrel_dir = np.roll(barrel_dir, 1, axis=0) barrel_pair = np.stack((n2f_barrel, next_barrel), axis=0) n2f_dir = (barrel_pair[0, 0, 0], barrel_pair[0, 0, 1]) elif step > 1: next_dir = (npath[step][0] - npath[step - 1][0], npath[step][1] - npath[step - 1][1]) while True: if barrel_pair[1, 0, 0] == next_dir[0] and barrel_pair[1, 0, 1] == next_dir[1]: next_barrel = barrel_pair.copy() break barrel_pair = np.roll(barrel_pair, 1, axis=1) n2f_dir = (barrel_pair[0, 0, 0], barrel_pair[0, 0, 1]) new_locs = [] if abs(n2f_dir[0]) == 1: new_locs.append((npath[step][0] + n2f_dir[0], npath[step][1])) if abs(n2f_dir[1]) == 1: new_locs.append((npath[step][0], npath[step][1] + n2f_dir[1])) if len(new_locs) > 1: new_locs = sorted(new_locs, key=lambda xx: np.hypot((xx[0] - fpath[-1][0]), (xx[1] - fpath[-1][1]))) break_flag = False for new_loc in new_locs: new_loc_nes = [xx for xx in [(new_loc[0] + 1, new_loc[1]), (new_loc[0] - 1, new_loc[1]), (new_loc[0], new_loc[1] + 1), (new_loc[0], new_loc[1] - 1)]\ if xx[0] >= 0 and xx[0] < my_fpath_map.shape[0] and xx[1] >= 0 and xx[1] < my_fpath_map.shape[1]] if fpath_map is not None and np.all([(fpath_map[nlne[0], nlne[1]] == -1) for nlne in new_loc_nes]) != True: break_flag = True break if np.all([(my_fpath_map[nlne[0], nlne[1]] == -1) for nlne in new_loc_nes]) != True: break_flag = True break if my_npath_map[new_loc[0], new_loc[1]] != -1: continue if npath_map is not None and npath_map[new_loc[0], new_loc[1]] != edge_id: break_flag = True break if valid_map[new_loc[0], new_loc[1]] == 0: break_flag = True break fpath.append(new_loc) if break_flag is True: break if step != len(npath) - 1: for xx in npath[step:]: if my_npath_map[xx[0], xx[1]] == edge_id: my_npath_map[xx[0], xx[1]] = -1 npath = npath[:step] if len(fpath) > 0: for fp_node in fpath: my_fpath_map[fp_node[0], fp_node[1]] = edge_id return my_fpath_map, my_npath_map, disp_diffdef refresh_node(old_node, old_feat, new_node, new_feat, mesh, stime=False): mesh.add_node(new_node) mesh.nodes[new_node].update(new_feat) mesh.nodes[new_node].update(old_feat) for ne in mesh.neighbors(old_node): mesh.add_edge(new_node, ne) if mesh.nodes[new_node].get('far') is not None: tmp_far_nodes = mesh.nodes[new_node]['far'] for far_node in tmp_far_nodes: if mesh.has_node(far_node) is False: mesh.nodes[new_node]['far'].remove(far_node) continue if mesh.nodes[far_node].get('near') is not None: for idx in range(len(mesh.nodes[far_node].get('near'))): if mesh.nodes[far_node]['near'][idx][0] == new_node[0] and mesh.nodes[far_node]['near'][idx][1] == new_node[1]: if len(mesh.nodes[far_node]['near'][idx]) == len(old_node): mesh.nodes[far_node]['near'][idx] = new_node if mesh.nodes[new_node].get('near') is not None: tmp_near_nodes = mesh.nodes[new_node]['near'] for near_node in tmp_near_nodes: if mesh.has_node(near_node) is False: mesh.nodes[new_node]['near'].remove(near_node) continue
if mesh.nodes[near_node].get('far') is not None: for idx in range(len(mesh.nodes[near_node].get('far'))): if mesh.nodes[near_node]['far'][idx][0] == new_node[0] and mesh.nodes[near_node]['far'][idx][1] == new_node[1]: if len(mesh.nodes[near_node]['far'][idx]) == len(old_node): mesh.nodes[near_node]['far'][idx] = new_node if new_node != old_node: mesh.remove_node(old_node) if stime is False: return mesh else: return mesh, None, Nonedef create_placeholder(context, mask, depth, fpath_map, npath_map, mesh, inpaint_id, edge_ccs, extend_edge_cc, all_edge_maps, self_edge_id): add_node_time = 0 add_edge_time = 0 add_far_near_time = 0 valid_area = context + mask H, W = mesh.graph['H'], mesh.graph['W'] edge_cc = edge_ccs[self_edge_id] num_com = len(edge_cc) + len(extend_edge_cc) hxs, hys = np.where(mask > 0) for hx, hy in zip(hxs, hys): mesh.add_node((hx, hy), inpaint_id=inpaint_id + 1, num_context=num_com) for hx, hy in zip(hxs, hys): four_nes = [(x, y) for x, y in [(hx + 1, hy), (hx - 1, hy), (hx, hy + 1), (hx, hy - 1)] if
0 <= x < mesh.graph['H'] and 0 <= y < mesh.graph['W'] and valid_area[x, y] != 0] for ne in four_nes: if mask[ne[0], ne[1]] != 0: if not mesh.has_edge((hx, hy), ne): mesh.add_edge((hx, hy), ne) elif depth[ne[0], ne[1]] != 0: if mesh.has_node((ne[0], ne[1], depth[ne[0], ne[1]])) and
not mesh.has_edge((hx, hy), (ne[0], ne[1], depth[ne[0], ne[1]])): mesh.add_edge((hx, hy), (ne[0], ne[1], depth[ne[0], ne[1]])) else: print("Undefined context node.") import pdb; pdb.set_trace() near_ids = np.unique(npath_map) if near_ids[0] == -1: near_ids = near_ids[1:] for near_id in near_ids: hxs, hys = np.where((fpath_map == near_id) & (mask > 0)) if hxs.shape[0] > 0: mesh.graph['max_edge_id'] = mesh.graph['max_edge_id'] + 1 else: break for hx, hy in zip(hxs, hys): mesh.nodes[(hx, hy)]['edge_id'] = int(round(mesh.graph['max_edge_id'])) four_nes = [(x, y) for x, y in [(hx + 1, hy), (hx - 1, hy), (hx, hy + 1), (hx, hy - 1)] if
x < mesh.graph['H'] and x >= 0 and y < mesh.graph['W'] and y >= 0 and npath_map[x, y] == near_id] for xx in four_nes: xx_n = copy.deepcopy(xx) if not mesh.has_node(xx_n): if mesh.has_node((xx_n[0], xx_n[1], depth[xx_n[0], xx_n[1]])): xx_n = (xx_n[0], xx_n[1], depth[xx_n[0], xx_n[1]]) if mesh.has_edge((hx, hy), xx_n): # pass mesh.remove_edge((hx, hy), xx_n) if mesh.nodes[(hx, hy)].get('near') is None: mesh.nodes[(hx, hy)]['near'] = [] mesh.nodes[(hx, hy)]['near'].append(xx_n) connect_point_exception = set() hxs, hys = np.where((npath_map == near_id) & (all_edge_maps > -1)) for hx, hy in zip(hxs, hys): unknown_id = int(round(all_edge_maps[hx, hy])) if unknown_id != near_id and unknown_id != self_edge_id: unknown_node = set([xx for xx in edge_ccs[unknown_id] if xx[0] == hx and xx[1] == hy]) connect_point_exception |= unknown_node hxs, hys = np.where((npath_map == near_id) & (mask > 0))
if hxs.shape[0] > 0: mesh.graph['max_edge_id'] = mesh.graph['max_edge_id'] + 1 else: break for hx, hy in zip(hxs, hys): mesh.nodes[(hx, hy)]['edge_id'] = int(round(mesh.graph['max_edge_id'])) mesh.nodes[(hx, hy)]['connect_point_id'] = int(round(near_id)) mesh.nodes[(hx, hy)]['connect_point_exception'] = connect_point_exception four_nes = [(x, y) for x, y in [(hx + 1, hy), (hx - 1, hy), (hx, hy + 1), (hx, hy - 1)] if
x < mesh.graph['H'] and x >= 0 and y < mesh.graph['W'] and y >= 0 and fpath_map[x, y] == near_id] for xx in four_nes: xx_n = copy.deepcopy(xx) if not mesh.has_node(xx_n): if mesh.has_node((xx_n[0], xx_n[1], depth[xx_n[0], xx_n[1]])): xx_n = (xx_n[0], xx_n[1], depth[xx_n[0], xx_n[1]]) if mesh.has_edge((hx, hy), xx_n): mesh.remove_edge((hx, hy), xx_n) if mesh.nodes[(hx, hy)].get('far') is None: mesh.nodes[(hx, hy)]['far'] = [] mesh.nodes[(hx, hy)]['far'].append(xx_n)return mesh, add_node_time, add_edge_time, add_far_near_timedef clean_far_edge(mask_edge, mask_edge_with_id, context_edge, mask, info_on_pix, global_mesh, anchor): if isinstance(mask_edge, torch.Tensor): if mask_edge.is_cuda: mask_edge = mask_edge.cpu() mask_edge = mask_edge.data mask_edge = mask_edge.numpy() if isinstance(context_edge, torch.Tensor): if context_edge.is_cuda: context_edge = context_edge.cpu() context_edge = context_edge.data context_edge = context_edge.numpy() if isinstance(mask, torch.Tensor): if mask.is_cuda: mask = mask.cpu() mask = mask.data mask = mask.numpy() mask = mask.squeeze() mask_edge = mask_edge.squeeze() context_edge = context_edge.squeeze() valid_near_edge = np.zeros_like(mask_edge) far_edge = np.zeros_like(mask_edge) far_edge_with_id = np.ones_like(mask_edge) * -1 near_edge_with_id = np.ones_like(mask_edge) * -1 uncleaned_far_edge = np.zeros_like(mask_edge) # Detect if there is any valid pixel mask_edge, if not ==> return default value if mask_edge.sum() == 0: return far_edge, uncleaned_far_edge, far_edge_with_id, near_edge_with_id mask_edge_ids = dict(collections.Counter(mask_edge_with_id.flatten())).keys() for edge_id in mask_edge_ids: if edge_id < 0: continue specific_edge_map = (mask_edge_with_id == edge_id).astype(np.uint8) _, sub_specific_edge_maps = cv2.connectedComponents(specific_edge_map.astype(np.uint8), connectivity=8) for sub_edge_id in range(1, sub_specific_edge_maps.max() + 1): specific_edge_map = (sub_specific_edge_maps == sub_edge_id).astype(np.uint8) edge_pxs, edge_pys = np.where(specific_edge_map > 0) edge_mesh = netx.Graph() for edge_px, edge_py in zip(edge_pxs, edge_pys): edge_mesh.add_node((edge_px, edge_py)) for ex in [edge_px-1, edge_px, edge_px+1]: for ey in [edge_py-1, edge_py, edge_py+1]: if edge_px == ex and edge_py == ey: continue if ex < 0 or ex >= specific_edge_map.shape[0] or ey < 0 or ey >= specific_edge_map.shape[1]: continue if specific_edge_map[ex, ey] == 1: if edge_mesh.has_node((ex, ey)): edge_mesh.add_edge((ex, ey), (edge_px, edge_py)) periphery_nodes = netx.periphery(edge_mesh) path_diameter = netx.diameter(edge_mesh) start_near_node = None for node_s in periphery_nodes: for node_e in periphery_nodes: if node_s != node_e: if netx.shortest_path_length(edge_mesh, node_s, node_e) == path_diameter: if np.any(context_edge[node_s[0]-1:node_s[0]+2, node_s[1]-1:node_s[1]+2].flatten()): start_near_node = (node_s[0], node_s[1]) end_near_node = (node_e[0], node_e[1]) break if np.any(context_edge[node_e[0]-1:node_e[0]+2, node_e[1]-1:node_e[1]+2].flatten()): start_near_node = (node_e[0], node_e[1]) end_near_node = (node_s[0], node_s[1]) break if start_near_node is not None: break if start_near_node is None: continue new_specific_edge_map = np.zeros_like(mask) for path_node in netx.shortest_path(edge_mesh, start_near_node, end_near_node): new_specific_edge_map[path_node[0], path_node[1]] = 1 context_near_pxs, context_near_pys = np.where(context_edge[start_near_node[0]-1:start_near_node[0]+2, start_near_node[1]-1:start_near_node[1]+2] > 0) distance = np.abs((context_near_pxs - 1)) + np.abs((context_near_pys - 1)) if (np.where(distance == distance.min())[0].shape[0]) > 1: closest_pxs = context_near_pxs[np.where(distance == distance.min())[0]] closest_pys = context_near_pys[np.where(distance == distance.min())[0]] closest_depths = [] for closest_px, closest_py in zip(closest_pxs, closest_pys): if info_on_pix.get((closest_px + start_near_node[0] - 1 + anchor[0], closest_py + start_near_node[1] - 1 + anchor[2])) is not None: for info in info_on_pix.get((closest_px + start_near_node[0] - 1 + anchor[0], closest_py + start_near_node[1] - 1 + anchor[2])): if info['synthesis'] is False: closest_depths.append(abs(info['depth'])) context_near_px, context_near_py = closest_pxs[np.array(closest_depths).argmax()], closest_pys[np.array(closest_depths).argmax()] else: context_near_px, context_near_py = context_near_pxs[distance.argmin()], context_near_pys[distance.argmin()] context_near_node = (start_near_node[0]-1 + context_near_px, start_near_node[1]-1 + context_near_py) far_node_list = [] global_context_near_node = (context_near_node[0] + anchor[0], context_near_node[1] + anchor[2]) if info_on_pix.get(global_context_near_node) is not None: for info in info_on_pix[global_context_near_node]: if info['synthesis'] is False: context_near_node_3d = (global_context_near_node[0], global_context_near_node[1], info['depth']) if global_mesh.nodes[context_near_node_3d].get('far') is not None: for far_node in global_mesh.nodes[context_near_node_3d].get('far'): far_node = (far_node[0] - anchor[0], far_node[1] - anchor[2], far_node[2]) if mask[far_node[0], far_node[1]] == 0: far_node_list.append([far_node[0], far_node[1]]) if len(far_node_list) > 0: far_nodes_dist = np.sum(np.abs(np.array(far_node_list) - np.array([[edge_px, edge_py]])), axis=1) context_far_node = tuple(far_node_list[far_nodes_dist.argmin()]) corresponding_far_edge = np.zeros_like(mask_edge) corresponding_far_edge[context_far_node[0], context_far_node[1]] = 1 surround_map = cv2.dilate(new_specific_edge_map.astype(np.uint8), np.array([[1,1,1],[1,1,1],[1,1,1]]).astype(np.uint8), iterations=1) specific_edge_map_wo_end_pt = new_specific_edge_map.copy() specific_edge_map_wo_end_pt[end_near_node[0], end_near_node[1]] = 0 surround_map_wo_end_pt = cv2.dilate(specific_edge_map_wo_end_pt.astype(np.uint8), np.array([[1,1,1],[1,1,1],[1,1,1]]).astype(np.uint8), iterations=1) surround_map_wo_end_pt[new_specific_edge_map > 0] = 0 surround_map_wo_end_pt[context_near_node[0], context_near_node[1]] = 0 surround_map = surround_map_wo_end_pt.copy() _, far_edge_cc = cv2.connectedComponents(surround_map.astype(np.uint8), connectivity=4) start_far_node = None accompany_far_node = None if surround_map[context_far_node[0], context_far_node[1]] == 1: start_far_node = context_far_node else: four_nes = [(context_far_node[0] - 1, context_far_node[1]), (context_far_node[0] + 1, context_far_node[1]), (context_far_node[0], context_far_node[1] - 1), (context_far_node[0], context_far_node[1] + 1)] candidate_bevel = []
for ne in four_nes: if surround_map[ne[0], ne[1]] == 1: start_far_node = (ne[0], ne[1]) break elif (ne[0] != context_near_node[0] or ne[1] != context_near_node[1]) and
(ne[0] != start_near_node[0] or ne[1] != start_near_node[1]): candidate_bevel.append((ne[0], ne[1])) if start_far_node is None: for ne in candidate_bevel: if ne[0] == context_far_node[0]: bevel_xys = [[ne[0] + 1, ne[1]], [ne[0] - 1, ne[1]]] if ne[1] == context_far_node[1]: bevel_xys = [[ne[0], ne[1] + 1], [ne[0], ne[1] - 1]] for bevel_x, bevel_y in bevel_xys: if surround_map[bevel_x, bevel_y] == 1: start_far_node = (bevel_x, bevel_y) accompany_far_node = (ne[0], ne[1]) break if start_far_node is not None: break if start_far_node is not None: for far_edge_id in range(1, far_edge_cc.max() + 1): specific_far_edge = (far_edge_cc == far_edge_id).astype(np.uint8) if specific_far_edge[start_far_node[0], start_far_node[1]] == 1: if accompany_far_node is not None: specific_far_edge[accompany_far_node] = 1 far_edge[specific_far_edge > 0] = 1 far_edge_with_id[specific_far_edge > 0] = edge_id end_far_candidates = np.zeros_like(far_edge) end_far_candidates[end_near_node[0], end_near_node[1]] = 1 end_far_candidates = cv2.dilate(end_far_candidates.astype(np.uint8), np.array([[0,1,0],[1,1,1],[0,1,0]]).astype(np.uint8), iterations=1) end_far_candidates[end_near_node[0], end_near_node[1]] = 0 invalid_nodes = (((far_edge_cc != far_edge_id).astype(np.uint8) *
(far_edge_cc != 0).astype(np.uint8)).astype(np.uint8) +
(new_specific_edge_map).astype(np.uint8) +
(mask == 0).astype(np.uint8)).clip(0, 1) end_far_candidates[invalid_nodes > 0] = 0 far_edge[end_far_candidates > 0] = 1 far_edge_with_id[end_far_candidates > 0] = edge_idfar_edge[context_far_node[0], context_far_node[1]] = 1 far_edge_with_id[context_far_node[0], context_far_node[1]] = edge_id near_edge_with_id[(mask_edge_with_id == edge_id) > 0] = edge_id uncleaned_far_edge = far_edge.copy() far_edge[mask == 0] = 0 return far_edge, uncleaned_far_edge, far_edge_with_id, near_edge_with_iddef get_MiDaS_samples(image_folder, depth_folder, config, specific=None, aft_certain=None): lines = [os.path.splitext(os.path.basename(xx))[0] for xx in glob.glob(os.path.join(image_folder, '*' + config['img_format']))] samples = [] generic_pose = np.eye(4) assert len(config['traj_types']) == len(config['x_shift_range']) ==
len(config['y_shift_range']) == len(config['z_shift_range']) == len(config['video_postfix']),
"The number of elements in 'traj_types', 'x_shift_range', 'y_shift_range', 'z_shift_range' and
'video_postfix' should be equal." tgt_pose = [[generic_pose * 1]] tgts_poses = [] for traj_idx in range(len(config['traj_types'])): tgt_poses = [] sx, sy, sz = path_planning(config['num_frames'], config['x_shift_range'][traj_idx], config['y_shift_range'][traj_idx], config['z_shift_range'][traj_idx], path_type=config['traj_types'][traj_idx]) for xx, yy, zz in zip(sx, sy, sz): tgt_poses.append(generic_pose * 1.) tgt_poses[-1][:3, -1] = np.array([xx, yy, zz]) tgts_poses += [tgt_poses]
tgt_pose = generic_pose * 1aft_flag = True if aft_certain is not None and len(aft_certain) > 0: aft_flag = False for seq_dir in lines: if specific is not None and len(specific) > 0: if specific != seq_dir: continue if aft_certain is not None and len(aft_certain) > 0: if aft_certain == seq_dir: aft_flag = True if aft_flag is False: continue samples.append({}) sdict = samples[-1] sdict['depth_fi'] = os.path.join(depth_folder, seq_dir + config['depth_format']) sdict['ref_img_fi'] = os.path.join(image_folder, seq_dir + config['img_format']) H, W = imageio.imread(sdict['ref_img_fi']).shape[:2] sdict['int_mtx'] = np.array([[max(H, W), 0, W//2], [0, max(H, W), H//2], [0, 0, 1]]).astype(np.float32) if sdict['int_mtx'].max() > 1: sdict['int_mtx'][0, :] = sdict['int_mtx'][0, :] / float(W) sdict['int_mtx'][1, :] = sdict['int_mtx'][1, :] / float(H) sdict['ref_pose'] = np.eye(4) sdict['tgt_pose'] = tgt_pose sdict['tgts_poses'] = tgts_poses sdict['video_postfix'] = config['video_postfix'] sdict['tgt_name'] = [os.path.splitext(os.path.basename(sdict['depth_fi']))[0]] sdict['src_pair_name'] = sdict['tgt_name'][0] return samplesdef get_valid_size(imap): x_max = np.where(imap.sum(1).squeeze() > 0)[0].max() + 1 x_min = np.where(imap.sum(1).squeeze() > 0)[0].min() y_max = np.where(imap.sum(0).squeeze() > 0)[0].max() + 1 y_min = np.where(imap.sum(0).squeeze() > 0)[0].min() size_dict = {'x_max':x_max, 'y_max':y_max, 'x_min':x_min, 'y_min':y_min}
return size_dictdef dilate_valid_size(isize_dict, imap, dilate=[0, 0]): osize_dict = copy.deepcopy(isize_dict) osize_dict['x_min'] = max(0, osize_dict['x_min'] - dilate[0]) osize_dict['x_max'] = min(imap.shape[0], osize_dict['x_max'] + dilate[0]) osize_dict['y_min'] = max(0, osize_dict['y_min'] - dilate[0]) osize_dict['y_max'] = min(imap.shape[1], osize_dict['y_max'] + dilate[1])
return osize_dictdef crop_maps_by_size(size, *imaps): omaps = [] for imap in imaps: omaps.append(imap[size['x_min']:size['x_max'], size['y_min']:size['y_max']].copy())
return omapsdef smooth_cntsyn_gap(init_depth_map, mask_region, context_region, init_mask_region=None): if init_mask_region is not None: curr_mask_region = init_mask_region * 1 else: curr_mask_region = mask_region * 0 depth_map = init_depth_map.copy() for _ in range(2): cm_mask = context_region + curr_mask_region depth_s1 = np.roll(depth_map, 1, 0) depth_s2 = np.roll(depth_map, -1, 0) depth_s3 = np.roll(depth_map, 1, 1) depth_s4 = np.roll(depth_map, -1, 1) mask_s1 = np.roll(cm_mask, 1, 0) mask_s2 = np.roll(cm_mask, -1, 0) mask_s3 = np.roll(cm_mask, 1, 1) mask_s4 = np.roll(cm_mask, -1, 1) fluxin_depths = (depth_s1 * mask_s1 + depth_s2 * mask_s2 + depth_s3 * mask_s3 + depth_s4 * mask_s4) /
((mask_s1 + mask_s2 + mask_s3 + mask_s4) + 1e-6) fluxin_mask = (fluxin_depths != 0) * mask_region init_mask = (fluxin_mask * (curr_mask_region >= 0).astype(np.float32) > 0).astype(np.uint8) depth_map[init_mask > 0] = fluxin_depths[init_mask > 0] if init_mask.shape[-1] > curr_mask_region.shape[-1]: curr_mask_region[init_mask.sum(-1, keepdims=True) > 0] = 1 else: curr_mask_region[init_mask > 0] = 1 depth_map[fluxin_mask > 0] = fluxin_depths[fluxin_mask > 0]return depth_mapdef read_MiDaS_depth(disp_fi, disp_rescale=10., h=None, w=None): if 'npy' in os.path.splitext(disp_fi)[-1]: disp = np.load(disp_fi) else: disp = imageio.imread(disp_fi).astype(np.float32) disp = disp - disp.min() disp = cv2.blur(disp / disp.max(), ksize=(3, 3)) * disp.max() disp = (disp / disp.max()) * disp_rescale if h is not None and w is not None: disp = resize(disp / disp.max(), (h, w), order=1) * disp.max() depth = 1. / np.maximum(disp, 0.05)
return depthdef follow_image_aspect_ratio(depth, image): H, W = image.shape[:2] image_aspect_ratio = H / W dH, dW = depth.shape[:2] depth_aspect_ratio = dH / dW if depth_aspect_ratio > image_aspect_ratio: resize_H = dH resize_W = dH / image_aspect_ratio else: resize_W = dW resize_H = dW * image_aspect_ratio depth = resize(depth / depth.max(), (int(resize_H), int(resize_W)), order=0) * depth.max()
return depthdef depth_resize(depth, origin_size, image_size): if origin_size[0] is not 0: max_depth = depth.max() depth = depth / max_depth depth = resize(depth, origin_size, order=1, mode='edge') depth = depth * max_depth else: max_depth = depth.max() depth = depth / max_depth depth = resize(depth, image_size, order=1, mode='edge') depth = depth * max_depth
return depthdef filter_irrelevant_edge(self_edge, other_edges, other_edges_with_id, current_edge_id, context, edge_ccs, mesh, anchor): other_edges = other_edges.squeeze() other_edges_with_id = other_edges_with_id.squeeze()
self_edge = self_edge.squeeze() dilate_self_edge = cv2.dilate(self_edge.astype(np.uint8), np.array([[1,1,1],[1,1,1],[1,1,1]]).astype(np.uint8), iterations=1) edge_ids = collections.Counter(other_edges_with_id.flatten()).keys() other_edges_info = [] # import ipdb # ipdb.set_trace() for edge_id in edge_ids: edge_id = int(edge_id) if edge_id >= 0: condition = ((other_edges_with_id == edge_id) * other_edges * context).astype(np.uint8) if dilate_self_edge[condition > 0].sum() == 0: other_edges[other_edges_with_id == edge_id] = 0 else: num_condition, condition_labels = cv2.connectedComponents(condition, connectivity=8) for condition_id in range(1, num_condition): isolate_condition = ((condition_labels == condition_id) > 0).astype(np.uint8) num_end_group, end_group = cv2.connectedComponents(((dilate_self_edge * isolate_condition) > 0).astype(np.uint8), connectivity=8) if num_end_group == 1: continue for end_id in range(1, num_end_group): end_pxs, end_pys = np.where((end_group == end_id)) end_px, end_py = end_pxs[0], end_pys[0] other_edges_info.append({}) other_edges_info[-1]['edge_id'] = edge_id # other_edges_info[-1]['near_depth'] = None other_edges_info[-1]['diff'] = None other_edges_info[-1]['edge_map'] = np.zeros_like(self_edge) other_edges_info[-1]['end_point_map'] = np.zeros_like(self_edge) other_edges_info[-1]['end_point_map'][(end_group == end_id)] = 1 other_edges_info[-1]['forbidden_point_map'] = np.zeros_like(self_edge) other_edges_info[-1]['forbidden_point_map'][(end_group != end_id) * (end_group != 0)] = 1 other_edges_info[-1]['forbidden_point_map'] = cv2.dilate(other_edges_info[-1]['forbidden_point_map'], kernel=np.array([[1,1,1],[1,1,1],[1,1,1]]), iterations=2) for x in edge_ccs[edge_id]: nx = x[0] - anchor[0] ny = x[1] - anchor[1] if nx == end_px and ny == end_py: # other_edges_info[-1]['near_depth'] = abs(nx) if mesh.nodes[x].get('far') is not None and len(mesh.nodes[x].get('far')) == 1: other_edges_info[-1]['diff'] = abs(1./abs([*mesh.nodes[x].get('far')][0][2]) - 1./abs(x[2])) else: other_edges_info[-1]['diff'] = 0 # if end_group[nx, ny] != end_id and end_group[nx, ny] > 0: # continue try: if isolate_condition[nx, ny] == 1: other_edges_info[-1]['edge_map'][nx, ny] = 1 except: pass try: other_edges_info = sorted(other_edges_info, key=lambda x : x['diff'], reverse=True) except: import pdb pdb.set_trace() # import pdb # pdb.set_trace() # other_edges = other_edges[..., None] for other_edge in other_edges_info: if other_edge['end_point_map'] is None: import pdb pdb.set_trace() other_edges = other_edges * context return other_edges, other_edges_infodef require_depth_edge(context_edge, mask): dilate_mask = cv2.dilate(mask, np.array([[1,1,1],[1,1,1],[1,1,1]]).astype(np.uint8), iterations=1) if (dilate_mask * context_edge).max() == 0: return False else: return True
def refine_color_around_edge(mesh, info_on_pix, edge_ccs, config, spdb=False): H, W = mesh.graph['H'], mesh.graph['W'] tmp_edge_ccs = copy.deepcopy(edge_ccs) for edge_id, edge_cc in enumerate(edge_ccs): if len(edge_cc) == 0: continue near_maps = np.zeros((H, W)).astype(np.bool) far_maps = np.zeros((H, W)).astype(np.bool) tmp_far_nodes = set() far_nodes = set() near_nodes = set() end_nodes = set()
for i in range(5): if i == 0: for edge_node in edge_cc: if mesh.nodes[edge_node].get('depth_edge_dilate_2_color_flag') is not True: break if mesh.nodes[edge_node].get('inpaint_id') == 1: near_nodes.add(edge_node) tmp_node = mesh.nodes[edge_node].get('far') tmp_node = set(tmp_node) if tmp_node is not None else set() tmp_far_nodes |= tmp_node rmv_tmp_far_nodes = set() for far_node in tmp_far_nodes: if not(mesh.has_node(far_node) and mesh.nodes[far_node].get('inpaint_id') == 1): rmv_tmp_far_nodes.add(far_node) if len(tmp_far_nodes - rmv_tmp_far_nodes) == 0: break
else: for near_node in near_nodes: near_maps[near_node[0], near_node[1]] = True mesh.nodes[near_node]['refine_rgbd'] = True mesh.nodes[near_node]['backup_depth'] = near_node[2]
if mesh.nodes[near_node].get('real_depth') is None else mesh.nodes[near_node]['real_depth'] mesh.nodes[near_node]['backup_color'] = mesh.nodes[near_node]['color'] for far_node in tmp_far_nodes: if mesh.has_node(far_node) and mesh.nodes[far_node].get('inpaint_id') == 1: far_nodes.add(far_node) far_maps[far_node[0], far_node[1]] = True mesh.nodes[far_node]['refine_rgbd'] = True mesh.nodes[far_node]['backup_depth'] = far_node[2]
if mesh.nodes[far_node].get('real_depth') is None else mesh.nodes[far_node]['real_depth'] mesh.nodes[far_node]['backup_color'] = mesh.nodes[far_node]['color'] tmp_far_nodes = far_nodes tmp_near_nodes = near_nodes else: tmp_far_nodes = new_tmp_far_nodes tmp_near_nodes = new_tmp_near_nodes new_tmp_far_nodes = None new_tmp_near_nodes = None new_tmp_far_nodes = set() new_tmp_near_nodes = set() for node in tmp_near_nodes: for ne_node in mesh.neighbors(node): if far_maps[ne_node[0], ne_node[1]] == False and
near_maps[ne_node[0], ne_node[1]] == False: if mesh.nodes[ne_node].get('inpaint_id') == 1: new_tmp_near_nodes.add(ne_node) near_maps[ne_node[0], ne_node[1]] = True mesh.nodes[ne_node]['refine_rgbd'] = True mesh.nodes[ne_node]['backup_depth'] = ne_node[2]
if mesh.nodes[ne_node].get('real_depth') is None else mesh.nodes[ne_node]['real_depth'] mesh.nodes[ne_node]['backup_color'] = mesh.nodes[ne_node]['color'] else: mesh.nodes[ne_node]['backup_depth'] = ne_node[2]
if mesh.nodes[ne_node].get('real_depth') is None else mesh.nodes[ne_node]['real_depth'] mesh.nodes[ne_node]['backup_color'] = mesh.nodes[ne_node]['color'] end_nodes.add(node) near_nodes.update(new_tmp_near_nodes) for node in tmp_far_nodes: for ne_node in mesh.neighbors(node): if far_maps[ne_node[0], ne_node[1]] == False and
near_maps[ne_node[0], ne_node[1]] == False: if mesh.nodes[ne_node].get('inpaint_id') == 1: new_tmp_far_nodes.add(ne_node) far_maps[ne_node[0], ne_node[1]] = True mesh.nodes[ne_node]['refine_rgbd'] = True mesh.nodes[ne_node]['backup_depth'] = ne_node[2]
if mesh.nodes[ne_node].get('real_depth') is None else mesh.nodes[ne_node]['real_depth'] mesh.nodes[ne_node]['backup_color'] = mesh.nodes[ne_node]['color'] else: mesh.nodes[ne_node]['backup_depth'] = ne_node[2]
if mesh.nodes[ne_node].get('real_depth') is None else mesh.nodes[ne_node]['real_depth'] mesh.nodes[ne_node]['backup_color'] = mesh.nodes[ne_node]['color'] end_nodes.add(node) far_nodes.update(new_tmp_far_nodes) if len(far_nodes) == 0: tmp_edge_ccs[edge_id] = set() continue for node in new_tmp_far_nodes | new_tmp_near_nodes: for ne_node in mesh.neighbors(node): if far_maps[ne_node[0], ne_node[1]] == False and near_maps[ne_node[0], ne_node[1]] == False: end_nodes.add(node) mesh.nodes[ne_node]['backup_depth'] = ne_node[2]
if mesh.nodes[ne_node].get('real_depth') is None else mesh.nodes[ne_node]['real_depth'] mesh.nodes[ne_node]['backup_color'] = mesh.nodes[ne_node]['color'] tmp_end_nodes = end_nodesrefine_nodes = near_nodes | far_nodes remain_refine_nodes = copy.deepcopy(refine_nodes) accum_idx = 0 while len(remain_refine_nodes) > 0: accum_idx += 1 if accum_idx > 100: break new_tmp_end_nodes = None new_tmp_end_nodes = set() survive_tmp_end_nodes = set() for node in tmp_end_nodes: re_depth, re_color, re_count = 0, np.array([0., 0., 0.]), 0 for ne_node in mesh.neighbors(node): if mesh.nodes[ne_node].get('refine_rgbd') is True: if ne_node not in tmp_end_nodes: new_tmp_end_nodes.add(ne_node) else: try: re_depth += mesh.nodes[ne_node]['backup_depth'] re_color += mesh.nodes[ne_node]['backup_color'].astype(np.float32) re_count += 1. except: import pdb; pdb.set_trace() if re_count > 0: re_depth = re_depth / re_count re_color = re_color / re_count mesh.nodes[node]['backup_depth'] = re_depth mesh.nodes[node]['backup_color'] = re_color mesh.nodes[node]['refine_rgbd'] = False else: survive_tmp_end_nodes.add(node) for node in tmp_end_nodes - survive_tmp_end_nodes: if node in remain_refine_nodes: remain_refine_nodes.remove(node) tmp_end_nodes = new_tmp_end_nodes if spdb == True: bfrd_canvas = np.zeros((H, W)) bfrc_canvas = np.zeros((H, W, 3)).astype(np.uint8) aftd_canvas = np.zeros((H, W)) aftc_canvas = np.zeros((H, W, 3)).astype(np.uint8) for node in refine_nodes: bfrd_canvas[node[0], node[1]] = abs(node[2]) aftd_canvas[node[0], node[1]] = abs(mesh.nodes[node]['backup_depth']) bfrc_canvas[node[0], node[1]] = mesh.nodes[node]['color'].astype(np.uint8) aftc_canvas[node[0], node[1]] = mesh.nodes[node]['backup_color'].astype(np.uint8) f, (ax1, ax2, ax3, ax4) = plt.subplots(1, 4, sharex=True, sharey=True); ax1.imshow(bfrd_canvas); ax2.imshow(aftd_canvas); ax3.imshow(bfrc_canvas); ax4.imshow(aftc_canvas); plt.show() import pdb; pdb.set_trace() for node in refine_nodes: if mesh.nodes[node].get('refine_rgbd') is not None: mesh.nodes[node].pop('refine_rgbd') mesh.nodes[node]['color'] = mesh.nodes[node]['backup_color'] for info in info_on_pix[(node[0], node[1])]: if info['depth'] == node[2]: info['color'] = mesh.nodes[node]['backup_color'] return mesh, info_on_pixdef refine_depth_around_edge(mask_depth, far_edge, uncleaned_far_edge, near_edge, mask, all_depth, config): if isinstance(mask_depth, torch.Tensor): if mask_depth.is_cuda: mask_depth = mask_depth.cpu() mask_depth = mask_depth.data mask_depth = mask_depth.numpy() if isinstance(far_edge, torch.Tensor): if far_edge.is_cuda: far_edge = far_edge.cpu() far_edge = far_edge.data far_edge = far_edge.numpy() if isinstance(uncleaned_far_edge, torch.Tensor): if uncleaned_far_edge.is_cuda: uncleaned_far_edge = uncleaned_far_edge.cpu() uncleaned_far_edge = uncleaned_far_edge.data uncleaned_far_edge = uncleaned_far_edge.numpy() if isinstance(near_edge, torch.Tensor): if near_edge.is_cuda: near_edge = near_edge.cpu() near_edge = near_edge.data near_edge = near_edge.numpy() if isinstance(mask, torch.Tensor): if mask.is_cuda: mask = mask.cpu() mask = mask.data mask = mask.numpy() mask = mask.squeeze() uncleaned_far_edge = uncleaned_far_edge.squeeze() far_edge = far_edge.squeeze() near_edge = near_edge.squeeze() mask_depth = mask_depth.squeeze() dilate_far_edge = cv2.dilate(uncleaned_far_edge.astype(np.uint8), kernel=np.array([[0,1,0],[1,1,1],[0,1,0]]).astype(np.uint8), iterations=1) near_edge[dilate_far_edge == 0] = 0 dilate_near_edge = cv2.dilate(near_edge.astype(np.uint8), kernel=np.array([[0,1,0],[1,1,1],[0,1,0]]).astype(np.uint8), iterations=1) far_edge[dilate_near_edge == 0] = 0 init_far_edge = far_edge.copy() init_near_edge = near_edge.copy() for i in range(config['depth_edge_dilate_2']): init_far_edge = cv2.dilate(init_far_edge, kernel=np.array([[0,1,0],[1,1,1],[0,1,0]]).astype(np.uint8), iterations=1) init_far_edge[init_near_edge == 1] = 0 init_near_edge = cv2.dilate(init_near_edge, kernel=np.array([[0,1,0],[1,1,1],[0,1,0]]).astype(np.uint8), iterations=1) init_near_edge[init_far_edge == 1] = 0 init_far_edge[mask == 0] = 0 init_near_edge[mask == 0] = 0 hole_far_edge = 1 - init_far_edge hole_near_edge = 1 - init_near_edge change = None while True: change = False hole_far_edge[init_near_edge == 1] = 0 hole_near_edge[init_far_edge == 1] = 0 far_pxs, far_pys = np.where((hole_far_edge == 0) * (init_far_edge == 1) > 0) current_hole_far_edge = hole_far_edge.copy() for far_px, far_py in zip(far_pxs, far_pys): min_px = max(far_px - 1, 0) max_px = min(far_px + 2, mask.shape[0]-1) min_py = max(far_py - 1, 0) max_py = min(far_py + 2, mask.shape[1]-1) hole_far = current_hole_far_edge[min_px: max_px, min_py: max_py] tmp_mask = mask[min_px: max_px, min_py: max_py] all_depth_patch = all_depth[min_px: max_px, min_py: max_py] * 0 all_depth_mask = (all_depth_patch != 0).astype(np.uint8) cross_element = np.array([[0,1,0],[1,1,1],[0,1,0]])[min_px - (far_px - 1): max_px - (far_px - 1), min_py - (far_py - 1): max_py - (far_py - 1)] combine_mask = (tmp_mask + all_depth_mask).clip(0, 1) * hole_far * cross_element tmp_patch = combine_mask * (mask_depth[min_px: max_px, min_py: max_py] + all_depth_patch) number = np.count_nonzero(tmp_patch) if number > 0: mask_depth[far_px, far_py] = np.sum(tmp_patch).astype(np.float32) / max(number, 1e-6) hole_far_edge[far_px, far_py] = 1 change = True near_pxs, near_pys = np.where((hole_near_edge == 0) * (init_near_edge == 1) > 0) current_hole_near_edge = hole_near_edge.copy() for near_px, near_py in zip(near_pxs, near_pys): min_px = max(near_px - 1, 0) max_px = min(near_px + 2, mask.shape[0]-1) min_py = max(near_py - 1, 0) max_py = min(near_py + 2, mask.shape[1]-1) hole_near = current_hole_near_edge[min_px: max_px, min_py: max_py] tmp_mask = mask[min_px: max_px, min_py: max_py] all_depth_patch = all_depth[min_px: max_px, min_py: max_py] * 0 all_depth_mask = (all_depth_patch != 0).astype(np.uint8)
cross_element = np.array([[0,1,0],[1,1,1],[0,1,0]])[min_px - near_px + 1:max_px - near_px + 1, min_py - near_py + 1:max_py - near_py + 1] combine_mask = (tmp_mask + all_depth_mask).clip(0, 1) * hole_near * cross_element tmp_patch = combine_mask * (mask_depth[min_px: max_px, min_py: max_py] + all_depth_patch) number = np.count_nonzero(tmp_patch) if number > 0:
mask_depth[near_px, near_py] = np.sum(tmp_patch) / max(number, 1e-6) hole_near_edge[near_px, near_py] = 1 change = True if change is False: breakreturn mask_depthdef vis_depth_edge_connectivity(depth, config): disp = 1./depth u_diff = (disp[1:, :] - disp[:-1, :])[:-1, 1:-1] b_diff = (disp[:-1, :] - disp[1:, :])[1:, 1:-1] l_diff = (disp[:, 1:] - disp[:, :-1])[1:-1, :-1] r_diff = (disp[:, :-1] - disp[:, 1:])[1:-1, 1:] u_over = (np.abs(u_diff) > config['depth_threshold']).astype(np.float32) b_over = (np.abs(b_diff) > config['depth_threshold']).astype(np.float32) l_over = (np.abs(l_diff) > config['depth_threshold']).astype(np.float32) r_over = (np.abs(r_diff) > config['depth_threshold']).astype(np.float32) concat_diff = np.stack([u_diff, b_diff, r_diff, l_diff], axis=-1) concat_over = np.stack([u_over, b_over, r_over, l_over], axis=-1) over_diff = concat_diff * concat_over pos_over = (over_diff > 0).astype(np.float32).sum(-1).clip(0, 1) neg_over = (over_diff < 0).astype(np.float32).sum(-1).clip(0, 1) neg_over[(over_diff > 0).astype(np.float32).sum(-1) > 0] = 0 _, edge_label = cv2.connectedComponents(pos_over.astype(np.uint8), connectivity=8) T_junction_maps = np.zeros_like(pos_over) for edge_id in range(1, edge_label.max() + 1): edge_map = (edge_label == edge_id).astype(np.uint8) edge_map = np.pad(edge_map, pad_width=((1,1),(1,1)), mode='constant') four_direc = np.roll(edge_map, 1, 1) + np.roll(edge_map, -1, 1) + np.roll(edge_map, 1, 0) + np.roll(edge_map, -1, 0) eight_direc = np.roll(np.roll(edge_map, 1, 1), 1, 0) + np.roll(np.roll(edge_map, 1, 1), -1, 0) +
np.roll(np.roll(edge_map, -1, 1), 1, 0) + np.roll(np.roll(edge_map, -1, 1), -1, 0) eight_direc = (eight_direc + four_direc)[1:-1,1:-1] pos_over[eight_direc > 2] = 0 T_junction_maps[eight_direc > 2] = 1 _, edge_label = cv2.connectedComponents(pos_over.astype(np.uint8), connectivity=8) edge_label = np.pad(edge_label, 1, mode='constant')return edge_labeldef max_size(mat, value=0): if not (mat and mat[0]): return (0, 0) it = iter(mat) prev = [(el==value) for el in next(it)] max_size = max_rectangle_size(prev) for row in it: hist = [(1+h) if el == value else 0 for h, el in zip(prev, row)] max_size = max(max_size, max_rectangle_size(hist), key=get_area) prev = hist
return max_sizedef max_rectangle_size(histogram): Info = namedtuple('Info', 'start height') stack = [] top = lambda: stack[-1] max_size = (0, 0) # height, width of the largest rectangle pos = 0 # current position in the histogram for pos, height in enumerate(histogram): start = pos # position where rectangle starts while True: if not stack or height > top().height: stack.append(Info(start, height)) # push if stack and height < top().height: max_size = max(max_size, (top().height, (pos-top().start)), key=get_area) start, _ = stack.pop() continue break # height == top().height goes here
pos += 1 for start, height in stack: max_size = max(max_size, (height, (pos-start)), key=get_area) return max_sizedef get_area(size): return reduce(mul, size)
def find_anchors(matrix): matrix = [[*x] for x in matrix] mh, mw = max_size(matrix) matrix = np.array(matrix) # element = np.zeros((mh, mw)) for i in range(matrix.shape[0] + 1 - mh): for j in range(matrix.shape[1] + 1 - mw): if matrix[i:i + mh, j:j + mw].max() == 0: return i, i + mh, j, j + mw
def find_largest_rect(dst_img, bg_color=(128, 128, 128)): valid = np.any(dst_img[..., :3] != bg_color, axis=-1) dst_h, dst_w = dst_img.shape[:2] ret, labels = cv2.connectedComponents(np.uint8(valid == False)) red_mat = np.zeros_like(labels) # denoise for i in range(1, np.max(labels)+1, 1): x, y, w, h = cv2.boundingRect(np.uint8(labels==i)) if x == 0 or (x+w) == dst_h or y == 0 or (y+h) == dst_w: red_mat[labels==i] = 1 # crop t, b, l, r = find_anchors(red_mat)
return t, b, l, r - Assistant
Your script contains a wide range of functions that seem to be centered around computer vision tasks, particularly those involving depth information from images, path planning, and edge detection. It heavily utilizes libraries like OpenCV, NumPy, and PyTorch.
Here's a brief summary of some of the functionalities evident in the code:
-
Path Planning: The function
path_planninggenerates different types of paths in 3D space, including straight lines and circular paths. -
Image and Edge Processing: There are multiple functions dedicated to manipulating images and masks, performing morphological operations like dilation and erosion, filtering irrelevant edges, and refining depth maps around edges.
-
Graph Construction and Manipulation: The code uses
cynetworkx(ornetworkxas a fallback) to create and manipulate graphs, specifically focusing on connectivity and components of pixels in images. -
Depth Map Handling: Functions like
read_MiDaS_depthhandle the reading and processing of depth maps, potentially converting disparity maps to depth information. -
Visualization: There are various plotting functions using Matplotlib for visualizing depth maps and other intermediate results.
-
Refinement and Cleaning: Functions like
refine_depth_around_edgeandclean_far_edge_newfocus on improving the accuracy and continuity of edges in the context of depth information, possibly aiming to smooth out depth discontinuities in images. -
Utilization of Collections: Many operations rely on dictionary and named tuples for organizing data, particularly in ways that store information about pixels and edges.
-
Miscellaneous Functions: Functions like
get_valid_sizeandfind_largest_rectare included for image bounding box calculations, validating regions in images, and ensuring that certain properties like connectivity are met.
If you have specific tasks or functionalities you want to modify or need help understanding, feel free to ask!
-
- User
this can make camera rotation?
- Assistant
The code you provided does not explicitly implement camera rotation, but it includes functions that might be part of a system that could manage camera movement, particularly in a 3D space. Here's a breakdown of how camera rotation could be implemented or influenced by the existing code:
Camera Rotation Concept
Camera rotation generally refers to changing the orientation of the camera in a 3D environment, which affects the perspective from which the scene is viewed. This can involve rotating around one or more axes (pitch, yaw, roll) and is typically combined with translation (movement through space).
Relevant Parts of Your Code:
-
Path Planning: The
path_planningfunction is capable of generating smooth trajectories in 3D space:- You could extend this function to include rotation for the camera by calculating orientations (using quaternions or Euler angles) corresponding to the generated path.
-
Pose Information: The code references "pose" in the function
get_MiDaS_samples, which likely pertains to the camera's position and orientation in the 3D space:- You can utilize transformation matrices to represent the camera's rotation and translation together. Each of the target poses generated can be extended to include orientation changes.
-
Depth Handling: Camera rotations are closely linked with depth perception (how far each point in the image is from the camera):
- Functions dealing with depth maps (like
read_MiDaS_depthand others) could help you simulate how the image would look from different orientations.
- Functions dealing with depth maps (like
-
3D Visualization: If you are planning to visualize the camera's path or orientation changes:
- You could use
matplotlibor similar libraries to graphically represent the camera's rotation over time or along a path. Thempl_toolkits.mplot3dcan facilitate 3D plotting.
- You could use
Example Steps to Implement Camera Rotation
To implement camera rotation, you might consider the following steps:
-
Define Rotation Logic: Decide how you want the camera to rotate (e.g., along the path, fixed axis rotation, etc.).
-
Calculate Transformation Matrices:
- Use rotation matrices for pitch, yaw, and roll to represent the camera's orientation. You typically combine these with the translation matrix for the camera position.
-
Modify Path Planning: Update
path_planningto also generate changes in orientation based on the desired camera rotation. -
Render Using Updated Poses: Whenever you render frames or update the scene, apply the transformation associated with the current pose (position and rotation) to adjust what is displayed.
Example of Applying Rotation Matrices
Here's a simple outline for applying rotation to a camera in 3D:
import numpy as np def rotation_matrix(axis, theta): """Generate a rotation matrix for a given axis and angle.""" axis = axis / np.linalg.norm(axis) a = np.cos(theta / 2.0) b, c, d = -axis * np.sin(theta / 2.0) return np.array([[a*a + b*b - c*c - d*d, 2*(b*c - a*d), 2*(b*d + a*c)], [2*(b*c + a*d), a*a + c*c - b*b - d*d, 2*(c*d - a*b)], [2*(b*d - a*c), 2*(c*d + a*b), a*a + d*d - b*b - c*c]]) # Example: Rotate around the y-axis (up) axis = np.array([0, 1, 0]) # y-axis theta = np.radians(30) # rotate 30 degrees rot_matrix = rotation_matrix(axis, theta) # Apply this rotation to camera position camera_position = np.array([x, y, z]) new_view = np.dot(rot_matrix, camera_position) # New view after rotationConclusion
While the current functions plan paths and manipulate depth information, they may not directly change camera orientation. However, leveraging the logic in the code allows you to develop a system where you can manage both translation and rotation of a camera based on the path and poses generated. Implementing camera rotation requires an understanding of 3D transformations, which can be integrated with the existing path planning and depth handling functions.
If you need help with a specific aspect of implementing camera rotation, feel free to ask!
-
Other models, same or similar prompt
Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.