import gradio as gr from PIL import Image import numpy as np from copy import deepcopy import cv2 import torch from ZoeDepth.zoedepth.utils.misc import colorize from objctrl_2_5d.utils.vis_camera import vis_camera_rescale from objctrl_2_5d.utils.objmask_util import trajectory_to_camera_poses_v1 from objctrl_2_5d.utils.customized_cam import rotation, clockwise, pan_and_zoom zc_threshold = 0.2 depth_scale_ = 5.2 center_margin = 10 height, width = 320, 576 num_frames = 14 intrinsics = np.array([[float(width), float(width), float(width) / 2, float(height) / 2]]) intrinsics = np.repeat(intrinsics, num_frames, axis=0) # [n_frame, 4] fx = intrinsics[0, 0] / width fy = intrinsics[0, 1] / height cx = intrinsics[0, 2] / width cy = intrinsics[0, 3] / height def process_image(raw_image): image, points = raw_image['image'], raw_image['points'] print(points) try: assert(len(points)) == 1, "Please select only one point" [x1, y1, _, x2, y2, _] = points[0] image = image.crop((x1, y1, x2, y2)) image = image.resize((width, height)) except: image = image.resize((width, height)) return image, gr.update(value={'image': image}) # -------------- general UI functionality -------------- def get_subject_points(canvas): return canvas["image"], canvas["points"] def run_segment(segmentor): def segment(canvas, image, logits): if logits is not None: logits *= 32.0 _, points = get_subject_points(canvas) image = np.array(image) with torch.inference_mode(), torch.autocast("cuda", dtype=torch.bfloat16): segmentor.set_image(image) input_points = [] input_boxes = [] for p in points: [x1, y1, _, x2, y2, _] = p if x2==0 and y2==0: input_points.append([x1, y1]) else: input_boxes.append([x1, y1, x2, y2]) if len(input_points) == 0: input_points = None input_labels = None else: input_points = np.array(input_points) input_labels = np.ones(len(input_points)) if len(input_boxes) == 0: input_boxes = None else: input_boxes = np.array(input_boxes) masks, _, logits = segmentor.predict( point_coords=input_points, point_labels=input_labels, box=input_boxes, multimask_output=False, return_logits=True, mask_input=logits, ) mask = masks > 0 masked_img = mask_image(image, mask[0], color=[252, 140, 90], alpha=0.9) masked_img = Image.fromarray(masked_img) return mask[0], masked_img, masked_img, logits / 32.0 return segment def mask_image(image, mask, color=[255,0,0], alpha=0.5): """ Overlay mask on image for visualization purpose. Args: image (H, W, 3) or (H, W): input image mask (H, W): mask to be overlaid color: the color of overlaid mask alpha: the transparency of the mask """ out = deepcopy(image) img = deepcopy(image) img[mask == 1] = color out = cv2.addWeighted(img, alpha, out, 1-alpha, 0, out) return out def get_points(img, sel_pix, evt: gr.SelectData): # collect the selected point img = np.array(img) img = deepcopy(img) sel_pix.append(evt.index) # only draw the last two points # if len(sel_pix) > 2: # sel_pix = sel_pix[-2:] points = [] for idx, point in enumerate(sel_pix): if idx % 2 == 0: # draw a red circle at the handle point cv2.circle(img, tuple(point), 10, (255, 0, 0), -1) else: # draw a blue circle at the handle point cv2.circle(img, tuple(point), 10, (0, 0, 255), -1) points.append(tuple(point)) # draw an arrow from handle point to target point # if len(points) == idx + 1: if idx > 0: cv2.arrowedLine(img, points[idx-1], points[idx], (255, 255, 255), 4, tipLength=0.5) # points = [] return img if isinstance(img, np.ndarray) else np.array(img), sel_pix # clear all handle/target points def undo_points(original_image): return original_image, [] def run_depth(d_model_NK): def get_depth(image, points): depth = d_model_NK.infer_pil(image) colored_depth = colorize(depth, cmap='gray_r') # [h, w, 4] 0-255 depth_img = deepcopy(colored_depth[:, :, :3]) if len(points) > 0: for idx, point in enumerate(points): if idx % 2 == 0: cv2.circle(depth_img, tuple(point), 10, (255, 0, 0), -1) else: cv2.circle(depth_img, tuple(point), 10, (0, 0, 255), -1) if idx > 0: cv2.arrowedLine(depth_img, points[idx-1], points[idx], (255, 255, 255), 4, tipLength=0.5) return depth, depth_img, colored_depth[:, :, :3] return get_depth def interpolate_points(points, num_points): x = points[:, 0] y = points[:, 1] # Interpolating the points t = np.linspace(0, 1, len(points)) t_new = np.linspace(0, 1, num_points) x_new = np.interp(t_new, t, x) y_new = np.interp(t_new, t, y) return np.vstack((x_new, y_new)).T # [] def traj2cam(traj, depth, rescale): traj = np.array(traj) trajectory = interpolate_points(traj, num_frames) center_h_margin, center_w_margin = center_margin, center_margin depth_center = np.mean(depth[height//2-center_h_margin:height//2+center_h_margin, width//2-center_w_margin:width//2+center_w_margin]) depth_rescale = round(depth_scale_ * rescale / depth_center, 2) r_depth = depth * depth_rescale zc = [] for i in range(num_frames): zc.append(r_depth[int(trajectory[i][1]), int(trajectory[i][0])]) # print(f'zc: {zc}') ## norm zc zc_norm = np.array(zc) zc_grad = zc_norm[1:] - zc_norm[:-1] zc_grad = np.abs(zc_grad) zc_grad = zc_grad[1:] - zc_grad[:-1] zc_grad_std = np.std(zc_grad) if zc_grad_std > zc_threshold: zc = [zc[0]] * num_frames # print(f'zc_grad_std: {zc_grad_std}, zc_threshold: {zc_threshold}') # print(f'zc: {zc}') traj_w2c = trajectory_to_camera_poses_v1(trajectory, intrinsics, num_frames, zc=zc) # numpy: [n_frame, 4, 4] RTs = traj_w2c[:, :3] fig = vis_camera_rescale(RTs) return RTs, fig def get_rotate_cam(angle, depth): # mean_depth = np.mean(depth * mask) center_h_margin, center_w_margin = center_margin, center_margin depth_center = np.mean(depth[height//2-center_h_margin:height//2+center_h_margin, width//2-center_w_margin:width//2+center_w_margin]) print(f'rotate depth_center: {depth_center}') RTs = rotation(num_frames, angle, depth_center, depth_center) fig = vis_camera_rescale(RTs) return RTs, fig def get_clockwise_cam(angle, depth, mask): mask = mask.astype(np.float32) # [0, 1] mean_depth = np.mean(depth * mask) # center_h_margin, center_w_margin = center_margin, center_margin # depth_center = np.mean(depth[height//2-center_h_margin:height//2+center_h_margin, width//2-center_w_margin:width//2+center_w_margin]) RTs = clockwise(angle, num_frames) RTs[:, -1, -1] = mean_depth fig = vis_camera_rescale(RTs) return RTs, fig def get_translate_cam(Tx, Ty, Tz, depth, mask, speed): mask = mask.astype(np.float32) # [0, 1] mean_depth = np.mean(depth * mask) T = np.array([Tx, Ty, Tz]) T = T.reshape(3, 1) T = T[None, ...].repeat(num_frames, axis=0) RTs = pan_and_zoom(T, speed, n=num_frames) RTs[:, -1, -1] += mean_depth fig = vis_camera_rescale(RTs) return RTs, fig def get_camera_pose(camera_mode): def trigger_camera_pose(camera_option, selected_points, depth, mask, rescale, angle, Tx, Ty, Tz, speed): if camera_option == camera_mode[0]: # traj2cam return traj2cam(selected_points, depth, rescale) elif camera_option == camera_mode[1]: # rotate return get_rotate_cam(angle, depth) elif camera_option == camera_mode[2]: # clockwise return get_clockwise_cam(angle, depth, mask) elif camera_option == camera_mode[3]: # translate return get_translate_cam(Tx, Ty, Tz, depth, mask, speed) return trigger_camera_pose