EMAGE / emage_utils /motion_rep_transfer.py
H-Liu1997's picture
newapp
b03a8f2
import smplx
import torch
import numpy as np
from . import rotation_conversions as rc
import os
import wget
download_path = "./emage_evaltools/"
smplx_model_dir = os.path.join(download_path, "smplx_models", "smplx")
if not os.path.exists(smplx_model_dir):
smplx_model_file_path = os.path.join(smplx_model_dir, "SMPLX_NEUTRAL_2020.npz")
os.makedirs(smplx_model_dir, exist_ok=True)
if not os.path.exists(smplx_model_file_path):
print(f"Downloading {smplx_model_file_path}")
wget.download(
"https://huggingface.co/spaces/H-Liu1997/EMAGE/resolve/main/EMAGE/smplx_models/smplx/SMPLX_NEUTRAL_2020.npz",
smplx_model_file_path,
)
smplx_model = smplx.create(
"./emage_evaltools/smplx_models/",
model_type='smplx',
gender='NEUTRAL_2020',
use_face_contour=False,
num_betas=300,
num_expression_coeffs=100,
ext='npz',
use_pca=False,
).eval()
def get_motion_rep_tensor(motion_tensor, pose_fps=30, device="cuda", betas=None):
global smplx_model
smplx_model = smplx_model.to(device)
bs, n, _ = motion_tensor.shape
motion_tensor = motion_tensor.float().to(device)
motion_tensor_reshaped = motion_tensor.reshape(bs * n, 165)
betas = torch.zeros(n, 300, device=device) if betas is None else betas.to(device).unsqueeze(0).repeat(n, 1)
output = smplx_model(
betas=torch.zeros(bs * n, 300, device=device),
transl=torch.zeros(bs * n, 3, device=device),
expression=torch.zeros(bs * n, 100, device=device),
jaw_pose=torch.zeros(bs * n, 3, device=device),
global_orient=torch.zeros(bs * n, 3, device=device),
body_pose=motion_tensor_reshaped[:, 3:21 * 3 + 3],
left_hand_pose=motion_tensor_reshaped[:, 25 * 3:40 * 3],
right_hand_pose=motion_tensor_reshaped[:, 40 * 3:55 * 3],
return_joints=True,
leye_pose=torch.zeros(bs * n, 3, device=device),
reye_pose=torch.zeros(bs * n, 3, device=device),
)
joints = output['joints'].reshape(bs, n, 127, 3)[:, :, :55, :]
dt = 1 / pose_fps
init_vel = (joints[:, 1:2] - joints[:, 0:1]) / dt
middle_vel = (joints[:, 2:] - joints[:, :-2]) / (2 * dt)
final_vel = (joints[:, -1:] - joints[:, -2:-1]) / dt
vel = torch.cat([init_vel, middle_vel, final_vel], dim=1)
position = joints
rot_matrices = rc.axis_angle_to_matrix(motion_tensor.reshape(bs, n, 55, 3))
rot6d = rc.matrix_to_rotation_6d(rot_matrices).reshape(bs, n, 55, 6)
init_vel_ang = (motion_tensor[:, 1:2] - motion_tensor[:, 0:1]) / dt
middle_vel_ang = (motion_tensor[:, 2:] - motion_tensor[:, :-2]) / (2 * dt)
final_vel_ang = (motion_tensor[:, -1:] - motion_tensor[:, -2:-1]) / dt
angular_velocity = torch.cat([init_vel_ang, middle_vel_ang, final_vel_ang], dim=1).reshape(bs, n, 55, 3)
rep15d = torch.cat([position, vel, rot6d, angular_velocity], dim=3).reshape(bs, n, 55 * 15)
return {
"position": position,
"velocity": vel,
"rotation": rot6d,
"axis_angle": motion_tensor,
"angular_velocity": angular_velocity,
"rep15d": rep15d,
}
def get_motion_rep_numpy(poses_np, pose_fps=30, device="cuda", expressions=None, expression_only=False, betas=None):
# motion["poses"] is expected to be numpy array of shape (n, 165)
# (n, 55*3), axis-angle for 55 joints
global smplx_model
smplx_model = smplx_model.to(device)
n = poses_np.shape[0]
# Convert numpy to torch tensor for SMPL-X forward pass
poses_ts = torch.from_numpy(poses_np).float().to(device).unsqueeze(0) # (1, n, 165)
poses_ts_reshaped = poses_ts.reshape(-1, 165) # (n, 165)
betas = torch.zeros(n, 300, device=device) if betas is None else torch.from_numpy(betas).to(device).unsqueeze(0).repeat(n, 1)
if expressions is not None and expression_only:
# print("xx")
expressions = torch.from_numpy(expressions).float().to(device)
output = smplx_model(
betas=betas,
transl=torch.zeros(n, 3, device=device),
expression=expressions,
jaw_pose=poses_ts_reshaped[:, 22 * 3:23 * 3],
global_orient=torch.zeros(n, 3, device=device),
body_pose=torch.zeros(n, 21*3, device=device),
left_hand_pose=torch.zeros(n, 15*3, device=device),
right_hand_pose=torch.zeros(n, 15*3, device=device),
return_joints=True,
leye_pose=torch.zeros(n, 3, device=device),
reye_pose=torch.zeros(n, 3, device=device),
)
joints = output["vertices"].detach().cpu().numpy().reshape(n, -1)
return {"vertices": joints}
# Run smplx model to get joints
output = smplx_model(
betas=betas,
transl=torch.zeros(n, 3, device=device),
expression=torch.zeros(n, 100, device=device),
jaw_pose=torch.zeros(n, 3, device=device),
global_orient=torch.zeros(n, 3, device=device),
body_pose=poses_ts_reshaped[:, 3:21 * 3 + 3],
left_hand_pose=poses_ts_reshaped[:, 25 * 3:40 * 3],
right_hand_pose=poses_ts_reshaped[:, 40 * 3:55 * 3],
return_joints=True,
leye_pose=torch.zeros(n, 3, device=device),
reye_pose=torch.zeros(n, 3, device=device),
)
joints = output["joints"].detach().cpu().numpy().reshape(n, 127, 3)[:, :55, :]
dt = 1 / pose_fps
# Compute linear velocity
init_vel = (joints[1:2] - joints[0:1]) / dt
middle_vel = (joints[2:] - joints[:-2]) / (2 * dt)
final_vel = (joints[-1:] - joints[-2:-1]) / dt
vel = np.concatenate([init_vel, middle_vel, final_vel], axis=0)
position = joints
# Compute rotation 6D from axis-angle
poses_ts_reshaped_aa = poses_ts.reshape(1, n, 55, 3)
rot_matrices = rc.axis_angle_to_matrix(poses_ts_reshaped_aa)[0] # (n, 55, 3, 3)
rot6d = rc.matrix_to_rotation_6d(rot_matrices).reshape(n, 55, 6).cpu().numpy()
# Compute angular velocity
init_vel_ang = (poses_np[1:2] - poses_np[0:1]) / dt
middle_vel_ang = (poses_np[2:] - poses_np[:-2]) / (2 * dt)
final_vel_ang = (poses_np[-1:] - poses_np[-2:-1]) / dt
angular_velocity = np.concatenate([init_vel_ang, middle_vel_ang, final_vel_ang], axis=0).reshape(n, 55, 3)
# rep15d: position(55*3), vel(55*3), rot6d(55*6), angular_velocity(55*3) => total 55*(3+3+6+3)=55*15
rep15d = np.concatenate([position, vel, rot6d, angular_velocity], axis=2).reshape(n, 55 * 15)
return {
"position": position,
"velocity": vel,
"rotation": rot6d,
"axis_angle": poses_np,
"angular_velocity": angular_velocity,
"rep15d": rep15d,
}