Spaces:
Sleeping
Sleeping
import smplx | |
import torch | |
import numpy as np | |
from . import rotation_conversions as rc | |
import os | |
import wget | |
download_path = "./emage_evaltools/" | |
smplx_model_dir = os.path.join(download_path, "smplx_models", "smplx") | |
if not os.path.exists(smplx_model_dir): | |
smplx_model_file_path = os.path.join(smplx_model_dir, "SMPLX_NEUTRAL_2020.npz") | |
os.makedirs(smplx_model_dir, exist_ok=True) | |
if not os.path.exists(smplx_model_file_path): | |
print(f"Downloading {smplx_model_file_path}") | |
wget.download( | |
"https://huggingface.co/spaces/H-Liu1997/EMAGE/resolve/main/EMAGE/smplx_models/smplx/SMPLX_NEUTRAL_2020.npz", | |
smplx_model_file_path, | |
) | |
smplx_model = smplx.create( | |
"./emage_evaltools/smplx_models/", | |
model_type='smplx', | |
gender='NEUTRAL_2020', | |
use_face_contour=False, | |
num_betas=300, | |
num_expression_coeffs=100, | |
ext='npz', | |
use_pca=False, | |
).eval() | |
def get_motion_rep_tensor(motion_tensor, pose_fps=30, device="cuda", betas=None): | |
global smplx_model | |
smplx_model = smplx_model.to(device) | |
bs, n, _ = motion_tensor.shape | |
motion_tensor = motion_tensor.float().to(device) | |
motion_tensor_reshaped = motion_tensor.reshape(bs * n, 165) | |
betas = torch.zeros(n, 300, device=device) if betas is None else betas.to(device).unsqueeze(0).repeat(n, 1) | |
output = smplx_model( | |
betas=torch.zeros(bs * n, 300, device=device), | |
transl=torch.zeros(bs * n, 3, device=device), | |
expression=torch.zeros(bs * n, 100, device=device), | |
jaw_pose=torch.zeros(bs * n, 3, device=device), | |
global_orient=torch.zeros(bs * n, 3, device=device), | |
body_pose=motion_tensor_reshaped[:, 3:21 * 3 + 3], | |
left_hand_pose=motion_tensor_reshaped[:, 25 * 3:40 * 3], | |
right_hand_pose=motion_tensor_reshaped[:, 40 * 3:55 * 3], | |
return_joints=True, | |
leye_pose=torch.zeros(bs * n, 3, device=device), | |
reye_pose=torch.zeros(bs * n, 3, device=device), | |
) | |
joints = output['joints'].reshape(bs, n, 127, 3)[:, :, :55, :] | |
dt = 1 / pose_fps | |
init_vel = (joints[:, 1:2] - joints[:, 0:1]) / dt | |
middle_vel = (joints[:, 2:] - joints[:, :-2]) / (2 * dt) | |
final_vel = (joints[:, -1:] - joints[:, -2:-1]) / dt | |
vel = torch.cat([init_vel, middle_vel, final_vel], dim=1) | |
position = joints | |
rot_matrices = rc.axis_angle_to_matrix(motion_tensor.reshape(bs, n, 55, 3)) | |
rot6d = rc.matrix_to_rotation_6d(rot_matrices).reshape(bs, n, 55, 6) | |
init_vel_ang = (motion_tensor[:, 1:2] - motion_tensor[:, 0:1]) / dt | |
middle_vel_ang = (motion_tensor[:, 2:] - motion_tensor[:, :-2]) / (2 * dt) | |
final_vel_ang = (motion_tensor[:, -1:] - motion_tensor[:, -2:-1]) / dt | |
angular_velocity = torch.cat([init_vel_ang, middle_vel_ang, final_vel_ang], dim=1).reshape(bs, n, 55, 3) | |
rep15d = torch.cat([position, vel, rot6d, angular_velocity], dim=3).reshape(bs, n, 55 * 15) | |
return { | |
"position": position, | |
"velocity": vel, | |
"rotation": rot6d, | |
"axis_angle": motion_tensor, | |
"angular_velocity": angular_velocity, | |
"rep15d": rep15d, | |
} | |
def get_motion_rep_numpy(poses_np, pose_fps=30, device="cuda", expressions=None, expression_only=False, betas=None): | |
# motion["poses"] is expected to be numpy array of shape (n, 165) | |
# (n, 55*3), axis-angle for 55 joints | |
global smplx_model | |
smplx_model = smplx_model.to(device) | |
n = poses_np.shape[0] | |
# Convert numpy to torch tensor for SMPL-X forward pass | |
poses_ts = torch.from_numpy(poses_np).float().to(device).unsqueeze(0) # (1, n, 165) | |
poses_ts_reshaped = poses_ts.reshape(-1, 165) # (n, 165) | |
betas = torch.zeros(n, 300, device=device) if betas is None else torch.from_numpy(betas).to(device).unsqueeze(0).repeat(n, 1) | |
if expressions is not None and expression_only: | |
# print("xx") | |
expressions = torch.from_numpy(expressions).float().to(device) | |
output = smplx_model( | |
betas=betas, | |
transl=torch.zeros(n, 3, device=device), | |
expression=expressions, | |
jaw_pose=poses_ts_reshaped[:, 22 * 3:23 * 3], | |
global_orient=torch.zeros(n, 3, device=device), | |
body_pose=torch.zeros(n, 21*3, device=device), | |
left_hand_pose=torch.zeros(n, 15*3, device=device), | |
right_hand_pose=torch.zeros(n, 15*3, device=device), | |
return_joints=True, | |
leye_pose=torch.zeros(n, 3, device=device), | |
reye_pose=torch.zeros(n, 3, device=device), | |
) | |
joints = output["vertices"].detach().cpu().numpy().reshape(n, -1) | |
return {"vertices": joints} | |
# Run smplx model to get joints | |
output = smplx_model( | |
betas=betas, | |
transl=torch.zeros(n, 3, device=device), | |
expression=torch.zeros(n, 100, device=device), | |
jaw_pose=torch.zeros(n, 3, device=device), | |
global_orient=torch.zeros(n, 3, device=device), | |
body_pose=poses_ts_reshaped[:, 3:21 * 3 + 3], | |
left_hand_pose=poses_ts_reshaped[:, 25 * 3:40 * 3], | |
right_hand_pose=poses_ts_reshaped[:, 40 * 3:55 * 3], | |
return_joints=True, | |
leye_pose=torch.zeros(n, 3, device=device), | |
reye_pose=torch.zeros(n, 3, device=device), | |
) | |
joints = output["joints"].detach().cpu().numpy().reshape(n, 127, 3)[:, :55, :] | |
dt = 1 / pose_fps | |
# Compute linear velocity | |
init_vel = (joints[1:2] - joints[0:1]) / dt | |
middle_vel = (joints[2:] - joints[:-2]) / (2 * dt) | |
final_vel = (joints[-1:] - joints[-2:-1]) / dt | |
vel = np.concatenate([init_vel, middle_vel, final_vel], axis=0) | |
position = joints | |
# Compute rotation 6D from axis-angle | |
poses_ts_reshaped_aa = poses_ts.reshape(1, n, 55, 3) | |
rot_matrices = rc.axis_angle_to_matrix(poses_ts_reshaped_aa)[0] # (n, 55, 3, 3) | |
rot6d = rc.matrix_to_rotation_6d(rot_matrices).reshape(n, 55, 6).cpu().numpy() | |
# Compute angular velocity | |
init_vel_ang = (poses_np[1:2] - poses_np[0:1]) / dt | |
middle_vel_ang = (poses_np[2:] - poses_np[:-2]) / (2 * dt) | |
final_vel_ang = (poses_np[-1:] - poses_np[-2:-1]) / dt | |
angular_velocity = np.concatenate([init_vel_ang, middle_vel_ang, final_vel_ang], axis=0).reshape(n, 55, 3) | |
# rep15d: position(55*3), vel(55*3), rot6d(55*6), angular_velocity(55*3) => total 55*(3+3+6+3)=55*15 | |
rep15d = np.concatenate([position, vel, rot6d, angular_velocity], axis=2).reshape(n, 55 * 15) | |
return { | |
"position": position, | |
"velocity": vel, | |
"rotation": rot6d, | |
"axis_angle": poses_np, | |
"angular_velocity": angular_velocity, | |
"rep15d": rep15d, | |
} | |