update joints

bodymocap/body_mocap_api.py

@@ -112,7 +112,7 @@ def regress(self, img_original, body_bbox_list):
                 pred_output['pred_joints_img'] = pred_joints_vis_img # SMPL joints in image space
 
                 pred_aa_tensor = gu.rotation_matrix_to_angle_axis(pred_rotmat.detach().cpu()[0])
-                pred_output['pred_body_pose'] = pred_aa_tensor.cpu().numpy().reshape(1, 72)
+                pred_output['pred_body_pose'] = pred_aa_tensor.cpu().numpy().reshape(1, 72) # (1, 72)
 
                 pred_output['pred_rotmat'] = pred_rotmat.detach().cpu().numpy() # (1, 24, 3, 3)
                 pred_output['pred_betas'] = pred_betas.detach().cpu().numpy() # (1, 10)

bodymocap/models/smpl.py

@@ -77,15 +77,14 @@ def forward(self, *args, **kwargs):
         extra_joints = vertices2joints(self.J_regressor_extra, smpl_output.vertices)
         # extra_joints = vertices2joints(self.J_regressor_extra, smpl_output.vertices[:,:6890])   *0      #TODO: implement this correctly
 
-
         #SMPL-X Joint order: https://docs.google.com/spreadsheets/d/1_1dLdaX-sbMkCKr_JzJW_RZCpwBwd7rcKkWT_VgAQ_0/edit#gid=0
-        smplx_to_smpl = list(range(0,22)) + [28,43] + list(range(55,76))        #28 left middle finger , 43: right middle finger 1
-        smpl_joints = smpl_output.joints[:,smplx_to_smpl,:]       #Convert SMPL-X to SMPL     127 ->45
-        joints = torch.cat([smpl_joints, extra_joints], dim=1)               #[N, 127, 3]->[N, 45, 3]  + [N, 9, 3]        #SMPL-X has more joints. should convert 45
+        smplx_to_smpl = list(range(0,22)) + [28,43] + list(range(55,76)) # 28 left middle finger , 43: right middle finger 1
+        smpl_joints = smpl_output.joints[:,smplx_to_smpl,:] # Convert SMPL-X to SMPL     127 ->45
+        joints = torch.cat([smpl_joints, extra_joints], dim=1) # [N, 127, 3]->[N, 45, 3]  + [N, 9, 3]  # SMPL-X has more joints. should convert 45
         joints = joints[:, self.joint_map, :]     
 
         # Hand joints
-        smplx_hand_to_panoptic = [0,   13,14,15,16, 1,2,3,17, 4,5,6,18, 10,11,12,19, 7,8,9,20] #Wrist Thumb to Pinky
+        smplx_hand_to_panoptic = [0, 13,14,15,16, 1,2,3,17, 4,5,6,18, 10,11,12,19, 7,8,9,20] #Wrist Thumb to Pinky
 
         smplx_lhand =  [20] + list(range(25,40)) + list(range(66, 71))         #20 for left wrist. 20 finger joints
         lhand_joints = smpl_output.joints[:,smplx_lhand, :]      #(N,21,3)

demo/demo_bodymocap.py

@@ -19,6 +19,7 @@
 import mocap_utils.general_utils as gnu
 from mocap_utils.timer import Timer
 
+import renderer.image_utils as imu
 from renderer.viewer2D import ImShow
 
 def run_body_mocap(args, body_bbox_detector, body_mocap, visualizer):
@@ -120,7 +121,7 @@ def run_body_mocap(args, body_bbox_detector, body_mocap, visualizer):
             img_original_bgr,
             pred_mesh_list = pred_mesh_list, 
             body_bbox_list = body_bbox_list)
-
+        
         # show result in the screen
         if not args.no_display:
             res_img = res_img.astype(np.uint8)

demo/demo_frankmocap.py

@@ -24,6 +24,7 @@
 from handmocap.hand_bbox_detector import HandBboxDetector
 from integration.copy_and_paste import integration_copy_paste
 
+import renderer.image_utils as imu
 from renderer.viewer2D import ImShow
 
 
@@ -193,7 +194,7 @@ def run_frank_mocap(args, bbox_detector, body_mocap, hand_mocap, visualizer):
             pred_mesh_list = pred_mesh_list,
             body_bbox_list = body_bbox_list,
             hand_bbox_list = hand_bbox_list)
-
+        
        # show result in the screen
         if not args.no_display:
             res_img = res_img.astype(np.uint8)

demo/demo_handmocap.py

@@ -14,6 +14,7 @@
 
 from handmocap.hand_mocap_api import HandMocap
 from handmocap.hand_bbox_detector import HandBboxDetector
+
 import renderer.image_utils as imu
 from renderer.viewer2D import ImShow
 import time

handmocap/hand_mocap_api.py

@@ -194,9 +194,9 @@ def regress(self, img_original, hand_bbox_list, add_margin=False):
                         pred_output[hand_type]['img_cropped'] = img_cropped
 
                         # pred hand pose & shape params & hand joints 3d
-                        pred_output[hand_type]['pred_hand_pose'] = pred_pose
+                        pred_output[hand_type]['pred_hand_pose'] = pred_pose # (1, 48): (1, 3) for hand rotation, (1, 45) for finger pose.
                         pred_output[hand_type]['pred_hand_betas'] = pred_res['pred_shape_params'] # (1, 10)
-                        
+
                         #Convert vertices into bbox & image space
                         cam_scale = cam[0]
                         cam_trans = cam[1:]

handmocap/hand_modules/h3dw_model.py

@@ -32,7 +32,7 @@ def extract_hand_output(output, hand_type, hand_info, top_finger_joints_type='av
         wrist_idx, hand_start_idx, middle_finger_idx = 20, 25, 28
     else:
         wrist_idx, hand_start_idx, middle_finger_idx = 21, 40, 43
-    
+
     vertices = output.vertices
     joints = output.joints
     vertices_shift = vertices - joints[:, hand_start_idx:hand_start_idx+1, :]
@@ -44,35 +44,13 @@ def extract_hand_output(output, hand_type, hand_info, top_finger_joints_type='av
     hand_verts = vertices[:, hand_verts_idx, :]
     hand_verts_shift = hand_verts - joints[:, hand_start_idx:hand_start_idx+1, :]
 
-    hand_joints = torch.cat((joints[:, wrist_idx:wrist_idx+1, :], 
-        joints[:, hand_start_idx:hand_start_idx+15, :] ), dim=1)
-
-    # add top hand joints
-    if len(top_finger_joints_type) > 0:
-        if top_finger_joints_type in ['long', 'manual']:
-            key = f'{hand_type}_top_finger_{top_finger_joints_type}_vert_idx'
-            top_joint_vert_idx = hand_info[key]
-            hand_joints = torch.cat((hand_joints, vertices[:, top_joint_vert_idx, :]), dim=1)
-        else:
-            assert top_finger_joints_type == 'ave'
-            key1 = f'{hand_type}_top_finger_{top_finger_joints_type}_vert_idx'
-            key2 = f'{hand_type}_top_finger_{top_finger_joints_type}_vert_weight'
-            top_joint_vert_idxs = hand_info[key1]
-            top_joint_vert_weight = hand_info[key2]
-            bs = vertices.size(0)
-
-            for top_joint_id, selected_verts in enumerate(top_joint_vert_idxs):
-                top_finger_vert_idx = hand_verts_idx[np.array(selected_verts)]
-                top_finger_verts = vertices[:, top_finger_vert_idx]
-                # weights = torch.from_numpy(np.repeat(top_joint_vert_weight[top_joint_id]).reshape(1, -1, 1)
-                weights = top_joint_vert_weight[top_joint_id].reshape(1, -1, 1)
-                weights = np.repeat(weights, bs, axis=0)
-                weights = torch.from_numpy(weights)
-                if use_cuda:
-                    weights = weights.cuda()
-                top_joint = torch.sum((weights * top_finger_verts),dim=1).view(bs, 1, 3)
-                hand_joints = torch.cat((hand_joints, top_joint), dim=1)
-
+   # Hand joints
+    if hand_type == 'left':
+        hand_idxs =  [20] + list(range(25,40)) + list(range(66, 71)) # 20 for left wrist. 20 finger joints
+    else:
+        hand_idxs = [21] + list(range(40,55)) + list(range(71, 76)) # 21 for right wrist. 20 finger joints
+    smplx_hand_to_panoptic = [0, 13,14,15,16, 1,2,3,17, 4,5,6,18, 10,11,12,19, 7,8,9,20] 
+    hand_joints = joints[:, hand_idxs, :][:, smplx_hand_to_panoptic, :]
     hand_joints_shift = hand_joints - joints[:, hand_start_idx:hand_start_idx+1, :]
 
     output = dict(
@@ -262,13 +240,6 @@ def forward(self):
         self.pred_verts, self.pred_joints_3d = self.get_smplx_output(
             self.pred_pose_params, self.pred_shape_params)
 
-        # generate additional visualization of mesh, with constant camera params
-        # self.generate_mesh_multi_view()
-
-        # generate predicted joints 2d
-        self.pred_joints_2d = self.batch_orth_proj_idrot(
-            self.pred_joints_3d, self.pred_cam_params)
-
 
     def test(self):
         with torch.no_grad():

integration/copy_and_paste.py

@@ -141,8 +141,12 @@ def integration_copy_paste(pred_body_list, pred_hand_list, smplx_model, image_sh
 
         pred_vertices = smplx_output.vertices
         pred_vertices = pred_vertices[0].detach().cpu().numpy()
-        pred_joints_3d = smplx_output.joints
-        pred_joints_3d = pred_joints_3d[0].detach().cpu().numpy()   
+        pred_body_joints = smplx_output.joints
+        pred_body_joints = pred_body_joints[0].detach().cpu().numpy()   
+        pred_lhand_joints = smplx_output.left_hand_joints
+        pred_lhand_joints = pred_lhand_joints[0].detach().cpu().numpy()
+        pred_rhand_joints = smplx_output.right_hand_joints
+        pred_rhand_joints = pred_rhand_joints[0].detach().cpu().numpy()
 
         camScale = body_info['pred_camera'][0]
         camTrans = body_info['pred_camera'][1:]
@@ -167,6 +171,26 @@ def integration_copy_paste(pred_body_list, pred_hand_list, smplx_model, image_sh
             pred_vertices_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0])
         integral_output['pred_vertices_img'] = pred_vertices_img
 
+        # convert joints to original image space (X, Y are aligned to image)
+        pred_body_joints_bbox = convert_smpl_to_bbox(
+            pred_body_joints, camScale, camTrans)
+        pred_body_joints_img = convert_bbox_to_oriIm(
+            pred_body_joints_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0])
+        integral_output['pred_body_joints_img'] = pred_body_joints_img
+
+        # convert left /right joints to original image space (X, Y are aligned to image)
+        pred_lhand_joints_bbox = convert_smpl_to_bbox(
+            pred_lhand_joints, camScale, camTrans)
+        pred_lhand_joints_img = convert_bbox_to_oriIm(
+            pred_lhand_joints_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0])
+        integral_output['pred_lhand_joints_img'] = pred_lhand_joints_img
+
+        pred_rhand_joints_bbox = convert_smpl_to_bbox(
+            pred_rhand_joints, camScale, camTrans)
+        pred_rhand_joints_img = convert_bbox_to_oriIm(
+            pred_rhand_joints_bbox, bbox_scale_ratio, bbox_top_left, image_shape[1], image_shape[0])
+        integral_output['pred_rhand_joints_img'] = pred_rhand_joints_img
+
         # keep hand info
         r_hand_local_orient_body = body_info['pred_rotmat'][:, 21] # rot-mat
         r_hand_global_orient_body = transfer_rotation(

renderer/od_renderer.py

@@ -33,7 +33,7 @@ def render(self, verts, faces, bg_img):
 
         input_size = 500
 
-        f = 5
+        f = 10
 
         verts[:, 0] = (verts[:, 0] - input_size) / input_size
         verts[:, 1] = (verts[:, 1] - input_size) / input_size