From 028331d707f0e275662a8618b91e939cc461f45d Mon Sep 17 00:00:00 2001 From: yinwanqi Date: Thu, 15 Jun 2023 00:51:29 +0800 Subject: [PATCH] clean the hardcode and debug code --- common/utils/preprocessing.py | 25 --------- data/AGORA/AGORA.py | 102 +--------------------------------- data/EHF/EHF.py | 1 - data/Human36M/Human36M.py | 9 --- data/MPII/MPII.py | 10 ---- data/MSCOCO/MSCOCO.py | 10 ---- data/PW3D/PW3D.py | 21 ------- data/SHAPY/SHAPY.py | 6 -- data/UBody/UBody.py | 74 +----------------------- data/humandata.py | 7 +-- main/SMPLer_X.py | 70 +---------------------- 11 files changed, 10 insertions(+), 325 deletions(-) diff --git a/common/utils/preprocessing.py b/common/utils/preprocessing.py index b880a6e..81d53b0 100644 --- a/common/utils/preprocessing.py +++ b/common/utils/preprocessing.py @@ -206,7 +206,6 @@ def process_db_coord(joint_img, joint_cam, joint_valid, do_flip, img_shape, flip joint_img[:, 1] = joint_img[:, 1] / cfg.input_img_shape[0] * cfg.output_hm_shape[1] # check truncation - # TODO joint_trunc = joint_valid * ((joint_img_original[:, 0] > 0) * (joint_img[:, 0] >= 0) * (joint_img[:, 0] < cfg.output_hm_shape[2]) * \ (joint_img_original[:, 1] > 0) *(joint_img[:, 1] >= 0) * (joint_img[:, 1] < cfg.output_hm_shape[1]) * \ (joint_img_original[:, 2] > 0) *(joint_img[:, 2] >= 0) * (joint_img[:, 2] < cfg.output_hm_shape[0])).reshape(-1, @@ -296,8 +295,6 @@ def process_human_model_output(human_model_param, cam_param, do_flip, img_shape, leye_pose=zero_pose, reye_pose=zero_pose, expression=expr) mesh_cam = output.vertices[0].numpy() joint_cam = output.joints[0].numpy()[smpl_x.joint_idx, :] - ### HARDCODE - # joint_cam_orig_ = joint_cam.copy() # apply camera exrinsic (translation) # compenstate rotation (translation from origin to root joint was not cancled) @@ -489,28 +486,6 @@ def process_human_model_output(human_model_param, cam_param, do_flip, img_shape, pose = pose.reshape(-1) expr = expr.numpy().reshape(-1) - ### ### HARDCODE temp save vis for debug - # POSE: torch.cat((root_pose, body_pose, lhand_pose, rhand_pose, jaw_pose)) - # pose_ = pose.numpy().copy() - - # root_pose = torch.FloatTensor(pose[0:3]).view(1, 3) # (1,3) - # body_pose = torch.FloatTensor(pose[3:66]).view(-1, 3) # (21,3) - # lhand_pose = torch.FloatTensor(pose[66:111]).view(-1, 3) # (15,3) - # rhand_pose = torch.FloatTensor(pose[111:156]).view(-1, 3) # (15,3) - # jaw_pose = torch.FloatTensor(pose[156:159]).view(-1, 3) # (1,3) - # shape = torch.FloatTensor(shape).view(1, -1) # SMPLX shape parameter - # expr = torch.FloatTensor(expr).view(1, -1) # SMPLX expression parameter - # trans = torch.FloatTensor(trans).view(1, -1) # translation vector - - # with torch.no_grad(): - # output = smpl_x.layer[gender](betas=shape, body_pose=body_pose.view(1, -1), global_orient=root_pose, - # transl=zero_pose, left_hand_pose=lhand_pose.view(1, -1), - # right_hand_pose=rhand_pose.view(1, -1), jaw_pose=jaw_pose.view(1, -1), - # leye_pose=zero_pose, reye_pose=zero_pose, expression=expr) - # mesh_rot = output.vertices[0].numpy() - # joint_rot = output.joints[0].numpy()[smpl_x.joint_idx, :] - # return mesh_rot, joint_cam, joint_trunc, pose, shape, expr, rotation_valid, coord_valid, expr_valid, mesh_cam_orig, joint_cam_orig_ - return joint_img, joint_cam, joint_trunc, pose, shape, expr, rotation_valid, coord_valid, expr_valid, mesh_cam_orig elif human_model_type == 'smpl': pose = pose.reshape(-1) diff --git a/data/AGORA/AGORA.py b/data/AGORA/AGORA.py index acfd53b..a8a0a51 100644 --- a/data/AGORA/AGORA.py +++ b/data/AGORA/AGORA.py @@ -157,14 +157,9 @@ class AGORA(torch.utils.data.Dataset): else: db = COCO(osp.join(self.data_path, 'AGORA_validation.json')) - ### HARDCODE vis for debug - # count = 0 i = 0 for aid in tqdm.tqdm(list(db.anns.keys())): - # if count > 50: - # continue - # count += 1 - + i += 1 if self.data_split == 'train' and i % getattr(cfg, 'AGORA_train_sample_interval', 1) != 0: continue @@ -418,8 +413,6 @@ class AGORA(torch.utils.data.Dataset): joint_img[:, 1] = joint_img[:, 1] / 2160 * self.resolution[0] with open(data['joints_3d_path']) as f: joint_cam = np.array(json.load(f)).reshape(-1, 3) - ### HARDCODE vis for debug - # joint_cam_orig = joint_cam.copy() with open(data['smplx_param_path'], 'rb') as f: smplx_param = pickle.load(f, encoding='latin1') @@ -438,30 +431,6 @@ class AGORA(torch.utils.data.Dataset): rhand_bbox_size = rhand_bbox[1] - rhand_bbox[0]; face_bbox_size = face_bbox[1] - face_bbox[0]; - """ - # for debug - _img = img.numpy().transpose(1,2,0)[:,:,::-1].copy() * 255 - if lhand_bbox_valid: - _tmp = lhand_bbox.copy().reshape(2,2) - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - cv2.rectangle(_img, (int(_tmp[0,0]), int(_tmp[0,1])), (int(_tmp[1,0]), int(_tmp[1,1])), (255,0,0), 3) - cv2.imwrite('agora_' + str(idx) + '_lhand.jpg', _img) - if rhand_bbox_valid: - _tmp = rhand_bbox.copy().reshape(2,2) - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - cv2.rectangle(_img, (int(_tmp[0,0]), int(_tmp[0,1])), (int(_tmp[1,0]), int(_tmp[1,1])), (255,0,0), 3) - cv2.imwrite('agora_' + str(idx) + '_rhand.jpg', _img) - if face_bbox_valid: - _tmp = face_bbox.copy().reshape(2,2) - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - cv2.rectangle(_img, (int(_tmp[0,0]), int(_tmp[0,1])), (int(_tmp[1,0]), int(_tmp[1,1])), (255,0,0), 3) - cv2.imwrite('agora_' + str(idx) + '_face.jpg', _img) - #cv2.imwrite('agora_' + str(idx) + '.jpg', _img) - """ - # coordinates joint_cam = joint_cam - joint_cam[self.joint_set['root_joint_idx'], None, :] # root-relative joint_cam[self.joint_set['joint_part']['lhand'], :] = joint_cam[self.joint_set['joint_part']['lhand'], @@ -507,27 +476,6 @@ class AGORA(torch.utils.data.Dataset): joint_cam_wo_ra[smpl_x.joint_part['face'], :] = joint_cam_wo_ra[smpl_x.joint_part['face'], :] \ + joint_cam_wo_ra[smpl_x.neck_idx, None,: ] # face root-relative - - """ - # for debug - _tmp = joint_img.copy() - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - _img = img.numpy().transpose(1,2,0)[:,:,::-1] * 255 - _img = vis_keypoints(_img.copy(), _tmp) - cv2.imwrite('agora_' + str(idx) + '.jpg', _img) - """ - - """ - # for debug - _tmp = joint_cam.copy()[:,:2] - _tmp[:,0] = _tmp[:,0] / (cfg.body_3d_size / 2) * cfg.input_img_shape[1] + cfg.input_img_shape[1]/2 - _tmp[:,1] = _tmp[:,1] / (cfg.body_3d_size / 2) * cfg.input_img_shape[0] + cfg.input_img_shape[0]/2 - _img = np.zeros((cfg.input_img_shape[0], cfg.input_img_shape[1], 3), dtype=np.float32) - _img = vis_keypoints(_img.copy(), _tmp) - cv2.imwrite('agora_' + str(idx) + '_cam.jpg', _img) - """ - # smplx parameters root_pose = np.array(smplx_param['global_orient'], dtype=np.float32).reshape( -1) # rotation to world coordinate @@ -553,9 +501,6 @@ class AGORA(torch.utils.data.Dataset): 'trans': trans} _, _, _, smplx_pose, smplx_shape, smplx_expr, smplx_pose_valid, _, smplx_expr_valid, _ = process_human_model_output( smplx_param, cam_param, do_flip, img_shape, img2bb_trans, rot, 'smplx') - ### HARDCODE vis for debug - # mesh_rot_, joint_cam_, _, smplx_pose, smplx_shape, smplx_expr, smplx_pose_valid, _, smplx_expr_valid, mesh_orig, joint_cam_orig_ = process_human_model_output( - # smplx_param, cam_param, do_flip, img_shape, img2bb_trans, rot, 'smplx') smplx_pose_valid = np.tile(smplx_pose_valid[:, None], (1, 3)).reshape(-1) if not getattr(cfg, 'agora_valid_root_pose', False): @@ -573,8 +518,6 @@ class AGORA(torch.utils.data.Dataset): 'smplx_pose_valid': smplx_pose_valid, 'smplx_shape_valid': float(smplx_shape_valid), 'smplx_expr_valid': float(smplx_expr_valid), 'is_3D': float(True), 'lhand_bbox_valid': lhand_bbox_valid, 'rhand_bbox_valid': rhand_bbox_valid, 'face_bbox_valid': face_bbox_valid} - ### HARDCODE vis for debug - # 'gt_3d_path': data['joints_3d_path'], 'smplx_path': data['smplx_param_path'], 'id': idx} return inputs, targets, meta_info else: # load crop and resize information (for the 4K setting) @@ -678,41 +621,8 @@ class AGORA(torch.utils.data.Dataset): eval_result['pa_mpvpe_face'].append( np.sqrt(np.sum((mesh_out_face_align - mesh_gt_face) ** 2, 1)).mean() * 1000) - ### HARDCODE + if vis: - - # from utils.vis import vis_keypoints, vis_mesh, save_obj, render_mesh - # # img = (out['img'].transpose(1,2,0)[:,:,::-1] * 255).copy() - # # joint_img = out['joint_img'].copy() - # # joint_img[:,0] = joint_img[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - # # joint_img[:,1] = joint_img[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - # # for j in range(len(joint_img)): - # # cv2.circle(img, (int(joint_img[j][0]), int(joint_img[j][1])), 3, (0,0,255), -1) - # # cv2.imwrite(str(cur_sample_idx + n) + '.jpg', img) - - # img_path = annot['img_path'] - # img_id = img_path.split('/')[-1][:-4] - # ann_id = 0 - # # ann_id = annot['ann_id'] - # img = load_img(img_path)[:, :, ::-1] - # bbox = annot['bbox'] - # focal = list(cfg.focal) - # princpt = list(cfg.princpt) - # focal[0] = focal[0] / cfg.input_body_shape[1] * bbox[2] - # focal[1] = focal[1] / cfg.input_body_shape[0] * bbox[3] - # princpt[0] = princpt[0] / cfg.input_body_shape[1] * bbox[2] + bbox[0] - # princpt[1] = princpt[1] / cfg.input_body_shape[0] * bbox[3] + bbox[1] - # img = render_mesh(img, out['smplx_mesh_cam'], smpl_x.face, {'focal': focal, 'princpt': princpt}, mesh_as_vertices=True) - # # img = cv2.resize(img, (512,512)) - # cv2.imwrite(osp.join(vis_save_dir, img_id + '_' + str(ann_id) + '.jpg'), img) - - # vis_mesh_out = out['smplx_mesh_cam'] - # vis_mesh_out = vis_mesh_out - np.dot(smpl_x.layer['neutral'].J_regressor, vis_mesh_out)[ - # smpl_x.J_regressor_idx['pelvis'], None, :] - # # vis_mesh_gt = out['smplx_mesh_cam_target'] - # # vis_mesh_gt = vis_mesh_gt - np.dot(smpl_x.layer['neutral'].J_regressor, vis_mesh_gt)[smpl_x.J_regressor_idx['pelvis'],None,:] - # # save_obj(vis_mesh_out, smpl_x.face, osp.join(img_id + '_' + str(ann_id) + '.obj')) - # # save_obj(vis_mesh_gt, smpl_x.face, str(cur_sample_idx + n) + '_gt.obj') img_path = out['img_path'] rel_img_path = img_path.split('..')[-1] smplx_pred = {} @@ -785,14 +695,6 @@ class AGORA(torch.utils.data.Dataset): with open(osp.join(cfg.result_dir, 'predictions', save_name), 'wb') as f: pickle.dump(save_dict, f) - """ - # for debug - img_path = annot['img_path'] - img_path = osp.join(self.data_path, '3840x2160', 'test', img_path.split('/')[-1].split('_')[0] + '.png') - img = cv2.imread(img_path) - img = vis_keypoints(img.copy(), joint_proj) - cv2.imwrite(img_path.split('/')[-1], img) - """ if getattr(cfg, 'vis', False): file.close() diff --git a/data/EHF/EHF.py b/data/EHF/EHF.py index e0c7972..ef47225 100644 --- a/data/EHF/EHF.py +++ b/data/EHF/EHF.py @@ -291,7 +291,6 @@ class EHF(torch.utils.data.Dataset): smplx_pred['expression'] = out['smplx_expr'].reshape(-1,10) smplx_pred['transl'] = out['cam_trans'].reshape(-1,3) - # import pdb; pdb.set_trace() np.savez(os.path.join(cfg.vis_dir, f'{self.save_idx}.npz'), **smplx_pred) # save img path and error diff --git a/data/Human36M/Human36M.py b/data/Human36M/Human36M.py index c3da5ee..7374486 100644 --- a/data/Human36M/Human36M.py +++ b/data/Human36M/Human36M.py @@ -192,15 +192,6 @@ class Human36M(torch.utils.data.Dataset): smplx_pose_valid, smplx_joint_valid, smplx_expr_valid, smplx_mesh_cam_orig = \ process_human_model_output(smplx_param, cam_param, do_flip, img_shape, img2bb_trans, rot, 'smplx') - """ - # for debug - _tmp = joint_img.copy() - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - _img = img.numpy().transpose(1,2,0)[:,:,::-1] * 255 - _img = vis_keypoints(_img, _tmp) - cv2.imwrite('h36m_' + str(idx) + '.jpg', _img) - """ # reverse ra smplx_joint_cam_wo_ra = smplx_joint_cam.copy() smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] = smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] \ diff --git a/data/MPII/MPII.py b/data/MPII/MPII.py index 06ed739..00f94a2 100644 --- a/data/MPII/MPII.py +++ b/data/MPII/MPII.py @@ -131,16 +131,6 @@ class MPII(torch.utils.data.Dataset): smplx_joint_img, smplx_joint_cam, smplx_joint_trunc, smplx_pose, smplx_shape, smplx_expr, smplx_pose_valid, smplx_joint_valid, smplx_expr_valid, smplx_mesh_cam_orig = process_human_model_output(smplx_param['smplx_param'], smplx_param['cam_param'], do_flip, img_shape, img2bb_trans, rot, 'smplx') is_valid_fit = True - """ - # for debug - _tmp = joint_img.copy() - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - _img = img.numpy().transpose(1,2,0)[:,:,::-1] * 255 - _img = vis_keypoints(_img.copy(), _tmp) - cv2.imwrite('mpii_' + str(idx) + '.jpg', _img) - """ - else: # dummy values smplx_joint_img = np.zeros((smpl_x.joint_num,3), dtype=np.float32) diff --git a/data/MSCOCO/MSCOCO.py b/data/MSCOCO/MSCOCO.py index d34ee7e..3a84727 100644 --- a/data/MSCOCO/MSCOCO.py +++ b/data/MSCOCO/MSCOCO.py @@ -321,16 +321,6 @@ class MSCOCO(torch.utils.data.Dataset): img_shape, img2bb_trans, rot, 'smplx') is_valid_fit = True - """ - # for debug - _tmp = joint_img.copy() - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - _img = img.numpy().transpose(1,2,0)[:,:,::-1] * 255 - _img = vis_keypoints(_img, _tmp) - cv2.imwrite('coco_' + str(idx) + '.jpg', _img) - """ - else: # dummy values smplx_joint_img = np.zeros((smpl_x.joint_num, 3), dtype=np.float32) diff --git a/data/PW3D/PW3D.py b/data/PW3D/PW3D.py index d7a9a2a..ae535af 100644 --- a/data/PW3D/PW3D.py +++ b/data/PW3D/PW3D.py @@ -165,10 +165,6 @@ class PW3D(torch.utils.data.Dataset): # 'left_elbow', 'right_elbow', 'left_wrist', 'right_wrist'] joint_mapper = [1, 2, 4, 5, 7, 8, 12, 15, 16, 17, 18, 19, 20, 21] - ### Save vis for debug - # joint_gt_body_to_save = np.zeros((sample_num, len(joint_mapper), 3)) - # joint_out_body_root_align_to_save = np.zeros((sample_num, len(joint_mapper), 3)) - # joint_out_body_pa_align_to_save = np.zeros((sample_num, len(joint_mapper), 3)) for n in range(sample_num): @@ -182,11 +178,6 @@ class PW3D(torch.utils.data.Dataset): mesh_out_align = mesh_out - np.dot(smpl_x.J_regressor, mesh_out)[smpl_x.J_regressor_idx['pelvis'], None, :] \ + np.dot(smpl.joint_regressor, mesh_gt)[smpl.root_joint_idx, None, :] - # only eval point0-21 since only smpl gt is given - # joint_gt_body = np.dot(smpl.joint_regressor, mesh_gt)[:22, :] - # joint_out_body = np.dot(smpl_x.J_regressor, mesh_out)[:22, :] - # joint_out_body_root_align = np.dot(smpl_x.J_regressor, mesh_out_align)[:22, :] - # only test 14 keypoints joint_gt_body = np.dot(smpl.joint_regressor, mesh_gt)[joint_mapper, :] joint_out_body = np.dot(smpl_x.J_regressor, mesh_out)[joint_mapper, :] @@ -199,18 +190,6 @@ class PW3D(torch.utils.data.Dataset): joint_out_body_pa_align = rigid_align(joint_out_body, joint_gt_body) eval_result['pa_mpjpe_body'].append( np.sqrt(np.sum((joint_out_body_pa_align - joint_gt_body) ** 2, 1)).mean() * 1000) - - ### Save vis for debug - # joint_gt_body_to_save[n, ...] = joint_gt_body - # joint_out_body_root_align_to_save[n, ...] = joint_out_body_root_align - # joint_out_body_pa_align_to_save[n, ...] = joint_out_body_pa_align - - ### Save vis for debug - # import numpy as np - # np.save(f'./vis/val_0509_joint_gt_body.npy', joint_gt_body_to_save) - # np.save(f'./vis/val_0509_joint_out_body_root_align.npy', joint_out_body_root_align_to_save) - # np.save(f'./vis/val_0509_joint_out_body_pa_align.npy', joint_out_body_pa_align_to_save) - # import pdb; pdb.set_trace() return eval_result diff --git a/data/SHAPY/SHAPY.py b/data/SHAPY/SHAPY.py index f976606..c354039 100644 --- a/data/SHAPY/SHAPY.py +++ b/data/SHAPY/SHAPY.py @@ -96,8 +96,6 @@ class SHAPY(HumanDataset): sample_num = len(outs) eval_result = {'v2v_t_errors': [], 'point_t_errors': [], 'height': [], 'chest': [], 'waist': [], 'hips': [], 'mass': []} - # sample_num = sample_num // 10 # TODO: debug only - for n in range(sample_num): annot = annots[cur_sample_idx + n] out = outs[n] @@ -107,8 +105,6 @@ class SHAPY(HumanDataset): # compute v_shaped betas_fit = torch.tensor(betas_fit.reshape(-1, 10)).cuda() - # betas_fit = torch.tensor(out['smplx_shape_target'].reshape(-1, 10)).cuda() # TODO: debug only - # betas_fit = torch.zeros((1, 10)).cuda() # TODO: debug only output = self.smplx_layer( betas=betas_fit, body_pose=torch.zeros((1, 63)).to(betas_fit.device), @@ -122,8 +118,6 @@ class SHAPY(HumanDataset): return_verts=True ) v_shaped_fit = output.vertices.detach().cpu().numpy().squeeze() - # v_shaped_gt = v_shaped_fit # TODO: debug only - # v_shaped_fit = self.smplx_layer.forward_shape(betas=betas_fit) image_name = '/'.join(img_path.split('/')[-4:]) self.images_names.append(image_name) diff --git a/data/UBody/UBody.py b/data/UBody/UBody.py index ad07fce..45ca158 100644 --- a/data/UBody/UBody.py +++ b/data/UBody/UBody.py @@ -235,9 +235,7 @@ class UBody_Part(torch.utils.data.Dataset): video_name = file_name.split('/')[-2] if 'Trim' in video_name: video_name = video_name.split('_Trim')[0] - # if video_name in test_video_list: - # # data to use in test - # import pdb; pdb.set_trace() + if video_name not in test_video_list: continue # exclude the train video img_path = osp.join(self.img_path, file_name) if not os.path.exists(img_path): continue @@ -403,15 +401,6 @@ class UBody_Part(torch.utils.data.Dataset): smplx_cam_trans = np.array(smplx_param['smplx_param']['trans']) is_valid_fit = True - """ - # for debug - _tmp = joint_img.copy() - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - _img = img.numpy().transpose(1,2,0)[:,:,::-1] * 255 - _img = vis_keypoints(_img, _tmp) - cv2.imwrite('coco_' + str(idx) + '.jpg', _img) - """ # reverse ra smplx_joint_cam_wo_ra = smplx_joint_cam.copy() smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] = smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] \ @@ -514,25 +503,14 @@ class UBody_Part(torch.utils.data.Dataset): # smplx coordinates and parameters smplx_param = data['smplx_param'] - # if str(data['ann_id'])=='184516': - # print(data['ann_id'], smplx_param) + if smplx_param is not None: smplx_joint_img, smplx_joint_cam, smplx_joint_trunc, smplx_pose, smplx_shape, smplx_expr, smplx_pose_valid, smplx_joint_valid, smplx_expr_valid, smplx_mesh_cam_orig = process_human_model_output( smplx_param['smplx_param'], smplx_param['cam_param'], do_flip, img_shape, img2bb_trans, rot, 'smplx') is_valid_fit = True smplx_cam_trans = np.array(smplx_param['smplx_param']['trans']) - # if str(data['ann_id']) == '184516': - # print(data['ann_id'], smplx_pose) - """ - # for debug - _tmp = joint_img.copy() - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - _img = img.numpy().transpose(1,2,0)[:,:,::-1] * 255 - _img = vis_keypoints(_img, _tmp) - cv2.imwrite('coco_' + str(idx) + '.jpg', _img) - """ + # reverse ra smplx_joint_cam_wo_ra = smplx_joint_cam.copy() smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] = smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] \ @@ -755,15 +733,6 @@ class UBody(Dataset): smplx_cam_trans = np.array(smplx_param['smplx_param']['trans']) is_valid_fit = True - """ - # for debug - _tmp = joint_img.copy() - _tmp[:,0] = _tmp[:,0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] - _tmp[:,1] = _tmp[:,1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] - _img = img.numpy().transpose(1,2,0)[:,:,::-1] * 255 - _img = vis_keypoints(_img, _tmp) - cv2.imwrite('coco_' + str(idx) + '.jpg', _img) - """ # reverse ra smplx_joint_cam_wo_ra = smplx_joint_cam.copy() smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] = smplx_joint_cam_wo_ra[smpl_x.joint_part['lhand'], :] \ @@ -1026,45 +995,10 @@ class UBody(Dataset): if len(pa_mpjpe_hand)>0: eval_result['pa_mpjpe_hand'].append(np.mean(pa_mpjpe_hand)) - # data_dict = {} - # data_dict['mpvpe_all'] = eval_result['mpvpe_all'][-1] - # data_dict['mpvpe_hand'] = eval_result['mpvpe_hand'][-1] - # data_dict['mpvpe_face'] = eval_result['mpvpe_face'][-1] - # data_dict['mesh'] = mesh_out - # data_dict['mesh_gt'] = mesh_gt - vis = cfg.vis save_folder = cfg.vis_dir data_folder = os.path.join(cfg.root_dir, 'dataset', 'UBody', 'images') if vis: - # from common.utils.vis import vis_keypoints, vis_mesh, save_obj, render_mesh - # img_path = annot['img_path'] - # render_img_save_path = img_path.replace(data_folder, f'{save_folder}/render/') - # if os.path.exists(render_img_save_path): - # img = load_img(render_img_save_path)[:, :, ::-1] - # else: - # img = load_img(img_path)[:, :, ::-1] - - - # ''' for debug - # kpt_path = render_img_save_path.replace('/render/', '/keypoints/') - # kpt_img = img.copy() - # kpt_img = vis_keypoints(kpt_img, joint_proj) - # # kpt_img = vis_keypoints(kpt_img, mesh_gt_proj) - # os.makedirs(os.path.dirname(kpt_path), exist_ok=True) - # cv2.imwrite(kpt_path, kpt_img) - # ''' - - # bbox = annot['bbox'] - # focal = list(cfg.focal) - # princpt = list(cfg.princpt) - # focal[0] = focal[0] / cfg.input_body_shape[1] * bbox[2] - # focal[1] = focal[1] / cfg.input_body_shape[0] * bbox[3] - # princpt[0] = princpt[0] / cfg.input_body_shape[1] * bbox[2] + bbox[0] - # princpt[1] = princpt[1] / cfg.input_body_shape[0] * bbox[3] + bbox[1] - # img = render_mesh(img, out['smplx_mesh_cam'], smpl_x.face, {'focal': focal, 'princpt': princpt}) - # os.makedirs(os.path.dirname(render_img_save_path), exist_ok=True) - # cv2.imwrite(render_img_save_path, img) vis_save_dir = cfg.vis_dir rel_img_path = img_path.split('..')[-1] smplx_pred = {} @@ -1080,8 +1014,6 @@ class UBody(Dataset): smplx_pred['transl'] = out['gt_smplx_transl'].reshape(-1,3) smplx_pred['img_path'] = rel_img_path - - # import pdb; pdb.set_trace() npz_path = os.path.join(cfg.vis_dir, f'{cur_sample_idx + n}.npz') np.savez(npz_path, **smplx_pred) diff --git a/data/humandata.py b/data/humandata.py index 82c884e..1987129 100644 --- a/data/humandata.py +++ b/data/humandata.py @@ -287,7 +287,6 @@ class HumanDataset(torch.utils.data.Dataset): smplx_param = {k: v[i] for k, v in smplx.items()} - # TODO: set invalid if None? smplx_param['root_pose'] = smplx_param.pop('global_orient', None) smplx_param['shape'] = smplx_param.pop('betas', None) smplx_param['trans'] = smplx_param.pop('transl', np.zeros(3)) @@ -298,9 +297,8 @@ class HumanDataset(torch.utils.data.Dataset): # TODO do not fix betas, give up shape supervision if 'betas_neutral' in smplx_param: smplx_param['shape'] = smplx_param.pop('betas_neutral') - # smplx_param['shape'] = np.zeros(10, dtype=np.float32) - # # TODO fix shape of poses + # TODO fix shape of poses if self.__class__.__name__ == 'Talkshow': smplx_param['body_pose'] = smplx_param['body_pose'].reshape(21, 3) smplx_param['lhand_pose'] = smplx_param['lhand_pose'].reshape(15, 3) @@ -466,7 +464,7 @@ class HumanDataset(torch.utils.data.Dataset): targets = {'joint_img': joint_img_aug, # keypoints2d 'smplx_joint_img': joint_img_aug, #smplx_joint_img, # projected smplx if valid cam_param, else same as keypoints2d 'joint_cam': joint_cam_wo_ra, # joint_cam actually not used in any loss, # raw kps3d probably without ra - 'smplx_joint_cam': smplx_joint_cam if (dummy_cord or getattr(cfg, 'debug', False)) else joint_cam_ra, # kps3d with body, face, hand ra + 'smplx_joint_cam': smplx_joint_cam if dummy_cord else joint_cam_ra, # kps3d with body, face, hand ra 'smplx_pose': smplx_pose, 'smplx_shape': smplx_shape, 'smplx_expr': smplx_expr, @@ -680,7 +678,6 @@ class HumanDataset(torch.utils.data.Dataset): smplx_pred['transl'] = out['gt_smplx_transl'].reshape(-1,3) smplx_pred['img_path'] = rel_img_path - # import pdb; pdb.set_trace() npz_path = os.path.join(cfg.vis_dir, f'{self.save_idx}.npz') np.savez(npz_path, **smplx_pred) diff --git a/main/SMPLer_X.py b/main/SMPLer_X.py index d8bf903..46fba3f 100644 --- a/main/SMPLer_X.py +++ b/main/SMPLer_X.py @@ -62,9 +62,8 @@ class Model(nn.Module): param_net = param_bb + param_neck + param_head - print('#parameters:') - print(f'{param_bb}, {param_neck}, {param_head}, {param_net}') - # import pdb; pdb.set_trace() + # print('#parameters:') + # print(f'{param_bb}, {param_neck}, {param_head}, {param_net}') def get_camera_trans(self, cam_param): # camera translation @@ -242,12 +241,7 @@ class Model(nn.Module): loss['smplx_orient'] = self.param_loss(pose, targets['smplx_pose'], meta_info['smplx_pose_valid'])[:, :3] * smplx_orient_weight loss['smplx_pose'] = self.param_loss(pose, targets['smplx_pose'], meta_info['smplx_pose_valid']) * smplx_pose_weight - ### debug - # import numpy as np - # check = torch.isnan(loss['smplx_pose']).cpu() - # pause = np.any(check.numpy()) - # if pause: - # import pdb; pdb.set_trace() + else: loss['smplx_pose'] = self.param_loss(pose, targets['smplx_pose'], meta_info['smplx_pose_valid'])[:, 3:] * smplx_pose_weight @@ -355,55 +349,6 @@ class Model(nn.Module): loss['smplx_joint_img'] = self.coord_loss(joint_img, smpl_x.reduce_joint_set(targets['smplx_joint_img']), smpl_x.reduce_joint_set(meta_info['smplx_joint_trunc'])) * net_kps_2d_weight - ### save vis for keypoints checking - # import pdb; pdb.set_trace() - if getattr(cfg, 'debug', False): - import numpy as np - import cv2 - out = {} - datalist = cfg.trainset_humandata + cfg.trainset_2d + cfg.trainset_3d - dataset = datalist[0] - out['img'] = inputs['img'] - np.save(f'./vis/train_{dataset}.npy', targets) - np.save(f'./vis/train_{dataset}_out.npy', out) - for key in ['joint_cam', 'smplx_joint_cam']: - to_save = targets[key].cpu().detach().numpy() - np.save(f'./vis/train_{dataset}_{key}.npy', to_save) - - for sample_id in range(5): - print(sample_id) - vis_joint_img = targets['original_joint_img'][sample_id,...].cpu().detach().numpy() - vis_smplx_joint_img = targets['original_smplx_joint_img'][sample_id,...].cpu().detach().numpy() - - # get image - image = out['img'][sample_id].cpu().numpy().transpose(1, 2, 0) * 255 - image = image.astype(np.uint8).copy() - image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - - color = [(0, 0, 255), (0, 255, 0)] # R: pred, G: gt - for set_id, joint_proj in enumerate([vis_joint_img, vis_smplx_joint_img]): #, joint_2d_gt]): - th = 3 - set_id - # restore kps - joint_proj[:, 0] = joint_proj[:, 0] / cfg.output_hm_shape[2] * cfg.input_img_shape[1] # - joint_proj[:, 1] = joint_proj[:, 1] / cfg.output_hm_shape[1] * cfg.input_img_shape[0] # - # R: pred, G: gt - n_kps = joint_proj.shape[0] - - for i in range (n_kps): - # import pdb; pdb.set_trace() - kps = joint_proj[i] - # import pdb;pdb.set_trace() - image = cv2.circle(image, (int(kps[0]),int(kps[1])), radius=th, color=color[set_id], thickness=th) - - # image = cv2.rectangle(image, (int(lhand_bbox[0]),int(lhand_bbox[1])), - # (int(lhand_bbox[2]),int(lhand_bbox[3])), (0, 255, 0), thickness=th) - # image = cv2.rectangle(image, (int(rhand_bbox[0]),int(rhand_bbox[1])), - # (int(rhand_bbox[2]),int(rhand_bbox[3])), (0, 0, 255), thickness=th) - # image = cv2.rectangle(image, (int(face_bbox[0]),int(face_bbox[1])), - # (int(face_bbox[2]),int(face_bbox[3])), (255, 0, 0), thickness=th) - cv2.imwrite(f'./vis/joint_img/{dataset}_{sample_id}.jpg',image) - - import pdb;pdb.set_trace() return loss else: @@ -459,15 +404,6 @@ class Model(nn.Module): if 'gt_smplx_transl' in meta_info: out['gt_smplx_transl'] = meta_info['gt_smplx_transl'] - - ### save result for vis and debug - # import numpy as np - # # np.save('./vis/val_exp38_wo_bbox_sup_out.npy', out) - # for key in ['joint_cam', 'smplx_joint_cam']: - # to_save = targets[key].cpu().detach().numpy() - # np.save(f'./vis/val_0517_{key}.npy', to_save) - - # import pdb;pdb.set_trace() return out def init_weights(m):