From 6cd40b7d8e6bcdcb3f205132b4d8772292c3f49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laure=CE=B7t?= Date: Fri, 13 Jan 2023 11:23:29 +0100 Subject: [PATCH] feat: POC true perspective transform --- src/bodyparts/head.py | 53 +++++++++++++++++++++++++++---------------- src/environment.py | 39 +++++++++++++++++++++++-------- 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/src/bodyparts/head.py b/src/bodyparts/head.py index 649fbb6..fb379d4 100644 --- a/src/bodyparts/head.py +++ b/src/bodyparts/head.py @@ -17,7 +17,7 @@ class Head: self.env = env self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED) self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( + self.bounding_box = np.array( [ [0, 0], [self.image.shape[1], 0], @@ -31,24 +31,38 @@ class Head: """Draw the head on the screen.""" # compute position if self.env.results.multi_face_landmarks: + + x = self.env.x - self.env.model_points[0] + y = self.env.y - self.env.model_points[0] + for face_landmarks in self.env.results.multi_face_landmarks: head_box = np.array( [ - 100 * self.ratio * self.env.x + 100 * self.env.y + 80 * self.env.z, - 100 * self.ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z, - 100 * self.ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z, - 100 * self.ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z, + self.ratio * x + y + self.env.model_points[0], + self.ratio * -x + y + self.env.model_points[0], + self.ratio * -x + -y + self.env.model_points[0], + self.ratio * x + -y + self.env.model_points[0], ] - )[:, :2] + ) - self.translated_head_box = ( - head_box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) + (box, _) = cv2.projectPoints( + head_box, + self.env.mp_rotation_vector, + self.env.mp_translation_vector, + self.env.camera_matrix, + self.env.dist_coeff, + ) + + a, b, c, d = box.squeeze().astype(int) + cv2.line(self.env.frame, a, b, (255, 255, 255), 2) + cv2.line(self.env.frame, b, c, (255, 255, 255), 2) + cv2.line(self.env.frame, c, d, (255, 255, 255), 2) + cv2.line(self.env.frame, d, a, (255, 255, 255), 2) # get perspective transform transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_head_box, + self.bounding_box, + box.astype(np.float32), ) # apply perspective transform to image @@ -64,11 +78,12 @@ class Head: def draw_debug(self) -> None: """Draw debug information on the screen.""" # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_head_box[i].astype(np.int32)), - tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) + pass + # for i in range(4): + # cv2.line( + # self.env.frame, + # tuple(self.translated_head_box[i].astype(np.int32)), + # tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)), + # (255, 255, 255), + # 2, + # ) diff --git a/src/environment.py b/src/environment.py index 2f16647..cac0661 100644 --- a/src/environment.py +++ b/src/environment.py @@ -3,6 +3,7 @@ import logging import cv2 import mediapipe as mp import numpy as np +from mediapipe.framework.formats import landmark_pb2 from bodyparts import ( Crown, @@ -28,8 +29,7 @@ points_idx = [33, 263, 61, 291, 199] points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis] points_idx = list(set(points_idx)) points_idx.sort() - -dist_coeff = np.zeros((4, 1)) +print(points_idx) class Environment: @@ -59,7 +59,7 @@ class Environment: self.body_parts = [ # LeftEar(self), # RightEar(self), - # Head(self), + Head(self), # RightMoustache(self), # LeftMoustache(self), # LeftEye(self), @@ -88,6 +88,8 @@ class Environment: dtype="double", ) + self.dist_coeff = np.zeros((4, 1)) + self.refine_landmarks = True def start(self) -> None: @@ -170,20 +172,27 @@ class Environment: self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3]) self.mp_translation_vector = pose_transform_mat[:3, 3, None] + self.pose_transform_mat = pose_transform_mat + def draw_axis(self) -> None: """Draw the face axis on the frame.""" nose_tip = self.model_points[0] - nose_tip_extended = 2.5 * self.model_points[0] - (nose_pointer2D, jacobian) = cv2.projectPoints( - np.array([nose_tip, nose_tip_extended]), + self.x = self.model_points[0] + [7, 0, 0] + self.y = self.model_points[0] + [0, 7, 0] + self.z = self.model_points[0] + [0, 0, 7] + + (nose_pointer2D, _) = cv2.projectPoints( + np.array([nose_tip, self.x, self.y, self.z]), self.mp_rotation_vector, self.mp_translation_vector, self.camera_matrix, - dist_coeff, + self.dist_coeff, ) - nose_tip_2D, nose_tip_2D_extended = nose_pointer2D.squeeze().astype(int) - cv2.line(self.frame, nose_tip_2D, nose_tip_2D_extended, (255, 0, 0), 2) + nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointer2D.squeeze().astype(int) + cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2) + cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2) + cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2) def draw_keypoints(self) -> None: """Draw the keypoints on the screen.""" @@ -193,7 +202,17 @@ class Environment: self.mp_drawing.draw_landmarks( self.frame, face_landmarks, - landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 0, 0), + landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0), + ) + + # draw subset of landmarks + landmark_subset = landmark_pb2.NormalizedLandmarkList( + landmark=[face_landmarks.landmark[i] for i in points_idx] + ) + self.mp_drawing.draw_landmarks( + self.frame, + landmark_subset, + landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 2, 0), ) def draw_avatar(self) -> None: