refactor: changing projection

README.md

@@ -18,3 +18,6 @@ TODO: https://github.com/Rassibassi/mediapipeFacegeometryPython/blob/main/head_p
 -> pnp -> pose estimation -> paramètres extrinsèques
 -> + param intrasèque (supposé connu, check site mediapipe)
 -> placer dans l'espace les textures -> et projeter dans le plan image
+
+
+https://github.com/Rassibassi/mediapipeDemos

src/environment.py

@@ -15,6 +15,7 @@ from bodyparts import (
     RightEye,
     RightMoustache,
 )
+from face_geometry import PCF, get_metric_landmarks, procrustes_landmark_basis
 from utils import (
     LANDMARKS_BOTTOM_SIDE,
     LANDMARKS_LEFT_SIDE,
@@ -23,6 +24,13 @@ from utils import (
     landmark2vec,
 )
 
+points_idx = [33, 263, 61, 291, 199]
+points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis]
+points_idx = list(set(points_idx))
+points_idx.sort()
+
+dist_coeff = np.zeros((4, 1))
+
 
 class Environment:
     """The environment is the main class of the application.
@@ -49,17 +57,39 @@ class Environment:
 
         # create body parts
         self.body_parts = [
-            LeftEar(self),
+            # LeftEar(self),
-            RightEar(self),
+            # RightEar(self),
-            Head(self),
+            # Head(self),
-            RightMoustache(self),
+            # RightMoustache(self),
-            LeftMoustache(self),
+            # LeftMoustache(self),
-            LeftEye(self),
+            # LeftEye(self),
-            RightEye(self),
+            # RightEye(self),
-            Crown(self),
+            # Crown(self),
-            Mouth(self),
+            # Mouth(self),
         ]
 
+        self.pcf = PCF(
+            near=1,
+            far=10000,
+            frame_height=self.camera_height,
+            frame_width=self.camera_width,
+            fy=self.camera_width,
+        )
+
+        # pseudo camera internals
+        self.focal_length = self.camera_width
+        self.center = (self.camera_width / 2, self.camera_height / 2)
+        self.camera_matrix = np.array(
+            [
+                [self.focal_length, 0, self.center[0]],
+                [0, self.focal_length, self.center[1]],
+                [0, 0, 1],
+            ],
+            dtype="double",
+        )
+
+        self.refine_landmarks = True
+
     def start(self) -> None:
         """Start the environment."""
         while self.cam.isOpened():
@@ -82,7 +112,7 @@ class Environment:
             self.draw_axis()
 
             # draw keypoints on top of frame
-            # self.draw_keypoints()
+            self.draw_keypoints()
 
             # draw avatar
             self.draw_avatar()
@@ -99,7 +129,8 @@ class Environment:
         """Detect the keypoints on the frame."""
         with self.mp_face_mesh.FaceMesh(
             max_num_faces=1,
-            refine_landmarks=True,
+            refine_landmarks=self.refine_landmarks,
+            static_image_mode=False,
             min_detection_confidence=0.5,
             min_tracking_confidence=0.5,
         ) as face_mesh:
@@ -122,46 +153,37 @@ class Environment:
     def compute_face_axis(self) -> None:
         """Compute the face axis."""
         if self.results.multi_face_landmarks:
-            for face_landmarks in self.results.multi_face_landmarks:
+            face_landmarks = self.results.multi_face_landmarks[0]
-                # retreive points
+            landmarks = np.array([(lm.x, lm.y, lm.z) for lm in face_landmarks.landmark])
-                left_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_LEFT_SIDE])
+            landmarks = landmarks.T
-                right_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_RIGHT_SIDE])
-                bottom_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_BOTTOM_SIDE])
-                top_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_TOP_SIDE])
 
-                # compute center
+            if self.refine_landmarks:
-                self.center = np.mean(np.concatenate((left_points, right_points, bottom_points, top_points)), axis=0)
+                landmarks = landmarks[:, :468]
 
-                # compute axis
+            metric_landmarks, pose_transform_mat = get_metric_landmarks(landmarks.copy(), self.pcf)
-                self.x = np.mean(right_points - left_points, axis=0)
-                self.y = np.mean(top_points - bottom_points, axis=0)
-                self.z = np.cross(self.x, self.y)
 
-                # normalize axis
+            self.model_points = metric_landmarks[0:3, points_idx].T
-                self.x = self.x / np.linalg.norm(self.x)
+
-                self.y = self.y / np.linalg.norm(self.y)
+            # see here:
-                self.z = self.z / np.linalg.norm(self.z)
+            # https://github.com/google/mediapipe/issues/1379#issuecomment-752534379
+            pose_transform_mat[1:3, :] = -pose_transform_mat[1:3, :]
+            self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
+            self.mp_translation_vector = pose_transform_mat[:3, 3, None]
 
     def draw_axis(self) -> None:
         """Draw the face axis on the frame."""
-        for (axis, color, letter) in [
+        nose_tip = self.model_points[0]
-            (self.x, (0, 0, 255), "X"),
+        nose_tip_extended = 2.5 * self.model_points[0]
-            (self.y, (0, 255, 0), "Y"),
+        (nose_pointer2D, jacobian) = cv2.projectPoints(
-            (self.z, (255, 0, 0), "Z"),
+            np.array([nose_tip, nose_tip_extended]),
-        ]:
+            self.mp_rotation_vector,
-            # compute start and end of axis
+            self.mp_translation_vector,
-            start = (
+            self.camera_matrix,
-                int(self.center[0] * self.camera_width),
+            dist_coeff,
-                int(self.center[1] * self.camera_height),
-            )
-            end = (
-                int(self.center[0] * self.camera_width + axis[0] * 100),
-                int(self.center[1] * self.camera_height + axis[1] * 100),
         )
 
-            # draw axis + letter
+        nose_tip_2D, nose_tip_2D_extended = nose_pointer2D.squeeze().astype(int)
-            cv2.line(self.frame, start, end, color, 2)
+        cv2.line(self.frame, nose_tip_2D, nose_tip_2D_extended, (255, 0, 0), 2)
-            cv2.putText(self.frame, letter, end, cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
 
     def draw_keypoints(self) -> None:
         """Draw the keypoints on the screen."""