refactor: changing projection

This commit is contained in:
Laureηt 2023-01-12 09:51:17 +01:00
parent fdfed4c1ff
commit 97fbc9bdb3
Signed by: Laurent
SSH key fingerprint: SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
3 changed files with 2728 additions and 44 deletions

View file

@ -18,3 +18,6 @@ TODO: https://github.com/Rassibassi/mediapipeFacegeometryPython/blob/main/head_p
-> pnp -> pose estimation -> paramètres extrinsèques
-> + param intrasèque (supposé connu, check site mediapipe)
-> placer dans l'espace les textures -> et projeter dans le plan image
https://github.com/Rassibassi/mediapipeDemos

View file

@ -15,6 +15,7 @@ from bodyparts import (
RightEye,
RightMoustache,
)
from face_geometry import PCF, get_metric_landmarks, procrustes_landmark_basis
from utils import (
LANDMARKS_BOTTOM_SIDE,
LANDMARKS_LEFT_SIDE,
@ -23,6 +24,13 @@ from utils import (
landmark2vec,
)
points_idx = [33, 263, 61, 291, 199]
points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis]
points_idx = list(set(points_idx))
points_idx.sort()
dist_coeff = np.zeros((4, 1))
class Environment:
"""The environment is the main class of the application.
@ -49,17 +57,39 @@ class Environment:
# create body parts
self.body_parts = [
LeftEar(self),
RightEar(self),
Head(self),
RightMoustache(self),
LeftMoustache(self),
LeftEye(self),
RightEye(self),
Crown(self),
Mouth(self),
# LeftEar(self),
# RightEar(self),
# Head(self),
# RightMoustache(self),
# LeftMoustache(self),
# LeftEye(self),
# RightEye(self),
# Crown(self),
# Mouth(self),
]
self.pcf = PCF(
near=1,
far=10000,
frame_height=self.camera_height,
frame_width=self.camera_width,
fy=self.camera_width,
)
# pseudo camera internals
self.focal_length = self.camera_width
self.center = (self.camera_width / 2, self.camera_height / 2)
self.camera_matrix = np.array(
[
[self.focal_length, 0, self.center[0]],
[0, self.focal_length, self.center[1]],
[0, 0, 1],
],
dtype="double",
)
self.refine_landmarks = True
def start(self) -> None:
"""Start the environment."""
while self.cam.isOpened():
@ -82,7 +112,7 @@ class Environment:
self.draw_axis()
# draw keypoints on top of frame
# self.draw_keypoints()
self.draw_keypoints()
# draw avatar
self.draw_avatar()
@ -99,7 +129,8 @@ class Environment:
"""Detect the keypoints on the frame."""
with self.mp_face_mesh.FaceMesh(
max_num_faces=1,
refine_landmarks=True,
refine_landmarks=self.refine_landmarks,
static_image_mode=False,
min_detection_confidence=0.5,
min_tracking_confidence=0.5,
) as face_mesh:
@ -122,46 +153,37 @@ class Environment:
def compute_face_axis(self) -> None:
"""Compute the face axis."""
if self.results.multi_face_landmarks:
for face_landmarks in self.results.multi_face_landmarks:
# retreive points
left_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_LEFT_SIDE])
right_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_RIGHT_SIDE])
bottom_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_BOTTOM_SIDE])
top_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_TOP_SIDE])
face_landmarks = self.results.multi_face_landmarks[0]
landmarks = np.array([(lm.x, lm.y, lm.z) for lm in face_landmarks.landmark])
landmarks = landmarks.T
# compute center
self.center = np.mean(np.concatenate((left_points, right_points, bottom_points, top_points)), axis=0)
if self.refine_landmarks:
landmarks = landmarks[:, :468]
# compute axis
self.x = np.mean(right_points - left_points, axis=0)
self.y = np.mean(top_points - bottom_points, axis=0)
self.z = np.cross(self.x, self.y)
metric_landmarks, pose_transform_mat = get_metric_landmarks(landmarks.copy(), self.pcf)
# normalize axis
self.x = self.x / np.linalg.norm(self.x)
self.y = self.y / np.linalg.norm(self.y)
self.z = self.z / np.linalg.norm(self.z)
self.model_points = metric_landmarks[0:3, points_idx].T
# see here:
# https://github.com/google/mediapipe/issues/1379#issuecomment-752534379
pose_transform_mat[1:3, :] = -pose_transform_mat[1:3, :]
self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
self.mp_translation_vector = pose_transform_mat[:3, 3, None]
def draw_axis(self) -> None:
"""Draw the face axis on the frame."""
for (axis, color, letter) in [
(self.x, (0, 0, 255), "X"),
(self.y, (0, 255, 0), "Y"),
(self.z, (255, 0, 0), "Z"),
]:
# compute start and end of axis
start = (
int(self.center[0] * self.camera_width),
int(self.center[1] * self.camera_height),
)
end = (
int(self.center[0] * self.camera_width + axis[0] * 100),
int(self.center[1] * self.camera_height + axis[1] * 100),
nose_tip = self.model_points[0]
nose_tip_extended = 2.5 * self.model_points[0]
(nose_pointer2D, jacobian) = cv2.projectPoints(
np.array([nose_tip, nose_tip_extended]),
self.mp_rotation_vector,
self.mp_translation_vector,
self.camera_matrix,
dist_coeff,
)
# draw axis + letter
cv2.line(self.frame, start, end, color, 2)
cv2.putText(self.frame, letter, end, cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
nose_tip_2D, nose_tip_2D_extended = nose_pointer2D.squeeze().astype(int)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_extended, (255, 0, 0), 2)
def draw_keypoints(self) -> None:
"""Draw the keypoints on the screen."""

2659
src/face_geometry.py Normal file

File diff suppressed because it is too large Load diff