feat: retreive axis from landmarks

This commit is contained in:
Laureηt 2022-12-14 11:19:04 +01:00
parent 731ed22156
commit 3a35f3dcf8
No known key found for this signature in database
GPG key ID: D88C6B294FD40994
2 changed files with 75 additions and 3 deletions

View file

@ -11,6 +11,7 @@ from bodyparts.eye import Eye
from bodyparts.head import Head
from bodyparts.moustache import Moustache
from bodyparts.mouth import Mouth
from utils import landmark3vec
class Environment:
@ -85,6 +86,9 @@ class Environment:
# detect keypoints on frame
self.detect_keypoints()
# compute face axis
self.compute_face_axis()
# draw keypoints on top of fram
self.draw_keypoints()
@ -115,6 +119,64 @@ class Environment:
# convert the image back to BGR format
self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR)
def compute_face_axis(self) -> None:
"""Compute the face axis."""
if self.results.multi_face_landmarks:
for face_landmarks in self.results.multi_face_landmarks:
# retreive points
left = landmark3vec(face_landmarks.landmark[234], self.screen)
right = landmark3vec(face_landmarks.landmark[454], self.screen)
bottom = landmark3vec(face_landmarks.landmark[152], self.screen)
top = landmark3vec(face_landmarks.landmark[10], self.screen)
center = (left + right + bottom + top) / 4
# compute axis
self.x = (right - left) / 2
self.y = (top - bottom) / 2
self.z = self.x.cross(self.y)
# normalize axis
self.x.normalize_ip()
self.y.normalize_ip()
self.z.normalize_ip()
# print horizontal angle to screen
self.angle_x = self.x.angle_to(pg.math.Vector3(1, 0, 0))
self.angle_y = self.x.angle_to(pg.math.Vector3(0, 0, 1)) - 90
self.angle_z = self.y.angle_to(pg.math.Vector3(0, 0, 1)) - 90
# draw axis on opencv screen
cv2.line(
self.frame,
(int(center.x * self.screen.get_width()), int(center.y * self.screen.get_height())),
(
int(center.x * self.screen.get_width() + self.x.x * 100),
int(center.y * self.screen.get_height() + self.x.y * 100),
),
(0, 0, 255),
2,
)
cv2.line(
self.frame,
(int(center.x * self.screen.get_width()), int(center.y * self.screen.get_height())),
(
int(center.x * self.screen.get_width() + self.y.x * 100),
int(center.y * self.screen.get_height() + self.y.y * 100),
),
(0, 255, 0),
2,
)
cv2.line(
self.frame,
(int(center.x * self.screen.get_width()), int(center.y * self.screen.get_height())),
(
int(center.x * self.screen.get_width() + self.z.x * 100),
int(center.y * self.screen.get_height() + self.z.y * 100),
),
(255, 0, 0),
2,
)
def draw_keypoints(self) -> None:
"""Draw the keypoints on the screen."""
# draw the face mesh annotations on the image.
@ -133,7 +195,8 @@ class Environment:
# )
# flip the image horizontally for a selfie-view display
cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1))
cv2.imshow("MediaPipe Face Mesh", self.frame)
# cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1))
def draw_avatar(self) -> None:
"""Draw the avatar on the screen."""
@ -142,7 +205,7 @@ class Environment:
for part in self.body_parts.values():
part.draw(self.screen)
# self.screen.blit(pg.transform.flip(self.screen, False, False), (0, 0))
self.screen.blit(pg.transform.flip(self.screen, True, False), (0, 0))
self.screen.blit(self.screen, (0, 0))
# self.screen.blit(pg.transform.flip(self.screen, True, False), (0, 0))
pg.display.flip()

View file

@ -35,3 +35,12 @@ def landmark2vec(landmark, screen):
screen.get_height(),
) # type: ignore
)
def landmark3vec(landmark, screen):
"""Convert a landmark to a pygame Vector3."""
return pg.Vector3(
np.clip(landmark.x, 0, 1),
np.clip(landmark.y, 0, 1),
np.clip(landmark.z, 0, 1),
)