feat: retreive axis from landmarks

This commit is contained in:
Laureηt 2022-12-14 11:19:04 +01:00
parent 731ed22156
commit 3a35f3dcf8
No known key found for this signature in database
GPG key ID: D88C6B294FD40994
2 changed files with 75 additions and 3 deletions

View file

@ -11,6 +11,7 @@ from bodyparts.eye import Eye
from bodyparts.head import Head from bodyparts.head import Head
from bodyparts.moustache import Moustache from bodyparts.moustache import Moustache
from bodyparts.mouth import Mouth from bodyparts.mouth import Mouth
from utils import landmark3vec
class Environment: class Environment:
@ -85,6 +86,9 @@ class Environment:
# detect keypoints on frame # detect keypoints on frame
self.detect_keypoints() self.detect_keypoints()
# compute face axis
self.compute_face_axis()
# draw keypoints on top of fram # draw keypoints on top of fram
self.draw_keypoints() self.draw_keypoints()
@ -115,6 +119,64 @@ class Environment:
# convert the image back to BGR format # convert the image back to BGR format
self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR) self.frame = cv2.cvtColor(self.frame, cv2.COLOR_RGB2BGR)
def compute_face_axis(self) -> None:
"""Compute the face axis."""
if self.results.multi_face_landmarks:
for face_landmarks in self.results.multi_face_landmarks:
# retreive points
left = landmark3vec(face_landmarks.landmark[234], self.screen)
right = landmark3vec(face_landmarks.landmark[454], self.screen)
bottom = landmark3vec(face_landmarks.landmark[152], self.screen)
top = landmark3vec(face_landmarks.landmark[10], self.screen)
center = (left + right + bottom + top) / 4
# compute axis
self.x = (right - left) / 2
self.y = (top - bottom) / 2
self.z = self.x.cross(self.y)
# normalize axis
self.x.normalize_ip()
self.y.normalize_ip()
self.z.normalize_ip()
# print horizontal angle to screen
self.angle_x = self.x.angle_to(pg.math.Vector3(1, 0, 0))
self.angle_y = self.x.angle_to(pg.math.Vector3(0, 0, 1)) - 90
self.angle_z = self.y.angle_to(pg.math.Vector3(0, 0, 1)) - 90
# draw axis on opencv screen
cv2.line(
self.frame,
(int(center.x * self.screen.get_width()), int(center.y * self.screen.get_height())),
(
int(center.x * self.screen.get_width() + self.x.x * 100),
int(center.y * self.screen.get_height() + self.x.y * 100),
),
(0, 0, 255),
2,
)
cv2.line(
self.frame,
(int(center.x * self.screen.get_width()), int(center.y * self.screen.get_height())),
(
int(center.x * self.screen.get_width() + self.y.x * 100),
int(center.y * self.screen.get_height() + self.y.y * 100),
),
(0, 255, 0),
2,
)
cv2.line(
self.frame,
(int(center.x * self.screen.get_width()), int(center.y * self.screen.get_height())),
(
int(center.x * self.screen.get_width() + self.z.x * 100),
int(center.y * self.screen.get_height() + self.z.y * 100),
),
(255, 0, 0),
2,
)
def draw_keypoints(self) -> None: def draw_keypoints(self) -> None:
"""Draw the keypoints on the screen.""" """Draw the keypoints on the screen."""
# draw the face mesh annotations on the image. # draw the face mesh annotations on the image.
@ -133,7 +195,8 @@ class Environment:
# ) # )
# flip the image horizontally for a selfie-view display # flip the image horizontally for a selfie-view display
cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1)) cv2.imshow("MediaPipe Face Mesh", self.frame)
# cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1))
def draw_avatar(self) -> None: def draw_avatar(self) -> None:
"""Draw the avatar on the screen.""" """Draw the avatar on the screen."""
@ -142,7 +205,7 @@ class Environment:
for part in self.body_parts.values(): for part in self.body_parts.values():
part.draw(self.screen) part.draw(self.screen)
# self.screen.blit(pg.transform.flip(self.screen, False, False), (0, 0)) self.screen.blit(self.screen, (0, 0))
self.screen.blit(pg.transform.flip(self.screen, True, False), (0, 0)) # self.screen.blit(pg.transform.flip(self.screen, True, False), (0, 0))
pg.display.flip() pg.display.flip()

View file

@ -35,3 +35,12 @@ def landmark2vec(landmark, screen):
screen.get_height(), screen.get_height(),
) # type: ignore ) # type: ignore
) )
def landmark3vec(landmark, screen):
"""Convert a landmark to a pygame Vector3."""
return pg.Vector3(
np.clip(landmark.x, 0, 1),
np.clip(landmark.y, 0, 1),
np.clip(landmark.z, 0, 1),
)