diff --git a/src/bodyparts/__init__.py b/src/bodyparts/__init__.py index 761faba..3fb34c5 100644 --- a/src/bodyparts/__init__.py +++ b/src/bodyparts/__init__.py @@ -1,8 +1 @@ -from .body import Body -from .crown import Crown -from .ear import Ear -from .eye import Eye from .head import Head -from .moustache import Moustache -from .mouth import Mouth -from .test import Test diff --git a/src/bodyparts/body.py b/src/bodyparts/body.py deleted file mode 100644 index c831808..0000000 --- a/src/bodyparts/body.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pygame as pg - -from utils import landmark2vec - -if TYPE_CHECKING: - from environment import Environment - -LANDMARKS = [152] - - -class Body: - def __init__(self, env: Environment) -> None: - self.env = env - self.texture = pg.image.load("assets/body.png").convert_alpha() - - def draw(self, screen) -> None: - - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - - body_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen) - - texture_scaled = pg.transform.scale( - self.texture, - self.env.size, - ) - - texture_pos = body_pos - pg.Vector2( - self.env.size[0] / 2, - self.env.size[1] / 10, - ) - - # draw texture - screen.blit(texture_scaled, texture_pos) diff --git a/src/bodyparts/crown.py b/src/bodyparts/crown.py deleted file mode 100644 index 419239e..0000000 --- a/src/bodyparts/crown.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pygame as pg - -from utils import blitRotate, landmark2vec - -if TYPE_CHECKING: - from environment import Environment - -LANDMARKS = [10] - - -class Crown: - def __init__(self, env: Environment) -> None: - self.env = env - self.texture = pg.image.load("assets/crown.png").convert_alpha() - - def draw(self, screen) -> None: - - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - - crown_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen) - - texture_scaled = pg.transform.scale( - self.texture, - (self.env.size[0] / 3, self.env.size[1] / 3), - ) - - # draw texture - blitRotate(screen, texture_scaled, crown_pos, self.env.angle_x) diff --git a/src/bodyparts/ear.py b/src/bodyparts/ear.py deleted file mode 100644 index 6357ea3..0000000 --- a/src/bodyparts/ear.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pygame as pg - -from utils import blitRotate, landmark2vec - -if TYPE_CHECKING: - from environment import Environment - -LANDMARKS = [103, 332] - - -class Ear: - def __init__(self, side: bool, env: Environment) -> None: - self.env = env - self.side = side - self.texture = ( - pg.image.load("assets/earL.png").convert_alpha() - if side - else pg.image.load("assets/earR.png").convert_alpha() - ) - - def draw(self, screen) -> None: - - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - - texture_scaled = pg.transform.scale( - self.texture, - (self.env.size[0] / 3, self.env.size[1]), - ) - - if self.side: # right moustache - moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen) - - pivot = pg.Vector2( - 3 * texture_scaled.get_width() / 4, - 0.8 * texture_scaled.get_height(), - ) - - else: # left moustache - moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen) - - pivot = pg.Vector2( - texture_scaled.get_width() / 4, - 0.8 * texture_scaled.get_height(), - ) - - # draw texture - blitRotate(screen, texture_scaled, moustache_pos, self.env.angle_x, pivot) diff --git a/src/bodyparts/eye.py b/src/bodyparts/eye.py deleted file mode 100644 index 0924abb..0000000 --- a/src/bodyparts/eye.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pygame as pg - -from utils import blitRotate, landmark2vec - -if TYPE_CHECKING: - from environment import Environment - -LANDMARK_LEFT_EYELID = [386, 374] -LANDMARK_RIGHT_EYELID = [159, 145] -LANDMARK_LEFT_IRIS = [473, 474, 475, 476, 477] -LANDMARK_RIGHT_IRIS = [468, 469, 470, 471, 472] - - -class Eye: - """The eye class is responsible of drawing an eye.""" - - def __init__(self, side: bool, env: Environment) -> None: - """Initialize the eye.""" - self.env = env - self.side = side - self.texture = ( - pg.image.load("assets/eyeL.png").convert_alpha() - if side - else pg.image.load("assets/eyeR.png").convert_alpha() - ) - - def draw(self, screen) -> None: - """Draw the eye.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - if self.side: # right eye - iris_pos = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_IRIS[0]], screen) - - iris_top = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_IRIS[2]], screen) - iris_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_IRIS[4]], screen) - - eyelid_top = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_EYELID[0]], screen) - eyelid_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_EYELID[1]], screen) - - else: # left eye - iris_pos = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_IRIS[0]], screen) - - iris_top = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_IRIS[2]], screen) - iris_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_IRIS[4]], screen) - - eyelid_top = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_EYELID[0]], screen) - eyelid_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_EYELID[1]], screen) - - angle = (iris_bottom - iris_top).angle_to(pg.Vector2(0, 1)) - width = (iris_bottom - iris_top).length() - height = (eyelid_bottom - eyelid_top).length() * 5 - - texture_scaled = pg.transform.scale( - self.texture, - (width, height), - ) - - # draw texture - blitRotate(screen, texture_scaled, iris_pos, angle) diff --git a/src/bodyparts/head.py b/src/bodyparts/head.py index 626233e..284ef67 100644 --- a/src/bodyparts/head.py +++ b/src/bodyparts/head.py @@ -2,41 +2,84 @@ from __future__ import annotations from typing import TYPE_CHECKING -import pygame as pg - -from utils import blitRotate, landmark2vec +import cv2 +import numpy as np if TYPE_CHECKING: from environment import Environment -LANDMARKS = [234, 10, 454, 152] - class Head: + """The head body part.""" + def __init__(self, env: Environment) -> None: + """Initialize the head.""" self.env = env - self.texture = pg.image.load("assets/head.png").convert_alpha() - - def draw(self, screen) -> None: + self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED) + self.ratio = self.image.shape[1] / self.image.shape[0] + self.bouding_box = np.array( + [ + [0, 0], + [self.image.shape[1], 0], + [self.image.shape[1], self.image.shape[0]], + [0, self.image.shape[0]], + ], + dtype=np.float32, + ) + def draw(self) -> None: + """Draw the head on the screen.""" # compute position if self.env.results.multi_face_landmarks: for face_landmarks in self.env.results.multi_face_landmarks: + head_box = np.array( + [ + 100 * self.ratio * self.env.x + 100 * self.env.y + 80 * self.env.z, + 100 * self.ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z, + 100 * self.ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z, + 100 * self.ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z, + ] + )[:, :2] - head_top = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen) - head_bottom = landmark2vec(face_landmarks.landmark[LANDMARKS[3]], screen) + self.translated_head_box = ( + head_box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) + ).astype(np.float32) - height = (head_bottom - head_top).length() - ratio = self.texture.get_width() / self.texture.get_height() - - self.env.size = (height * ratio, height) - - head_pos = (head_bottom + head_top) / 2 - - texture_scaled = pg.transform.scale( - self.texture, - self.env.size, + # get perspective transform + transform_mat = cv2.getPerspectiveTransform( + self.bouding_box, + self.translated_head_box, ) - # draw texture - blitRotate(screen, texture_scaled, head_pos, self.env.angle_x) + # apply perspective transform to image + warped = cv2.warpPerspective( + self.image, + transform_mat, + self.env.frame.shape[1::-1], + ) + + # replace non black pixels of warped image by frame + self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] + + def draw_debug(self) -> None: + """Draw debug information on the screen.""" + # link points + for i in range(4): + cv2.line( + self.env.frame, + tuple(self.translated_head_box[i].astype(np.int32)), + tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)), + (255, 255, 255), + 2, + ) + + # TODO: faire un POC où je place juste un cvGetPerspectiveTransform suivi d'un cvWarpPerspective + # -> comme sur cet exemple, mais où l'image de gauche et droite sont inversées + # https://docs.adaptive-vision.com/studio/filters/GeometricImageTransformations/cvGetPerspectiveTransform.html + + # instrisics ? -> https://github.dev/google/mediapipe/blob/master/mediapipe/modules/face_geometry/libs/effect_renderer.cc#L573-L599 + + # TODO: https://github.com/Rassibassi/mediapipeFacegeometryPython/blob/main/head_posture_rt.py + # -> pnp -> pose estimation -> paramètres extrinsèques + # -> + param intrasèque (supposé connu, check site mediapipe) + # -> placer dans l'espace les textures -> et projeter dans le plan image diff --git a/src/bodyparts/moustache.py b/src/bodyparts/moustache.py deleted file mode 100644 index 72550ab..0000000 --- a/src/bodyparts/moustache.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pygame as pg - -from utils import blitRotate, landmark2vec - -if TYPE_CHECKING: - from environment import Environment - -LANDMARKS = [138, 367] - - -class Moustache: - def __init__(self, side: bool, env: Environment) -> None: - self.env = env - self.side = side - self.texture = ( - pg.image.load("assets/moustacheL.png").convert_alpha() - if side - else pg.image.load("assets/moustacheR.png").convert_alpha() - ) - - def draw(self, screen) -> None: - - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - - texture_scaled = pg.transform.scale( - self.texture, - (self.env.size[0] / 1.5, self.env.size[1] / 1.5), - ) - - if self.side: # right moustache - moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen) - - pivot = pg.Vector2( - texture_scaled.get_width(), - texture_scaled.get_height() / 2, - ) - - else: # left moustache - moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen) - - pivot = pg.Vector2( - 0, - texture_scaled.get_height() / 2, - ) - - # draw texture - blitRotate(screen, texture_scaled, moustache_pos, self.env.angle_x, pivot) diff --git a/src/bodyparts/mouth.py b/src/bodyparts/mouth.py deleted file mode 100644 index 2a4818f..0000000 --- a/src/bodyparts/mouth.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import pygame as pg - -from utils import blitRotate, landmark2vec - -if TYPE_CHECKING: - from environment import Environment - -LANDMARKS = [78, 13, 308, 14] -MIN_HEIGHT = 4 -MAX_HEIGHT = 40 - - -class Mouth: - def __init__(self, env: Environment) -> None: - self.env = env - self.texture = pg.image.load("assets/mouth.png").convert_alpha() - self.closed_texture = pg.image.load("assets/mouth.png").convert_alpha() - self.closed_texture.fill((0, 0, 0, 255), special_flags=pg.BLEND_RGBA_MULT) - - def draw(self, screen) -> None: - - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - - mouth_top = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen) - mouth_bottom = landmark2vec(face_landmarks.landmark[LANDMARKS[3]], screen) - - mouth_left = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen) - mouth_right = landmark2vec(face_landmarks.landmark[LANDMARKS[2]], screen) - - height = (mouth_bottom - mouth_top).length() - height = min(height, MAX_HEIGHT) - width = (mouth_right - mouth_left).length() - - mouth_pos = (mouth_bottom + mouth_top) / 2 - - if height < MIN_HEIGHT: - height = MIN_HEIGHT - texture = self.closed_texture - else: - height *= 1.5 - texture = self.texture - - texture_scaled = pg.transform.scale( - texture, - (width, height), - ) - - # draw texture - blitRotate(screen, texture_scaled, mouth_pos, self.env.angle_x) diff --git a/src/bodyparts/test.py b/src/bodyparts/test.py deleted file mode 100644 index 0ee5bdd..0000000 --- a/src/bodyparts/test.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np -import pygame as pg - -if TYPE_CHECKING: - from environment import Environment - - -class Test: - def __init__(self, env: Environment) -> None: - self.env = env - self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED) - - def draw(self, screen: pg.Surface) -> None: - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - - ratio = self.image.shape[1] / self.image.shape[0] - - head_box = np.array( - [ - 100 * ratio * self.env.x + 100 * self.env.y + 80 * self.env.z, - 100 * ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z, - 100 * ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z, - 100 * ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z, - ] - )[:, :2] - - # link points - cv2.line( - self.env.frame, - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[0][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[0][1]), - ), - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[1][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[1][1]), - ), - (0, 0, 255), - 2, - ) - cv2.line( - self.env.frame, - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[1][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[1][1]), - ), - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[2][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[2][1]), - ), - (0, 0, 255), - 2, - ) - cv2.line( - self.env.frame, - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[2][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[2][1]), - ), - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[3][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[3][1]), - ), - (0, 0, 255), - 2, - ) - cv2.line( - self.env.frame, - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[3][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[3][1]), - ), - ( - int(self.env.center.x * self.env.screen.get_width() + head_box[0][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[0][1]), - ), - (0, 0, 255), - 2, - ) - - a = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - b = np.array( - [ - [ - int(self.env.center.x * self.env.screen.get_width() + head_box[0][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[0][1]), - ], - [ - int(self.env.center.x * self.env.screen.get_width() + head_box[1][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[1][1]), - ], - [ - int(self.env.center.x * self.env.screen.get_width() + head_box[2][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[2][1]), - ], - [ - int(self.env.center.x * self.env.screen.get_width() + head_box[3][0]), - int(self.env.center.y * self.env.screen.get_height() + head_box[3][1]), - ], - ], - dtype=np.float32, - ) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - a, - b, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.frame[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] diff --git a/src/bodyparts/test.py.old b/src/bodyparts/test.py.old deleted file mode 100644 index d8fb45c..0000000 --- a/src/bodyparts/test.py.old +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import numpy as np -import pygame as pg -import skimage -import skimage.io -import skimage.transform -from skimage import data, img_as_float - -if TYPE_CHECKING: - from environment import Environment - -LANDMARKS = [6] - -# https://scikit-image.org/docs/stable/auto_examples/transform/plot_geometric.html#sphx-glr-auto-examples-transform-plot-geometric-py -# https://scikit-image.org/docs/stable/auto_examples/transform/plot_transform_types.html#sphx-glr-auto-examples-transform-plot-transform-types-py -# https://github.com/google/mediapipe/issues/1379 -# https://en.wikipedia.org/wiki/3D_projection#Perspective_projection - - -class Test: - def __init__(self, env: Environment) -> None: - self.env = env - self.image = skimage.io.imread("assets/head.png")[:, :, :3] - self.image = skimage.transform.resize( - self.image, (self.image.shape[0] // 10, self.image.shape[1] // 10), anti_aliasing=False - ) - - self.anglex = 0 - self.angley = 0 - self.anglez = 0 - - def draw(self, screen: pg.Surface) -> None: - - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - - body_pos = pg.Vector2(*screen.get_size()) / 2 - img = img_as_float(data.chelsea()) - - self.anglez += 1 / 180 * np.pi - self.anglez += 1 / 180 * np.pi - - # matrix = np.array( - # [ - # [1, -0.5, 100], - # [0.1, 0.9, 50], - # [0.0015, 0.0015, 1], - # ] - # ) - - # self.angley += 0.1 / 180 * np.pi - # print(self.angley * 180 / np.pi) - - # scale = np.array( - # [ - # [0.5, 0, 0], - # [0, 0.5, 0], - # [0, 0, 1], - # ] - # ) - - translation = np.array( - [ - [1, 0, -img.shape[1] / 2], - [0, 1, img.shape[0] / 2], - [0, 0, 1], - ] - ) - - anti_translation = np.array( - [ - [1, 0, img.shape[1] / 2], - [0, 1, -img.shape[0] / 2], - [0, 0, 1], - ] - ) - - rotation_x = np.array( - [ - [1, 0, 0], - [0, np.cos(self.anglex), -np.sin(self.anglex)], - [0, np.sin(self.anglex), np.cos(self.anglex)], - ] - ) - - rotation_y = np.array( - [ - [np.cos(self.angley), 0, np.sin(self.angley)], - [0, 1, 0], - [-np.sin(self.angley), 0, np.cos(self.angley)], - ] - ) - - rotation_z = np.array( - [ - [np.cos(self.anglez), -np.sin(self.anglez), 0], - [np.sin(self.anglez), np.cos(self.anglez), 0], - [0, 0, 1], - ] - ) - - # matrix = translation @ rotation_x @ rotation_y @ rotation_z @ anti_translation - matrix = rotation_x @ rotation_y @ rotation_z - matrix = translation @ rotation_x @ rotation_y @ rotation_z - - tform = skimage.transform.ProjectiveTransform(matrix=matrix) - tf_img = skimage.transform.warp(img, tform.inverse) - - yes = pg.surfarray.make_surface((tf_img * 255).astype(np.uint8).transpose(1, 0, 2)) - yes.set_colorkey((0, 0, 0)) - - texture_scaled = pg.transform.scale( - yes, - (yes.get_width() / 2, yes.get_height() / 2), - ) - - texture_pos = body_pos - pg.Vector2( - texture_scaled.get_width() / 2, - texture_scaled.get_height() / 2, - ) - - screen.blit(texture_scaled, texture_pos) diff --git a/src/environment.py b/src/environment.py index 39fe4f6..b16a026 100644 --- a/src/environment.py +++ b/src/environment.py @@ -2,10 +2,16 @@ import logging import cv2 import mediapipe as mp -import pygame as pg +import numpy as np -from bodyparts import Body, Crown, Ear, Eye, Head, Moustache, Mouth, Test -from utils import landmark3vec +from bodyparts import Head +from utils import ( + LANDMARKS_BOTTOM_SIDE, + LANDMARKS_LEFT_SIDE, + LANDMARKS_RIGHT_SIDE, + LANDMARKS_TOP_SIDE, + landmark2vec, +) class Environment: @@ -27,70 +33,56 @@ class Environment: self.mp_drawing_styles = mp.solutions.drawing_styles # type: ignore self.mp_face_mesh = mp.solutions.face_mesh # type: ignore - # init pygame - pg.init() - # get screen size from webcam - self.screen_width = camera.get(3) - self.screen_height = camera.get(4) - - # create screen - self.screen: pg.Surface = pg.display.set_mode( - (self.screen_width, self.screen_height), - pg.DOUBLEBUF | pg.HWSURFACE, - ) # type: ignore - pg.display.set_caption("Projet APP") - - self.size = (0.0, 0.0) + self.camera_width = camera.get(3) + self.camera_height = camera.get(4) # create body parts - self.body_parts = { - # "body": Body(self), - # "left_ear": Ear(False, self), - # "right_ear": Ear(True, self), - # "head": Head(self), - # "left_moustache": Moustache(False, self), - # "right_moustache": Moustache(True, self), - # "left_eye": Eye(False, self), - # "right_eye": Eye(True, self), - # "crown": Crown(self), - # "mouth": Mouth(self), - "test": Test(self), - } + self.body_parts = [ + # Body(self), + # Ear(False, self), + # Ear(True, self), + Head(self), + # Moustache(False, self), + # Moustache(True, self), + # Eye(False, self), + # Eye(True, self), + # Crown(self), + # Mouth(self), + ] def start(self) -> None: """Start the environment.""" while self.cam.isOpened(): + + # stop if q is pressed (opencv) + if cv2.waitKey(5) & 0xFF == ord("q"): + break + # read webcam success, self.frame = self.cam.read() if not success: logging.debug("Ignoring empty camera frame.") continue - # stop if q is pressed (opencv) - if cv2.waitKey(5) & 0xFF == ord("q"): - break - - # quit the game - if any( - [event.type == pg.KEYDOWN and event.key == pg.K_q for event in pg.event.get()], - ): - break - # detect keypoints on frame self.detect_keypoints() # compute face axis self.compute_face_axis() + self.draw_axis() # draw keypoints on top of frame - # self.draw_keypoints() + self.draw_keypoints() # draw avatar self.draw_avatar() - # tmp - cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1)) + # show frame + cv2.imshow("Camera", cv2.flip(self.frame, 1)) + + # show avatar + cv2.imshow("Avatar", cv2.flip(self.avatar, 1)) def detect_keypoints(self) -> None: """Detect the keypoints on the frame.""" @@ -121,58 +113,44 @@ class Environment: if self.results.multi_face_landmarks: for face_landmarks in self.results.multi_face_landmarks: # retreive points - left = landmark3vec(face_landmarks.landmark[234], self.screen) - right = landmark3vec(face_landmarks.landmark[454], self.screen) - bottom = landmark3vec(face_landmarks.landmark[152], self.screen) - top = landmark3vec(face_landmarks.landmark[10], self.screen) - self.center = (left + right + bottom + top) / 4 + left_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_LEFT_SIDE]) + right_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_RIGHT_SIDE]) + bottom_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_BOTTOM_SIDE]) + top_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_TOP_SIDE]) + + # compute center + self.center = np.mean(np.concatenate((left_points, right_points, bottom_points, top_points)), axis=0) # compute axis - self.x = (right - left) / 2 - self.y = (top - bottom) / 2 - self.z = self.x.cross(self.y) + self.x = np.mean(right_points - left_points, axis=0) + self.y = np.mean(top_points - bottom_points, axis=0) + self.z = np.cross(self.x, self.y) # normalize axis - self.x.normalize_ip() - self.y.normalize_ip() - self.z.normalize_ip() + self.x = self.x / np.linalg.norm(self.x) + self.y = self.y / np.linalg.norm(self.y) + self.z = self.z / np.linalg.norm(self.z) - # print horizontal angle to screen - self.angle_x = self.x.angle_to(pg.math.Vector3(0, 1, 0)) - 90 - self.angle_y = self.x.angle_to(pg.math.Vector3(0, 0, 1)) - 90 - self.angle_z = self.y.angle_to(pg.math.Vector3(0, 0, 1)) - 90 + def draw_axis(self) -> None: + """Draw the face axis on the frame.""" + for (axis, color, letter) in [ + (self.x, (0, 0, 255), "X"), + (self.y, (0, 255, 0), "Y"), + (self.z, (255, 0, 0), "Z"), + ]: + # compute start and end of axis + start = ( + int(self.center[0] * self.camera_width), + int(self.center[1] * self.camera_height), + ) + end = ( + int(self.center[0] * self.camera_width + axis[0] * 100), + int(self.center[1] * self.camera_height + axis[1] * 100), + ) - # draw axis on opencv screen - # cv2.line( - # self.frame, - # (int(self.center.x * self.screen.get_width()), int(self.center.y * self.screen.get_height())), - # ( - # int(self.center.x * self.screen.get_width() + self.x.x * 100), - # int(self.center.y * self.screen.get_height() + self.x.y * 100), - # ), - # (0, 0, 255), - # 2, - # ) - # cv2.line( - # self.frame, - # (int(self.center.x * self.screen.get_width()), int(self.center.y * self.screen.get_height())), - # ( - # int(self.center.x * self.screen.get_width() + self.y.x * 100), - # int(self.center.y * self.screen.get_height() + self.y.y * 100), - # ), - # (0, 255, 0), - # 2, - # ) - # cv2.line( - # self.frame, - # (int(self.center.x * self.screen.get_width()), int(self.center.y * self.screen.get_height())), - # ( - # int(self.center.x * self.screen.get_width() + self.z.x * 100), - # int(self.center.y * self.screen.get_height() + self.z.y * 100), - # ), - # (255, 0, 0), - # 2, - # ) + # draw axis + letter + cv2.line(self.frame, start, end, color, 2) + cv2.putText(self.frame, letter, end, cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2) def draw_keypoints(self) -> None: """Draw the keypoints on the screen.""" @@ -185,20 +163,12 @@ class Environment: landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 0, 0), ) - # flip the image horizontally for a selfie-view display - cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1)) - def draw_avatar(self) -> None: """Draw the avatar on the screen.""" - # clear image with green background - self.screen.fill(pg.Color("green")) + # clear avatar frame + self.avatar = np.zeros(self.frame.shape, dtype=np.uint8) # draw each body part - for part in self.body_parts.values(): - part.draw(self.screen) - - # flip screen - self.screen.blit(pg.transform.flip(self.screen, True, False), (0, 0)) - - # display screen - pg.display.flip() + for part in self.body_parts: + part.draw() + part.draw_debug() diff --git a/src/utils.py b/src/utils.py index 0d14595..8c15e22 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,46 +1,32 @@ import numpy as np -import pygame as pg from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates -# def blitRotate(surf, image, topleft, angle): -# """Rotate an image while drawing it.""" -# rotated_image = pg.transform.rotate(image, angle) -# new_rect = rotated_image.get_rect(center=image.get_rect(topleft=topleft).center) -# surf.blit(rotated_image, new_rect.topleft) +# def landmark2vec(landmark, screen): +# """Convert a landmark to a pygame Vector2.""" +# return pg.Vector2( +# _normalized_to_pixel_coordinates( +# np.clip(landmark.x, 0, 1), +# np.clip(landmark.y, 0, 1), +# screen.get_width(), +# screen.get_height(), +# ) # type: ignore +# ) -# https://stackoverflow.com/questions/70819750/rotating-and-scaling-an-image-around-a-pivot-while-scaling-width-and-height-sep/70820034#70820034 -def blitRotate(surf, image, origin, angle, pivot=None): - - if pivot is None: - pivot = pg.math.Vector2(image.get_size()) / 2 - - image_rect = image.get_rect(topleft=(origin[0] - pivot[0], origin[1] - pivot[1])) - offset_center_to_pivot = pg.math.Vector2(origin) - image_rect.center - rotated_offset = offset_center_to_pivot.rotate(-angle) - rotated_image_center = (origin[0] - rotated_offset.x, origin[1] - rotated_offset.y) - rotated_image = pg.transform.rotate(image, angle) - rect = rotated_image.get_rect(center=rotated_image_center) - - surf.blit(rotated_image, rect) - - -def landmark2vec(landmark, screen): - """Convert a landmark to a pygame Vector2.""" - return pg.Vector2( - _normalized_to_pixel_coordinates( - np.clip(landmark.x, 0, 1), - np.clip(landmark.y, 0, 1), - screen.get_width(), - screen.get_height(), - ) # type: ignore +def landmark2vec(landmark) -> np.ndarray: + """Convert a landmark to numpy array.""" + return np.clip( + [ + landmark.x, + landmark.y, + landmark.z, + ], + a_min=0, + a_max=1, ) -def landmark3vec(landmark, screen): - """Convert a landmark to a pygame Vector3.""" - return pg.Vector3( - np.clip(landmark.x, 0, 1), - np.clip(landmark.y, 0, 1), - np.clip(landmark.z, 0, 1), - ) +LANDMARKS_LEFT_SIDE = [109, 67, 103, 54, 21, 162, 127, 234, 93, 132, 58, 172, 136, 150, 149, 176, 148] +LANDMARKS_RIGHT_SIDE = [338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377] +LANDMARKS_BOTTOM_SIDE = [93, 132, 152, 361, 323] +LANDMARKS_TOP_SIDE = [127, 162, 10, 389, 356]