diff --git a/src/bodyparts/head.py b/src/bodypart.py similarity index 57% rename from src/bodyparts/head.py rename to src/bodypart.py index ba38cf7..a5bfb12 100644 --- a/src/bodyparts/head.py +++ b/src/bodypart.py @@ -9,13 +9,13 @@ if TYPE_CHECKING: from environment import Environment -class Head: - """The head body part.""" +class BodyPart: + """A basic body part.""" - def __init__(self, env: Environment) -> None: - """Initialize the head.""" + def __init__(self, env: Environment, image_path: str, position, height) -> None: + """Initialize the part.""" self.env = env - self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED) + self.image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED) self.ratio = self.image.shape[1] / self.image.shape[0] self.bounding_box = np.array( [ @@ -28,24 +28,33 @@ class Head: ) self.old_box = np.zeros((4, 1, 2)) + self.position = position + self.height = height + def draw(self) -> None: - """Draw the head on the screen.""" + """Draw the part on the screen.""" # compute position if self.env.results.multi_face_landmarks: - x = self.env.x - self.env.model_points[0] - y = self.env.y - self.env.model_points[0] + x = self.env.x + y = self.env.y for face_landmarks in self.env.results.multi_face_landmarks: - head_box = np.array( - [ - self.ratio * x + y + self.env.model_points[0], - self.ratio * -x + y + self.env.model_points[0], - self.ratio * -x + -y + self.env.model_points[0], - self.ratio * x + -y + self.env.model_points[0], - ] + # compute bounding box from position and height + head_box = ( + np.array( + [ + self.ratio * x * self.height + y * self.height, + self.ratio * -x * self.height + y * self.height, + self.ratio * -x * self.height + -y * self.height, + self.ratio * x * self.height + -y * self.height, + ] + ) + + self.position + + self.env.center ) + # project bounding box to camera (box, _) = cv2.projectPoints( head_box, self.env.mp_rotation_vector, @@ -58,12 +67,6 @@ class Head: box = (box + self.old_box) / 2 self.old_box = box - a, b, c, d = box.squeeze().astype(int) - cv2.line(self.env.frame, a, b, (255, 255, 255), 2) - cv2.line(self.env.frame, b, c, (255, 255, 255), 2) - cv2.line(self.env.frame, c, d, (255, 255, 255), 2) - cv2.line(self.env.frame, d, a, (255, 255, 255), 2) - # get perspective transform transform_mat = cv2.getPerspectiveTransform( self.bounding_box, @@ -82,13 +85,10 @@ class Head: def draw_debug(self) -> None: """Draw debug information on the screen.""" - # link points - pass - # for i in range(4): - # cv2.line( - # self.env.frame, - # tuple(self.translated_head_box[i].astype(np.int32)), - # tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)), - # (255, 255, 255), - # 2, - # ) + cv2.polylines( + self.env.frame, + [self.old_box.squeeze().astype(int)], + True, + (255, 255, 255), + 2, + ) diff --git a/src/bodyparts/__init__.py b/src/bodyparts/__init__.py deleted file mode 100644 index 9e6195a..0000000 --- a/src/bodyparts/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .crown import Crown -from .head import Head -from .left_ear import LeftEar -from .left_eye import LeftEye -from .left_moustache import LeftMoustache -from .mouth import Mouth -from .right_ear import RightEar -from .right_eye import RightEye -from .right_moustache import RightMoustache diff --git a/src/bodyparts/crown.py b/src/bodyparts/crown.py deleted file mode 100644 index 4e1ba8c..0000000 --- a/src/bodyparts/crown.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class Crown: - """The crown body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the crown.""" - self.env = env - self.image = cv2.imread("assets/crown.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the crown on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - 30 * self.ratio * self.env.x + 135 * self.env.y + 90 * self.env.z, - -30 * self.ratio * self.env.x + 135 * self.env.y + 90 * self.env.z, - -30 * self.ratio * self.env.x + 75 * self.env.y + 90 * self.env.z, - 30 * self.ratio * self.env.x + 75 * self.env.y + 90 * self.env.z, - ] - )[:, :2] - - self.translated_box = ( - box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/bodyparts/left_ear.py b/src/bodyparts/left_ear.py deleted file mode 100644 index a839ace..0000000 --- a/src/bodyparts/left_ear.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class LeftEar: - """The left eye body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the left eye.""" - self.env = env - self.image = cv2.imread("assets/earL.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the left eye on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - 100 * self.env.x + 250 * self.env.y + 70 * self.env.z, - 30 * self.env.x + 250 * self.env.y + 70 * self.env.z, - 30 * self.env.x + 50 * self.env.y + 70 * self.env.z, - 100 * self.env.x + 50 * self.env.y + 70 * self.env.z, - ] - )[:, :2] - - self.translated_box = ( - box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/bodyparts/left_eye.py b/src/bodyparts/left_eye.py deleted file mode 100644 index a624c2c..0000000 --- a/src/bodyparts/left_eye.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class LeftEye: - """The left eye body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the left eye.""" - self.env = env - self.image = cv2.imread("assets/eyeL.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the left eye on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - 40 * self.env.x + 70 * self.env.y + 80 * self.env.z, - 25 * self.env.x + 70 * self.env.y + 80 * self.env.z, - 25 * self.env.x + 15 * self.env.y + 80 * self.env.z, - 40 * self.env.x + 15 * self.env.y + 80 * self.env.z, - ] - )[:, :2] - - self.translated_box = ( - box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/bodyparts/left_moustache.py b/src/bodyparts/left_moustache.py deleted file mode 100644 index 1eb270b..0000000 --- a/src/bodyparts/left_moustache.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class LeftMoustache: - """The left eye body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the left eye.""" - self.env = env - self.image = cv2.imread("assets/moustacheL.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the left eye on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - 270 * self.env.x + 20 * self.env.y + 80 * self.env.z, - 70 * self.env.x + 20 * self.env.y + 80 * self.env.z, - 70 * self.env.x + -130 * self.env.y + 80 * self.env.z, - 270 * self.env.x + -130 * self.env.y + 80 * self.env.z, - ] - )[:, :2] - - self.translated_box = ( - box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/bodyparts/mouth.py b/src/bodyparts/mouth.py deleted file mode 100644 index 59115fd..0000000 --- a/src/bodyparts/mouth.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class Mouth: - """The mouth body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the mouth.""" - self.env = env - self.image = cv2.imread("assets/mouth.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the mouth on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - 15 * self.ratio * self.env.x + -30 * self.env.y + 80 * self.env.z, - -15 * self.ratio * self.env.x + -30 * self.env.y + 80 * self.env.z, - -15 * self.ratio * self.env.x + -50 * self.env.y + 80 * self.env.z, - 15 * self.ratio * self.env.x + -50 * self.env.y + 80 * self.env.z, - ] - )[:, :2] - - self.translated_box = ( - box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/bodyparts/right_ear.py b/src/bodyparts/right_ear.py deleted file mode 100644 index d5a0e68..0000000 --- a/src/bodyparts/right_ear.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class RightEar: - """The left eye body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the left eye.""" - self.env = env - self.image = cv2.imread("assets/earR.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the left eye on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - -30 * self.env.x + 250 * self.env.y + 70 * self.env.z, - -100 * self.env.x + 250 * self.env.y + 70 * self.env.z, - -100 * self.env.x + 50 * self.env.y + 70 * self.env.z, - -30 * self.env.x + 50 * self.env.y + 70 * self.env.z, - ] - )[:, :2] - - self.translated_box = ( - box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/bodyparts/right_eye.py b/src/bodyparts/right_eye.py deleted file mode 100644 index e7f0a30..0000000 --- a/src/bodyparts/right_eye.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class RightEye: - """The right eye body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the right eye.""" - self.env = env - self.image = cv2.imread("assets/eyeR.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the right eye on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - -25 * self.env.x + 70 * self.env.y + 80 * self.env.z, - -40 * self.env.x + 70 * self.env.y + 80 * self.env.z, - -40 * self.env.x + 15 * self.env.y + 80 * self.env.z, - -25 * self.env.x + 15 * self.env.y + 80 * self.env.z, - ] - ) - - self.translated_box = ( - box[:, :2] + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/bodyparts/right_moustache.py b/src/bodyparts/right_moustache.py deleted file mode 100644 index c847952..0000000 --- a/src/bodyparts/right_moustache.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -import cv2 -import numpy as np - -if TYPE_CHECKING: - from environment import Environment - - -class RightMoustache: - """The left eye body part.""" - - def __init__(self, env: Environment) -> None: - """Initialize the left eye.""" - self.env = env - self.image = cv2.imread("assets/moustacheR.png", cv2.IMREAD_UNCHANGED) - self.ratio = self.image.shape[1] / self.image.shape[0] - self.bouding_box = np.array( - [ - [0, 0], - [self.image.shape[1], 0], - [self.image.shape[1], self.image.shape[0]], - [0, self.image.shape[0]], - ], - dtype=np.float32, - ) - - def draw(self) -> None: - """Draw the left eye on the screen.""" - # compute position - if self.env.results.multi_face_landmarks: - for face_landmarks in self.env.results.multi_face_landmarks: - box = np.array( - [ - -70 * self.env.x + 20 * self.env.y + 80 * self.env.z, - -270 * self.env.x + 20 * self.env.y + 80 * self.env.z, - -270 * self.env.x + -130 * self.env.y + 80 * self.env.z, - -70 * self.env.x + -130 * self.env.y + 80 * self.env.z, - ] - )[:, :2] - - self.translated_box = ( - box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) - ).astype(np.float32) - - # get perspective transform - transform_mat = cv2.getPerspectiveTransform( - self.bouding_box, - self.translated_box, - ) - - # apply perspective transform to image - warped = cv2.warpPerspective( - self.image, - transform_mat, - self.env.frame.shape[1::-1], - ) - - # replace non black pixels of warped image by frame - self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3] - - def draw_debug(self) -> None: - """Draw debug information on the screen.""" - # link points - for i in range(4): - cv2.line( - self.env.frame, - tuple(self.translated_box[i].astype(np.int32)), - tuple(self.translated_box[(i + 1) % 4].astype(np.int32)), - (255, 255, 255), - 2, - ) diff --git a/src/environment.py b/src/environment.py index cac0661..9eb4bf2 100644 --- a/src/environment.py +++ b/src/environment.py @@ -3,33 +3,12 @@ import logging import cv2 import mediapipe as mp import numpy as np -from mediapipe.framework.formats import landmark_pb2 -from bodyparts import ( - Crown, - Head, - LeftEar, - LeftEye, - LeftMoustache, - Mouth, - RightEar, - RightEye, - RightMoustache, -) -from face_geometry import PCF, get_metric_landmarks, procrustes_landmark_basis -from utils import ( - LANDMARKS_BOTTOM_SIDE, - LANDMARKS_LEFT_SIDE, - LANDMARKS_RIGHT_SIDE, - LANDMARKS_TOP_SIDE, - landmark2vec, -) +from bodypart import BodyPart +from face_geometry import PCF, get_metric_landmarks -points_idx = [33, 263, 61, 291, 199] -points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis] -points_idx = list(set(points_idx)) -points_idx.sort() -print(points_idx) +NOSE_LANDMARK = 4 +UNREFINED_LANDMARKS = 468 class Environment: @@ -50,6 +29,7 @@ class Environment: self.mp_drawing = mp.solutions.drawing_utils # type: ignore self.mp_drawing_styles = mp.solutions.drawing_styles # type: ignore self.mp_face_mesh = mp.solutions.face_mesh # type: ignore + self.refine_landmarks = True # get screen size from webcam self.camera_width = camera.get(3) @@ -57,26 +37,64 @@ class Environment: # create body parts self.body_parts = [ - # LeftEar(self), - # RightEar(self), - Head(self), - # RightMoustache(self), - # LeftMoustache(self), - # LeftEye(self), - # RightEye(self), - # Crown(self), - # Mouth(self), + BodyPart( + self, + "assets/earL.png", + np.array([6, 11, -0.5]), + 1, + ), + BodyPart( + self, + "assets/earR.png", + np.array([-6, 11, -0.5]), + 1, + ), + BodyPart( + self, + "assets/head.png", + np.array([0, 0, 0]), + 1, + ), + BodyPart( + self, + "assets/moustacheL.png", + np.array([13, -6, 0.1]), + 1, + ), + BodyPart( + self, + "assets/moustacheR.png", + np.array([-13, -6, 0.1]), + 1, + ), + BodyPart( + self, + "assets/eyeL.png", + np.array([2.5, 2.5, 0.1]), + 0.3, + ), + BodyPart( + self, + "assets/eyeR.png", + np.array([-2.5, 2.5, 0.1]), + 0.3, + ), + BodyPart( + self, + "assets/crown.png", + np.array([0, 6.5, 0.5]), + 0.3, + ), + BodyPart( + self, + "assets/mouth.png", + np.array([0, -3.5, 0.1]), + 0.15, + ), ] - self.pcf = PCF( - near=1, - far=10000, - frame_height=self.camera_height, - frame_width=self.camera_width, - fy=self.camera_width, - ) - # pseudo camera internals + self.dist_coeff = np.zeros((4, 1)) self.focal_length = self.camera_width self.center = (self.camera_width / 2, self.camera_height / 2) self.camera_matrix = np.array( @@ -87,10 +105,13 @@ class Environment: ], dtype="double", ) - - self.dist_coeff = np.zeros((4, 1)) - - self.refine_landmarks = True + self.pcf = PCF( + near=1, + far=10000, + frame_height=self.camera_height, + frame_width=self.camera_width, + fy=self.focal_length, + ) def start(self) -> None: """Start the environment.""" @@ -155,41 +176,46 @@ class Environment: def compute_face_axis(self) -> None: """Compute the face axis.""" if self.results.multi_face_landmarks: + # get landmarks, suppose only one face is detected face_landmarks = self.results.multi_face_landmarks[0] + + # convert landmarks to numpy array landmarks = np.array([(lm.x, lm.y, lm.z) for lm in face_landmarks.landmark]) landmarks = landmarks.T + # remove refined landmarks if self.refine_landmarks: - landmarks = landmarks[:, :468] + landmarks = landmarks[:, :UNREFINED_LANDMARKS] - metric_landmarks, pose_transform_mat = get_metric_landmarks(landmarks.copy(), self.pcf) + # get pose from landmarks + metric_landmarks, pose_transform_mat = get_metric_landmarks(landmarks, self.pcf) - self.model_points = metric_landmarks[0:3, points_idx].T - - # see here: - # https://github.com/google/mediapipe/issues/1379#issuecomment-752534379 + # extract rotation and translation vectors pose_transform_mat[1:3, :] = -pose_transform_mat[1:3, :] self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3]) self.mp_translation_vector = pose_transform_mat[:3, 3, None] - self.pose_transform_mat = pose_transform_mat + # define axis + self.center = metric_landmarks[:, NOSE_LANDMARK].T + self.x = np.array([7, 0, 0]) + self.y = np.array([0, 7, 0]) + self.z = np.array([0, 0, 7]) def draw_axis(self) -> None: """Draw the face axis on the frame.""" - nose_tip = self.model_points[0] - self.x = self.model_points[0] + [7, 0, 0] - self.y = self.model_points[0] + [0, 7, 0] - self.z = self.model_points[0] + [0, 0, 7] - - (nose_pointer2D, _) = cv2.projectPoints( - np.array([nose_tip, self.x, self.y, self.z]), + # project axis + (nose_pointers, _) = cv2.projectPoints( + np.array([np.zeros(3), self.x, self.y, self.z]) + self.center, self.mp_rotation_vector, self.mp_translation_vector, self.camera_matrix, self.dist_coeff, ) - nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointer2D.squeeze().astype(int) + # extract projected vectors + nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointers.squeeze().astype(int) + + # draw axis cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2) cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2) cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2) @@ -205,16 +231,6 @@ class Environment: landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0), ) - # draw subset of landmarks - landmark_subset = landmark_pb2.NormalizedLandmarkList( - landmark=[face_landmarks.landmark[i] for i in points_idx] - ) - self.mp_drawing.draw_landmarks( - self.frame, - landmark_subset, - landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 2, 0), - ) - def draw_avatar(self) -> None: """Draw the avatar on the screen.""" # clear avatar frame diff --git a/src/utils.py b/src/utils.py deleted file mode 100644 index 8c15e22..0000000 --- a/src/utils.py +++ /dev/null @@ -1,32 +0,0 @@ -import numpy as np -from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates - -# def landmark2vec(landmark, screen): -# """Convert a landmark to a pygame Vector2.""" -# return pg.Vector2( -# _normalized_to_pixel_coordinates( -# np.clip(landmark.x, 0, 1), -# np.clip(landmark.y, 0, 1), -# screen.get_width(), -# screen.get_height(), -# ) # type: ignore -# ) - - -def landmark2vec(landmark) -> np.ndarray: - """Convert a landmark to numpy array.""" - return np.clip( - [ - landmark.x, - landmark.y, - landmark.z, - ], - a_min=0, - a_max=1, - ) - - -LANDMARKS_LEFT_SIDE = [109, 67, 103, 54, 21, 162, 127, 234, 93, 132, 58, 172, 136, 150, 149, 176, 148] -LANDMARKS_RIGHT_SIDE = [338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377] -LANDMARKS_BOTTOM_SIDE = [93, 132, 152, 361, 323] -LANDMARKS_TOP_SIDE = [127, 162, 10, 389, 356]