ENSEEIHT/projet-transversal
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

refactor: reify code

Browse source
This commit is contained in:
Laureηt 2023-01-15 16:33:31 +01:00
parent 5af7cf3ee8
commit 3a263937ab
Signed by: Laurent
SSH key fingerprint: SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
12 changed files with 118 additions and 735 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
src/bodyparts/head.py → src/bodypart.py
View file

@ -9,13 +9,13 @@ if TYPE_CHECKING:
from environment import Environment from environment import Environment
class Head: class BodyPart:
"""The head body part.""" """A basic body part."""
def __init__(self, env: Environment) -> None: def __init__(self, env: Environment, image_path: str, position, height) -> None:
"""Initialize the head.""" """Initialize the part."""
self.env = env self.env = env
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED) self.image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0] self.ratio = self.image.shape[1] / self.image.shape[0]
self.bounding_box = np.array( self.bounding_box = np.array(
[ [
@ -28,24 +28,33 @@ class Head:
) )
self.old_box = np.zeros((4, 1, 2)) self.old_box = np.zeros((4, 1, 2))
self.position = position
self.height = height
def draw(self) -> None: def draw(self) -> None:
"""Draw the head on the screen.""" """Draw the part on the screen."""
# compute position # compute position
if self.env.results.multi_face_landmarks: if self.env.results.multi_face_landmarks:
x = self.env.x - self.env.model_points[0] x = self.env.x
y = self.env.y - self.env.model_points[0] y = self.env.y
for face_landmarks in self.env.results.multi_face_landmarks: for face_landmarks in self.env.results.multi_face_landmarks:
head_box = np.array( # compute bounding box from position and height
[ head_box = (
self.ratio * x + y + self.env.model_points[0], np.array(
self.ratio * -x + y + self.env.model_points[0], [
self.ratio * -x + -y + self.env.model_points[0], self.ratio * x * self.height + y * self.height,
self.ratio * x + -y + self.env.model_points[0], self.ratio * -x * self.height + y * self.height,
] self.ratio * -x * self.height + -y * self.height,
self.ratio * x * self.height + -y * self.height,
]
)
+ self.position
+ self.env.center
) )
# project bounding box to camera
(box, _) = cv2.projectPoints( (box, _) = cv2.projectPoints(
head_box, head_box,
self.env.mp_rotation_vector, self.env.mp_rotation_vector,
@ -58,12 +67,6 @@ class Head:
box = (box + self.old_box) / 2 box = (box + self.old_box) / 2
self.old_box = box self.old_box = box
a, b, c, d = box.squeeze().astype(int)
cv2.line(self.env.frame, a, b, (255, 255, 255), 2)
cv2.line(self.env.frame, b, c, (255, 255, 255), 2)
cv2.line(self.env.frame, c, d, (255, 255, 255), 2)
cv2.line(self.env.frame, d, a, (255, 255, 255), 2)
# get perspective transform # get perspective transform
transform_mat = cv2.getPerspectiveTransform( transform_mat = cv2.getPerspectiveTransform(
self.bounding_box, self.bounding_box,
@ -82,13 +85,10 @@ class Head:
def draw_debug(self) -> None: def draw_debug(self) -> None:
"""Draw debug information on the screen.""" """Draw debug information on the screen."""
# link points cv2.polylines(
pass self.env.frame,
# for i in range(4): [self.old_box.squeeze().astype(int)],
# cv2.line( True,
# self.env.frame, (255, 255, 255),
# tuple(self.translated_head_box[i].astype(np.int32)), 2,
# tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)), )
# (255, 255, 255),
# 2,
# )

9
src/bodyparts/__init__.py
View file

@ -1,9 +0,0 @@
from .crown import Crown
from .head import Head
from .left_ear import LeftEar
from .left_eye import LeftEye
from .left_moustache import LeftMoustache
from .mouth import Mouth
from .right_ear import RightEar
from .right_eye import RightEye
from .right_moustache import RightMoustache

74
src/bodyparts/crown.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class Crown:
"""The crown body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the crown."""
self.env = env
self.image = cv2.imread("assets/crown.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the crown on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
30 * self.ratio * self.env.x + 135 * self.env.y + 90 * self.env.z,
-30 * self.ratio * self.env.x + 135 * self.env.y + 90 * self.env.z,
-30 * self.ratio * self.env.x + 75 * self.env.y + 90 * self.env.z,
30 * self.ratio * self.env.x + 75 * self.env.y + 90 * self.env.z,
]
)[:, :2]
self.translated_box = (
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

74
src/bodyparts/left_ear.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class LeftEar:
"""The left eye body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the left eye."""
self.env = env
self.image = cv2.imread("assets/earL.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the left eye on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
100 * self.env.x + 250 * self.env.y + 70 * self.env.z,
30 * self.env.x + 250 * self.env.y + 70 * self.env.z,
30 * self.env.x + 50 * self.env.y + 70 * self.env.z,
100 * self.env.x + 50 * self.env.y + 70 * self.env.z,
]
)[:, :2]
self.translated_box = (
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

74
src/bodyparts/left_eye.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class LeftEye:
"""The left eye body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the left eye."""
self.env = env
self.image = cv2.imread("assets/eyeL.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the left eye on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
40 * self.env.x + 70 * self.env.y + 80 * self.env.z,
25 * self.env.x + 70 * self.env.y + 80 * self.env.z,
25 * self.env.x + 15 * self.env.y + 80 * self.env.z,
40 * self.env.x + 15 * self.env.y + 80 * self.env.z,
]
)[:, :2]
self.translated_box = (
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

74
src/bodyparts/left_moustache.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class LeftMoustache:
"""The left eye body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the left eye."""
self.env = env
self.image = cv2.imread("assets/moustacheL.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the left eye on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
270 * self.env.x + 20 * self.env.y + 80 * self.env.z,
70 * self.env.x + 20 * self.env.y + 80 * self.env.z,
70 * self.env.x + -130 * self.env.y + 80 * self.env.z,
270 * self.env.x + -130 * self.env.y + 80 * self.env.z,
]
)[:, :2]
self.translated_box = (
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

74
src/bodyparts/mouth.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class Mouth:
"""The mouth body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the mouth."""
self.env = env
self.image = cv2.imread("assets/mouth.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the mouth on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
15 * self.ratio * self.env.x + -30 * self.env.y + 80 * self.env.z,
-15 * self.ratio * self.env.x + -30 * self.env.y + 80 * self.env.z,
-15 * self.ratio * self.env.x + -50 * self.env.y + 80 * self.env.z,
15 * self.ratio * self.env.x + -50 * self.env.y + 80 * self.env.z,
]
)[:, :2]
self.translated_box = (
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

74
src/bodyparts/right_ear.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class RightEar:
"""The left eye body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the left eye."""
self.env = env
self.image = cv2.imread("assets/earR.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the left eye on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
-30 * self.env.x + 250 * self.env.y + 70 * self.env.z,
-100 * self.env.x + 250 * self.env.y + 70 * self.env.z,
-100 * self.env.x + 50 * self.env.y + 70 * self.env.z,
-30 * self.env.x + 50 * self.env.y + 70 * self.env.z,
]
)[:, :2]
self.translated_box = (
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

74
src/bodyparts/right_eye.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class RightEye:
"""The right eye body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the right eye."""
self.env = env
self.image = cv2.imread("assets/eyeR.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the right eye on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
-25 * self.env.x + 70 * self.env.y + 80 * self.env.z,
-40 * self.env.x + 70 * self.env.y + 80 * self.env.z,
-40 * self.env.x + 15 * self.env.y + 80 * self.env.z,
-25 * self.env.x + 15 * self.env.y + 80 * self.env.z,
]
)
self.translated_box = (
box[:, :2] + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

74
src/bodyparts/right_moustache.py
View file

@ -1,74 +0,0 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import cv2
import numpy as np
if TYPE_CHECKING:
from environment import Environment
class RightMoustache:
"""The left eye body part."""
def __init__(self, env: Environment) -> None:
"""Initialize the left eye."""
self.env = env
self.image = cv2.imread("assets/moustacheR.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
[self.image.shape[1], self.image.shape[0]],
[0, self.image.shape[0]],
],
dtype=np.float32,
)
def draw(self) -> None:
"""Draw the left eye on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
for face_landmarks in self.env.results.multi_face_landmarks:
box = np.array(
[
-70 * self.env.x + 20 * self.env.y + 80 * self.env.z,
-270 * self.env.x + 20 * self.env.y + 80 * self.env.z,
-270 * self.env.x + -130 * self.env.y + 80 * self.env.z,
-70 * self.env.x + -130 * self.env.y + 80 * self.env.z,
]
)[:, :2]
self.translated_box = (
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_box,
)
# apply perspective transform to image
warped = cv2.warpPerspective(
self.image,
transform_mat,
self.env.frame.shape[1::-1],
)
# replace non black pixels of warped image by frame
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_box[i].astype(np.int32)),
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)

158
src/environment.py
View file

@ -3,33 +3,12 @@ import logging
import cv2 import cv2
import mediapipe as mp import mediapipe as mp
import numpy as np import numpy as np
from mediapipe.framework.formats import landmark_pb2
from bodyparts import ( from bodypart import BodyPart
Crown, from face_geometry import PCF, get_metric_landmarks
Head,
LeftEar,
LeftEye,
LeftMoustache,
Mouth,
RightEar,
RightEye,
RightMoustache,
)
from face_geometry import PCF, get_metric_landmarks, procrustes_landmark_basis
from utils import (
LANDMARKS_BOTTOM_SIDE,
LANDMARKS_LEFT_SIDE,
LANDMARKS_RIGHT_SIDE,
LANDMARKS_TOP_SIDE,
landmark2vec,
)
points_idx = [33, 263, 61, 291, 199] NOSE_LANDMARK = 4
points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis] UNREFINED_LANDMARKS = 468
points_idx = list(set(points_idx))
points_idx.sort()
print(points_idx)
class Environment: class Environment:
@ -50,6 +29,7 @@ class Environment:
self.mp_drawing = mp.solutions.drawing_utils # type: ignore self.mp_drawing = mp.solutions.drawing_utils # type: ignore
self.mp_drawing_styles = mp.solutions.drawing_styles # type: ignore self.mp_drawing_styles = mp.solutions.drawing_styles # type: ignore
self.mp_face_mesh = mp.solutions.face_mesh # type: ignore self.mp_face_mesh = mp.solutions.face_mesh # type: ignore
self.refine_landmarks = True
# get screen size from webcam # get screen size from webcam
self.camera_width = camera.get(3) self.camera_width = camera.get(3)
@ -57,26 +37,64 @@ class Environment:
# create body parts # create body parts
self.body_parts = [ self.body_parts = [
# LeftEar(self), BodyPart(
# RightEar(self), self,
Head(self), "assets/earL.png",
# RightMoustache(self), np.array([6, 11, -0.5]),
# LeftMoustache(self), 1,
# LeftEye(self), ),
# RightEye(self), BodyPart(
# Crown(self), self,
# Mouth(self), "assets/earR.png",
np.array([-6, 11, -0.5]),
1,
),
BodyPart(
self,
"assets/head.png",
np.array([0, 0, 0]),
1,
),
BodyPart(
self,
"assets/moustacheL.png",
np.array([13, -6, 0.1]),
1,
),
BodyPart(
self,
"assets/moustacheR.png",
np.array([-13, -6, 0.1]),
1,
),
BodyPart(
self,
"assets/eyeL.png",
np.array([2.5, 2.5, 0.1]),
0.3,
),
BodyPart(
self,
"assets/eyeR.png",
np.array([-2.5, 2.5, 0.1]),
0.3,
),
BodyPart(
self,
"assets/crown.png",
np.array([0, 6.5, 0.5]),
0.3,
),
BodyPart(
self,
"assets/mouth.png",
np.array([0, -3.5, 0.1]),
0.15,
),
] ]
self.pcf = PCF(
near=1,
far=10000,
frame_height=self.camera_height,
frame_width=self.camera_width,
fy=self.camera_width,
)
# pseudo camera internals # pseudo camera internals
self.dist_coeff = np.zeros((4, 1))
self.focal_length = self.camera_width self.focal_length = self.camera_width
self.center = (self.camera_width / 2, self.camera_height / 2) self.center = (self.camera_width / 2, self.camera_height / 2)
self.camera_matrix = np.array( self.camera_matrix = np.array(
@ -87,10 +105,13 @@ class Environment:
], ],
dtype="double", dtype="double",
) )
self.pcf = PCF(
self.dist_coeff = np.zeros((4, 1)) near=1,
far=10000,
self.refine_landmarks = True frame_height=self.camera_height,
frame_width=self.camera_width,
fy=self.focal_length,
)
def start(self) -> None: def start(self) -> None:
"""Start the environment.""" """Start the environment."""
@ -155,41 +176,46 @@ class Environment:
def compute_face_axis(self) -> None: def compute_face_axis(self) -> None:
"""Compute the face axis.""" """Compute the face axis."""
if self.results.multi_face_landmarks: if self.results.multi_face_landmarks:
# get landmarks, suppose only one face is detected
face_landmarks = self.results.multi_face_landmarks[0] face_landmarks = self.results.multi_face_landmarks[0]
# convert landmarks to numpy array
landmarks = np.array([(lm.x, lm.y, lm.z) for lm in face_landmarks.landmark]) landmarks = np.array([(lm.x, lm.y, lm.z) for lm in face_landmarks.landmark])
landmarks = landmarks.T landmarks = landmarks.T
# remove refined landmarks
if self.refine_landmarks: if self.refine_landmarks:
landmarks = landmarks[:, :468] landmarks = landmarks[:, :UNREFINED_LANDMARKS]
metric_landmarks, pose_transform_mat = get_metric_landmarks(landmarks.copy(), self.pcf) # get pose from landmarks
metric_landmarks, pose_transform_mat = get_metric_landmarks(landmarks, self.pcf)
self.model_points = metric_landmarks[0:3, points_idx].T # extract rotation and translation vectors
# see here:
# https://github.com/google/mediapipe/issues/1379#issuecomment-752534379
pose_transform_mat[1:3, :] = -pose_transform_mat[1:3, :] pose_transform_mat[1:3, :] = -pose_transform_mat[1:3, :]
self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3]) self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
self.mp_translation_vector = pose_transform_mat[:3, 3, None] self.mp_translation_vector = pose_transform_mat[:3, 3, None]
self.pose_transform_mat = pose_transform_mat # define axis
self.center = metric_landmarks[:, NOSE_LANDMARK].T
self.x = np.array([7, 0, 0])
self.y = np.array([0, 7, 0])
self.z = np.array([0, 0, 7])
def draw_axis(self) -> None: def draw_axis(self) -> None:
"""Draw the face axis on the frame.""" """Draw the face axis on the frame."""
nose_tip = self.model_points[0] # project axis
self.x = self.model_points[0] + [7, 0, 0] (nose_pointers, _) = cv2.projectPoints(
self.y = self.model_points[0] + [0, 7, 0] np.array([np.zeros(3), self.x, self.y, self.z]) + self.center,
self.z = self.model_points[0] + [0, 0, 7]
(nose_pointer2D, _) = cv2.projectPoints(
np.array([nose_tip, self.x, self.y, self.z]),
self.mp_rotation_vector, self.mp_rotation_vector,
self.mp_translation_vector, self.mp_translation_vector,
self.camera_matrix, self.camera_matrix,
self.dist_coeff, self.dist_coeff,
) )
nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointer2D.squeeze().astype(int) # extract projected vectors
nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointers.squeeze().astype(int)
# draw axis
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2) cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2) cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2) cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2)
@ -205,16 +231,6 @@ class Environment:
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0), landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0),
) )
# draw subset of landmarks
landmark_subset = landmark_pb2.NormalizedLandmarkList(
landmark=[face_landmarks.landmark[i] for i in points_idx]
)
self.mp_drawing.draw_landmarks(
self.frame,
landmark_subset,
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 2, 0),
)
def draw_avatar(self) -> None: def draw_avatar(self) -> None:
"""Draw the avatar on the screen.""" """Draw the avatar on the screen."""
# clear avatar frame # clear avatar frame

32
src/utils.py
View file

@ -1,32 +0,0 @@
import numpy as np
from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates
# def landmark2vec(landmark, screen):
# """Convert a landmark to a pygame Vector2."""
# return pg.Vector2(
# _normalized_to_pixel_coordinates(
# np.clip(landmark.x, 0, 1),
# np.clip(landmark.y, 0, 1),
# screen.get_width(),
# screen.get_height(),
# ) # type: ignore
# )
def landmark2vec(landmark) -> np.ndarray:
"""Convert a landmark to numpy array."""
return np.clip(
[
landmark.x,
landmark.y,
landmark.z,
],
a_min=0,
a_max=1,
)
LANDMARKS_LEFT_SIDE = [109, 67, 103, 54, 21, 162, 127, 234, 93, 132, 58, 172, 136, 150, 149, 176, 148]
LANDMARKS_RIGHT_SIDE = [338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377]
LANDMARKS_BOTTOM_SIDE = [93, 132, 152, 361, 323]
LANDMARKS_TOP_SIDE = [127, 162, 10, 389, 356]