feat: active bodypart

This commit is contained in:
Laureηt 2023-01-19 09:08:25 +01:00
parent f0c99df51c
commit 005ee85d38
Signed by: Laurent
SSH key fingerprint: SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
3 changed files with 104 additions and 5 deletions

88
src/active_bodypart.py Normal file
View file

@ -0,0 +1,88 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import numpy as np
from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates
from bodypart import BodyPart
if TYPE_CHECKING:
from environment import Environment
def landmark2vec(landmark, width, height):
"""Convert a landmark to numpy vector."""
return np.array(
_normalized_to_pixel_coordinates(
np.clip(landmark.x, 0, 1),
np.clip(landmark.y, 0, 1),
width,
height,
)
)
class ActiveBodyPart(BodyPart):
"""An active body part."""
def __init__(
self,
env: Environment,
image_path: str,
position: np.ndarray,
height: float,
keypoints: tuple[int, int, int, int],
) -> None:
"""Initialize the active part."""
super().__init__(env, image_path, position, height)
self.keypoints = keypoints
def modulate_height(self) -> None:
"""Modulate the height of the part."""
if self.env.results.multi_face_landmarks:
face_landmarks = self.env.results.multi_face_landmarks[0]
left = landmark2vec(
face_landmarks.landmark[self.keypoints[0]],
self.env.camera_width,
self.env.camera_height,
)
right = landmark2vec(
face_landmarks.landmark[self.keypoints[1]],
self.env.camera_width,
self.env.camera_height,
)
top = landmark2vec(
face_landmarks.landmark[self.keypoints[3]],
self.env.camera_width,
self.env.camera_height,
)
bottom = landmark2vec(
face_landmarks.landmark[self.keypoints[2]],
self.env.camera_width,
self.env.camera_height,
)
vertical = np.linalg.norm(top - bottom)
horizontal = np.linalg.norm(right - left)
active_ratio = np.clip(0, 1, horizontal / vertical * 4)
print(active_ratio)
self.nominal_box = (
np.array(
[
self.ratio * self.env.x + self.env.y * active_ratio,
self.ratio * -self.env.x + self.env.y * active_ratio,
self.ratio * -self.env.x + -self.env.y * active_ratio,
self.ratio * self.env.x + -self.env.y * active_ratio,
]
)
* self.height
)
def draw(self) -> None:
"""Draw the active part on the screen."""
self.modulate_height()
super().draw()

View file

@ -12,13 +12,20 @@ if TYPE_CHECKING:
class BodyPart: class BodyPart:
"""A basic body part.""" """A basic body part."""
def __init__(self, env: Environment, image_path: str, position, height) -> None: def __init__(
self,
env: Environment,
image_path: str,
position: np.ndarray,
height: float,
) -> None:
"""Initialize the part.""" """Initialize the part."""
self.env = env self.env = env
self.image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED) self.image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0] self.ratio = self.image.shape[1] / self.image.shape[0]
self.height = height
self.position = position self.position = position
self.old_bounding_box = np.zeros((4, 1, 2)) self.old_bounding_box = np.zeros((4, 1, 2))
@ -61,7 +68,7 @@ class BodyPart:
self.env.dist_coeff, self.env.dist_coeff,
) )
# interpolation with self.old_box # interpolation with self.old_bounding_box
bounding_box = (bounding_box + self.old_bounding_box) / 2 bounding_box = (bounding_box + self.old_bounding_box) / 2
self.old_bounding_box = bounding_box self.old_bounding_box = bounding_box

View file

@ -4,6 +4,7 @@ import cv2
import mediapipe as mp import mediapipe as mp
import numpy as np import numpy as np
from active_bodypart import ActiveBodyPart
from bodypart import BodyPart from bodypart import BodyPart
from face_geometry import PCF, get_metric_landmarks from face_geometry import PCF, get_metric_landmarks
@ -72,17 +73,19 @@ class Environment:
np.array([-13, -6, 0.1]), np.array([-13, -6, 0.1]),
1, 1,
), ),
BodyPart( ActiveBodyPart(
self, self,
"assets/eyeL.png", "assets/eyeL.png",
np.array([2.5, 2.5, 0.1]), np.array([2.5, 2.5, 0.1]),
0.3, 0.3,
(145, 159, 133, 33),
), ),
BodyPart( ActiveBodyPart(
self, self,
"assets/eyeR.png", "assets/eyeR.png",
np.array([-2.5, 2.5, 0.1]), np.array([-2.5, 2.5, 0.1]),
0.3, 0.3,
(374, 386, 362, 263),
), ),
BodyPart( BodyPart(
self, self,
@ -90,11 +93,12 @@ class Environment:
np.array([0, 6.5, 0.5]), np.array([0, 6.5, 0.5]),
0.3, 0.3,
), ),
BodyPart( ActiveBodyPart(
self, self,
"assets/mouth.png", "assets/mouth.png",
np.array([0, -3.5, 0.1]), np.array([0, -3.5, 0.1]),
0.15, 0.15,
(14, 13, 308, 78),
), ),
] ]