feat: active bodypart
This commit is contained in:
parent
f0c99df51c
commit
005ee85d38
88
src/active_bodypart.py
Normal file
88
src/active_bodypart.py
Normal file
|
@ -0,0 +1,88 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates
|
||||||
|
|
||||||
|
from bodypart import BodyPart
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from environment import Environment
|
||||||
|
|
||||||
|
|
||||||
|
def landmark2vec(landmark, width, height):
|
||||||
|
"""Convert a landmark to numpy vector."""
|
||||||
|
return np.array(
|
||||||
|
_normalized_to_pixel_coordinates(
|
||||||
|
np.clip(landmark.x, 0, 1),
|
||||||
|
np.clip(landmark.y, 0, 1),
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ActiveBodyPart(BodyPart):
|
||||||
|
"""An active body part."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
env: Environment,
|
||||||
|
image_path: str,
|
||||||
|
position: np.ndarray,
|
||||||
|
height: float,
|
||||||
|
keypoints: tuple[int, int, int, int],
|
||||||
|
) -> None:
|
||||||
|
"""Initialize the active part."""
|
||||||
|
super().__init__(env, image_path, position, height)
|
||||||
|
self.keypoints = keypoints
|
||||||
|
|
||||||
|
def modulate_height(self) -> None:
|
||||||
|
"""Modulate the height of the part."""
|
||||||
|
if self.env.results.multi_face_landmarks:
|
||||||
|
face_landmarks = self.env.results.multi_face_landmarks[0]
|
||||||
|
|
||||||
|
left = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[0]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
right = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[1]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
top = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[3]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
bottom = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[2]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
|
||||||
|
vertical = np.linalg.norm(top - bottom)
|
||||||
|
horizontal = np.linalg.norm(right - left)
|
||||||
|
|
||||||
|
active_ratio = np.clip(0, 1, horizontal / vertical * 4)
|
||||||
|
print(active_ratio)
|
||||||
|
|
||||||
|
self.nominal_box = (
|
||||||
|
np.array(
|
||||||
|
[
|
||||||
|
self.ratio * self.env.x + self.env.y * active_ratio,
|
||||||
|
self.ratio * -self.env.x + self.env.y * active_ratio,
|
||||||
|
self.ratio * -self.env.x + -self.env.y * active_ratio,
|
||||||
|
self.ratio * self.env.x + -self.env.y * active_ratio,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
* self.height
|
||||||
|
)
|
||||||
|
|
||||||
|
def draw(self) -> None:
|
||||||
|
"""Draw the active part on the screen."""
|
||||||
|
self.modulate_height()
|
||||||
|
super().draw()
|
|
@ -12,13 +12,20 @@ if TYPE_CHECKING:
|
||||||
class BodyPart:
|
class BodyPart:
|
||||||
"""A basic body part."""
|
"""A basic body part."""
|
||||||
|
|
||||||
def __init__(self, env: Environment, image_path: str, position, height) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
env: Environment,
|
||||||
|
image_path: str,
|
||||||
|
position: np.ndarray,
|
||||||
|
height: float,
|
||||||
|
) -> None:
|
||||||
"""Initialize the part."""
|
"""Initialize the part."""
|
||||||
self.env = env
|
self.env = env
|
||||||
|
|
||||||
self.image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
|
self.image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
self.ratio = self.image.shape[1] / self.image.shape[0]
|
||||||
|
|
||||||
|
self.height = height
|
||||||
self.position = position
|
self.position = position
|
||||||
|
|
||||||
self.old_bounding_box = np.zeros((4, 1, 2))
|
self.old_bounding_box = np.zeros((4, 1, 2))
|
||||||
|
@ -61,7 +68,7 @@ class BodyPart:
|
||||||
self.env.dist_coeff,
|
self.env.dist_coeff,
|
||||||
)
|
)
|
||||||
|
|
||||||
# interpolation with self.old_box
|
# interpolation with self.old_bounding_box
|
||||||
bounding_box = (bounding_box + self.old_bounding_box) / 2
|
bounding_box = (bounding_box + self.old_bounding_box) / 2
|
||||||
self.old_bounding_box = bounding_box
|
self.old_bounding_box = bounding_box
|
||||||
|
|
||||||
|
|
|
@ -4,6 +4,7 @@ import cv2
|
||||||
import mediapipe as mp
|
import mediapipe as mp
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
|
from active_bodypart import ActiveBodyPart
|
||||||
from bodypart import BodyPart
|
from bodypart import BodyPart
|
||||||
from face_geometry import PCF, get_metric_landmarks
|
from face_geometry import PCF, get_metric_landmarks
|
||||||
|
|
||||||
|
@ -72,17 +73,19 @@ class Environment:
|
||||||
np.array([-13, -6, 0.1]),
|
np.array([-13, -6, 0.1]),
|
||||||
1,
|
1,
|
||||||
),
|
),
|
||||||
BodyPart(
|
ActiveBodyPart(
|
||||||
self,
|
self,
|
||||||
"assets/eyeL.png",
|
"assets/eyeL.png",
|
||||||
np.array([2.5, 2.5, 0.1]),
|
np.array([2.5, 2.5, 0.1]),
|
||||||
0.3,
|
0.3,
|
||||||
|
(145, 159, 133, 33),
|
||||||
),
|
),
|
||||||
BodyPart(
|
ActiveBodyPart(
|
||||||
self,
|
self,
|
||||||
"assets/eyeR.png",
|
"assets/eyeR.png",
|
||||||
np.array([-2.5, 2.5, 0.1]),
|
np.array([-2.5, 2.5, 0.1]),
|
||||||
0.3,
|
0.3,
|
||||||
|
(374, 386, 362, 263),
|
||||||
),
|
),
|
||||||
BodyPart(
|
BodyPart(
|
||||||
self,
|
self,
|
||||||
|
@ -90,11 +93,12 @@ class Environment:
|
||||||
np.array([0, 6.5, 0.5]),
|
np.array([0, 6.5, 0.5]),
|
||||||
0.3,
|
0.3,
|
||||||
),
|
),
|
||||||
BodyPart(
|
ActiveBodyPart(
|
||||||
self,
|
self,
|
||||||
"assets/mouth.png",
|
"assets/mouth.png",
|
||||||
np.array([0, -3.5, 0.1]),
|
np.array([0, -3.5, 0.1]),
|
||||||
0.15,
|
0.15,
|
||||||
|
(14, 13, 308, 78),
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue