Compare commits
10 commits
eb2cf131d5
...
ff99203af6
Author | SHA1 | Date | |
---|---|---|---|
Laureηt | ff99203af6 | ||
Laureηt | 61c6baed4f | ||
Laureηt | 005ee85d38 | ||
Laureηt | f0c99df51c | ||
Laureηt | 5455a4f962 | ||
Laureηt | 3a263937ab | ||
Laureηt | 5af7cf3ee8 | ||
Laureηt | 6cd40b7d8e | ||
Laureηt | 97fbc9bdb3 | ||
Laureηt | fdfed4c1ff |
22
README.md
22
README.md
|
@ -1 +1,23 @@
|
||||||
Créez vos propre branches
|
Créez vos propre branches
|
||||||
|
|
||||||
|
# random stuff
|
||||||
|
|
||||||
|
https://scikit-image.org/docs/stable/auto_examples/transform/plot_geometric.html#sphx-glr-auto-examples-transform-plot-geometric-py
|
||||||
|
https://scikit-image.org/docs/stable/auto_examples/transform/plot_transform_types.html#sphx-glr-auto-examples-transform-plot-transform-types-py
|
||||||
|
https://github.com/google/mediapipe/issues/1379
|
||||||
|
https://en.wikipedia.org/wiki/3D_projection#Perspective_projection
|
||||||
|
|
||||||
|
|
||||||
|
TODO: faire un POC où je place juste un cvGetPerspectiveTransform suivi d'un cvWarpPerspective
|
||||||
|
-> comme sur cet exemple, mais où l'image de gauche et droite sont inversées
|
||||||
|
https://docs.adaptive-vision.com/studio/filters/GeometricImageTransformations/cvGetPerspectiveTransform.html
|
||||||
|
|
||||||
|
instrisics ? -> https://github.dev/google/mediapipe/blob/master/mediapipe/modules/face_geometry/libs/effect_renderer.cc#L573-L599
|
||||||
|
|
||||||
|
TODO: https://github.com/Rassibassi/mediapipeFacegeometryPython/blob/main/head_posture_rt.py
|
||||||
|
-> pnp -> pose estimation -> paramètres extrinsèques
|
||||||
|
-> + param intrasèque (supposé connu, check site mediapipe)
|
||||||
|
-> placer dans l'espace les textures -> et projeter dans le plan image
|
||||||
|
|
||||||
|
|
||||||
|
https://github.com/Rassibassi/mediapipeDemos
|
||||||
|
|
43
flake.lock
Normal file
43
flake.lock
Normal file
|
@ -0,0 +1,43 @@
|
||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"flake-utils": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1667395993,
|
||||||
|
"narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "numtide",
|
||||||
|
"repo": "flake-utils",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1673796341,
|
||||||
|
"narHash": "sha256-1kZi9OkukpNmOaPY7S5/+SlCDOuYnP3HkXHvNDyLQcc=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "6dccdc458512abce8d19f74195bb20fdb067df50",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixos-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"flake-utils": "flake-utils",
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
24
flake.nix
Normal file
24
flake.nix
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
{
|
||||||
|
description = "Proj APP";
|
||||||
|
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||||
|
flake-utils.url = "github:numtide/flake-utils";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs = { self, nixpkgs, flake-utils }:
|
||||||
|
flake-utils.lib.eachDefaultSystem (system:
|
||||||
|
let pkgs = nixpkgs.legacyPackages.${system};
|
||||||
|
in {
|
||||||
|
devShell = pkgs.mkShell {
|
||||||
|
buildInputs = with pkgs; [
|
||||||
|
poetry
|
||||||
|
python3
|
||||||
|
(python310Packages.opencv4.override {
|
||||||
|
enableGtk2 = true;
|
||||||
|
gtk2 = pkgs.gtk2;
|
||||||
|
})
|
||||||
|
];
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
12
shell.nix
12
shell.nix
|
@ -1,12 +0,0 @@
|
||||||
{ pkgs ? import <nixpkgs> { } }:
|
|
||||||
|
|
||||||
pkgs.mkShell {
|
|
||||||
buildInputs = with pkgs; [
|
|
||||||
poetry
|
|
||||||
python3
|
|
||||||
(python310Packages.opencv4.override {
|
|
||||||
enableGtk2 = true;
|
|
||||||
gtk2 = pkgs.gtk2;
|
|
||||||
})
|
|
||||||
];
|
|
||||||
}
|
|
87
src/active_bodypart.py
Normal file
87
src/active_bodypart.py
Normal file
|
@ -0,0 +1,87 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates
|
||||||
|
|
||||||
|
from bodypart import BodyPart
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from environment import Environment
|
||||||
|
|
||||||
|
|
||||||
|
def landmark2vec(landmark, width, height):
|
||||||
|
"""Convert a landmark to numpy vector."""
|
||||||
|
return np.array(
|
||||||
|
_normalized_to_pixel_coordinates(
|
||||||
|
np.clip(landmark.x, 0, 1),
|
||||||
|
np.clip(landmark.y, 0, 1),
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ActiveBodyPart(BodyPart):
|
||||||
|
"""An active body part."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
env: Environment,
|
||||||
|
image_path: str,
|
||||||
|
position: np.ndarray,
|
||||||
|
height: float,
|
||||||
|
keypoints: tuple[int, int, int, int],
|
||||||
|
) -> None:
|
||||||
|
"""Initialize the active part."""
|
||||||
|
super().__init__(env, image_path, position, height)
|
||||||
|
self.keypoints = keypoints
|
||||||
|
|
||||||
|
def modulate_height(self) -> None:
|
||||||
|
"""Modulate the height of the part."""
|
||||||
|
if self.env.results.multi_face_landmarks:
|
||||||
|
face_landmarks = self.env.results.multi_face_landmarks[0]
|
||||||
|
|
||||||
|
left = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[0]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
right = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[1]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
top = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[3]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
bottom = landmark2vec(
|
||||||
|
face_landmarks.landmark[self.keypoints[2]],
|
||||||
|
self.env.camera_width,
|
||||||
|
self.env.camera_height,
|
||||||
|
)
|
||||||
|
|
||||||
|
vertical = np.linalg.norm(top - bottom)
|
||||||
|
horizontal = np.linalg.norm(right - left)
|
||||||
|
|
||||||
|
active_ratio = np.clip(horizontal / vertical * 4, 0.1, 1)
|
||||||
|
|
||||||
|
self.nominal_box = (
|
||||||
|
np.array(
|
||||||
|
[
|
||||||
|
self.ratio * self.env.x + self.env.y * active_ratio,
|
||||||
|
self.ratio * -self.env.x + self.env.y * active_ratio,
|
||||||
|
self.ratio * -self.env.x + -self.env.y * active_ratio,
|
||||||
|
self.ratio * self.env.x + -self.env.y * active_ratio,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
* self.height
|
||||||
|
)
|
||||||
|
|
||||||
|
def draw(self) -> None:
|
||||||
|
"""Draw the active part on the screen."""
|
||||||
|
self.modulate_height()
|
||||||
|
super().draw()
|
127
src/bodypart.py
Normal file
127
src/bodypart.py
Normal file
|
@ -0,0 +1,127 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from environment import Environment
|
||||||
|
|
||||||
|
|
||||||
|
class BodyPart:
|
||||||
|
"""A basic body part."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
env: Environment,
|
||||||
|
image_path: str,
|
||||||
|
position: np.ndarray,
|
||||||
|
height: float,
|
||||||
|
wavy=None,
|
||||||
|
) -> None:
|
||||||
|
"""Initialize the part."""
|
||||||
|
self.env = env
|
||||||
|
|
||||||
|
self.image = cv2.imread(image_path, cv2.IMREAD_UNCHANGED)
|
||||||
|
self.ratio = self.image.shape[1] / self.image.shape[0]
|
||||||
|
|
||||||
|
self.height = height
|
||||||
|
self.position = position
|
||||||
|
|
||||||
|
self.old_bounding_box = np.zeros((4, 1, 2))
|
||||||
|
self.nominal_box = (
|
||||||
|
np.array(
|
||||||
|
[
|
||||||
|
self.ratio * self.env.x + self.env.y,
|
||||||
|
self.ratio * -self.env.x + self.env.y,
|
||||||
|
self.ratio * -self.env.x + -self.env.y,
|
||||||
|
self.ratio * self.env.x + -self.env.y,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
* height
|
||||||
|
)
|
||||||
|
self.image_box = np.array(
|
||||||
|
[
|
||||||
|
[0, 0],
|
||||||
|
[self.image.shape[1], 0],
|
||||||
|
[self.image.shape[1], self.image.shape[0]],
|
||||||
|
[0, self.image.shape[0]],
|
||||||
|
],
|
||||||
|
dtype=np.float32,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.wavy = wavy
|
||||||
|
|
||||||
|
def draw(self) -> None:
|
||||||
|
"""Draw the part on the screen."""
|
||||||
|
# compute position
|
||||||
|
if self.env.results.multi_face_landmarks:
|
||||||
|
|
||||||
|
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||||
|
# compute bounding box from position and height
|
||||||
|
bounding_box = self.nominal_box + self.position + self.env.center
|
||||||
|
|
||||||
|
# project bounding box to camera
|
||||||
|
(bounding_box, _) = cv2.projectPoints(
|
||||||
|
bounding_box,
|
||||||
|
self.env.mp_rotation_vector,
|
||||||
|
self.env.mp_translation_vector,
|
||||||
|
self.env.camera_matrix,
|
||||||
|
self.env.dist_coeff,
|
||||||
|
)
|
||||||
|
|
||||||
|
# interpolation with self.old_bounding_box
|
||||||
|
bounding_box = (bounding_box + self.old_bounding_box) / 2
|
||||||
|
self.old_bounding_box = bounding_box
|
||||||
|
|
||||||
|
# get perspective transform
|
||||||
|
transform_mat = cv2.getPerspectiveTransform(
|
||||||
|
self.image_box,
|
||||||
|
bounding_box.astype(np.float32),
|
||||||
|
)
|
||||||
|
|
||||||
|
# apply perspective transform to image
|
||||||
|
warped = cv2.warpPerspective(
|
||||||
|
self.image,
|
||||||
|
transform_mat,
|
||||||
|
self.env.frame.shape[1::-1],
|
||||||
|
)
|
||||||
|
|
||||||
|
if self.wavy:
|
||||||
|
# move left side of bounding box up and down sinusoidally
|
||||||
|
self.sin_box = self.wavy(bounding_box.copy(), self.env.frame_count)
|
||||||
|
|
||||||
|
# compute affine transform
|
||||||
|
sin_mat = cv2.getAffineTransform(
|
||||||
|
bounding_box[[0, 1, 3]].astype(np.float32),
|
||||||
|
self.sin_box[[0, 1, 3]].astype(np.float32),
|
||||||
|
)
|
||||||
|
|
||||||
|
# apply affine transform to image
|
||||||
|
warped = cv2.warpAffine(
|
||||||
|
warped,
|
||||||
|
sin_mat,
|
||||||
|
self.env.frame.shape[1::-1],
|
||||||
|
)
|
||||||
|
|
||||||
|
# replace non black pixels of warped image by frame
|
||||||
|
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
||||||
|
|
||||||
|
def draw_debug(self) -> None:
|
||||||
|
"""Draw debug information on the screen."""
|
||||||
|
cv2.polylines(
|
||||||
|
self.env.frame,
|
||||||
|
[self.old_bounding_box.squeeze().astype(int)],
|
||||||
|
True,
|
||||||
|
(255, 255, 255),
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
if self.wavy:
|
||||||
|
cv2.polylines(
|
||||||
|
self.env.frame,
|
||||||
|
[self.sin_box.squeeze().astype(int)],
|
||||||
|
True,
|
||||||
|
(0, 0, 255),
|
||||||
|
2,
|
||||||
|
)
|
|
@ -1,9 +0,0 @@
|
||||||
from .crown import Crown
|
|
||||||
from .head import Head
|
|
||||||
from .left_ear import LeftEar
|
|
||||||
from .left_eye import LeftEye
|
|
||||||
from .left_moustache import LeftMoustache
|
|
||||||
from .mouth import Mouth
|
|
||||||
from .right_ear import RightEar
|
|
||||||
from .right_eye import RightEye
|
|
||||||
from .right_moustache import RightMoustache
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class Crown:
|
|
||||||
"""The crown body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the crown."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/crown.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the crown on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
30 * self.ratio * self.env.x + 135 * self.env.y + 90 * self.env.z,
|
|
||||||
-30 * self.ratio * self.env.x + 135 * self.env.y + 90 * self.env.z,
|
|
||||||
-30 * self.ratio * self.env.x + 75 * self.env.y + 90 * self.env.z,
|
|
||||||
30 * self.ratio * self.env.x + 75 * self.env.y + 90 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class Head:
|
|
||||||
"""The head body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the head."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the head on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
head_box = np.array(
|
|
||||||
[
|
|
||||||
100 * self.ratio * self.env.x + 100 * self.env.y + 80 * self.env.z,
|
|
||||||
100 * self.ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z,
|
|
||||||
100 * self.ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
|
||||||
100 * self.ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_head_box = (
|
|
||||||
head_box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_head_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_head_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class LeftEar:
|
|
||||||
"""The left eye body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the left eye."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/earL.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the left eye on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
100 * self.env.x + 250 * self.env.y + 70 * self.env.z,
|
|
||||||
30 * self.env.x + 250 * self.env.y + 70 * self.env.z,
|
|
||||||
30 * self.env.x + 50 * self.env.y + 70 * self.env.z,
|
|
||||||
100 * self.env.x + 50 * self.env.y + 70 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class LeftEye:
|
|
||||||
"""The left eye body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the left eye."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/eyeL.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the left eye on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
40 * self.env.x + 70 * self.env.y + 80 * self.env.z,
|
|
||||||
25 * self.env.x + 70 * self.env.y + 80 * self.env.z,
|
|
||||||
25 * self.env.x + 15 * self.env.y + 80 * self.env.z,
|
|
||||||
40 * self.env.x + 15 * self.env.y + 80 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class LeftMoustache:
|
|
||||||
"""The left eye body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the left eye."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/moustacheL.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the left eye on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
270 * self.env.x + 20 * self.env.y + 80 * self.env.z,
|
|
||||||
70 * self.env.x + 20 * self.env.y + 80 * self.env.z,
|
|
||||||
70 * self.env.x + -130 * self.env.y + 80 * self.env.z,
|
|
||||||
270 * self.env.x + -130 * self.env.y + 80 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class Mouth:
|
|
||||||
"""The mouth body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the mouth."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/mouth.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the mouth on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
15 * self.ratio * self.env.x + -30 * self.env.y + 80 * self.env.z,
|
|
||||||
-15 * self.ratio * self.env.x + -30 * self.env.y + 80 * self.env.z,
|
|
||||||
-15 * self.ratio * self.env.x + -50 * self.env.y + 80 * self.env.z,
|
|
||||||
15 * self.ratio * self.env.x + -50 * self.env.y + 80 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class RightEar:
|
|
||||||
"""The left eye body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the left eye."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/earR.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the left eye on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
-30 * self.env.x + 250 * self.env.y + 70 * self.env.z,
|
|
||||||
-100 * self.env.x + 250 * self.env.y + 70 * self.env.z,
|
|
||||||
-100 * self.env.x + 50 * self.env.y + 70 * self.env.z,
|
|
||||||
-30 * self.env.x + 50 * self.env.y + 70 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class RightEye:
|
|
||||||
"""The right eye body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the right eye."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/eyeR.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the right eye on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
-25 * self.env.x + 70 * self.env.y + 80 * self.env.z,
|
|
||||||
-40 * self.env.x + 70 * self.env.y + 80 * self.env.z,
|
|
||||||
-40 * self.env.x + 15 * self.env.y + 80 * self.env.z,
|
|
||||||
-25 * self.env.x + 15 * self.env.y + 80 * self.env.z,
|
|
||||||
]
|
|
||||||
)
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box[:, :2] + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -1,74 +0,0 @@
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from environment import Environment
|
|
||||||
|
|
||||||
|
|
||||||
class RightMoustache:
|
|
||||||
"""The left eye body part."""
|
|
||||||
|
|
||||||
def __init__(self, env: Environment) -> None:
|
|
||||||
"""Initialize the left eye."""
|
|
||||||
self.env = env
|
|
||||||
self.image = cv2.imread("assets/moustacheR.png", cv2.IMREAD_UNCHANGED)
|
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
|
||||||
self.bouding_box = np.array(
|
|
||||||
[
|
|
||||||
[0, 0],
|
|
||||||
[self.image.shape[1], 0],
|
|
||||||
[self.image.shape[1], self.image.shape[0]],
|
|
||||||
[0, self.image.shape[0]],
|
|
||||||
],
|
|
||||||
dtype=np.float32,
|
|
||||||
)
|
|
||||||
|
|
||||||
def draw(self) -> None:
|
|
||||||
"""Draw the left eye on the screen."""
|
|
||||||
# compute position
|
|
||||||
if self.env.results.multi_face_landmarks:
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
|
||||||
box = np.array(
|
|
||||||
[
|
|
||||||
-70 * self.env.x + 20 * self.env.y + 80 * self.env.z,
|
|
||||||
-270 * self.env.x + 20 * self.env.y + 80 * self.env.z,
|
|
||||||
-270 * self.env.x + -130 * self.env.y + 80 * self.env.z,
|
|
||||||
-70 * self.env.x + -130 * self.env.y + 80 * self.env.z,
|
|
||||||
]
|
|
||||||
)[:, :2]
|
|
||||||
|
|
||||||
self.translated_box = (
|
|
||||||
box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
|
||||||
).astype(np.float32)
|
|
||||||
|
|
||||||
# get perspective transform
|
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
|
||||||
self.bouding_box,
|
|
||||||
self.translated_box,
|
|
||||||
)
|
|
||||||
|
|
||||||
# apply perspective transform to image
|
|
||||||
warped = cv2.warpPerspective(
|
|
||||||
self.image,
|
|
||||||
transform_mat,
|
|
||||||
self.env.frame.shape[1::-1],
|
|
||||||
)
|
|
||||||
|
|
||||||
# replace non black pixels of warped image by frame
|
|
||||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
|
||||||
|
|
||||||
def draw_debug(self) -> None:
|
|
||||||
"""Draw debug information on the screen."""
|
|
||||||
# link points
|
|
||||||
for i in range(4):
|
|
||||||
cv2.line(
|
|
||||||
self.env.frame,
|
|
||||||
tuple(self.translated_box[i].astype(np.int32)),
|
|
||||||
tuple(self.translated_box[(i + 1) % 4].astype(np.int32)),
|
|
||||||
(255, 255, 255),
|
|
||||||
2,
|
|
||||||
)
|
|
|
@ -4,24 +4,36 @@ import cv2
|
||||||
import mediapipe as mp
|
import mediapipe as mp
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
from bodyparts import (
|
from active_bodypart import ActiveBodyPart
|
||||||
Crown,
|
from bodypart import BodyPart
|
||||||
Head,
|
from face_geometry import PCF, get_metric_landmarks
|
||||||
LeftEar,
|
|
||||||
LeftEye,
|
NOSE_LANDMARK = 4
|
||||||
LeftMoustache,
|
UNREFINED_LANDMARKS = 468
|
||||||
Mouth,
|
|
||||||
RightEar,
|
|
||||||
RightEye,
|
def earL_wavy(box, t):
|
||||||
RightMoustache,
|
box[0, 0, 0] -= np.sin(t / 7) * 11
|
||||||
)
|
box[1, 0, 0] -= np.sin(t / 7) * 11
|
||||||
from utils import (
|
return box
|
||||||
LANDMARKS_BOTTOM_SIDE,
|
|
||||||
LANDMARKS_LEFT_SIDE,
|
|
||||||
LANDMARKS_RIGHT_SIDE,
|
def earR_wavy(box, t):
|
||||||
LANDMARKS_TOP_SIDE,
|
box[0, 0, 0] += np.sin(t / 7 + 1) * 11
|
||||||
landmark2vec,
|
box[1, 0, 0] += np.sin(t / 7 + 1) * 11
|
||||||
)
|
return box
|
||||||
|
|
||||||
|
|
||||||
|
def moustacheL_wavy(box, t):
|
||||||
|
box[0, 0, 1] += np.sin(t / 7) * 11
|
||||||
|
box[3, 0, 1] += np.sin(t / 7) * 11
|
||||||
|
return box
|
||||||
|
|
||||||
|
|
||||||
|
def moustacheR_wavy(box, t):
|
||||||
|
box[1, 0, 1] += np.sin(t / 7 + 1) * 11
|
||||||
|
box[2, 0, 1] += np.sin(t / 7 + 1) * 11
|
||||||
|
return box
|
||||||
|
|
||||||
|
|
||||||
class Environment:
|
class Environment:
|
||||||
|
@ -38,28 +50,108 @@ class Environment:
|
||||||
# store reference to webcam
|
# store reference to webcam
|
||||||
self.cam = camera
|
self.cam = camera
|
||||||
|
|
||||||
|
self.frame_count = 0
|
||||||
|
|
||||||
# mediapipe stuff
|
# mediapipe stuff
|
||||||
self.mp_drawing = mp.solutions.drawing_utils # type: ignore
|
self.mp_drawing = mp.solutions.drawing_utils # type: ignore
|
||||||
self.mp_drawing_styles = mp.solutions.drawing_styles # type: ignore
|
self.mp_drawing_styles = mp.solutions.drawing_styles # type: ignore
|
||||||
self.mp_face_mesh = mp.solutions.face_mesh # type: ignore
|
self.mp_face_mesh = mp.solutions.face_mesh # type: ignore
|
||||||
|
self.refine_landmarks = True
|
||||||
|
|
||||||
# get screen size from webcam
|
# get screen size from webcam
|
||||||
self.camera_width = camera.get(3)
|
self.camera_width = camera.get(3)
|
||||||
self.camera_height = camera.get(4)
|
self.camera_height = camera.get(4)
|
||||||
|
|
||||||
|
# setup face axis
|
||||||
|
self.x = np.array([7, 0, 0]) # TODO: replace 7s by 1s
|
||||||
|
self.y = np.array([0, 7, 0])
|
||||||
|
self.z = np.array([0, 0, 7])
|
||||||
|
|
||||||
# create body parts
|
# create body parts
|
||||||
self.body_parts = [
|
self.body_parts = [
|
||||||
LeftEar(self),
|
BodyPart(
|
||||||
RightEar(self),
|
self,
|
||||||
Head(self),
|
"assets/earL.png",
|
||||||
RightMoustache(self),
|
np.array([6, 11, -0.5]),
|
||||||
LeftMoustache(self),
|
1,
|
||||||
LeftEye(self),
|
wavy=earL_wavy,
|
||||||
RightEye(self),
|
),
|
||||||
Crown(self),
|
BodyPart(
|
||||||
Mouth(self),
|
self,
|
||||||
|
"assets/earR.png",
|
||||||
|
np.array([-6, 11, -0.5]),
|
||||||
|
1,
|
||||||
|
wavy=earR_wavy,
|
||||||
|
),
|
||||||
|
BodyPart(
|
||||||
|
self,
|
||||||
|
"assets/head.png",
|
||||||
|
np.array([0, 0, 0]),
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
BodyPart(
|
||||||
|
self,
|
||||||
|
"assets/moustacheL.png",
|
||||||
|
np.array([13, -6, 0.1]),
|
||||||
|
1,
|
||||||
|
wavy=moustacheL_wavy,
|
||||||
|
),
|
||||||
|
BodyPart(
|
||||||
|
self,
|
||||||
|
"assets/moustacheR.png",
|
||||||
|
np.array([-13, -6, 0.1]),
|
||||||
|
1,
|
||||||
|
wavy=moustacheR_wavy,
|
||||||
|
),
|
||||||
|
ActiveBodyPart(
|
||||||
|
self,
|
||||||
|
"assets/eyeL.png",
|
||||||
|
np.array([2.5, 2.5, 0.1]),
|
||||||
|
0.3,
|
||||||
|
(145, 159, 133, 33),
|
||||||
|
),
|
||||||
|
ActiveBodyPart(
|
||||||
|
self,
|
||||||
|
"assets/eyeR.png",
|
||||||
|
np.array([-2.5, 2.5, 0.1]),
|
||||||
|
0.3,
|
||||||
|
(374, 386, 362, 263),
|
||||||
|
),
|
||||||
|
BodyPart(
|
||||||
|
self,
|
||||||
|
"assets/crown.png",
|
||||||
|
np.array([0, 6.5, 0.5]),
|
||||||
|
0.3,
|
||||||
|
),
|
||||||
|
ActiveBodyPart(
|
||||||
|
self,
|
||||||
|
"assets/mouth.png",
|
||||||
|
np.array([0, -3.5, 0.1]),
|
||||||
|
0.15,
|
||||||
|
(14, 13, 308, 78),
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# pseudo camera internals
|
||||||
|
self.dist_coeff = np.zeros((4, 1))
|
||||||
|
self.focal_length = self.camera_width
|
||||||
|
self.center = (self.camera_width / 2, self.camera_height / 2)
|
||||||
|
self.camera_matrix = np.array(
|
||||||
|
[
|
||||||
|
[self.focal_length, 0, self.center[0]],
|
||||||
|
[0, self.focal_length, self.center[1]],
|
||||||
|
[0, 0, 1],
|
||||||
|
],
|
||||||
|
dtype="double",
|
||||||
|
)
|
||||||
|
self.pcf = PCF(
|
||||||
|
near=1,
|
||||||
|
far=10000,
|
||||||
|
frame_height=self.camera_height,
|
||||||
|
frame_width=self.camera_width,
|
||||||
|
fy=self.focal_length,
|
||||||
|
)
|
||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
"""Start the environment."""
|
"""Start the environment."""
|
||||||
while self.cam.isOpened():
|
while self.cam.isOpened():
|
||||||
|
@ -74,6 +166,8 @@ class Environment:
|
||||||
logging.debug("Ignoring empty camera frame.")
|
logging.debug("Ignoring empty camera frame.")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
self.frame_count += 1
|
||||||
|
|
||||||
# detect keypoints on frame
|
# detect keypoints on frame
|
||||||
self.detect_keypoints()
|
self.detect_keypoints()
|
||||||
|
|
||||||
|
@ -82,7 +176,7 @@ class Environment:
|
||||||
self.draw_axis()
|
self.draw_axis()
|
||||||
|
|
||||||
# draw keypoints on top of frame
|
# draw keypoints on top of frame
|
||||||
# self.draw_keypoints()
|
self.draw_keypoints()
|
||||||
|
|
||||||
# draw avatar
|
# draw avatar
|
||||||
self.draw_avatar()
|
self.draw_avatar()
|
||||||
|
@ -99,7 +193,8 @@ class Environment:
|
||||||
"""Detect the keypoints on the frame."""
|
"""Detect the keypoints on the frame."""
|
||||||
with self.mp_face_mesh.FaceMesh(
|
with self.mp_face_mesh.FaceMesh(
|
||||||
max_num_faces=1,
|
max_num_faces=1,
|
||||||
refine_landmarks=True,
|
refine_landmarks=self.refine_landmarks,
|
||||||
|
static_image_mode=False,
|
||||||
min_detection_confidence=0.5,
|
min_detection_confidence=0.5,
|
||||||
min_tracking_confidence=0.5,
|
min_tracking_confidence=0.5,
|
||||||
) as face_mesh:
|
) as face_mesh:
|
||||||
|
@ -122,46 +217,46 @@ class Environment:
|
||||||
def compute_face_axis(self) -> None:
|
def compute_face_axis(self) -> None:
|
||||||
"""Compute the face axis."""
|
"""Compute the face axis."""
|
||||||
if self.results.multi_face_landmarks:
|
if self.results.multi_face_landmarks:
|
||||||
for face_landmarks in self.results.multi_face_landmarks:
|
# get landmarks, suppose only one face is detected
|
||||||
# retreive points
|
face_landmarks = self.results.multi_face_landmarks[0]
|
||||||
left_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_LEFT_SIDE])
|
|
||||||
right_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_RIGHT_SIDE])
|
|
||||||
bottom_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_BOTTOM_SIDE])
|
|
||||||
top_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_TOP_SIDE])
|
|
||||||
|
|
||||||
# compute center
|
# convert landmarks to numpy array
|
||||||
self.center = np.mean(np.concatenate((left_points, right_points, bottom_points, top_points)), axis=0)
|
landmarks = np.array([(lm.x, lm.y, lm.z) for lm in face_landmarks.landmark])
|
||||||
|
landmarks = landmarks.T
|
||||||
|
|
||||||
# compute axis
|
# remove refined landmarks
|
||||||
self.x = np.mean(right_points - left_points, axis=0)
|
if self.refine_landmarks:
|
||||||
self.y = np.mean(top_points - bottom_points, axis=0)
|
landmarks = landmarks[:, :UNREFINED_LANDMARKS]
|
||||||
self.z = np.cross(self.x, self.y)
|
|
||||||
|
|
||||||
# normalize axis
|
# get pose from landmarks
|
||||||
self.x = self.x / np.linalg.norm(self.x)
|
metric_landmarks, pose_transform_mat = get_metric_landmarks(landmarks, self.pcf)
|
||||||
self.y = self.y / np.linalg.norm(self.y)
|
|
||||||
self.z = self.z / np.linalg.norm(self.z)
|
# extract rotation and translation vectors
|
||||||
|
pose_transform_mat[1:3, :] = -pose_transform_mat[1:3, :]
|
||||||
|
self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
|
||||||
|
self.mp_translation_vector = pose_transform_mat[:3, 3, None]
|
||||||
|
|
||||||
|
# retrieve center of face
|
||||||
|
self.center = metric_landmarks[:, NOSE_LANDMARK].T
|
||||||
|
|
||||||
def draw_axis(self) -> None:
|
def draw_axis(self) -> None:
|
||||||
"""Draw the face axis on the frame."""
|
"""Draw the face axis on the frame."""
|
||||||
for (axis, color, letter) in [
|
# project axis
|
||||||
(self.x, (0, 0, 255), "X"),
|
(nose_pointers, _) = cv2.projectPoints(
|
||||||
(self.y, (0, 255, 0), "Y"),
|
np.array([np.zeros(3), self.x, self.y, self.z]) + self.center,
|
||||||
(self.z, (255, 0, 0), "Z"),
|
self.mp_rotation_vector,
|
||||||
]:
|
self.mp_translation_vector,
|
||||||
# compute start and end of axis
|
self.camera_matrix,
|
||||||
start = (
|
self.dist_coeff,
|
||||||
int(self.center[0] * self.camera_width),
|
)
|
||||||
int(self.center[1] * self.camera_height),
|
|
||||||
)
|
|
||||||
end = (
|
|
||||||
int(self.center[0] * self.camera_width + axis[0] * 100),
|
|
||||||
int(self.center[1] * self.camera_height + axis[1] * 100),
|
|
||||||
)
|
|
||||||
|
|
||||||
# draw axis + letter
|
# extract projected vectors
|
||||||
cv2.line(self.frame, start, end, color, 2)
|
nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointers.squeeze().astype(int)
|
||||||
cv2.putText(self.frame, letter, end, cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
|
|
||||||
|
# draw axis
|
||||||
|
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2)
|
||||||
|
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2)
|
||||||
|
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2)
|
||||||
|
|
||||||
def draw_keypoints(self) -> None:
|
def draw_keypoints(self) -> None:
|
||||||
"""Draw the keypoints on the screen."""
|
"""Draw the keypoints on the screen."""
|
||||||
|
@ -171,7 +266,7 @@ class Environment:
|
||||||
self.mp_drawing.draw_landmarks(
|
self.mp_drawing.draw_landmarks(
|
||||||
self.frame,
|
self.frame,
|
||||||
face_landmarks,
|
face_landmarks,
|
||||||
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 0, 0),
|
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0),
|
||||||
)
|
)
|
||||||
|
|
||||||
def draw_avatar(self) -> None:
|
def draw_avatar(self) -> None:
|
||||||
|
|
2659
src/face_geometry.py
Normal file
2659
src/face_geometry.py
Normal file
File diff suppressed because it is too large
Load diff
32
src/utils.py
32
src/utils.py
|
@ -1,32 +0,0 @@
|
||||||
import numpy as np
|
|
||||||
from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates
|
|
||||||
|
|
||||||
# def landmark2vec(landmark, screen):
|
|
||||||
# """Convert a landmark to a pygame Vector2."""
|
|
||||||
# return pg.Vector2(
|
|
||||||
# _normalized_to_pixel_coordinates(
|
|
||||||
# np.clip(landmark.x, 0, 1),
|
|
||||||
# np.clip(landmark.y, 0, 1),
|
|
||||||
# screen.get_width(),
|
|
||||||
# screen.get_height(),
|
|
||||||
# ) # type: ignore
|
|
||||||
# )
|
|
||||||
|
|
||||||
|
|
||||||
def landmark2vec(landmark) -> np.ndarray:
|
|
||||||
"""Convert a landmark to numpy array."""
|
|
||||||
return np.clip(
|
|
||||||
[
|
|
||||||
landmark.x,
|
|
||||||
landmark.y,
|
|
||||||
landmark.z,
|
|
||||||
],
|
|
||||||
a_min=0,
|
|
||||||
a_max=1,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
LANDMARKS_LEFT_SIDE = [109, 67, 103, 54, 21, 162, 127, 234, 93, 132, 58, 172, 136, 150, 149, 176, 148]
|
|
||||||
LANDMARKS_RIGHT_SIDE = [338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377]
|
|
||||||
LANDMARKS_BOTTOM_SIDE = [93, 132, 152, 361, 323]
|
|
||||||
LANDMARKS_TOP_SIDE = [127, 162, 10, 389, 356]
|
|
Loading…
Reference in a new issue