refactor!: perspective? (orthogonal) projection
This commit is contained in:
parent
0aa7c42781
commit
7212e2e8ec
SSH key fingerprint:
SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
|
|
@ -1,8 +1 @@
|
|||
from .body import Body
|
||||
from .crown import Crown
|
||||
from .ear import Ear
|
||||
from .eye import Eye
|
||||
from .head import Head
|
||||
from .moustache import Moustache
|
||||
from .mouth import Mouth
|
||||
from .test import Test
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pygame as pg
|
||||
|
||||
from utils import landmark2vec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARKS = [152]
|
||||
|
||||
|
||||
class Body:
|
||||
def __init__(self, env: Environment) -> None:
|
||||
self.env = env
|
||||
self.texture = pg.image.load("assets/body.png").convert_alpha()
|
||||
|
||||
def draw(self, screen) -> None:
|
||||
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
|
||||
body_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen)
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
self.texture,
|
||||
self.env.size,
|
||||
)
|
||||
|
||||
texture_pos = body_pos - pg.Vector2(
|
||||
self.env.size[0] / 2,
|
||||
self.env.size[1] / 10,
|
||||
)
|
||||
|
||||
# draw texture
|
||||
screen.blit(texture_scaled, texture_pos)
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pygame as pg
|
||||
|
||||
from utils import blitRotate, landmark2vec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARKS = [10]
|
||||
|
||||
|
||||
class Crown:
|
||||
def __init__(self, env: Environment) -> None:
|
||||
self.env = env
|
||||
self.texture = pg.image.load("assets/crown.png").convert_alpha()
|
||||
|
||||
def draw(self, screen) -> None:
|
||||
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
|
||||
crown_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen)
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
self.texture,
|
||||
(self.env.size[0] / 3, self.env.size[1] / 3),
|
||||
)
|
||||
|
||||
# draw texture
|
||||
blitRotate(screen, texture_scaled, crown_pos, self.env.angle_x)
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pygame as pg
|
||||
|
||||
from utils import blitRotate, landmark2vec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARKS = [103, 332]
|
||||
|
||||
|
||||
class Ear:
|
||||
def __init__(self, side: bool, env: Environment) -> None:
|
||||
self.env = env
|
||||
self.side = side
|
||||
self.texture = (
|
||||
pg.image.load("assets/earL.png").convert_alpha()
|
||||
if side
|
||||
else pg.image.load("assets/earR.png").convert_alpha()
|
||||
)
|
||||
|
||||
def draw(self, screen) -> None:
|
||||
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
self.texture,
|
||||
(self.env.size[0] / 3, self.env.size[1]),
|
||||
)
|
||||
|
||||
if self.side: # right moustache
|
||||
moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen)
|
||||
|
||||
pivot = pg.Vector2(
|
||||
3 * texture_scaled.get_width() / 4,
|
||||
0.8 * texture_scaled.get_height(),
|
||||
)
|
||||
|
||||
else: # left moustache
|
||||
moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen)
|
||||
|
||||
pivot = pg.Vector2(
|
||||
texture_scaled.get_width() / 4,
|
||||
0.8 * texture_scaled.get_height(),
|
||||
)
|
||||
|
||||
# draw texture
|
||||
blitRotate(screen, texture_scaled, moustache_pos, self.env.angle_x, pivot)
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pygame as pg
|
||||
|
||||
from utils import blitRotate, landmark2vec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARK_LEFT_EYELID = [386, 374]
|
||||
LANDMARK_RIGHT_EYELID = [159, 145]
|
||||
LANDMARK_LEFT_IRIS = [473, 474, 475, 476, 477]
|
||||
LANDMARK_RIGHT_IRIS = [468, 469, 470, 471, 472]
|
||||
|
||||
|
||||
class Eye:
|
||||
"""The eye class is responsible of drawing an eye."""
|
||||
|
||||
def __init__(self, side: bool, env: Environment) -> None:
|
||||
"""Initialize the eye."""
|
||||
self.env = env
|
||||
self.side = side
|
||||
self.texture = (
|
||||
pg.image.load("assets/eyeL.png").convert_alpha()
|
||||
if side
|
||||
else pg.image.load("assets/eyeR.png").convert_alpha()
|
||||
)
|
||||
|
||||
def draw(self, screen) -> None:
|
||||
"""Draw the eye."""
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
if self.side: # right eye
|
||||
iris_pos = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_IRIS[0]], screen)
|
||||
|
||||
iris_top = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_IRIS[2]], screen)
|
||||
iris_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_IRIS[4]], screen)
|
||||
|
||||
eyelid_top = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_EYELID[0]], screen)
|
||||
eyelid_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_RIGHT_EYELID[1]], screen)
|
||||
|
||||
else: # left eye
|
||||
iris_pos = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_IRIS[0]], screen)
|
||||
|
||||
iris_top = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_IRIS[2]], screen)
|
||||
iris_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_IRIS[4]], screen)
|
||||
|
||||
eyelid_top = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_EYELID[0]], screen)
|
||||
eyelid_bottom = landmark2vec(face_landmarks.landmark[LANDMARK_LEFT_EYELID[1]], screen)
|
||||
|
||||
angle = (iris_bottom - iris_top).angle_to(pg.Vector2(0, 1))
|
||||
width = (iris_bottom - iris_top).length()
|
||||
height = (eyelid_bottom - eyelid_top).length() * 5
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
self.texture,
|
||||
(width, height),
|
||||
)
|
||||
|
||||
# draw texture
|
||||
blitRotate(screen, texture_scaled, iris_pos, angle)
|
||||
|
|
@ -2,41 +2,84 @@ from __future__ import annotations
|
|||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pygame as pg
|
||||
|
||||
from utils import blitRotate, landmark2vec
|
||||
import cv2
|
||||
import numpy as np
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARKS = [234, 10, 454, 152]
|
||||
|
||||
|
||||
class Head:
|
||||
"""The head body part."""
|
||||
|
||||
def __init__(self, env: Environment) -> None:
|
||||
"""Initialize the head."""
|
||||
self.env = env
|
||||
self.texture = pg.image.load("assets/head.png").convert_alpha()
|
||||
|
||||
def draw(self, screen) -> None:
|
||||
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED)
|
||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
||||
self.bouding_box = np.array(
|
||||
[
|
||||
[0, 0],
|
||||
[self.image.shape[1], 0],
|
||||
[self.image.shape[1], self.image.shape[0]],
|
||||
[0, self.image.shape[0]],
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
def draw(self) -> None:
|
||||
"""Draw the head on the screen."""
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
head_box = np.array(
|
||||
[
|
||||
100 * self.ratio * self.env.x + 100 * self.env.y + 80 * self.env.z,
|
||||
100 * self.ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z,
|
||||
100 * self.ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
||||
100 * self.ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
||||
]
|
||||
)[:, :2]
|
||||
|
||||
head_top = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen)
|
||||
head_bottom = landmark2vec(face_landmarks.landmark[LANDMARKS[3]], screen)
|
||||
self.translated_head_box = (
|
||||
head_box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
||||
).astype(np.float32)
|
||||
|
||||
height = (head_bottom - head_top).length()
|
||||
ratio = self.texture.get_width() / self.texture.get_height()
|
||||
|
||||
self.env.size = (height * ratio, height)
|
||||
|
||||
head_pos = (head_bottom + head_top) / 2
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
self.texture,
|
||||
self.env.size,
|
||||
# get perspective transform
|
||||
transform_mat = cv2.getPerspectiveTransform(
|
||||
self.bouding_box,
|
||||
self.translated_head_box,
|
||||
)
|
||||
|
||||
# draw texture
|
||||
blitRotate(screen, texture_scaled, head_pos, self.env.angle_x)
|
||||
# apply perspective transform to image
|
||||
warped = cv2.warpPerspective(
|
||||
self.image,
|
||||
transform_mat,
|
||||
self.env.frame.shape[1::-1],
|
||||
)
|
||||
|
||||
# replace non black pixels of warped image by frame
|
||||
self.env.avatar[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
||||
|
||||
def draw_debug(self) -> None:
|
||||
"""Draw debug information on the screen."""
|
||||
# link points
|
||||
for i in range(4):
|
||||
cv2.line(
|
||||
self.env.frame,
|
||||
tuple(self.translated_head_box[i].astype(np.int32)),
|
||||
tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)),
|
||||
(255, 255, 255),
|
||||
2,
|
||||
)
|
||||
|
||||
# TODO: faire un POC où je place juste un cvGetPerspectiveTransform suivi d'un cvWarpPerspective
|
||||
# -> comme sur cet exemple, mais où l'image de gauche et droite sont inversées
|
||||
# https://docs.adaptive-vision.com/studio/filters/GeometricImageTransformations/cvGetPerspectiveTransform.html
|
||||
|
||||
# instrisics ? -> https://github.dev/google/mediapipe/blob/master/mediapipe/modules/face_geometry/libs/effect_renderer.cc#L573-L599
|
||||
|
||||
# TODO: https://github.com/Rassibassi/mediapipeFacegeometryPython/blob/main/head_posture_rt.py
|
||||
# -> pnp -> pose estimation -> paramètres extrinsèques
|
||||
# -> + param intrasèque (supposé connu, check site mediapipe)
|
||||
# -> placer dans l'espace les textures -> et projeter dans le plan image
|
||||
|
|
|
|||
|
|
@ -1,53 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pygame as pg
|
||||
|
||||
from utils import blitRotate, landmark2vec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARKS = [138, 367]
|
||||
|
||||
|
||||
class Moustache:
|
||||
def __init__(self, side: bool, env: Environment) -> None:
|
||||
self.env = env
|
||||
self.side = side
|
||||
self.texture = (
|
||||
pg.image.load("assets/moustacheL.png").convert_alpha()
|
||||
if side
|
||||
else pg.image.load("assets/moustacheR.png").convert_alpha()
|
||||
)
|
||||
|
||||
def draw(self, screen) -> None:
|
||||
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
self.texture,
|
||||
(self.env.size[0] / 1.5, self.env.size[1] / 1.5),
|
||||
)
|
||||
|
||||
if self.side: # right moustache
|
||||
moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen)
|
||||
|
||||
pivot = pg.Vector2(
|
||||
texture_scaled.get_width(),
|
||||
texture_scaled.get_height() / 2,
|
||||
)
|
||||
|
||||
else: # left moustache
|
||||
moustache_pos = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen)
|
||||
|
||||
pivot = pg.Vector2(
|
||||
0,
|
||||
texture_scaled.get_height() / 2,
|
||||
)
|
||||
|
||||
# draw texture
|
||||
blitRotate(screen, texture_scaled, moustache_pos, self.env.angle_x, pivot)
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pygame as pg
|
||||
|
||||
from utils import blitRotate, landmark2vec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARKS = [78, 13, 308, 14]
|
||||
MIN_HEIGHT = 4
|
||||
MAX_HEIGHT = 40
|
||||
|
||||
|
||||
class Mouth:
|
||||
def __init__(self, env: Environment) -> None:
|
||||
self.env = env
|
||||
self.texture = pg.image.load("assets/mouth.png").convert_alpha()
|
||||
self.closed_texture = pg.image.load("assets/mouth.png").convert_alpha()
|
||||
self.closed_texture.fill((0, 0, 0, 255), special_flags=pg.BLEND_RGBA_MULT)
|
||||
|
||||
def draw(self, screen) -> None:
|
||||
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
|
||||
mouth_top = landmark2vec(face_landmarks.landmark[LANDMARKS[1]], screen)
|
||||
mouth_bottom = landmark2vec(face_landmarks.landmark[LANDMARKS[3]], screen)
|
||||
|
||||
mouth_left = landmark2vec(face_landmarks.landmark[LANDMARKS[0]], screen)
|
||||
mouth_right = landmark2vec(face_landmarks.landmark[LANDMARKS[2]], screen)
|
||||
|
||||
height = (mouth_bottom - mouth_top).length()
|
||||
height = min(height, MAX_HEIGHT)
|
||||
width = (mouth_right - mouth_left).length()
|
||||
|
||||
mouth_pos = (mouth_bottom + mouth_top) / 2
|
||||
|
||||
if height < MIN_HEIGHT:
|
||||
height = MIN_HEIGHT
|
||||
texture = self.closed_texture
|
||||
else:
|
||||
height *= 1.5
|
||||
texture = self.texture
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
texture,
|
||||
(width, height),
|
||||
)
|
||||
|
||||
# draw texture
|
||||
blitRotate(screen, texture_scaled, mouth_pos, self.env.angle_x)
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
import pygame as pg
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
|
||||
class Test:
|
||||
def __init__(self, env: Environment) -> None:
|
||||
self.env = env
|
||||
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED)
|
||||
|
||||
def draw(self, screen: pg.Surface) -> None:
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
|
||||
ratio = self.image.shape[1] / self.image.shape[0]
|
||||
|
||||
head_box = np.array(
|
||||
[
|
||||
100 * ratio * self.env.x + 100 * self.env.y + 80 * self.env.z,
|
||||
100 * ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z,
|
||||
100 * ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
||||
100 * ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
||||
]
|
||||
)[:, :2]
|
||||
|
||||
# link points
|
||||
cv2.line(
|
||||
self.env.frame,
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[0][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[0][1]),
|
||||
),
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[1][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[1][1]),
|
||||
),
|
||||
(0, 0, 255),
|
||||
2,
|
||||
)
|
||||
cv2.line(
|
||||
self.env.frame,
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[1][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[1][1]),
|
||||
),
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[2][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[2][1]),
|
||||
),
|
||||
(0, 0, 255),
|
||||
2,
|
||||
)
|
||||
cv2.line(
|
||||
self.env.frame,
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[2][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[2][1]),
|
||||
),
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[3][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[3][1]),
|
||||
),
|
||||
(0, 0, 255),
|
||||
2,
|
||||
)
|
||||
cv2.line(
|
||||
self.env.frame,
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[3][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[3][1]),
|
||||
),
|
||||
(
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[0][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[0][1]),
|
||||
),
|
||||
(0, 0, 255),
|
||||
2,
|
||||
)
|
||||
|
||||
a = np.array(
|
||||
[
|
||||
[0, 0],
|
||||
[self.image.shape[1], 0],
|
||||
[self.image.shape[1], self.image.shape[0]],
|
||||
[0, self.image.shape[0]],
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
b = np.array(
|
||||
[
|
||||
[
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[0][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[0][1]),
|
||||
],
|
||||
[
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[1][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[1][1]),
|
||||
],
|
||||
[
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[2][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[2][1]),
|
||||
],
|
||||
[
|
||||
int(self.env.center.x * self.env.screen.get_width() + head_box[3][0]),
|
||||
int(self.env.center.y * self.env.screen.get_height() + head_box[3][1]),
|
||||
],
|
||||
],
|
||||
dtype=np.float32,
|
||||
)
|
||||
|
||||
# get perspective transform
|
||||
transform_mat = cv2.getPerspectiveTransform(
|
||||
a,
|
||||
b,
|
||||
)
|
||||
|
||||
# apply perspective transform to image
|
||||
warped = cv2.warpPerspective(
|
||||
self.image,
|
||||
transform_mat,
|
||||
self.env.frame.shape[1::-1],
|
||||
)
|
||||
|
||||
# replace non black pixels of warped image by frame
|
||||
self.env.frame[warped[:, :, 3] != 0] = warped[warped[:, :, 3] != 0][:, :3]
|
||||
|
|
@ -1,126 +0,0 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import pygame as pg
|
||||
import skimage
|
||||
import skimage.io
|
||||
import skimage.transform
|
||||
from skimage import data, img_as_float
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from environment import Environment
|
||||
|
||||
LANDMARKS = [6]
|
||||
|
||||
# https://scikit-image.org/docs/stable/auto_examples/transform/plot_geometric.html#sphx-glr-auto-examples-transform-plot-geometric-py
|
||||
# https://scikit-image.org/docs/stable/auto_examples/transform/plot_transform_types.html#sphx-glr-auto-examples-transform-plot-transform-types-py
|
||||
# https://github.com/google/mediapipe/issues/1379
|
||||
# https://en.wikipedia.org/wiki/3D_projection#Perspective_projection
|
||||
|
||||
|
||||
class Test:
|
||||
def __init__(self, env: Environment) -> None:
|
||||
self.env = env
|
||||
self.image = skimage.io.imread("assets/head.png")[:, :, :3]
|
||||
self.image = skimage.transform.resize(
|
||||
self.image, (self.image.shape[0] // 10, self.image.shape[1] // 10), anti_aliasing=False
|
||||
)
|
||||
|
||||
self.anglex = 0
|
||||
self.angley = 0
|
||||
self.anglez = 0
|
||||
|
||||
def draw(self, screen: pg.Surface) -> None:
|
||||
|
||||
# compute position
|
||||
if self.env.results.multi_face_landmarks:
|
||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||
|
||||
body_pos = pg.Vector2(*screen.get_size()) / 2
|
||||
img = img_as_float(data.chelsea())
|
||||
|
||||
self.anglez += 1 / 180 * np.pi
|
||||
self.anglez += 1 / 180 * np.pi
|
||||
|
||||
# matrix = np.array(
|
||||
# [
|
||||
# [1, -0.5, 100],
|
||||
# [0.1, 0.9, 50],
|
||||
# [0.0015, 0.0015, 1],
|
||||
# ]
|
||||
# )
|
||||
|
||||
# self.angley += 0.1 / 180 * np.pi
|
||||
# print(self.angley * 180 / np.pi)
|
||||
|
||||
# scale = np.array(
|
||||
# [
|
||||
# [0.5, 0, 0],
|
||||
# [0, 0.5, 0],
|
||||
# [0, 0, 1],
|
||||
# ]
|
||||
# )
|
||||
|
||||
translation = np.array(
|
||||
[
|
||||
[1, 0, -img.shape[1] / 2],
|
||||
[0, 1, img.shape[0] / 2],
|
||||
[0, 0, 1],
|
||||
]
|
||||
)
|
||||
|
||||
anti_translation = np.array(
|
||||
[
|
||||
[1, 0, img.shape[1] / 2],
|
||||
[0, 1, -img.shape[0] / 2],
|
||||
[0, 0, 1],
|
||||
]
|
||||
)
|
||||
|
||||
rotation_x = np.array(
|
||||
[
|
||||
[1, 0, 0],
|
||||
[0, np.cos(self.anglex), -np.sin(self.anglex)],
|
||||
[0, np.sin(self.anglex), np.cos(self.anglex)],
|
||||
]
|
||||
)
|
||||
|
||||
rotation_y = np.array(
|
||||
[
|
||||
[np.cos(self.angley), 0, np.sin(self.angley)],
|
||||
[0, 1, 0],
|
||||
[-np.sin(self.angley), 0, np.cos(self.angley)],
|
||||
]
|
||||
)
|
||||
|
||||
rotation_z = np.array(
|
||||
[
|
||||
[np.cos(self.anglez), -np.sin(self.anglez), 0],
|
||||
[np.sin(self.anglez), np.cos(self.anglez), 0],
|
||||
[0, 0, 1],
|
||||
]
|
||||
)
|
||||
|
||||
# matrix = translation @ rotation_x @ rotation_y @ rotation_z @ anti_translation
|
||||
matrix = rotation_x @ rotation_y @ rotation_z
|
||||
matrix = translation @ rotation_x @ rotation_y @ rotation_z
|
||||
|
||||
tform = skimage.transform.ProjectiveTransform(matrix=matrix)
|
||||
tf_img = skimage.transform.warp(img, tform.inverse)
|
||||
|
||||
yes = pg.surfarray.make_surface((tf_img * 255).astype(np.uint8).transpose(1, 0, 2))
|
||||
yes.set_colorkey((0, 0, 0))
|
||||
|
||||
texture_scaled = pg.transform.scale(
|
||||
yes,
|
||||
(yes.get_width() / 2, yes.get_height() / 2),
|
||||
)
|
||||
|
||||
texture_pos = body_pos - pg.Vector2(
|
||||
texture_scaled.get_width() / 2,
|
||||
texture_scaled.get_height() / 2,
|
||||
)
|
||||
|
||||
screen.blit(texture_scaled, texture_pos)
|
||||
|
|
@ -2,10 +2,16 @@ import logging
|
|||
|
||||
import cv2
|
||||
import mediapipe as mp
|
||||
import pygame as pg
|
||||
import numpy as np
|
||||
|
||||
from bodyparts import Body, Crown, Ear, Eye, Head, Moustache, Mouth, Test
|
||||
from utils import landmark3vec
|
||||
from bodyparts import Head
|
||||
from utils import (
|
||||
LANDMARKS_BOTTOM_SIDE,
|
||||
LANDMARKS_LEFT_SIDE,
|
||||
LANDMARKS_RIGHT_SIDE,
|
||||
LANDMARKS_TOP_SIDE,
|
||||
landmark2vec,
|
||||
)
|
||||
|
||||
|
||||
class Environment:
|
||||
|
|
@ -27,70 +33,56 @@ class Environment:
|
|||
self.mp_drawing_styles = mp.solutions.drawing_styles # type: ignore
|
||||
self.mp_face_mesh = mp.solutions.face_mesh # type: ignore
|
||||
|
||||
# init pygame
|
||||
pg.init()
|
||||
|
||||
# get screen size from webcam
|
||||
self.screen_width = camera.get(3)
|
||||
self.screen_height = camera.get(4)
|
||||
|
||||
# create screen
|
||||
self.screen: pg.Surface = pg.display.set_mode(
|
||||
(self.screen_width, self.screen_height),
|
||||
pg.DOUBLEBUF | pg.HWSURFACE,
|
||||
) # type: ignore
|
||||
pg.display.set_caption("Projet APP")
|
||||
|
||||
self.size = (0.0, 0.0)
|
||||
self.camera_width = camera.get(3)
|
||||
self.camera_height = camera.get(4)
|
||||
|
||||
# create body parts
|
||||
self.body_parts = {
|
||||
# "body": Body(self),
|
||||
# "left_ear": Ear(False, self),
|
||||
# "right_ear": Ear(True, self),
|
||||
# "head": Head(self),
|
||||
# "left_moustache": Moustache(False, self),
|
||||
# "right_moustache": Moustache(True, self),
|
||||
# "left_eye": Eye(False, self),
|
||||
# "right_eye": Eye(True, self),
|
||||
# "crown": Crown(self),
|
||||
# "mouth": Mouth(self),
|
||||
"test": Test(self),
|
||||
}
|
||||
self.body_parts = [
|
||||
# Body(self),
|
||||
# Ear(False, self),
|
||||
# Ear(True, self),
|
||||
Head(self),
|
||||
# Moustache(False, self),
|
||||
# Moustache(True, self),
|
||||
# Eye(False, self),
|
||||
# Eye(True, self),
|
||||
# Crown(self),
|
||||
# Mouth(self),
|
||||
]
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start the environment."""
|
||||
while self.cam.isOpened():
|
||||
|
||||
# stop if q is pressed (opencv)
|
||||
if cv2.waitKey(5) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
# read webcam
|
||||
success, self.frame = self.cam.read()
|
||||
if not success:
|
||||
logging.debug("Ignoring empty camera frame.")
|
||||
continue
|
||||
|
||||
# stop if q is pressed (opencv)
|
||||
if cv2.waitKey(5) & 0xFF == ord("q"):
|
||||
break
|
||||
|
||||
# quit the game
|
||||
if any(
|
||||
[event.type == pg.KEYDOWN and event.key == pg.K_q for event in pg.event.get()],
|
||||
):
|
||||
break
|
||||
|
||||
# detect keypoints on frame
|
||||
self.detect_keypoints()
|
||||
|
||||
# compute face axis
|
||||
self.compute_face_axis()
|
||||
self.draw_axis()
|
||||
|
||||
# draw keypoints on top of frame
|
||||
# self.draw_keypoints()
|
||||
self.draw_keypoints()
|
||||
|
||||
# draw avatar
|
||||
self.draw_avatar()
|
||||
|
||||
# tmp
|
||||
cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1))
|
||||
# show frame
|
||||
cv2.imshow("Camera", cv2.flip(self.frame, 1))
|
||||
|
||||
# show avatar
|
||||
cv2.imshow("Avatar", cv2.flip(self.avatar, 1))
|
||||
|
||||
def detect_keypoints(self) -> None:
|
||||
"""Detect the keypoints on the frame."""
|
||||
|
|
@ -121,58 +113,44 @@ class Environment:
|
|||
if self.results.multi_face_landmarks:
|
||||
for face_landmarks in self.results.multi_face_landmarks:
|
||||
# retreive points
|
||||
left = landmark3vec(face_landmarks.landmark[234], self.screen)
|
||||
right = landmark3vec(face_landmarks.landmark[454], self.screen)
|
||||
bottom = landmark3vec(face_landmarks.landmark[152], self.screen)
|
||||
top = landmark3vec(face_landmarks.landmark[10], self.screen)
|
||||
self.center = (left + right + bottom + top) / 4
|
||||
left_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_LEFT_SIDE])
|
||||
right_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_RIGHT_SIDE])
|
||||
bottom_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_BOTTOM_SIDE])
|
||||
top_points = np.array([landmark2vec(face_landmarks.landmark[i]) for i in LANDMARKS_TOP_SIDE])
|
||||
|
||||
# compute center
|
||||
self.center = np.mean(np.concatenate((left_points, right_points, bottom_points, top_points)), axis=0)
|
||||
|
||||
# compute axis
|
||||
self.x = (right - left) / 2
|
||||
self.y = (top - bottom) / 2
|
||||
self.z = self.x.cross(self.y)
|
||||
self.x = np.mean(right_points - left_points, axis=0)
|
||||
self.y = np.mean(top_points - bottom_points, axis=0)
|
||||
self.z = np.cross(self.x, self.y)
|
||||
|
||||
# normalize axis
|
||||
self.x.normalize_ip()
|
||||
self.y.normalize_ip()
|
||||
self.z.normalize_ip()
|
||||
self.x = self.x / np.linalg.norm(self.x)
|
||||
self.y = self.y / np.linalg.norm(self.y)
|
||||
self.z = self.z / np.linalg.norm(self.z)
|
||||
|
||||
# print horizontal angle to screen
|
||||
self.angle_x = self.x.angle_to(pg.math.Vector3(0, 1, 0)) - 90
|
||||
self.angle_y = self.x.angle_to(pg.math.Vector3(0, 0, 1)) - 90
|
||||
self.angle_z = self.y.angle_to(pg.math.Vector3(0, 0, 1)) - 90
|
||||
def draw_axis(self) -> None:
|
||||
"""Draw the face axis on the frame."""
|
||||
for (axis, color, letter) in [
|
||||
(self.x, (0, 0, 255), "X"),
|
||||
(self.y, (0, 255, 0), "Y"),
|
||||
(self.z, (255, 0, 0), "Z"),
|
||||
]:
|
||||
# compute start and end of axis
|
||||
start = (
|
||||
int(self.center[0] * self.camera_width),
|
||||
int(self.center[1] * self.camera_height),
|
||||
)
|
||||
end = (
|
||||
int(self.center[0] * self.camera_width + axis[0] * 100),
|
||||
int(self.center[1] * self.camera_height + axis[1] * 100),
|
||||
)
|
||||
|
||||
# draw axis on opencv screen
|
||||
# cv2.line(
|
||||
# self.frame,
|
||||
# (int(self.center.x * self.screen.get_width()), int(self.center.y * self.screen.get_height())),
|
||||
# (
|
||||
# int(self.center.x * self.screen.get_width() + self.x.x * 100),
|
||||
# int(self.center.y * self.screen.get_height() + self.x.y * 100),
|
||||
# ),
|
||||
# (0, 0, 255),
|
||||
# 2,
|
||||
# )
|
||||
# cv2.line(
|
||||
# self.frame,
|
||||
# (int(self.center.x * self.screen.get_width()), int(self.center.y * self.screen.get_height())),
|
||||
# (
|
||||
# int(self.center.x * self.screen.get_width() + self.y.x * 100),
|
||||
# int(self.center.y * self.screen.get_height() + self.y.y * 100),
|
||||
# ),
|
||||
# (0, 255, 0),
|
||||
# 2,
|
||||
# )
|
||||
# cv2.line(
|
||||
# self.frame,
|
||||
# (int(self.center.x * self.screen.get_width()), int(self.center.y * self.screen.get_height())),
|
||||
# (
|
||||
# int(self.center.x * self.screen.get_width() + self.z.x * 100),
|
||||
# int(self.center.y * self.screen.get_height() + self.z.y * 100),
|
||||
# ),
|
||||
# (255, 0, 0),
|
||||
# 2,
|
||||
# )
|
||||
# draw axis + letter
|
||||
cv2.line(self.frame, start, end, color, 2)
|
||||
cv2.putText(self.frame, letter, end, cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2)
|
||||
|
||||
def draw_keypoints(self) -> None:
|
||||
"""Draw the keypoints on the screen."""
|
||||
|
|
@ -185,20 +163,12 @@ class Environment:
|
|||
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 0, 0),
|
||||
)
|
||||
|
||||
# flip the image horizontally for a selfie-view display
|
||||
cv2.imshow("MediaPipe Face Mesh", cv2.flip(self.frame, 1))
|
||||
|
||||
def draw_avatar(self) -> None:
|
||||
"""Draw the avatar on the screen."""
|
||||
# clear image with green background
|
||||
self.screen.fill(pg.Color("green"))
|
||||
# clear avatar frame
|
||||
self.avatar = np.zeros(self.frame.shape, dtype=np.uint8)
|
||||
|
||||
# draw each body part
|
||||
for part in self.body_parts.values():
|
||||
part.draw(self.screen)
|
||||
|
||||
# flip screen
|
||||
self.screen.blit(pg.transform.flip(self.screen, True, False), (0, 0))
|
||||
|
||||
# display screen
|
||||
pg.display.flip()
|
||||
for part in self.body_parts:
|
||||
part.draw()
|
||||
part.draw_debug()
|
||||
|
|
|
|||
62
src/utils.py
62
src/utils.py
|
|
@ -1,46 +1,32 @@
|
|||
import numpy as np
|
||||
import pygame as pg
|
||||
from mediapipe.python.solutions.drawing_utils import _normalized_to_pixel_coordinates
|
||||
|
||||
# def blitRotate(surf, image, topleft, angle):
|
||||
# """Rotate an image while drawing it."""
|
||||
# rotated_image = pg.transform.rotate(image, angle)
|
||||
# new_rect = rotated_image.get_rect(center=image.get_rect(topleft=topleft).center)
|
||||
# surf.blit(rotated_image, new_rect.topleft)
|
||||
# def landmark2vec(landmark, screen):
|
||||
# """Convert a landmark to a pygame Vector2."""
|
||||
# return pg.Vector2(
|
||||
# _normalized_to_pixel_coordinates(
|
||||
# np.clip(landmark.x, 0, 1),
|
||||
# np.clip(landmark.y, 0, 1),
|
||||
# screen.get_width(),
|
||||
# screen.get_height(),
|
||||
# ) # type: ignore
|
||||
# )
|
||||
|
||||
|
||||
# https://stackoverflow.com/questions/70819750/rotating-and-scaling-an-image-around-a-pivot-while-scaling-width-and-height-sep/70820034#70820034
|
||||
def blitRotate(surf, image, origin, angle, pivot=None):
|
||||
|
||||
if pivot is None:
|
||||
pivot = pg.math.Vector2(image.get_size()) / 2
|
||||
|
||||
image_rect = image.get_rect(topleft=(origin[0] - pivot[0], origin[1] - pivot[1]))
|
||||
offset_center_to_pivot = pg.math.Vector2(origin) - image_rect.center
|
||||
rotated_offset = offset_center_to_pivot.rotate(-angle)
|
||||
rotated_image_center = (origin[0] - rotated_offset.x, origin[1] - rotated_offset.y)
|
||||
rotated_image = pg.transform.rotate(image, angle)
|
||||
rect = rotated_image.get_rect(center=rotated_image_center)
|
||||
|
||||
surf.blit(rotated_image, rect)
|
||||
|
||||
|
||||
def landmark2vec(landmark, screen):
|
||||
"""Convert a landmark to a pygame Vector2."""
|
||||
return pg.Vector2(
|
||||
_normalized_to_pixel_coordinates(
|
||||
np.clip(landmark.x, 0, 1),
|
||||
np.clip(landmark.y, 0, 1),
|
||||
screen.get_width(),
|
||||
screen.get_height(),
|
||||
) # type: ignore
|
||||
def landmark2vec(landmark) -> np.ndarray:
|
||||
"""Convert a landmark to numpy array."""
|
||||
return np.clip(
|
||||
[
|
||||
landmark.x,
|
||||
landmark.y,
|
||||
landmark.z,
|
||||
],
|
||||
a_min=0,
|
||||
a_max=1,
|
||||
)
|
||||
|
||||
|
||||
def landmark3vec(landmark, screen):
|
||||
"""Convert a landmark to a pygame Vector3."""
|
||||
return pg.Vector3(
|
||||
np.clip(landmark.x, 0, 1),
|
||||
np.clip(landmark.y, 0, 1),
|
||||
np.clip(landmark.z, 0, 1),
|
||||
)
|
||||
LANDMARKS_LEFT_SIDE = [109, 67, 103, 54, 21, 162, 127, 234, 93, 132, 58, 172, 136, 150, 149, 176, 148]
|
||||
LANDMARKS_RIGHT_SIDE = [338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377]
|
||||
LANDMARKS_BOTTOM_SIDE = [93, 132, 152, 361, 323]
|
||||
LANDMARKS_TOP_SIDE = [127, 162, 10, 389, 356]
|
||||
|
|
|
|||
Loading…
Reference in a new issue