ENSEEIHT/projet-transversal
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

feat: POC true perspective transform

Browse source
This commit is contained in:
Laureηt 2023-01-13 11:23:29 +01:00
parent 97fbc9bdb3
commit 6cd40b7d8e
Signed by: Laurent
SSH key fingerprint: SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
2 changed files with 63 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
src/bodyparts/head.py
View file

@ -17,7 +17,7 @@ class Head:
self.env = env
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array(
self.bounding_box = np.array(
[
[0, 0],
[self.image.shape[1], 0],
@ -31,24 +31,38 @@ class Head:
"""Draw the head on the screen."""
# compute position
if self.env.results.multi_face_landmarks:
x = self.env.x - self.env.model_points[0]
y = self.env.y - self.env.model_points[0]
for face_landmarks in self.env.results.multi_face_landmarks:
head_box = np.array(
[
100 * self.ratio * self.env.x + 100 * self.env.y + 80 * self.env.z,
100 * self.ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z,
100 * self.ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z,
100 * self.ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z,
self.ratio * x + y + self.env.model_points[0],
self.ratio * -x + y + self.env.model_points[0],
self.ratio * -x + -y + self.env.model_points[0],
self.ratio * x + -y + self.env.model_points[0],
]
)[:, :2]
)
self.translated_head_box = (
head_box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
).astype(np.float32)
(box, _) = cv2.projectPoints(
head_box,
self.env.mp_rotation_vector,
self.env.mp_translation_vector,
self.env.camera_matrix,
self.env.dist_coeff,
)
a, b, c, d = box.squeeze().astype(int)
cv2.line(self.env.frame, a, b, (255, 255, 255), 2)
cv2.line(self.env.frame, b, c, (255, 255, 255), 2)
cv2.line(self.env.frame, c, d, (255, 255, 255), 2)
cv2.line(self.env.frame, d, a, (255, 255, 255), 2)
# get perspective transform
transform_mat = cv2.getPerspectiveTransform(
self.bouding_box,
self.translated_head_box,
self.bounding_box,
box.astype(np.float32),
)
# apply perspective transform to image
@ -64,11 +78,12 @@ class Head:
def draw_debug(self) -> None:
"""Draw debug information on the screen."""
# link points
for i in range(4):
cv2.line(
self.env.frame,
tuple(self.translated_head_box[i].astype(np.int32)),
tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)),
(255, 255, 255),
2,
)
pass
# for i in range(4):
# cv2.line(
# self.env.frame,
# tuple(self.translated_head_box[i].astype(np.int32)),
# tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)),
# (255, 255, 255),
# 2,
# )

39
src/environment.py
View file

@ -3,6 +3,7 @@ import logging
import cv2
import mediapipe as mp
import numpy as np
from mediapipe.framework.formats import landmark_pb2
from bodyparts import (
Crown,
@ -28,8 +29,7 @@ points_idx = [33, 263, 61, 291, 199]
points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis]
points_idx = list(set(points_idx))
points_idx.sort()
dist_coeff = np.zeros((4, 1))
print(points_idx)
class Environment:
@ -59,7 +59,7 @@ class Environment:
self.body_parts = [
# LeftEar(self),
# RightEar(self),
# Head(self),
Head(self),
# RightMoustache(self),
# LeftMoustache(self),
# LeftEye(self),
@ -88,6 +88,8 @@ class Environment:
dtype="double",
)
self.dist_coeff = np.zeros((4, 1))
self.refine_landmarks = True
def start(self) -> None:
@ -170,20 +172,27 @@ class Environment:
self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
self.mp_translation_vector = pose_transform_mat[:3, 3, None]
self.pose_transform_mat = pose_transform_mat
def draw_axis(self) -> None:
"""Draw the face axis on the frame."""
nose_tip = self.model_points[0]
nose_tip_extended = 2.5 * self.model_points[0]
(nose_pointer2D, jacobian) = cv2.projectPoints(
np.array([nose_tip, nose_tip_extended]),
self.x = self.model_points[0] + [7, 0, 0]
self.y = self.model_points[0] + [0, 7, 0]
self.z = self.model_points[0] + [0, 0, 7]
(nose_pointer2D, _) = cv2.projectPoints(
np.array([nose_tip, self.x, self.y, self.z]),
self.mp_rotation_vector,
self.mp_translation_vector,
self.camera_matrix,
dist_coeff,
self.dist_coeff,
)
nose_tip_2D, nose_tip_2D_extended = nose_pointer2D.squeeze().astype(int)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_extended, (255, 0, 0), 2)
nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointer2D.squeeze().astype(int)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2)
def draw_keypoints(self) -> None:
"""Draw the keypoints on the screen."""
@ -193,7 +202,17 @@ class Environment:
self.mp_drawing.draw_landmarks(
self.frame,
face_landmarks,
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 0, 0),
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0),
)
# draw subset of landmarks
landmark_subset = landmark_pb2.NormalizedLandmarkList(
landmark=[face_landmarks.landmark[i] for i in points_idx]
)
self.mp_drawing.draw_landmarks(
self.frame,
landmark_subset,
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 2, 0),
)
def draw_avatar(self) -> None: