ENSEEIHT/projet-transversal
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

feat: POC true perspective transform

Browse source
This commit is contained in:
Laureηt 2023-01-13 11:23:29 +01:00
parent 97fbc9bdb3
commit 6cd40b7d8e
Signed by: Laurent
SSH key fingerprint: SHA256:kZEpW8cMJ54PDeCvOhzreNr4FSh6R13CMGH/POoO8DI
2 changed files with 63 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

53
src/bodyparts/head.py
View file

@ -17,7 +17,7 @@ class Head:
self.env = env self.env = env
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED) self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED)
self.ratio = self.image.shape[1] / self.image.shape[0] self.ratio = self.image.shape[1] / self.image.shape[0]
self.bouding_box = np.array( self.bounding_box = np.array(
[ [
[0, 0], [0, 0],
[self.image.shape[1], 0], [self.image.shape[1], 0],
@ -31,24 +31,38 @@ class Head:
"""Draw the head on the screen.""" """Draw the head on the screen."""
# compute position # compute position
if self.env.results.multi_face_landmarks: if self.env.results.multi_face_landmarks:
x = self.env.x - self.env.model_points[0]
y = self.env.y - self.env.model_points[0]
for face_landmarks in self.env.results.multi_face_landmarks: for face_landmarks in self.env.results.multi_face_landmarks:
head_box = np.array( head_box = np.array(
[ [
100 * self.ratio * self.env.x + 100 * self.env.y + 80 * self.env.z, self.ratio * x + y + self.env.model_points[0],
100 * self.ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z, self.ratio * -x + y + self.env.model_points[0],
100 * self.ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z, self.ratio * -x + -y + self.env.model_points[0],
100 * self.ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z, self.ratio * x + -y + self.env.model_points[0],
] ]
)[:, :2] )
self.translated_head_box = ( (box, _) = cv2.projectPoints(
head_box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height]) head_box,
).astype(np.float32) self.env.mp_rotation_vector,
self.env.mp_translation_vector,
self.env.camera_matrix,
self.env.dist_coeff,
)
a, b, c, d = box.squeeze().astype(int)
cv2.line(self.env.frame, a, b, (255, 255, 255), 2)
cv2.line(self.env.frame, b, c, (255, 255, 255), 2)
cv2.line(self.env.frame, c, d, (255, 255, 255), 2)
cv2.line(self.env.frame, d, a, (255, 255, 255), 2)
# get perspective transform # get perspective transform
transform_mat = cv2.getPerspectiveTransform( transform_mat = cv2.getPerspectiveTransform(
self.bouding_box, self.bounding_box,
self.translated_head_box, box.astype(np.float32),
) )
# apply perspective transform to image # apply perspective transform to image
@ -64,11 +78,12 @@ class Head:
def draw_debug(self) -> None: def draw_debug(self) -> None:
"""Draw debug information on the screen.""" """Draw debug information on the screen."""
# link points # link points
for i in range(4): pass
cv2.line( # for i in range(4):
self.env.frame, # cv2.line(
tuple(self.translated_head_box[i].astype(np.int32)), # self.env.frame,
tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)), # tuple(self.translated_head_box[i].astype(np.int32)),
(255, 255, 255), # tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)),
2, # (255, 255, 255),
) # 2,
# )

39
src/environment.py
View file

@ -3,6 +3,7 @@ import logging
import cv2 import cv2
import mediapipe as mp import mediapipe as mp
import numpy as np import numpy as np
from mediapipe.framework.formats import landmark_pb2
from bodyparts import ( from bodyparts import (
Crown, Crown,
@ -28,8 +29,7 @@ points_idx = [33, 263, 61, 291, 199]
points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis] points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis]
points_idx = list(set(points_idx)) points_idx = list(set(points_idx))
points_idx.sort() points_idx.sort()
print(points_idx)
dist_coeff = np.zeros((4, 1))
class Environment: class Environment:
@ -59,7 +59,7 @@ class Environment:
self.body_parts = [ self.body_parts = [
# LeftEar(self), # LeftEar(self),
# RightEar(self), # RightEar(self),
# Head(self), Head(self),
# RightMoustache(self), # RightMoustache(self),
# LeftMoustache(self), # LeftMoustache(self),
# LeftEye(self), # LeftEye(self),
@ -88,6 +88,8 @@ class Environment:
dtype="double", dtype="double",
) )
self.dist_coeff = np.zeros((4, 1))
self.refine_landmarks = True self.refine_landmarks = True
def start(self) -> None: def start(self) -> None:
@ -170,20 +172,27 @@ class Environment:
self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3]) self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
self.mp_translation_vector = pose_transform_mat[:3, 3, None] self.mp_translation_vector = pose_transform_mat[:3, 3, None]
self.pose_transform_mat = pose_transform_mat
def draw_axis(self) -> None: def draw_axis(self) -> None:
"""Draw the face axis on the frame.""" """Draw the face axis on the frame."""
nose_tip = self.model_points[0] nose_tip = self.model_points[0]
nose_tip_extended = 2.5 * self.model_points[0] self.x = self.model_points[0] + [7, 0, 0]
(nose_pointer2D, jacobian) = cv2.projectPoints( self.y = self.model_points[0] + [0, 7, 0]
np.array([nose_tip, nose_tip_extended]), self.z = self.model_points[0] + [0, 0, 7]
(nose_pointer2D, _) = cv2.projectPoints(
np.array([nose_tip, self.x, self.y, self.z]),
self.mp_rotation_vector, self.mp_rotation_vector,
self.mp_translation_vector, self.mp_translation_vector,
self.camera_matrix, self.camera_matrix,
dist_coeff, self.dist_coeff,
) )
nose_tip_2D, nose_tip_2D_extended = nose_pointer2D.squeeze().astype(int) nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointer2D.squeeze().astype(int)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_extended, (255, 0, 0), 2) cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2)
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2)
def draw_keypoints(self) -> None: def draw_keypoints(self) -> None:
"""Draw the keypoints on the screen.""" """Draw the keypoints on the screen."""
@ -193,7 +202,17 @@ class Environment:
self.mp_drawing.draw_landmarks( self.mp_drawing.draw_landmarks(
self.frame, self.frame,
face_landmarks, face_landmarks,
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 0, 0), landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0),
)
# draw subset of landmarks
landmark_subset = landmark_pb2.NormalizedLandmarkList(
landmark=[face_landmarks.landmark[i] for i in points_idx]
)
self.mp_drawing.draw_landmarks(
self.frame,
landmark_subset,
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 2, 0),
) )
def draw_avatar(self) -> None: def draw_avatar(self) -> None: