feat: POC true perspective transform
This commit is contained in:
parent
97fbc9bdb3
commit
6cd40b7d8e
|
@ -17,7 +17,7 @@ class Head:
|
||||||
self.env = env
|
self.env = env
|
||||||
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED)
|
self.image = cv2.imread("assets/head.png", cv2.IMREAD_UNCHANGED)
|
||||||
self.ratio = self.image.shape[1] / self.image.shape[0]
|
self.ratio = self.image.shape[1] / self.image.shape[0]
|
||||||
self.bouding_box = np.array(
|
self.bounding_box = np.array(
|
||||||
[
|
[
|
||||||
[0, 0],
|
[0, 0],
|
||||||
[self.image.shape[1], 0],
|
[self.image.shape[1], 0],
|
||||||
|
@ -31,24 +31,38 @@ class Head:
|
||||||
"""Draw the head on the screen."""
|
"""Draw the head on the screen."""
|
||||||
# compute position
|
# compute position
|
||||||
if self.env.results.multi_face_landmarks:
|
if self.env.results.multi_face_landmarks:
|
||||||
|
|
||||||
|
x = self.env.x - self.env.model_points[0]
|
||||||
|
y = self.env.y - self.env.model_points[0]
|
||||||
|
|
||||||
for face_landmarks in self.env.results.multi_face_landmarks:
|
for face_landmarks in self.env.results.multi_face_landmarks:
|
||||||
head_box = np.array(
|
head_box = np.array(
|
||||||
[
|
[
|
||||||
100 * self.ratio * self.env.x + 100 * self.env.y + 80 * self.env.z,
|
self.ratio * x + y + self.env.model_points[0],
|
||||||
100 * self.ratio * -self.env.x + 100 * self.env.y + 80 * self.env.z,
|
self.ratio * -x + y + self.env.model_points[0],
|
||||||
100 * self.ratio * -self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
self.ratio * -x + -y + self.env.model_points[0],
|
||||||
100 * self.ratio * self.env.x + 100 * -self.env.y + 80 * self.env.z,
|
self.ratio * x + -y + self.env.model_points[0],
|
||||||
]
|
]
|
||||||
)[:, :2]
|
)
|
||||||
|
|
||||||
self.translated_head_box = (
|
(box, _) = cv2.projectPoints(
|
||||||
head_box + self.env.center[:2] * np.array([self.env.camera_width, self.env.camera_height])
|
head_box,
|
||||||
).astype(np.float32)
|
self.env.mp_rotation_vector,
|
||||||
|
self.env.mp_translation_vector,
|
||||||
|
self.env.camera_matrix,
|
||||||
|
self.env.dist_coeff,
|
||||||
|
)
|
||||||
|
|
||||||
|
a, b, c, d = box.squeeze().astype(int)
|
||||||
|
cv2.line(self.env.frame, a, b, (255, 255, 255), 2)
|
||||||
|
cv2.line(self.env.frame, b, c, (255, 255, 255), 2)
|
||||||
|
cv2.line(self.env.frame, c, d, (255, 255, 255), 2)
|
||||||
|
cv2.line(self.env.frame, d, a, (255, 255, 255), 2)
|
||||||
|
|
||||||
# get perspective transform
|
# get perspective transform
|
||||||
transform_mat = cv2.getPerspectiveTransform(
|
transform_mat = cv2.getPerspectiveTransform(
|
||||||
self.bouding_box,
|
self.bounding_box,
|
||||||
self.translated_head_box,
|
box.astype(np.float32),
|
||||||
)
|
)
|
||||||
|
|
||||||
# apply perspective transform to image
|
# apply perspective transform to image
|
||||||
|
@ -64,11 +78,12 @@ class Head:
|
||||||
def draw_debug(self) -> None:
|
def draw_debug(self) -> None:
|
||||||
"""Draw debug information on the screen."""
|
"""Draw debug information on the screen."""
|
||||||
# link points
|
# link points
|
||||||
for i in range(4):
|
pass
|
||||||
cv2.line(
|
# for i in range(4):
|
||||||
self.env.frame,
|
# cv2.line(
|
||||||
tuple(self.translated_head_box[i].astype(np.int32)),
|
# self.env.frame,
|
||||||
tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)),
|
# tuple(self.translated_head_box[i].astype(np.int32)),
|
||||||
(255, 255, 255),
|
# tuple(self.translated_head_box[(i + 1) % 4].astype(np.int32)),
|
||||||
2,
|
# (255, 255, 255),
|
||||||
)
|
# 2,
|
||||||
|
# )
|
||||||
|
|
|
@ -3,6 +3,7 @@ import logging
|
||||||
import cv2
|
import cv2
|
||||||
import mediapipe as mp
|
import mediapipe as mp
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
from mediapipe.framework.formats import landmark_pb2
|
||||||
|
|
||||||
from bodyparts import (
|
from bodyparts import (
|
||||||
Crown,
|
Crown,
|
||||||
|
@ -28,8 +29,7 @@ points_idx = [33, 263, 61, 291, 199]
|
||||||
points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis]
|
points_idx = points_idx + [key for (key, val) in procrustes_landmark_basis]
|
||||||
points_idx = list(set(points_idx))
|
points_idx = list(set(points_idx))
|
||||||
points_idx.sort()
|
points_idx.sort()
|
||||||
|
print(points_idx)
|
||||||
dist_coeff = np.zeros((4, 1))
|
|
||||||
|
|
||||||
|
|
||||||
class Environment:
|
class Environment:
|
||||||
|
@ -59,7 +59,7 @@ class Environment:
|
||||||
self.body_parts = [
|
self.body_parts = [
|
||||||
# LeftEar(self),
|
# LeftEar(self),
|
||||||
# RightEar(self),
|
# RightEar(self),
|
||||||
# Head(self),
|
Head(self),
|
||||||
# RightMoustache(self),
|
# RightMoustache(self),
|
||||||
# LeftMoustache(self),
|
# LeftMoustache(self),
|
||||||
# LeftEye(self),
|
# LeftEye(self),
|
||||||
|
@ -88,6 +88,8 @@ class Environment:
|
||||||
dtype="double",
|
dtype="double",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
self.dist_coeff = np.zeros((4, 1))
|
||||||
|
|
||||||
self.refine_landmarks = True
|
self.refine_landmarks = True
|
||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
|
@ -170,20 +172,27 @@ class Environment:
|
||||||
self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
|
self.mp_rotation_vector, _ = cv2.Rodrigues(pose_transform_mat[:3, :3])
|
||||||
self.mp_translation_vector = pose_transform_mat[:3, 3, None]
|
self.mp_translation_vector = pose_transform_mat[:3, 3, None]
|
||||||
|
|
||||||
|
self.pose_transform_mat = pose_transform_mat
|
||||||
|
|
||||||
def draw_axis(self) -> None:
|
def draw_axis(self) -> None:
|
||||||
"""Draw the face axis on the frame."""
|
"""Draw the face axis on the frame."""
|
||||||
nose_tip = self.model_points[0]
|
nose_tip = self.model_points[0]
|
||||||
nose_tip_extended = 2.5 * self.model_points[0]
|
self.x = self.model_points[0] + [7, 0, 0]
|
||||||
(nose_pointer2D, jacobian) = cv2.projectPoints(
|
self.y = self.model_points[0] + [0, 7, 0]
|
||||||
np.array([nose_tip, nose_tip_extended]),
|
self.z = self.model_points[0] + [0, 0, 7]
|
||||||
|
|
||||||
|
(nose_pointer2D, _) = cv2.projectPoints(
|
||||||
|
np.array([nose_tip, self.x, self.y, self.z]),
|
||||||
self.mp_rotation_vector,
|
self.mp_rotation_vector,
|
||||||
self.mp_translation_vector,
|
self.mp_translation_vector,
|
||||||
self.camera_matrix,
|
self.camera_matrix,
|
||||||
dist_coeff,
|
self.dist_coeff,
|
||||||
)
|
)
|
||||||
|
|
||||||
nose_tip_2D, nose_tip_2D_extended = nose_pointer2D.squeeze().astype(int)
|
nose_tip_2D, nose_tip_2D_x, nose_tip_2D_y, nose_tip_2D_z = nose_pointer2D.squeeze().astype(int)
|
||||||
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_extended, (255, 0, 0), 2)
|
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_x, (0, 0, 255), 2)
|
||||||
|
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_y, (0, 255, 0), 2)
|
||||||
|
cv2.line(self.frame, nose_tip_2D, nose_tip_2D_z, (255, 0, 0), 2)
|
||||||
|
|
||||||
def draw_keypoints(self) -> None:
|
def draw_keypoints(self) -> None:
|
||||||
"""Draw the keypoints on the screen."""
|
"""Draw the keypoints on the screen."""
|
||||||
|
@ -193,7 +202,17 @@ class Environment:
|
||||||
self.mp_drawing.draw_landmarks(
|
self.mp_drawing.draw_landmarks(
|
||||||
self.frame,
|
self.frame,
|
||||||
face_landmarks,
|
face_landmarks,
|
||||||
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 0, 0),
|
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 0), 0, 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
# draw subset of landmarks
|
||||||
|
landmark_subset = landmark_pb2.NormalizedLandmarkList(
|
||||||
|
landmark=[face_landmarks.landmark[i] for i in points_idx]
|
||||||
|
)
|
||||||
|
self.mp_drawing.draw_landmarks(
|
||||||
|
self.frame,
|
||||||
|
landmark_subset,
|
||||||
|
landmark_drawing_spec=self.mp_drawing_styles.DrawingSpec((0, 0, 255), 2, 0),
|
||||||
)
|
)
|
||||||
|
|
||||||
def draw_avatar(self) -> None:
|
def draw_avatar(self) -> None:
|
||||||
|
|
Loading…
Reference in a new issue