Mediapipe Rotations

Currently in development process.

Future python package to calculate rotation data for Googles Mediapipe Detection results. The calculations heavly rely on the crate cgt_math which is also in development process.

Setup dev enviroment

Either run the setup.bash or setup manually:

python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
maturin develop
pytest

Usage Example

import mediapipe as mp
import mediapipe_rotations as mpr
import cv2

mp_drawing = mp.solutions.drawing_utils
mp_drawing_styles = mp.solutions.drawing_styles
mp_holistic = mp.solutions.holistic


def cvt2array(landmark_list):
    """ converts landmark list to list. """
    return [[landmark.x, landmark.y, landmark.z] for landmark in landmark_list.landmark]


cap = cv2.VideoCapture(0)
with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:
    while cap.isOpened():
        success, image = cap.read()
        if not success:
            print("Ignoring empty camera frame.")
            # If loading a video, use 'break' instead of 'continue'.
            continue

        # To improve performance, optionally mark the image as not writeable to
        # pass by reference.
        image.flags.writeable = False
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        results = holistic.process(image)

        # Get Detection Data
        pose, face, l_hand, r_hand = [], [], [], []

        if results.pose_landmarks:
            pose = cvt2array(results.pose_landmarks)

        if results.face_landmarks:
            face = cvt2array(results.face_landmarks)

        if results.left_hand_landmarks and results.right_hand_landmarks:
            l_hand = cvt2array(results.left_hand_landmarks)
            r_hand = cvt2array(results.right_hand_landmarks)
        elif results.left_hand_landmarks:
            l_hand = cvt2array(results.left_hand_landmarks)
        elif results.right_hand_landmarks:
            r_hand = cvt2array(results.right_hand_landmarks)

        # Calculate rotations
        pose_rotation_quaternion = mpr.pose(pose)
        hand_rotation_quaternionL = mpr.hand(l_hand)
        hand_rotation_quaternionR = mpr.hand(r_hand)
        hands_rotation_quaternion = mpr.hands([l_hand, r_hand])
        face_rotation_quaternion = mpr.face(face)
        holistic_rotation_quaternion = mpr.holistic([pose, face, l_hand, r_hand])

        # Draw landmark annotation on the image.
        image.flags.writeable = True
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        mp_drawing.draw_landmarks(
            image,
            results.face_landmarks,
            mp_holistic.FACEMESH_CONTOURS,
            landmark_drawing_spec=None,
            connection_drawing_spec=mp_drawing_styles
                .get_default_face_mesh_contours_style())

        mp_drawing.draw_landmarks(
            image,
            results.pose_landmarks,
            mp_holistic.POSE_CONNECTIONS,
            landmark_drawing_spec=mp_drawing_styles
                .get_default_pose_landmarks_style())

        # Flip the image horizontally for a selfie-view display.
        cv2.imshow('MediaPipe Holistic', cv2.flip(image, 1))
        if cv2.waitKey(5) & 0xFF == 27:
            break
cap.release()

Results

The results come in quaternions. There are some blank spots in the returned array, mainly for easier indexing.

Face

Idx	Target	+
0	head	+
1	chin	+
2	mouth corner.L	+
3	mouth corner.R	+

Hand

Idx	Target	+
0	wrist	+
1	thumb cmc	+
2	thumb mcp	+
3	thumb ip	+
4	thumb tip
5	index finger mcp	+
6	index finger pip	+
7	index finger dip	+
8	index finger tip
9	middle finger mcp	+
10	middle finger pip	+
11	middle finger dip	+
12	middle finger tip
13	ring finger mcp	+
14	ring finger pip	+
15	ring finger dip	+
16	ring finger tip
17	pinky mcp	+
18	pinky pip	+
19	pinky dip	+
20	pinky tip

Pose

0	nose	+
1	left eye inner
2	left eye
3	left eye outer
4	right eye inner
5	right eye
6	right eyerouter
7	left ear
8	right ear
9	mouth left
10	mouth right
11	left shoulder	+
12	right shoulder	+
13	left elbow	+
14	right elbow	+
15	left wrist	+
16	right wrist	+
17	left pinky
18	right pinky
19	left index
20	right index
21	left thumb
22	right thumb
23	left hip	+
24	right hip	+
25	left knee	+
26	right knee	+
27	left ankle	+
28	right ankle	+
29	left heel
30	right heel
31	left foot index
32	right foot index
33	hip center	+
34	shoulder center	+