gesturecontrolpc/gesture_control.py at main · BerrF35/gesturecontrolpc · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
import cv2
import mediapipe as mp
import os
import numpy as np
import time

# MediaPipe Hands setup
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(max_num_hands=1, min_detection_confidence=0.7)
mp_draw = mp.solutions.drawing_utils

# MediaPipe Face Mesh setup
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh(
    static_image_mode=False,
    max_num_faces=1,
    refine_landmarks=True,
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5
)

# Helper functions for fingers
def finger_is_up(hand_landmarks, tip_id, pip_id, threshold=0.05):
    return (hand_landmarks.landmark[pip_id].y - hand_landmarks.landmark[tip_id].y) > threshold

def finger_is_down(hand_landmarks, tip_id, mcp_id, threshold=0.03):
    return (hand_landmarks.landmark[tip_id].y - hand_landmarks.landmark[mcp_id].y) > threshold

def spider_man_gesture(hand_landmarks, spacing_thresh=0.1, fold_thresh=0.05):
    index_up = finger_is_up(hand_landmarks, 8, 6)
    pinky_up = finger_is_up(hand_landmarks, 20, 18)
    middle_fold = finger_is_down(hand_landmarks, 12, 9, fold_thresh)
    ring_fold = finger_is_down(hand_landmarks, 16, 13, fold_thresh)
    tip_dist = abs(hand_landmarks.landmark[8].x - hand_landmarks.landmark[20].x)
    return (index_up and pinky_up and middle_fold and ring_fold and tip_dist > spacing_thresh)

def is_fist(hand_landmarks, fold_threshold=0.03):
    return all([
        finger_is_down(hand_landmarks, 4, 3, fold_threshold),
        finger_is_down(hand_landmarks, 8, 6, fold_threshold),
        finger_is_down(hand_landmarks, 12, 10, fold_threshold),
        finger_is_down(hand_landmarks, 16, 14, fold_threshold),
        finger_is_down(hand_landmarks, 20, 18, fold_threshold)
    ])

def all_fingers_up(hand_landmarks, threshold=0.05):
    return all([
        finger_is_up(hand_landmarks, 4, 3, threshold),
        finger_is_up(hand_landmarks, 8, 6, threshold),
        finger_is_up(hand_landmarks, 12, 10, threshold),
        finger_is_up(hand_landmarks, 16, 14, threshold),
        finger_is_up(hand_landmarks, 20, 18, threshold)
    ])

# Eye landmarks for wink detection (right eye)
RIGHT_EYE_TOP = 386
RIGHT_EYE_BOTTOM = 374
RIGHT_EYE_LEFT = 263
RIGHT_EYE_RIGHT = 362

def eye_aspect_ratio(landmarks, top_id, bottom_id, left_id, right_id):
    top = landmarks.landmark[top_id]
    bottom = landmarks.landmark[bottom_id]
    left = landmarks.landmark[left_id]
    right = landmarks.landmark[right_id]
    vertical_dist = np.linalg.norm(np.array([top.x, top.y]) - np.array([bottom.x, bottom.y]))
    horizontal_dist = np.linalg.norm(np.array([left.x, left.y]) - np.array([right.x, right.y]))
    if horizontal_dist == 0:
        return 0
    return vertical_dist / horizontal_dist

# Gesture sequence state for screenshot (fist → open hand → fist)
gesture_state = 0
state_start_time = 0

wink_cooldown = 0
WINK_THRESHOLD = 0.2  # Adjust based on testing
WINK_COOLDOWN_TIME = 2  # seconds

cap = cv2.VideoCapture(0)

while True:
    success, img = cap.read()
    if not success:
        break

    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    # Process hands
    hand_results = hands.process(img_rgb)

    # Process face
    face_results = face_mesh.process(img_rgb)

    now = time.time()

    # -- Wink Detection --
    if face_results.multi_face_landmarks:
        face_landmarks = face_results.multi_face_landmarks[0]
        ear_right = eye_aspect_ratio(face_landmarks, RIGHT_EYE_TOP, RIGHT_EYE_BOTTOM, RIGHT_EYE_LEFT, RIGHT_EYE_RIGHT)
        if ear_right < WINK_THRESHOLD and now - wink_cooldown > WINK_COOLDOWN_TIME:
            wink_cooldown = now
            print("Wink detected — opening Snip & Sketch!")
            os.system("start ms-screenclip:")

    # -- Hand Gestures --
    if hand_results.multi_hand_landmarks:
        hand = hand_results.multi_hand_landmarks[0]
        mp_draw.draw_landmarks(img, hand, mp_hands.HAND_CONNECTIONS)

        thumb_up = finger_is_up(hand, 4, 3)
        index_up = finger_is_up(hand, 8, 6)
        middle_up = finger_is_up(hand, 12, 10)
        ring_up = finger_is_up(hand, 16, 14)
        pinky_up = finger_is_up(hand, 20, 18)

        thumb_down = finger_is_down(hand, 4, 3)
        index_down = finger_is_down(hand, 8, 6)
        middle_down = finger_is_down(hand, 12, 10)
        ring_down = finger_is_down(hand, 16, 14)
        pinky_down = finger_is_down(hand, 20, 18)

        # More sensitive middle finger lock
        if middle_up and not (index_up or ring_up):
            print("Middle finger detected — locking screen!")
            os.system("rundll32.exe user32.dll,LockWorkStation")
            break

        # Open Chrome: Index finger only but stricter
        elif index_up and all([thumb_down, middle_down, ring_down, pinky_down]):
            print("Index finger detected — opening Chrome!")
            os.system("start chrome")
            break

        # Open Chrome: Spider-Man finger refined
        elif spider_man_gesture(hand):
            print("Spider-Man finger detected — opening Chrome!")
            os.system("start chrome")
            break

        # Screenshot sequence (fist → open hand → fist)
        if gesture_state == 0:
            if is_fist(hand):
                gesture_state = 1
                state_start_time = now
                print("Gesture step 1: Fist detected")
        elif gesture_state == 1:
            if all_fingers_up(hand):
                gesture_state = 2
                print("Gesture step 2: Open hand detected")
        elif gesture_state == 2:
            if is_fist(hand):
                if now - state_start_time <= 3:
                    print("Gesture complete: Opening Snip & Sketch for screenshot")
                    os.system("start ms-screenclip:")
                gesture_state = 0
        if gesture_state != 0 and now - state_start_time > 3:
            gesture_state = 0

    else:
        gesture_state = 0

    cv2.imshow("Gesture and Wink Control", img)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()