idek anymore

Author: li-pearl

emotion_classifier.py

@@ -1,10 +1,14 @@
 import cv2
-import torch
 import numpy as np
-from fer import FER
+from feat import Detector
 
-# Initialize Face Detector & Emotion Recognizer
-detector = FER()
+# Initialize Py-Feat Detector
+detector = Detector(
+    face_model="retinaface", 
+    landmark_model="mobilenet", 
+    au_model="xgb", 
+    emotion_model="resmasknet"
+)
 
 # Open webcam
 cap = cv2.VideoCapture(0)
@@ -14,21 +18,37 @@
     if not ret:
         break
 
-    # Detect emotion using FER
-    results = detector.detect_emotions(frame)
+    try:
+        # Detect faces
+        face_results = detector.detect_faces(frame)
 
-    for result in results:
-        (x, y, w, h) = result["box"]
-        emotion, score = max(result["emotions"].items(), key=lambda item: item[1])
+        if face_results and len(face_results) > 0:
+            for face in face_results:
+                bbox = face["bbox"]  # Correct way to extract bounding box
+                landmarks = face["landmarks"]  # Correct way to extract landmarks
 
-        # Draw face box
-        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
+                # Ensure bbox is correctly formatted as (x, y, w, h)
+                x, y, w, h = map(int, bbox)
 
-        # Display emotion classification
-        cv2.putText(frame, f"{emotion} ({score:.2f})", (x, y-10), cv2.FONT_HERSHEY_SIMPLEX,
-                    0.8, (0, 255, 0), 2, cv2.LINE_AA)
+                # Get emotion predictions
+                emotion_scores = detector.detect_emotions(frame, facebox=bbox, landmarks=landmarks)
+                
+                if not emotion_scores.empty:
+                    dominant_emotion = emotion_scores.idxmax(axis=1).values[0]  # Get most probable emotion
+                else:
+                    dominant_emotion = "Unknown"
 
-    cv2.imshow("Emotion Classification", frame)
+                # Draw face bounding box
+                cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
+
+                # Display dominant emotion
+                cv2.putText(frame, f"Emotion: {dominant_emotion}", (x, y - 10),
+                            cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 0), 2, cv2.LINE_AA)
+
+    except Exception as e:
+        print(f"Error: {e}")
+
+    cv2.imshow("Py-Feat Emotion Classification", frame)
 
     if cv2.waitKey(1) & 0xFF == ord("q"):  # Press 'q' to quit
         break

gemini_prompt.py

@@ -8,7 +8,6 @@ def speak(text):
     engine.say(text)
     engine.runAndWait()
 
-# Configure Gemini API
 genai.configure(api_key=GEMINI_API_KEY)
 
 # Load Gemini model
@@ -18,13 +17,13 @@ def speak(text):
 def generate_visual_context_summary(speech_text, pose_label, emotion_label, gesture_label):
     prompt = f"""
     Here is the information available from the other person talking to the user:
-    - **Speech**: "{speech_text}"
+    - **Speech from the other person in the conversation**: "{speech_text}"
     - **Pose**: {pose_label}
     - **Emotion**: {emotion_label}
     - **Gesture**: {gesture_label}
 
     You are assisting a blind or neurodivergent user by providing a concise one-sentence summary of the visual context they are missing using the information provided. In addition to the analysis this sentence should provide, it should also include all pieces of information available except for the speech in the sentence.
-    The summary should be natural, clear, and informative without unnecessary details.
+    The summary should be natural, clear, and informative without unnecessary details. But, it should give context to the user about visual information. For example, if the speech is a greeting and a hand is raised, it means they are waving. 
     """
 
     # Send prompt to Gemini API
@@ -33,12 +32,11 @@ def generate_visual_context_summary(speech_text, pose_label, emotion_label, gest
     return response.text  # Get the AI-generated response
 
 # Example data (Replace with real values)
-speech_text = "I'm really excited about this project!"
+speech_text = "Raya has a ball!"
 pose_label = "Open stance"
 emotion_label = "Happy"
-gesture_label = "Hand raised"
+gesture_label = "Right Hand Raised"
 
-# Call function and print result
 summary = generate_visual_context_summary(speech_text, pose_label, emotion_label, gesture_label)
 print(summary)
 speak(summary)

recorded_audio.wav

@@ -7,7 +7,7 @@
 
 # Audio settings
 FORMAT = pyaudio.paInt16
-CHANNELS = 1
+CHANNELS = 1                                                                                                                        
 RATE = 44100  # Sample rate
 CHUNK = 1024  # Buffer size
 OUTPUT_FILENAME = "recorded_audio.wav"
@@ -18,7 +18,7 @@ def record_speech(button="space"):
     print(f"Press and hold '{button}' to record speech...")
 
     # Wait until the button is pressed
-    while not keyboard.is_pressed(button):
+    while not keyboard.is_pressed(button):                                                                     
         time.sleep(0.01)  # Prevent CPU overuse
 
     print("Recording... Speak now.")