Merge pull request #27 from Gerosullivan/llm-recall-attempt-feedback

Llm recall attempt feedback
Gerosullivan · Oct 17, 2024 · 1829afb · 1829afb
2 parents b406fe9 + 7d59924
commit 1829afb
Show file tree

Hide file tree

Showing 10 changed files with 138 additions and 180 deletions.
diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts
@@ -16,94 +16,44 @@ export const maxDuration = 180
 export async function POST(request: Request) {
   try {
     const json = await request.json()
-    const {
-      messages,
-      chatId,
-      studyState,
-      studySheet,
-      chatRecallInfo,
-      randomRecallFact,
-      systemContext
-    } = json
-
-    const studentMessage = messages[messages.length - 1]
 
     const defaultModel = openai("gpt-4o-mini") as LanguageModel
-    const scoringModel = openai("gpt-4o", {
-      structuredOutputs: true
-    }) as LanguageModel
-    const hintingModel = defaultModel
-
     const nextStudyState: StudyState =
-      getNextStudyState(studyState) || studyState
+      getNextStudyState(json.studyState) || json.studyState
 
-    switch (studyState as StudyState) {
+    const context = {
+      ...json,
+      defaultModel,
+      nextStudyState
+    }
+
+    switch (context.studyState as StudyState) {
       case "topic_name_saved":
       case "topic_describe_upload":
       case "topic_no_description_in_db":
-        return await handleTopicGeneration(
-          defaultModel,
-          messages,
-          systemContext,
-          nextStudyState
-        )
+        return await handleTopicGeneration(context)
 
       case "recall_first_attempt":
-        return await handleRecallAttempt(
-          scoringModel,
-          defaultModel,
-          nextStudyState,
-          studySheet,
-          chatId,
-          studentMessage,
-          systemContext
-        )
+        return await handleRecallAttempt(context)
 
       case "recall_show_hints":
-        return await handleRecallShowHints(
-          defaultModel,
-          studySheet,
-          chatRecallInfo,
-          systemContext,
-          nextStudyState
-        )
+        return await handleRecallShowHints(context)
+
       case "recall_final_suboptimal_feedback":
       case "recall_answer_hints":
-        return await handleHinting(
-          hintingModel,
-          messages,
-          nextStudyState,
-          studySheet,
-          chatRecallInfo,
-          systemContext
-        )
+        return await handleHinting(context)
 
       case "recall_finished":
       case "reviewing":
-        return await handleReview(defaultModel, messages, systemContext)
+        return await handleReview(context)
 
       case "quick_quiz_question":
-        return await handleQuickQuizQuestion(
-          defaultModel,
-          studySheet,
-          randomRecallFact,
-          systemContext,
-          nextStudyState
-        )
+        return await handleQuickQuizQuestion(context)
 
       case "quick_quiz_answer_next":
       case "quick_quiz_user_answer":
       case "quick_quiz_finished":
-        const noMoreQuizQuestions = studyState === "quick_quiz_finished"
-        return await handleQuickQuizAnswer(
-          defaultModel,
-          messages,
-          nextStudyState,
-          studySheet,
-          studentMessage,
-          systemContext,
-          noMoreQuizQuestions
-        )
+        return await handleQuickQuizAnswer(context)
 
       default:
         // Handle other states or error

diff --git a/lib/server/hinting-handler.ts b/lib/server/hinting-handler.ts
@@ -2,54 +2,50 @@ import { StudyState } from "@/lib/studyStates"
 import { streamText, LanguageModel, convertToCoreMessages } from "ai"
 import { formatDistanceToNow } from "date-fns/esm"
 
-export async function handleHinting(
-  hintingModel: LanguageModel,
-  messages: any[],
-  nextStudyState: StudyState,
-  studySheet: string,
-  chatRecallInfo: any,
+export async function handleHinting(context: {
+  defaultModel: LanguageModel
+  messages: any[]
+  nextStudyState: StudyState
+  studySheet: string
+  chatRecallInfo: any
   systemContext: string
-) {
-  const dueDateFromNow = formatDistanceToNow(new Date(chatRecallInfo.due_date))
+}) {
+  const dueDateFromNow = formatDistanceToNow(
+    new Date(context.chatRecallInfo.due_date)
+  )
+
   const chatStreamResponse = await streamText({
-    model: hintingModel,
+    model: context.defaultModel,
     temperature: 0.3,
     messages: convertToCoreMessages([
       {
         role: "system",
-        content: `${systemContext}
-When constructing feedback for a student's attempt at answering hints on a recall test, follow these guidelines:
-
-Use this topic source only when providing feedback:
-  <StudySheet>
-  ${studySheet}.
-  <StudySheet>
+        content: `${context.systemContext}
+        When providing feedback on a student's recall attempt, use only the information from this topic source:
+        
+        <StudySheet>
+        ${context.studySheet}
+        </StudySheet>
 
-Your response should follow this exact structure:
-1. Positive reinforcement
-2. Highlight of correct answers
-3. Address of incorrect answers
-4. Additional information
-5. Encouragement for continued effort
-6. Next steps and scheduling;   Inform the student about their next recall session based on this due date: ${dueDateFromNow}.
-  Provide a specific timeframe for the next session.
-  Use calendar emoji for visual reinforcement.
-  Example: "Your next recall session is due in {{dueDateFromNow}}. 📅"
-7. Study recommendations
-8. Focus areas
-9. Future outlook
+        Respond in a conversational tone, as if you're chatting with a friend. Avoid sounding like a teacher giving formal feedback. Instead of using a structured format, blend your feedback into a natural conversation. Here are some points to cover in your response:
 
-IMPORTANT: Do not include any meta-commentary about your response or offer additional assistance. Your response should only contain the feedback for the student as instructed above.
+        - Start with a friendly, encouraging comment about their effort.
+        - Mention what they got right, but do it casually.
+        - If they missed anything, bring it up gently without making a big deal of it.
+        - Add a bit of extra info if it's relevant, but keep it light.
+        - Throw in some encouragement for their next study session.
+        - Let them know when their next recall is due, like this: "By the way, your next recall is coming up in ${dueDateFromNow}. 📅"
+        - Suggest a quick study tip if you think it'll help.
 
-After providing the feedback as structured above, end your response immediately. Do not add any concluding remarks or offers for further assistance`
+        Keep your tone upbeat and supportive throughout. Wrap up your response naturally, as if you're ending a chat with a friend. Don't offer additional help or add any closing remarks – just end it when you've covered these points.`
       },
-      ...messages
+      ...context.messages
     ])
   })
 
   return chatStreamResponse.toDataStreamResponse({
     headers: {
-      "NEW-STUDY-STATE": nextStudyState
+      "NEW-STUDY-STATE": context.nextStudyState
     }
   })
 }
diff --git a/lib/server/quick-quiz-answer-handler.ts b/lib/server/quick-quiz-answer-handler.ts
@@ -1,22 +1,24 @@
 import { StudyState } from "@/lib/studyStates"
 import { streamText, LanguageModel, convertToCoreMessages } from "ai"
 
-export async function handleQuickQuizAnswer(
-  defaultModel: LanguageModel,
-  messages: any[],
-  nextStudyState: StudyState,
-  studySheet: string,
-  studentMessage: any,
-  systemContext: string,
-  noMoreQuizQuestions: boolean
-) {
-  const previousQuizQuestion = messages[messages.length - 2].content
+export async function handleQuickQuizAnswer(context: {
+  defaultModel: LanguageModel
+  messages: any[]
+  nextStudyState: StudyState
+  studySheet: string
+  systemContext: string
+  studyState: StudyState
+}) {
+  const noMoreQuizQuestions = context.studyState === "quick_quiz_finished"
+  const studentMessage = context.messages[context.messages.length - 1].content
+  const previousQuizQuestion =
+    context.messages[context.messages.length - 2].content
   const finalFeedback = noMoreQuizQuestions
     ? "Finally advise the student there are no more quiz questions available. Come back again another time."
     : ""
 
   const chatStreamResponse = await streamText({
-    model: defaultModel,
+    model: context.defaultModel,
     temperature: 0.3,
     messages: convertToCoreMessages([
       {
@@ -27,16 +29,16 @@ Never give the answer to the question when generating the question text.
 Do not state which step of the instructions you are on.
 Always provide the answer when giving feedback to the student. 
 If the student answers "I don't know.", just give the answer.
-${systemContext}`
+${context.systemContext}`
       },
       {
         role: "user",
         content: `Provide feedback and answer to the following quiz question:
           """${previousQuizQuestion}"""
           Based on the following student response:
-          """${studentMessage.content}"""
+          """${studentMessage}"""
           Given this topic study sheet as context:
-          """${studySheet}"""
+          """${context.studySheet}"""
           ${finalFeedback}
           `
       }
@@ -45,7 +47,7 @@ ${systemContext}`
 
   return chatStreamResponse.toDataStreamResponse({
     headers: {
-      "NEW-STUDY-STATE": nextStudyState
+      "NEW-STUDY-STATE": context.nextStudyState
     }
   })
 }
diff --git a/lib/server/quick-quiz-question-handler.ts b/lib/server/quick-quiz-question-handler.ts
@@ -1,15 +1,15 @@
 import { StudyState } from "@/lib/studyStates"
 import { streamText, LanguageModel, convertToCoreMessages } from "ai"
 
-export async function handleQuickQuizQuestion(
-  defaultModel: LanguageModel,
-  studySheet: string,
-  randomRecallFact: string,
-  systemContext: string,
+export async function handleQuickQuizQuestion(context: {
+  defaultModel: LanguageModel
+  studySheet: string
+  randomRecallFact: string
+  systemContext: string
   nextStudyState: StudyState
-) {
+}) {
   const chatStreamResponse = await streamText({
-    model: defaultModel,
+    model: context.defaultModel,
     temperature: 0.3,
     messages: convertToCoreMessages([
       {
@@ -18,22 +18,22 @@ export async function handleQuickQuizQuestion(
 Generate short answer quiz questions based on a provided fact. 
 Never give the answer to the question when generating the question text. 
 Do not state which step of the instructions you are on.
-${systemContext}`
+${context.systemContext}`
       },
       {
         role: "user",
         content: `Given this topic study sheet as context:
-<StudySheet>${studySheet}</StudySheet>
+<StudySheet>${context.studySheet}</StudySheet>
 Generate a short answer quiz question based on the following fact the student previously got incorrect:
-<Fact>${randomRecallFact}</Fact>
+<Fact>${context.randomRecallFact}</Fact>
 Important: Do not provide the answer when generating the question or mention the fact used to generate quiz question.`
       }
     ])
   })
 
   return chatStreamResponse.toDataStreamResponse({
     headers: {
-      "NEW-STUDY-STATE": nextStudyState
+      "NEW-STUDY-STATE": context.nextStudyState
     }
   })
 }