feat(audiorag): acutal app development

1p22geo · May 5, 2024 · 60c2529 · 60c2529
1 parent 1e4b976
commit 60c2529
Show file tree

Hide file tree

Showing 23 changed files with 559 additions and 159 deletions.
diff --git a/audio-rag/endpoint/app.py b/audio-rag/endpoint/app.py
@@ -1,6 +1,8 @@
 from flask import Flask, request, redirect
 from rag_set import query, load
 from werkzeug.utils import secure_filename
+import mimetypes
+from datetime import datetime
 import os
 
 
@@ -18,11 +20,14 @@ def index():
 @app.route('/api/query', methods=['POST'])
 def api_query():
     json = request.json
-    res = query(json['prompt'])
-    context = []
-    for doc in res['context']:
-        context.append(doc.dict())
-    return {"answer": res['answer'], "context": context}
+    try:
+        res = query(json['prompt'])
+        context = []
+        for doc in res['context']:
+            context.append(doc.dict())
+        return {"answer": res['answer'], "context": context}
+    except:
+        return {"answer": "An error occured. Make sure to upload some documents before chatting with them."}
 
 
 @app.route('/upload', methods=['POST'])
@@ -33,11 +38,21 @@ def upload_file():
     if file.filename == '':
         raise 'No selected file'
     if file:
-        filename = secure_filename(file.filename)
+        filename = datetime.now().isoformat() + \
+            secure_filename(file.filename).split(
+                ".")[0] + mimetypes.guess_extension(file.mimetype)
         file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
         load(filename)
     return redirect("/", code=302)
 
 
+@app.route("/files", methods=['GET'])
+def list_files():
+    files = []
+    for file in os.listdir("data/"):
+        files.append(file)
+    return files
+
+
 if __name__ == "__main__":
     app.run()
diff --git a/audio-rag/endpoint/rag_set.py b/audio-rag/endpoint/rag_set.py
@@ -1,3 +1,4 @@
+import speech_recognition as sr
 from langchain_text_splitters import CharacterTextSplitter
 from langchain_community.vectorstores import FAISS
 from langchain_community.document_loaders import TextLoader
@@ -6,6 +7,7 @@
 from langchain_community.llms import Ollama
 from langchain.chains.combine_documents import create_stuff_documents_chain
 from langchain.chains import create_retrieval_chain
+from langchain_core.documents import Document
 from langchain import hub
 import os
 
@@ -55,6 +57,31 @@ def load(f):
     global chain
     global texts
     match f.split(".")[-1]:
+        case "mp3":
+            name = ".".join(f.split(".")[:-1])
+            os.system(f"ffmpeg -i data/{name}.mp3 data/{name}.wav")
+            os.remove(f"data/{name}.mp3")
+            r = sr.Recognizer()
+            file = sr.AudioFile(f"data/{name}.wav")
+            with file as source:
+                audio = r.record(source)
+            txt = str(r.recognize_sphinx(audio))
+            documents = [
+                Document(page_content=txt, metadata={"source": f"data/{name}.wav"})]
+            text_splitter = CharacterTextSplitter(
+                chunk_size=500, chunk_overlap=10)
+            texts.extend(text_splitter.split_documents(documents))
+        case "wav":
+            r = sr.Recognizer()
+            file = sr.AudioFile("data/"+f)
+            with file as source:
+                audio = r.record(source)
+            txt = str(r.recognize_sphinx(audio))
+            documents = [
+                Document(page_content=txt, metadata={"source": "data/"+f})]
+            text_splitter = CharacterTextSplitter(
+                chunk_size=500, chunk_overlap=10)
+            texts.extend(text_splitter.split_documents(documents))
         case "txt":
             loader = TextLoader("data/"+f)
             documents = loader.load()
@@ -67,6 +94,9 @@ def load(f):
             text_splitter = CharacterTextSplitter(
                 chunk_size=500, chunk_overlap=10)
             texts.extend(text_splitter.split_documents(documents))
+        case _:
+            print(f, "not accepted")
+            os.remove("data/"+f)
 
     db = FAISS.from_documents(texts, embeddings)
     retriever = db.as_retriever()

diff --git a/audio-rag/endpoint/static/index.html b/audio-rag/endpoint/static/index.html
@@ -0,0 +1,34 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Ollama RAG with LLAMA 3</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <script src="./main.js" defer></script>
+  </head>
+  <body class="w-screen h-screen bg-emerald-100 overflow-scroll flex flex-row">
+    <div id="files" class="flex flex-col w-[30%] p-8 gap-4"><h1 class="text-xl">Files in RAG store</h1></div>
+    <div id="main" class="flex flex-col w-[70%] p-8 gap-4"><h1 class="text-xl">Chat</h1></div>
+    <div class="fixed bottom-8 right-8 h-12 flex items-stretch justify-stretch">
+      <form method="post" enctype="multipart/form-data" action="/upload">
+      <input type="file" name="file" class="ml-4 outline-none p-2 self-center">
+      <input type="submit" value="Upload" class="cursor-pointer hover:underline">
+    </form>
+      <div id="spinner" class="w-12 h-12 outline hidden animate-spin"></div>
+      <input
+        type="text"
+        name="q"
+        id="q"
+        value=""
+        class="ml-4 outline-none w-[600px] rounded-l-full p-2 px-8"
+      />
+      <button
+        id="send"
+        class="bg-emerald-600 text-white p-2 px-4 rounded-r-full"
+      >
+        Send
+      </button>
+    </div>
+  </body>
+</html>
diff --git a/audio-rag/endpoint/static/main.js b/audio-rag/endpoint/static/main.js
@@ -0,0 +1,74 @@
+document.querySelector("#send").onclick = async () => {
+  const prompt = document.querySelector("#q").value;
+  document.querySelector("#q").value = "";
+
+  document.querySelector("#spinner").classList.toggle("hidden");
+
+  const q_div = document.createElement("div");
+  q_div.className =
+    "m-4 mr-24 flex flex-col gap-2 items-start bg-emerald-400 p-4 rounded-md";
+  const user_div = document.createElement("div");
+  user_div.innerText = "User:";
+  const prompt_div = document.createElement("div");
+  prompt_div.innerText = prompt;
+  q_div.appendChild(user_div);
+  q_div.appendChild(prompt_div);
+  document.querySelector("#main").appendChild(q_div);
+
+  const req = await fetch("/api/query", {
+    body: JSON.stringify({
+      prompt,
+    }),
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+  const res = await req.json();
+  console.log(res);
+  const r_div = document.createElement("div");
+  r_div.className =
+    "m-4 ml-24 flex flex-col gap-2 bg-emerald-300 p-4 rounded-md";
+  const a_div = document.createElement("div");
+  a_div.innerText = "RAG:";
+  const answer_div = document.createElement("div");
+  answer_div.innerText = res.answer;
+  const context_div = document.createElement("div");
+  res.context.forEach((elem) => {
+    const context_item = document.createElement("div");
+    context_item.className = "m-4 border-b-2";
+    const text = document.createElement("div");
+    text.innerText = elem.page_content;
+    context_item.appendChild(text);
+    const source = document.createElement("div");
+    source.innerText = elem.metadata.source;
+    source.className = "text-right font-semibold";
+    context_item.appendChild(source);
+    context_div.appendChild(context_item);
+  });
+  r_div.appendChild(a_div);
+  r_div.appendChild(answer_div);
+  r_div.appendChild(context_div);
+
+  document.querySelector("#main").appendChild(r_div);
+
+  document.querySelector("#spinner").classList.toggle("hidden");
+};
+const getFileType = (filename) => {
+  switch (filename.split('.').at(-1)) {
+    case 'txt':
+      return 'Text file';
+    case 'wav':
+      return 'WAV Sound recording';
+    case 'pdf':
+      return 'PDF document';
+    default:
+      return '<<unknown>>';
+  }
+};
+
+fetch("/files").then(res=>res.json().then(json=>{
+  document.querySelector("#files").innerHTML = `<h1 class="text-xl">Files in RAG store</h1>` + json.map(file=>`
+  <div class="p-4 bg-white rounded-md flex flex-row items-center justify-between"><div>${file}</div><div class="italic">${getFileType(file)}</div></div>
+`).join("")
+}))
diff --git a/audio-rag/native/README.md b/audio-rag/native/README.md
@@ -2,7 +2,7 @@ This is a new [**React Native**](https://reactnative.dev) project, bootstrapped
 
 # Getting Started
 
->**Note**: Make sure you have completed the [React Native - Environment Setup](https://reactnative.dev/docs/environment-setup) instructions till "Creating a new application" step, before proceeding.
+> **Note**: Make sure you have completed the [React Native - Environment Setup](https://reactnative.dev/docs/environment-setup) instructions till "Creating a new application" step, before proceeding.
 
 ## Step 1: Start the Metro Server
 

diff --git a/audio-rag/native/__tests__/App.test.tsx b/audio-rag/native/__tests__/App.test.tsx
diff --git a/audio-rag/native/index.js b/audio-rag/native/index.js
@@ -2,8 +2,8 @@
  * @format
  */
 
-import { AppRegistry } from 'react-native';
+import {AppRegistry} from 'react-native';
 import App from './src/App';
-import { name as appName } from './app.json';
+import {name as appName} from './app.json';
 
 AppRegistry.registerComponent(appName, () => App);
diff --git a/audio-rag/native/ios/audiorag/Images.xcassets/AppIcon.appiconset/Contents.json b/audio-rag/native/ios/audiorag/Images.xcassets/AppIcon.appiconset/Contents.json
@@ -1,53 +1,53 @@
 {
-  "images" : [
+  "images": [
     {
-      "idiom" : "iphone",
-      "scale" : "2x",
-      "size" : "20x20"
+      "idiom": "iphone",
+      "scale": "2x",
+      "size": "20x20"
     },
     {
-      "idiom" : "iphone",
-      "scale" : "3x",
-      "size" : "20x20"
+      "idiom": "iphone",
+      "scale": "3x",
+      "size": "20x20"
     },
     {
-      "idiom" : "iphone",
-      "scale" : "2x",
-      "size" : "29x29"
+      "idiom": "iphone",
+      "scale": "2x",
+      "size": "29x29"
     },
     {
-      "idiom" : "iphone",
-      "scale" : "3x",
-      "size" : "29x29"
+      "idiom": "iphone",
+      "scale": "3x",
+      "size": "29x29"
     },
     {
-      "idiom" : "iphone",
-      "scale" : "2x",
-      "size" : "40x40"
+      "idiom": "iphone",
+      "scale": "2x",
+      "size": "40x40"
     },
     {
-      "idiom" : "iphone",
-      "scale" : "3x",
-      "size" : "40x40"
+      "idiom": "iphone",
+      "scale": "3x",
+      "size": "40x40"
     },
     {
-      "idiom" : "iphone",
-      "scale" : "2x",
-      "size" : "60x60"
+      "idiom": "iphone",
+      "scale": "2x",
+      "size": "60x60"
     },
     {
-      "idiom" : "iphone",
-      "scale" : "3x",
-      "size" : "60x60"
+      "idiom": "iphone",
+      "scale": "3x",
+      "size": "60x60"
     },
     {
-      "idiom" : "ios-marketing",
-      "scale" : "1x",
-      "size" : "1024x1024"
+      "idiom": "ios-marketing",
+      "scale": "1x",
+      "size": "1024x1024"
     }
   ],
-  "info" : {
-    "author" : "xcode",
-    "version" : 1
+  "info": {
+    "author": "xcode",
+    "version": 1
   }
 }
diff --git a/audio-rag/native/ios/audiorag/Images.xcassets/Contents.json b/audio-rag/native/ios/audiorag/Images.xcassets/Contents.json
@@ -1,6 +1,6 @@
 {
-  "info" : {
-    "version" : 1,
-    "author" : "xcode"
+  "info": {
+    "version": 1,
+    "author": "xcode"
   }
 }
diff --git a/audio-rag/native/package.json b/audio-rag/native/package.json
@@ -29,7 +29,6 @@
     "eslint": "^8.19.0",
     "jest": "^29.6.3",
     "prettier": "2.8.8",
-    "react-test-renderer": "18.2.0",
     "typescript": "5.0.4"
   },
   "engines": {