diff --git a/audio-rag/endpoint/app.py b/audio-rag/endpoint/app.py
index aeadbb3..e7de249 100644
--- a/audio-rag/endpoint/app.py
+++ b/audio-rag/endpoint/app.py
@@ -1,6 +1,8 @@
from flask import Flask, request, redirect
from rag_set import query, load
from werkzeug.utils import secure_filename
+import mimetypes
+from datetime import datetime
import os
@@ -18,11 +20,14 @@ def index():
@app.route('/api/query', methods=['POST'])
def api_query():
json = request.json
- res = query(json['prompt'])
- context = []
- for doc in res['context']:
- context.append(doc.dict())
- return {"answer": res['answer'], "context": context}
+ try:
+ res = query(json['prompt'])
+ context = []
+ for doc in res['context']:
+ context.append(doc.dict())
+ return {"answer": res['answer'], "context": context}
+ except:
+ return {"answer": "An error occured. Make sure to upload some documents before chatting with them."}
@app.route('/upload', methods=['POST'])
@@ -33,11 +38,21 @@ def upload_file():
if file.filename == '':
raise 'No selected file'
if file:
- filename = secure_filename(file.filename)
+ filename = datetime.now().isoformat() + \
+ secure_filename(file.filename).split(
+ ".")[0] + mimetypes.guess_extension(file.mimetype)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
load(filename)
return redirect("/", code=302)
+@app.route("/files", methods=['GET'])
+def list_files():
+ files = []
+ for file in os.listdir("data/"):
+ files.append(file)
+ return files
+
+
if __name__ == "__main__":
app.run()
diff --git a/audio-rag/endpoint/rag_set.py b/audio-rag/endpoint/rag_set.py
index b2fd404..e370c94 100644
--- a/audio-rag/endpoint/rag_set.py
+++ b/audio-rag/endpoint/rag_set.py
@@ -1,3 +1,4 @@
+import speech_recognition as sr
from langchain_text_splitters import CharacterTextSplitter
from langchain_community.vectorstores import FAISS
from langchain_community.document_loaders import TextLoader
@@ -6,6 +7,7 @@
from langchain_community.llms import Ollama
from langchain.chains.combine_documents import create_stuff_documents_chain
from langchain.chains import create_retrieval_chain
+from langchain_core.documents import Document
from langchain import hub
import os
@@ -55,6 +57,31 @@ def load(f):
global chain
global texts
match f.split(".")[-1]:
+ case "mp3":
+ name = ".".join(f.split(".")[:-1])
+ os.system(f"ffmpeg -i data/{name}.mp3 data/{name}.wav")
+ os.remove(f"data/{name}.mp3")
+ r = sr.Recognizer()
+ file = sr.AudioFile(f"data/{name}.wav")
+ with file as source:
+ audio = r.record(source)
+ txt = str(r.recognize_sphinx(audio))
+ documents = [
+ Document(page_content=txt, metadata={"source": f"data/{name}.wav"})]
+ text_splitter = CharacterTextSplitter(
+ chunk_size=500, chunk_overlap=10)
+ texts.extend(text_splitter.split_documents(documents))
+ case "wav":
+ r = sr.Recognizer()
+ file = sr.AudioFile("data/"+f)
+ with file as source:
+ audio = r.record(source)
+ txt = str(r.recognize_sphinx(audio))
+ documents = [
+ Document(page_content=txt, metadata={"source": "data/"+f})]
+ text_splitter = CharacterTextSplitter(
+ chunk_size=500, chunk_overlap=10)
+ texts.extend(text_splitter.split_documents(documents))
case "txt":
loader = TextLoader("data/"+f)
documents = loader.load()
@@ -67,6 +94,9 @@ def load(f):
text_splitter = CharacterTextSplitter(
chunk_size=500, chunk_overlap=10)
texts.extend(text_splitter.split_documents(documents))
+ case _:
+ print(f, "not accepted")
+ os.remove("data/"+f)
db = FAISS.from_documents(texts, embeddings)
retriever = db.as_retriever()
diff --git a/audio-rag/endpoint/static/index.html b/audio-rag/endpoint/static/index.html
new file mode 100644
index 0000000..c293ed5
--- /dev/null
+++ b/audio-rag/endpoint/static/index.html
@@ -0,0 +1,34 @@
+
+
+
+
+
+ Ollama RAG with LLAMA 3
+
+
+
+
+ Files in RAG store
+ Chat
+
+
+
diff --git a/audio-rag/endpoint/static/main.js b/audio-rag/endpoint/static/main.js
new file mode 100644
index 0000000..ea6dae1
--- /dev/null
+++ b/audio-rag/endpoint/static/main.js
@@ -0,0 +1,74 @@
+document.querySelector("#send").onclick = async () => {
+ const prompt = document.querySelector("#q").value;
+ document.querySelector("#q").value = "";
+
+ document.querySelector("#spinner").classList.toggle("hidden");
+
+ const q_div = document.createElement("div");
+ q_div.className =
+ "m-4 mr-24 flex flex-col gap-2 items-start bg-emerald-400 p-4 rounded-md";
+ const user_div = document.createElement("div");
+ user_div.innerText = "User:";
+ const prompt_div = document.createElement("div");
+ prompt_div.innerText = prompt;
+ q_div.appendChild(user_div);
+ q_div.appendChild(prompt_div);
+ document.querySelector("#main").appendChild(q_div);
+
+ const req = await fetch("/api/query", {
+ body: JSON.stringify({
+ prompt,
+ }),
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ });
+ const res = await req.json();
+ console.log(res);
+ const r_div = document.createElement("div");
+ r_div.className =
+ "m-4 ml-24 flex flex-col gap-2 bg-emerald-300 p-4 rounded-md";
+ const a_div = document.createElement("div");
+ a_div.innerText = "RAG:";
+ const answer_div = document.createElement("div");
+ answer_div.innerText = res.answer;
+ const context_div = document.createElement("div");
+ res.context.forEach((elem) => {
+ const context_item = document.createElement("div");
+ context_item.className = "m-4 border-b-2";
+ const text = document.createElement("div");
+ text.innerText = elem.page_content;
+ context_item.appendChild(text);
+ const source = document.createElement("div");
+ source.innerText = elem.metadata.source;
+ source.className = "text-right font-semibold";
+ context_item.appendChild(source);
+ context_div.appendChild(context_item);
+ });
+ r_div.appendChild(a_div);
+ r_div.appendChild(answer_div);
+ r_div.appendChild(context_div);
+
+ document.querySelector("#main").appendChild(r_div);
+
+ document.querySelector("#spinner").classList.toggle("hidden");
+};
+const getFileType = (filename) => {
+ switch (filename.split('.').at(-1)) {
+ case 'txt':
+ return 'Text file';
+ case 'wav':
+ return 'WAV Sound recording';
+ case 'pdf':
+ return 'PDF document';
+ default:
+ return '<>';
+ }
+};
+
+fetch("/files").then(res=>res.json().then(json=>{
+ document.querySelector("#files").innerHTML = `Files in RAG store
` + json.map(file=>`
+ ${file}
${getFileType(file)}