From 0e13e6d2ace2d784579980521998da6de30024d4 Mon Sep 17 00:00:00 2001 From: tmeftah <46863341+tmeftah@users.noreply.github.com> Date: Fri, 30 Aug 2024 13:12:08 +0200 Subject: [PATCH] Feature 46/add model params (#57) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * oauth refactored and added to all routes * Added auth to query service * Initial auths added * Autorization moved to the request level as return objects are not needed for many functions * addedd params model_name * List all downloaded ollama models * feat ✨: Extend llm_utils module with OLLAMA model list - This commit extends the `llm_utils` module to provide a function for listing all available OLLAMA models. - Improved the query service and added model list functionality. * feat ✨: Improved user interface - Improved the user interface by adding a select dropdown to filter models. - Addthe ability to send a question along with the model name when pressing enter. * feat ✨: Enhanced data modeling and documentation - Implemented enhanced `pydantic` models for increased data modeling and clarity. - Implemented added status field and timestamps for Documents model objects. - The service file has been updated to incorporate document progress tracking and returns the validated DocumentPydantic model object for each document. - Improved document processing for better query response clarity and structure. * docs 📝:: Error handling for document retrieval updated. - Replaced error handling for document retrieval with a default empty list. * feat ✨: Consistent datetime format for timestamps - The codebase now uses a consistent datetime for both `created_at` and `updated_at`. * feat ✨: UI Document upload feature implementation - Implemented a document upload feature with a popup and modal. * feat ✨: UI Store Document management functionality added - Added document management functionality to the main store. * feat ✨: User authentication token clearing - Functionality for clearing the user's authentication token has been added. - The code implements a mechanism to handle unauthorized access attempts and redirect the user to the login page. * feat ✨: New response template implementation - The code implements a new response template for user queries based on the provided context. * feat ✨: Support for different document types - Added support for accepting different file types for document upload. * feat ✨: Improved vectorstore and AI assistant - The code updates the vectorstore initialization and embedding function to improve efficiency and accuracy. - Implemented an improved prompt template for AI assistant responses. * feat ✨: Monitoring system for document updates - Implemented a monitoring system to check for changes in the uploaded documents and update their status. * feat ✨: Update watchdog with new vector database - The code updates the `watchdog` to use a new vector database and update document status in the background. * fix 🐛: Simplified database configuration - Simplified database configuration for improved consistency and reduced complexity. * feat ✨: Use 'aora.db' for SQLite persistence - Changed database connection settings to use the 'aora.db' file for SQLite persistence. * feat ✨: Build and store vector embeddings for PDF documents - The code rewrites the logic to build and store vector embeddings of PDF documents into a persistent vector database. * feat ✨: Document saving and hashing method - The code defines a method to save documents into the designated directory and hash them for persistence in the database. * feat ✨: Implement environment variable loading - The code implements environment variables loading for the application, then sets up a context manager. --------- Co-authored-by: raikarn Co-authored-by: NikhilRaikar17 --- backend/api/document.py | 2 +- backend/db/sessions.py | 4 +- backend/embeddings/ingest.py | 24 ++-- backend/main.py | 4 + backend/models/pydantic_models.py | 14 +- backend/models/sqlalchemy_models.py | 6 + backend/service/document_service.py | 18 +-- backend/service/llm_utils.py | 8 +- backend/service/query_service.py | 11 +- backend/watchdog.py | 108 ++++++++++++++++ frontend/src/layouts/MainLayout.vue | 20 +++ frontend/src/pages/DocumentsPage.vue | 64 +++++---- frontend/src/pages/LoginPage.vue | 4 + frontend/src/pages/QueryPage.vue | 6 +- frontend/src/stores/auth.js | 5 + frontend/src/stores/main-store.js | 187 ++++++++++++++++++++++++++- 16 files changed, 420 insertions(+), 65 deletions(-) create mode 100644 backend/watchdog.py diff --git a/backend/api/document.py b/backend/api/document.py index b6d615d..b34fb19 100644 --- a/backend/api/document.py +++ b/backend/api/document.py @@ -59,7 +59,7 @@ def list_documents( return document_list(db) except NoDocumentsFoundException as e: - raise HTTPException(status_code=404, detail=str(e)) + return [] except Exception as e: raise HTTPException(status_code=500, detail=str(e)) diff --git a/backend/db/sessions.py b/backend/db/sessions.py index 5e86387..58fe186 100644 --- a/backend/db/sessions.py +++ b/backend/db/sessions.py @@ -5,8 +5,8 @@ from backend.models.sqlalchemy_models import Base -DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///aora.db") -engine = create_engine(DATABASE_URL) +DATABASE_URL = os.getenv("DATABASE_URL", "aora.db") +engine = create_engine("sqlite:///"+DATABASE_URL) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/backend/embeddings/ingest.py b/backend/embeddings/ingest.py index ed4ec2a..150a739 100644 --- a/backend/embeddings/ingest.py +++ b/backend/embeddings/ingest.py @@ -11,28 +11,22 @@ from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_chroma import Chroma import chromadb -from dotenv import load_dotenv -load_dotenv() +vectordatastore_directory = os.getenv("VECTORSTORE_DATABASE_PATH") +documenst_directory = os.getenv("DOCUMENTS_DIRECTORY") -# Get the path value from .env file -relative_path = os.getenv("DATABASE_PATH") -# Get directory of script -script_dir = os.path.dirname(os.path.abspath(__file__)) -# Append the relative path to the script directory -persist_directory = os.path.join(script_dir, relative_path) def create_vectorstore(): text_splitter = RecursiveCharacterTextSplitter( # Set a really small chunk size, just to show. - chunk_size=1300, - chunk_overlap=110, + chunk_size=1500, + chunk_overlap=120, length_function=len, ) documents = [] - for file in os.listdir("docs"): + for file in os.listdir(documenst_directory): if file.endswith(".pdf"): pdf_path = "./docs/" + file loader = PyPDFLoader(pdf_path) @@ -44,25 +38,25 @@ def create_vectorstore(): collection_name=os.environ.get("COLLECTION_NAME"), documents=documents, embedding=OllamaEmbeddings(model="mxbai-embed-large"), - persist_directory=persist_directory, + persist_directory=vectordatastore_directory, ) print("vectorstore created...") def get_vectorstore(): - persistent_client = chromadb.PersistentClient(path=persist_directory) + persistent_client = chromadb.PersistentClient(path=vectordatastore_directory) langchain_chroma = Chroma( client=persistent_client, collection_name=os.environ.get("COLLECTION_NAME"), embedding_function=OllamaEmbeddings(model="mxbai-embed-large"), + collection_metadata={"hnsw:space": "cosine"} ) print("There are", langchain_chroma._collection.count(), "in the collection") # print("There are", langchain_chroma.similarity_search("bmw?")) - return langchain_chroma.as_retriever( - search_type="mmr", search_kwargs={"k": 3, "lambda_mult": 0.25} + return langchain_chroma.as_retriever(search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.2,"k":3} ) diff --git a/backend/main.py b/backend/main.py index a80739e..3e54ce9 100644 --- a/backend/main.py +++ b/backend/main.py @@ -14,6 +14,10 @@ from backend.db.sessions import get_db from backend.db.utils import populate_admin_user +from dotenv import load_dotenv + +load_dotenv() + @asynccontextmanager async def lifespan(app: FastAPI): diff --git a/backend/models/pydantic_models.py b/backend/models/pydantic_models.py index 7985300..5381f48 100644 --- a/backend/models/pydantic_models.py +++ b/backend/models/pydantic_models.py @@ -1,6 +1,6 @@ from typing import Optional - -from pydantic import BaseModel +from datetime import date, datetime +from pydantic import BaseModel, ConfigDict class Token(BaseModel): @@ -18,5 +18,15 @@ class UserPydantic(BaseModel): class DocumentPydantic(BaseModel): + model_config = ConfigDict(from_attributes=True) + filename: str content_type: str + status:str + created_at:datetime + + + + + + diff --git a/backend/models/sqlalchemy_models.py b/backend/models/sqlalchemy_models.py index 43e0eeb..9ea95b2 100644 --- a/backend/models/sqlalchemy_models.py +++ b/backend/models/sqlalchemy_models.py @@ -1,6 +1,9 @@ + from sqlalchemy import Column from sqlalchemy import Integer from sqlalchemy import String +from sqlalchemy import DateTime +import datetime from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() @@ -28,3 +31,6 @@ class Documents(Base): filename = Column(String, unique=True, index=True) filehash = Column(String, unique=True) content_type = Column(String) + status = Column(String) + created_at = Column(DateTime, default= datetime.datetime.now) + updated_at = Column(DateTime, default= datetime.datetime.now) diff --git a/backend/service/document_service.py b/backend/service/document_service.py index 789acde..a9a2358 100644 --- a/backend/service/document_service.py +++ b/backend/service/document_service.py @@ -9,16 +9,18 @@ from backend.models.pydantic_models import DocumentPydantic from backend.models.sqlalchemy_models import Documents +documenst_directory = os.getenv("DOCUMENTS_DIRECTORY") + def save_document(file: File, db: Session) -> DocumentPydantic: """Gets the uploaded document, saves the document in the docs folder and creates a hash of the document and saves it in db""" - file_directory = "docs" - os.makedirs(file_directory, exist_ok=True) + #file_directory = "docs" + os.makedirs(documenst_directory, exist_ok=True) - file_location = f"docs/{file.filename}" + file_location = os.path.join(documenst_directory, file.filename) with open(file_location, "wb+") as file_object: file_object.write(file.file.read()) @@ -27,19 +29,19 @@ def save_document(file: File, db: Session) -> DocumentPydantic: new_document = Documents( filename=file.filename, filehash=file_hash, + status="on progress", content_type=file.content_type, ) db.add(new_document) db.commit() - return DocumentPydantic.model_validate( - {"filename": file.filename, "content_type": file.content_type} - ) + return DocumentPydantic.model_validate(new_document) def get_all_documents(db: Session) -> List[DocumentPydantic]: """Get all documents""" documents = db.query(Documents).all() + return documents if documents else None @@ -52,8 +54,8 @@ def document_list(db: Session) -> List[DocumentPydantic]: raise NoDocumentsFoundException() return [ - DocumentPydantic( - filename=doc.filename, content_type=str(doc.content_type) + DocumentPydantic.model_validate( + doc ) for doc in documents ] diff --git a/backend/service/llm_utils.py b/backend/service/llm_utils.py index 696b2b5..0a4c220 100644 --- a/backend/service/llm_utils.py +++ b/backend/service/llm_utils.py @@ -7,13 +7,11 @@ from backend.exceptions import ModelsNotRetrievedException -TEMPLATE = """<|begin_of_text|><|start_header_id|>system<|end_header_id|> -You are an AI assistant, you only answer questions on the folwing -context and nothing else. If you do not know the answer please strictly say -'see Documentation'<|eot_id|><|start_header_id|>user<|end_header_id|> +TEMPLATE = """You are an AI assistant and based on the context provided below, please provide an answer starting with 'Based on the given context'. Do not use external knowledge or make assumptions beyond the context +context and nothing else. If you do not know the answer please strictly say "I couldn't find the answer to that question. Please contact our support team for more assistance." Question: {input} Context: {context} -<|eot_id|><|start_header_id|>assistant<|end_header_id|>""" +""" def get_list_available_models(): diff --git a/backend/service/query_service.py b/backend/service/query_service.py index 01b1a6d..7d8f338 100644 --- a/backend/service/query_service.py +++ b/backend/service/query_service.py @@ -5,14 +5,10 @@ from fastapi.responses import StreamingResponse from backend.embeddings.ingest import get_vectorstore - from backend.exceptions import ModelsNotRetrievedException from backend.service.llm_utils import create_chain, get_list_available_models -from backend.rag_llms_langchain import chain - - async def query_service(query: str, model_name: str): """ @@ -24,10 +20,15 @@ async def query_service(query: str, model_name: str): chain = create_chain(model_name) print(20 * "*", "docs", 20 * "*", "\n", docs) + + context ="" + + for doc in docs: + context = context +"\n" + doc.page_content async def stream_generator(): print(20 * "*", "\n", query) - async for text in chain.astream({"input": query, "context": docs}): + async for text in chain.astream({"input": query, "context": context}): yield json.dumps({"event_id": str(uuid.uuid4()), "data": text}) # TODO here we have to add the metadata/source diff --git a/backend/watchdog.py b/backend/watchdog.py new file mode 100644 index 0000000..aafea46 --- /dev/null +++ b/backend/watchdog.py @@ -0,0 +1,108 @@ +import os +import sys +import time +import hashlib +from sqlalchemy import create_engine +from sqlalchemy.orm import declarative_base + + +from sqlalchemy.orm import sessionmaker +from datetime import datetime +from models.sqlalchemy_models import Documents + + +from langchain_community.document_loaders import PyPDFLoader +from langchain_community.embeddings import OllamaEmbeddings +from langchain_text_splitters import RecursiveCharacterTextSplitter +from langchain_chroma import Chroma + +from dotenv import load_dotenv + + +load_dotenv() + + +vectordatastore_directory = os.getenv("VECTORSTORE_DATABASE_PATH") +documenst_directory = os.getenv("DOCUMENTS_DIRECTORY") + +DATABASE_URL = os.getenv("DATABASE_URL", "aora.db") +engine = create_engine("sqlite:///"+DATABASE_URL) +Base = declarative_base() + + + + + +Base.metadata.create_all(engine) +Session = sessionmaker(bind=engine) +session = Session() + +def check_file_in_db(filename): + return session.query(Documents).filter_by(filename=filename).first() + +def update_file_status(filename): + file = check_file_in_db(filename) + # file_hash = hashlib.sha256(open(os.path.join(documenst_directory, filename), "rb").read()).hexdigest() + + if file and file.status != "done": + + file.status = "uploaded" + file.updated_at = datetime.now() + session.commit() + create_vectorstore(filename) + + file.status = "done" + file.updated_at = datetime.now() + session.commit() + + +def create_vectorstore(filename): + + text_splitter = RecursiveCharacterTextSplitter( + # Set a really small chunk size, just to show. + chunk_size=1300, + chunk_overlap=110, + length_function=len, + ) + + + loader = PyPDFLoader(os.path.join(documenst_directory, filename)) + doc = loader.load() + document_split = text_splitter.split_documents(doc) + + Chroma.from_documents( + collection_name=os.environ.get("COLLECTION_NAME"), + documents=document_split, + embedding=OllamaEmbeddings(model="mxbai-embed-large"), + persist_directory=vectordatastore_directory, + collection_metadata={"hnsw:space": "cosine"} + ) + + print("vectorstore created...") + +def monitor_directory(directory): + previous_files = set() + while True: + try: + current_files = set(os.listdir(directory)) + print("current_files:", current_files) + new_files = current_files - previous_files + for filename in new_files: + file_path = os.path.join(directory, filename) + if os.path.isfile(file_path): + update_file_status(filename) + previous_files = current_files + print(50* "*") + time.sleep(3) + except KeyboardInterrupt: + print('Stopping script...') + session.close() + sys.exit(0) + + +def main(): + + monitor_directory(documenst_directory) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/frontend/src/layouts/MainLayout.vue b/frontend/src/layouts/MainLayout.vue index be698cf..552284e 100644 --- a/frontend/src/layouts/MainLayout.vue +++ b/frontend/src/layouts/MainLayout.vue @@ -23,6 +23,21 @@ v-bind="link" /> + + +
+ +
@@ -33,7 +48,9 @@ diff --git a/frontend/src/pages/LoginPage.vue b/frontend/src/pages/LoginPage.vue index 7bbdfed..b7c6e81 100644 --- a/frontend/src/pages/LoginPage.vue +++ b/frontend/src/pages/LoginPage.vue @@ -112,12 +112,15 @@ import { ref, reactive } from "vue"; import { useRouter } from "vue-router"; import { useAuthStore } from "../stores/auth"; +import { useMainStore } from "src/stores/main-store"; const isPwd = ref(true); const authStore = useAuthStore(); const router = useRouter(); +const mainStore = useMainStore(); + const user = reactive({ email: null, password: null, @@ -133,6 +136,7 @@ async function submit() { authStore.getCurrentUser().then(() => { if (authStore.user) { router.push("/"); + mainStore.get_models(); } }); }) diff --git a/frontend/src/pages/QueryPage.vue b/frontend/src/pages/QueryPage.vue index 21ce1a5..d91fe6b 100644 --- a/frontend/src/pages/QueryPage.vue +++ b/frontend/src/pages/QueryPage.vue @@ -3,7 +3,7 @@
@@ -57,5 +57,5 @@ defineOptions({ }); const MainStore = useMainStore(); -const { loading, solution, question } = storeToRefs(MainStore); +const { loading, solution, question, model_name } = storeToRefs(MainStore); diff --git a/frontend/src/stores/auth.js b/frontend/src/stores/auth.js index 1149455..cd23523 100644 --- a/frontend/src/stores/auth.js +++ b/frontend/src/stores/auth.js @@ -17,6 +17,11 @@ export const useAuthStore = defineStore("auth", { }, actions: { + clearToken() { + this.token = null; + localStorage.removeItem("token"); // Remove token from storage + }, + async login(email, password) { const formData = new URLSearchParams(); formData.append("grant_type", "password"); diff --git a/frontend/src/stores/main-store.js b/frontend/src/stores/main-store.js index 21eafba..3a1f3da 100644 --- a/frontend/src/stores/main-store.js +++ b/frontend/src/stores/main-store.js @@ -1,4 +1,8 @@ import { defineStore } from "pinia"; +import { Notify } from "quasar"; +import { useAuthStore } from "stores/auth"; + +const authStore = useAuthStore(); const baseUrl = `${process.env.API}`; @@ -7,14 +11,85 @@ export const useMainStore = defineStore("main", { loading: false, question: "", solution: "", + models: JSON.parse(localStorage.getItem("models")) || [], + model_name: localStorage.getItem("model_name") || "", + documents: [], + show_uploader: false, }), getters: {}, actions: { - async sendQA(question) { + async get_models() { + try { + const response = await fetch(`${baseUrl}/query/list_models`, { + method: "GET", + headers: { + Accept: "application/json", + Authorization: `Bearer ${authStore.token}`, + }, + }); + + if (response.status === 401) { + authStore.clearToken(); + window.location.href = "/login"; // Or use Vue Router + return; + } + + if (!response.ok) { + // Extract error message if available + const errorData = await response.json().catch(() => ({})); // Parsing might fail, default to empty object + const errorMessage = + errorData.message || `Error: ${response.statusText}`; + throw new Error(errorMessage); + } + + const data = await response.json(); + this.models = data; + localStorage.setItem("models", JSON.stringify(this.models)); + + // + } catch (error) { + // Handle network errors and HTTP errors + if (error.name === "TypeError") { + // This typically indicates a network error + console.error("Network error: Could not reach the server"); + Notify.create({ + color: "negative", + position: "bottom", + message: error.message, + icon: "report_problem", + }); + } else { + // HTTP error, or some other error + console.error(`API error: ${error.message}`); + Notify.create({ + color: "negative", + position: "bottom", + message: error.message, + icon: "report_problem", + }); + } + + // You can rethrow the error or handle it in some way, e.g., user notification + } + }, + + set_model_name(name) { + localStorage.setItem("model_name", name); + }, + + async sendQA(question, model_name) { this.solution = ""; this.loading = true; - await fetch(`${baseUrl}/query?query=${question}`) + await fetch( + `${baseUrl}/query?query=${question}&model_name=${model_name}`, + { + headers: { + Accept: "application/json", + Authorization: `Bearer ${authStore.token}`, + }, + } + ) .then((response) => { console.log(response.body); if (!response.ok) { @@ -46,11 +121,119 @@ export const useMainStore = defineStore("main", { readChunk(reader, this.solution); }) .catch((error) => { + // + if (error.status === 401) { + authStore.clearToken(); + window.location.href = "/login"; // Or use Vue Router + return; + } + + // this.loading = false; console.error("Error fetching the data:", error); }); // }, + + async get_documents_list() { + try { + const response = await fetch(`${baseUrl}/documents`, { + method: "GET", + headers: { + Accept: "application/json", + Authorization: `Bearer ${authStore.token}`, + }, + }); + + if (response.status === 401) { + authStore.clearToken(); + window.location.href = "/login"; // Or use Vue Router + return; + } + + if (!response.ok) { + // Extract error message if available + const errorData = await response.json().catch(() => ({})); // Parsing might fail, default to empty object + const errorMessage = + errorData.message || `Error: ${response.statusText}`; + throw new Error(errorMessage); + } + + const data = await response.json(); + this.documents = data; + + // + } catch (error) { + // Handle network errors and HTTP errors + if (error.name === "TypeError") { + // This typically indicates a network error + console.error("Network error: Could not reach the server"); + Notify.create({ + color: "negative", + position: "bottom", + message: error.message, + icon: "report_problem", + }); + } else { + // HTTP error, or some other error + console.error(`API error: ${error.message}`); + Notify.create({ + color: "negative", + position: "bottom", + message: error.message, + icon: "report_problem", + }); + } + + // You can rethrow the error or handle it in some way, e.g., user notification + } + }, + + async upload_documents() { + // returning a Promise + + return new Promise((resolve) => { + // simulating a delay of 2 seconds + setTimeout(() => { + resolve({ + url: `${baseUrl}/documents/upload`, + method: "POST", + headers: [ + { + name: "Authorization", + value: `Bearer ${authStore.token}`, + }, + ], + }); + }, 2000); + }); + }, + + async uploaded_success() { + Notify.create({ + color: "positive", + position: "bottom", + message: "uploaded", + icon: "done", + }); + + setTimeout(() => { + this.show_uploader = false; + }, 2000); + + this.get_documents_list(); + }, + + async upload_failed() { + // FIXME: chech if token is not valid anymore + Notify.create({ + color: "negative", + position: "bottom", + message: "could not be uploaded", + icon: "report_problem", + }); + this.show_uploader = true; + }, }, });