Feature 46/add model params (#57)

* oauth refactored and added to all routes

* Added auth to query service

* Initial auths added

* Autorization moved to the request level as return objects are not needed for many functions

* addedd params model_name

* List all downloaded ollama models

* feat ✨: Extend llm_utils module with OLLAMA model list

- This commit extends the `llm_utils` module to provide a function for listing all available OLLAMA models.

- Improved the query service and added model list functionality.

* feat ✨: Improved user interface

- Improved the user interface by adding a select dropdown to filter models.

- Addthe ability to send a question along with the model name when pressing enter.

* feat ✨: Enhanced data modeling and documentation

- Implemented enhanced `pydantic` models for increased data modeling and clarity.

- Implemented added status field and timestamps for Documents model objects.

- The service file has been updated to incorporate document progress tracking and returns the validated DocumentPydantic model object for each document.

- Improved document processing for better query response clarity and structure.

* docs 📝:: Error handling for document retrieval updated.

- Replaced error handling for document retrieval with a default empty list.

* feat ✨: Consistent datetime format for timestamps

- The codebase now uses a consistent datetime for both `created_at` and `updated_at`.

* feat ✨: UI Document upload feature implementation

- Implemented a document upload feature with a popup and modal.

* feat ✨: UI Store Document management functionality added

- Added document management functionality to the main store.

* feat ✨: User authentication token clearing

- Functionality for clearing the user's authentication token has been added.

- The code implements a mechanism to handle unauthorized access attempts and redirect the user to the login page.

* feat ✨: New response template implementation

- The code implements a new response template for user queries based on the provided context.

* feat ✨: Support for different document types

- Added support for accepting different file types for document upload.

* feat ✨: Improved vectorstore and AI assistant

- The code updates the vectorstore initialization and embedding function to improve efficiency and accuracy.

- Implemented an improved prompt template for AI assistant responses.

* feat ✨: Monitoring system for document updates

- Implemented a monitoring system to check for changes in the uploaded documents and update their status.

* feat ✨: Update watchdog with new vector database

- The code updates the `watchdog` to use a new vector database and update document status in the background.

* fix 🐛: Simplified database configuration

- Simplified database configuration for improved consistency and reduced complexity.

* feat ✨: Use 'aora.db' for SQLite persistence

- Changed database connection settings to use the 'aora.db' file for SQLite persistence.

* feat ✨: Build and store vector embeddings for PDF documents

- The code rewrites the logic to build and store vector embeddings of PDF documents into a persistent vector database.

* feat ✨: Document saving and hashing method

- The code defines a method to save documents into the designated directory and hash them for persistence in the database.

* feat ✨: Implement environment variable loading

- The code implements environment variables loading for the application, then sets up a context manager.

---------

Co-authored-by: raikarn <nikhil.raikar@apl-landau.de>
Co-authored-by: NikhilRaikar17 <nikhilraikar88@gmail.com>

backend/api/document.py

@@ -59,7 +59,7 @@ def list_documents(
         return document_list(db)
 
     except NoDocumentsFoundException as e:
-        raise HTTPException(status_code=404, detail=str(e))
+        return []
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
 

backend/db/sessions.py

@@ -5,8 +5,8 @@
 
 from backend.models.sqlalchemy_models import Base
 
-DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///aora.db")
-engine = create_engine(DATABASE_URL)
+DATABASE_URL = os.getenv("DATABASE_URL", "aora.db")
+engine = create_engine("sqlite:///"+DATABASE_URL)
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 

backend/embeddings/ingest.py

@@ -11,28 +11,22 @@
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_chroma import Chroma
 import chromadb
-from dotenv import load_dotenv
 
-load_dotenv()
+vectordatastore_directory = os.getenv("VECTORSTORE_DATABASE_PATH")
+documenst_directory = os.getenv("DOCUMENTS_DIRECTORY")
 
-# Get the path value from .env file
-relative_path = os.getenv("DATABASE_PATH")
-# Get directory of script
-script_dir = os.path.dirname(os.path.abspath(__file__))
-# Append the relative path to the script directory
-persist_directory = os.path.join(script_dir, relative_path)
 
 
 def create_vectorstore():
     text_splitter = RecursiveCharacterTextSplitter(
         # Set a really small chunk size, just to show.
-        chunk_size=1300,
-        chunk_overlap=110,
+        chunk_size=1500,
+        chunk_overlap=120,
         length_function=len,
     )
 
     documents = []
-    for file in os.listdir("docs"):
+    for file in os.listdir(documenst_directory):
         if file.endswith(".pdf"):
             pdf_path = "./docs/" + file
             loader = PyPDFLoader(pdf_path)
@@ -44,25 +38,25 @@ def create_vectorstore():
         collection_name=os.environ.get("COLLECTION_NAME"),
         documents=documents,
         embedding=OllamaEmbeddings(model="mxbai-embed-large"),
-        persist_directory=persist_directory,
+        persist_directory=vectordatastore_directory,
     )
 
     print("vectorstore created...")
 
 
 def get_vectorstore():
-    persistent_client = chromadb.PersistentClient(path=persist_directory)
+    persistent_client = chromadb.PersistentClient(path=vectordatastore_directory)
 
     langchain_chroma = Chroma(
         client=persistent_client,
         collection_name=os.environ.get("COLLECTION_NAME"),
         embedding_function=OllamaEmbeddings(model="mxbai-embed-large"),
+        collection_metadata={"hnsw:space": "cosine"}
     )
     print("There are", langchain_chroma._collection.count(), "in the collection")
 
     # print("There are", langchain_chroma.similarity_search("bmw?"))
-    return langchain_chroma.as_retriever(
-        search_type="mmr", search_kwargs={"k": 3, "lambda_mult": 0.25}
+    return langchain_chroma.as_retriever(search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.2,"k":3}
     )
 
 

backend/main.py

@@ -14,6 +14,10 @@
 from backend.db.sessions import get_db
 from backend.db.utils import populate_admin_user
 
+from dotenv import load_dotenv
+
+load_dotenv()
+
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):

backend/models/pydantic_models.py

@@ -1,6 +1,6 @@
 from typing import Optional
-
-from pydantic import BaseModel
+from datetime import date, datetime
+from pydantic import BaseModel, ConfigDict
 
 
 class Token(BaseModel):
@@ -18,5 +18,15 @@ class UserPydantic(BaseModel):
 
 
 class DocumentPydantic(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
     filename: str
     content_type: str
+    status:str
+    created_at:datetime
+
+
+
+
+
+

backend/models/sqlalchemy_models.py

@@ -1,6 +1,9 @@
+
 from sqlalchemy import Column
 from sqlalchemy import Integer
 from sqlalchemy import String
+from sqlalchemy import DateTime
+import datetime
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
@@ -28,3 +31,6 @@ class Documents(Base):
     filename = Column(String, unique=True, index=True)
     filehash = Column(String, unique=True)
     content_type = Column(String)
+    status = Column(String)
+    created_at = Column(DateTime, default= datetime.datetime.now)
+    updated_at = Column(DateTime, default= datetime.datetime.now)

backend/service/document_service.py

@@ -9,16 +9,18 @@
 from backend.models.pydantic_models import DocumentPydantic
 from backend.models.sqlalchemy_models import Documents
 
+documenst_directory = os.getenv("DOCUMENTS_DIRECTORY")
+
 
 def save_document(file: File, db: Session) -> DocumentPydantic:
     """Gets the uploaded document, saves the document
     in the docs folder and creates a hash of the document
     and saves it in db"""
 
-    file_directory = "docs"
-    os.makedirs(file_directory, exist_ok=True)
+    #file_directory = "docs"
+    os.makedirs(documenst_directory, exist_ok=True)
 
-    file_location = f"docs/{file.filename}"
+    file_location = os.path.join(documenst_directory, file.filename) 
     with open(file_location, "wb+") as file_object:
         file_object.write(file.file.read())
 
@@ -27,19 +29,19 @@ def save_document(file: File, db: Session) -> DocumentPydantic:
     new_document = Documents(
         filename=file.filename,
         filehash=file_hash,
+        status="on progress",
         content_type=file.content_type,
     )
     db.add(new_document)
     db.commit()
 
-    return DocumentPydantic.model_validate(
-        {"filename": file.filename, "content_type": file.content_type}
-    )
+    return DocumentPydantic.model_validate(new_document)
 
 
 def get_all_documents(db: Session) -> List[DocumentPydantic]:
     """Get all documents"""
     documents = db.query(Documents).all()
+
     return documents if documents else None
 
 
@@ -52,8 +54,8 @@ def document_list(db: Session) -> List[DocumentPydantic]:
         raise NoDocumentsFoundException()
 
     return [
-        DocumentPydantic(
-            filename=doc.filename, content_type=str(doc.content_type)
+        DocumentPydantic.model_validate(
+            doc
         )
         for doc in documents
     ]

backend/service/llm_utils.py

@@ -7,13 +7,11 @@
 
 from backend.exceptions import ModelsNotRetrievedException
 
-TEMPLATE = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
-You are an AI assistant, you only answer questions on the folwing
-context and nothing else. If you do not know the answer please strictly say
-'see Documentation'<|eot_id|><|start_header_id|>user<|end_header_id|>
+TEMPLATE = """You are an AI assistant and based on the context provided below, please provide an answer starting with 'Based on the given context'. Do not use external knowledge or make assumptions beyond the context
+context and nothing else. If you do not know the answer please strictly say "I couldn't find the answer to that question. Please contact our support team for more assistance."
 Question: {input}
 Context: {context}
-<|eot_id|><|start_header_id|>assistant<|end_header_id|>"""
+"""
 
 
 def get_list_available_models():

backend/service/query_service.py

@@ -5,14 +5,10 @@
 from fastapi.responses import StreamingResponse
 
 from backend.embeddings.ingest import get_vectorstore
-
 from backend.exceptions import ModelsNotRetrievedException
 from backend.service.llm_utils import create_chain, get_list_available_models
 
 
-from backend.rag_llms_langchain import chain
-
-
 
 async def query_service(query: str, model_name: str):
     """
@@ -24,10 +20,15 @@ async def query_service(query: str, model_name: str):
     chain = create_chain(model_name)
 
     print(20 * "*", "docs", 20 * "*", "\n", docs)
+
+    context =""
+
+    for doc in docs:
+        context = context +"\n" + doc.page_content
 
     async def stream_generator():
         print(20 * "*", "\n", query)
-        async for text in chain.astream({"input": query, "context": docs}):
+        async for text in chain.astream({"input": query, "context": context}):
             yield json.dumps({"event_id": str(uuid.uuid4()), "data": text})
 
         # TODO  here we have to add the metadata/source

backend/watchdog.py

@@ -0,0 +1,108 @@
+import os
+import sys
+import time
+import hashlib
+from sqlalchemy import create_engine
+from sqlalchemy.orm import declarative_base
+
+
+from sqlalchemy.orm import sessionmaker
+from datetime import datetime
+from models.sqlalchemy_models import Documents
+
+
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_chroma import Chroma
+
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+
+vectordatastore_directory = os.getenv("VECTORSTORE_DATABASE_PATH")
+documenst_directory = os.getenv("DOCUMENTS_DIRECTORY")
+
+DATABASE_URL = os.getenv("DATABASE_URL", "aora.db")
+engine = create_engine("sqlite:///"+DATABASE_URL)
+Base = declarative_base()
+
+
+
+
+
+Base.metadata.create_all(engine)
+Session = sessionmaker(bind=engine)
+session = Session()
+
+def check_file_in_db(filename):
+    return session.query(Documents).filter_by(filename=filename).first()
+
+def update_file_status(filename):
+    file = check_file_in_db(filename)
+    # file_hash = hashlib.sha256(open(os.path.join(documenst_directory, filename), "rb").read()).hexdigest()
+
+    if file and file.status != "done":
+
+        file.status = "uploaded"
+        file.updated_at = datetime.now()
+        session.commit()        
+        create_vectorstore(filename)
+
+        file.status = "done"
+        file.updated_at = datetime.now()
+        session.commit()   
+
+
+def create_vectorstore(filename):
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        # Set a really small chunk size, just to show.
+        chunk_size=1300,
+        chunk_overlap=110,
+        length_function=len,
+    )
+
+
+    loader = PyPDFLoader(os.path.join(documenst_directory, filename))
+    doc = loader.load()
+    document_split = text_splitter.split_documents(doc)
+
+    Chroma.from_documents(
+        collection_name=os.environ.get("COLLECTION_NAME"),
+        documents=document_split,
+        embedding=OllamaEmbeddings(model="mxbai-embed-large"),
+        persist_directory=vectordatastore_directory,
+        collection_metadata={"hnsw:space": "cosine"}
+    )
+
+    print("vectorstore created...")
+
+def monitor_directory(directory):
+    previous_files = set()
+    while True:
+        try:
+            current_files = set(os.listdir(directory))
+            print("current_files:", current_files)
+            new_files = current_files - previous_files
+            for filename in new_files:
+                file_path = os.path.join(directory, filename)
+                if os.path.isfile(file_path):
+                    update_file_status(filename)
+            previous_files = current_files
+            print(50* "*")
+            time.sleep(3)
+        except KeyboardInterrupt:
+            print('Stopping script...')
+            session.close()
+            sys.exit(0)
+
+
+def main():
+
+    monitor_directory(documenst_directory)
+
+if __name__ == '__main__':
+    main()

frontend/src/layouts/MainLayout.vue

@@ -23,6 +23,21 @@
           v-bind="link"
         />
       </q-list>
+
+      <q-separator />
+      <div class="q-ma-sm fixed-bottom">
+        <q-select
+          transition-show="flip-up"
+          transition-hide="flip-down"
+          dense
+          options-dense
+          outlined
+          v-model="model_name"
+          :options="models"
+          label="Model"
+          @update:model-value="(val) => mainStore.set_model_name(val)"
+        />
+      </div>
     </q-drawer>
 
     <q-page-container>
@@ -33,7 +48,9 @@
 
 <script setup>
 import { ref } from "vue";
+import { storeToRefs } from "pinia";
 import { useAuthStore } from "../stores/auth";
+import { useMainStore } from "src/stores/main-store";
 import { useRouter } from "vue-router";
 
 import EssentialLink from "components/EssentialLink.vue";
@@ -68,6 +85,9 @@ const miniState = ref(true);
 const authStore = useAuthStore();
 const router = useRouter();
 
+const mainStore = useMainStore();
+const { model_name, models } = storeToRefs(mainStore);
+
 function toggleLeftDrawer() {
   leftDrawerOpen.value = !leftDrawerOpen.value;
 }