From f3415d397a24ab5f1c70d52baf6387810dfba5c1 Mon Sep 17 00:00:00 2001 From: NikhilRaikar17 Date: Sun, 4 Aug 2024 19:34:54 +0200 Subject: [PATCH 1/4] Refactoring according to pep8 and fixed imports --- .github/workflows/ci.yml | 14 +++++------ backend/.gitignore | 2 +- backend/.pre-commit-config.yaml | 42 +++++++++++++++++++++++++++++++++ backend/api/health.py | 1 - backend/api/query.py | 10 ++++---- backend/api/token.py | 3 +-- backend/api/user.py | 42 +++++++++++++++++++++++---------- backend/config.py | 4 +--- backend/embeddings/ingest.py | 29 ++++++++++++----------- backend/main.py | 4 ++-- backend/oauth.py | 10 +++++--- backend/pydantic_models.py | 3 ++- backend/sessions.py | 2 +- backend/sqlalchemy_models.py | 3 ++- backend/utils.py | 8 +++---- 15 files changed, 120 insertions(+), 57 deletions(-) create mode 100644 backend/.pre-commit-config.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee073b1..e7d5148 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,20 @@ name: FastAPI CI -on: [push, pull_request] +on: [push, pull_request] jobs: test: - runs-on: ubuntu-latest + runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v4 with: python-version: '3.10' - cache: 'pip' + cache: 'pip' - name: Install dependencies run: | @@ -25,8 +25,8 @@ jobs: run: | cd backend nohup fastapi dev > /dev/null 2>&1 & - echo $! > uvicorn.pid - sleep 10 + echo $! > uvicorn.pid + sleep 10 - name: Check FastAPI /health endpoint run: | diff --git a/backend/.gitignore b/backend/.gitignore index 00f0076..abdb69c 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -162,4 +162,4 @@ cython_debug/ #.idea/ *.db *docs -chroma_db \ No newline at end of file +chroma_db diff --git a/backend/.pre-commit-config.yaml b/backend/.pre-commit-config.yaml new file mode 100644 index 0000000..0d260a3 --- /dev/null +++ b/backend/.pre-commit-config.yaml @@ -0,0 +1,42 @@ +# .pre-commit-config.yaml + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 # Replace with the latest version if needed + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: check-merge-conflict + + - repo: https://github.com/psf/black + rev: 23.1.0 # Replace with the latest version if needed + hooks: + - id: black + language_version: python3 # Adjust if you use a specific Python version + + - repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + + + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.4.1 # Replace with the latest version if needed + hooks: + - id: mypy + args: [--ignore-missing-imports, --disallow-untyped-calls, --disallow-untyped-defs] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 # Replace with the latest version if needed + hooks: + - id: check-json + - id: check-xml + + - repo: https://github.com/pre-commit/mirrors-pylint + rev: v3.0.0a5 # Replace with the latest version if needed + hooks: + - id: pylint + args: ["--max-line-length=88"] diff --git a/backend/api/health.py b/backend/api/health.py index 6c7982a..e51e4dd 100644 --- a/backend/api/health.py +++ b/backend/api/health.py @@ -1,4 +1,3 @@ - from fastapi import APIRouter health_router = APIRouter() diff --git a/backend/api/query.py b/backend/api/query.py index 58e1136..640090e 100644 --- a/backend/api/query.py +++ b/backend/api/query.py @@ -14,18 +14,20 @@ @query_router.get("/query") async def query(query: str): # if current_user.role < 5: - # raise HTTPException(status_code=403, detail="Only admin users can delete other users") + # raise HTTPException(status_code=403, + # detail="Only admin users can delete other users") store = get_vectorstore() docs = store.invoke(query) - print(20*"*", "docs", 20*"*", "\n", docs) + print(20 * "*", "docs", 20 * "*", "\n", docs) async def stream_generator(): # Use the LangChain model to generate text - print(20*'*', "\n", query) + print(20 * "*", "\n", query) async for text in chain.astream({"input": query, "context": docs}): yield json.dumps({"event_id": str(uuid.uuid4()), "data": text}) # TODO here we have to add the metadata/source - return StreamingResponse(stream_generator(), media_type="application/x-ndjson") \ No newline at end of file + return StreamingResponse(stream_generator(), + media_type="application/x-ndjson") diff --git a/backend/api/token.py b/backend/api/token.py index 3b04164..fb07650 100644 --- a/backend/api/token.py +++ b/backend/api/token.py @@ -1,4 +1,3 @@ - from fastapi import APIRouter, Depends, HTTPException from backend.pydantic_models import Token from backend.oauth import authenticate_user, create_access_token @@ -23,4 +22,4 @@ async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends( access_token = create_access_token( data={"sub": user.username}, expires_delta=access_token_expires ) - return {"access_token": access_token, "token_type": "bearer"} \ No newline at end of file + return {"access_token": access_token, "token_type": "bearer"} diff --git a/backend/api/user.py b/backend/api/user.py index 74d9e5a..5f74aea 100644 --- a/backend/api/user.py +++ b/backend/api/user.py @@ -1,4 +1,3 @@ - from fastapi import APIRouter, Depends, HTTPException from backend.sqlalchemy_models import User from backend.sessions import session @@ -8,7 +7,6 @@ user_router = APIRouter() - @user_router.get("/users/me") async def read_users_me(current_user: User = Depends(get_current_user)): return {"username": current_user.username, "role": current_user.role} @@ -18,15 +16,20 @@ async def read_users_me(current_user: User = Depends(get_current_user)): async def read_users(current_user: User = Depends(get_current_user)): if current_user.role < 5: raise HTTPException( - status_code=403, detail="Only admin users can view all users") - return [{"username": user.username, "role": user.role} for user in session.query(User).all()] + status_code=403, detail="Only admin users can view all users" + ) + return [ + {"username": user.username, "role": user.role} + for user in session.query(User).all() + ] @user_router.get("/users/{user_id}") async def read_user(user_id: int, current_user: User = Depends(get_current_user)): if current_user.id != user_id and current_user.role < 5: raise HTTPException( - status_code=403, detail="Only admin users can view other users") + status_code=403, detail="Only admin users can view other users" + ) user = session.query(User).filter(User.id == user_id).first() if not user: raise HTTPException(status_code=404, detail="User not found") @@ -34,22 +37,34 @@ async def read_user(user_id: int, current_user: User = Depends(get_current_user) @user_router.post("/users/") -async def create_user(username: str, password: str, role: int, current_user: User = Depends(get_current_user)): +async def create_user( + username: str, + password: str, + role: int, + current_user: User = Depends(get_current_user), +): if current_user.role < 5: raise HTTPException( - status_code=403, detail="Only admin users can create new users") - user = User(username=username, - password_hash=encrypt_password(password), role=role) + status_code=403, detail="Only admin users can create new users" + ) + user = User(username=username, password_hash=encrypt_password(password), role=role) session.add(user) session.commit() return {"username": user.username, "role": user.role} @user_router.put("/users/{user_id}") -async def update_user(user_id: int, username: str, password: str, role: int, current_user: User = Depends(get_current_user)): +async def update_user( + user_id: int, + username: str, + password: str, + role: int, + current_user: User = Depends(get_current_user), +): if current_user.id != user_id and current_user.role < 5: raise HTTPException( - status_code=403, detail="Only admin users can update other users") + status_code=403, detail="Only admin users can update other users" + ) user = session.query(User).filter(User.id == user_id).first() if not user: raise HTTPException(status_code=404, detail="User not found") @@ -64,10 +79,11 @@ async def update_user(user_id: int, username: str, password: str, role: int, cur async def delete_user(user_id: int, current_user: User = Depends(get_current_user)): if current_user.id != user_id and current_user.role < 5: raise HTTPException( - status_code=403, detail="Only admin users can delete other users") + status_code=403, detail="Only admin users can delete other users" + ) user = session.query(User).filter(User.id == user_id).first() if not user: raise HTTPException(status_code=404, detail="User not found") session.delete(user) session.commit() - return {"message": "User deleted"} \ No newline at end of file + return {"message": "User deleted"} diff --git a/backend/config.py b/backend/config.py index 41dca8d..701e25b 100644 --- a/backend/config.py +++ b/backend/config.py @@ -3,11 +3,9 @@ ACCESS_TOKEN_EXPIRE_MINUTES = 30 # No origins are limited for now -# All origins are allowed for now and this +# All origins are allowed for now and this # must be changed origins = [ "http://localhost", "http://127.0.0.1:8000", ] - - diff --git a/backend/embeddings/ingest.py b/backend/embeddings/ingest.py index c1d6e67..e80d449 100644 --- a/backend/embeddings/ingest.py +++ b/backend/embeddings/ingest.py @@ -1,14 +1,15 @@ # LangChain supports many other chat models. Here, we're using Ollama -from langchain_community.chat_models import ChatOllama -from langchain_core.output_parsers import StrOutputParser -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.runnables import RunnablePassthrough +# from langchain_community.chat_models import ChatOllama +# from langchain_core.output_parsers import StrOutputParser +# from langchain_core.prompts import ChatPromptTemplate +# from langchain_core.runnables import RunnablePassthrough +# from langchain_core.documents import Document + from langchain_community.document_loaders import PyPDFLoader from langchain_text_splitters import RecursiveCharacterTextSplitter from langchain_chroma import Chroma from langchain_community.embeddings import OllamaEmbeddings -from langchain_core.documents import Document import os import argparse @@ -23,13 +24,13 @@ def create_vectorstore(): # Set a really small chunk size, just to show. chunk_size=1300, chunk_overlap=110, - length_function=len + length_function=len, ) documents = [] - for file in os.listdir('docs'): - if file.endswith('.pdf'): - pdf_path = './docs/' + file + for file in os.listdir("docs"): + if file.endswith(".pdf"): + pdf_path = "./docs/" + file loader = PyPDFLoader(pdf_path) doc = loader.load() document_split = text_splitter.split_documents(doc) @@ -39,15 +40,14 @@ def create_vectorstore(): collection_name="kardex", documents=documents, embedding=OllamaEmbeddings(model="mxbai-embed-large"), - persist_directory="./vectorstore/chroma_db" + persist_directory="./vectorstore/chroma_db", ) print("vectorstore created...") def get_vectorstore(): - persistent_client = chromadb.PersistentClient( - path="./vectorstore/chroma_db") + persistent_client = chromadb.PersistentClient(path="./vectorstore/chroma_db") langchain_chroma = Chroma( client=persistent_client, collection_name="kardex", @@ -55,8 +55,9 @@ def get_vectorstore(): ) # print("There are", langchain_chroma._collection.count(), "in the collection") # print("There are", langchain_chroma.similarity_search("bmw?")) - return langchain_chroma.as_retriever(search_type="mmr", - search_kwargs={'k': 3, 'lambda_mult': 0.25}) + return langchain_chroma.as_retriever( + search_type="mmr", search_kwargs={"k": 3, "lambda_mult": 0.25} + ) if __name__ == "__main__": diff --git a/backend/main.py b/backend/main.py index 971f3aa..0a0abe5 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,7 +1,7 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from backend.api.health import health_router +from backend.api.health import health_router from backend.api.user import user_router from backend.api.token import token_router from backend.api.query import query_router @@ -12,7 +12,7 @@ app.add_middleware( CORSMiddleware, - allow_origins=['*'], + allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], diff --git a/backend/oauth.py b/backend/oauth.py index 68b8eca..96b3875 100644 --- a/backend/oauth.py +++ b/backend/oauth.py @@ -14,13 +14,15 @@ oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token") -def encrypt_password(password: str): + +def encrypt_password(password: str) -> bytes: return hashpw(password.encode(), gensalt()) -def verify_password(plain_password: str, hashed_password: str): +def verify_password(plain_password: str, hashed_password: str) -> bytes: return checkpw(plain_password.encode(), hashed_password) + def get_user(username: str): return session.query(User).filter(User.username == username).first() @@ -31,6 +33,7 @@ def authenticate_user(username: str, password: str): return False return user + async def get_current_user(token: str = Depends(oauth2_scheme)): credentials_exception = HTTPException( status_code=401, @@ -52,6 +55,7 @@ async def get_current_user(token: str = Depends(oauth2_scheme)): raise credentials_exception return user + def create_access_token(data: dict, expires_delta: Optional[timedelta] = None): to_encode = data.copy() if expires_delta: @@ -60,4 +64,4 @@ def create_access_token(data: dict, expires_delta: Optional[timedelta] = None): expire = datetime.utcnow() + timedelta(minutes=15) to_encode.update({"exp": expire}) encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) - return encoded_jwt \ No newline at end of file + return encoded_jwt diff --git a/backend/pydantic_models.py b/backend/pydantic_models.py index 428270a..22d8fc1 100644 --- a/backend/pydantic_models.py +++ b/backend/pydantic_models.py @@ -1,10 +1,11 @@ from pydantic import BaseModel from typing import Optional + class Token(BaseModel): access_token: str token_type: str class TokenData(BaseModel): - username: Optional[str] = None \ No newline at end of file + username: Optional[str] = None diff --git a/backend/sessions.py b/backend/sessions.py index 1a1eb73..c90c7a7 100644 --- a/backend/sessions.py +++ b/backend/sessions.py @@ -10,4 +10,4 @@ Base.metadata.create_all(engine) session_maker = sessionmaker(bind=engine) -session = session_maker() \ No newline at end of file +session = session_maker() diff --git a/backend/sqlalchemy_models.py b/backend/sqlalchemy_models.py index 3524b65..efb6bc3 100644 --- a/backend/sqlalchemy_models.py +++ b/backend/sqlalchemy_models.py @@ -3,10 +3,11 @@ Base = declarative_base() + class User(Base): __tablename__ = "users" id = Column(Integer, primary_key=True) username = Column(String, unique=True) password_hash = Column(String) # 1 = user, 4 = manager, 5 = admin, 6 = superadmin - role = Column(Integer, default=1) \ No newline at end of file + role = Column(Integer, default=1) diff --git a/backend/utils.py b/backend/utils.py index cc77236..9e15338 100644 --- a/backend/utils.py +++ b/backend/utils.py @@ -3,11 +3,11 @@ from backend.oauth import encrypt_password, get_user - def populate_admin_user(): admin_user = get_user("admin") if not admin_user: - admin_user = User(username="admin", - password_hash=encrypt_password("admin"), role=6) + admin_user = User( + username="admin", password_hash=encrypt_password("admin"), role=6 + ) session.add(admin_user) - session.commit() \ No newline at end of file + session.commit() From f5f6a64062115a7af6ba3a2a02e229b6fb28a33f Mon Sep 17 00:00:00 2001 From: NikhilRaikar17 Date: Sun, 4 Aug 2024 21:16:31 +0200 Subject: [PATCH 2/4] Removed icons and trying to add components into frontend --- frontend/src/layouts/MainLayout.vue | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/frontend/src/layouts/MainLayout.vue b/frontend/src/layouts/MainLayout.vue index 66ac648..c63c532 100644 --- a/frontend/src/layouts/MainLayout.vue +++ b/frontend/src/layouts/MainLayout.vue @@ -1,6 +1,6 @@ @@ -32,6 +35,7 @@