From f75c05e714bfb2834bffc4a4d3c8b79ba5c8a6fc Mon Sep 17 00:00:00 2001 From: lvalics Date: Tue, 23 Jan 2024 11:58:00 +0000 Subject: [PATCH 1/5] Added Data Sources Added Over Time on dashboard and an Errors Check link in menu to see errors. --- dj_backend_server/web/interfaces/dashboard.py | 49 ++++++++++++ .../web/templates/errors_check.html | 77 +++++++++++++++++++ dj_backend_server/web/templates/index.html | 37 ++++++++- .../templates/layout/sidebar-bot-page.html | 6 ++ dj_backend_server/web/urls.py | 4 +- dj_backend_server/web/views/views_chatbot.py | 29 +++++-- dj_backend_server/web/views/views_errors.py | 17 ++++ 7 files changed, 209 insertions(+), 10 deletions(-) create mode 100644 dj_backend_server/web/templates/errors_check.html create mode 100644 dj_backend_server/web/views/views_errors.py diff --git a/dj_backend_server/web/interfaces/dashboard.py b/dj_backend_server/web/interfaces/dashboard.py index 26d98458..10947694 100644 --- a/dj_backend_server/web/interfaces/dashboard.py +++ b/dj_backend_server/web/interfaces/dashboard.py @@ -1,12 +1,18 @@ from django.db.models import Count from django.db.models.functions import TruncDay +from django.utils import timezone from web.models.chat_histories import ChatHistory +from web.models.crawled_pages import CrawledPages +from web.models.pdf_data_sources import PdfDataSource +from web.models.website_data_sources import WebsiteDataSource def get_discussion_counts(): # Aggregate discussion counts per day grouped by session_id + thirty_days_ago = timezone.now() - timezone.timedelta(days=30) discussion_counts = ( ChatHistory.objects + .filter(created_at__gte=thirty_days_ago) .values('session_id') .annotate(created_date=TruncDay('created_at')) .values('created_date') @@ -20,3 +26,46 @@ def get_discussion_counts(): return formatted_counts +def get_data_source_counts(): + # Aggregate data source counts per day + thirty_days_ago = timezone.now() - timezone.timedelta(days=30) + crawled_pages_counts = ( + CrawledPages.objects + .filter(created_at__gte=thirty_days_ago) + .annotate(created_date=TruncDay('created_at')) + .values('created_date') + .annotate(count=Count('id')) + .order_by('created_date') + ) + pdf_data_sources_counts = ( + PdfDataSource.objects + .filter(created_at__gte=thirty_days_ago) + .annotate(created_date=TruncDay('created_at')) + .values('created_date') + .annotate(count=Count('id')) + .order_by('created_date') + ) + website_data_sources_counts = ( + WebsiteDataSource.objects + .filter(created_at__gte=thirty_days_ago) + .annotate(created_date=TruncDay('created_at')) + .values('created_date') + .annotate(count=Count('id')) + .order_by('created_date') + ) + # Combine and format the counts + all_dates = sorted(set( + [cp['created_date'] for cp in crawled_pages_counts] + + [pdf['created_date'] for pdf in pdf_data_sources_counts] + + [ws['created_date'] for ws in website_data_sources_counts] + )) + formatted_counts = [] + for date in all_dates: + formatted_counts.append({ + 'date': str(date.date()), + 'crawled_pages': next((cp['count'] for cp in crawled_pages_counts if cp['created_date'] == date), 0), + 'pdf_data_sources': next((pdf['count'] for pdf in pdf_data_sources_counts if pdf['created_date'] == date), 0), + 'website_data_sources': next((ws['count'] for ws in website_data_sources_counts if ws['created_date'] == date), 0), + }) + return formatted_counts + diff --git a/dj_backend_server/web/templates/errors_check.html b/dj_backend_server/web/templates/errors_check.html new file mode 100644 index 00000000..801396ed --- /dev/null +++ b/dj_backend_server/web/templates/errors_check.html @@ -0,0 +1,77 @@ +{% extends 'layout/app.html' %} +{% block content %} +
+
+
+ +
+

Error Check

+ + + + + + + + + + + {% for error in errors %} + + + + + + + {% empty %} + + + + {% endfor %} + +
+ Payload + + UUID + + Exception + + Failed At +
+ {{ error.payload }} + + {{ error.uuid }} + + {{ error.exception }} + + {{ error.failed_at }} +
+ No errors found. +
+ + +
+ +
+
+ +
+{% endblock %} \ No newline at end of file diff --git a/dj_backend_server/web/templates/index.html b/dj_backend_server/web/templates/index.html index d6a66108..35b110c9 100644 --- a/dj_backend_server/web/templates/index.html +++ b/dj_backend_server/web/templates/index.html @@ -65,7 +65,23 @@

Statistics of discussions in the last days

- +
+
+
+
+
+
+
+

Data Sources Added Over Time

+
+
+
+
+
+
+
+
+ @@ -94,7 +110,6 @@

Statistics of discussions in the last days

$(function () { 'use strict'; var discussionCountsData = JSON.parse('{{ discussion_counts_json }}'); - console.log(discussionCountsData); if ($("#morris_discussion_counts").length) { Morris.Bar({ element: 'morris_discussion_counts', @@ -109,6 +124,24 @@

Statistics of discussions in the last days

labels: ['Discussions'] }); } + + var dataSourcesCountsData = JSON.parse('{{ data_sources_counts_json }}'); + console.log(dataSourcesCountsData); + if ($("#morris_data_sources_counts").length) { + Morris.Bar({ + element: 'morris_data_sources_counts', + barColors: ['#ffad46', '#00b19d', '#ff5b5b'], + data: dataSourcesCountsData, + xkey: 'date', + ykeys: ['crawled_pages', 'pdf_data_sources', 'website_data_sources'], + hideHover: 'auto', + gridLineColor: '#eef0f2', + resize: true, + barSizeRatio: 0.4, + labels: ['Crawled Pages', 'PDF Data Sources', 'Website Data Sources'] + }); + } + }); diff --git a/dj_backend_server/web/templates/layout/sidebar-bot-page.html b/dj_backend_server/web/templates/layout/sidebar-bot-page.html index 9fef785f..11530c3b 100644 --- a/dj_backend_server/web/templates/layout/sidebar-bot-page.html +++ b/dj_backend_server/web/templates/layout/sidebar-bot-page.html @@ -47,6 +47,11 @@ {% endif %} {% if user.is_superuser %} +
  • + + + Errors check +
  • @@ -62,6 +67,7 @@ + diff --git a/dj_backend_server/web/urls.py b/dj_backend_server/web/urls.py index 7d1a110f..c18c615a 100644 --- a/dj_backend_server/web/urls.py +++ b/dj_backend_server/web/urls.py @@ -1,6 +1,6 @@ # api/urls.py from django.urls import path -from web.views import views_chatbot_settings, views_onboarding, views_chatbot, views_pdf_data_source, views_website_datasource, views_chatbot_createuser; +from web.views import views_chatbot_settings, views_onboarding, views_chatbot, views_pdf_data_source, views_website_datasource, views_chatbot_createuser, views_errors; urlpatterns = [ # Dashboard @@ -8,6 +8,8 @@ path('login', views_chatbot.login_view, name='login'), path('logout', views_chatbot.logout_view, name='logout'), path('modify_user/', views_chatbot_createuser.modify_user, name='modify_user'), + path('errors/', views_errors.errors_check, name='checkerrors'), + # Chatbot Settings path('app//', views_chatbot_settings.general_settings, name='chatbot.settings'), diff --git a/dj_backend_server/web/views/views_chatbot.py b/dj_backend_server/web/views/views_chatbot.py index 3eb16b57..05bdfd23 100644 --- a/dj_backend_server/web/views/views_chatbot.py +++ b/dj_backend_server/web/views/views_chatbot.py @@ -1,7 +1,6 @@ from uuid import uuid4 import os import requests -import re import json from django.http import JsonResponse, HttpResponseRedirect, HttpResponseServerError from django.shortcuts import render @@ -10,6 +9,7 @@ from django.contrib.auth import authenticate, login, logout from django.views.decorators.http import require_POST from django.core.serializers.json import DjangoJSONEncoder +from django.core.cache import cache from django.utils.safestring import mark_safe from django.utils import timezone from django.urls import reverse @@ -21,7 +21,7 @@ from web.signals.codebase_datasource_was_created import codebase_data_source_added from web.signals.pdf_datasource_was_added import pdf_data_source_added from web.services.handle_pdf_datasource import HandlePdfDataSource -from web.interfaces.dashboard import get_discussion_counts +from web.interfaces.dashboard import get_discussion_counts, get_data_source_counts from web.signals.codebase_datasource_was_created import codebase_data_source_added from web.signals.chatbot_was_created import chatbot_was_created from web.enums.chatbot_initial_prompt_enum import ChatBotInitialPromptEnum @@ -64,18 +64,33 @@ def logout_view(request): @check_authentication def index(request): chatbots = Chatbot.objects.filter(status=1) + # Cache keys + discussion_counts_cache_key = 'discussion_counts' + data_source_counts_cache_key = 'data_source_counts' + + # Try to get cached data + discussion_counts = cache.get(discussion_counts_cache_key) + data_source_counts = cache.get(data_source_counts_cache_key) + + # If cache is empty, compute the data and set it in the cache if not request.user.is_superuser: chatbots = chatbots.filter(user=request.user) - # Get discussion counts from the new dashboard interface - discussion_counts = get_discussion_counts() - discussion_counts_json = mark_safe(json.dumps(list(discussion_counts), cls=DjangoJSONEncoder)) - print (discussion_counts_json) + # If cache is empty, compute the data and set it in the cache + if not discussion_counts: + discussion_counts = get_discussion_counts() + cache.set(discussion_counts_cache_key, discussion_counts, 3600) # Cache for 1 hour + + if not data_source_counts: + data_source_counts = get_data_source_counts() + cache.set(data_source_counts_cache_key, data_source_counts, 3600) # Cache for 1 hour + # Pass the discussion counts to the template return render(request, 'index.html', { 'chatbots': chatbots, - 'discussion_counts_json': discussion_counts_json + 'discussion_counts_json': mark_safe(json.dumps(list(discussion_counts), cls=DjangoJSONEncoder)), + 'data_sources_counts_json': mark_safe(json.dumps(list(data_source_counts), cls=DjangoJSONEncoder)), }) diff --git a/dj_backend_server/web/views/views_errors.py b/dj_backend_server/web/views/views_errors.py new file mode 100644 index 00000000..187e961a --- /dev/null +++ b/dj_backend_server/web/views/views_errors.py @@ -0,0 +1,17 @@ +from django.shortcuts import render +from django.contrib.auth.decorators import login_required +from django.core.paginator import Paginator +from web.models.failed_jobs import FailedJob # Assuming the model for the failed_jobs table is named FailedJob + +@login_required +def errors_check(request): + if request.user.is_superuser: + error_list = FailedJob.objects.all().values('payload', 'uuid', 'exception', 'failed_at') + else: + error_list = FailedJob.objects.filter(user=request.user).values('payload', 'uuid', 'exception', 'failed_at') + + paginator = Paginator(error_list, 25) # Show 25 errors per page + page_number = request.GET.get('page') + errors = paginator.get_page(page_number) + return render(request, 'errors_check.html', {'errors': errors, 'paginator': paginator, 'page_obj': errors}) + From ff4029d78bd57f0e446ceba281b27088fba10d55 Mon Sep 17 00:00:00 2001 From: lvalics Date: Tue, 23 Jan 2024 12:21:27 +0000 Subject: [PATCH 2/5] Small tweaks on messages. --- .../web/templates/create_user.html | 24 +++++++++++-------- .../web/templates/modify_user.html | 7 ++++-- .../web/views/views_chatbot_createuser.py | 4 ++++ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/dj_backend_server/web/templates/create_user.html b/dj_backend_server/web/templates/create_user.html index d7d369ab..ae622c12 100644 --- a/dj_backend_server/web/templates/create_user.html +++ b/dj_backend_server/web/templates/create_user.html @@ -12,11 +12,15 @@

    Create a new user

    Please note down the username and password to send to the user. They will be required to change these credentials upon their first login.

    - {% if error_message %} - - {% endif %} + + {% if error_messages %} + + {% endif %} + @@ -31,7 +35,7 @@

    Create a new user

    - +
    • @@ -40,7 +44,7 @@

    Create a new user

    +
    @@ -50,7 +54,7 @@

    Create a new user

    +
    @@ -71,7 +75,7 @@

    Create a new user

    - +

    Suggested password (user will change) {{ SUGGESTEDPASS }}

    @@ -82,7 +86,7 @@

    Create a new user

    - +
    diff --git a/dj_backend_server/web/templates/modify_user.html b/dj_backend_server/web/templates/modify_user.html index 6b01da26..81797a5e 100644 --- a/dj_backend_server/web/templates/modify_user.html +++ b/dj_backend_server/web/templates/modify_user.html @@ -29,7 +29,7 @@

    Modify your user data

    +
    @@ -39,7 +39,7 @@

    Modify your user data

    +
    @@ -77,6 +77,9 @@

    Modify your user data

    +
    diff --git a/dj_backend_server/web/views/views_chatbot_createuser.py b/dj_backend_server/web/views/views_chatbot_createuser.py index 6ed54827..665b215d 100644 --- a/dj_backend_server/web/views/views_chatbot_createuser.py +++ b/dj_backend_server/web/views/views_chatbot_createuser.py @@ -12,6 +12,8 @@ from django.utils.crypto import get_random_string from django.urls import reverse from django.contrib import messages +import logging +logger = logging.getLogger(__name__) @login_required(login_url='/login/') @@ -35,6 +37,7 @@ def createuser(request): if not request.user.is_superuser: return HttpResponseRedirect(reverse('index')) if request.method == 'POST': + logger.debug("Processing POST request for user creation.") username = request.POST.get('username') first_name = request.POST.get('first_name') last_name = request.POST.get('last_name') @@ -95,6 +98,7 @@ def createuser(request): is_active=True, date_joined=now() ) + logger.info(f"User created successfully: {user.username} (ID: {user.id})") # Redirect to a new URL: request.session['created_username'] = username # Store the username in the session From 10e78a28d843a5bc354262deddac9e1e2f7d06b0 Mon Sep 17 00:00:00 2001 From: lvalics Date: Wed, 24 Jan 2024 08:08:38 +0000 Subject: [PATCH 3/5] Organized in menu, submenu --- dj_backend_server/web/templates/layout/sidebar-bot-page.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dj_backend_server/web/templates/layout/sidebar-bot-page.html b/dj_backend_server/web/templates/layout/sidebar-bot-page.html index 11530c3b..32c9bf81 100644 --- a/dj_backend_server/web/templates/layout/sidebar-bot-page.html +++ b/dj_backend_server/web/templates/layout/sidebar-bot-page.html @@ -11,15 +11,15 @@
    diff --git a/dj_backend_server/web/templates/onboarding/step-2-codebase.html b/dj_backend_server/web/templates/onboarding/step-2-codebase.html index d14cc4b6..435edf04 100644 --- a/dj_backend_server/web/templates/onboarding/step-2-codebase.html +++ b/dj_backend_server/web/templates/onboarding/step-2-codebase.html @@ -91,7 +91,7 @@

    GitHub Repo information ✨ <- Back -

    diff --git a/dj_backend_server/web/templates/onboarding/step-2-pdf.html b/dj_backend_server/web/templates/onboarding/step-2-pdf.html index a68aea60..b19be1e3 100644 --- a/dj_backend_server/web/templates/onboarding/step-2-pdf.html +++ b/dj_backend_server/web/templates/onboarding/step-2-pdf.html @@ -91,9 +91,9 @@

    Upload PDF files as sources {% endif %}
    {% csrf_token %} - +
    - +
    @@ -104,9 +104,10 @@

    Upload PDF files as sources you can upload up to 5 pdf files, and we will process the first ±100k words

    - - - + + +
    @@ -120,19 +121,20 @@

    Upload PDF files as sources

    -
    Make sure that your files are scannable (text not images) 🫶
    -
    You can upload multiple files at once and we will process them in the background.
    +
    We are accepting PDF/DOCX/XLSX/TXT(CSV/JSON). 🫶
    +
    PDF files can contain both text and images. We will send to OCR.
    +
    You can upload multiple files at once and we will process them in the background.
    - +

    - +
    <- Back - - + +
    - + @@ -142,6 +144,28 @@

    Upload PDF files as sources {% block scripts %}