Merge pull request #2 from jesvijonathan/jesvi_ml

Jesvi ML stuff

.gitignore

@@ -1,2 +1,3 @@
 .venv/
 app/Backend/.env
+app/Backend/google_credentials.json

app/Backend/.gitignore

@@ -1 +1,2 @@
-__pycache__
+__pycache__
+tmp

app/Backend/Modelfile

@@ -0,0 +1,67 @@
+FROM llama3.2
+
+
+# Params
+
+# Sets the temperature for creative responses
+PARAMETER temperature 0.3
+# Sets the context window size for token management (increase to improve context but will yoink more memory)
+PARAMETER num_ctx 30192
+
+# Enable Mirostat sampling
+PARAMETER mirostat 2
+
+# Adjust repeat penalty and look back
+# PARAMETER repeat_last_n 55
+PARAMETER repeat_penalty 1.2
+
+# Adjust sampling parameters for diversity
+# PARAMETER top_k 40
+# PARAMETER top_p 0.92
+# PARAMETER min_p 0.02
+
+# Maximum tokens to predict
+PARAMETER num_predict 180
+
+
+SYSTEM """You are an IT application/expert engineer who has to sort and work on incoming tickets from merchants or clients for Worldline's products in the payment/fintech industry. 
+Start with a greeting and understand the issue. 
+Ask specific questions only if required and try to auto-fill or get all the details with minimal queries. 
+Have a conversation, and when the conversation ends (once the user sends a 'create' text), provide the JSON as the last parameter and indicate that the ticket has been created. 
+Do not provide the JSON before the conversation ends and keep the conversation natural in a flow. 
+Once the JSON is provided after 'create', end the conversation. Also, give them an option to mention 'create' to continue creating a ticket (only if the text/description is sufficient/retrieved from the user). 
+You will be given the history of chat, to conitue with further, refer the chat and attachments to continue meaningful conversation.
+Analyse the text and attachments thoroughly and debug the issue.
+Give brief replies. Do not nudge. Upon command 'status', explain what you have understood till now, and Try to resolve or understand and create a ticket. Do not close the chat if the user has not mentioned 'create' or '/create' command. 
+on receiving "create" or "/create" command, STRICTLY print the json, do nothing extra. start json with ```json and end with ```.
+
+create 
+return json:
+```json
+{
+"subject" : "", // Generate the title for provided text
+"summary": "", // Generate a summary of the text, with all description 
+"attachments": [ {"attachment_name" : "The attachement name ", "attachment details" :  " analysis, file information about the attachent and issue" } ],
+"product_type": "", // retrieve the product type from the text & attachments [ webgui, wlpfo, pass, wlsi, ]
+"issue_type": "", // retrieve the issue type from the text & attachments [\'bug\', \'error\', \'issue\', \'story\', \'others\', \'feature\', \'enhancement\', \'support\']
+"priority": "", // Analyse the priority from the text & attachments [\'crtical\', \'high\', \'medium\', \'low\']\\
+"story_points": "", // Analyse the story points from the text & attachments, linked with the priority & ticket type
+"estimation": "", // Analyse the estimation from the text & attachments, linked with the story points & priority
+"analysis": "", // Analysis of the issue, what could be the possible cause of the issue, and how can it be resolved from an support or engineer\'s point of view. If an legit error/bug, give solution on worldline\'s product.
+"reply": "" // reply to the support text, conversation end text.
+}
+"""
+
+
+TEMPLATE """{{ if .System }}<|im_start|>system
+{{ .System }}<|im_end|>
+{{ end }}{{ if .Prompt }}<|im_start|>user
+{{ .Prompt }}<|im_end|>
+{{ end }}<|im_start|>assistant
+"""
+
+PARAMETER stop "<|im_start|>"
+PARAMETER stop "<|im_end|>"
+PARAMETER stop "<|eot_id|>"
+
+MESSAGE assistant Hello! How can I help you today?

app/Backend/README.md

@@ -0,0 +1 @@
+Simulated content of the file

app/Backend/app.py

@@ -1,29 +1,115 @@
-from flask import Flask, request
+from flask import Flask, request, session, render_template, make_response
 from flask_cors import CORS
-from auth import auth_ldap
+from flask_socketio import SocketIO, emit, send, disconnect
+from flask import jsonify
+from flask import request
+import datetime
+import time
+from google.api_core.exceptions import ResourceExhausted
+import json
+from modules.auth.auth import auth_ldap, jwt_required, cleanup_user
 from ml_image_eval import vision
-from ml_text_eval import text
+from modules.text import text
+from modules.ml.ml_handler import ChatbotHandler
 from modules.ticket import ticket
 from config import *
+from modules.log import *
+import base64
+import shutil
 
-#flask configurations
+
+# Flask configurations
 app = Flask(__name__)
+app.secret_key = secret_key
 app.config.from_prefixed_env()
-CORS(app, resources={r"/*": {"origins": "*"}})
+CORS(app, resources={r"/*": {"origins": "*", "supports_credentials": True}})
 
-# app configurations
-app.register_blueprint(auth_ldap,url_prefix='/sso')
-app.register_blueprint(ticket,url_prefix='/v1/ticket')
+socketio = SocketIO(app, cors_allowed_origins="*", async_mode='threading')
 
-# a demo page to pass screenshots 
-# # and on submit evaluate the screen shot and find the product/division
-app.register_blueprint(vision,url_prefix='/v1/vision')
+# Global routes
+app.register_blueprint(auth_ldap, url_prefix='/sso')
+app.register_blueprint(ticket, url_prefix='/ticket')
+app.register_blueprint(vision, url_prefix='/vision')
+app.register_blueprint(text, url_prefix='/text')
 
+@app.route('/')
+def home():
+    return render_template('index.html', token_param="")
 
-# get text and other details
-# get product, division, team, issue, summary, issue-level, 
-app.register_blueprint(text,url_prefix='/v1/text')
+# dummy route to test json data from ./dataset/test.json
+@app.route('/test', methods=['GET'])
+def test():
+    with open('dataset/test.json') as f:
+        data = json.load(f)
+    return jsonify(data)
+
+# Socket IO event handling
+@socketio.on('connect')
+@jwt_required
+def connect():
+    sid = request.sid
+    token = request.cookies.get('session')
+    if token not in sockets:
+        sockets[token] = {"sid": sid, "connected": True, "history": {}}
+        socket_connection[token] = ChatbotHandler(token, socketio)
+    sockets[token]["sid"] = sid 
+    sockets[token]["connected"] = True
+
+    chat_history = socket_connection[token].history if token in socket_connection else {}
+    socketio.emit("live_chat" if chat_history else "message",
+                  {"live_chat": chat_history} if chat_history else {"message": {**sockets[token]}},
+                  room=sid)
+
+@socketio.on('message')
+@jwt_required
+def message(msg):
+    sid = request.sid
+    print(f"Message from {sid}: {msg}")
+    token = request.cookies.get('session')
+    if token in socket_connection:
+        socket_connection[token].response_add(msg)
+    else:
+        socketio.emit("message", {"message": "No active session found"}, room=sid)
+
+@socketio.on('user_attachment')
+@jwt_required
+def handle_user_attachment(data):
+    token = request.cookies.get('session')
+    chat_handler = socket_connection[token]
+    print("######", data)
+    socket_connection[token].response_add(data['message'], data['attachments'])
 
-if __name__ == '__main__':
-    app.run(host='0.0.0.0', port=5000, debug=True)
 
+@socketio.on('disconnect')
+@jwt_required
+def disconnect():
+    print(f"User disconnected: {request.sid}")
+    token = request.cookies.get('session')
+    sockets[token]["connected"] = False
+    chat= socket_connection[token]
+    connec= chat.result["connection"] 
+    if connec == "closed":
+        chat.destroy()
+        chat = None
+        del socket_connection[token]
+        del sockets[token]
+    else:
+        chat.result["connection"] = "offline"
+
+# cleanup during startup
+if tmp_folders_cleanup:
+    if os.path.exists(chats_folder) and os.path.isdir(chats_folder):
+            for filename in os.listdir(chats_folder):
+                file_path = os.path.join(chats_folder, filename)
+                try:
+                    if os.path.isfile(file_path) or os.path.islink(file_path):
+                        os.unlink(file_path)  
+                    elif os.path.isdir(file_path):
+                        shutil.rmtree(file_path)
+                except Exception as e:
+                    print(f"Failed to delete {file_path}. Reason: {e}")
+
+
+
+if __name__ == '__main__':
+    socketio.run(app, debug=True, host="0.0.0.0", port=5000)

app/Backend/bucket/tickets/SVC-53331/chat.json

@@ -0,0 +1 @@
+{"1": {"recipient": "wl_vertex", "time": "2024-10-09T21:52:27.670787", "message": "Hello! How can I help you today?"}, "2": {"recipient": "admin", "time": "2024-10-09T21:52:32.155105", "message": "", "attachment": {"0": {"filename": "log.txt", "path": "./bucket/chats/nD94jLywg2TKka5lAAAD/log.txt", "extension": "txt", "mime_type": "text/plain", "size": 4382}, "1": {"filename": "Invalid_LoginID.jpg", "path": "./bucket/chats/nD94jLywg2TKka5lAAAD/Invalid_LoginID.jpg", "extension": "jpg", "mime_type": "image/jpeg", "size": 31031}}}, "3": {"recipient": "wl_vertex", "time": "2024-10-09T21:52:46.683341", "message": "Thank you for providing the logs.  We see the failed login attempts and the instance management activities.  To help us troubleshoot further, could you please confirm the username you were using to log in?  In the meantime, we can initiate a password reset for you. Please provide the email address associated with your account."}}

app/Backend/bucket/tickets/SVC-53331/log.txt

@@ -0,0 +1,108 @@
+[27:7:2023:18:32:22] 
+
+Application Started
+[27:7:2023:18:34:6] User wl2028 logged in
+[27:7:2023:18:34:6] Initializing Training Manager
+[27:7:2023:18:34:27] Course Menu
+[27:7:2023:18:34:27] Course 1 instance created
+[27:7:2023:18:34:42] Course 1 created
+[27:7:2023:18:34:42] Training Manager Invoked
+[27:7:2023:18:34:44] Generate report
+[27:7:2023:18:34:51] Training Manager Invoked
+[27:7:2023:18:34:51] Course Menu
+[27:7:2023:18:34:51] Course 2 instance created
+[27:7:2023:18:35:8] Course 2 created
+[27:7:2023:18:35:8] Training Manager Invoked
+[27:7:2023:18:35:10] Employee Menu
+[27:7:2023:18:35:10] Employee 1 instance created
+[27:7:2023:18:35:25] Employee 1 created
+[27:7:2023:18:35:25] Training Manager Invoked
+[27:7:2023:18:35:26] Employee Menu
+[27:7:2023:18:35:26] Employee 2 instance created
+[27:7:2023:18:36:2] Employee 2 created
+[27:7:2023:18:36:2] Training Manager Invoked
+[27:7:2023:18:36:3] Employee Menu
+[27:7:2023:18:36:3] Employee 3 instance created
+[27:7:2023:18:36:29] Employee 3 created
+[27:7:2023:18:36:29] Training Manager Invoked
+[27:7:2023:18:36:31] Generate report
+[27:7:2023:18:36:57] Employee 1 instance deleted
+[27:7:2023:18:36:57] Employee 2 instance deleted
+[27:7:2023:18:36:57] Employee 3 instance deleted
+[27:7:2023:18:36:57] Employee 3 instance deleted
+[27:7:2023:18:36:57] Employee 1 instance deleted
+[27:7:2023:18:36:57] Employee 2 instance deleted
+[27:7:2023:18:36:57] Course Report Generated
+[27:7:2023:18:37:27] Training Manager Invoked
+[27:7:2023:18:37:30] Enroll employee Menu
+[27:7:2023:18:37:32] Returning to Main Menu
+[27:7:2023:18:37:32] Training Manager Invoked
+[27:7:2023:18:37:35] Update Menu
+[27:7:2023:18:37:41] Course 2 deleted
+[27:7:2023:18:37:41] Course 2 instance deleted
+[27:7:2023:18:37:43] Training Manager Invoked
+[27:7:2023:18:37:44] Generate report
+[27:7:2023:18:37:46] Training Manager Invoked
+[27:7:2023:18:37:49] Enroll employee Menu
+[27:7:2023:18:37:50] Enroll Employee
+[27:7:2023:18:37:55] Employee 1 selected
+[27:7:2023:18:37:58] Course 1 selected
+[27:7:2023:18:37:59] Employee 1 enrolled in course 1
+[27:7:2023:18:38:0] Enroll Employee
+[27:7:2023:18:38:2] Employee 3 selected
+[27:7:2023:18:38:3] Course 1 selected
+[27:7:2023:18:38:4] Employee 3 enrolled in course 1
+[27:7:2023:18:38:5] View Enrollments
+[27:7:2023:18:38:16] Returning to Main Menu
+[27:7:2023:18:38:16] Training Manager Invoked
+[27:7:2023:18:38:18] Generate report
+[27:7:2023:18:38:20] Training Manager Invoked
+[27:7:2023:18:38:33] Course 2 instance created
+[27:7:2023:18:38:33] Course 3 instance created
+[27:7:2023:18:38:33] Course 4 instance created
+[27:7:2023:18:38:33] Courses data loaded
+[27:7:2023:18:38:33] Employee -2 instance created
+[27:7:2023:18:38:33] Employee -1 instance created
+[27:7:2023:18:38:33] Employee 0 instance created
+[27:7:2023:18:38:33] Employee 1 instance created
+[27:7:2023:18:38:33] Employees data loaded
+[27:7:2023:18:38:33] Employee 1 enrolled in course 1
+[27:7:2023:18:38:33] Employee 2 enrolled in course 2
+[27:7:2023:18:38:33] Employee 3 enrolled in course 1
+[27:7:2023:18:38:33] Enrollment data loaded
+[27:7:2023:18:38:33] Loaded
+[27:7:2023:18:38:35] Training Manager Invoked
+[27:7:2023:18:38:37] Generate report
+[27:7:2023:18:38:40] Training Manager Invoked
+[27:7:2023:18:38:53] Course Menu
+[27:7:2023:18:38:53] Course 5 instance created
+[27:7:2023:18:38:56] Course 5 instance deleted
+[27:7:2023:18:38:56] Course creation cancelled
+[27:7:2023:18:38:56] Training Manager Invoked
+[27:7:2023:18:39:0] Courses data loaded
+[27:7:2023:18:39:0] Employees data loaded
+[27:7:2023:18:39:0] Enrollment data loaded
+[27:7:2023:18:39:0] Loaded
+[27:7:2023:18:39:1] Training Manager Invoked
+[27:7:2023:18:39:1] Course Menu
+[27:7:2023:18:39:1] Course 5 instance created
+[27:7:2023:18:39:3] Course 5 instance deleted
+[27:7:2023:18:39:3] Course creation cancelled
+[27:7:2023:18:39:3] Training Manager Invoked
+[27:7:2023:18:39:4] Generate report
+[27:7:2023:18:39:11] Training Manager Invoked
+[27:7:2023:18:39:12] Database Reset !
+[27:7:2023:18:39:13] Application Exited
+[27:7:2023:18:39:15] 
+
+Application Started
+[28:7:2023:23:11:38] Exception : 
+
+Invalid username or password !
+[28:7:2023:23:11:40] Exception : 
+
+Invalid username or password !
+[28:7:2023:23:11:44] Exception : 
+
+Invalid username or password !
+[31:7:2023:11:16:49]