From 994ef2570a9a1f7e450de87c03ecbc4a6c6c4964 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Fri, 27 Oct 2023 11:30:00 -0400 Subject: [PATCH 01/29] Version bump and fixed documentation url in python/README.md --- teraserver/CMakeLists.txt | 2 +- teraserver/python/README.md | 2 +- teraserver/python/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/teraserver/CMakeLists.txt b/teraserver/CMakeLists.txt index 72ff9ace2..6df8d0a02 100755 --- a/teraserver/CMakeLists.txt +++ b/teraserver/CMakeLists.txt @@ -10,7 +10,7 @@ endif(NOT CMAKE_BUILD_TYPE) # Software version SET(OPENTERA_VERSION_MAJOR "1") SET(OPENTERA_VERSION_MINOR "2") -SET(OPENTERA_VERSION_PATCH "4") +SET(OPENTERA_VERSION_PATCH "5") SET(OPENTERA_SERVER_VERSION OpenTera_v${OPENTERA_VERSION_MAJOR}.${OPENTERA_VERSION_MINOR}.${OPENTERA_VERSION_PATCH}) diff --git a/teraserver/python/README.md b/teraserver/python/README.md index 5a95fe0b1..82c895ed8 100644 --- a/teraserver/python/README.md +++ b/teraserver/python/README.md @@ -10,7 +10,7 @@ External microservices can use this package as a base. OpenTera is licensed under [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.txt) . # Documentation -Please visit our [WiKi documentation on GitHub](https://github.com/introlab/opentera/wiki) +Please visit our [Documentation on GitHub](https://introlab.github.io/opentera/) # Dependencies OpenTera is based or uses the following Open Source technologies : diff --git a/teraserver/python/setup.py b/teraserver/python/setup.py index 2962575a1..0ac132c2d 100644 --- a/teraserver/python/setup.py +++ b/teraserver/python/setup.py @@ -9,7 +9,7 @@ setuptools.setup( name="opentera", - version="1.2.4", + version="1.2.5", author="Dominic Létourneau, Simon Brière", author_email="dominic.letourneau@usherbrooke.ca, simon.briere@usherbrooke.ca", description="OpenTera base package", From e842ce1d398761cdb5e92c099dde608d2e1ccdae Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Tue, 5 Dec 2023 15:27:08 -0500 Subject: [PATCH 02/29] Updated requirements Added JOSS paper in publications. --- .gitignore | 21 ++----------------- README.md | 1 + teraserver/python/env/requirements.txt | 29 +++++++++++++------------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 03f4c71da..6a58469e2 100644 --- a/.gitignore +++ b/.gitignore @@ -51,20 +51,14 @@ docs/_build/ docs/--no-check-certificate docs/swagger.json -build-teraplus-Desktop_Qt_5_12_0_MSVC2017_64bit-Debug *.user -build-teraserver-Desktop_Qt_5_12_0_MSVC2017_64bit-Debug .idea -python-3.6 # Docker docker/prod/certificates/*.pem # Node modules node_modules/ -/teraplus/CMakeLists.txt.user.4.9-pre1 -/teraserver/CMakeLists.txt.user.4.9-pre1 -/teraserver/python/config/TeraServerConfig.ini~0b30c4625222f7c8696ca353a9e47d17d15d6cb5 /teraserver/python/messages/python/* /teraserver/python/certificates/ca_cert.pem /teraserver/python/certificates/ca_key.pem @@ -72,28 +66,17 @@ node_modules/ /teraserver/python/certificates/devices/client_key.pem /teraserver/python/certificates/site_cert.pem /teraserver/python/certificates/site_key.pem -/teraplus/client/resources/icons/device.psd /teraserver/python/uploads -build-teraplus-Desktop_Qt_5_13_1_MSVC2017_64bit-Debug -build-teraserver-Desktop_Qt_5_13_1_MSVC2017_64bit-Debug -build-teraplus-Desktop_Qt_5_13_1_MSVC2017_64bit-Release -build-teraserver-Desktop_Qt_5_13_2_MSVC2017_64bit-Debug -build-teraserver-Desktop_Qt_5_14_0_MSVC2017_64bit-Debug -build-teraserver-Desktop_Qt_5_14_1_MSVC2017_64bit-Debug protobuf -teraserver/python/env/_python-3.6 -build-teraserver-Desktop_Qt_5_14_2_MSVC2017_64bit-Debug -python-3.8 -teraserver/python/services/BureauActif/uploads/ /teraserver/python/OpenTeraServerVersion.py /teraserver/python/services/FileTransferService/upload files build-teraserver-Desktop_Qt_5_15_2_MSVC2019_64bit-Debug -python-3.10 /teraserver/easyrtc/package-lock.json venv joss-paper/paper.jats joss-paper/paper.pdf /.vscode -_python-3.10 python-3.11 +build-teraserver-Desktop_Qt_6_6_1_MSVC2019_64bit-Debug +python-3.10 diff --git a/README.md b/README.md index 998358ad2..cb864f21a 100755 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ You are welcome to participate in this effort. Leave us comments or report [Issu ## Publication(s) +* [![DOI](https://joss.theoj.org/papers/10.21105/joss.05497/status.svg)](https://doi.org/10.21105/joss.05497) Létourneau, D., Brière , S., et al., [OpenTera: A Framework for Telehealth Applications](https://doi.org/10.21105/joss.05497), Journal of Open Source Software, vol. 8, no 91, p. 5497 (2023) * Panchea, A.M., Létourneau, D., Brière, S. et al., [OpenTera: A microservice architecture solution for rapid prototyping of robotic solutions to COVID-19 challenges in care facilities](https://rdcu.be/cHzmf), Health Technol. 12, 583–596 (2022) ## Videos diff --git a/teraserver/python/env/requirements.txt b/teraserver/python/env/requirements.txt index 5dda09c8e..f08148bbc 100644 --- a/teraserver/python/env/requirements.txt +++ b/teraserver/python/env/requirements.txt @@ -1,20 +1,20 @@ pypiwin32==223; sys_platform == 'win32' -Twisted==23.8.0 -treq==22.2.0 -cryptography==41.0.4 +Twisted==23.10.0 +treq==23.11.0 +cryptography==41.0.7 autobahn==23.6.2 -SQLAlchemy==2.0.21 +SQLAlchemy==2.0.23 sqlalchemy-schemadisplay==1.3 pydot==1.4.2 psycopg2-binary==2.9.9 Flask==2.3.3 Flask-SQLAlchemy==3.1.1 -Flask-Login==0.6.2 +Flask-Login==0.6.3 Flask-Login-Multi==0.1.2 Flask-HTTPAuth==4.8.0 Flask-SocketIO==5.3.6 Flask-Session==0.5.0 -flask-restx==1.1.0 +flask-restx==1.2.0 Flask-Security==3.0.0 Flask-Babel==4.0.0 Flask-BabelEx==0.9.4 @@ -26,18 +26,19 @@ Flask-Principal==0.4.0 redis==5.0.1 txredisapi==1.4.10 passlib==1.7.4 -bcrypt==4.0.1 -WTForms==3.0.1 -pyOpenSSL==23.2.0 +bcrypt==4.1.1 +WTForms==3.1.1 +pyOpenSSL==23.3.0 service-identity==23.1.0 PyJWT==2.8.0 pylzma==0.5.0 bz2file==0.98 python-slugify==8.0.1 -websocket-client==1.6.3 -pytest==7.4.2 -jsonschema==4.19.1 +websocket-client==1.7.0 +pytest==7.4.3 +# Hold jsonschema until flask-restx gets updated +jsonschema==4.17.3 Jinja2==3.1.2 ua-parser==0.18.0 -#Remove this when Flask-Login is updated with latest Werkseug 3.x.x -Werkzeug==2.3.7 +#Remove this when Flask-Login / flask-restx is updated with latest Werkseug 3.x.x +Werkzeug==2.3.8 From 448aa560f802dbaccf4ea70351356cef86e4d4b4 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Thu, 7 Dec 2023 09:13:39 -0500 Subject: [PATCH 03/29] Improved response time when querying project stats with participants. --- .../FlaskModule/API/user/UserQueryStats.py | 11 ++++--- .../opentera/db/models/TeraParticipant.py | 32 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/user/UserQueryStats.py b/teraserver/python/modules/FlaskModule/API/user/UserQueryStats.py index fbed9622e..b51f3b71b 100644 --- a/teraserver/python/modules/FlaskModule/API/user/UserQueryStats.py +++ b/teraserver/python/modules/FlaskModule/API/user/UserQueryStats.py @@ -2,6 +2,7 @@ from modules.LoginModule.LoginModule import user_multi_auth, current_user from modules.FlaskModule.FlaskModule import user_api_ns as api from opentera.db.models.TeraParticipant import TeraParticipant +from opentera.db.models.TeraSessionParticipants import TeraSessionParticipants from flask_babel import gettext from modules.DatabaseModule.DBManager import DBManager, DBManagerTeraUserAccess @@ -248,9 +249,9 @@ def get_project_stats(user_access: DBManagerTeraUserAccess, item_id: int, with_p devices_total = len(project.project_devices) devices_enabled = len([dev for dev in project.project_devices if dev.device_enabled]) sessions_total = 0 - for part in project.project_participants: - sessions_total += TeraSessionParticipants.get_session_count_for_participant( - id_participant=part.id_participant) + # for part in project.project_participants: + # sessions_total += TeraSessionParticipants.get_session_count_for_participant( + # id_participant=part.id_participant) stats = {'users_total_count': len(project_users), 'users_enabled_count': len(project_users_enabled), @@ -359,6 +360,7 @@ def get_device_stats(user_access: DBManagerTeraUserAccess, item_id: int) -> dict @staticmethod def get_participant_list_stats(participant: TeraParticipant): + first_session = participant.get_first_session() first_session_date = None if first_session: @@ -377,7 +379,8 @@ def get_participant_list_stats(participant: TeraParticipant): stats = {'id_participant': participant.id_participant, 'participant_name': participant.participant_name, 'participant_enabled': participant.participant_enabled, - 'participant_sessions_count': len(participant.participant_sessions), + 'participant_sessions_count': + TeraSessionParticipants.get_session_count_for_participant(participant.id_participant), 'participant_first_session': first_session_date, 'participant_last_session': last_session_date, 'participant_last_online': last_online_date diff --git a/teraserver/python/opentera/db/models/TeraParticipant.py b/teraserver/python/opentera/db/models/TeraParticipant.py index 6cc64f73d..009efd775 100644 --- a/teraserver/python/opentera/db/models/TeraParticipant.py +++ b/teraserver/python/opentera/db/models/TeraParticipant.py @@ -1,7 +1,7 @@ from opentera.db.Base import BaseModel from opentera.db.SoftDeleteMixin import SoftDeleteMixin from sqlalchemy import Column, ForeignKey, Integer, String, Sequence, Boolean, TIMESTAMP -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, lazyload from sqlalchemy.exc import IntegrityError from opentera.db.models.TeraParticipantGroup import TeraParticipantGroup from opentera.db.models.TeraServerSettings import TeraServerSettings @@ -161,19 +161,29 @@ def get_id(self): return self.participant_uuid def get_first_session(self): - sessions = sorted(self.participant_sessions, key=lambda session: session.session_start_datetime) - if sessions: - return sessions[0] + session = (TeraSessionParticipants.query.filter_by(id_participant=self.id_participant) + .order_by(TeraSessionParticipants.id_session.asc()).limit(1).first()) + if session: + # Turn off lazy loading for session + return TeraSession.query.filter_by(id_session=session.id_session).options(lazyload("*")).first() + # sessions = sorted(self.participant_sessions, key=lambda session: session.session_start_datetime) + # if sessions: + # return sessions[0] return None def get_last_session(self): - from opentera.db.models.TeraSession import TeraSessionStatus - sessions = [session for session in self.participant_sessions - if session.session_status == TeraSessionStatus.STATUS_COMPLETED.value or - session.session_status == TeraSessionStatus.STATUS_TERMINATED.value] - sessions = sorted(sessions, key=lambda session: session.session_start_datetime) - if sessions: - return sessions[-1] + # from opentera.db.models.TeraSession import TeraSessionStatus + # sessions = [session for session in self.participant_sessions + # if session.session_status == TeraSessionStatus.STATUS_COMPLETED.value or + # session.session_status == TeraSessionStatus.STATUS_TERMINATED.value] + # sessions = sorted(sessions, key=lambda session: session.session_start_datetime) + # if sessions: + # return sessions[-1] + session = (TeraSessionParticipants.query.filter_by(id_participant=self.id_participant) + .order_by(TeraSessionParticipants.id_session.desc()).limit(1).first()) + if session: + # Turn off lazy loading for session + return TeraSession.query.filter_by(id_session=session.id_session).options(lazyload("*")).first() return None @staticmethod From d11c13cc8a99eab4cbfb471573a58433624a856b Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Fri, 8 Dec 2023 11:23:23 -0500 Subject: [PATCH 04/29] Fixed issue when added participants / users / devices to a WebRTC session. --- .../modules/UserManagerModule/UserManagerModule.py | 11 +++++++---- .../python/opentera/services/BaseWebRTCService.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/teraserver/python/modules/UserManagerModule/UserManagerModule.py b/teraserver/python/modules/UserManagerModule/UserManagerModule.py index 951e1fdc2..5ac099d22 100755 --- a/teraserver/python/modules/UserManagerModule/UserManagerModule.py +++ b/teraserver/python/modules/UserManagerModule/UserManagerModule.py @@ -449,12 +449,15 @@ def set_participants_in_session(self, session_uuid: str, participant_uuids: list self.participant_registry.participant_leave_session(participant, session_uuid) participant_event.type = ParticipantEvent.PARTICIPANT_LEFT_SESSION - # TODO: Get others infos for that participant from opentera.db.models.TeraParticipant import TeraParticipant part_data = TeraParticipant.get_participant_by_uuid(participant) - participant_event.participant_name = part_data.participant_name - participant_event.participant_project_name = part_data.participant_project.project_name - participant_event.participant_site_name = part_data.participant_project.project_site.site_name + if part_data: + participant_event.participant_name = part_data.participant_name + participant_event.participant_project_name = part_data.participant_project.project_name + participant_event.participant_site_name = part_data.participant_project.project_site.site_name + else: + # TODO: Find when this can happen! + pass self.send_event_message(participant_event, self.event_topic_name()) def set_devices_in_session(self, session_uuid: str, device_uuids: list, in_session: bool): diff --git a/teraserver/python/opentera/services/BaseWebRTCService.py b/teraserver/python/opentera/services/BaseWebRTCService.py index a9a086bc3..f62070438 100644 --- a/teraserver/python/opentera/services/BaseWebRTCService.py +++ b/teraserver/python/opentera/services/BaseWebRTCService.py @@ -578,7 +578,7 @@ def manage_invite_to_session(self, session_manage_args: dict): api_req = {'session': {'id_session': id_session, # New session 'session_participants_uuids': session_info['session_participants'], 'session_users_uuids': session_info['session_users'], - 'sessiom_devices_uuids': session_info['session_devices'], + 'session_devices_uuids': session_info['session_devices'], } } api_response = self.post_to_opentera('/api/service/sessions', api_req) From a9cea8da8072d000b7b432e932adc8e165257f88 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 11 Dec 2023 11:10:22 -0500 Subject: [PATCH 05/29] Refs #232. Added preloading of cameras, fixed issue on participant "welcome" page. --- .../static/js/tera_layout_participants.js | 3 +- teraserver/easyrtc/static/js/tera_webrtc.js | 67 +++++++++++++++++-- .../static/js/opentera_localvideo.js | 8 +++ 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/teraserver/easyrtc/static/js/tera_layout_participants.js b/teraserver/easyrtc/static/js/tera_layout_participants.js index ba269e156..0170b9bf0 100644 --- a/teraserver/easyrtc/static/js/tera_layout_participants.js +++ b/teraserver/easyrtc/static/js/tera_layout_participants.js @@ -98,7 +98,7 @@ function updateUserLocalViewLayout(){ let remote_num = usedRemoteVideosIndexes.length; let usedLocalVideosIndexes = getVideoStreamsIndexes(localStreams); let local_num = usedLocalVideosIndexes.length; - //console.log(usedLocalVideosIndexes); + if (currentLargeViewId.startsWith('local') && local_num === 1){ setColWidth(largeView, 10); @@ -112,6 +112,7 @@ function updateUserLocalViewLayout(){ } switch(local_num){ + case 0: case 1: selfViewRow2.hide(); break; diff --git a/teraserver/easyrtc/static/js/tera_webrtc.js b/teraserver/easyrtc/static/js/tera_webrtc.js index e841b28e9..7814841a3 100644 --- a/teraserver/easyrtc/static/js/tera_webrtc.js +++ b/teraserver/easyrtc/static/js/tera_webrtc.js @@ -12,10 +12,13 @@ let localStreams = []; // {peerid, streamname, stream: MediaStream}, order is im var connected = false; var needToCallOtherUsers = false; +let preinitCameras = true; + function connect() { console.log("Connecting..."); - playSound("audioCalling"); + if (!preinitCameras) + playSound("audioCalling"); /*var localFilter = easyrtc.buildLocalSdpFilter( { audioRecvBitrate:20, videoRecvBitrate:30 ,videoRecvCodec:"h264" @@ -41,11 +44,60 @@ function connect() { //Post-connect Event listeners //easyrtc.setOnHangup(streamDisconnected); //easyrtc.setOnCall(newStreamStarted); + if (preinitCameras) + preloadCameras(); + else{ + connected = true; + updateLocalAudioVideoSource(1); + showLayout(true); + } + + +} - connected = true; - updateLocalAudioVideoSource(1); +// On some devices, there's a strange bug that delays access to the camera, unless we try to access it at least once... +function preloadCameras(){ + navigator.mediaDevices.enumerateDevices() + .then(function(devices) { + let preload_devices = []; + devices.forEach(function(device) { + if (device.kind === "videoinput"){ + if (!device.label.includes(" IR ")) { // Filter "IR" camera, since they won't work. + preload_devices.push(device); + } + } + //console.log(device.kind + ": " + device.label + " id = " + device.deviceId); + }); + preloadCamera(preload_devices, 0); + }) + .catch(function(err) { + console.log(err.name + ": " + err.message); + }); - showLayout(true); +} + +function preloadCamera(devices, current_index){ + if (current_index >= devices.length || current_index < 0){ + return; + } + + navigator.mediaDevices.getUserMedia({video: {deviceId: { exact: devices[current_index].deviceId }}, + audio: false}).then(function(stream){ + console.log("Preloaded camera " + devices[current_index].label); + stream.getTracks().forEach(track => track.stop()); + // Did we get at least the first stream? If so, start everything! + if (current_index === 0){ + playSound("audioCalling"); + connected = true; + updateLocalAudioVideoSource(1); + showLayout(true); + } + preloadCamera(devices, current_index + 1); + }).catch(async function() { + console.log("Can't preload camera: " + devices[current_index].label); + /*await new Promise(resolve => setTimeout(resolve, 1000))*/ + preloadCamera(devices, current_index + 1); + }); } function muteMicro(local, index, new_state){ @@ -264,14 +316,15 @@ function setPrimaryView(peer_id, streamname){ setPrimaryViewIcon(primaryView.peerid, primaryView.streamName); } -function updateLocalAudioVideoSource(streamindex){ +async function updateLocalAudioVideoSource(streamindex){ if (connected === true){ let streamname = "localStream" + streamindex; if (streamindex === 1) // Default stream = no name. streamname = ""; - if (streamindex < localStreams.length){ + if (streamindex <= localStreams.length){ console.log("Updating audio/video source: " + streamname); - + // Stopping previous stream + localStreams[streamindex-1].stream.getTracks().forEach(track => track.stop()); }else { console.log("Creating audio/video source: " + streamname); } diff --git a/teraserver/python/services/VideoRehabService/static/js/opentera_localvideo.js b/teraserver/python/services/VideoRehabService/static/js/opentera_localvideo.js index 04f360726..011e87239 100644 --- a/teraserver/python/services/VideoRehabService/static/js/opentera_localvideo.js +++ b/teraserver/python/services/VideoRehabService/static/js/opentera_localvideo.js @@ -1,6 +1,7 @@ let videoSources = []; let currentVideoSourceIndex = 0; let timerHandle = 0; +let currentVideoStream = undefined; let currentConfig = {'currentVideoName': undefined}; @@ -59,6 +60,7 @@ function handleVideo(stream) { //console.log("Success! Device Name: " + stream.getVideoTracks()[0].label); video.srcObject = stream; + currentVideoStream = stream; } function videoError(err) { @@ -106,6 +108,12 @@ function fillVideoSourceList(selected_source=undefined){ function updateVideoSource(){ let select = document.getElementById('videoSelect'); if (select.selectedIndex>=0){ + + // Stop other camera tracks, otherwise, won't work on some devices + if (currentVideoStream){ + currentVideoStream.getVideoTracks()[0].stop(); + currentVideoStream = undefined; + } currentVideoSourceIndex = select.selectedIndex; currentConfig.currentVideoName = videoSources[currentVideoSourceIndex].label; if (typeof(localPTZCapabilities) !== 'undefined'){ From ed0878879c0b298e4afc13eb1e259c11b6ed43d2 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 11 Dec 2023 14:14:34 -0500 Subject: [PATCH 06/29] Added "with_sites" and "with_projects" to user/online_participants API to provide context. --- .../API/user/UserQueryOnlineParticipants.py | 20 +++++++--- .../user/test_UserQueryOnlineParticipants.py | 38 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/user/UserQueryOnlineParticipants.py b/teraserver/python/modules/FlaskModule/API/user/UserQueryOnlineParticipants.py index 60ddd6f69..c645852e9 100644 --- a/teraserver/python/modules/FlaskModule/API/user/UserQueryOnlineParticipants.py +++ b/teraserver/python/modules/FlaskModule/API/user/UserQueryOnlineParticipants.py @@ -1,5 +1,5 @@ from flask import session -from flask_restx import Resource +from flask_restx import Resource, inputs from flask_babel import gettext from modules.LoginModule.LoginModule import user_multi_auth, current_user from modules.FlaskModule.FlaskModule import user_api_ns as api @@ -11,6 +11,8 @@ from modules.DatabaseModule.DBManager import DBManager get_parser = api.parser() +get_parser.add_argument('with_sites', type=inputs.boolean, help='Include site informations for each participant.') +get_parser.add_argument('with_projects', type=inputs.boolean, help='Include project informations for each participant.') class UserQueryOnlineParticipants(Resource): @@ -45,10 +47,18 @@ def get(self): participants = TeraParticipant.query.filter(TeraParticipant.participant_uuid.in_( filtered_participants_uuids)).order_by(TeraParticipant.participant_name.asc()).all() - participants_json = [participant.to_json(minimal=True) for participant in participants] - for participant in participants_json: - participant['participant_online'] = status_participants[participant['participant_uuid']]['online'] - participant['participant_busy'] = status_participants[participant['participant_uuid']]['busy'] + participants_json = [] + for participant in participants: + part_json = participant.to_json(minimal=True) + if args['with_projects']: + part_json['id_project'] = participant.id_project + part_json['project_name'] = participant.participant_project.project_name + if args['with_sites']: + part_json['id_site'] = participant.participant_project.id_site + part_json['site_name'] = participant.participant_project.project_site.site_name + part_json['participant_online'] = status_participants[part_json['participant_uuid']]['online'] + part_json['participant_busy'] = status_participants[part_json['participant_uuid']]['busy'] + participants_json.append(part_json) return participants_json diff --git a/teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryOnlineParticipants.py b/teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryOnlineParticipants.py index 7d5db382c..1ccfd533c 100644 --- a/teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryOnlineParticipants.py +++ b/teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryOnlineParticipants.py @@ -27,3 +27,41 @@ def test_with_admin_auth(self): for participant_info in response.json: self.assertTrue('participant_online' in participant_info) self.assertTrue('participant_busy' in participant_info) + + def test_with_projects(self): + response = self._get_with_user_http_auth(self.test_client, username='admin', password='admin', + params={'with_projects': True}) + self.assertEqual(response.status_code, 200) + + # Check for important status fields + for participant_info in response.json: + self.assertTrue('participant_online' in participant_info) + self.assertTrue('participant_busy' in participant_info) + self.assertTrue('id_project' in participant_info) + self.assertTrue('project_name' in participant_info) + + def test_with_sites(self): + response = self._get_with_user_http_auth(self.test_client, username='admin', password='admin', + params={'with_sites': True}) + self.assertEqual(response.status_code, 200) + + # Check for important status fields + for participant_info in response.json: + self.assertTrue('participant_online' in participant_info) + self.assertTrue('participant_busy' in participant_info) + self.assertTrue('id_site' in participant_info) + self.assertTrue('site_name' in participant_info) + + def test_with_projects_and_sites(self): + response = self._get_with_user_http_auth(self.test_client, username='admin', password='admin', + params={'with_projects': True, 'with_sites': True}) + self.assertEqual(response.status_code, 200) + + # Check for important status fields + for participant_info in response.json: + self.assertTrue('participant_online' in participant_info) + self.assertTrue('participant_busy' in participant_info) + self.assertTrue('id_project' in participant_info) + self.assertTrue('project_name' in participant_info) + self.assertTrue('id_site' in participant_info) + self.assertTrue('site_name' in participant_info) From 488ad414c158b79a7fba8fbc24fa302f84cdeb62 Mon Sep 17 00:00:00 2001 From: mouhamb <127998244+mouhamb@users.noreply.github.com> Date: Wed, 17 Jan 2024 16:34:17 -0500 Subject: [PATCH 07/29] Add files via upload --- .../service/ServiceQueryParticipantGroups.py | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py diff --git a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py new file mode 100644 index 000000000..f9612f29c --- /dev/null +++ b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py @@ -0,0 +1,179 @@ +from flask import request +from flask_restx import Resource +from flask_babel import gettext +from sqlalchemy import exc +from sqlalchemy.exc import IntegrityError +from modules.LoginModule.LoginModule import LoginModule +from modules.FlaskModule.FlaskModule import service_api_ns as api +from opentera.db.models.TeraParticipantGroup import TeraParticipantGroup + +# Parser definition(s) +get_parser = api.parser() +get_parser.add_argument('id_participant_group', type=int, help='ID to query') + +post_parser = api.parser() + +participant_group_schema = api.schema_model('participant_group', { + 'properties': { + 'participant_group': { + 'type': 'object', + 'properties': { + 'id_participant_group': { + 'type': 'integer', + }, + + 'id_project': { + 'type': 'integer', + }, + + 'participant_group_name': { + 'type': 'string' + } + + }, + 'required': ['id_participant_group', 'id_project', 'participant_group_name'] + }, + + }, + 'type': 'object', + 'required': ['participant_group'] +}) + +delete_parser = api.parser() +delete_parser.add_argument('id', type=int, help='ID to delete') + + +class ServiceQueryParticipantGroups(Resource): + + # Handle auth + def __init__(self, _api, *args, **kwargs): + Resource.__init__(self, _api, *args, **kwargs) + self.module = kwargs.get('flaskModule', None) + self.test = kwargs.get('test', False) + + @api.doc(description='Return participant group information.', + responses={200: 'Success', + 500: 'Required parameter is missing', + 501: 'Not implemented.', + 403: 'Service doesn\'t have permission to access the requested data'}, + params={'token': 'Secret token'}) + @api.expect(get_parser) + @LoginModule.service_token_or_certificate_required + def get(self): + args = get_parser.parse_args() + + # Check if 'id_participant_group' is specified in args + if args['id_participant_group']: + # Retrieve participant group by ID + participant_group = TeraParticipantGroup.get_participant_group_by_id(args['id_participant_group']) + # If participant group is found, return its JSON representation + if participant_group: + return participant_group.to_json() + + # Return error message for missing arguments + return gettext('Missing arguments'), 400 + + @api.doc(description='Update participant group', + responses={200: 'Success - To be documented', + 500: 'Required parameter is missing', + 501: 'Not implemented.', + 403: 'Logged user doesn\'t have permission to access the requested data'}, + params={'token': 'Secret token'}) + @api.expect(participant_group_schema) + @LoginModule.service_token_or_certificate_required + def post(self): + # Parse arguments + args = post_parser.parse_args() + + # Check if 'participant_group' is present in the JSON request + if 'participant_group' not in request.json: + return gettext('Missing arguments'), 400 + + # Extract participant group information from the JSON request + participant_group_info = request.json['participant_group'] + + # Check if the project ID is valid + if participant_group_info['id_project'] < 1: + return gettext('Unknown project'), 403 + + # Check if the participant group name is provided + if not participant_group_info['participant_group_name']: + return gettext('Invalid participant group name'), 403 + + # Initialize participant group instance + participant_group: TeraParticipantGroup = TeraParticipantGroup() + + # Check if it's a new participant group or an update + if participant_group_info['id_participant_group'] == 0: + # Create participant group + participant_group = TeraParticipantGroup() + participant_group.participant_group_name = participant_group_info['participant_group_name'] + participant_group.id_project = participant_group_info['id_project'] + + try: + TeraParticipantGroup.insert(participant_group) + except IntegrityError as e: + self.module.logger.log_warning(self.module.module_name, ServiceQueryParticipantGroups.__name__, + 'insert', + 400, 'Integrity error', str(e)) + + # Handle integrity error related to projects + if 't_projects' in str(e.args): + return gettext('Can\'t insert participant group: participant group\'s project ' + 'is disabled or invalid.'), 400 + else: + # Update existing participant group + try: + TeraParticipantGroup.update(participant_group_info['id_participant_group'], participant_group_info) + except IntegrityError as e: + self.module.logger.log_warning(self.module.module_name, ServiceQueryParticipantGroups.__name__, + 'update', + 400, 'Integrity error', str(e)) + + # Handle integrity error related to projects + if 't_projects' in str(e.args): + return gettext('Can\'t update participant group: participant group\'s project is disabled.'), 400 + + # Retrieve the updated participant group + participant_group = TeraParticipantGroup.get_participant_group_by_id(participant_group_info + ['id_participant_group']) + + # Return the JSON representation of the participant group + return participant_group.to_json(minimal=False) + + + + @api.doc(description='Delete a specific participant group.', + responses={200: 'Success', + 403: 'Logged user doesn\'t have permission to access the requested data', + 500: 'Database error.'}, + params={'token': 'Secret token'}) + @api.expect(delete_parser) + @LoginModule.service_token_or_certificate_required + def delete(self): + args = delete_parser.parse_args() + id_todel = args['id'] + + + # Check deletion integrity + group_to_del = TeraParticipantGroup.get_participant_group_by_id(id_todel) + if group_to_del is None: + return gettext('The id_participant_group given was not found'), + + deletion_integrity = group_to_del.delete_check_integrity() + + if deletion_integrity is not None: + return gettext('Deletion impossible: Participant group still has participant(s)') + + # If we are here, we are allowed to delete. Do so. + try: + TeraParticipantGroup.delete(id_todel=id_todel) + except exc.SQLAlchemyError as e: + import sys + print(sys.exc_info()) + self.module.logger.log_error(self.module.module_name, + ServiceQueryParticipantGroups.__name__, + 'delete', 500, 'Database error', str(e)) + return gettext('Database error'), 500 + + return '', 200 \ No newline at end of file From 6deb66c92dff0a2241e22ac0ca1b53968519b607 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Thu, 18 Jan 2024 11:02:36 -0500 Subject: [PATCH 08/29] Added test mode for LoggingService and test API to reset database. --- teraserver/python/TeraServer.py | 2 +- .../modules/DatabaseModule/DBManager.py | 14 +++++++++++ .../DatabaseModule/DBManagerTeraUserAccess.py | 2 +- .../FlaskModule/API/test/TestDBReset.py | 25 +++++++++++++++++++ .../modules/FlaskModule/API/test/__init__.py | 0 .../FlaskModule/API/user/UserQueryVersions.py | 7 ++++-- .../python/modules/FlaskModule/FlaskModule.py | 15 ++++++++++- .../ServiceLauncherModule.py | 6 +++-- .../FileTransferService.py | 2 +- .../libfiletransferservice/db/DBManager.py | 2 +- .../services/LoggingService/LoggingService.py | 25 +++++++++++++------ .../libloggingservice/db/DBManager.py | 16 +++++++++--- 12 files changed, 96 insertions(+), 20 deletions(-) create mode 100644 teraserver/python/modules/FlaskModule/API/test/TestDBReset.py create mode 100644 teraserver/python/modules/FlaskModule/API/test/__init__.py diff --git a/teraserver/python/TeraServer.py b/teraserver/python/TeraServer.py index 2e921cf7f..adc519fb8 100755 --- a/teraserver/python/TeraServer.py +++ b/teraserver/python/TeraServer.py @@ -168,7 +168,7 @@ def init_opentera_service(config: ConfigManager): init_opentera_service(config=config_man) # Main Flask module - flask_module = FlaskModule(config_man) + flask_module = FlaskModule(config_man, test_mode=args.enable_tests) # LOGIN MANAGER, must be initialized after Flask ################################################# diff --git a/teraserver/python/modules/DatabaseModule/DBManager.py b/teraserver/python/modules/DatabaseModule/DBManager.py index 02cb8e5db..db26b2858 100755 --- a/teraserver/python/modules/DatabaseModule/DBManager.py +++ b/teraserver/python/modules/DatabaseModule/DBManager.py @@ -78,6 +78,7 @@ def __init__(self, config: ConfigManager, app=flask_app): self.db = SQLAlchemy(engine_options={'future': True}, session_options={'future': True}) self.db_uri = None self.app = app + self.db_in_ram = False # Database cleanup task set to run at next midnight self.cleanup_database_task = self.start_cleanup_task() @@ -319,6 +320,7 @@ def open_local(self, db_infos, echo=False, ram=True): # IN RAM if ram: self.db_uri = 'sqlite://' + self.db_in_ram = True else: self.db_uri = 'sqlite:///%(filename)s' % db_infos @@ -400,6 +402,18 @@ def stamp_db(self): # Stamp database command.stamp(config, revision, sql, tag) + def reset_db(self): + if not self.db_in_ram: + return # Safety: only possible to reset a db if database is in RAM! + BaseModel.metadata.drop_all(self.db.engine.connect()) + BaseModel.create_all() + self.create_defaults(self.config, True) + + # Set versions + from opentera.utils.TeraVersions import TeraVersions + versions = TeraVersions() + versions.save_to_db() + def cleanup_database(self): print("Cleaning up database...") # Updating session states diff --git a/teraserver/python/modules/DatabaseModule/DBManagerTeraUserAccess.py b/teraserver/python/modules/DatabaseModule/DBManagerTeraUserAccess.py index dd795fd28..4c8a82462 100644 --- a/teraserver/python/modules/DatabaseModule/DBManagerTeraUserAccess.py +++ b/teraserver/python/modules/DatabaseModule/DBManagerTeraUserAccess.py @@ -284,7 +284,7 @@ def get_accessible_session_types(self, admin_only=False): site_id_list = self.get_accessible_sites_ids(admin_only=admin_only) query = TeraSessionType.query.join(TeraSessionTypeSite)\ - .filter(TeraSessionTypeSite.id_site.in_(site_id_list)) + .filter(TeraSessionTypeSite.id_site.in_(site_id_list)).order_by(TeraSessionType.session_type_name.asc()) return query.all() def get_accessible_session_types_ids(self, admin_only=False): diff --git a/teraserver/python/modules/FlaskModule/API/test/TestDBReset.py b/teraserver/python/modules/FlaskModule/API/test/TestDBReset.py new file mode 100644 index 000000000..dffd0b372 --- /dev/null +++ b/teraserver/python/modules/FlaskModule/API/test/TestDBReset.py @@ -0,0 +1,25 @@ +from flask_restx import Resource + +from modules.FlaskModule.FlaskModule import test_api_ns as api +import modules.Globals +import json + + +# Parser definition(s) +# GET +get_parser = api.parser() + + +class TestDBReset(Resource): + + def __init__(self, _api, *args, **kwargs): + Resource.__init__(self, _api, *args, **kwargs) + self.module = kwargs.get('flaskModule', None) + self.test = kwargs.get('test', False) + + @api.doc(description='Reset database to default values', + responses={200: 'Success'}) + @api.expect(get_parser) + def get(self): + modules.Globals.db_man.reset_db() + return 200 diff --git a/teraserver/python/modules/FlaskModule/API/test/__init__.py b/teraserver/python/modules/FlaskModule/API/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py b/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py index 43407388f..f2c77e139 100644 --- a/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py +++ b/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py @@ -36,8 +36,11 @@ def get(self): # As soon as we are authorized, we can output the server versions args = get_parser.parse_args() - current_settings = json.loads(TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerVersions)) - return current_settings + current_settings = TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerVersions) + if not current_settings: + return gettext('No version information found'), 500 + + return json.loads(current_settings) @api.doc(description='Post server versions', responses={200: 'Success - asset posted', diff --git a/teraserver/python/modules/FlaskModule/FlaskModule.py b/teraserver/python/modules/FlaskModule/FlaskModule.py index d2da7003c..3d31856fb 100755 --- a/teraserver/python/modules/FlaskModule/FlaskModule.py +++ b/teraserver/python/modules/FlaskModule/FlaskModule.py @@ -73,16 +73,18 @@ def specs_url(self): device_api_ns = api.namespace('device', description='API for device calls') participant_api_ns = api.namespace('participant', description='API for participant calls') service_api_ns = api.namespace('service', description='API for service calls') +test_api_ns = api.namespace('test', description='API for tests') class FlaskModule(BaseModule): - def __init__(self, config: ConfigManager): + def __init__(self, config: ConfigManager, test_mode=False): BaseModule.__init__(self, ModuleNames.FLASK_MODULE_NAME.value, config) # Use debug mode flag flask_app.debug = config.server_config['debug_mode'] + self.test_mode = test_mode # Change secret key to use server UUID # This is used for session encryption @@ -110,6 +112,8 @@ def __init__(self, config: ConfigManager): FlaskModule.init_device_api(self, device_api_ns) FlaskModule.init_participant_api(self, participant_api_ns) FlaskModule.init_service_api(self, service_api_ns) + if self.test_mode: + FlaskModule.init_test_api(self, test_api_ns) # Init Views self.init_views() @@ -329,6 +333,15 @@ def init_service_api(module: object, namespace: Namespace, additional_args: dict namespace.add_resource(ServiceQueryUsers, '/users', resource_class_kwargs=kwargs) namespace.add_resource(ServiceQuerySiteProjectAccessRoles, '/users/access', resource_class_kwargs=kwargs) + @staticmethod + def init_test_api(module: object, namespace: Namespace, additional_args: dict = dict()): + # Default arguments + kwargs = {'flaskModule': module} + kwargs |= additional_args + + from modules.FlaskModule.API.test.TestDBReset import TestDBReset + namespace.add_resource(TestDBReset, '/database/reset', resource_class_kwargs=kwargs) + def init_views(self): from modules.FlaskModule.Views.About import About from modules.FlaskModule.Views.DisabledDoc import DisabledDoc diff --git a/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py b/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py index 0da8b5959..eb05b36c1 100644 --- a/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py +++ b/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py @@ -125,8 +125,6 @@ def launch_service(self, service: TeraService): elif service.service_key == 'FileTransferService': path = os.path.join(os.getcwd(), 'services', 'FileTransferService', 'FileTransferService.py') executable_args.append(path) - if self.enable_tests: - executable_args.append('--enable_tests=1') working_directory = os.path.join(os.getcwd(), 'services', 'FileTransferService') # elif service.service_key == 'BureauActif': # path = os.path.join(os.getcwd(), 'services', 'BureauActif', 'BureauActifService.py') @@ -141,6 +139,10 @@ def launch_service(self, service: TeraService): self.logger.log_error(self.module_name, 'Unable to start', service.service_key) return + # Append test mode argument to all launched services + if self.enable_tests: + executable_args.append('--enable_tests=1') + # Start process process = subprocess.Popen(executable_args, cwd=os.path.realpath(working_directory)) process_dict = { diff --git a/teraserver/python/services/FileTransferService/FileTransferService.py b/teraserver/python/services/FileTransferService/FileTransferService.py index 5d890034f..df433dadf 100644 --- a/teraserver/python/services/FileTransferService/FileTransferService.py +++ b/teraserver/python/services/FileTransferService/FileTransferService.py @@ -111,7 +111,7 @@ def asset_event_received(self, event: messages.DatabaseEvent): try: if args.enable_tests: - Globals.db_man.open_local(None, echo=True) + Globals.db_man.open_local(echo=True) else: Globals.db_man.open(POSTGRES, Globals.config_man.service_config['debug_mode']) except OperationalError as e: diff --git a/teraserver/python/services/FileTransferService/libfiletransferservice/db/DBManager.py b/teraserver/python/services/FileTransferService/libfiletransferservice/db/DBManager.py index 230ebaad1..554f44ef4 100644 --- a/teraserver/python/services/FileTransferService/libfiletransferservice/db/DBManager.py +++ b/teraserver/python/services/FileTransferService/libfiletransferservice/db/DBManager.py @@ -56,7 +56,7 @@ def open(self, db_infos, echo=False): # Apply any database upgrade, if needed self.upgrade_db() - def open_local(self, db_infos, echo=False): + def open_local(self, echo=False): self.db_uri = 'sqlite://' flask_app.config.update({ diff --git a/teraserver/python/services/LoggingService/LoggingService.py b/teraserver/python/services/LoggingService/LoggingService.py index 24e9db800..45325c8a3 100644 --- a/teraserver/python/services/LoggingService/LoggingService.py +++ b/teraserver/python/services/LoggingService/LoggingService.py @@ -123,6 +123,12 @@ def set_loglevel(self, loglevel): print('Invalid config') exit(1) + import argparse + + parser = argparse.ArgumentParser(description='Logging Service') + parser.add_argument('--enable_tests', help='Test mode for service.', default=False) + args = parser.parse_args() + # Global redis client Globals.redis_client = RedisClient(Globals.config_man.redis_config) Globals.api_user_token_key = Globals.redis_client.redisGet(RedisVars.RedisVar_UserTokenAPIKey) @@ -171,14 +177,19 @@ def set_loglevel(self, loglevel): 'port': Globals.config_man.db_config['port'] } - try: - Globals.db_man.open(POSTGRES, Globals.config_man.service_config['debug_mode']) - except OperationalError as e: - print("Unable to connect to database - please check settings in config file!", e) - quit() + Globals.db_man.test = args.enable_tests + + if not args.enable_tests: + try: + Globals.db_man.open(POSTGRES, Globals.config_man.service_config['debug_mode']) + except OperationalError as e: + print("Unable to connect to database - please check settings in config file!", e) + quit() + + with flask_app.app_context(): + Globals.db_man.create_defaults(Globals.config_man) - with flask_app.app_context(): - Globals.db_man.create_defaults(Globals.config_man) + # In test mode, db manager will not save anything into a database # Create the Service Globals.service = LoggingService(Globals.config_man, service_info) diff --git a/teraserver/python/services/LoggingService/libloggingservice/db/DBManager.py b/teraserver/python/services/LoggingService/libloggingservice/db/DBManager.py index 5f8951a22..e27643adb 100644 --- a/teraserver/python/services/LoggingService/libloggingservice/db/DBManager.py +++ b/teraserver/python/services/LoggingService/libloggingservice/db/DBManager.py @@ -168,8 +168,12 @@ def store_log_event(self, event: LogEvent): entry.sender = event.sender entry.timestamp = datetime.datetime.fromtimestamp(event.timestamp) entry.message = event.message - self.db.session.add(entry) - self.db.session.commit() + if not self.test: + self.db.session.add(entry) + self.db.session.commit() + else: + import json + print('Logging: ' + json.dumps(entry.to_json())) def store_login_event(self, event: LoginEvent): with self.app.app_context(): @@ -190,8 +194,12 @@ def store_login_event(self, event: LoginEvent): entry.login_os_name = event.os_name entry.login_os_version = event.os_version entry.login_message = event.log_header.message - self.db.session.add(entry) - self.db.session.commit() + if not self.test: + self.db.session.add(entry) + self.db.session.commit() + else: + import json + print('Logging - Login: ' + json.dumps(entry.to_json())) # Fix foreign_keys on sqlite From 0d5288a0a6abb53ced500d3261cd341fc9ebffe3 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Thu, 18 Jan 2024 13:13:09 -0500 Subject: [PATCH 09/29] Removed check for valid Client-Name in UserLogin --- teraserver/python/modules/FlaskModule/API/user/UserLogin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/user/UserLogin.py b/teraserver/python/modules/FlaskModule/API/user/UserLogin.py index 523f794c4..596c2ca89 100644 --- a/teraserver/python/modules/FlaskModule/API/user/UserLogin.py +++ b/teraserver/python/modules/FlaskModule/API/user/UserLogin.py @@ -62,7 +62,7 @@ def get(self): if current_user.user_uuid not in online_users: websocket_url = "wss://" + servername + ":" + str(port) + "/wss/user?id=" + session['_id'] - print('Login - setting key with expiration in 60s', session['_id'], session['_user_id']) + # print('Login - setting key with expiration in 60s', session['_id'], session['_user_id']) self.module.redisSet(session['_id'], session['_user_id'], ex=60) elif args['with_websocket']: # User is online and a websocket is required @@ -134,8 +134,8 @@ def get(self): message=gettext('Client version mismatch')) return gettext('Client major version too old, not accepting login'), 426 - else: - return gettext('Invalid client name :') + client_name, 403 + # else: + # return gettext('Invalid client name :') + client_name, 403 except BaseException as e: self.module.logger.log_error(self.module.module_name, UserLogin.__name__, From 3c2f9c3bd4625fd21fc2f6e7f24137d07239a4e6 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Tue, 23 Jan 2024 08:29:38 -0500 Subject: [PATCH 10/29] Refs #237. Added Device Registration key in server settings. --- .gitignore | 2 ++ .../e6ee93ef205b_add_device_register_key.py | 27 +++++++++++++++++++ .../opentera/db/models/TeraServerSettings.py | 7 +++-- 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py diff --git a/.gitignore b/.gitignore index 6a58469e2..eaeec4463 100644 --- a/.gitignore +++ b/.gitignore @@ -80,3 +80,5 @@ joss-paper/paper.pdf python-3.11 build-teraserver-Desktop_Qt_6_6_1_MSVC2019_64bit-Debug python-3.10 +/teraserver/python/config/certificates +/teraserver/python/tests/*.pem diff --git a/teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py b/teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py new file mode 100644 index 000000000..6b7651815 --- /dev/null +++ b/teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py @@ -0,0 +1,27 @@ +"""Add device register key + +Revision ID: e6ee93ef205b +Revises: f41b70d6513e +Create Date: 2024-01-23 08:15:07.224075 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy import orm +from opentera.db.models.TeraServerSettings import TeraServerSettings + + +# revision identifiers, used by Alembic. +revision = 'e6ee93ef205b' +down_revision = 'f41b70d6513e' +branch_labels = None +depends_on = None + + +def upgrade(): + TeraServerSettings.set_server_setting(TeraServerSettings.ServerDeviceRegisterKey, + TeraServerSettings.generate_token_key(10)) + + +def downgrade(): + pass diff --git a/teraserver/python/opentera/db/models/TeraServerSettings.py b/teraserver/python/opentera/db/models/TeraServerSettings.py index aece9a98e..bc9d90923 100644 --- a/teraserver/python/opentera/db/models/TeraServerSettings.py +++ b/teraserver/python/opentera/db/models/TeraServerSettings.py @@ -18,6 +18,7 @@ class TeraServerSettings(BaseModel): ServerParticipantTokenKey = "ParticipantTokenEncryptionKey" ServerUUID = "ServerUUID" ServerVersions = "ServerVersions" + ServerDeviceRegisterKey = "DeviceRegisterKey" @staticmethod def create_defaults(test=False): @@ -29,6 +30,9 @@ def create_defaults(test=False): TeraServerSettings.set_server_setting(TeraServerSettings.ServerParticipantTokenKey, TeraServerSettings.generate_token_key(32)) + TeraServerSettings.set_server_setting(TeraServerSettings.ServerDeviceRegisterKey, + TeraServerSettings.generate_token_key(10)) + # Unique server id server_uuid = str(uuid.uuid4()) TeraServerSettings.set_server_setting(TeraServerSettings.ServerUUID, server_uuid) @@ -49,8 +53,7 @@ def get_server_setting_value(setting_name: string): @staticmethod def get_server_setting(setting_name: string): - return TeraServerSettings.query.filter_by( - server_settings_name=setting_name).first() + return TeraServerSettings.query.filter_by(server_settings_name=setting_name).first() @staticmethod def set_server_setting(setting_name: string, setting_value: string): From 84789ab00add1b04a58b0204d39e9e6d4c365161 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Tue, 23 Jan 2024 08:49:50 -0500 Subject: [PATCH 11/29] Updated python env to latest packages version --- .../e6ee93ef205b_add_device_register_key.py | 3 -- teraserver/python/env/requirements.txt | 29 +++++++++---------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py b/teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py index 6b7651815..5db162f1a 100644 --- a/teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py +++ b/teraserver/python/alembic/versions/e6ee93ef205b_add_device_register_key.py @@ -5,9 +5,6 @@ Create Date: 2024-01-23 08:15:07.224075 """ -from alembic import op -import sqlalchemy as sa -from sqlalchemy import orm from opentera.db.models.TeraServerSettings import TeraServerSettings diff --git a/teraserver/python/env/requirements.txt b/teraserver/python/env/requirements.txt index f08148bbc..29f5bfdf1 100644 --- a/teraserver/python/env/requirements.txt +++ b/teraserver/python/env/requirements.txt @@ -1,20 +1,20 @@ pypiwin32==223; sys_platform == 'win32' Twisted==23.10.0 treq==23.11.0 -cryptography==41.0.7 +cryptography==42.0.0 autobahn==23.6.2 -SQLAlchemy==2.0.23 +SQLAlchemy==2.0.25 sqlalchemy-schemadisplay==1.3 -pydot==1.4.2 +pydot==2.0.0 psycopg2-binary==2.9.9 -Flask==2.3.3 +Flask==3.0.1 Flask-SQLAlchemy==3.1.1 Flask-Login==0.6.3 Flask-Login-Multi==0.1.2 Flask-HTTPAuth==4.8.0 Flask-SocketIO==5.3.6 -Flask-Session==0.5.0 -flask-restx==1.2.0 +Flask-Session==0.6.0 +flask-restx==1.3.0 Flask-Security==3.0.0 Flask-Babel==4.0.0 Flask-BabelEx==0.9.4 @@ -26,19 +26,16 @@ Flask-Principal==0.4.0 redis==5.0.1 txredisapi==1.4.10 passlib==1.7.4 -bcrypt==4.1.1 -WTForms==3.1.1 -pyOpenSSL==23.3.0 -service-identity==23.1.0 +bcrypt==4.1.2 +WTForms==3.1.2 +pyOpenSSL==24.0.0 +service-identity==24.1.0 PyJWT==2.8.0 pylzma==0.5.0 bz2file==0.98 python-slugify==8.0.1 websocket-client==1.7.0 -pytest==7.4.3 -# Hold jsonschema until flask-restx gets updated -jsonschema==4.17.3 -Jinja2==3.1.2 +pytest==7.4.4 +Jinja2==3.1.3 ua-parser==0.18.0 -#Remove this when Flask-Login / flask-restx is updated with latest Werkseug 3.x.x -Werkzeug==2.3.8 + From 4b51d49c906de81a7b49cda14614404dabf8fe84 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Tue, 23 Jan 2024 09:52:56 -0500 Subject: [PATCH 12/29] Refs #237. Added User API to query server settings --- .../API/user/UserQueryServerSettings.py | 38 ++++++++++ .../FlaskModule/API/user/UserQueryVersions.py | 5 +- .../python/modules/FlaskModule/FlaskModule.py | 2 + .../API/user/test_UserQueryServerSettings.py | 71 +++++++++++++++++++ 4 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 teraserver/python/modules/FlaskModule/API/user/UserQueryServerSettings.py create mode 100644 teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryServerSettings.py diff --git a/teraserver/python/modules/FlaskModule/API/user/UserQueryServerSettings.py b/teraserver/python/modules/FlaskModule/API/user/UserQueryServerSettings.py new file mode 100644 index 000000000..ac10146cf --- /dev/null +++ b/teraserver/python/modules/FlaskModule/API/user/UserQueryServerSettings.py @@ -0,0 +1,38 @@ +from flask_restx import Resource, inputs +from modules.LoginModule.LoginModule import user_multi_auth +from modules.FlaskModule.FlaskModule import user_api_ns as api +from opentera.db.models.TeraServerSettings import TeraServerSettings + +# Parser definition(s) +# GET +get_parser = api.parser() +get_parser.add_argument('uuid', type=inputs.boolean, help='Get server UUID') +get_parser.add_argument('device_register_key', type=inputs.boolean, help='Get device registration key') + + +class UserQueryServerSettings(Resource): + + def __init__(self, _api, *args, **kwargs): + Resource.__init__(self, _api, *args, **kwargs) + self.module = kwargs.get('flaskModule', None) + self.test = kwargs.get('test', False) + + @api.doc(description='Get server setting key', + responses={200: 'Success - returns setting value', + 401: 'Logged user doesn\'t have permission to access the requested data'}, + params={'token': 'Secret token'}) + @api.expect(get_parser) + @user_multi_auth.login_required + def get(self): + # As soon as we are authorized, we can output the server versions + args = get_parser.parse_args() + + settings = {} + if args['uuid']: + settings |= {'server_uuid': TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerUUID)} + + if args['device_register_key']: + settings |= {'device_register_key': + TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerDeviceRegisterKey)} + + return settings diff --git a/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py b/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py index f2c77e139..121522475 100644 --- a/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py +++ b/teraserver/python/modules/FlaskModule/API/user/UserQueryVersions.py @@ -1,4 +1,4 @@ -from flask import session, request +from flask import request from flask_restx import Resource from flask_babel import gettext from modules.LoginModule.LoginModule import user_multi_auth, current_user @@ -6,7 +6,6 @@ from opentera.db.models.TeraServerSettings import TeraServerSettings from opentera.utils.TeraVersions import TeraVersions, ClientVersions import json -from opentera.db.models.TeraUser import TeraUser # Parser definition(s) # GET @@ -26,7 +25,7 @@ def __init__(self, _api, *args, **kwargs): self.test = kwargs.get('test', False) @api.doc(description='Get server versions', - responses={200: 'Success - returns list of assets', + responses={200: 'Success - returns versions information', 400: 'Required parameter is missing', 403: 'Logged user doesn\'t have permission to access the requested data'}, params={'token': 'Secret token'}) diff --git a/teraserver/python/modules/FlaskModule/FlaskModule.py b/teraserver/python/modules/FlaskModule/FlaskModule.py index 3d31856fb..e99746f31 100755 --- a/teraserver/python/modules/FlaskModule/FlaskModule.py +++ b/teraserver/python/modules/FlaskModule/FlaskModule.py @@ -180,6 +180,7 @@ def init_user_api(module: object, namespace: Namespace, additional_args: dict = from modules.FlaskModule.API.user.UserQueryTestType import UserQueryTestTypes from modules.FlaskModule.API.user.UserQueryTests import UserQueryTests from modules.FlaskModule.API.user.UserQueryDisconnect import UserQueryDisconnect + from modules.FlaskModule.API.user.UserQueryServerSettings import UserQueryServerSettings from modules.FlaskModule.API.user.UserQueryUndelete import UserQueryUndelete # Resources @@ -210,6 +211,7 @@ def init_user_api(module: object, namespace: Namespace, additional_args: dict = namespace.add_resource(UserQuerySessionTypeProjects, '/sessiontypes/projects', resource_class_kwargs=kwargs) namespace.add_resource(UserQuerySessionTypeSites, '/sessiontypes/sites', resource_class_kwargs=kwargs) namespace.add_resource(UserQuerySessionEvents, '/sessions/events', resource_class_kwargs=kwargs) + namespace.add_resource(UserQueryServerSettings, '/server/settings', resource_class_kwargs=kwargs) namespace.add_resource(UserQueryServices, '/services', resource_class_kwargs=kwargs) namespace.add_resource(UserQueryServiceProjects, '/services/projects', resource_class_kwargs=kwargs) namespace.add_resource(UserQueryServiceSites, '/services/sites', resource_class_kwargs=kwargs) diff --git a/teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryServerSettings.py b/teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryServerSettings.py new file mode 100644 index 000000000..fb2fcb8b7 --- /dev/null +++ b/teraserver/python/tests/modules/FlaskModule/API/user/test_UserQueryServerSettings.py @@ -0,0 +1,71 @@ +from BaseUserAPITest import BaseUserAPITest +from opentera.db.models.TeraServerSettings import TeraServerSettings + + +class UserQueryServerSettingsTest(BaseUserAPITest): + test_endpoint = '/api/user/server/settings' + + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + + def test_no_auth(self): + with self._flask_app.app_context(): + response = self.test_client.get(self.test_endpoint) + self.assertEqual(401, response.status_code) + + def test_get_endpoint_invalid_http_auth(self): + with self._flask_app.app_context(): + response = self._get_with_user_http_auth(self.test_client, username='invalid', password='invalid') + self.assertEqual(401, response.status_code) + + def test_get_endpoint_invalid_token_auth(self): + with self._flask_app.app_context(): + response = self._get_with_user_token_auth(self.test_client, token='invalid') + self.assertEqual(401, response.status_code) + + def test_post(self): + with self._flask_app.app_context(): + response = self.test_client.post(self.test_endpoint) + self.assertEqual(405, response.status_code) + + def test_delete(self): + with self._flask_app.app_context(): + response = self.test_client.delete(self.test_endpoint) + self.assertEqual(405, response.status_code) + + def test_get_server_uuid(self): + with self._flask_app.app_context(): + response = self._get_with_user_http_auth(self.test_client, 'user', 'user', {'uuid': True}) + self.assertEqual(200, response.status_code) + self.assertTrue('server_uuid' in response.json) + server_uuid = TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerUUID) + self.assertEqual(server_uuid, response.json['server_uuid']) + + def test_get_device_register_key(self): + with self._flask_app.app_context(): + response = self._get_with_user_http_auth(self.test_client, 'user3', 'user3', {'device_register_key': True}) + self.assertEqual(200, response.status_code) + self.assertTrue('device_register_key' in response.json) + key = TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerDeviceRegisterKey) + self.assertEqual(key, response.json['device_register_key']) + + def test_get_nothing(self): + with self._flask_app.app_context(): + response = self._get_with_user_http_auth(self.test_client, 'siteadmin', 'siteadmin') + self.assertEqual(200, response.status_code) + self.assertTrue(len(response.json) == 0) + + def test_get_server_uuid_and_device_register_key(self): + with self._flask_app.app_context(): + response = self._get_with_user_http_auth(self.test_client, 'admin', 'admin', {'uuid': True, + 'device_register_key': True}) + self.assertEqual(200, response.status_code) + self.assertTrue('server_uuid' in response.json) + self.assertTrue('device_register_key' in response.json) + server_uuid = TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerUUID) + self.assertEqual(server_uuid, response.json['server_uuid']) + key = TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerDeviceRegisterKey) + self.assertEqual(key, response.json['device_register_key']) From d9f1f895efcc2d6e02af8b928c9a524774035ccc Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Tue, 23 Jan 2024 11:55:01 -0500 Subject: [PATCH 13/29] Refs #237. Revised device register API. --- .../FlaskModule/API/device/DeviceRegister.py | 210 +++++++------- .../API/user/UserQueryDeviceSubTypes.py | 7 +- .../python/opentera/db/models/TeraDevice.py | 3 +- .../opentera/db/models/TeraDeviceSubType.py | 5 +- .../API/device/BaseDeviceAPITest.py | 7 + .../API/device/test_DeviceRegister.py | 264 ++++++++++++++---- 6 files changed, 334 insertions(+), 162 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/device/DeviceRegister.py b/teraserver/python/modules/FlaskModule/API/device/DeviceRegister.py index 8dbe3efde..7b40596c9 100644 --- a/teraserver/python/modules/FlaskModule/API/device/DeviceRegister.py +++ b/teraserver/python/modules/FlaskModule/API/device/DeviceRegister.py @@ -1,26 +1,32 @@ -from flask_restx import Resource, reqparse +from flask_restx import Resource, inputs from flask_babel import gettext -from flask import jsonify from flask import request from flask_limiter import Limiter from flask_limiter.util import get_remote_address -import base64 from opentera.crypto.crypto_utils import generate_device_certificate, load_private_pem_key, load_pem_certificate + from cryptography import x509 -from cryptography.x509.oid import NameOID from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives import serialization from opentera.db.models.TeraDevice import TeraDevice from opentera.db.models.TeraDeviceType import TeraDeviceType -from opentera.db.models.TeraSessionType import TeraSessionType +from opentera.db.models.TeraDeviceSubType import TeraDeviceSubType +from opentera.db.models.TeraServerSettings import TeraServerSettings from modules.FlaskModule.FlaskModule import device_api_ns as api -import uuid + from modules.FlaskModule.FlaskModule import flask_app -from sqlalchemy.exc import SQLAlchemyError +import uuid limiter = Limiter(get_remote_address, app=flask_app, storage_uri="memory://") +api_parser = api.parser() +api_parser.add_argument('key', type=str, required=True, help='Server device registration key') +api_parser.add_argument('name', type=str, required=True, help='Device name to use') +api_parser.add_argument('type_key', type=str, required=True, help='Device type key to use') +api_parser.add_argument('subtype_name', type=str, help='Device subtype name to use') +api_parser.add_argument('onlineable', type=inputs.boolean, help='Device can get online status') + class DeviceRegister(Resource): """ @@ -53,108 +59,112 @@ def __init__(self, _api, *args, **kwargs): self.ca_info['private_key'] = info['private_key'] self.ca_info['certificate'] = info['certificate'] - def create_device(self, name, device_json=None): - # Create TeraDevice - device = TeraDevice() - - if device_json: - device.from_json(device_json) - else: - # Name should be taken from CSR or JSON request - device.device_name = name - # TODO set flags properly - device.device_onlineable = False - # TODO FORCING 'capteur' as default? - device.id_device_type = TeraDeviceType.get_device_type_by_key('capteur').id_device_type - - # Force disabled by default - device.device_enabled = False - - return device - - @api.doc(description='Register a device with certificate or token request. This endpoint is rate limited. ' - 'Use application/octet-stream to send CSR or application/json Content-Type for token ' - 'generation.', - responses={200: 'Success, will return registration information. Devices must then be enabled by admin.', - 400: 'Missing parameter(s)', - 500: 'Internal server error'}) + @api.doc(description='Register a device to use token identification. This endpoint is rate limited. If the device ' + 'type key doesn\'t exist, a new one will be created. Same behavior for subtype name.', + reponses={200: 'Success - returns registration information. Devices must then be enabled by admin.', + 400: 'Missing or invalid parameter', + 401: 'Unauthorized - provided registration key is invalid'}) + @api.expect(api_parser) + def get(self): + args = api_parser.parse_args(strict=True) + + # Check if provided registration key is ok + if args['key'] != TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerDeviceRegisterKey): + return gettext('Invalid registration key'), 401 + + new_device = self.get_new_device(args) + TeraDevice.insert(new_device) + + device_json = new_device.to_json(minimal=True, ignore_fields=['id_device', 'id_device_type', + 'id_device_subtype']) + device_json['device_token'] = new_device.device_token + + self.module.logger.log_info(self.module.module_name, DeviceRegister.__name__, + 'post', 'Device registered (token)', + new_device.device_name + '(' + new_device.device_uuid + ')') + + return device_json + + @api.doc(description='Register a device with certificate request. This endpoint is rate limited. If the device ' + 'type key doesn\'t exist, a new one will be created. Same behavior for subtype name.' + 'Use application/octet-stream to send CSR.', + responses={200: 'Success - returns registration information. Devices must then be enabled by admin.', + 400: 'Missing or invalid parameter', + 401: 'Unauthorized - provided registration key is invalid'}) def post(self): - # We should receive a certificate signing request (base64) in an octet-stream - if request.content_type == 'application/octet-stream': - # try: - # Read certificate request - req = x509.load_pem_x509_csr(request.data, default_backend()) - - if req.is_signature_valid: + args = api_parser.parse_args(strict=True) - # Name should be taken from CSR - device = self.create_device(str(req.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value)) + # Check if provided registration key is ok + if args['key'] != TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerDeviceRegisterKey): + return gettext('Invalid registration key'), 401 - # Must sign request with CA/key and generate certificate - cert = generate_device_certificate(req, self.ca_info, device.device_uuid) - - # Update certificate - device.device_certificate = cert.public_bytes(serialization.Encoding.PEM).decode('utf-8') - - # Store - TeraDevice.insert(device) - - result = dict() - result['certificate'] = device.device_certificate - result['ca_info'] = self.ca_info['certificate'].public_bytes(serialization.Encoding.PEM).decode('utf-8') - result['token'] = device.device_token - - self.module.logger.log_info(self.module.module_name, DeviceRegister.__name__, - 'post', 'Device registered (certificate)', - device.device_uuid, result['certificate']) - - # Return certificate... - return jsonify(result) - else: - self.module.logger.log_error(self.module.module_name, - DeviceRegister.__name__, - 'post', 400, 'Invalid CSR signature', request.data) - - return gettext('Invalid CSR signature'), 400 - # except: - # return 'Error processing request', 400 - - elif request.content_type == 'application/json': - - if 'device_info' not in request.json: - return gettext('Invalid content type'), 400 + # We should receive a certificate signing request (base64) in an octet-stream + if request.content_type != 'application/octet-stream': + return gettext('Invalid content type'), 400 - device_info = request.json['device_info'] + # Read certificate request + req = x509.load_pem_x509_csr(request.data, default_backend()) - # Check if we have device name - if 'device_name' not in device_info: - return gettext('Invalid content type'), 400 + if req.is_signature_valid: - if 'id_device_type' not in device_info: - return gettext('Invalid content type'), 400 + new_device = self.get_new_device(args) + new_device.device_uuid = str(uuid.uuid4()) # Device uuid is required to generate certificate - try: - device_name = device_info['device_name'] - device = self.create_device(device_name, device_info) + # Must sign request with CA/key and generate certificate + cert = generate_device_certificate(req, self.ca_info, new_device.device_uuid) - # Store - TeraDevice.insert(device) + # Update certificate + new_device.device_certificate = cert.public_bytes(serialization.Encoding.PEM).decode('utf-8') - result = dict() - result['token'] = device.device_token + # Store + TeraDevice.insert(new_device) - self.module.logger.log_info(self.module.module_name, DeviceRegister.__name__, - 'post', 'Device registered (token)', device.device_uuid, result['token']) + device_json = new_device.to_json(minimal=True, ignore_fields=['id_device', 'id_device_type', + 'id_device_subtype']) + device_json['device_certificate'] = new_device.device_certificate + device_json['ca_info'] = self.ca_info['certificate'].public_bytes(serialization.Encoding.PEM).decode('utf-8') + device_json['device_token'] = new_device.device_token - # Return token - return jsonify(result) - except SQLAlchemyError as e: - import sys - print(sys.exc_info()) - self.module.logger.log_error(self.module.module_name, - DeviceRegister.__name__, - 'post', 500, 'Database error', str(e)) - return e.args, 500 + self.module.logger.log_info(self.module.module_name, DeviceRegister.__name__, + 'post', 'Device registered (certificate)', + new_device.device_name + '(' + new_device.device_uuid + ')') + return device_json else: - return gettext('Invalid content type'), 400 + self.module.logger.log_error(self.module.module_name, DeviceRegister.__name__, + 'post', 400, 'Invalid CSR signature', request.data) + + return gettext('Invalid CSR signature'), 400 + + @staticmethod + def get_new_device(args) -> TeraDevice: + # Get device type + device_type = TeraDeviceType.get_device_type_by_key(args['type_key']) + if not device_type: + # Create a new device type with the appropriate key + device_type = TeraDeviceType() + device_type.device_type_key = args['type_key'] + device_type.device_type_name = device_type.device_type_key + TeraDeviceType.insert(device_type) + + # Get device subtype, if required + device_subtype = None + if args['subtype_name']: + device_subtype = TeraDeviceSubType.get_device_subtype_by_name(args['subtype_name'], + device_type.id_device_type) + if not device_subtype: + # Create a new device subtype + device_subtype = TeraDeviceSubType() + device_subtype.id_device_type = device_type.id_device_type + device_subtype.device_subtype_name = args['subtype_name'] + TeraDeviceSubType.insert(device_subtype) + + # Create new device + device = TeraDevice() + device.device_name = args['name'] + device.id_device_type = device_type.id_device_type + if device_subtype: + device.id_device_subtype = device_subtype.id_device_subtype + if 'onlineable' in args: + device.device_onlineable = args['onlineable'] + return device diff --git a/teraserver/python/modules/FlaskModule/API/user/UserQueryDeviceSubTypes.py b/teraserver/python/modules/FlaskModule/API/user/UserQueryDeviceSubTypes.py index 389e89891..03463e129 100644 --- a/teraserver/python/modules/FlaskModule/API/user/UserQueryDeviceSubTypes.py +++ b/teraserver/python/modules/FlaskModule/API/user/UserQueryDeviceSubTypes.py @@ -116,7 +116,7 @@ def post(self): # Already existing try: device_subtype: TeraDeviceSubType = \ - TeraDeviceSubType.get_device_subtype(json_device_subtype['id_device_subtype']) + TeraDeviceSubType.get_device_subtype_by_id(json_device_subtype['id_device_subtype']) if not device_subtype: return gettext('Invalid device subtype'), 400 json_device_subtype['id_device_type'] = device_subtype.id_device_type @@ -144,8 +144,7 @@ def post(self): 'post', 500, 'Database error', str(e)) return gettext('Database error'), 500 - # TODO: Publish update to everyone who is subscribed to devices update... - update_device = TeraDeviceSubType.get_device_subtype(json_device_subtype['id_device_subtype']) + update_device = TeraDeviceSubType.get_device_subtype_by_id(json_device_subtype['id_device_subtype']) return [update_device.to_json()] @@ -165,7 +164,7 @@ def delete(self): if not user_access.user.user_superadmin: return gettext('Forbidden'), 403 - todel = TeraDeviceSubType.get_device_subtype(id_todel) + todel = TeraDeviceSubType.get_device_subtype_by_id(id_todel) if not todel: return gettext('Device subtype not found'), 400 diff --git a/teraserver/python/opentera/db/models/TeraDevice.py b/teraserver/python/opentera/db/models/TeraDevice.py index d48b9f458..be1d31f51 100644 --- a/teraserver/python/opentera/db/models/TeraDevice.py +++ b/teraserver/python/opentera/db/models/TeraDevice.py @@ -215,7 +215,8 @@ def create_defaults(test=False): @classmethod def insert(cls, device): # Generate UUID - device.device_uuid = str(uuid.uuid4()) + if not device.device_uuid: + device.device_uuid = str(uuid.uuid4()) # Clear last online field device.device_lastonline = None diff --git a/teraserver/python/opentera/db/models/TeraDeviceSubType.py b/teraserver/python/opentera/db/models/TeraDeviceSubType.py index 28768dd58..390d43485 100644 --- a/teraserver/python/opentera/db/models/TeraDeviceSubType.py +++ b/teraserver/python/opentera/db/models/TeraDeviceSubType.py @@ -49,8 +49,9 @@ def get_devices_subtypes(): return TeraDeviceSubType.query.all() @staticmethod - def get_device_subtype(dev_subtype: int): - return TeraDeviceSubType.query.filter_by(id_device_subtype=dev_subtype).first() + def get_device_subtype_by_name(name: str, id_device_type: int): + return (TeraDeviceSubType.query.filter_by(device_subtype_name=name).filter_by(id_device_type=id_device_type) + .first()) @staticmethod def get_device_subtype_by_id(dev_subtype: int): diff --git a/teraserver/python/tests/modules/FlaskModule/API/device/BaseDeviceAPITest.py b/teraserver/python/tests/modules/FlaskModule/API/device/BaseDeviceAPITest.py index a23eef645..29a594daa 100644 --- a/teraserver/python/tests/modules/FlaskModule/API/device/BaseDeviceAPITest.py +++ b/teraserver/python/tests/modules/FlaskModule/API/device/BaseDeviceAPITest.py @@ -205,6 +205,13 @@ def _get_with_device_token_auth(self, client: FlaskClient, token: str = '', para headers = {'Authorization': 'OpenTera ' + token} return client.get(endpoint, headers=headers, query_string=params) + def _get_data_no_auth(self, client: FlaskClient, token: str = '', params={}, endpoint=None): + if params is None: + params = {} + if endpoint is None: + endpoint = self.test_endpoint + return client.get(endpoint, query_string=params) + def _post_with_device_token_auth(self, client: FlaskClient, token: str = '', json: dict = {}, params: dict = {}, endpoint: str = None): if params is None: diff --git a/teraserver/python/tests/modules/FlaskModule/API/device/test_DeviceRegister.py b/teraserver/python/tests/modules/FlaskModule/API/device/test_DeviceRegister.py index d0991253b..c347e1038 100644 --- a/teraserver/python/tests/modules/FlaskModule/API/device/test_DeviceRegister.py +++ b/teraserver/python/tests/modules/FlaskModule/API/device/test_DeviceRegister.py @@ -1,9 +1,10 @@ from BaseDeviceAPITest import BaseDeviceAPITest +from opentera.db.models.TeraServerSettings import TeraServerSettings from opentera.db.models.TeraDevice import TeraDevice from opentera.db.models.TeraDeviceType import TeraDeviceType +from opentera.db.models.TeraDeviceSubType import TeraDeviceSubType import opentera.crypto.crypto_utils as crypto -from cryptography.hazmat.primitives import hashes, serialization -import time +from cryptography.hazmat.primitives import serialization class DeviceRegisterTest(BaseDeviceAPITest): @@ -13,64 +14,156 @@ class DeviceRegisterTest(BaseDeviceAPITest): def setUp(self): super().setUp() self.sleep_time = 0 + with self._flask_app.app_context(): + self.device_register_key = ( + TeraServerSettings.get_server_setting_value(TeraServerSettings.ServerDeviceRegisterKey) + ) def tearDown(self): super().tearDown() - def test_post_endpoint_device_register_empty_json(self): - with self._flask_app.app_context(): - # This is required since the server will throttle device creations - time.sleep(self.sleep_time) + def test_register_token_missing_args(self): + response = self._get_data_no_auth(self.test_client) + self.assertEqual(400, response.status_code) - response = self._post_data_no_auth(self.test_client, json={}) - self.assertEqual(response.status_code, 400) + def test_register_token_bad_key(self): + data = {'key': 'Bad key', + 'name': 'New Device', + 'type_key': 'capteur'} + response = self._get_data_no_auth(self.test_client, params=data) + self.assertEqual(401, response.status_code) - def test_post_endpoint_device_register_json_incomplete_post(self): + def test_register_token_existing_device_key(self): with self._flask_app.app_context(): - # This is required since the server will throttle device creations - time.sleep(self.sleep_time) + device_type_key = TeraDeviceType.get_device_type_by_id(1).device_type_key + device_type_count = TeraDeviceType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key} + response = self._get_data_no_auth(self.test_client, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json) + + device = TeraDevice.get_device_by_token(response.json['device_token']) + self.assertIsNotNone(device) + self.assertEqual('New Device', device.device_name) + self.assertFalse(device.device_enabled) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertIsNone(device.id_device_subtype) + self.assertEqual(device_type_count, TeraDeviceType.get_count()) - device_info = {'device_info': {'device_name': 'Device Name'}} - response = self._post_data_no_auth(self.test_client, json=device_info) - self.assertEqual(response.status_code, 400) + def test_register_token_existing_device_subtype(self): + with self._flask_app.app_context(): + device_type_key = 'bureau_actif' + device_type_count = TeraDeviceType.get_count() + subtype_name = 'Bureau modèle #1' + subtype_count = TeraDeviceSubType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key, + 'subtype_name': subtype_name} + response = self._get_data_no_auth(self.test_client, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json) + + device = TeraDevice.get_device_by_token(response.json['device_token']) + self.assertIsNotNone(device) + self.assertEqual('New Device', device.device_name) + self.assertFalse(device.device_enabled) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertEqual(device_type_count, TeraDeviceType.get_count()) + self.assertEqual(subtype_name, device.device_subtype.device_subtype_name) + self.assertEqual(subtype_count, TeraDeviceSubType.get_count()) - device_info = {'device_info': {'id_device_type': 0}} - response = self._post_data_no_auth(self.test_client, json=device_info) - self.assertEqual(response.status_code, 400) + def test_register_token_new_device_key(self): + with self._flask_app.app_context(): + device_type_key = 'New Token Device Type' + device_type_count = TeraDeviceType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key} + response = self._get_data_no_auth(self.test_client, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json) + + device = TeraDevice.get_device_by_token(response.json['device_token']) + self.assertIsNotNone(device) + self.assertEqual('New Device', device.device_name) + self.assertFalse(device.device_enabled) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertIsNone(device.id_device_subtype) + self.assertEqual(device_type_count+1, TeraDeviceType.get_count()) - def test_post_endpoint_device_register_invalid_id_device_type(self): + def test_register_token_new_device_subtype(self): with self._flask_app.app_context(): - # This is required since the server will throttle device creations - time.sleep(self.sleep_time) + device_type_key = 'bureau_actif' + device_type_count = TeraDeviceType.get_count() + subtype_name = 'Bureau modèle #4' + subtype_count = TeraDeviceSubType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key, + 'subtype_name': subtype_name} + response = self._get_data_no_auth(self.test_client, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json) + + device = TeraDevice.get_device_by_token(response.json['device_token']) + self.assertIsNotNone(device) + self.assertEqual('New Device', device.device_name) + self.assertFalse(device.device_enabled) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertEqual(device_type_count, TeraDeviceType.get_count()) + self.assertEqual(subtype_name, device.device_subtype.device_subtype_name) + self.assertEqual(subtype_count+1, TeraDeviceSubType.get_count()) + + def test_register_certificate_bad_key(self): + data = {'key': 'Bad key', + 'name': 'New Device', + 'type_key': 'capteur'} + response = self._post_data_no_auth(self.test_client, params=data) + self.assertEqual(401, response.status_code) + + def test_register_certificate_missing_args(self): + response = self._post_data_no_auth(self.test_client) + self.assertEqual(400, response.status_code) + + def test_register_certificate_existing_device_key(self): + with self._flask_app.app_context(): + device_type_key = TeraDeviceType.get_device_type_by_id(1).device_type_key + device_type_count = TeraDeviceType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key} + # This will generate private key and signing request for the CA + client_info = crypto.create_certificate_signing_request('Test Device with Certificate') - device_info = {'device_info': {'device_name': 'Device Name', 'id_device_type': 0}} - response = self._post_data_no_auth(self.test_client, json=device_info) - self.assertEqual(response.status_code, 500) + # Encode in PEM format + encoded_csr = client_info['csr'].public_bytes(serialization.Encoding.PEM) - def test_post_endpoint_device_register_json_ok(self): - with self._flask_app.app_context(): - # This is required since the server will throttle device creations - time.sleep(self.sleep_time) - - device_info = {'device_info': {'device_name': 'Device Name', 'id_device_type': 1}} - response = self._post_data_no_auth(self.test_client, json=device_info) - self.assertEqual(response.status_code, 200) - self.assertTrue('token' in response.json) - self.assertGreater(len(response.json['token']), 0) - # Validate DB - device: TeraDevice = TeraDevice.get_device_by_token(response.json['token']) + response = self._post_data_no_auth(self.test_client, data=encoded_csr, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json, True) + + device = TeraDevice.get_device_by_certificate(response.json['device_certificate']) self.assertIsNotNone(device) + self.assertIsNotNone(device.device_certificate) + self.assertEqual('New Device', device.device_name) self.assertFalse(device.device_enabled) - self.assertFalse(device.device_onlineable) - self.assertEqual(device.id_device_type, 1) - # Delete device - TeraDevice.delete(device.id_device) - self.assertIsNone(TeraDevice.get_device_by_token(response.json['token'])) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertEqual(device_type_count, TeraDeviceType.get_count()) + self.assertIsNone(device.id_device_subtype) - def test_post_endpoint_with_device_register_with_certificate_csr(self): + def test_register_certificate_existing_device_subtype(self): with self._flask_app.app_context(): - # This is required since the server will throttle device creations - time.sleep(self.sleep_time) + device_type_key = 'bureau_actif' + device_type_count = TeraDeviceType.get_count() + subtype_name = 'Bureau modèle #1' + subtype_count = TeraDeviceSubType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key, + 'subtype_name': subtype_name} # This will generate private key and signing request for the CA client_info = crypto.create_certificate_signing_request('Test Device with Certificate') @@ -78,20 +171,81 @@ def test_post_endpoint_with_device_register_with_certificate_csr(self): # Encode in PEM format encoded_csr = client_info['csr'].public_bytes(serialization.Encoding.PEM) - response = self._post_data_no_auth(self.test_client, data=encoded_csr) - self.assertEqual(response.status_code, 200) + response = self._post_data_no_auth(self.test_client, data=encoded_csr, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json, True) - self.assertTrue('ca_info' in response.json) - self.assertTrue('certificate' in response.json) - self.assertGreater(len(response.json['certificate']), 0) + device = TeraDevice.get_device_by_certificate(response.json['device_certificate']) + self.assertIsNotNone(device) + self.assertEqual('New Device', device.device_name) + self.assertFalse(device.device_enabled) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertEqual(device_type_count, TeraDeviceType.get_count()) + self.assertEqual(subtype_name, device.device_subtype.device_subtype_name) + self.assertEqual(subtype_count, TeraDeviceSubType.get_count()) + + def test_register_certificate_new_device_key(self): + with self._flask_app.app_context(): + device_type_key = 'New Device Type' + device_type_count = TeraDeviceType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key} + # This will generate private key and signing request for the CA + client_info = crypto.create_certificate_signing_request('Test Device with Certificate') - device: TeraDevice = TeraDevice.get_device_by_certificate(response.json['certificate']) + # Encode in PEM format + encoded_csr = client_info['csr'].public_bytes(serialization.Encoding.PEM) + + response = self._post_data_no_auth(self.test_client, data=encoded_csr, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json, True) + + device = TeraDevice.get_device_by_certificate(response.json['device_certificate']) self.assertIsNotNone(device) + self.assertEqual('New Device', device.device_name) self.assertFalse(device.device_enabled) - self.assertFalse(device.device_onlineable) - # TODO device type default is 'capteur' - self.assertEqual(device.id_device_type, TeraDeviceType.get_device_type_by_key('capteur').id_device_type) - # Delete device - TeraDevice.delete(device.id_device) - self.assertIsNone(TeraDevice.get_device_by_certificate(response.json['certificate'])) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertIsNone(device.id_device_subtype) + self.assertEqual(device_type_count + 1, TeraDeviceType.get_count()) + def test_register_certificate_new_device_subtype(self): + with self._flask_app.app_context(): + device_type_key = 'bureau_actif' + device_type_count = TeraDeviceType.get_count() + subtype_name = 'Bureau modèle #3' + subtype_count = TeraDeviceSubType.get_count() + data = {'key': self.device_register_key, + 'name': 'New Device', + 'type_key': device_type_key, + 'subtype_name': subtype_name} + # This will generate private key and signing request for the CA + client_info = crypto.create_certificate_signing_request('Test Device with Certificate') + + # Encode in PEM format + encoded_csr = client_info['csr'].public_bytes(serialization.Encoding.PEM) + + response = self._post_data_no_auth(self.test_client, data=encoded_csr, params=data) + self.assertEqual(200, response.status_code) + self._checkJson(response.json, True) + + device = TeraDevice.get_device_by_certificate(response.json['device_certificate']) + self.assertIsNotNone(device) + self.assertEqual('New Device', device.device_name) + self.assertFalse(device.device_enabled) + self.assertEqual(device_type_key, device.device_type.device_type_key) + self.assertEqual(device_type_count, TeraDeviceType.get_count()) + self.assertEqual(subtype_name, device.device_subtype.device_subtype_name) + self.assertEqual(subtype_count + 1, TeraDeviceSubType.get_count()) + + def _checkJson(self, json_data, certificate=False): + self.assertFalse(json_data.__contains__('id_device')) + self.assertFalse(json_data.__contains__('id_device_type')) + self.assertFalse(json_data.__contains__('id_device_subtype')) + self.assertTrue(json_data.__contains__('device_name')) + self.assertTrue(json_data.__contains__('device_uuid')) + self.assertTrue(json_data.__contains__('device_token')) + self.assertTrue(json_data.__contains__('device_enabled')) + if certificate: + self.assertTrue(json_data.__contains__('device_certificate')) + self.assertTrue(json_data.__contains__('ca_info')) From f86e7bb65ce87336fece3acc5cdb6570d11dfeb0 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 29 Jan 2024 10:17:36 -0500 Subject: [PATCH 14/29] Fixed bug in POST in DeviceQuerySessionEvents --- .../API/device/DeviceQuerySessionEvents.py | 5 ++--- .../FlaskModule/API/device/DeviceQuerySessions.py | 13 +++++++------ .../python/opentera/db/models/TeraSessionEvent.py | 1 - 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessionEvents.py b/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessionEvents.py index f1b0f879d..706e53370 100644 --- a/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessionEvents.py +++ b/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessionEvents.py @@ -79,7 +79,7 @@ def post(self): new_event.from_json(json_event) TeraSessionEvent.insert(new_event) # Update ID for further use - json_event['id_session_event'] = new_event.id_session + json_event['id_session_event'] = new_event.id_session_event except exc.SQLAlchemyError as e: import sys print(sys.exc_info()) @@ -88,10 +88,9 @@ def post(self): 'post', 500, 'Database error', str(e)) return gettext('Database error'), 500 - # TODO: Publish update to everyone who is subscribed to sites update... update_event = TeraSessionEvent.get_session_event_by_id(json_event['id_session_event']) - return jsonify([update_event.to_json()]) + return [update_event.to_json()] @LoginModule.device_token_or_certificate_required def delete(self): diff --git a/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessions.py b/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessions.py index fd14de52d..f36df815a 100644 --- a/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessions.py +++ b/teraserver/python/modules/FlaskModule/API/device/DeviceQuerySessions.py @@ -84,7 +84,7 @@ def post(self): # args = post_parser.parse_args() # Using request.json instead of parser, since parser messes up the json! if 'session' not in request.json: - return gettext('Missing arguments'), 400 + return gettext('Missing session'), 400 json_session = request.json['session'] @@ -92,16 +92,16 @@ def post(self): # Validate if we have an id if 'id_session' not in json_session: - return gettext('Missing arguments'), 400 + return gettext('Missing id_session value'), 400 # Validate if we have an id if 'id_session_type' not in json_session: - return gettext('Missing arguments'), 400 + return gettext('Missing id_session_type value'), 400 # Validate that we have session participants or users for new sessions if ('session_participants' not in json_session and 'session_users' not in json_session) \ - and json_session['id_session'] == 0: - return gettext('Missing arguments'), 400 + and 'session_devices' not in json_session and json_session['id_session'] == 0: + return gettext('Missing session participants and/or users and/or devices'), 400 # We know we have a device # Avoid identity thief @@ -111,7 +111,7 @@ def post(self): session_types = device_access.get_accessible_session_types_ids() if not json_session['id_session_type'] in session_types: - return gettext('Unauthorized'), 403 + return gettext('No access to session type'), 403 # Check if a session of that type and name already exists. If so, don't create it, just returns it. if json_session['id_session'] == 0: @@ -141,6 +141,7 @@ def post(self): # Already existing # TODO handle participant list (remove, add) in session + try: if 'session_participants' in json_session: participants = json_session.pop('session_participants') diff --git a/teraserver/python/opentera/db/models/TeraSessionEvent.py b/teraserver/python/opentera/db/models/TeraSessionEvent.py index aef0b2e99..9444990d9 100644 --- a/teraserver/python/opentera/db/models/TeraSessionEvent.py +++ b/teraserver/python/opentera/db/models/TeraSessionEvent.py @@ -42,7 +42,6 @@ class SessionEventTypes(Enum): __tablename__ = 't_sessions_events' id_session_event = Column(Integer, Sequence('id_session_events_sequence'), primary_key=True, autoincrement=True) id_session = Column(Integer, ForeignKey('t_sessions.id_session', ondelete='cascade'), nullable=False) - # TODO: Typo that should be fixed someday... id_session_event_type = Column(Integer, nullable=False) session_event_datetime = Column(TIMESTAMP(timezone=True), nullable=False) session_event_text = Column(String, nullable=True) From 403931a7631b9aebe0aba79dd3fc3e9be6d136f7 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Tue, 13 Feb 2024 15:14:11 -0500 Subject: [PATCH 15/29] Managed TeraServerSettings redis keys when creating/updating --- teraserver/python/env/requirements.txt | 12 ++-- .../ServiceLauncherModule.py | 62 ++++++++++++------- .../opentera/db/models/TeraServerSettings.py | 8 +++ .../python/opentera/db/models/__init__.py | 3 +- 4 files changed, 57 insertions(+), 28 deletions(-) diff --git a/teraserver/python/env/requirements.txt b/teraserver/python/env/requirements.txt index 29f5bfdf1..00f41abaa 100644 --- a/teraserver/python/env/requirements.txt +++ b/teraserver/python/env/requirements.txt @@ -1,13 +1,13 @@ pypiwin32==223; sys_platform == 'win32' Twisted==23.10.0 treq==23.11.0 -cryptography==42.0.0 +cryptography==42.0.2 autobahn==23.6.2 -SQLAlchemy==2.0.25 +SQLAlchemy==2.0.26 sqlalchemy-schemadisplay==1.3 pydot==2.0.0 psycopg2-binary==2.9.9 -Flask==3.0.1 +Flask==3.0.2 Flask-SQLAlchemy==3.1.1 Flask-Login==0.6.3 Flask-Login-Multi==0.1.2 @@ -20,7 +20,7 @@ Flask-Babel==4.0.0 Flask-BabelEx==0.9.4 Flask-Migrate==4.0.5 flask-swagger-ui==4.11.1 -Flask-Limiter==3.5.0 +Flask-Limiter==3.5.1 Flask-Mail==0.9.1 Flask-Principal==0.4.0 redis==5.0.1 @@ -33,9 +33,9 @@ service-identity==24.1.0 PyJWT==2.8.0 pylzma==0.5.0 bz2file==0.98 -python-slugify==8.0.1 +python-slugify==8.0.4 websocket-client==1.7.0 -pytest==7.4.4 +pytest==8.0.0 Jinja2==3.1.3 ua-parser==0.18.0 diff --git a/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py b/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py index eb05b36c1..f58c40a7e 100644 --- a/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py +++ b/teraserver/python/modules/ServiceLauncherModule/ServiceLauncherModule.py @@ -1,6 +1,7 @@ from opentera.modules.BaseModule import BaseModule, ModuleNames, create_module_event_topic_from_name from opentera.config.ConfigManager import ConfigManager from opentera.db.models.TeraService import TeraService +from opentera.db.models.TeraServerSettings import TeraServerSettings import opentera.messages.python as messages from twisted.internet import defer import os @@ -45,7 +46,9 @@ def setup_module_pubsub(self): print('ServiceLauncherModule - Registering to events...') # Always register to user events yield self.subscribe_pattern_with_callback(create_module_event_topic_from_name( - ModuleNames.DATABASE_MODULE_NAME, 'service'), self.database_event_received_for_service) + ModuleNames.DATABASE_MODULE_NAME, 'service'), self.database_event_received) + yield self.subscribe_pattern_with_callback(create_module_event_topic_from_name( + ModuleNames.DATABASE_MODULE_NAME, 'server_settings'), self.database_event_received) # Launch all internal services services = TeraService.query.all() @@ -61,7 +64,7 @@ def setup_module_pubsub(self): # Need to register to events (base class) super().setup_module_pubsub() - def database_event_received_for_service(self, pattern, channel, message): + def database_event_received(self, pattern, channel, message): # Process database event try: tera_event = messages.TeraEvent() @@ -75,30 +78,47 @@ def database_event_received_for_service(self, pattern, channel, message): # Look for DatabaseEvent for any_msg in tera_event.events: if any_msg.Unpack(database_event): - # Process event - try: - service_dict = json.loads(database_event.object_value) - - if database_event.type == database_event.DB_CREATE or \ - database_event.type == database_event.DB_UPDATE: - # Update redis values - if 'id_service' in service_dict: - if service_dict['service_enabled'] and 'deleted_at' not in service_dict: - self.update_specific_service_info(service_dict['service_key'], service_dict) - else: + if database_event.object_type == 'service': + # Process event + try: + service_dict = json.loads(database_event.object_value) + + if database_event.type == database_event.DB_CREATE or \ + database_event.type == database_event.DB_UPDATE: + # Update redis values + if 'id_service' in service_dict: + if service_dict['service_enabled'] and 'deleted_at' not in service_dict: + self.update_specific_service_info(service_dict['service_key'], service_dict) + else: + self.delete_specific_service_info(service_dict['service_key']) + elif database_event.type == database_event.DB_DELETE: + if 'service_key' in service_dict: self.delete_specific_service_info(service_dict['service_key']) - elif database_event.type == database_event.DB_DELETE: - if 'service_key' in service_dict: - self.delete_specific_service_info(service_dict['service_key']) - except json.JSONDecodeError as json_decode_error: - print('ServiceLauncherModule:database_event_received_for_service - JSONDecodeError ', - str(database_event.object_value), str(json_decode_error)) + except json.JSONDecodeError as json_decode_error: + print('ServiceLauncherModule:database_event_received service - JSONDecodeError ', + str(database_event.object_value), str(json_decode_error)) + if database_event.object_type == 'server_settings': + try: + settings_dict = json.loads(database_event.object_value) + if database_event.type == database_event.DB_CREATE or \ + database_event.type == database_event.DB_UPDATE: + if settings_dict['server_settings_name'] == TeraServerSettings.ServerDeviceTokenKey: + self.redisSet(RedisVars.RedisVar_DeviceStaticTokenAPIKey, + settings_dict['server_settings_value']) + if (settings_dict['server_settings_name'] == + TeraServerSettings.ServerParticipantTokenKey): + self.redisSet(RedisVars.RedisVar_ParticipantStaticTokenAPIKey, + settings_dict['server_settings_value']) + except json.JSONDecodeError as json_decode_error: + print('ServiceLauncherModule:database_event_received server settings - JSONDecodeError ', + str(database_event.object_value), str(json_decode_error)) + except DecodeError as decode_error: - print('ServiceLauncherModule:database_event_received_for_service - DecodeError ', pattern, channel, message, + print('ServiceLauncherModule:database_event_received - DecodeError ', pattern, channel, message, decode_error) except ParseError as parse_error: - print('ServiceLauncherModule:database_event_received_for_service - Failure in database_event_received', + print('ServiceLauncherModule:database_event_received - Failure in database_event_received', parse_error) def notify_module_messages(self, pattern, channel, message): diff --git a/teraserver/python/opentera/db/models/TeraServerSettings.py b/teraserver/python/opentera/db/models/TeraServerSettings.py index bc9d90923..87057743d 100644 --- a/teraserver/python/opentera/db/models/TeraServerSettings.py +++ b/teraserver/python/opentera/db/models/TeraServerSettings.py @@ -72,3 +72,11 @@ def set_server_setting(setting_name: string, setting_value: string): TeraServerSettings.db().session.add(current_setting) # Store object current_setting.commit() + + def to_json_create_event(self): + return self.to_json(ignore_fields=['ServerDeviceTokenKey', 'ServerParticipantTokenKey', 'ServerUUID', + 'ServerVersions', 'ServerDeviceRegisterKey']) + + def to_json_update_event(self): + return self.to_json(ignore_fields=['ServerDeviceTokenKey', 'ServerParticipantTokenKey', 'ServerUUID', + 'ServerVersions', 'ServerDeviceRegisterKey']) diff --git a/teraserver/python/opentera/db/models/__init__.py b/teraserver/python/opentera/db/models/__init__.py index ba3e0bfad..c0d5f08e3 100644 --- a/teraserver/python/opentera/db/models/__init__.py +++ b/teraserver/python/opentera/db/models/__init__.py @@ -54,7 +54,8 @@ TeraTest.get_model_name(): TeraTest, TeraService.get_model_name(): TeraService, TeraSessionTypeSite.get_model_name(): TeraSessionTypeSite, - TeraSessionTypeProject.get_model_name(): TeraSessionTypeProject + TeraSessionTypeProject.get_model_name(): TeraSessionTypeProject, + TeraServerSettings.get_model_name(): TeraServerSettings } # All exported symbols From be1ff7ad94e33cff49f762a8afcedd43c333fdbc Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Thu, 15 Feb 2024 08:04:53 -0500 Subject: [PATCH 16/29] Fixed bad access check in ServiceQueryProjects --- .../modules/FlaskModule/API/service/ServiceQueryProjects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py index ed53bcec7..3c51fb3b6 100644 --- a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py +++ b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py @@ -100,9 +100,9 @@ def post(self): # Do the update! if json_project['id_project'] > 0: - # Already existing - can only modifify is service is associated to that project + # Already existing - can only modifify if service is associated to that project project: TeraProject = TeraProject.get_project_by_id(json_project['id_project']) - if not project or project.id_site not in service_access.get_accessible_projects_ids(): + if not project or project.id_project not in service_access.get_accessible_projects_ids(): return gettext('Forbidden'), 403 try: From a8f7e0819f7f7bf62351c45f66199efb68e15228 Mon Sep 17 00:00:00 2001 From: mouhamb Date: Thu, 15 Feb 2024 09:58:27 -0500 Subject: [PATCH 17/29] For the pull request of ServiceQueryParticipantGroups There is the update of the file ServiceQueryParticipantGroups.py and the implementation of its tests too. --- .../service/ServiceQueryParticipantGroups.py | 156 +++++++----- .../test_ServiceQueryParticipantGroups.py | 228 ++++++++++++++++++ 2 files changed, 321 insertions(+), 63 deletions(-) create mode 100644 teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py diff --git a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py index f9612f29c..aa98c5f2b 100644 --- a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py +++ b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py @@ -2,42 +2,21 @@ from flask_restx import Resource from flask_babel import gettext from sqlalchemy import exc -from sqlalchemy.exc import IntegrityError -from modules.LoginModule.LoginModule import LoginModule + +from modules.DatabaseModule.DBManager import DBManager +from modules.LoginModule.LoginModule import LoginModule, current_service from modules.FlaskModule.FlaskModule import service_api_ns as api from opentera.db.models.TeraParticipantGroup import TeraParticipantGroup # Parser definition(s) get_parser = api.parser() get_parser.add_argument('id_participant_group', type=int, help='ID to query') +get_parser.add_argument('id_project', type=int, help='ID project to query information') post_parser = api.parser() -participant_group_schema = api.schema_model('participant_group', { - 'properties': { - 'participant_group': { - 'type': 'object', - 'properties': { - 'id_participant_group': { - 'type': 'integer', - }, - - 'id_project': { - 'type': 'integer', - }, - - 'participant_group_name': { - 'type': 'string' - } - - }, - 'required': ['id_participant_group', 'id_project', 'participant_group_name'] - }, - - }, - 'type': 'object', - 'required': ['participant_group'] -}) +post_schema = api.schema_model('participant_group', {'properties': TeraParticipantGroup.get_json_schema(), + 'type': 'object', 'location': 'json'}) delete_parser = api.parser() delete_parser.add_argument('id', type=int, help='ID to delete') @@ -60,15 +39,43 @@ def __init__(self, _api, *args, **kwargs): @api.expect(get_parser) @LoginModule.service_token_or_certificate_required def get(self): + # Get service access manager, that allows to check for access + service_access = DBManager.serviceAccess(current_service) + + # Parse arguments args = get_parser.parse_args() # Check if 'id_participant_group' is specified in args if args['id_participant_group']: + # Check if service has access to the specified participant group + if args['id_participant_group'] not in service_access.get_accessible_participants_groups_ids(): + return gettext('Forbidden'), 403 # Retrieve participant group by ID participant_group = TeraParticipantGroup.get_participant_group_by_id(args['id_participant_group']) - # If participant group is found, return its JSON representation + # If participant group is found, check if 'id_project' matches if participant_group: - return participant_group.to_json() + # Check if the 'id_project' and 'id_participant_group' is matching + if args['id_project'] and args['id_project'] != participant_group.id_project: + # Check if user has access to the specified project + if args['id_project'] not in service_access.get_accessible_projects_ids(): + return gettext('Forbidden'), 403 + return gettext('No group matching the provided id_participant_group and id_project was found'), 404 + + # Return the JSON representation of the participant group + return participant_group.to_json(minimal=True) + + # Check if 'id_project' is specified in args + if args['id_project']: + # Check if user has access to the specified project + if args['id_project'] not in service_access.get_accessible_projects_ids(): + return gettext('Forbidden'), 403 + # Retrieve participant groups by id_project + participant_groups = TeraParticipantGroup.get_participant_group_for_project(args['id_project']) + # If at least one participant group is found, convert result to JSON + if participant_groups: + # Convert result to JSON + participant_group_json = [group.to_json(minimal=True) for group in participant_groups] + return participant_group_json # Return error message for missing arguments return gettext('Missing arguments'), 400 @@ -79,32 +86,46 @@ def get(self): 501: 'Not implemented.', 403: 'Logged user doesn\'t have permission to access the requested data'}, params={'token': 'Secret token'}) - @api.expect(participant_group_schema) + @api.expect(post_schema) @LoginModule.service_token_or_certificate_required def post(self): # Parse arguments args = post_parser.parse_args() + # Get service access manager, that allows to check for access + service_access = DBManager.serviceAccess(current_service) + # Check if 'participant_group' is present in the JSON request if 'participant_group' not in request.json: - return gettext('Missing arguments'), 400 + return gettext('Missing participant_group'), 400 # Extract participant group information from the JSON request participant_group_info = request.json['participant_group'] - # Check if the project ID is valid - if participant_group_info['id_project'] < 1: - return gettext('Unknown project'), 403 + # Check if 'id_participant_group' is missing + if 'id_participant_group' not in participant_group_info: + return gettext('Missing id_participant_group'), 400 - # Check if the participant group name is provided - if not participant_group_info['participant_group_name']: - return gettext('Invalid participant group name'), 403 + # Check if creating a new participant group and 'id_project' is missing + if participant_group_info['id_participant_group'] == 0 and ('id_project' not in participant_group_info + or participant_group_info['id_project'] is None): + return gettext('Missing id_project'), 400 - # Initialize participant group instance - participant_group: TeraParticipantGroup = TeraParticipantGroup() + # Check if creating a new participant group and 'participant_group_name' is missing + if (participant_group_info['id_participant_group'] == 0 and ('participant_group_name' + not in participant_group_info or + participant_group_info[ + 'participant_group_name'] is None)): + return gettext('Missing group name'), 400 # Check if it's a new participant group or an update if participant_group_info['id_participant_group'] == 0: + verif = service_access.get_accessible_projects_ids() + # Check if the project ID is valid + if ('id_project' in participant_group_info and participant_group_info['id_project'] + not in service_access.get_accessible_projects_ids()): + return gettext('Forbidden'), 403 + # Create participant group participant_group = TeraParticipantGroup() participant_group.participant_group_name = participant_group_info['participant_group_name'] @@ -112,27 +133,30 @@ def post(self): try: TeraParticipantGroup.insert(participant_group) - except IntegrityError as e: - self.module.logger.log_warning(self.module.module_name, ServiceQueryParticipantGroups.__name__, - 'insert', - 400, 'Integrity error', str(e)) - - # Handle integrity error related to projects - if 't_projects' in str(e.args): - return gettext('Can\'t insert participant group: participant group\'s project ' - 'is disabled or invalid.'), 400 + except exc.SQLAlchemyError as e: + import sys + print(sys.exc_info()) + self.module.logger.log_error(self.module.module_name, + ServiceQueryParticipantGroups.__name__, + 'post', 500, 'Database error', str(e)) + return gettext('Database error'), 500 else: # Update existing participant group try: - TeraParticipantGroup.update(participant_group_info['id_participant_group'], participant_group_info) - except IntegrityError as e: - self.module.logger.log_warning(self.module.module_name, ServiceQueryParticipantGroups.__name__, - 'update', - 400, 'Integrity error', str(e)) + # Check if updating an existing participant group - # Handle integrity error related to projects - if 't_projects' in str(e.args): - return gettext('Can\'t update participant group: participant group\'s project is disabled.'), 400 + if (participant_group_info['id_participant_group'] + not in service_access.get_accessible_participants_groups_ids()): + return gettext('Forbidden'), 403 + + TeraParticipantGroup.update(participant_group_info['id_participant_group'], participant_group_info) + except exc.SQLAlchemyError as e: + import sys + print(sys.exc_info()) + self.module.logger.log_error(self.module.module_name, + ServiceQueryParticipantGroups.__name__, + 'post', 500, 'Database error', str(e)) + return gettext('Database error'), 500 # Retrieve the updated participant group participant_group = TeraParticipantGroup.get_participant_group_by_id(participant_group_info @@ -141,8 +165,6 @@ def post(self): # Return the JSON representation of the participant group return participant_group.to_json(minimal=False) - - @api.doc(description='Delete a specific participant group.', responses={200: 'Success', 403: 'Logged user doesn\'t have permission to access the requested data', @@ -151,19 +173,27 @@ def post(self): @api.expect(delete_parser) @LoginModule.service_token_or_certificate_required def delete(self): + # Parse arguments args = delete_parser.parse_args() id_todel = args['id'] + # Get service access manager, that allows to check for access + service_access = DBManager.serviceAccess(current_service) - # Check deletion integrity + # Check if the user has access to delete participant groups + if id_todel not in service_access.get_accessible_participants_groups_ids(): + return gettext('Forbidden'), 403 + + # If the participant group with the given ID is not found, return an error group_to_del = TeraParticipantGroup.get_participant_group_by_id(id_todel) if group_to_del is None: - return gettext('The id_participant_group given was not found'), + return gettext('The id_participant_group given was not found'), 400 + # Check deletion integrity deletion_integrity = group_to_del.delete_check_integrity() - + # Check if deletion is possible without violating integrity constraints if deletion_integrity is not None: - return gettext('Deletion impossible: Participant group still has participant(s)') + return gettext('Deletion impossible: Participant group still has participant(s)'), 500 # If we are here, we are allowed to delete. Do so. try: @@ -176,4 +206,4 @@ def delete(self): 'delete', 500, 'Database error', str(e)) return gettext('Database error'), 500 - return '', 200 \ No newline at end of file + return '', 200 diff --git a/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py new file mode 100644 index 000000000..1d5f09d21 --- /dev/null +++ b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py @@ -0,0 +1,228 @@ +from BaseServiceAPITest import BaseServiceAPITest +from opentera.db.models import TeraParticipantGroup + + +class ServiceQueryParticipantGroupsTest(BaseServiceAPITest): + test_endpoint = '/api/service/groups' + + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + + def test_get_endpoint_no_auth(self): + with self._flask_app.app_context(): + response = self.test_client.get(self.test_endpoint) + self.assertEqual(401, response.status_code) + + + def test_get_endpoint_with_token_auth_no_params(self): + with self._flask_app.app_context(): + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=None, endpoint=self.test_endpoint) + self.assertEqual(400, response.status_code) + + + def test_get_endpoint_with_token_auth_and_invalid_id_project(self): + with self._flask_app.app_context(): + params = {'id_project': -1} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(403, response.status_code) + + + def test_get_endpoint_with_token_auth_and_invalid_id_participant_group(self): + with self._flask_app.app_context(): + params = {'id_participant_group': -1} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(403, response.status_code) + + + def test_get_endpoint_with_token_auth_and_invalid_id_project_and_invalid_id_participant_group(self): + with self._flask_app.app_context(): + params = {'id_project': -1, 'id_participant_group': -1} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(403, response.status_code) + + + def test_get_endpoint_with_token_auth_and_valid_id_participant_group(self): + with self._flask_app.app_context(): + params = {'id_participant_group': 1} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + self.assertEqual(3, len(response.json)) + group: TeraParticipantGroup = TeraParticipantGroup.get_participant_group_by_id(1) + self.assertEqual(group.to_json(minimal=True), response.json) + + + def test_get_endpoint_with_token_auth_and_valid_id_project(self): + with self._flask_app.app_context(): + params = {'id_project': 1} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + self.assertGreaterEqual(len(response.json),1 ) + groups: TeraParticipantGroup = TeraParticipantGroup.get_participant_group_for_project(1) + i = 0 + for group in groups: + self.assertEqual(group.to_json(minimal=True), response.json[i]) + i = i + 1 + + def test_get_endpoint_with_token_auth_and_valid_id_project_and_valid_id_participant_group(self): + with self._flask_app.app_context(): + params = {'id_project': 1, 'id_participant_group': 1} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + + def test_get_endpoint_with_token_auth_and_valid_but_denied_id_project(self): + with self._flask_app.app_context(): + denied_id_projects = [2, 3] + + for id_project in denied_id_projects: + params = {'id_project': id_project} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(403, response.status_code) + + + def test_get_endpoint_with_token_auth_and_valid_but_denied_id_participant_group(self): + with self._flask_app.app_context(): + params = {'id_participant_group': 2} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(403, response.status_code) + + + + def test_post_and_delete_endpoint_with_token(self): + with self._flask_app.app_context(): + # Test case: Post with missing information + json_data = { + 'participant_group_name': 'Testing123', + } + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(400, response.status_code, msg="Missing project struct") + + # Test case: Post with missing id_project + json_data = { + 'participant_group': { + 'participant_group_name': 'Testing123' + } + } + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(400, response.status_code, msg="Missing id_project") + + # Test case: Post with missing id_participant_group + json_data = { + 'participant_group': { + 'participant_group_name': 'Testing123', + 'id_project': 1 + } + } + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(400, response.status_code, msg="Missing id_participant_group") + + json_data['participant_group']['id_project'] = 2 + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(400, response.status_code, msg="Missing id_participant_group") + + # Test case: Post in a project where service isn't associated + json_data['participant_group']['id_project'] = 2 + json_data['participant_group']['id_participant_group'] = 0 + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(403, response.status_code, msg="No access to project") + + # Test case: Modification + json_data['participant_group']['id_participant_group'] = 1 + json_data['participant_group']['id_project'] = 1 + json_data['participant_group']['participant_group_name'] = "New name" + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(200, response.status_code, msg="New OK") + group_data = response.json + project_id = group_data['id_project'] + name = group_data['participant_group_name'] + self.assertEqual(1, project_id) + self.assertEqual("New name", name) + + + # Test case: Creation + json_data['participant_group']['id_participant_group'] = 0 + json_data['participant_group']['id_project'] = 1 + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(200, response.status_code, msg="New OK") + group_data = response.json + project_id = group_data['id_project'] + self.assertEqual(1, project_id) + + + # Test case: Post update to project without association to service + json_data = { + 'participant_group': { + 'id_project': 3, + 'id_participant_group': 0, + 'participant_group_name': 'Testing123' + } + } + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(403, response.status_code, msg="No access to project") + + # Test case: Post update to group without association to service + json_data = { + 'participant_group': { + 'id_project': 1, + 'id_participant_group': 2, + 'participant_group_name': 'Testing123', + 'invalid_parameter': -1 + } + } + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(403, response.status_code, msg="No access to the group") + + # Test case: Post update with invalid parameter + del json_data['participant_group']['id_project'] + json_data['participant_group']['id_participant_group'] = 3 + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(500, response.status_code, msg="Invalid parameter") + + + del json_data['participant_group']['invalid_parameter'] + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(200, response.status_code, msg="Update OK") + + # Test case: Modification + json_data['participant_group']['id_participant_group'] = 1 + json_data['participant_group']['id_project'] = 1 + response = self._post_with_service_token_auth(self.test_client, token=self.service_token, + json=json_data) + self.assertEqual(200, response.status_code, msg="New OK") + + # Test case: Delete denied (not associated to service) + response = self._delete_with_service_token_auth(self.test_client, token=self.service_token, + params={'id': 2}) + self.assertEqual(403, response.status_code, msg="Delete denied") + + # Test case: Delete with integrity error + response = self._delete_with_service_token_auth(self.test_client, token=self.service_token, + params={'id': 1}) + self.assertEqual(500, response.status_code, msg="Delete denied (integrity)") + + # Test case: Delete with no problem + response = self._delete_with_service_token_auth(self.test_client, token=self.service_token, + params={'id': 3}) + self.assertEqual(200, response.status_code, msg="Delete OK") \ No newline at end of file From 64cb2a0040aa06215c9c23f6bb20aacff1b3f7df Mon Sep 17 00:00:00 2001 From: mouhamb Date: Thu, 15 Feb 2024 10:02:22 -0500 Subject: [PATCH 18/29] Update of ServiceQueryProjects --- .../modules/FlaskModule/API/service/ServiceQueryProjects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py index ed53bcec7..3c51fb3b6 100644 --- a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py +++ b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryProjects.py @@ -100,9 +100,9 @@ def post(self): # Do the update! if json_project['id_project'] > 0: - # Already existing - can only modifify is service is associated to that project + # Already existing - can only modifify if service is associated to that project project: TeraProject = TeraProject.get_project_by_id(json_project['id_project']) - if not project or project.id_site not in service_access.get_accessible_projects_ids(): + if not project or project.id_project not in service_access.get_accessible_projects_ids(): return gettext('Forbidden'), 403 try: From 079e0026f8ec494b65d8133cb33eaedca41b7918 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 4 Mar 2024 05:07:05 -0500 Subject: [PATCH 19/29] Refs #234. Merged ServiceQueryParticipantGroups --- .../service/ServiceQueryParticipantGroups.py | 18 ++++++------------ .../python/modules/FlaskModule/FlaskModule.py | 2 ++ .../test_ServiceQueryParticipantGroups.py | 14 +------------- 3 files changed, 9 insertions(+), 25 deletions(-) diff --git a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py index aa98c5f2b..27b5a4fbd 100644 --- a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py +++ b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipantGroups.py @@ -52,17 +52,12 @@ def get(self): return gettext('Forbidden'), 403 # Retrieve participant group by ID participant_group = TeraParticipantGroup.get_participant_group_by_id(args['id_participant_group']) - # If participant group is found, check if 'id_project' matches - if participant_group: - # Check if the 'id_project' and 'id_participant_group' is matching - if args['id_project'] and args['id_project'] != participant_group.id_project: - # Check if user has access to the specified project - if args['id_project'] not in service_access.get_accessible_projects_ids(): - return gettext('Forbidden'), 403 - return gettext('No group matching the provided id_participant_group and id_project was found'), 404 - - # Return the JSON representation of the participant group - return participant_group.to_json(minimal=True) + + if not participant_group: + return gettext('Not found'), 404 + + # Return the JSON representation of the participant group + return participant_group.to_json(minimal=True) # Check if 'id_project' is specified in args if args['id_project']: @@ -120,7 +115,6 @@ def post(self): # Check if it's a new participant group or an update if participant_group_info['id_participant_group'] == 0: - verif = service_access.get_accessible_projects_ids() # Check if the project ID is valid if ('id_project' in participant_group_info and participant_group_info['id_project'] not in service_access.get_accessible_projects_ids()): diff --git a/teraserver/python/modules/FlaskModule/FlaskModule.py b/teraserver/python/modules/FlaskModule/FlaskModule.py index e99746f31..c957a9fc4 100755 --- a/teraserver/python/modules/FlaskModule/FlaskModule.py +++ b/teraserver/python/modules/FlaskModule/FlaskModule.py @@ -313,11 +313,13 @@ def init_service_api(module: object, namespace: Namespace, additional_args: dict from modules.FlaskModule.API.service.ServiceQueryRoles import ServiceQueryRoles from modules.FlaskModule.API.service.ServiceQueryServiceAccess import ServiceQueryServiceAccess from modules.FlaskModule.API.service.ServiceQueryUserGroups import ServiceQueryUserGroups + from modules.FlaskModule.API.service.ServiceQueryParticipantGroups import ServiceQueryParticipantGroups namespace.add_resource(ServiceQueryAccess, '/access', resource_class_kwargs=kwargs) namespace.add_resource(ServiceQueryAssets, '/assets', resource_class_kwargs=kwargs) namespace.add_resource(ServiceQueryDevices, '/devices', resource_class_kwargs=kwargs) namespace.add_resource(ServiceQueryDisconnect, '/disconnect', resource_class_kwargs=kwargs) + namespace.add_resource(ServiceQueryParticipantGroups, '/groups', resource_class_kwargs=kwargs) namespace.add_resource(ServiceQueryParticipants, '/participants', resource_class_kwargs=kwargs) namespace.add_resource(ServiceQueryProjects, '/projects', resource_class_kwargs=kwargs) namespace.add_resource(ServiceQueryRoles, '/roles', resource_class_kwargs=kwargs) diff --git a/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py index 1d5f09d21..4138754d5 100644 --- a/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py +++ b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipantGroups.py @@ -16,14 +16,12 @@ def test_get_endpoint_no_auth(self): response = self.test_client.get(self.test_endpoint) self.assertEqual(401, response.status_code) - def test_get_endpoint_with_token_auth_no_params(self): with self._flask_app.app_context(): response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, params=None, endpoint=self.test_endpoint) self.assertEqual(400, response.status_code) - def test_get_endpoint_with_token_auth_and_invalid_id_project(self): with self._flask_app.app_context(): params = {'id_project': -1} @@ -31,7 +29,6 @@ def test_get_endpoint_with_token_auth_and_invalid_id_project(self): params=params, endpoint=self.test_endpoint) self.assertEqual(403, response.status_code) - def test_get_endpoint_with_token_auth_and_invalid_id_participant_group(self): with self._flask_app.app_context(): params = {'id_participant_group': -1} @@ -39,7 +36,6 @@ def test_get_endpoint_with_token_auth_and_invalid_id_participant_group(self): params=params, endpoint=self.test_endpoint) self.assertEqual(403, response.status_code) - def test_get_endpoint_with_token_auth_and_invalid_id_project_and_invalid_id_participant_group(self): with self._flask_app.app_context(): params = {'id_project': -1, 'id_participant_group': -1} @@ -47,7 +43,6 @@ def test_get_endpoint_with_token_auth_and_invalid_id_project_and_invalid_id_part params=params, endpoint=self.test_endpoint) self.assertEqual(403, response.status_code) - def test_get_endpoint_with_token_auth_and_valid_id_participant_group(self): with self._flask_app.app_context(): params = {'id_participant_group': 1} @@ -58,14 +53,13 @@ def test_get_endpoint_with_token_auth_and_valid_id_participant_group(self): group: TeraParticipantGroup = TeraParticipantGroup.get_participant_group_by_id(1) self.assertEqual(group.to_json(minimal=True), response.json) - def test_get_endpoint_with_token_auth_and_valid_id_project(self): with self._flask_app.app_context(): params = {'id_project': 1} response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, params=params, endpoint=self.test_endpoint) self.assertEqual(200, response.status_code) - self.assertGreaterEqual(len(response.json),1 ) + self.assertGreaterEqual(len(response.json), 1) groups: TeraParticipantGroup = TeraParticipantGroup.get_participant_group_for_project(1) i = 0 for group in groups: @@ -89,7 +83,6 @@ def test_get_endpoint_with_token_auth_and_valid_but_denied_id_project(self): params=params, endpoint=self.test_endpoint) self.assertEqual(403, response.status_code) - def test_get_endpoint_with_token_auth_and_valid_but_denied_id_participant_group(self): with self._flask_app.app_context(): params = {'id_participant_group': 2} @@ -97,8 +90,6 @@ def test_get_endpoint_with_token_auth_and_valid_but_denied_id_participant_group( params=params, endpoint=self.test_endpoint) self.assertEqual(403, response.status_code) - - def test_post_and_delete_endpoint_with_token(self): with self._flask_app.app_context(): # Test case: Post with missing information @@ -155,7 +146,6 @@ def test_post_and_delete_endpoint_with_token(self): self.assertEqual(1, project_id) self.assertEqual("New name", name) - # Test case: Creation json_data['participant_group']['id_participant_group'] = 0 json_data['participant_group']['id_project'] = 1 @@ -166,7 +156,6 @@ def test_post_and_delete_endpoint_with_token(self): project_id = group_data['id_project'] self.assertEqual(1, project_id) - # Test case: Post update to project without association to service json_data = { 'participant_group': { @@ -199,7 +188,6 @@ def test_post_and_delete_endpoint_with_token(self): json=json_data) self.assertEqual(500, response.status_code, msg="Invalid parameter") - del json_data['participant_group']['invalid_parameter'] response = self._post_with_service_token_auth(self.test_client, token=self.service_token, json=json_data) From fa2282f4695a3b23cd3b08c673735147705738a6 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 4 Mar 2024 05:23:16 -0500 Subject: [PATCH 20/29] Fixes #239. Added support for "datetime.date" in to_json method --- teraserver/python/opentera/db/Base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/teraserver/python/opentera/db/Base.py b/teraserver/python/opentera/db/Base.py index 7d177f413..0574e3156 100755 --- a/teraserver/python/opentera/db/Base.py +++ b/teraserver/python/opentera/db/Base.py @@ -60,7 +60,7 @@ def to_json(self, ignore_fields=None): if name == 'deleted_at' and value is None: continue # If deleted field, but not deleted, don't add to the json if self.is_valid_property_value(value): - if isinstance(value, datetime.datetime): + if isinstance(value, datetime.datetime) or isinstance(value, datetime.date): value = value.isoformat() if isinstance(value, datetime.timedelta): # Strip too many zeros at the end @@ -132,10 +132,10 @@ def clean_values(cls, values: dict): @classmethod def get_count(cls, filters: dict = None, with_deleted: bool = False) -> int: - query = cls.db().session.query(cls).execution_options(include_deleted=with_deleted) + count_query = cls.db().session.query(cls).execution_options(include_deleted=with_deleted) if filters: - query = query.filter_by(**filters) - return query.count() + count_query = count_query.filter_by(**filters) + return count_query.count() @classmethod def get_primary_key_name(cls) -> str: From 4710321b91cab1491e396cc799e6233a17363186 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 4 Mar 2024 06:38:35 -0500 Subject: [PATCH 21/29] Refs #240. Added search by name feature for ServiceQueryParticipants --- .../API/service/ServiceQueryParticipants.py | 37 +++++++- .../opentera/db/models/TeraParticipant.py | 9 ++ .../service/test_ServiceQueryParticipants.py | 93 +++++++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipants.py b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipants.py index 43d8da92e..6b7e8f0f1 100644 --- a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipants.py +++ b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryParticipants.py @@ -2,8 +2,9 @@ from flask_restx import Resource from flask_babel import gettext from sqlalchemy.exc import IntegrityError -from modules.LoginModule.LoginModule import LoginModule +from modules.LoginModule.LoginModule import LoginModule, current_service from modules.FlaskModule.FlaskModule import service_api_ns as api +from modules.DatabaseModule.DBManager import DBManager from opentera.db.models.TeraParticipant import TeraParticipant import uuid from datetime import datetime @@ -11,6 +12,9 @@ # Parser definition(s) get_parser = api.parser() get_parser.add_argument('participant_uuid', type=str, help='Participant uuid of the participant to query') +get_parser.add_argument('id_project', type=int, help='Project ID to query all participants for') +get_parser.add_argument('id_participant_group', type=int, help='Participant group to query all participants for') +get_parser.add_argument('name', type=str, help='Return participants with at least a partial match on their name.') post_parser = api.parser() @@ -60,12 +64,43 @@ def __init__(self, _api, *args, **kwargs): def get(self): args = get_parser.parse_args() + service_access = DBManager.serviceAccess(current_service) + # args['participant_uuid'] Will be None if not specified in args if args['participant_uuid']: + if args['participant_uuid'] not in service_access.get_accessible_participants_uuids(): + return gettext('Forbidden'), 403 participant = TeraParticipant.get_participant_by_uuid(args['participant_uuid']) if participant: return participant.to_json() + if args['id_project']: + if args['id_project'] not in service_access.get_accessible_projects_ids(): + return gettext('Forbidden'), 403 + filters = {'id_project': args['id_project']} + if not args['name']: + participants = TeraParticipant.query_with_filters(filters) + else: + participants = TeraParticipant.search_participant_by_name(args['name'], filters) + return [participant.to_json() for participant in participants] + + if args['id_participant_group']: + if args['id_participant_group'] not in service_access.get_accessible_participants_groups_ids(): + return gettext('Forbidden'), 403 + filters = {'id_participant_group': args['id_participant_group']} + if not args['name']: + participants = TeraParticipant.query_with_filters(filters) + else: + participants = TeraParticipant.search_participant_by_name(args['name'], filters) + return [participant.to_json() for participant in participants] + + if args['name']: + # Search for participants with name in all availables + participants = (TeraParticipant.query.filter( + TeraParticipant.id_participant.in_(service_access.get_accessible_participants_ids())) + .filter(TeraParticipant.participant_name.like('%' + args['name'] + '%')).all()) + return [participant.to_json(minimal=True) for participant in participants] + return gettext('Missing arguments'), 400 @api.doc(description='Update participant', diff --git a/teraserver/python/opentera/db/models/TeraParticipant.py b/teraserver/python/opentera/db/models/TeraParticipant.py index 009efd775..51a369c62 100644 --- a/teraserver/python/opentera/db/models/TeraParticipant.py +++ b/teraserver/python/opentera/db/models/TeraParticipant.py @@ -272,6 +272,15 @@ def is_participant_username_available(username: str) -> bool: return TeraParticipant.query.filter_by(participant_username=username).first() is None + @staticmethod + def search_participant_by_name(name: str, other_filters: dict | None): + if other_filters is None: + other_filters = dict() + search_query = (TeraParticipant.query.filter_by(**other_filters). + filter(TeraParticipant.participant_name.like('%' + name + '%'))) + return search_query.all() + + @staticmethod def create_defaults(test=False): if test: diff --git a/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipants.py b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipants.py index 3e112d2a4..e354afe37 100644 --- a/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipants.py +++ b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryParticipants.py @@ -35,6 +35,17 @@ def test_get_endpoint_with_token_auth_with_wrong_params(self): params=params, endpoint=self.test_endpoint) self.assertEqual(400, response.status_code) + def test_get_endpoint_with_token_auth_with_forbidden_uuid(self): + with self._flask_app.app_context(): + # Get all participants from DB + secret_participant = TeraParticipant.get_participant_by_name('Secret Participant') + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params={'participant_uuid': + secret_participant.participant_uuid}, + endpoint=self.test_endpoint) + + self.assertEqual(403, response.status_code) + def test_get_endpoint_with_token_auth_with_participant_uuid(self): with self._flask_app.app_context(): # Get all participants from DB @@ -47,6 +58,88 @@ def test_get_endpoint_with_token_auth_with_participant_uuid(self): participant_json = participant.to_json() self.assertEqual(participant_json, response.json) + def test_get_endpoint_with_project_id(self): + with self._flask_app.app_context(): + project_id = 1 + params = {'id_project': project_id} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + participants_count = TeraParticipant.get_count({'id_project': project_id}) + self.assertEqual(participants_count, len(response.json)) + + def test_get_endpoint_with_forbidden_project_id(self): + with self._flask_app.app_context(): + project_id = 2 + params = {'id_project': project_id} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(403, response.status_code) + + def test_get_endpoint_with_participant_group_id(self): + with self._flask_app.app_context(): + group_id = 1 + params = {'id_participant_group': group_id} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + participants_count = TeraParticipant.get_count({'id_participant_group': group_id}) + self.assertEqual(participants_count, len(response.json)) + + def test_get_endpoint_with_forbidden_participant_group_id(self): + with self._flask_app.app_context(): + group_id = 2 + params = {'id_participant_group': group_id} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(403, response.status_code) + + def test_get_endpoint_search_by_name_in_project(self): + with self._flask_app.app_context(): + params = {'id_project': 1, 'name': '#1'} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(response.json)) # Only one participant has "#1" in their name + + params = {'id_project': 1, 'name': 'iciPAnt'} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + # Should return all participants in the project, since they all have that pattern in their name + self.assertEqual(len(TeraParticipant.query_with_filters({'id_project': 1})), len(response.json)) + + def test_get_endpoint_search_by_name_in_group(self): + with self._flask_app.app_context(): + params = {'id_participant_group': 1, 'name': '#2'} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + self.assertEqual(0, len(response.json)) # No participant has "#1" in their name in that group + + params = {'id_participant_group': 1, 'name': 'ICipant'} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + # Should return all participants in the project, since they all have that pattern in their name + self.assertEqual(len(TeraParticipant.query_with_filters({'id_participant_group': 1})), len(response.json)) + + def test_get_endpoint_search_by_name_global(self): + with self._flask_app.app_context(): + params = {'name': 'Secret'} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + # No participant available with that pattern (even if it exists) + self.assertEqual(0, len(response.json)) + + params = {'name': 'ICipant'} + response = self._get_with_service_token_auth(client=self.test_client, token=self.service_token, + params=params, endpoint=self.test_endpoint) + self.assertEqual(200, response.status_code) + # Should return all participants, but only from the accessible project (1) for this service + self.assertEqual(len(TeraParticipant.query_with_filters({'id_project': 1})), len(response.json)) + def test_post_endpoint_without_token_auth(self): with self._flask_app.app_context(): response = self.test_client.post(self.test_endpoint, json={}) From 864a7a7c20e7208d796ffb1c56e5858d2474ec86 Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 4 Mar 2024 09:48:26 -0500 Subject: [PATCH 22/29] Refs #241. Added projects and sites access from ServiceQueryUserGroups API --- .../DBManagerTeraServiceAccess.py | 43 +++++++++++++++++-- .../API/service/ServiceQueryUserGroups.py | 10 +++++ .../service/test_ServiceQueryUserGroups.py | 28 ++++++++++++ 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/teraserver/python/modules/DatabaseModule/DBManagerTeraServiceAccess.py b/teraserver/python/modules/DatabaseModule/DBManagerTeraServiceAccess.py index 126815f87..0bfe63961 100644 --- a/teraserver/python/modules/DatabaseModule/DBManagerTeraServiceAccess.py +++ b/teraserver/python/modules/DatabaseModule/DBManagerTeraServiceAccess.py @@ -5,7 +5,7 @@ from opentera.db.models.TeraServiceRole import TeraServiceRole from opentera.db.models.TeraProject import TeraProject -from sqlalchemy import or_, not_ +from sqlalchemy import or_, not_, and_ class DBManagerTeraServiceAccess: @@ -143,10 +143,22 @@ def get_accessible_users_ids(self, admin_only=False): return [user.id_user for user in self.get_accessible_users(admin_only=admin_only)] def get_accessible_usergroups(self): + # Usergroup is accessible if it has a direct association to this service + access = TeraServiceAccess.query.join(TeraServiceRole). \ + filter(TeraServiceRole.id_service == self.service.id_service). \ + filter(TeraServiceAccess.id_user_group != None).all() + # Usergroup is accessible if it has access to a service site / project or if it has a role in the service - access = TeraServiceAccess.query.join(TeraServiceRole).\ - filter(TeraServiceRole.id_service == self.service.id_service).\ - filter(TeraServiceAccess.id_user_group is not None).all() + # project_ids = self.get_accessible_projects_ids() + # site_ids = self.get_accessibles_sites_ids() + # access = TeraServiceAccess.query.join(TeraServiceRole).\ + # filter(or_(TeraServiceRole.id_service == self.service.id_service, + # and_(TeraServiceRole.id_service == TeraService.get_openteraserver_service().id_service, + # or_(TeraServiceRole.id_project == None, TeraServiceRole.id_project.in_(project_ids)), + # or_(TeraServiceRole.id_site == None, TeraServiceRole.id_site.in_(site_ids)), + # ).self_group() + # ) + # ).filter(TeraServiceAccess.id_user_group != None).all() usergroups = [] if access: @@ -274,3 +286,26 @@ def query_session(self, session_id: int): return session return None + + def query_usergroups_for_site(self, site_id: int): + all_users_groups = self.get_accessible_usergroups() + users_groups = {} + for user_group in all_users_groups: + sites = {key.id_site: value for key, value in user_group.get_sites_roles().items() + if key.id_site == site_id} + if site_id in sites: + users_groups[user_group] = sites[site_id] + return users_groups + + def query_usergroups_for_project(self, project_id: int): + all_users_groups = self.get_accessible_usergroups() + users_groups = {} + for user_group in all_users_groups: + + projects = {key.id_project: value for key, value in user_group.get_projects_roles().items() + if key.id_project == project_id} + + if project_id in projects: + users_groups[user_group] = projects[project_id] + + return users_groups diff --git a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryUserGroups.py b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryUserGroups.py index 58e39638b..faf0455f8 100644 --- a/teraserver/python/modules/FlaskModule/API/service/ServiceQueryUserGroups.py +++ b/teraserver/python/modules/FlaskModule/API/service/ServiceQueryUserGroups.py @@ -13,6 +13,8 @@ # Parser definition(s) get_parser = api.parser() get_parser.add_argument('id_user_group', type=int, help='ID of the user group to query') +get_parser.add_argument('id_project', type=int, help='ID of the project to query user group with access to') +get_parser.add_argument('id_site', type=int, help='ID of the site to query user group with access to') post_parser = api.parser() post_schema = api.schema_model('service_user_group', {'properties': TeraUserGroup.get_json_schema(), 'type': 'object', @@ -43,6 +45,14 @@ def get(self): if args['id_user_group']: if args['id_user_group'] in service_access.get_accessible_usergroups_ids(): user_groups.append(TeraUserGroup.get_user_group_by_id(args['id_user_group'])) + elif args['id_project']: + if args['id_project'] not in service_access.get_accessible_projects_ids(): + return gettext('Forbidden'), 403 + user_groups = service_access.query_usergroups_for_project(args['id_project']) + elif args['id_site']: + if args['id_site'] not in service_access.get_accessibles_sites_ids(): + return gettext('Forbidden'), 403 + user_groups = service_access.query_usergroups_for_site(args['id_site']) else: # If we have no arguments, return all accessible user groups user_groups = service_access.get_accessible_usergroups() diff --git a/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryUserGroups.py b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryUserGroups.py index d9434942e..2d81f19c0 100644 --- a/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryUserGroups.py +++ b/teraserver/python/tests/modules/FlaskModule/API/service/test_ServiceQueryUserGroups.py @@ -87,6 +87,34 @@ def test_query_specific_user_group(self): self._checkJson(json_data[0], minimal=True) self.assertEqual(json_data[0]['id_user_group'], 1) + def test_get_usergroups_for_forbidden_project(self): + with self._flask_app.app_context(): + params = {'id_project': 2} + response = self._get_with_service_token_auth(self.test_client, token=self.service_token, params=params) + self.assertEqual(403, response.status_code) + + def test_get_usergroups_for_project(self): + with self._flask_app.app_context(): + params = {'id_project': 1} + response = self._get_with_service_token_auth(self.test_client, token=self.service_token, params=params) + self.assertEqual(200, response.status_code) + for json_data in response.json: + self._checkJson(json_data) + + def test_get_usergroups_for_forbidden_site(self): + with self._flask_app.app_context(): + params = {'id_site': 2} + response = self._get_with_service_token_auth(self.test_client, token=self.service_token, params=params) + self.assertEqual(403, response.status_code) + + def test_get_usergroups_for_site(self): + with self._flask_app.app_context(): + params = {'id_site': 1} + response = self._get_with_service_token_auth(self.test_client, token=self.service_token, params=params) + self.assertEqual(200, response.status_code) + for json_data in response.json: + self._checkJson(json_data) + def test_post_and_delete(self): with self._flask_app.app_context(): json_data = { From b114b7d3829c496c15e00ebee7e2872368d07c4a Mon Sep 17 00:00:00 2001 From: Simon Briere Date: Mon, 4 Mar 2024 10:52:58 -0500 Subject: [PATCH 23/29] Refs #242. Reworked camera management when unable to get a stream from the selected one (and first one in case of enumeration). --- teraserver/easyrtc/protected/index_users.html | 2 +- teraserver/easyrtc/static/js/tera_medias.js | 69 ++++++++++++++----- teraserver/easyrtc/static/js/tera_webrtc.js | 13 +++- 3 files changed, 62 insertions(+), 22 deletions(-) diff --git a/teraserver/easyrtc/protected/index_users.html b/teraserver/easyrtc/protected/index_users.html index e14690b37..7b0a42687 100644 --- a/teraserver/easyrtc/protected/index_users.html +++ b/teraserver/easyrtc/protected/index_users.html @@ -130,7 +130,7 @@