diff --git a/.github/workflows/testing-all-oses.yml b/.github/workflows/testing-all-oses.yml index 86fb996fc..58df9ccdb 100644 --- a/.github/workflows/testing-all-oses.yml +++ b/.github/workflows/testing-all-oses.yml @@ -48,6 +48,4 @@ jobs: # I have no idea yet on why this happens and how to fix it. # Even a module level skip is not enough, they need to be completely ignored. # TODO: fix those tests and drop the ignores - run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib - --ignore=tests/_test_msui/test_sideview.py --ignore=tests/_test_msui/test_topview.py --ignore=tests/_test_msui/test_wms_control.py - tests + run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib tests diff --git a/.github/workflows/testing-scheduled.yml b/.github/workflows/testing-scheduled.yml index a4bab8ae0..580003604 100644 --- a/.github/workflows/testing-scheduled.yml +++ b/.github/workflows/testing-scheduled.yml @@ -10,7 +10,7 @@ jobs: permissions: actions: write steps: - - uses: benc-uk/workflow-dispatch@v1.2.3 + - uses: benc-uk/workflow-dispatch@v1.2.4 with: workflow: testing-stable.yml ref: stable @@ -20,7 +20,7 @@ jobs: permissions: actions: write steps: - - uses: benc-uk/workflow-dispatch@v1.2.3 + - uses: benc-uk/workflow-dispatch@v1.2.4 with: workflow: testing-develop.yml ref: develop diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 0e1f739f6..ae1eb46c7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -40,7 +40,7 @@ jobs: # scheduled tests should just check that new dependency versions do not break the # tests, but should not update the image. if: ${{ (github.ref_name == 'stable' || github.ref_name == 'develop') && github.event_name == 'push' && env.triggerdockerbuild == 'yes' && matrix.order == 'normal' }} - uses: benc-uk/workflow-dispatch@v1.2.3 + uses: benc-uk/workflow-dispatch@v1.2.4 with: workflow: Update Image testing-${{ inputs.image_suffix }} repo: Open-MSS/dockertesting diff --git a/CHANGES.rst b/CHANGES.rst index a92d20429..b1aef8422 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,16 @@ Changelog ========= +Version 9.2.0 +~~~~~~~~~~~~~ + +Bug fix release and minor enhancements: +We added a verification for xml data and changed the startup of the SocketsManager. + +All changes: +https://github.com/Open-MSS/MSS/milestone/106?closed=1 + + Version 9.1.0 ~~~~~~~~~~~~~ diff --git a/docs/conf.py b/docs/conf.py index bbd933540..80e5e185d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -226,7 +226,7 @@ def get_tutorial_images(): html_style = 'css/mss.css' else: htmls_static_path = ['_static'] - html_css_files = ['mss.css'] + html_css_files = ['css/mss.css'] html_context = { 'display_github': False, # Add 'Edit on Bitbucket' link instead of 'View page source' 'last_updated': True, diff --git a/docs/development.rst b/docs/development.rst index c7968a2d0..ecbf42d07 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -399,6 +399,26 @@ like e.g. a running MSColab server or a QApplication instance for GUI tests, are collected in :mod:`tests.fixtures` in the form of pytest fixtures that can be requested as needed in tests. +Changing the database model +--------------------------- + +Changing the database model requires adding a corresponding migration script to MSS, +so that existing databases can be migrated automatically. + +To generate such a migration script you can run:: + + flask --app mslib.mscolab.app db migrate -d mslib/mscolab/migrations -m "To version " + +Depending on the complexity of the changes that were made, +the generated migration script might need some tweaking. + +If there is already a migration script for the next release, +then please incorporate the generated migration script into this existing one, +instead of adding a new one. +You can still generate a script with the above command first +to get a starting point for the changes. + + Pushing your changes -------------------- diff --git a/docs/environment.yml b/docs/environment.yml index 8e703ec7b..2a0fbaf92 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -15,7 +15,7 @@ dependencies: - xstatic - defusedxml - sphinx_rtd_theme - - sphinxcontrib-video + - sphinxcontrib-video>=0.2.1 - sphinx - fs - netCDF4 diff --git a/docs/mscolab.rst b/docs/mscolab.rst index ec482b220..80459d6ad 100644 --- a/docs/mscolab.rst +++ b/docs/mscolab.rst @@ -150,75 +150,24 @@ by `pg_dump `_ using a pg_dump -d mscolab -f "/home/mscolab/dump/$timestamp.sql" -Data Base Migration from version 8 -.................................. - -.. important:: - This manual migration on the server side by a user is deprecated and will become removed with version 10.0.0. - With version 10.0.0, the initialization of the database will be refactored and migrations will be performed automatically when mscolab is started - -For an easy way to update the database scheme we implemented `flask migrate `_. - -You have to create based on your configuration a migration script and call that afterwards. :: - - mamba activate instance - cd ~/INSTANCE/config - export PYTHONPATH=`pwd` - cd ~/INSTANCE/wsgi - flask --app mscolab.py db init - flask --app mscolab.py db migrate -m "To version 9.0.0" - flask --app mscolab.py db upgrade - -The migration script builder does the base but the created script needs first allow nullable so that we afterwards set the default for the existing data. -Use this as an example for your script :: - - """To version 9.0.0 - - Revision ID: e62d08ce88a4 - Revises: 27a026b0daec - Create Date: 2024-06-07 16:53:43.314338 - - """ - from alembic import op - import sqlalchemy as sa - - - # revision identifiers, used by Alembic. - revision = 'e62d08ce88a4' - down_revision = '27a026b0daec' - branch_labels = None - depends_on = None - - - def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.add_column(sa.Column('authentication_backend', sa.String(length=255), nullable=True)) - op.execute('UPDATE users SET authentication_backend = \'local\' WHERE authentication_backend IS NULL') - - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.alter_column('authentication_backend', existing_type=sa.String(length=255), nullable=False) - batch_op.drop_constraint('users_password_key', type_='unique') - - # ### end Alembic commands ### - - - def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.create_unique_constraint('users_password_key', ['password']) - batch_op.drop_column('authentication_backend') - - # ### end Alembic commands ### - - -The output looks like :: - - ~/INSTANCE/wsgi$ flask --app mscolab.py db upgrade - INFO [alembic.runtime.migration] Context impl SQLiteImpl. - INFO [alembic.runtime.migration] Will assume non-transactional DDL. - INFO [alembic.runtime.migration] Running upgrade -> e62d08ce88a, To version 9.0.0 - +Database Migration from Version 8 or 9 +................................. + +From v10 onwards MSColab uses `Flask-Migrate ` to automatically deal with database migrations. +To upgrade from v8 or v9 a recreation of the database and subsequent copy of existing data is necessary. +To do this follow these steps: + +#. Stop MSColab completely, no process interacting with the MSColab database should remain running +#. **Make a backup of your existing database** +#. Set ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` to your existing database +#. Set ``SQLALCHEMY_DB_URI`` to a new database +#. If you are not using SQLite: create the new database +#. Start MSColab +#. Check that everything was migrated successfully +#. Unset ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` + +If you want to keep using your old database URI you can first rename your existing database so that it has a different URI +and just set ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` to that. Steps to use the MSColab UI features diff --git a/docs/mss_theme/css/mss.css b/docs/mss_theme/css/mss.css index de441f52a..58fa4d1b7 100644 --- a/docs/mss_theme/css/mss.css +++ b/docs/mss_theme/css/mss.css @@ -14,3 +14,7 @@ margin-bottom: auto; margin-left: auto; } + +video { + max-width: 100%; +} diff --git a/docs/samples/automation/retriever.py b/docs/samples/automation/retriever.py index 0226c9c6c..989faa95f 100644 --- a/docs/samples/automation/retriever.py +++ b/docs/samples/automation/retriever.py @@ -31,6 +31,7 @@ import io import os import xml +import defusedxml.minidom import requests from fs import open_fs import PIL.Image @@ -58,7 +59,7 @@ def load_from_ftml(filename): _fs = open_fs(_dirname) datasource = _fs.open(_name) try: - doc = xml.dom.minidom.parse(datasource) + doc = defusedxml.minidom.parse(datasource) except xml.parsers.expat.ExpatError as ex: raise SyntaxError(str(ex)) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index a188127e3..8fe213c4b 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -9,7 +9,7 @@ a `public share `_ Get familiar by some videos about the Mission Support System (MSS). - .. video:: _static/mp4/tutorial_waypoints.mp4 + .. video:: videos/mp4/tutorial_waypoints.mp4 :autoplay: :loop: :alt: This is the waypoints tutorial,i.e., whenever we are going to plan diff --git a/docs/tutorials/tutorial_hexagoncontrol.rst b/docs/tutorials/tutorial_hexagoncontrol.rst index 0c3dae428..a212a70e1 100644 --- a/docs/tutorials/tutorial_hexagoncontrol.rst +++ b/docs/tutorials/tutorial_hexagoncontrol.rst @@ -3,7 +3,7 @@ Table View and Hexagon Flight Patterns For tomographic imaging, a hexagonal flight pattern can be integrated by a docking widget of Table View - .. video:: ../_static/mp4/tutorial_hexagoncontrol.mp4 + .. video:: ../videos/mp4/tutorial_hexagoncontrol.mp4 :alt: Top View windows is opened (CTRL+H). We select the "global (cyl)" for the world map. Zooming in for the required region. diff --git a/docs/tutorials/tutorial_introduction_topview.rst b/docs/tutorials/tutorial_introduction_topview.rst index b4edd8a1f..b72815f05 100644 --- a/docs/tutorials/tutorial_introduction_topview.rst +++ b/docs/tutorials/tutorial_introduction_topview.rst @@ -3,7 +3,7 @@ Top View and Selecting of Layers Selection and display of different data in the Top View with the help of the layer chooser. - .. video:: ../_static/mp4/tutorial_wms.mp4 + .. video:: ../videos/mp4/tutorial_wms.mp4 :alt: When we open the Top View (CTRL+H) of the map, the Web Map Service is already opened by default. It collects its data from the server: "open-mss dot org" that provides demodata for the meteorological or atmospheric information as layer lists. diff --git a/docs/tutorials/tutorial_kml.rst b/docs/tutorials/tutorial_kml.rst index 5d8c3d951..68a85903a 100644 --- a/docs/tutorials/tutorial_kml.rst +++ b/docs/tutorials/tutorial_kml.rst @@ -3,7 +3,7 @@ Top View and KML Data es can be displayed in the Top View. Color and line width can be adjusted. - .. video:: ../_static/mp4/tutorial_kml.mp4 + .. video:: ../videos/mp4/tutorial_kml.mp4 :alt: Open the TopView (CTRL+H) After clicking on "(select to open control)", click on KML OVERLAY. The UI will look as shown. KML files can be used to show the geographical boundary which helps in planning the WAY POINTS. @@ -23,4 +23,4 @@ es can be displayed in the Top View. Color and line width can be adjusted. Changing it for some time... Now, we change line width by changing its numerical value whose range is 0 to 10. We change it to different values to obtain different linewidths. - The top view is closed and the tutorial ends. \ No newline at end of file + The top view is closed and the tutorial ends. diff --git a/docs/tutorials/tutorial_mscolab.rst b/docs/tutorials/tutorial_mscolab.rst index c4164fa9c..7d9cb4a92 100644 --- a/docs/tutorials/tutorial_mscolab.rst +++ b/docs/tutorials/tutorial_mscolab.rst @@ -5,7 +5,7 @@ Using the different views of the MSUI with a fictitious flight path and demo dat In comparison to the standalone mode of the MSUI an example setup of users is shown on a MSColab server and the possibilities of interactions. - .. video:: ../_static/mp4/tutorial_mscolab.mp4 + .. video:: ../videos/mp4/tutorial_mscolab.mp4 :alt: MSColab stores data in an online server, and can be used to access the data remotely as also working in a team where everyone contributes his part. It is used for collaborating with the users as a team together and working on a shared MSColab operation. diff --git a/docs/tutorials/tutorial_msui_views.rst b/docs/tutorials/tutorial_msui_views.rst index 7e1ba7dcd..27910e9ab 100644 --- a/docs/tutorials/tutorial_msui_views.rst +++ b/docs/tutorials/tutorial_msui_views.rst @@ -4,7 +4,7 @@ Introduction to MSUI Using the different views of the MSUI with a fictitious flight path and demo data. - .. video:: ../_static/mp4//tutorial_views.mp4 + .. video:: ../videos/mp4/tutorial_views.mp4 :alt: Lets look at the tutorial of the various views required for flight planning: Top View (CTRL+H), Side View (CTRL+V), Linear View (CTRL+L) and Table View (CTRL+T). At first, lets open Top View and Side View. diff --git a/docs/tutorials/tutorial_performance_settings.rst b/docs/tutorials/tutorial_performance_settings.rst index 82954355a..59820e2c2 100644 --- a/docs/tutorials/tutorial_performance_settings.rst +++ b/docs/tutorials/tutorial_performance_settings.rst @@ -3,7 +3,7 @@ Table View and Aircraft Performance Data The range-specific data of an aircraft can be taken into account in Tableview for flight planning. - .. video:: ../_static/mp4/tutorial_performancesettings.mp4 + .. video:: ../videos/mp4/tutorial_performancesettings.mp4 :alt: This is the Performance Settings dockwidget opened in the Table View (CTRL+T) of the MSS software where by putting and changing some parameters, we can evaluate the performance of the aircraft. Parameters like Flight Altitude, Aviation fuel, Aircraft weight, Maximum take off weight diff --git a/docs/tutorials/tutorial_remotesensing.rst b/docs/tutorials/tutorial_remotesensing.rst index e1a1d1253..65d1dbf4e 100644 --- a/docs/tutorials/tutorial_remotesensing.rst +++ b/docs/tutorials/tutorial_remotesensing.rst @@ -4,7 +4,7 @@ Top View and Remotesensing Tools In order to be able to take into account the viewing angle and solar level for measuring instruments, the remotesensing tools are used - .. video:: ../_static/mp4/tutorial_remotesensing.mp4 + .. video:: ../videos/mp4/tutorial_remotesensing.mp4 :alt: This is the Remote Sensing Section of the Top View. It shows the position and angle of the flight from any particular object in the sky. Azimuth is the forward direction line of the flight. If we go above tHE AZIMUTH, angle is in positive, diff --git a/docs/tutorials/tutorial_satellitetrack.rst b/docs/tutorials/tutorial_satellitetrack.rst index 04561d811..7c6b55550 100644 --- a/docs/tutorials/tutorial_satellitetrack.rst +++ b/docs/tutorials/tutorial_satellitetrack.rst @@ -4,7 +4,7 @@ Top View and Satellite Overflight To combine a flight path with a satellite overflight, the remotesensing widget is used. - .. video:: ../_static/mp4/tutorial_satellitetrack.mp4 + .. video:: ../videos/mp4/tutorial_satellitetrack.mp4 :alt: This is Satellite Tracking Prediction System that can be used to check the accuracy of the path travelled by a Satellite by the help of data collected from different space agencies and planning a flight accordingly. diff --git a/docs/tutorials/tutorial_waypoints.rst b/docs/tutorials/tutorial_waypoints.rst index e43d21f3d..87938a01c 100644 --- a/docs/tutorials/tutorial_waypoints.rst +++ b/docs/tutorials/tutorial_waypoints.rst @@ -3,7 +3,7 @@ Top View Drawing Waypoints Waypoints for a flight path are defined, shifted and deleted. - .. video:: ../_static/mp4/tutorial_waypoints.mp4 + .. video:: ../videos/mp4/tutorial_waypoints.mp4 :alt: This is the waypoints tutorial,i.e., whenever we are going to plan a flight track, we have to place the waypoints in some places and, form a flight path. diff --git a/localbuild/meta.yaml b/localbuild/meta.yaml index 3088dfce2..9e2b52fdc 100644 --- a/localbuild/meta.yaml +++ b/localbuild/meta.yaml @@ -77,7 +77,7 @@ requirements: - metpy - pycountry - websocket-client - - libtiff <4.5.0 + - libtiff - flask-wtf - email_validator - keyring diff --git a/mslib/mscolab/app/__init__.py b/mslib/mscolab/app/__init__.py index 9cf33a150..374383e1c 100644 --- a/mslib/mscolab/app/__init__.py +++ b/mslib/mscolab/app/__init__.py @@ -25,6 +25,7 @@ """ import os +import sqlalchemy from flask_migrate import Migrate @@ -62,8 +63,23 @@ APP.config['MAIL_USE_TLS'] = getattr(mscolab_settings, "MAIL_USE_TLS", None) APP.config['MAIL_USE_SSL'] = getattr(mscolab_settings, "MAIL_USE_SSL", None) -db = SQLAlchemy(APP) -migrate = Migrate(APP, db, render_as_batch=True) +db = SQLAlchemy( + metadata=sqlalchemy.MetaData( + naming_convention={ + # For reference: https://alembic.sqlalchemy.org/en/latest/naming.html#the-importance-of-naming-constraints + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_`%(constraint_name)s`", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", + }, + ), +) +db.init_app(APP) +import mslib.mscolab.models + +migrate = Migrate(render_as_batch=True, user_module_prefix="cu.") +migrate.init_app(APP, db) def get_topmenu(): diff --git a/mslib/mscolab/chat_manager.py b/mslib/mscolab/chat_manager.py index e1d35624e..0d0c92bdb 100644 --- a/mslib/mscolab/chat_manager.py +++ b/mslib/mscolab/chat_manager.py @@ -38,16 +38,16 @@ class ChatManager: def __init__(self): pass - def add_message(self, user, text, operation_name, message_type=MessageType.TEXT, reply_id=None): + def add_message(self, user, text, op_id, message_type=MessageType.TEXT, reply_id=None): """ - text: message to be emitted to operation and saved to db - operation_name: operation-name(op_id) to which message is emitted, user: User object, one which emits the message + text: message to be emitted to operation and saved to db + op_id: operation id to which message is emitted, message_type: Enum of type MessageType. values: TEXT, SYSTEM_MESSAGE, IMAGE, DOCUMENT """ if reply_id == -1: reply_id = None - message = Message(operation_name, user.id, text, message_type, reply_id) + message = Message(op_id, user.id, text, message_type, reply_id) db.session.add(message) db.session.commit() return message diff --git a/mslib/mscolab/conf.py b/mslib/mscolab/conf.py index af083085c..4321f7ab2 100644 --- a/mslib/mscolab/conf.py +++ b/mslib/mscolab/conf.py @@ -63,6 +63,9 @@ class default_mscolab_settings: # MYSQL CONNECTION STRING: "mysql+pymysql://:@:/?charset=utf8mb4" SQLALCHEMY_DB_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'mscolab.db') + # SQLAlchemy connection string to migrate data from, if set + SQLALCHEMY_DB_URI_TO_MIGRATE_FROM = None + # Set to True for testing and False for production SQLALCHEMY_ECHO = False diff --git a/mslib/mscolab/custom_migration_types.py b/mslib/mscolab/custom_migration_types.py new file mode 100644 index 000000000..f54f3a90d --- /dev/null +++ b/mslib/mscolab/custom_migration_types.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" + + mslib.mscolab.custom_migration_types + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Consolidated module of all custom/non-standard database types used in MSColab. + + This file is part of MSS. + + :copyright: Copyright 2024 Matthias Riße + :copyright: Copyright 2024 by the MSS team, see AUTHORS. + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +from mslib.mscolab.models import AwareDateTime # noqa: F401 diff --git a/mslib/mscolab/file_manager.py b/mslib/mscolab/file_manager.py index 3cfdada71..a80333f89 100644 --- a/mslib/mscolab/file_manager.py +++ b/mslib/mscolab/file_manager.py @@ -36,6 +36,7 @@ import mimetypes from werkzeug.utils import secure_filename from sqlalchemy.exc import IntegrityError +from mslib.utils.verify_waypoint_data import verify_waypoint_data from mslib.mscolab.models import db, Operation, Permission, User, Change, Message from mslib.mscolab.conf import mscolab_settings @@ -58,9 +59,19 @@ def _get_operation_lock(self, op_id): def create_operation(self, path, description, user, last_used=None, content=None, category="default", active=True): """ - path: path to the operation - description: description of the operation + Creates a new operation in the mscolab system. + + :param path: The path of the operation. + :param description: The description of the operation. + :param user: The user object creating the operation. + :param last_used: The last used datetime of the operation. Default is None. + :param content: The content of the operation. Default is None. + :param category: The category of the operation. Default is 'default'. + :param active: The activity status of the operation. Default is True. + :return: True if the operation is created successfully, False otherwise. """ + if content is not None and not verify_waypoint_data(content): + return False # set codes on these later if path.find("/") != -1 or path.find("\\") != -1 or (" " in path): logging.debug("malicious request: %s", user) @@ -339,14 +350,14 @@ def update_operation(self, op_id, attribute, value, user): if value.find("/") != -1 or value.find("\\") != -1 or (" " in value): logging.debug("malicious request: %s", user) return False - data = fs.open_fs(self.data_dir) - if data.exists(value): - return False - # will be move when operations are introduced - # make a directory, else movedir - data.makedir(value) - data.movedir(operation.path, value) - # when renamed to a Group operation + with fs.open_fs(self.data_dir) as data: + if data.exists(value): + return False + # will be move when operations are introduced + # make a directory, else movedir + data.makedir(value) + data.movedir(operation.path, value) + # when renamed to a Group operation if value.endswith(mscolab_settings.GROUP_POSTFIX): # getting the category category = value.split(mscolab_settings.GROUP_POSTFIX)[0] @@ -395,6 +406,8 @@ def save_file(self, op_id, content, user, comment=""): content: content of the file to be saved # ToDo save change in schema """ + if not verify_waypoint_data(content): + return False # ToDo use comment operation = Operation.query.filter_by(id=op_id).first() if not operation: diff --git a/mslib/mscolab/migrations/env.py b/mslib/mscolab/migrations/env.py index 9bb35bb93..7b3a3322d 100644 --- a/mslib/mscolab/migrations/env.py +++ b/mslib/mscolab/migrations/env.py @@ -45,7 +45,7 @@ # target_metadata = mymodel.Base.metadata config.set_main_option( 'sqlalchemy.url', - str(current_app.extensions['migrate'].db.get_engine().url).replace( + str(current_app.extensions['migrate'].db.engine.url).replace( '%', '%%')) target_metadata = current_app.extensions['migrate'].db.metadata @@ -94,7 +94,7 @@ def process_revision_directives(context, revision, directives): directives[:] = [] logger.info('No changes in schema detected.') - connectable = current_app.extensions['migrate'].db.get_engine() + connectable = current_app.extensions['migrate'].db.engine with connectable.connect() as connection: context.configure( diff --git a/mslib/mscolab/migrations/script.py.mako b/mslib/mscolab/migrations/script.py.mako index 2c0156303..2943f763e 100644 --- a/mslib/mscolab/migrations/script.py.mako +++ b/mslib/mscolab/migrations/script.py.mako @@ -7,6 +7,7 @@ Create Date: ${create_date} """ from alembic import op import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu ${imports if imports else ""} # revision identifiers, used by Alembic. diff --git a/mslib/mscolab/migrations/versions/83993fcdf5ef_.py b/mslib/mscolab/migrations/versions/83993fcdf5ef_.py deleted file mode 100644 index 1fa5d6c53..000000000 --- a/mslib/mscolab/migrations/versions/83993fcdf5ef_.py +++ /dev/null @@ -1,92 +0,0 @@ -# flake8: noqa -""" - -DB setup 6.x - -Revision ID: 83993fcdf5ef -Revises: cd8b7108713a -Create Date: 2021-10-04 09:22:36.987652 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '83993fcdf5ef' -down_revision = 'cd8b7108713a' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('operations', - sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('path', sa.String(length=255), nullable=True), - sa.Column('category', sa.String(length=255), nullable=True), - sa.Column('description', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('path') - ) - op.drop_table('projects') - with op.batch_alter_table('changes', schema=None) as batch_op: - batch_op.add_column(sa.Column('op_id', sa.Integer(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'operations', ['op_id'], ['id']) - batch_op.drop_column('p_id') - - with op.batch_alter_table('messages', schema=None) as batch_op: - batch_op.add_column(sa.Column('op_id', sa.Integer(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'operations', ['op_id'], ['id']) - batch_op.drop_column('p_id') - - with op.batch_alter_table('permissions', schema=None) as batch_op: - batch_op.add_column(sa.Column('op_id', sa.Integer(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'operations', ['op_id'], ['id']) - batch_op.drop_column('p_id') - - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.add_column(sa.Column('registered_on', sa.DateTime(), nullable=False)) - batch_op.add_column(sa.Column('confirmed', sa.Boolean(), nullable=False)) - batch_op.add_column(sa.Column('confirmed_on', sa.DateTime(), nullable=True)) - - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - with op.batch_alter_table('users', schema=None) as batch_op: - batch_op.drop_column('confirmed_on') - batch_op.drop_column('confirmed') - batch_op.drop_column('registered_on') - - with op.batch_alter_table('permissions', schema=None) as batch_op: - batch_op.add_column(sa.Column('p_id', sa.INTEGER(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'projects', ['p_id'], ['id']) - batch_op.drop_column('op_id') - - with op.batch_alter_table('messages', schema=None) as batch_op: - batch_op.add_column(sa.Column('p_id', sa.INTEGER(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'projects', ['p_id'], ['id']) - batch_op.drop_column('op_id') - - with op.batch_alter_table('changes', schema=None) as batch_op: - batch_op.add_column(sa.Column('p_id', sa.INTEGER(), nullable=True)) - batch_op.drop_constraint(None, type_='foreignkey') - batch_op.create_foreign_key(None, 'projects', ['p_id'], ['id']) - batch_op.drop_column('op_id') - - op.create_table('projects', - sa.Column('id', sa.INTEGER(), nullable=False), - sa.Column('path', sa.VARCHAR(length=255), nullable=True), - sa.Column('description', sa.VARCHAR(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('path') - ) - op.drop_table('operations') - # ### end Alembic commands ### diff --git a/mslib/mscolab/migrations/versions/922e4d9c94e2_to_version_10_0_0.py b/mslib/mscolab/migrations/versions/922e4d9c94e2_to_version_10_0_0.py new file mode 100644 index 000000000..c03710ee4 --- /dev/null +++ b/mslib/mscolab/migrations/versions/922e4d9c94e2_to_version_10_0_0.py @@ -0,0 +1,33 @@ +"""To version 10.0.0 + +Revision ID: 922e4d9c94e2 +Revises: c171019fe3ee +Create Date: 2024-07-24 15:28:42.009581 + +""" +from alembic import op +import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu + + +# revision identifiers, used by Alembic. +revision = '922e4d9c94e2' +down_revision = 'c171019fe3ee' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('profile_image_path', sa.String(length=255), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.drop_column('profile_image_path') + + # ### end Alembic commands ### diff --git a/mslib/mscolab/migrations/versions/cd8b7108713a_.py b/mslib/mscolab/migrations/versions/92eaba86a92e_to_version_8_3_5_initial_migration.py similarity index 51% rename from mslib/mscolab/migrations/versions/cd8b7108713a_.py rename to mslib/mscolab/migrations/versions/92eaba86a92e_to_version_8_3_5_initial_migration.py index 970b96816..fc307551a 100644 --- a/mslib/mscolab/migrations/versions/cd8b7108713a_.py +++ b/mslib/mscolab/migrations/versions/92eaba86a92e_to_version_8_3_5_initial_migration.py @@ -1,18 +1,17 @@ -# flake8: noqa -""" -DB setup until v6.x +"""To version 8.3.5 - Initial migration -Revision ID: cd8b7108713a +Revision ID: 92eaba86a92e Revises: -Create Date: 2021-10-04 09:10:35.606793 +Create Date: 2024-07-08 15:47:24.916851 """ from alembic import op import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu # revision identifiers, used by Alembic. -revision = 'cd8b7108713a' +revision = '92eaba86a92e' down_revision = None branch_labels = None depends_on = None @@ -20,55 +19,61 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('projects', + op.create_table('operations', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), sa.Column('path', sa.String(length=255), nullable=True), + sa.Column('category', sa.String(length=255), nullable=True), sa.Column('description', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('path') + sa.Column('active', sa.Boolean(), nullable=True), + sa.Column('last_used', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_operations')), + sa.UniqueConstraint('path', name=op.f('uq_operations_path')) ) op.create_table('users', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), sa.Column('username', sa.String(length=255), nullable=True), sa.Column('emailid', sa.String(length=255), nullable=True), sa.Column('password', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('emailid'), - sa.UniqueConstraint('password') + sa.Column('registered_on', sa.DateTime(), nullable=False), + sa.Column('confirmed', sa.Boolean(), nullable=False), + sa.Column('confirmed_on', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_users')), + sa.UniqueConstraint('emailid', name=op.f('uq_users_emailid')), + sa.UniqueConstraint('password', name=op.f('uq_users_password')) ) op.create_table('changes', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('p_id', sa.Integer(), nullable=True), + sa.Column('op_id', sa.Integer(), nullable=True), sa.Column('u_id', sa.Integer(), nullable=True), sa.Column('commit_hash', sa.String(length=255), nullable=True), sa.Column('version_name', sa.String(length=255), nullable=True), sa.Column('comment', sa.String(length=255), nullable=True), sa.Column('created_at', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['p_id'], ['projects.id'], ), - sa.ForeignKeyConstraint(['u_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') + sa.ForeignKeyConstraint(['op_id'], ['operations.id'], name=op.f('fk_changes_op_id_operations')), + sa.ForeignKeyConstraint(['u_id'], ['users.id'], name=op.f('fk_changes_u_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_changes')) ) op.create_table('messages', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('p_id', sa.Integer(), nullable=True), + sa.Column('op_id', sa.Integer(), nullable=True), sa.Column('u_id', sa.Integer(), nullable=True), sa.Column('text', sa.Text(), nullable=True), sa.Column('message_type', sa.Enum('TEXT', 'SYSTEM_MESSAGE', 'IMAGE', 'DOCUMENT', name='messagetype'), nullable=True), sa.Column('reply_id', sa.Integer(), nullable=True), sa.Column('created_at', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['p_id'], ['projects.id'], ), - sa.ForeignKeyConstraint(['reply_id'], ['messages.id'], ), - sa.ForeignKeyConstraint(['u_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') + sa.ForeignKeyConstraint(['op_id'], ['operations.id'], name=op.f('fk_messages_op_id_operations')), + sa.ForeignKeyConstraint(['reply_id'], ['messages.id'], name=op.f('fk_messages_reply_id_messages')), + sa.ForeignKeyConstraint(['u_id'], ['users.id'], name=op.f('fk_messages_u_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_messages')) ) op.create_table('permissions', sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), - sa.Column('p_id', sa.Integer(), nullable=True), + sa.Column('op_id', sa.Integer(), nullable=True), sa.Column('u_id', sa.Integer(), nullable=True), sa.Column('access_level', sa.Enum('admin', 'collaborator', 'viewer', 'creator', name='access_level'), nullable=True), - sa.ForeignKeyConstraint(['p_id'], ['projects.id'], ), - sa.ForeignKeyConstraint(['u_id'], ['users.id'], ), - sa.PrimaryKeyConstraint('id') + sa.ForeignKeyConstraint(['op_id'], ['operations.id'], name=op.f('fk_permissions_op_id_operations')), + sa.ForeignKeyConstraint(['u_id'], ['users.id'], name=op.f('fk_permissions_u_id_users')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_permissions')) ) # ### end Alembic commands ### @@ -76,8 +81,10 @@ def upgrade(): def downgrade(): # ### commands auto generated by Alembic - please adjust! ### op.drop_table('permissions') + sa.Enum(name='access_level').drop(op.get_bind(), checkfirst=False) op.drop_table('messages') + sa.Enum(name='messagetype').drop(op.get_bind(), checkfirst=False) op.drop_table('changes') op.drop_table('users') - op.drop_table('projects') + op.drop_table('operations') # ### end Alembic commands ### diff --git a/mslib/mscolab/migrations/versions/c171019fe3ee_to_version_9_0_0.py b/mslib/mscolab/migrations/versions/c171019fe3ee_to_version_9_0_0.py new file mode 100644 index 000000000..6b3e4ebda --- /dev/null +++ b/mslib/mscolab/migrations/versions/c171019fe3ee_to_version_9_0_0.py @@ -0,0 +1,35 @@ +"""To version 9.0.0 + +Revision ID: c171019fe3ee +Revises: 92eaba86a92e +Create Date: 2024-07-08 15:49:08.277483 + +""" +from alembic import op +import sqlalchemy as sa +import mslib.mscolab.custom_migration_types as cu + + +# revision identifiers, used by Alembic. +revision = 'c171019fe3ee' +down_revision = '92eaba86a92e' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.add_column(sa.Column('authentication_backend', sa.String(length=255), nullable=False, default='local')) + batch_op.drop_constraint('uq_users_password', type_='unique') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('users', schema=None) as batch_op: + batch_op.create_unique_constraint('uq_users_password', ['password']) + batch_op.drop_column('authentication_backend') + + # ### end Alembic commands ### diff --git a/mslib/mscolab/models.py b/mslib/mscolab/models.py index abac5406c..742583fb7 100644 --- a/mslib/mscolab/models.py +++ b/mslib/mscolab/models.py @@ -67,14 +67,14 @@ class User(db.Model): def __init__(self, emailid, username, password, profile_image_path=None, confirmed=False, confirmed_on=None, authentication_backend='local'): - self.username = username - self.emailid = emailid + self.username = str(username) + self.emailid = str(emailid) self.hash_password(password) self.profile_image_path = profile_image_path self.registered_on = datetime.datetime.now(tz=datetime.timezone.utc) - self.confirmed = confirmed + self.confirmed = bool(confirmed) self.confirmed_on = confirmed_on - self.authentication_backend = authentication_backend + self.authentication_backend = str(authentication_backend) def __repr__(self): return f'' @@ -137,9 +137,9 @@ def __init__(self, u_id, op_id, access_level): op_id: process-id access_level: the type of authorization to the operation """ - self.u_id = u_id - self.op_id = op_id - self.access_level = access_level + self.u_id = int(u_id) + self.op_id = int(op_id) + self.access_level = str(access_level) def __repr__(self): return f'' @@ -161,10 +161,10 @@ def __init__(self, path, description, last_used=None, category="default", active description: small description of operation category: name of category """ - self.path = path - self.description = description - self.category = category - self.active = active + self.path = str(path) + self.description = str(description) + self.category = str(category) + self.active = bool(active) if self.last_used is None: self.last_used = datetime.datetime.now(tz=datetime.timezone.utc) else: @@ -190,9 +190,9 @@ class Message(db.Model): replies = db.relationship('Message', cascade='all,delete,delete-orphan', single_parent=True) def __init__(self, op_id, u_id, text, message_type=MessageType.TEXT, reply_id=None): - self.op_id = op_id - self.u_id = u_id - self.text = text + self.op_id = int(op_id) + self.u_id = int(u_id) + self.text = str(text) self.message_type = message_type self.reply_id = reply_id @@ -213,8 +213,8 @@ class Change(db.Model): user = db.relationship('User') def __init__(self, op_id, u_id, commit_hash, version_name=None, comment=None): - self.op_id = op_id - self.u_id = u_id - self.commit_hash = commit_hash - self.version_name = version_name - self.comment = comment + self.op_id = int(op_id) + self.u_id = int(u_id) + self.commit_hash = str(commit_hash) + self.version_name = str(version_name) + self.comment = str(comment) diff --git a/mslib/mscolab/mscolab.py b/mslib/mscolab/mscolab.py index b0a79969d..445dc2f7c 100644 --- a/mslib/mscolab/mscolab.py +++ b/mslib/mscolab/mscolab.py @@ -34,11 +34,15 @@ import secrets import subprocess import git +import flask_migrate +import pathlib from mslib import __version__ +from mslib.mscolab import migrations from mslib.mscolab.conf import mscolab_settings from mslib.mscolab.seed import seed_data, add_user, add_all_users_default_operation, \ add_all_users_to_all_operations, delete_user +from mslib.mscolab.server import APP from mslib.mscolab.utils import create_files from mslib.utils import setup_logging from mslib.utils.qt import Worker, Updater @@ -65,24 +69,25 @@ def confirm_action(confirmation_prompt): print("Invalid input! Please select an option between y or n") -def handle_db_init(): - from mslib.mscolab.models import db - from mslib.mscolab.server import APP - create_files() - with APP.app_context(): - db.create_all() - print("Database initialised successfully!") - - def handle_db_reset(verbose=True): - from mslib.mscolab.models import db - from mslib.mscolab.server import APP - if os.path.exists(mscolab_settings.DATA_DIR): + if mscolab_settings.SQLALCHEMY_DB_URI.startswith("sqlite:///") and ( + db_path := pathlib.Path(mscolab_settings.SQLALCHEMY_DB_URI.removeprefix("sqlite:///")) + ).is_relative_to(mscolab_settings.DATA_DIR): + # Don't remove the database file + # This would be easier if the database wasn't stored in DATA_DIR... + p = pathlib.Path(mscolab_settings.DATA_DIR) + for root, dirs, files in os.walk(p, topdown=False): + for name in files: + full_file_path = pathlib.Path(root) / name + if full_file_path != db_path: + full_file_path.unlink() + for name in dirs: + (pathlib.Path(root) / name).rmdir() + elif os.path.exists(mscolab_settings.DATA_DIR): shutil.rmtree(mscolab_settings.DATA_DIR) create_files() - with APP.app_context(): - db.drop_all() - db.create_all() + flask_migrate.downgrade(directory=migrations.__path__[0], revision="base") + flask_migrate.upgrade(directory=migrations.__path__[0]) if verbose is True: print("Database has been reset successfully!") @@ -365,7 +370,6 @@ def main(): database_parser = subparsers.add_parser("db", help="Manage mscolab database") database_parser = database_parser.add_mutually_exclusive_group(required=True) - database_parser.add_argument("--init", help="Initialise database", action="store_true") database_parser.add_argument("--reset", help="Reset database", action="store_true") database_parser.add_argument("--seed", help="Seed database", action="store_true") database_parser.add_argument("--users_by_file", type=argparse.FileType('r'), @@ -417,18 +421,18 @@ def main(): handle_start(args) elif args.action == "db": - if args.init: - handle_db_init() - elif args.reset: + if args.reset: confirmation = confirm_action("Are you sure you want to reset the database? This would delete " "all your data! (y/[n]):") if confirmation is True: - handle_db_reset() + with APP.app_context(): + handle_db_reset() elif args.seed: confirmation = confirm_action("Are you sure you want to seed the database? Seeding will delete all your " "existing data and replace it with seed data (y/[n]):") if confirmation is True: - handle_db_seed() + with APP.app_context(): + handle_db_seed() elif args.users_by_file is not None: # fileformat: suggested_username name confirmation = confirm_action("Are you sure you want to add users to the database? (y/[n]):") diff --git a/mslib/mscolab/seed.py b/mslib/mscolab/seed.py index 366e221c5..8d9f23457 100644 --- a/mslib/mscolab/seed.py +++ b/mslib/mscolab/seed.py @@ -226,178 +226,175 @@ def archive_operation(path=None, emailid=None): def seed_data(): - app.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI - app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - with app.app_context(): - # create users - users = [{ - 'username': 'a', - 'id': 8, - 'password': 'a', - 'emailid': 'a@notexisting.org' - }, { - 'username': 'b', - 'id': 9, - 'password': 'b', - 'emailid': 'b@notexisting.org' - }, { - 'username': 'c', - 'id': 10, - 'password': 'c', - 'emailid': 'c@notexisting.org' - }, { - 'username': 'd', - 'id': 11, - 'password': 'd', - 'emailid': 'd@notexisting.org' - }, { - 'username': 'test1', - 'id': 12, - 'password': 'test1', - 'emailid': 'test1@notexisting.org' - }, { - 'username': 'test2', - 'id': 13, - 'password': 'test2', - 'emailid': 'test2@notexisting.org' - }, { - 'username': 'test3', - 'id': 14, - 'password': 'test3', - 'emailid': 'test3@notexisting.org' - }, { - 'username': 'test4', - 'id': 15, - 'password': 'test4', - 'emailid': 'test4@notexisting.org' - }, { - 'username': 'mscolab_user', - 'id': 16, - 'password': 'password', - 'emailid': 'mscolab_user@notexisting.org' - }, { - 'username': 'merge_waypoints_user', - 'id': 17, - 'password': 'password', - 'emailid': 'merge_waypoints_user@notexisting.org' - }] - for user in users: - db_user = User(user['emailid'], user['username'], user['password']) - db_user.id = user['id'] - db.session.add(db_user) - - # create operations - operations = [{ - 'id': 1, - 'path': 'one', - 'description': 'a, b', - 'category': 'default' - }, { - 'id': 2, - 'path': 'two', - 'description': 'b, c', - 'category': 'default' - }, { - 'id': 3, - 'path': 'three', - 'description': 'a, c', - 'category': 'default' - }, { - 'id': 4, - 'path': 'four', - 'description': 'd', - 'category': 'default' - }, { - 'id': 5, - 'path': 'Admin_Test', - 'description': 'Operation for testing admin window', - 'category': 'default' - }, { - 'id': 6, - 'path': 'test_mscolab', - 'description': 'Operation for testing mscolab main window', - 'category': 'default' - }] - for operation in operations: - db_operation = Operation(operation['path'], operation['description'], operation['category']) - db_operation.id = operation['id'] - db.session.add(db_operation) - - # create permissions - permissions = [{ - 'u_id': 8, - 'op_id': 1, - 'access_level': "creator" - }, { - 'u_id': 9, - 'op_id': 1, - 'access_level': "collaborator" - }, { - 'u_id': 9, - 'op_id': 2, - 'access_level': "creator" - }, { - 'u_id': 10, - 'op_id': 2, - 'access_level': "collaborator" - }, { - 'u_id': 10, - 'op_id': 3, - 'access_level': "creator" - }, { - 'u_id': 8, - 'op_id': 3, - 'access_level': "collaborator" - }, { - 'u_id': 10, - 'op_id': 1, - 'access_level': "viewer" - }, { - 'u_id': 11, - 'op_id': 4, - 'access_level': 'creator' - }, { - 'u_id': 8, - 'op_id': 4, - 'access_level': 'admin' - }, { - 'u_id': 13, - 'op_id': 3, - 'access_level': 'viewer' - }, { - 'u_id': 12, - 'op_id': 5, - 'access_level': 'creator' - }, { - 'u_id': 12, - 'op_id': 3, - 'access_level': 'collaborator' - }, { - 'u_id': 15, - 'op_id': 5, - 'access_level': 'viewer' - }, { - 'u_id': 14, - 'op_id': 3, - 'access_level': 'collaborator' - }, { - 'u_id': 15, - 'op_id': 3, - 'access_level': 'collaborator' - }, { - 'u_id': 16, - 'op_id': 6, - 'access_level': 'creator' - }, { - 'u_id': 17, - 'op_id': 6, - 'access_level': 'admin' - }] - for perm in permissions: - db_perm = Permission(perm['u_id'], perm['op_id'], perm['access_level']) - db.session.add(db_perm) - db.session.commit() - db.session.close() + # create users + users = [{ + 'username': 'a', + 'id': 8, + 'password': 'a', + 'emailid': 'a@notexisting.org' + }, { + 'username': 'b', + 'id': 9, + 'password': 'b', + 'emailid': 'b@notexisting.org' + }, { + 'username': 'c', + 'id': 10, + 'password': 'c', + 'emailid': 'c@notexisting.org' + }, { + 'username': 'd', + 'id': 11, + 'password': 'd', + 'emailid': 'd@notexisting.org' + }, { + 'username': 'test1', + 'id': 12, + 'password': 'test1', + 'emailid': 'test1@notexisting.org' + }, { + 'username': 'test2', + 'id': 13, + 'password': 'test2', + 'emailid': 'test2@notexisting.org' + }, { + 'username': 'test3', + 'id': 14, + 'password': 'test3', + 'emailid': 'test3@notexisting.org' + }, { + 'username': 'test4', + 'id': 15, + 'password': 'test4', + 'emailid': 'test4@notexisting.org' + }, { + 'username': 'mscolab_user', + 'id': 16, + 'password': 'password', + 'emailid': 'mscolab_user@notexisting.org' + }, { + 'username': 'merge_waypoints_user', + 'id': 17, + 'password': 'password', + 'emailid': 'merge_waypoints_user@notexisting.org' + }] + for user in users: + db_user = User(user['emailid'], user['username'], user['password']) + db_user.id = user['id'] + db.session.add(db_user) + + # create operations + operations = [{ + 'id': 1, + 'path': 'one', + 'description': 'a, b', + 'category': 'default' + }, { + 'id': 2, + 'path': 'two', + 'description': 'b, c', + 'category': 'default' + }, { + 'id': 3, + 'path': 'three', + 'description': 'a, c', + 'category': 'default' + }, { + 'id': 4, + 'path': 'four', + 'description': 'd', + 'category': 'default' + }, { + 'id': 5, + 'path': 'Admin_Test', + 'description': 'Operation for testing admin window', + 'category': 'default' + }, { + 'id': 6, + 'path': 'test_mscolab', + 'description': 'Operation for testing mscolab main window', + 'category': 'default' + }] + for operation in operations: + db_operation = Operation(operation['path'], operation['description'], operation['category']) + db_operation.id = operation['id'] + db.session.add(db_operation) + + # create permissions + permissions = [{ + 'u_id': 8, + 'op_id': 1, + 'access_level': "creator" + }, { + 'u_id': 9, + 'op_id': 1, + 'access_level': "collaborator" + }, { + 'u_id': 9, + 'op_id': 2, + 'access_level': "creator" + }, { + 'u_id': 10, + 'op_id': 2, + 'access_level': "collaborator" + }, { + 'u_id': 10, + 'op_id': 3, + 'access_level': "creator" + }, { + 'u_id': 8, + 'op_id': 3, + 'access_level': "collaborator" + }, { + 'u_id': 10, + 'op_id': 1, + 'access_level': "viewer" + }, { + 'u_id': 11, + 'op_id': 4, + 'access_level': 'creator' + }, { + 'u_id': 8, + 'op_id': 4, + 'access_level': 'admin' + }, { + 'u_id': 13, + 'op_id': 3, + 'access_level': 'viewer' + }, { + 'u_id': 12, + 'op_id': 5, + 'access_level': 'creator' + }, { + 'u_id': 12, + 'op_id': 3, + 'access_level': 'collaborator' + }, { + 'u_id': 15, + 'op_id': 5, + 'access_level': 'viewer' + }, { + 'u_id': 14, + 'op_id': 3, + 'access_level': 'collaborator' + }, { + 'u_id': 15, + 'op_id': 3, + 'access_level': 'collaborator' + }, { + 'u_id': 16, + 'op_id': 6, + 'access_level': 'creator' + }, { + 'u_id': 17, + 'op_id': 6, + 'access_level': 'admin' + }] + for perm in permissions: + db_perm = Permission(perm['u_id'], perm['op_id'], perm['access_level']) + db.session.add(db_perm) + db.session.commit() + db.session.close() with fs.open_fs(mscolab_settings.MSCOLAB_DATA_DIR) as file_dir: file_paths = ['one', 'two', 'three', 'four', 'Admin_Test', 'test_mscolab'] diff --git a/mslib/mscolab/server.py b/mslib/mscolab/server.py index 0034d6455..9edc0747a 100644 --- a/mslib/mscolab/server.py +++ b/mslib/mscolab/server.py @@ -34,6 +34,7 @@ import socketio import sqlalchemy.exc import werkzeug +import flask_migrate from itsdangerous import URLSafeTimedSerializer, BadSignature from flask import g, jsonify, request, render_template, flash @@ -53,9 +54,101 @@ from mslib.utils import conditional_decorator from mslib.index import create_app from mslib.mscolab.forms import ResetRequestForm, ResetPasswordForm +from mslib.mscolab import migrations + + +def _handle_db_upgrade(): + from mslib.mscolab.models import db + + create_files() + inspector = sqlalchemy.inspect(db.engine) + existing_tables = inspector.get_table_names() + if ("alembic_version" not in existing_tables and len(existing_tables) > 0) or ( + "alembic_version" in existing_tables + and len(existing_tables) > 1 + and db.session.execute(sqlalchemy.text("SELECT * FROM alembic_version")).first() is None + ): + sys.exit( + """Your database contains no alembic_version revision identifier, but it has a schema. This suggests \ +that you have a pre-existing database but haven't followed the database migration instructions. To prevent damage to \ +your database MSColab will abort. Please follow the documentation for a manual database migration from MSColab v8/v9.""" + ) + + is_empty_database = len(existing_tables) == 0 or ( + len(existing_tables) == 1 + and "alembic_version" in existing_tables + and db.session.execute(sqlalchemy.text("SELECT * FROM alembic_version")).first() is None + ) + # If a database connection to migrate from is set and the target database is empty, then migrate the existing data + if is_empty_database and mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM is not None: + logging.info("The target database is empty and a database to migrate from is set, starting the data migration") + source_engine = sqlalchemy.create_engine(mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM) + source_metadata = sqlalchemy.MetaData() + source_metadata.reflect(bind=source_engine) + # Determine the previous MSColab version based on the database content and upgrade to the corresponding revision + if "authentication_backend" in source_metadata.tables["users"].columns: + # It should be v9 + flask_migrate.upgrade(directory=migrations.__path__[0], revision="c171019fe3ee") + else: + # It's probably v8 + flask_migrate.upgrade(directory=migrations.__path__[0], revision="92eaba86a92e") + # Copy over the existing data + target_engine = sqlalchemy.create_engine(mscolab_settings.SQLALCHEMY_DB_URI) + target_metadata = sqlalchemy.MetaData() + target_metadata.reflect(bind=target_engine) + with source_engine.connect() as src_connection, target_engine.connect() as target_connection: + for table in source_metadata.sorted_tables: + if table.name == "alembic_version": + # Do not migrate the alembic_version table! + continue + logging.debug("Copying table %s", table.name) + stmt = target_metadata.tables[table.name].insert() + for row in src_connection.execute(table.select()): + logging.debug("Copying row %s", row) + row = tuple( + r.replace(tzinfo=datetime.timezone.utc) if isinstance(r, datetime.datetime) else r for r in row + ) + target_connection.execute(stmt.values(row)) + target_connection.commit() + if target_engine.name == "postgresql": + # Fix the databases auto-increment sequences, if it is a PostgreSQL database + # For reference, see: https://wiki.postgresql.org/wiki/Fixing_Sequences + logging.info("Using a PostgreSQL database, will fix up sequences") + cur = target_connection.execute(sqlalchemy.text(r""" +SELECT + 'SELECT SETVAL(' || + quote_literal(quote_ident(sequence_namespace.nspname) || '.' || quote_ident(class_sequence.relname)) || + ', COALESCE(MAX(' ||quote_ident(pg_attribute.attname)|| '), 1) ) FROM ' || + quote_ident(table_namespace.nspname)|| '.'||quote_ident(class_table.relname)|| ';' +FROM pg_depend + INNER JOIN pg_class AS class_sequence + ON class_sequence.oid = pg_depend.objid + AND class_sequence.relkind = 'S' + INNER JOIN pg_class AS class_table + ON class_table.oid = pg_depend.refobjid + INNER JOIN pg_attribute + ON pg_attribute.attrelid = class_table.oid + AND pg_depend.refobjsubid = pg_attribute.attnum + INNER JOIN pg_namespace as table_namespace + ON table_namespace.oid = class_table.relnamespace + INNER JOIN pg_namespace AS sequence_namespace + ON sequence_namespace.oid = class_sequence.relnamespace +ORDER BY sequence_namespace.nspname, class_sequence.relname; +""")) + for stmt, in cur.all(): + target_connection.execute(sqlalchemy.text(stmt)) + target_connection.commit() + logging.info("Data migration finished") + + # Upgrade to the latest database revision + flask_migrate.upgrade(directory=migrations.__path__[0]) + + logging.info("Database initialised successfully!") APP = create_app(__name__, imprint=mscolab_settings.IMPRINT, gdpr=mscolab_settings.GDPR) +with APP.app_context(): + _handle_db_upgrade() mail = Mail(APP) CORS(APP, origins=mscolab_settings.CORS_ORIGINS if hasattr(mscolab_settings, "CORS_ORIGINS") else ["*"]) auth = HTTPBasicAuth() diff --git a/mslib/msui/flighttrack.py b/mslib/msui/flighttrack.py index 5f9903f7a..3ca4760bc 100644 --- a/mslib/msui/flighttrack.py +++ b/mslib/msui/flighttrack.py @@ -48,6 +48,7 @@ from mslib.utils.units import units from mslib.utils.coordinate import find_location, path_points, get_distance from mslib.utils import thermolib +from mslib.utils.verify_waypoint_data import verify_waypoint_data from mslib.utils.config import config_loader, save_settings_qsettings, load_settings_qsettings from mslib.utils.config import MSUIDefaultConfig as mss_default from mslib.utils.qt import variant_to_string, variant_to_float @@ -665,8 +666,11 @@ def load_from_ftml(self, filename): def load_from_xml_data(self, xml_content, name="Flight track"): self.name = name - _waypoints_list = load_from_xml_data(xml_content, name) - self.replace_waypoints(_waypoints_list) + if verify_waypoint_data(xml_content): + _waypoints_list = load_from_xml_data(xml_content, name) + self.replace_waypoints(_waypoints_list) + else: + raise SyntaxError(f"Invalid flight track filename: {name}") def get_filename(self): return self.filename diff --git a/mslib/msui/mscolab.py b/mslib/msui/mscolab.py index b977fb475..6b9fa8993 100644 --- a/mslib/msui/mscolab.py +++ b/mslib/msui/mscolab.py @@ -61,6 +61,7 @@ from mslib.utils.auth import get_password_from_keyring, save_password_to_keyring from mslib.utils.verify_user_token import verify_user_token +from mslib.utils.verify_waypoint_data import verify_waypoint_data from mslib.utils.qt import get_open_filename, get_save_filename, dropEvent, dragEnterEvent, show_popup from mslib.msui.qt5 import ui_mscolab_help_dialog as msc_help_dialog from mslib.msui.qt5 import ui_add_operation_dialog as add_operation_ui @@ -656,6 +657,14 @@ def after_login(self, emailid, url, r): self.user = _json["user"] self.mscolab_server_url = url + if config_loader(dataset="MSCOLAB_skip_archived_operations"): + self.ui.pbOpenOperationArchive.setEnabled(False) + self.ui.pbOpenOperationArchive.setToolTip( + "This button is disabled to the config option 'MSCOLAB_skip_archived_operations'") + else: + self.ui.pbOpenOperationArchive.setEnabled(True) + self.ui.pbOpenOperationArchive.setToolTip("") + # create socket connection here try: self.conn = sc.ConnectionManager(self.token, user=self.user, mscolab_server_url=self.mscolab_server_url) @@ -2070,10 +2079,12 @@ def handle_import_msc(self, file_path, extension, function, pickertype): return dir_path, file_name = fs.path.split(file_path) file_name = fs.path.basename(file_path) - name, file_ext = fs.path.splitext(file_name) if function is None: with open_fs(dir_path) as file_dir: xml_content = file_dir.readtext(file_name) + if not verify_waypoint_data(xml_content): + show_popup(self.ui, "Import Failed", f"The file - {file_name}, does not contain valid XML") + return try: model = ft.WaypointsTableModel(xml_content=xml_content) except SyntaxError: @@ -2085,6 +2096,9 @@ def handle_import_msc(self, file_path, extension, function, pickertype): model = ft.WaypointsTableModel(waypoints=new_waypoints) xml_doc = self.waypoints_model.get_xml_doc() xml_content = xml_doc.toprettyxml(indent=" ", newl="\n") + if not verify_waypoint_data(xml_content): + show_popup(self.ui, "Import Failed", f"The file - {file_name}, was not imported!", 0) + return self.waypoints_model.dataChanged.disconnect(self.handle_waypoints_changed) self.waypoints_model = model self.waypoints_model.changeMessageSignal.connect(self.handle_change_message) diff --git a/mslib/static/docs/help.md b/mslib/static/docs/help.md index 5ea50caa7..c0a2c0777 100644 --- a/mslib/static/docs/help.md +++ b/mslib/static/docs/help.md @@ -8,7 +8,7 @@ software that simplifies the process for planning a scientific flight. # Drawing waypoints in the MSUI Topview The example shows defining of waypoints for a flight path, moved and deleted. -![Waypoint Tutorial](https://mss.readthedocs.io/en/stable/_static/mp4/tutorial_waypoints.mp4) +![Waypoint Tutorial](https://mss.readthedocs.io/en/stable/_images/tutorial_waypoints.mp4) Further tutorials about the Mission Support System Software on: diff --git a/mslib/static/docs/installation.md b/mslib/static/docs/installation.md index 8c8051e33..b31e57473 100644 --- a/mslib/static/docs/installation.md +++ b/mslib/static/docs/installation.md @@ -7,7 +7,7 @@ The Mission Support System (MSS) including a Web Map Service a Collaboration Ser [conda-forge](https://anaconda.org/conda-forge/mss) package. -We strongly recommend to start from [Mambaforge](https://mamba.readthedocs.io/en/latest/installation.html) +We strongly recommend to start from [Miniforge3](https://github.com/conda-forge/miniforge?tab=readme-ov-file#miniforge3) a community project of the conda-forge community. You can install it either automatically with the help of a script or manually. @@ -31,8 +31,8 @@ You can install it either automatically with the help of a script or manually. ### Manually -As **Beginner** start with an installation of Mambaforge -Get [mambaforge](https://github.com/conda-forge/miniforge#mambaforge) for your Operation System +As **Beginner** start with an installation of Miniforge3 +Get [Miniforge3](https://github.com/conda-forge/miniforge?tab=readme-ov-file#miniforge3) for your Operation System You must install mss into a new environment to ensure the most recent @@ -65,10 +65,10 @@ user for the apache2 wsgi script. We suggest to create a mss user. - login as mss user - create a *src* directory in /home/mss - cd src -- get [mambaforge](https://github.com/conda-forge/miniforge#mambaforge) +- get [Miniforge3](https://github.com/conda-forge/miniforge?tab=readme-ov-file#miniforge3) - set execute bit on install script - execute script, enable environment in .bashrc -- login again or export PATH="/home/mss/mambaforge/bin:\$PATH" +- start your shell again (new login) - python --version should tell Python 3.X.X - mamba create -n mssenv - mamba activate mssenv diff --git a/mslib/utils/verify_waypoint_data.py b/mslib/utils/verify_waypoint_data.py new file mode 100644 index 000000000..455912bfe --- /dev/null +++ b/mslib/utils/verify_waypoint_data.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" + + mslib.utils.verify_waypoint_data + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + basic checks for xml waypoint data. + + This file is part of MSS. + + :copyright: Copyright 2024 Reimar Bauer + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + + +import defusedxml.minidom +import xml.parsers.expat + + +def verify_waypoint_data(xml_content): + try: + doc = defusedxml.minidom.parseString(xml_content) + except xml.parsers.expat.ExpatError: + return False + + ft_el = doc.getElementsByTagName("FlightTrack")[0] + waypoints = ft_el.getElementsByTagName("Waypoint") + if (len(waypoints)) < 2: + return False + + for wp_el in ft_el.getElementsByTagName("Waypoint"): + try: + wp_el.getAttribute("location") + float(wp_el.getAttribute("lat")) + float(wp_el.getAttribute("lon")) + float(wp_el.getAttribute("flightlevel")) + wp_el.getElementsByTagName("Comments")[0] + except ValueError: + return False + + return True diff --git a/mslib/version.py b/mslib/version.py index a179012a9..9181d64ab 100644 --- a/mslib/version.py +++ b/mslib/version.py @@ -24,4 +24,4 @@ See the License for the specific language governing permissions and limitations under the License. """ -__version__ = u'9.1.0' +__version__ = u'9.2.0' diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 232063ce6..d6403b27c 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -20,4 +20,3 @@ eventlet>0.30.2 dnspython>=2.0.0, <2.3.0 gsl==2.7.0 boa -xmlschema<2.5.0 diff --git a/setup.cfg b/setup.cfg index 147b23cfb..1c8297f6e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,6 @@ omit = norecursedirs = .git .idea .cache [flake8] -ignore = E124,E125,E402,W504,A005 +ignore = E124,E125,E402,W503,W504,A005 max-line-length = 120 exclude = mslib/msui/qt5/*.py, mslib/mscolab/migrations/*.py diff --git a/tests/_test_mscolab/test_chat_manager.py b/tests/_test_mscolab/test_chat_manager.py index 4cc1691c4..0405f04b8 100644 --- a/tests/_test_mscolab/test_chat_manager.py +++ b/tests/_test_mscolab/test_chat_manager.py @@ -27,7 +27,7 @@ import pytest from mslib.mscolab.models import Message, MessageType -from mslib.mscolab.seed import add_user, get_user, add_operation, add_user_to_operation +from mslib.mscolab.seed import add_user, get_user, add_operation, add_user_to_operation, get_operation class Test_Chat_Manager: @@ -42,20 +42,21 @@ def setup(self, mscolab_app, mscolab_managers): assert add_operation(self.operation_name, "test europe") assert add_user_to_operation(path=self.operation_name, emailid=self.userdata[0]) self.user = get_user(self.userdata[0]) + self.operation = get_operation(self.operation_name) with self.app.app_context(): yield def test_add_message(self): with self.app.test_client(): message = self.cm.add_message(self.user, 'some message', - self.operation_name, message_type=MessageType.TEXT, + self.operation.id, message_type=MessageType.TEXT, reply_id=None) assert message.text == 'some message' def test_edit_messages(self): with self.app.test_client(): message = self.cm.add_message(self.user, 'some test message', - self.operation_name, message_type=MessageType.TEXT, + self.operation.id, message_type=MessageType.TEXT, reply_id=None) new_message_text = "Wonderland" self.cm.edit_message(message.id, new_message_text) @@ -65,7 +66,7 @@ def test_edit_messages(self): def test_delete_messages(self): with self.app.test_client(): message = self.cm.add_message(self.user, 'some test example message', - self.operation_name, message_type=MessageType.TEXT, + self.operation.id, message_type=MessageType.TEXT, reply_id=None) assert 'some test example message' in message.text self.cm.delete_message(message.id) diff --git a/tests/_test_mscolab/test_files.py b/tests/_test_mscolab/test_files.py index 1710ec9d5..35f0da5d2 100644 --- a/tests/_test_mscolab/test_files.py +++ b/tests/_test_mscolab/test_files.py @@ -32,6 +32,7 @@ from mslib.mscolab.models import User, Operation, Permission, Change, Message from mslib.mscolab.seed import add_user, get_user from mslib.mscolab.utils import get_recent_op_id +from tests.utils import XML_CONTENT1, XML_CONTENT2, XML_CONTENT3 class Test_Files: @@ -94,25 +95,54 @@ def test_is_creator(self): def test_file_save(self): with self.app.test_client(): flight_path, operation = self._create_operation(flight_path="operation77") - assert self.fm.save_file(operation.id, "beta", self.user) - assert self.fm.get_file(operation.id, self.user) == "beta" - assert self.fm.save_file(operation.id, "gamma", self.user) - assert self.fm.get_file(operation.id, self.user) == "gamma" + assert self.fm.save_file(operation.id, XML_CONTENT1, self.user) + assert self.fm.get_file(operation.id, self.user) == XML_CONTENT1 + assert self.fm.save_file(operation.id, XML_CONTENT2, self.user) + assert self.fm.get_file(operation.id, self.user) == XML_CONTENT2 # check if change is saved properly changes = self.fm.get_all_changes(operation.id, self.user) assert len(changes) == 2 + def test_cant_save(self): + with self.app.test_client(): + flight_path, operation = self._create_operation(flight_path="operation911") + assert self.fm.save_file(operation.id, "text", self.user) is False + incomplete = """ + + + """ + assert self.fm.save_file(operation.id, incomplete, self.user) is False + incomplete = """ + + + + + + + + + + """ + assert self.fm.save_file(operation.id, incomplete, self.user) is False + + def test_stub_data(self): + with self.app.test_client(): + flight_path, operation = self._create_operation(flight_path="operationstub") + content = self.fm.get_file(operation.id, self.user) + assert flight_path == "operationstub" + assert content == mscolab_settings.STUB_CODE + def test_undo(self): with self.app.test_client(): - flight_path, operation = self._create_operation(flight_path="operation7", content="alpha") - assert self.fm.save_file(operation.id, "beta", self.user) - assert self.fm.save_file(operation.id, "gamma", self.user) + flight_path, operation = self._create_operation(flight_path="operation7", content=XML_CONTENT1) + assert self.fm.save_file(operation.id, XML_CONTENT2, self.user) + assert self.fm.save_file(operation.id, XML_CONTENT3, self.user) changes = Change.query.filter_by(op_id=operation.id).all() assert changes is not None assert changes[0].id == 1 assert self.fm.undo_changes(changes[0].id, self.user) is True assert len(self.fm.get_all_changes(operation.id, self.user)) == 3 - assert "beta" in self.fm.get_file(operation.id, self.user) + assert XML_CONTENT2 == self.fm.get_file(operation.id, self.user) def test_get_operation(self): with self.app.test_client(): @@ -156,8 +186,7 @@ def test_delete_operation(self): assert len(messages) == 0 def _example_data(self): - self.content1 = """\ - + self.content1 = """ new flight track (1) @@ -178,8 +207,7 @@ def _example_data(self): """ - self.content2 = """\ - + self.content2 = """ new flight track (1) diff --git a/tests/_test_mscolab/test_files_api.py b/tests/_test_mscolab/test_files_api.py index e7c8ba3de..fea89f4fd 100644 --- a/tests/_test_mscolab/test_files_api.py +++ b/tests/_test_mscolab/test_files_api.py @@ -29,6 +29,7 @@ from mslib.mscolab.models import Operation from mslib.mscolab.seed import add_user, get_user +from tests.utils import XML_CONTENT1, XML_CONTENT2, XML_CONTENT3 class Test_Files: @@ -170,8 +171,8 @@ def test_delete_operation(self): def test_get_all_changes(self): with self.app.test_client(): flight_path, operation = self._create_operation(flight_path="V11") - assert self.fm.save_file(operation.id, "content1", self.user) - assert self.fm.save_file(operation.id, "content2", self.user) + assert self.fm.save_file(operation.id, XML_CONTENT1, self.user) + assert self.fm.save_file(operation.id, XML_CONTENT2, self.user) all_changes = self.fm.get_all_changes(operation.id, self.user) # the newest change is on index 0, because it has a recent created_at time assert len(all_changes) == 2 @@ -181,20 +182,20 @@ def test_get_all_changes(self): def test_get_change_content(self): with self.app.test_client(): - flight_path, operation = self._create_operation(flight_path="V12", content='initial') - assert self.fm.save_file(operation.id, "content1", self.user) - assert self.fm.save_file(operation.id, "content2", self.user) - assert self.fm.save_file(operation.id, "content3", self.user) + flight_path, operation = self._create_operation(flight_path="V12", content=XML_CONTENT3) + assert self.fm.save_file(operation.id, XML_CONTENT1, self.user) + assert self.fm.save_file(operation.id, XML_CONTENT2, self.user) + assert self.fm.save_file(operation.id, XML_CONTENT3, self.user) all_changes = self.fm.get_all_changes(operation.id, self.user) previous_change = self.fm.get_change_content(all_changes[2]["id"], self.user) - assert previous_change == "content1" + assert previous_change == XML_CONTENT1 previous_change = self.fm.get_change_content(all_changes[1]["id"], self.user) - assert previous_change == "content2" + assert previous_change == XML_CONTENT2 def test_set_version_name(self): with self.app.test_client(): - flight_path, operation = self._create_operation(flight_path="V13", content='initial') - assert self.fm.save_file(operation.id, "content1", self.user) + flight_path, operation = self._create_operation(flight_path="V13", content=XML_CONTENT3) + assert self.fm.save_file(operation.id, XML_CONTENT1, self.user) all_changes = self.fm.get_all_changes(operation.id, self.user) ch_id = all_changes[-1]["id"] self.fm.set_version_name(ch_id, operation.id, self.user.id, "berlin") diff --git a/tests/_test_mscolab/test_migrations.py b/tests/_test_mscolab/test_migrations.py new file mode 100644 index 000000000..75cd0e651 --- /dev/null +++ b/tests/_test_mscolab/test_migrations.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +""" + + tests._test_mscolab.test_migrations + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Tests for database migrations. + + This file is part of MSS. + + :copyright: Copyright 2024 Matthias Riße + :copyright: Copyright 2024 by the MSS team, see AUTHORS. + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import pytest +import itertools +import flask +import flask_migrate +import sqlalchemy +import mslib.mscolab.migrations +import mslib.index +from mslib.mscolab.app import db, migrate +from mslib.mscolab.conf import mscolab_settings + + +def test_migrations(mscolab_app): + migrations_path = mslib.mscolab.migrations.__path__[0] + with mscolab_app.app_context(): + # Seed the database and try to downgrade to each previous revision and then upgrade to the latest again + mslib.mscolab.mscolab.handle_db_seed() + backward_steps = 1 + all_revisions_tested = False + while not all_revisions_tested: + for _ in range(backward_steps): + try: + flask_migrate.downgrade(directory=migrations_path) + except SystemExit as e: + if e.code == 1: + all_revisions_tested = True + flask_migrate.upgrade(directory=migrations_path) + backward_steps += 1 + # Check that there are no differences between the now-current database schema and the defined model + try: + flask_migrate.check(directory=migrations_path) + except SystemExit as e: + assert ( + e.code == 0 + ), "The database models are inconsistent with the migration scripts. Did you forget to add a migration?" + + +_revision_to_name = { + "92eaba86a92e": "v8", + "c171019fe3ee": "v9", +} + +_cases = list( + pytest.param(revision, iterations, id=f"{_revision_to_name[revision]}-iterations={iterations}") + for revision, iterations in itertools.product(["92eaba86a92e", "c171019fe3ee"], [1, 2, 100]) +) + + +@pytest.mark.parametrize("revision,iterations", _cases) +def test_upgrade_from(revision, iterations, mscolab_app, tmp_path): + """Test upgrading from a pre-v10 database that wasn't yet automatically managed with flask-migrate.""" + migrations_path = mslib.mscolab.migrations.__path__[0] + # Construct a dummy flask app to create a separate database to migrate from + # TODO: this would be easier if it was possible to create multiple canonical MSColab Flask app instances, + # i.e. if there was a factory function instead of one global instance. This test could then check the correct + # functioning of the data migration while creating such an app instance, instead of having to call a private method. + app = flask.Flask("whatever") + # TODO: make this somehow configurable to use something other than sqlite as the source database + app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///" + str(tmp_path.absolute() / "mscolab.db") + db.init_app(app) + migrate.init_app(app, db) + with app.app_context(): + # Seed the database and downgrade to the supplied revision + mslib.mscolab.mscolab.handle_db_seed() + flask_migrate.downgrade(directory=migrations_path, revision=revision) + # Set the alembic_version to a non-existing revision to simulate a manual migration following the old migration + # instructions. + db.session.execute(sqlalchemy.text("UPDATE alembic_version SET version_num = 'e62d08ce88a4'")) + # Collect all data for comparison with what's copied over + metadata = sqlalchemy.MetaData() + metadata.reflect(bind=db.engine) + expected_data = {name: db.session.execute(table.select()).all() for name, table in metadata.tables.items()} + del expected_data["alembic_version"] # the alembic_version table will be different, but that is expected + + try: + mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM = app.config["SQLALCHEMY_DATABASE_URI"] + with mscolab_app.app_context(): + db.drop_all() + db.session.execute(sqlalchemy.text("DROP TABLE alembic_version")) + db.session.commit() + inspector = sqlalchemy.inspect(db.engine) + existing_tables = inspector.get_table_names() + assert existing_tables == [] + + # Also try multiple applications of the db upgrade to ensure idempotence of the operation + for _ in range(iterations): + mslib.mscolab.server._handle_db_upgrade() + + # Check that no further migration is required + flask_migrate.check(directory=migrations_path) + actual_data = {name: db.session.execute(table.select()).all() for name, table in db.metadata.tables.items()} + # Check that all tables have the right number of entries with matching ids copied over + assert {k: [e[0] for e in v] for k, v in expected_data.items()} == { + k: [e[0] for e in v] for k, v in actual_data.items() + } + # TODO: Maybe add more asserts? Basically anything could break with future migrations though, if the schema + # is fundamentally changed. Having an id as the first column is already an assumption that might not always + # hold (but should be reliable enough). + flask_migrate.downgrade(directory=migrations_path, revision=revision) + metadata = sqlalchemy.MetaData() + metadata.reflect(bind=db.engine) + actual_data_after_downgrade = { + name: db.session.execute(table.select()).all() for name, table in metadata.tables.items() + } + del actual_data_after_downgrade["alembic_version"] # expected data doesn't have the revision table + # Check that after a downgrade the data is definitely the same + assert expected_data == actual_data_after_downgrade + + # Try to add a new user after the migration + flask_migrate.upgrade(directory=migrations_path) + assert mslib.mscolab.seed.add_user('test123@test456', 'test123', 'test456') + finally: + mscolab_settings.SQLALCHEMY_DB_URI_TO_MIGRATE_FROM = None diff --git a/tests/_test_mscolab/test_models.py b/tests/_test_mscolab/test_models.py index 8b0a2140f..bf38a4894 100644 --- a/tests/_test_mscolab/test_models.py +++ b/tests/_test_mscolab/test_models.py @@ -30,7 +30,7 @@ from zoneinfo import ZoneInfo from mslib.mscolab.server import register_user -from mslib.mscolab.models import AwareDateTime, User, Permission, Operation, Message, Change +from mslib.mscolab.models import AwareDateTime, User, Permission, Operation, Message, MessageType, Change def test_aware_datetime_conversion(): @@ -90,12 +90,12 @@ def test_operation_repr(): def test_message_creation(): - message = Message(1, 1, "Hello, this is a test message", "TEXT", None) + message = Message(1, 1, "Hello, this is a test message", MessageType.TEXT, None) assert message.op_id == 1 assert message.u_id == 1 assert message.text == "Hello, this is a test message" - assert message.message_type == "TEXT" + assert message.message_type == MessageType.TEXT assert message.reply_id is None diff --git a/tests/_test_mscolab/test_mscolab.py b/tests/_test_mscolab/test_mscolab.py index 6e6882d39..ca395d9f6 100644 --- a/tests/_test_mscolab/test_mscolab.py +++ b/tests/_test_mscolab/test_mscolab.py @@ -53,7 +53,7 @@ def test_main(): with mock.patch("mslib.mscolab.mscolab.argparse.ArgumentParser.parse_args", return_value=argparse.Namespace(version=False, update=False, action="db", - init=False, reset=False, seed=False, users_by_file=None, + reset=False, seed=False, users_by_file=None, default_operation=False, add_all_to_all_operation=False, delete_users_by_file=False)): main() diff --git a/tests/_test_mscolab/test_server.py b/tests/_test_mscolab/test_server.py index 6d4caf172..f92264b00 100644 --- a/tests/_test_mscolab/test_server.py +++ b/tests/_test_mscolab/test_server.py @@ -36,12 +36,14 @@ from mslib.mscolab.server import check_login, register_user from mslib.mscolab.file_manager import FileManager from mslib.mscolab.seed import add_user, get_user +from tests.utils import XML_CONTENT1, XML_CONTENT2 class Test_Server: @pytest.fixture(autouse=True) - def setup(self, mscolab_app): + def setup(self, mscolab_app, mscolab_managers): self.app = mscolab_app + self.sockio, _, self.fm = mscolab_managers self.userdata = 'UV10@uv10', 'UV10', 'uv10' with self.app.app_context(): yield @@ -233,6 +235,17 @@ def test_create_operation(self): assert operation.active is False assert token is not None + def test_dont_create_operation(self): + content = """ + + + +""" + assert add_user(self.userdata[0], self.userdata[1], self.userdata[2]) + with self.app.test_client() as test_client: + operation, token = self._create_operation(test_client, self.userdata, content=content) + assert operation is None + def test_get_operation_by_id(self): assert add_user(self.userdata[0], self.userdata[1], self.userdata[2]) with self.app.test_client() as test_client: @@ -271,8 +284,16 @@ def test_get_all_changes(self): assert add_user(self.userdata[0], self.userdata[1], self.userdata[2]) with self.app.test_client() as test_client: operation, token = self._create_operation(test_client, self.userdata) - fm, user = self._save_content(operation, self.userdata) - fm.save_file(operation.id, "content2", user) + self._save_content(operation, self.userdata) + sio = self.sockio.test_client(self.app) + # ToDo implement storing comment + sio.emit('file-save', { + "op_id": operation.id, + "token": token, + "content": XML_CONTENT2, + "comment": "XML_CONTENT2"}) + sio.emit('disconnect') + # the newest change is on index 0, because it has a recent created_at time response = test_client.get('/get_all_changes', data={"token": token, "op_id": operation.id}) @@ -288,22 +309,35 @@ def test_get_change_content(self): assert add_user(self.userdata[0], self.userdata[1], self.userdata[2]) with self.app.test_client() as test_client: operation, token = self._create_operation(test_client, self.userdata) - fm, user = self._save_content(operation, self.userdata) - fm.save_file(operation.id, "content2", user) - all_changes = fm.get_all_changes(operation.id, user) + user = self._save_content(operation, self.userdata) + sio = self.sockio.test_client(self.app) + # ToDo implement storing comment + sio.emit('file-save', { + "op_id": operation.id, + "token": token, + "content": XML_CONTENT2, + "comment": "XML_CONTENT2"}) + sio.emit('disconnect') + all_changes = self.fm.get_all_changes(operation.id, user) response = test_client.get('/get_change_content', data={"token": token, "ch_id": all_changes[1]["id"]}) assert response.status_code == 200 data = json.loads(response.data.decode('utf-8')) - assert data == {'content': 'content1'} + assert data == {'content': XML_CONTENT1} def test_set_version_name(self): assert add_user(self.userdata[0], self.userdata[1], self.userdata[2]) with self.app.test_client() as test_client: operation, token = self._create_operation(test_client, self.userdata) - fm, user = self._save_content(operation, self.userdata) - fm.save_file(operation.id, "content2", user) - all_changes = fm.get_all_changes(operation.id, user) + user = self._save_content(operation, self.userdata) + sio = self.sockio.test_client(self.app) + sio.emit('file-save', { + "op_id": operation.id, + "token": token, + "content": XML_CONTENT2, + "comment": "XML_CONTENT2"}) + sio.emit("disconnect") + all_changes = self.fm.get_all_changes(operation.id, user) ch_id = all_changes[1]["id"] version_name = "THIS" response = test_client.post('/set_version_name', data={"token": token, @@ -419,7 +453,8 @@ def test_import_permissions(self): # creator is not listed assert data["success"] is True - def _create_operation(self, test_client, userdata=None, path="firstflight", description="simple test", active=True): + def _create_operation(self, test_client, userdata=None, path="firstflight", description="simple test", active=True, + content=None): if userdata is None: userdata = self.userdata response = test_client.post('/token', data={"email": userdata[0], "password": userdata[2]}) @@ -428,9 +463,9 @@ def _create_operation(self, test_client, userdata=None, path="firstflight", desc response = test_client.post('/create_operation', data={"token": token, "path": path, "description": description, + "content": content, "active": str(active)}) assert response.status_code == 200 - assert response.data.decode('utf-8') == "True" operation = Operation.query.filter_by(path=path).first() return operation, token @@ -448,6 +483,8 @@ def _save_content(self, operation, userdata=None): if userdata is None: userdata = self.userdata user = get_user(userdata[0]) + self.fm.save_file(operation.id, XML_CONTENT1, user) + return user fm = FileManager(self.app.config["MSCOLAB_DATA_DIR"]) fm.save_file(operation.id, "content1", user) return fm, user diff --git a/tests/_test_utils/test_verify_waypoint_data.py b/tests/_test_utils/test_verify_waypoint_data.py new file mode 100644 index 000000000..c87d802e5 --- /dev/null +++ b/tests/_test_utils/test_verify_waypoint_data.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +""" + + tests._test_utils.test_verify_xml_waypoint + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This tests for valid xml data of waypoint data. + + This file is part of MSS. + + :copyright: Copyright 2024 Reimar Bauer + :license: APACHE-2.0, see LICENSE for details. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" +import pytest + +from mslib.utils.verify_waypoint_data import verify_waypoint_data + + +flight_track_with_waypoints = """ + + + + + + + + + + + + + + + + """ + +flight_track_with_waypoints_and_broken_by_linebreak = """ + + + + + + + + + + + + + + + + + """ + +flight_track_empty = """ + + + + """ + +flight_track_incomplete = """ + + + + + + + + + + + + + + + + """ + +flight_track_with_typo = """ + + + + + + + + + + + + + """ # typo is "233.0"" + +cases = [ + (flight_track_with_waypoints, True), + (flight_track_with_waypoints_and_broken_by_linebreak, False), + (flight_track_empty, False), + (flight_track_incomplete, False), + (flight_track_with_typo, False), +] + + +@pytest.mark.parametrize("xml_content, verification_result", cases) +def test_verify_xml_waypoint(xml_content, verification_result): + """Test xml verification.""" + assert verify_waypoint_data(xml_content) is verification_result diff --git a/tests/fixtures.py b/tests/fixtures.py index dfd29a39e..0a4e03f9c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -37,7 +37,7 @@ from contextlib import contextmanager from mslib.mscolab.conf import mscolab_settings from mslib.mscolab.server import APP, sockio, cm, fm -from mslib.mscolab.mscolab import handle_db_init, handle_db_reset +from mslib.mscolab.mscolab import handle_db_reset from mslib.utils.config import modify_config_file from tests.utils import is_url_response_ok @@ -94,7 +94,6 @@ def mscolab_session_app(): _app.config['SQLALCHEMY_DATABASE_URI'] = mscolab_settings.SQLALCHEMY_DB_URI _app.config['MSCOLAB_DATA_DIR'] = mscolab_settings.MSCOLAB_DATA_DIR _app.config['UPLOAD_FOLDER'] = mscolab_settings.UPLOAD_FOLDER - handle_db_init() return _app @@ -138,7 +137,8 @@ def reset_mscolab(mscolab_session_app): This fixture is not explicitly needed in tests, it is used in the other fixtures to do the cleanup actions. """ - handle_db_reset() + with mscolab_session_app.app_context(): + handle_db_reset() @pytest.fixture diff --git a/tests/utils.py b/tests/utils.py index 7e0083f5d..b7a47b7e9 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -34,6 +34,57 @@ from tests.constants import MSUI_CONFIG_PATH +XML_CONTENT1 = """ + + + + + + + + + + + + + + + + """ + + +XML_CONTENT2 = """ + + + + + + + + + + """ + + +XML_CONTENT3 = """ + + + + + + + + + + + + + + + + """ + + def callback_ok_image(status, response_headers): assert status == "200 OK" assert response_headers[0] == ('Content-type', 'image/png')