merge develop into GSOC2024-AryanGupta

.github/workflows/testing-all-oses.yml

@@ -48,6 +48,4 @@ jobs:
       # I have no idea yet on why this happens and how to fix it.
       # Even a module level skip is not enough, they need to be completely ignored.
       # TODO: fix those tests and drop the ignores
-      run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib
-        --ignore=tests/_test_msui/test_sideview.py --ignore=tests/_test_msui/test_topview.py --ignore=tests/_test_msui/test_wms_control.py
-        tests
+      run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib tests

.github/workflows/testing-scheduled.yml

@@ -10,7 +10,7 @@ jobs:
     permissions:
       actions: write
     steps:
-      - uses: benc-uk/workflow-dispatch@v1.2.3
+      - uses: benc-uk/workflow-dispatch@v1.2.4
         with:
           workflow: testing-stable.yml
           ref: stable
@@ -20,7 +20,7 @@ jobs:
     permissions:
       actions: write
     steps:
-      - uses: benc-uk/workflow-dispatch@v1.2.3
+      - uses: benc-uk/workflow-dispatch@v1.2.4
         with:
           workflow: testing-develop.yml
           ref: develop

.github/workflows/testing.yml

@@ -40,7 +40,7 @@ jobs:
       # scheduled tests should just check that new dependency versions do not break the
       # tests, but should not update the image.
       if: ${{ (github.ref_name == 'stable' || github.ref_name == 'develop') && github.event_name == 'push' && env.triggerdockerbuild == 'yes' && matrix.order == 'normal' }}
-      uses: benc-uk/workflow-dispatch@v1.2.3
+      uses: benc-uk/workflow-dispatch@v1.2.4
       with:
         workflow: Update Image testing-${{ inputs.image_suffix }}
         repo: Open-MSS/dockertesting

CHANGES.rst

@@ -1,6 +1,16 @@
 Changelog
 =========
 
+Version 9.2.0
+~~~~~~~~~~~~~
+
+Bug fix release and minor enhancements:
+We added a verification for xml data and changed the startup of the SocketsManager.
+
+All changes:
+https://github.com/Open-MSS/MSS/milestone/106?closed=1
+
+
 Version 9.1.0
 ~~~~~~~~~~~~~
 

docs/conf.py

@@ -226,7 +226,7 @@ def get_tutorial_images():
     html_style = 'css/mss.css'
 else:
     htmls_static_path = ['_static']
-    html_css_files = ['mss.css']
+    html_css_files = ['css/mss.css']
     html_context = {
         'display_github': False,  # Add 'Edit on Bitbucket' link instead of 'View page source'
         'last_updated': True,

docs/development.rst

@@ -399,6 +399,26 @@ like e.g. a running MSColab server or a QApplication instance for GUI tests,
 are collected in :mod:`tests.fixtures` in the form of pytest fixtures that can be requested as needed in tests.
 
 
+Changing the database model
+---------------------------
+
+Changing the database model requires adding a corresponding migration script to MSS,
+so that existing databases can be migrated automatically.
+
+To generate such a migration script you can run::
+
+  flask --app mslib.mscolab.app db migrate -d mslib/mscolab/migrations -m "To version <next-major-version>"
+
+Depending on the complexity of the changes that were made,
+the generated migration script might need some tweaking.
+
+If there is already a migration script for the next release,
+then please incorporate the generated migration script into this existing one,
+instead of adding a new one.
+You can still generate a script with the above command first
+to get a starting point for the changes.
+
+
 Pushing your changes
 --------------------
 

docs/environment.yml

@@ -15,7 +15,7 @@ dependencies:
       - xstatic
       - defusedxml
       - sphinx_rtd_theme
-      - sphinxcontrib-video
+      - sphinxcontrib-video>=0.2.1
       - sphinx
       - fs
       - netCDF4

docs/mscolab.rst

@@ -150,75 +150,24 @@ by `pg_dump <https://www.postgresql.org/docs/current/app-pgdump.html>`_ using a
     pg_dump -d mscolab -f "/home/mscolab/dump/$timestamp.sql"
 
 
-Data Base Migration from version 8
-..................................
-
-.. important::
-   This manual migration on the server side by a user is deprecated and will become removed with version 10.0.0.
-   With version 10.0.0, the initialization of the database will be refactored and migrations will be performed automatically when mscolab is started
-
-For an easy way to update the database scheme we implemented  `flask migrate <https://flask-migrate.readthedocs.io/en/latest/>`_.
-
-You have to create based on your configuration a migration script and call that afterwards. ::
-
-    mamba activate instance
-    cd ~/INSTANCE/config
-    export PYTHONPATH=`pwd`
-    cd ~/INSTANCE/wsgi
-    flask --app mscolab.py db init
-    flask --app mscolab.py db migrate -m "To version 9.0.0"
-    flask --app mscolab.py db upgrade
-
-The migration script builder does the base but the created script needs first allow nullable so that we afterwards set the default for the existing data.
-Use this as an example for your script ::
-
-    """To version 9.0.0
-
-    Revision ID: e62d08ce88a4
-    Revises: 27a026b0daec
-    Create Date: 2024-06-07 16:53:43.314338
-
-    """
-    from alembic import op
-    import sqlalchemy as sa
-
-
-    # revision identifiers, used by Alembic.
-    revision = 'e62d08ce88a4'
-    down_revision = '27a026b0daec'
-    branch_labels = None
-    depends_on = None
-
-
-    def upgrade():
-        # ### commands auto generated by Alembic - please adjust! ###
-        with op.batch_alter_table('users', schema=None) as batch_op:
-            batch_op.add_column(sa.Column('authentication_backend', sa.String(length=255), nullable=True))
-        op.execute('UPDATE users SET authentication_backend = \'local\' WHERE authentication_backend IS NULL')
-
-        with op.batch_alter_table('users', schema=None) as batch_op:
-            batch_op.alter_column('authentication_backend', existing_type=sa.String(length=255), nullable=False)
-            batch_op.drop_constraint('users_password_key', type_='unique')
-
-        # ### end Alembic commands ###
-
-
-    def downgrade():
-        # ### commands auto generated by Alembic - please adjust! ###
-        with op.batch_alter_table('users', schema=None) as batch_op:
-            batch_op.create_unique_constraint('users_password_key', ['password'])
-            batch_op.drop_column('authentication_backend')
-
-        # ### end Alembic commands ###
-
-
-The output looks like ::
-
-    ~/INSTANCE/wsgi$ flask --app mscolab.py db upgrade
-    INFO  [alembic.runtime.migration] Context impl SQLiteImpl.
-    INFO  [alembic.runtime.migration] Will assume non-transactional DDL.
-    INFO  [alembic.runtime.migration] Running upgrade  -> e62d08ce88a, To version 9.0.0
-
+Database Migration from Version 8 or 9
+.................................
+
+From v10 onwards MSColab uses `Flask-Migrate <https://flask-migrate.readthedocs.io/en/latest/>` to automatically deal with database migrations.
+To upgrade from v8 or v9 a recreation of the database and subsequent copy of existing data is necessary.
+To do this follow these steps:
+
+#. Stop MSColab completely, no process interacting with the MSColab database should remain running
+#. **Make a backup of your existing database**
+#. Set ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` to your existing database
+#. Set ``SQLALCHEMY_DB_URI`` to a new database
+#. If you are not using SQLite: create the new database
+#. Start MSColab
+#. Check that everything was migrated successfully
+#. Unset ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM``
+
+If you want to keep using your old database URI you can first rename your existing database so that it has a different URI
+and just set ``SQLALCHEMY_DB_URI_TO_MIGRATE_FROM`` to that.
 
 
 Steps to use the MSColab UI features

docs/mss_theme/css/mss.css

@@ -14,3 +14,7 @@
   margin-bottom: auto;
   margin-left: auto;
 }
+
+video {
+  max-width: 100%;
+}

docs/samples/automation/retriever.py

@@ -31,6 +31,7 @@
 import io
 import os
 import xml
+import defusedxml.minidom
 import requests
 from fs import open_fs
 import PIL.Image
@@ -58,7 +59,7 @@ def load_from_ftml(filename):
     _fs = open_fs(_dirname)
     datasource = _fs.open(_name)
     try:
-        doc = xml.dom.minidom.parse(datasource)
+        doc = defusedxml.minidom.parse(datasource)
     except xml.parsers.expat.ExpatError as ex:
         raise SyntaxError(str(ex))
 

docs/tutorial.rst

@@ -9,7 +9,7 @@ a `public share <https://fz-juelich.sciebo.de/s/IzNGi24Ki68iV7t#pdfviewer>`_
 Get familiar by some videos about the Mission Support System (MSS).
 
 
-  .. video:: _static/mp4/tutorial_waypoints.mp4
+  .. video:: videos/mp4/tutorial_waypoints.mp4
      :autoplay:
      :loop:
      :alt: This is the waypoints tutorial,i.e., whenever we are going to plan

docs/tutorials/tutorial_hexagoncontrol.rst

@@ -3,7 +3,7 @@ Table View and Hexagon Flight Patterns
 
 For tomographic imaging, a hexagonal flight pattern can be integrated by a docking widget of Table View
 
-   .. video:: ../_static/mp4/tutorial_hexagoncontrol.mp4
+   .. video:: ../videos/mp4/tutorial_hexagoncontrol.mp4
      :alt: Top View windows is opened (CTRL+H).
            We select the "global (cyl)" for the world map.
            Zooming in for the required region.

docs/tutorials/tutorial_introduction_topview.rst

@@ -3,7 +3,7 @@ Top View and Selecting of Layers
 
 Selection and display of different data in the Top View with the help of the layer chooser.
 
-  .. video:: ../_static/mp4/tutorial_wms.mp4
+  .. video:: ../videos/mp4/tutorial_wms.mp4
      :alt: When we open the Top View (CTRL+H) of the map, the Web Map Service is already opened by default.
            It collects its data from the server: "open-mss dot org" that provides demodata for the meteorological or
            atmospheric information as layer lists.

docs/tutorials/tutorial_kml.rst

@@ -3,7 +3,7 @@ Top View and KML Data
 
 es can be displayed in the Top View. Color and line width can be adjusted.
 
-  .. video:: ../_static/mp4/tutorial_kml.mp4
+  .. video:: ../videos/mp4/tutorial_kml.mp4
      :alt: Open the TopView (CTRL+H)
            After clicking on "(select to open control)", click on KML OVERLAY. The UI will look as shown.
            KML files can be used to show the geographical boundary which helps in planning the WAY POINTS.
@@ -23,4 +23,4 @@ es can be displayed in the Top View. Color and line width can be adjusted.
            Changing it for some time...
            Now, we change line width by changing its numerical value whose range is 0 to 10.
            We change it to different values to obtain different linewidths.
-           The top view is closed and the tutorial ends.
+           The top view is closed and the tutorial ends.

docs/tutorials/tutorial_mscolab.rst

@@ -5,7 +5,7 @@ Using the different views of the MSUI with a fictitious flight path and demo dat
 In comparison to the standalone mode of the MSUI an example setup of users is
 shown on a MSColab server and the possibilities of interactions.
 
-  .. video:: ../_static/mp4/tutorial_mscolab.mp4
+  .. video:: ../videos/mp4/tutorial_mscolab.mp4
      :alt: MSColab stores data in an online server, and can be used to access the data remotely as also working in a
            team where everyone contributes his part. It is used for collaborating with the users as a team together
            and working on a shared MSColab operation.

docs/tutorials/tutorial_msui_views.rst

@@ -4,7 +4,7 @@ Introduction to MSUI
 Using the different views of the MSUI with a fictitious flight path and demo data.
 
 
-  .. video:: ../_static/mp4//tutorial_views.mp4
+  .. video:: ../videos/mp4/tutorial_views.mp4
      :alt: Lets look at the tutorial of the various views required for flight planning:
            Top View (CTRL+H), Side View (CTRL+V), Linear View (CTRL+L) and Table View (CTRL+T).
            At first, lets open Top View and Side View.

docs/tutorials/tutorial_performance_settings.rst

@@ -3,7 +3,7 @@ Table View and Aircraft Performance Data
 
 The range-specific data of an aircraft can be taken into account in Tableview for flight planning.
 
-  .. video:: ../_static/mp4/tutorial_performancesettings.mp4
+  .. video:: ../videos/mp4/tutorial_performancesettings.mp4
      :alt: This is the Performance Settings dockwidget opened in the Table View (CTRL+T) of the MSS software
            where by putting and changing some parameters, we can evaluate the performance of the aircraft.
            Parameters like Flight Altitude, Aviation fuel, Aircraft weight, Maximum take off weight

docs/tutorials/tutorial_remotesensing.rst

@@ -4,7 +4,7 @@ Top View and Remotesensing Tools
 In order to be able to take into account the viewing angle and solar level for measuring instruments, the remotesensing tools are used
 
 
-  .. video:: ../_static/mp4/tutorial_remotesensing.mp4
+  .. video:: ../videos/mp4/tutorial_remotesensing.mp4
      :alt: This is the Remote Sensing Section of the Top View.
            It shows the position and angle of the flight from any particular object in the sky.
            Azimuth is the forward direction line of the flight. If we go above tHE AZIMUTH, angle is in positive,

docs/tutorials/tutorial_satellitetrack.rst

@@ -4,7 +4,7 @@ Top View and  Satellite Overflight
 To combine a flight path with a satellite overflight, the remotesensing widget is used.
 
 
-  .. video:: ../_static/mp4/tutorial_satellitetrack.mp4
+  .. video:: ../videos/mp4/tutorial_satellitetrack.mp4
      :alt: This is Satellite Tracking Prediction System that can be used to check the accuracy of the path
            travelled by a Satellite by the help of data collected from different space agencies and planning
            a flight accordingly.

docs/tutorials/tutorial_waypoints.rst

@@ -3,7 +3,7 @@ Top View Drawing Waypoints
 
 Waypoints for a flight path are defined, shifted and deleted.
 
-  .. video:: ../_static/mp4/tutorial_waypoints.mp4
+  .. video:: ../videos/mp4/tutorial_waypoints.mp4
      :alt: This is the waypoints tutorial,i.e., whenever we are going to plan
            a flight track, we have to place the waypoints in some places and,
            form a flight path.

localbuild/meta.yaml

@@ -77,7 +77,7 @@ requirements:
     - metpy
     - pycountry
     - websocket-client
-    - libtiff <4.5.0
+    - libtiff
     - flask-wtf
     - email_validator
     - keyring

mslib/mscolab/app/__init__.py

@@ -25,6 +25,7 @@
 """
 
 import os
+import sqlalchemy
 
 from flask_migrate import Migrate
 
@@ -62,8 +63,23 @@
 APP.config['MAIL_USE_TLS'] = getattr(mscolab_settings, "MAIL_USE_TLS", None)
 APP.config['MAIL_USE_SSL'] = getattr(mscolab_settings, "MAIL_USE_SSL", None)
 
-db = SQLAlchemy(APP)
-migrate = Migrate(APP, db, render_as_batch=True)
+db = SQLAlchemy(
+    metadata=sqlalchemy.MetaData(
+        naming_convention={
+            # For reference: https://alembic.sqlalchemy.org/en/latest/naming.html#the-importance-of-naming-constraints
+            "ix": "ix_%(column_0_label)s",
+            "uq": "uq_%(table_name)s_%(column_0_name)s",
+            "ck": "ck_%(table_name)s_`%(constraint_name)s`",
+            "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+            "pk": "pk_%(table_name)s",
+        },
+    ),
+)
+db.init_app(APP)
+import mslib.mscolab.models
+
+migrate = Migrate(render_as_batch=True, user_module_prefix="cu.")
+migrate.init_app(APP, db)
 
 
 def get_topmenu():

mslib/mscolab/chat_manager.py

@@ -38,16 +38,16 @@ class ChatManager:
     def __init__(self):
         pass
 
-    def add_message(self, user, text, operation_name, message_type=MessageType.TEXT, reply_id=None):
+    def add_message(self, user, text, op_id, message_type=MessageType.TEXT, reply_id=None):
         """
-        text: message to be emitted to operation and saved to db
-        operation_name: operation-name(op_id) to which message is emitted,
         user: User object, one which emits the message
+        text: message to be emitted to operation and saved to db
+        op_id: operation id to which message is emitted,
         message_type: Enum of type MessageType. values: TEXT, SYSTEM_MESSAGE, IMAGE, DOCUMENT
         """
         if reply_id == -1:
             reply_id = None
-        message = Message(operation_name, user.id, text, message_type, reply_id)
+        message = Message(op_id, user.id, text, message_type, reply_id)
         db.session.add(message)
         db.session.commit()
         return message

mslib/mscolab/conf.py

@@ -63,6 +63,9 @@ class default_mscolab_settings:
     # MYSQL CONNECTION STRING: "mysql+pymysql://<username>:<password>@<host>:<port>/<db_name>?charset=utf8mb4"
     SQLALCHEMY_DB_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'mscolab.db')
 
+    # SQLAlchemy connection string to migrate data from, if set
+    SQLALCHEMY_DB_URI_TO_MIGRATE_FROM = None
+
     # Set to True for testing and False for production
     SQLALCHEMY_ECHO = False