Merge pull request #2724 from shwethanidd/pytest_fixes_ops

Pytest fixes ops

services/ops/MessageDebuggerAgent/tests/test_message_debugging.py → services/contrib/MessageDebuggerAgent/tests/test_message_debugging.py

@@ -82,7 +82,7 @@ class TestMessageDebugger:
         Regression tests for the MessageDebuggerAgent.
     """
 
-    # @pytest.mark.skip(reason="Dependency on SQLAlchemy library")
+    @pytest.mark.skip(reason="Community contributed agent")
     def test_rpc_calls(self, agent):
         """Test the full range of RPC calls to the MessageDebuggerAgent, except those related to streaming."""
 

services/ops/FileWatchPublisher/Tests/test_file_watcher.py

@@ -46,26 +46,12 @@
 from volttron.platform import get_ops, get_home
 from volttron.platform.vip.agent import Agent
 
-test_path = os.path.join(get_home(), "test.txt")
-
-test_config = {
-    "files": [
-        {
-            "file": test_path,
-            "topic": "platform/test_topic"
-        }
-    ]
-}
-
 
 @pytest.fixture(scope="module")
 def publish_agent(request, volttron_instance):
     # 1: Start a fake agent to publish to message bus
     agent = volttron_instance.build_agent(identity='test-agent')
 
-    with open(test_path, "w") as textfile:
-        textfile.write("test_data")
-
     agent.callback = MagicMock(name="callback")
     agent.callback.reset_mock()
 
@@ -75,7 +61,6 @@ def stop_agent():
         print("In teardown method of publish_agent")
         if isinstance(agent, Agent):
             agent.core.stop()
-        os.remove(test_path)
 
     request.addfinalizer(stop_agent)
     return agent
@@ -98,18 +83,33 @@ def test_default_config(volttron_instance, publish_agent):
     volttron_instance.remove_agent(watcher_uuid)
 
 
-# def test_file_watcher(volttron_instance, publish_agent):
-#     watcher_uuid = volttron_instance.install_agent(
-#         agent_dir=get_ops("FileWatchPublisher"),
-#         config_file=test_config,
-#         start=True,
-#         vip_identity="health_test")
-#
-#     with open(test_path, "w+") as textfile:
-#         textfile.write("more test_data")
-#
-#     gevent.sleep(1)
-#
-#     assert publish_agent.callback.call_count == 1
-#     print(publish_agent.callback.call_args)
-#     volttron_instance.remove_agent(watcher_uuid)
+def test_file_watcher(volttron_instance, publish_agent):
+    test_path = os.path.join(get_home(), "test.txt")
+
+    with open(test_path, "w") as textfile:
+        textfile.write("test_data")
+
+    test_config = {
+        "files": [
+            {
+                "file": test_path,
+                "topic": "platform/test_topic"
+            }
+        ]
+    }
+
+    watcher_uuid = volttron_instance.install_agent(
+        agent_dir=get_ops("FileWatchPublisher"),
+        config_file=test_config,
+        start=True,
+        vip_identity="health_test")
+
+    with open(test_path, "w+") as textfile:
+        textfile.write("more test_data")
+
+    gevent.sleep(1)
+
+    assert publish_agent.callback.call_count == 1
+    print(publish_agent.callback.call_args)
+    volttron_instance.remove_agent(watcher_uuid)
+    os.remove(test_path)

services/ops/LogStatisticsAgent/Tests/test_log_statistics.py

@@ -45,25 +45,29 @@
 from volttron.platform.vip.agent import Agent
 from volttron.platform import get_ops, get_home
 
-# TODO fix finding test logs
 test_config = {
-    "file_path": os.path.join(get_home(), "volttron.log"),
     "analysis_interval_sec": 2,
     "publish_topic": "platform/log_statistics",
     "historian_topic": "analysis/log_statistics"
 }
 
-
 @pytest.fixture(scope="module")
 def publish_agent(request, volttron_instance):
+    test_config = {
+        "file_path": os.path.join(volttron_instance.volttron_home, "volttron.log"),
+        "analysis_interval_sec": 2,
+        "publish_topic": "platform/log_statistics",
+        "historian_topic": "analysis/log_statistics"
+    }
     # 1: Start a fake agent to publish to message bus
     agent = volttron_instance.build_agent()
 
-    publish_agent.callback = MagicMock(name="callback")
-    publish_agent.callback.reset_mock()
+    agent.callback = MagicMock(name="callback")
+    agent.callback.reset_mock()
 
-    agent.vip.pubsub.subscribe(peer='pubsub', prefix=test_config.get("publish_topic"),
-                               callback=publish_agent.callback).get()
+    agent.vip.pubsub.subscribe(peer='pubsub',
+                               prefix=test_config.get("publish_topic"),
+                               callback=agent.callback).get()
 
     def stop_agent():
         print("In teardown method of publish_agent")
@@ -91,20 +95,24 @@ def test_default_config(volttron_instance, publish_agent):
     volttron_instance.remove_agent(stats_uuid)
 
 
-# def test_log_stats(volttron_instance, publish_agent):
-#     stats_uuid = volttron_instance.install_agent(
-#         agent_dir=get_ops("LogStatisticsAgent"),
-#         config_file=test_config,
-#         start=True,
-#         vip_identity="health_test")
-#
-#     gevent.sleep(1)
-#
-#     # building another agent should populate the logs
-#     volttron_instance.build_agent(identity="log_populate")
-#
-#     gevent.sleep(2)
-#
-#     # TODO do mock asserts
-#
-#     volttron_instance.remove_agent(stats_uuid)
+def test_log_stats(volttron_instance, publish_agent):
+    test_config["file_path"] = volttron_instance.log_path
+    print(f"File path: {test_config['file_path']}")
+
+    stats_uuid = volttron_instance.install_agent(
+        agent_dir=get_ops("LogStatisticsAgent"),
+        config_file=test_config,
+        start=True,
+        vip_identity="health_test")
+
+    import gevent
+    gevent.sleep(1)
+
+    # building another agent should populate the logs
+    volttron_instance.build_agent(identity="log_populate")
+
+    gevent.sleep(1)
+
+    assert publish_agent.callback.call_count >= 1
+
+    volttron_instance.remove_agent(stats_uuid)

services/ops/SysMonAgent/tests/test_sysmonagent.py

@@ -102,8 +102,8 @@ def add_topic(peer, sender, bus, topic, headers, messages):
 
     agent.vip.pubsub.subscribe('pubsub', base_topic, callback=add_topic)
 
-    max_wait = 1 + max(value for key, value in _test_config.items() if key.endswith('_interval'))
-
+    max_wait = 1 + max(value for key, value in _test_config.items() if key.endswith('_interval')) + 8
+    print(f"Max wait: {max_wait}, topics: {topics}, seen_topics: {seen_topics}")
     assert poll_gevent_sleep(max_wait, lambda: set(topics) <= seen_topics)
 
 
@@ -124,6 +124,7 @@ def test_reconfigure_then_listen(sysmon_tester_agent):
     listen(sysmon_tester_agent, new_config)
 
 
+@pytest.mark.dev
 def test_default_config(sysmon_tester_agent):
     """
     Test that the topic can be reconfigured

services/ops/TopicWatcher/tests/test_remote_topic_watcher.py

@@ -182,4 +182,5 @@ def onmessage(peer, sender, bus, topic, headers, message):
            u"Topic(s) not published within time limit: [('fakedevice2/all', 'point'), " \
            u"'fakedevice2/all', 'fakedevice']" in \
            alert_messages
+
     alert_messages.clear()

volttron/platform/__init__.py

@@ -213,6 +213,17 @@ def is_rabbitmq_available():
     return rabbitmq_available
 
 
+def is_web_available():
+    web_available = True
+    try:
+        import jwt
+        from jinja2 import Environment, FileSystemLoader, select_autoescape
+        from ws4py.server.geventserver import WSGIServer
+    except ImportError:
+        web_available = False
+    return web_available
+
+
 __config__ = None
 
 

volttron/platform/main.py

@@ -963,7 +963,6 @@ def rmq_router(stop):
             del event
 
             protected_topics = auth.get_protected_topics()
-
             # Spawn Greenlet friendly ZMQ router
             # Necessary for backward compatibility with ZMQ message bus
             green_router = GreenRouter(opts.vip_local_address, opts.vip_address,
@@ -1029,7 +1028,6 @@ def rmq_router(stop):
                               enable_store=False,
                               message_bus='zmq')
         ]
-
         entry = AuthEntry(credentials=services[0].core.publickey,
                           user_id=CONTROL,
                           capabilities=[{'edit_config_store': {'identity': '/.*/'}},
@@ -1040,6 +1038,7 @@ def rmq_router(stop):
         # Begin the webserver based options here.
         if opts.bind_web_address is not None:
             if not HAS_WEB:
+                _log.info(f"Web libraries not installed, but bind web address specified\n")
                 sys.stderr.write("Web libraries not installed, but bind web address specified\n")
                 sys.stderr.write("Please install web libraries using python3 bootstrap.py --web\n")
                 sys.exit(-1)
@@ -1090,7 +1089,6 @@ def rmq_router(stop):
         #                   capabilities=['allow_auth_modifications'],
         #                   comments='Automatically added by platform on start')
         # AuthFile().add(entry, overwrite=True)
-
         health_service = HealthService(address=address,
                                        identity=PLATFORM_HEALTH, heartbeat_autostart=True,
                                        enable_store=False,

volttrontesting/fixtures/volttron_platform_fixtures.py

@@ -8,7 +8,7 @@
 import psutil
 import pytest
 
-from volttron.platform import is_rabbitmq_available
+from volttron.platform import is_rabbitmq_available, is_web_available
 from volttron.platform import update_platform_config
 from volttron.utils import get_random_key
 from volttrontesting.fixtures.cert_fixtures import certs_profile_1
@@ -20,9 +20,12 @@
 
 PRINT_LOG_ON_SHUTDOWN = False
 HAS_RMQ = is_rabbitmq_available()
+HAS_WEB = is_web_available()
+
 ci_skipif = pytest.mark.skipif(os.getenv('CI', None) == 'true', reason='SSL does not work in CI')
 rmq_skipif = pytest.mark.skipif(not HAS_RMQ,
                                 reason='RabbitMQ is not setup and/or SSL does not work in CI')
+web_skipif = pytest.mark.skipif(not HAS_WEB, reason='Web libraries are not installed')
 
 
 def print_log(volttron_home):
@@ -273,7 +276,7 @@ def volttron_instance_web(request):
 
 @pytest.fixture(scope="module",
                 params=[
-                    dict(sink='zmq_web', source='zmq', zmq_ssl=False),
+                    pytest.param(dict(sink='zmq_web', source='zmq', zmq_ssl=False), marks=web_skipif),
                     pytest.param(dict(sink='zmq_web', source='zmq', zmq_ssl=True), marks=ci_skipif),
                     pytest.param(dict(sink='rmq_web', source='zmq', zmq_ssl=False), marks=rmq_skipif),
                     pytest.param(dict(sink='rmq_web', source='rmq', zmq_ssl=False), marks=rmq_skipif),