diff --git a/ci-integration/run-test-docker.sh b/ci-integration/run-test-docker.sh index 79f3484478..9457d0b648 100755 --- a/ci-integration/run-test-docker.sh +++ b/ci-integration/run-test-docker.sh @@ -70,8 +70,8 @@ run_test(){ base_filename="$(basename "$filename")" # Start the docker run module. docker run -e "IGNORE_ENV_CHECK=1" -e "CI=$CI" --name "$base_filename" \ - -t --network="host" -v /var/run/docker.sock:/var/run/docker.sock volttron_test_image \ - pytest "$filename" > "$base_filename.result.txt" 2>&1 & + -t --network="host" -v /var/run/docker.sock:/var/run/docker.sock volttron_test_image \ + pytest "$filename" > "$base_filename.result.txt" 2>&1 & runningprocs+=($!) outputfiles+=("$base_filename.result.txt") containernames+=("$base_filename") diff --git a/ci-integration/virtualization/requirements_test.txt b/ci-integration/virtualization/requirements_test.txt index f658997a36..50a08e0506 100644 --- a/ci-integration/virtualization/requirements_test.txt +++ b/ci-integration/virtualization/requirements_test.txt @@ -11,3 +11,8 @@ cryptography==2.3 docker psycopg2 mysql-connector-python-rf +watchdog +watchdog-gevent +docker +cryptography +pymongo diff --git a/pytest.ini b/pytest.ini index ee425efb25..eb86088ee0 100644 --- a/pytest.ini +++ b/pytest.ini @@ -58,3 +58,4 @@ markers = postgresqlfuncts_timescaledb: level one integreation tests for timescaledb postgresqlfuncts: level one integration tests for postgresqlfuncts dbutils: test all the level one integrations tests for dbfuncts classes + mongoutils: level one integration tests for mongoutils diff --git a/volttrontesting/platform/dbutils/test_mongoutils.py b/volttrontesting/platform/dbutils/test_mongoutils.py new file mode 100644 index 0000000000..33117a751a --- /dev/null +++ b/volttrontesting/platform/dbutils/test_mongoutils.py @@ -0,0 +1,202 @@ +import os +from time import time + +from gevent import sleep +import pytest + +import volttron.platform.dbutils.mongoutils as mongoutils +from volttrontesting.fixtures.docker_wrapper import create_container +from volttrontesting.utils.utils import get_rand_port + + +IMAGES = ["mongo:3-xenial", "mongo:bionic"] + +if "CI" not in os.environ: + IMAGES.extend( + [ + "mongo:3.6-xenial", + "mongo:3.6.19-xenial", + "mongo:4.0-xenial", + "mongo:4.0.19-xenial", + "mongo:4-bionic", + "mongo:4.2-bionic", + "mongo:4.2.8-bionic", + "mongo:4.4-bionic", + "mongo:4.4.0-bionic", + ] + ) + +TEST_DATABASE = "test_historian" +ROOT_USERNAME = "mongoadmin" +ROOT_PASSWORD = "12345" +ENV_MONGODB = { + "MONGO_INITDB_ROOT_USERNAME": ROOT_USERNAME, + "MONGO_INITDB_ROOT_PASSWORD": ROOT_PASSWORD, + "MONGO_INITDB_DATABASE": TEST_DATABASE, +} +ALLOW_CONNECTION_TIME = 10 + + +@pytest.mark.mongoutils +@pytest.mark.parametrize( + "query, expected_topic_id_map, expected_topic_name_map", + [ + ( + '\'db.topics.insertOne({topic_name:"foobar", _id:"42"})\'', + {"foobar": "42"}, + {"foobar": "foobar"}, + ), + ( + '\'db.topics.insertOne({topic_name:"ROMA", _id:"17"})\'', + {"roma": "17"}, + {"roma": "ROMA"}, + ), + ], +) +def test_get_topic_map( + get_container_func, + ports_config, + query, + expected_topic_id_map, + expected_topic_name_map, +): + get_container, image = get_container_func + with get_container( + image, ports=ports_config["ports"], env=ENV_MONGODB + ) as container: + wait_for_connection(container) + query_database(container, query) + + actual_topic_id_map, actual_topic_name_map = mongoutils.get_topic_map( + mongo_client(ports_config["port_on_host"]), "topics" + ) + + assert actual_topic_id_map == expected_topic_id_map + assert actual_topic_name_map == expected_topic_name_map + + +@pytest.mark.mongoutils +@pytest.mark.parametrize( + "query, agg_topics_collection, expected_agg_topic_map", + [ + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + "aggregate_topics", + {("foobar", "AVG", "2001"): "42"}, + ), + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"ROmA", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + "aggregate_topics", + {("roma", "AVG", "2001"): "42"}, + ), + ], +) +def test_get_agg_topic_map( + get_container_func, + ports_config, + query, + agg_topics_collection, + expected_agg_topic_map, +): + get_container, image = get_container_func + with get_container( + image, ports=ports_config["ports"], env=ENV_MONGODB + ) as container: + wait_for_connection(container) + query_database(container, query) + + actual_agg_topic_map = mongoutils.get_agg_topic_map( + mongo_client(ports_config["port_on_host"]), agg_topics_collection + ) + + assert actual_agg_topic_map == expected_agg_topic_map + + +@pytest.mark.mongoutils +@pytest.mark.parametrize( + "query_agg_topics, query_agg_meta, expected_agg_topics", + [ + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + '\'db.aggregate_meta.insertOne({agg_topic_id:"42", meta:{configured_topics: "topic1"}})\'', + [("foobar", "AVG", "2001", "topic1")], + ), + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"FOO", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + '\'db.aggregate_meta.insertOne({agg_topic_id:"42", meta:{configured_topics: "topic1"}})\'', + [("foo", "AVG", "2001", "topic1")], + ), + ], +) +def test_get_agg_topics( + get_container_func, + ports_config, + query_agg_topics, + query_agg_meta, + expected_agg_topics, +): + get_container, image = get_container_func + with get_container( + image, ports=ports_config["ports"], env=ENV_MONGODB + ) as container: + wait_for_connection(container) + query_database(container, query_agg_topics) + query_database(container, query_agg_meta) + + actual_agg_topics = mongoutils.get_agg_topics( + mongo_client(ports_config["port_on_host"]), + "aggregate_topics", + "aggregate_meta", + ) + + assert actual_agg_topics == expected_agg_topics + + +def mongo_client(port): + connection_params = { + "host": "localhost", + "port": port, + "database": TEST_DATABASE, + "user": ROOT_USERNAME, + "passwd": ROOT_PASSWORD, + "authSource": "admin", + } + + return mongoutils.get_mongo_client(connection_params) + + +@pytest.fixture(params=IMAGES) +def get_container_func(request): + return create_container, request.param + + +@pytest.fixture() +def ports_config(): + port_on_host = get_rand_port(ip="27017") + return {"port_on_host": port_on_host, "ports": {"27017/tcp": port_on_host}} + + +def wait_for_connection(container): + command = f'mongo --username="{ROOT_USERNAME}" --password="{ROOT_PASSWORD}" --authenticationDatabase admin {TEST_DATABASE} --eval "db.getName()"' + query_database(container, None, command=command) + + +def query_database(container, query, command=None): + if command is None: + cmd = ( + f'mongo --username "{ROOT_USERNAME}" --password "{ROOT_PASSWORD}" ' + f"--authenticationDatabase admin {TEST_DATABASE} --eval={query}" + ) + else: + cmd = command + + start_time = time() + while time() - start_time < ALLOW_CONNECTION_TIME: + r = container.exec_run(cmd=cmd, tty=True) + if r[0] != 0: + continue + else: + sleep(0.5) + return + + return RuntimeError(r)