From 07877bf7fedde42654c2e75f01993b0a261037f8 Mon Sep 17 00:00:00 2001 From: Mark Bonicillo Date: Mon, 27 Jul 2020 17:56:07 -0700 Subject: [PATCH 1/2] Add mongoutils level one integ tests --- pytest.ini | 1 + .../platform/dbutils/test_mongoutils.py | 178 ++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 volttrontesting/platform/dbutils/test_mongoutils.py diff --git a/pytest.ini b/pytest.ini index 571e122e4f..d9297885da 100644 --- a/pytest.ini +++ b/pytest.ini @@ -52,3 +52,4 @@ markers = rmq_shutdown: rabbitmq shutdown tests secure: Test platform and agents with secure platform options mysqlfuncts: level one integration tests for mysqlfuncts + mongoutils: level one integration tests for mongoutils diff --git a/volttrontesting/platform/dbutils/test_mongoutils.py b/volttrontesting/platform/dbutils/test_mongoutils.py new file mode 100644 index 0000000000..484f46c405 --- /dev/null +++ b/volttrontesting/platform/dbutils/test_mongoutils.py @@ -0,0 +1,178 @@ +from time import time + +import pytest + + +import volttron.platform.dbutils.mongoutils as mongoutils +from volttrontesting.fixtures.docker_wrapper import create_container +from volttrontesting.utils.utils import get_rand_port + + +IMAGES = ["mongo:3-xenial"] # To test more images, add them here +TEST_DATABASE = "test_historian" +ROOT_USERNAME = "mongoadmin" +ROOT_PASSWORD = "12345" +ENV_MONGODB = { + "MONGO_INITDB_ROOT_USERNAME": ROOT_USERNAME, + "MONGO_INITDB_ROOT_PASSWORD": ROOT_PASSWORD, + "MONGO_INITDB_DATABASE": TEST_DATABASE, +} +ALLOW_CONNECTION_TIME = 10 + + +test_data_get_topic_map = [ + ( + "'db.topics.insertOne({topic_name:\"foobar\"})'", + ({"foobar": "foobar"}), + {"foobar"}, + ) +] + + +@pytest.mark.mongoutils +@pytest.mark.parametrize( + "query, expected_topic_name_map, expected_topic_id_keys", test_data_get_topic_map +) +def test_get_topic_map( + get_container_func, + ports_config, + query, + expected_topic_name_map, + expected_topic_id_keys, +): + get_container, image = get_container_func + with get_container( + image, ports=ports_config["ports"], env=ENV_MONGODB + ) as container: + wait_for_connection(container) + seed_database(container, query) + + actual_topic_map = mongoutils.get_topic_map( + mongo_client(ports_config["port_on_host"]), "topics" + ) + + assert actual_topic_map[1] == expected_topic_name_map + assert actual_topic_map[0].keys() == expected_topic_id_keys + + +test_data_get_agg_topic_map = [ + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + "aggregate_topics", + {("foobar", "AVG", "2001")}, + ) +] + + +@pytest.mark.mongoutils +@pytest.mark.parametrize( + "query, agg_topics_collection, expected_topic_id_map_keys", + test_data_get_agg_topic_map, +) +def test_get_agg_topic_map( + get_container_func, + ports_config, + query, + agg_topics_collection, + expected_topic_id_map_keys, +): + get_container, image = get_container_func + with get_container( + image, ports=ports_config["ports"], env=ENV_MONGODB + ) as container: + wait_for_connection(container) + seed_database(container, query) + + actual_agg_topic_map = mongoutils.get_agg_topic_map( + mongo_client(ports_config["port_on_host"]), agg_topics_collection + ) + + assert actual_agg_topic_map.keys() == expected_topic_id_map_keys + + +test_data_get_agg_topics = [ + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + '\'db.aggregate_meta.insertOne({agg_topic_id:"42", meta:{configured_topics: "topic1"}})\'', + "aggregate_topics", + "aggregate_meta", + [("foobar", "AVG", "2001", "topic1")], + ) +] + + +@pytest.mark.mongoutils +@pytest.mark.parametrize( + "query_agg_topics, query_agg_meta, agg_topics_collection, agg_meta_collection, expected_agg_topics", + test_data_get_agg_topics, +) +def test_get_agg_topics( + get_container_func, + ports_config, + query_agg_topics, + query_agg_meta, + agg_topics_collection, + agg_meta_collection, + expected_agg_topics, +): + get_container, image = get_container_func + with get_container( + image, ports=ports_config["ports"], env=ENV_MONGODB + ) as container: + wait_for_connection(container) + seed_database(container, query_agg_topics) + seed_database(container, query_agg_meta) + + actual_agg_topics = mongoutils.get_agg_topics( + mongo_client(ports_config["port_on_host"]), + agg_topics_collection, + agg_meta_collection, + ) + + assert actual_agg_topics == expected_agg_topics + + +def mongo_client(port): + connection_params = { + "host": "localhost", + "port": port, + "database": TEST_DATABASE, + "user": ROOT_USERNAME, + "passwd": ROOT_PASSWORD, + "authSource": "admin", + } + + return mongoutils.get_mongo_client(connection_params) + + +@pytest.fixture(params=IMAGES) +def get_container_func(request): + return create_container, request.param + + +@pytest.fixture() +def ports_config(): + port_on_host = get_rand_port(ip="27017") + return {"port_on_host": port_on_host, "ports": {"27017/tcp": port_on_host}} + + +def wait_for_connection(container): + start_time = time() + # exit codes for MongoDb can be referenced at https://docs.mongodb.com/manual/reference/exit-codes/ + while time() - start_time < ALLOW_CONNECTION_TIME: + command = f'mongo --username="{ROOT_USERNAME}" --password="{ROOT_PASSWORD}" --authenticationDatabase admin {TEST_DATABASE} --eval "db.getName()"' + r = container.exec_run(command, tty=True) + if r[0] == 0: + return + else: + continue + + return RuntimeError(r) + + +def seed_database(container, query): + command = ( + f'mongo --username "{ROOT_USERNAME}" --password "{ROOT_PASSWORD}" ' + f"--authenticationDatabase admin {TEST_DATABASE} --eval={query}" + ) + container.exec_run(cmd=command, tty=True) From 1f0e7789f77a9391bf00e14e5b16cc962d50df0e Mon Sep 17 00:00:00 2001 From: Mark Date: Thu, 6 Aug 2020 21:04:20 -0700 Subject: [PATCH 2/2] Test TravisCI config for mongo tests --- ci-integration/run-test-docker.sh | 5 +- .../virtualization/core/entrypoint.sh | 9 ++ .../virtualization/requirements_test.txt | 5 + .../platform/dbutils/test_mongoutils.py | 152 ++++++++++-------- 4 files changed, 105 insertions(+), 66 deletions(-) diff --git a/ci-integration/run-test-docker.sh b/ci-integration/run-test-docker.sh index 8c9bfc41d1..9c51e828ff 100755 --- a/ci-integration/run-test-docker.sh +++ b/ci-integration/run-test-docker.sh @@ -69,8 +69,9 @@ run_test(){ echo "Running test module $filename" base_filename="$(basename "$filename")" # Start the docker run module. - docker run -e "IGNORE_ENV_CHECK=1" --name "$base_filename" \ - -t volttron_test_image pytest "$filename" > "$base_filename.result.txt" 2>&1 & + docker run -e "IGNORE_ENV_CHECK=1" -e "CI=$CI" --name "$base_filename" \ + -t --network="host" -v /var/run/docker.sock:/var/run/docker.sock volttron_test_image \ + pytest "$filename" > "$base_filename.result.txt" 2>&1 & runningprocs+=($!) outputfiles+=("$base_filename.result.txt") containernames+=("$base_filename") diff --git a/ci-integration/virtualization/core/entrypoint.sh b/ci-integration/virtualization/core/entrypoint.sh index d2ee36ece3..8532454888 100644 --- a/ci-integration/virtualization/core/entrypoint.sh +++ b/ci-integration/virtualization/core/entrypoint.sh @@ -45,6 +45,15 @@ fi # # exit 1; # # fi +# For tests that need to use Docker, we need to restart the virtual machine so that user added to 'docker' group, such as user 'volttron' can have privileges to run Docker. +# See step 3 of "Manage Docker as a non-root user" https://docs.docker.com/engine/install/linux-postinstall/ +# However, rebooting a container results in a "System has not been booted with systemd as init system (PID 1)." +# Thus, in order to give Docker privileges to 'volttron' user, '/var/run/docker/sock' is modified to be readable, writable, and executable by all users. +# This is obviously a security risk, however, given that the container that uses this `entrypoint.sh` is ephemeral and used only for testing, the risk +# is minimal and more importantly allows successful level one integration testing on Travis CI. +chmod 777 /var/run/docker.sock + + if [[ $# -lt 1 ]]; then echo "Please provide a command to run (e.g. /bin/bash, volttron -vv)"; exit 1; diff --git a/ci-integration/virtualization/requirements_test.txt b/ci-integration/virtualization/requirements_test.txt index 7d67750050..98c264f9f9 100644 --- a/ci-integration/virtualization/requirements_test.txt +++ b/ci-integration/virtualization/requirements_test.txt @@ -6,3 +6,8 @@ websocket-client numpy>1.13<2 pandas mysql-connector-python-rf +watchdog +watchdog-gevent +docker +cryptography +pymongo diff --git a/volttrontesting/platform/dbutils/test_mongoutils.py b/volttrontesting/platform/dbutils/test_mongoutils.py index 484f46c405..33117a751a 100644 --- a/volttrontesting/platform/dbutils/test_mongoutils.py +++ b/volttrontesting/platform/dbutils/test_mongoutils.py @@ -1,14 +1,31 @@ +import os from time import time +from gevent import sleep import pytest - import volttron.platform.dbutils.mongoutils as mongoutils from volttrontesting.fixtures.docker_wrapper import create_container from volttrontesting.utils.utils import get_rand_port -IMAGES = ["mongo:3-xenial"] # To test more images, add them here +IMAGES = ["mongo:3-xenial", "mongo:bionic"] + +if "CI" not in os.environ: + IMAGES.extend( + [ + "mongo:3.6-xenial", + "mongo:3.6.19-xenial", + "mongo:4.0-xenial", + "mongo:4.0.19-xenial", + "mongo:4-bionic", + "mongo:4.2-bionic", + "mongo:4.2.8-bionic", + "mongo:4.4-bionic", + "mongo:4.4.0-bionic", + ] + ) + TEST_DATABASE = "test_historian" ROOT_USERNAME = "mongoadmin" ROOT_PASSWORD = "12345" @@ -20,99 +37,102 @@ ALLOW_CONNECTION_TIME = 10 -test_data_get_topic_map = [ - ( - "'db.topics.insertOne({topic_name:\"foobar\"})'", - ({"foobar": "foobar"}), - {"foobar"}, - ) -] - - @pytest.mark.mongoutils @pytest.mark.parametrize( - "query, expected_topic_name_map, expected_topic_id_keys", test_data_get_topic_map + "query, expected_topic_id_map, expected_topic_name_map", + [ + ( + '\'db.topics.insertOne({topic_name:"foobar", _id:"42"})\'', + {"foobar": "42"}, + {"foobar": "foobar"}, + ), + ( + '\'db.topics.insertOne({topic_name:"ROMA", _id:"17"})\'', + {"roma": "17"}, + {"roma": "ROMA"}, + ), + ], ) def test_get_topic_map( get_container_func, ports_config, query, + expected_topic_id_map, expected_topic_name_map, - expected_topic_id_keys, ): get_container, image = get_container_func with get_container( image, ports=ports_config["ports"], env=ENV_MONGODB ) as container: wait_for_connection(container) - seed_database(container, query) + query_database(container, query) - actual_topic_map = mongoutils.get_topic_map( + actual_topic_id_map, actual_topic_name_map = mongoutils.get_topic_map( mongo_client(ports_config["port_on_host"]), "topics" ) - assert actual_topic_map[1] == expected_topic_name_map - assert actual_topic_map[0].keys() == expected_topic_id_keys - - -test_data_get_agg_topic_map = [ - ( - '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', - "aggregate_topics", - {("foobar", "AVG", "2001")}, - ) -] + assert actual_topic_id_map == expected_topic_id_map + assert actual_topic_name_map == expected_topic_name_map @pytest.mark.mongoutils @pytest.mark.parametrize( - "query, agg_topics_collection, expected_topic_id_map_keys", - test_data_get_agg_topic_map, + "query, agg_topics_collection, expected_agg_topic_map", + [ + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + "aggregate_topics", + {("foobar", "AVG", "2001"): "42"}, + ), + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"ROmA", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + "aggregate_topics", + {("roma", "AVG", "2001"): "42"}, + ), + ], ) def test_get_agg_topic_map( get_container_func, ports_config, query, agg_topics_collection, - expected_topic_id_map_keys, + expected_agg_topic_map, ): get_container, image = get_container_func with get_container( image, ports=ports_config["ports"], env=ENV_MONGODB ) as container: wait_for_connection(container) - seed_database(container, query) + query_database(container, query) actual_agg_topic_map = mongoutils.get_agg_topic_map( mongo_client(ports_config["port_on_host"]), agg_topics_collection ) - assert actual_agg_topic_map.keys() == expected_topic_id_map_keys - - -test_data_get_agg_topics = [ - ( - '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', - '\'db.aggregate_meta.insertOne({agg_topic_id:"42", meta:{configured_topics: "topic1"}})\'', - "aggregate_topics", - "aggregate_meta", - [("foobar", "AVG", "2001", "topic1")], - ) -] + assert actual_agg_topic_map == expected_agg_topic_map @pytest.mark.mongoutils @pytest.mark.parametrize( - "query_agg_topics, query_agg_meta, agg_topics_collection, agg_meta_collection, expected_agg_topics", - test_data_get_agg_topics, + "query_agg_topics, query_agg_meta, expected_agg_topics", + [ + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"foobar", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + '\'db.aggregate_meta.insertOne({agg_topic_id:"42", meta:{configured_topics: "topic1"}})\'', + [("foobar", "AVG", "2001", "topic1")], + ), + ( + '\'db.aggregate_topics.insertOne({agg_topic_name:"FOO", agg_type:"AVG", agg_time_period:"2001", _id:"42"})\'', + '\'db.aggregate_meta.insertOne({agg_topic_id:"42", meta:{configured_topics: "topic1"}})\'', + [("foo", "AVG", "2001", "topic1")], + ), + ], ) def test_get_agg_topics( get_container_func, ports_config, query_agg_topics, query_agg_meta, - agg_topics_collection, - agg_meta_collection, expected_agg_topics, ): get_container, image = get_container_func @@ -120,13 +140,13 @@ def test_get_agg_topics( image, ports=ports_config["ports"], env=ENV_MONGODB ) as container: wait_for_connection(container) - seed_database(container, query_agg_topics) - seed_database(container, query_agg_meta) + query_database(container, query_agg_topics) + query_database(container, query_agg_meta) actual_agg_topics = mongoutils.get_agg_topics( mongo_client(ports_config["port_on_host"]), - agg_topics_collection, - agg_meta_collection, + "aggregate_topics", + "aggregate_meta", ) assert actual_agg_topics == expected_agg_topics @@ -157,22 +177,26 @@ def ports_config(): def wait_for_connection(container): + command = f'mongo --username="{ROOT_USERNAME}" --password="{ROOT_PASSWORD}" --authenticationDatabase admin {TEST_DATABASE} --eval "db.getName()"' + query_database(container, None, command=command) + + +def query_database(container, query, command=None): + if command is None: + cmd = ( + f'mongo --username "{ROOT_USERNAME}" --password "{ROOT_PASSWORD}" ' + f"--authenticationDatabase admin {TEST_DATABASE} --eval={query}" + ) + else: + cmd = command + start_time = time() - # exit codes for MongoDb can be referenced at https://docs.mongodb.com/manual/reference/exit-codes/ while time() - start_time < ALLOW_CONNECTION_TIME: - command = f'mongo --username="{ROOT_USERNAME}" --password="{ROOT_PASSWORD}" --authenticationDatabase admin {TEST_DATABASE} --eval "db.getName()"' - r = container.exec_run(command, tty=True) - if r[0] == 0: - return - else: + r = container.exec_run(cmd=cmd, tty=True) + if r[0] != 0: continue + else: + sleep(0.5) + return return RuntimeError(r) - - -def seed_database(container, query): - command = ( - f'mongo --username "{ROOT_USERNAME}" --password "{ROOT_PASSWORD}" ' - f"--authenticationDatabase admin {TEST_DATABASE} --eval={query}" - ) - container.exec_run(cmd=command, tty=True)